rclone/fs/fs.go

// Package fs is a generic file system interface for rclone object storage systems
package fs

import (
	"context"
	"crypto/md5"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math"
	"os"
	"path/filepath"
	"reflect"
	"sort"
	"strings"
	"time"

	"github.com/pkg/errors"
	"github.com/rclone/rclone/fs/config/configmap"
	"github.com/rclone/rclone/fs/config/configstruct"
	"github.com/rclone/rclone/fs/fserrors"
	"github.com/rclone/rclone/fs/fspath"
	"github.com/rclone/rclone/fs/hash"
	"github.com/rclone/rclone/lib/pacer"
)

// EntryType can be associated with remote paths to identify their type
type EntryType int

// Constants
const (
	// ModTimeNotSupported is a very large precision value to show
	// mod time isn't supported on this Fs
	ModTimeNotSupported = 100 * 365 * 24 * time.Hour
	// MaxLevel is a sentinel representing an infinite depth for listings
	MaxLevel = math.MaxInt32
	// EntryDirectory should be used to classify remote paths in directories
	EntryDirectory EntryType = iota // 0
	// EntryObject should be used to classify remote paths in objects
	EntryObject // 1
)

// Globals
var (
	// Filesystem registry
	Registry []*RegInfo
	// ErrorNotFoundInConfigFile is returned by NewFs if not found in config file
	ErrorNotFoundInConfigFile        = errors.New("didn't find section in config file")
	ErrorCantPurge                   = errors.New("can't purge directory")
	ErrorCantCopy                    = errors.New("can't copy object - incompatible remotes")
	ErrorCantMove                    = errors.New("can't move object - incompatible remotes")
	ErrorCantDirMove                 = errors.New("can't move directory - incompatible remotes")
	ErrorCantUploadEmptyFiles        = errors.New("can't upload empty files to this remote")
	ErrorDirExists                   = errors.New("can't copy directory - destination already exists")
	ErrorCantSetModTime              = errors.New("can't set modified time")
	ErrorCantSetModTimeWithoutDelete = errors.New("can't set modified time without deleting existing object")
	ErrorDirNotFound                 = errors.New("directory not found")
	ErrorObjectNotFound              = errors.New("object not found")
	ErrorLevelNotSupported           = errors.New("level value not supported")
	ErrorListAborted                 = errors.New("list aborted")
	ErrorListBucketRequired          = errors.New("bucket or container name is needed in remote")
	ErrorIsFile                      = errors.New("is a file not a directory")
	ErrorNotAFile                    = errors.New("is not a regular file")
	ErrorNotDeleting                 = errors.New("not deleting files as there were IO errors")
	ErrorNotDeletingDirs             = errors.New("not deleting directories as there were IO errors")
	ErrorOverlapping                 = errors.New("can't sync or move files on overlapping remotes")
	ErrorDirectoryNotEmpty           = errors.New("directory not empty")
	ErrorImmutableModified           = errors.New("immutable file modified")
	ErrorPermissionDenied            = errors.New("permission denied")
	ErrorCantShareDirectories        = errors.New("this backend can't share directories with link")
	ErrorNotImplemented              = errors.New("optional feature not implemented")
	ErrorCommandNotFound             = errors.New("command not found")
	ErrorFileNameTooLong             = errors.New("file name too long")
)

// RegInfo provides information about a filesystem
type RegInfo struct {
	// Name of this fs
	Name string
	// Description of this fs - defaults to Name
	Description string
	// Prefix for command line flags for this fs - defaults to Name if not set
	Prefix string
	// Create a new file system.  If root refers to an existing
	// object, then it should return an Fs which which points to
	// the parent of that object and ErrorIsFile.
	NewFs func(ctx context.Context, name string, root string, config configmap.Mapper) (Fs, error) `json:"-"`
	// Function to call to help with config - see docs for ConfigIn for more info
	Config func(ctx context.Context, name string, m configmap.Mapper, configIn ConfigIn) (*ConfigOut, error) `json:"-"`
	// Options for the Fs configuration
	Options Options
	// The command help, if any
	CommandHelp []CommandHelp
}

// FileName returns the on disk file name for this backend
func (ri *RegInfo) FileName() string {
	return strings.Replace(ri.Name, " ", "", -1)
}

// Options is a slice of configuration Option for a backend
type Options []Option

// Set the default values for the options
func (os Options) setValues() {
	for i := range os {
		o := &os[i]
		if o.Default == nil {
			o.Default = ""
		}
	}
}

// Get the Option corresponding to name or return nil if not found
func (os Options) Get(name string) *Option {
	for i := range os {
		opt := &os[i]
		if opt.Name == name {
			return opt
		}
	}
	return nil
}

// Overridden discovers which config items have been overridden in the
// configmap passed in, either by the config string, command line
// flags or environment variables
func (os Options) Overridden(m *configmap.Map) configmap.Simple {
	var overridden = configmap.Simple{}
	for i := range os {
		opt := &os[i]
		value, isSet := m.GetPriority(opt.Name, configmap.PriorityNormal)
		if isSet {
			overridden.Set(opt.Name, value)
		}
	}
	return overridden
}

// NonDefault discovers which config values aren't at their default
func (os Options) NonDefault(m configmap.Getter) configmap.Simple {
	var nonDefault = configmap.Simple{}
	for i := range os {
		opt := &os[i]
		value, isSet := m.Get(opt.Name)
		if !isSet {
			continue
		}
		defaultValue := fmt.Sprint(opt.Default)
		if value != defaultValue {
			nonDefault.Set(opt.Name, value)
		}
	}
	return nonDefault
}

// OptionVisibility controls whether the options are visible in the
// configurator or the command line.
type OptionVisibility byte

// Constants Option.Hide
const (
	OptionHideCommandLine OptionVisibility = 1 << iota
	OptionHideConfigurator
	OptionHideBoth = OptionHideCommandLine | OptionHideConfigurator
)

// Option is describes an option for the config wizard
//
// This also describes command line options and environment variables
type Option struct {
	Name       string           // name of the option in snake_case
	Help       string           // Help, the first line only is used for the command line help
	Provider   string           // Set to filter on provider
	Default    interface{}      // default value, nil => ""
	Value      interface{}      // value to be set by flags
	Examples   OptionExamples   `json:",omitempty"` // config examples
	ShortOpt   string           // the short option for this if required
	Hide       OptionVisibility // set this to hide the config from the configurator or the command line
	Required   bool             // this option is required
	IsPassword bool             // set if the option is a password
	NoPrefix   bool             // set if the option for this should not use the backend prefix
	Advanced   bool             // set if this is an advanced config option
}

// BaseOption is an alias for Option used internally
type BaseOption Option

// MarshalJSON turns an Option into JSON
//
// It adds some generated fields for ease of use
// - DefaultStr - a string rendering of Default
// - ValueStr - a string rendering of Value
// - Type - the type of the option
func (o *Option) MarshalJSON() ([]byte, error) {
	return json.Marshal(struct {
		BaseOption
		DefaultStr string
		ValueStr   string
		Type       string
	}{
		BaseOption: BaseOption(*o),
		DefaultStr: fmt.Sprint(o.Default),
		ValueStr:   o.String(),
		Type:       o.Type(),
	})
}

// GetValue gets the current current value which is the default if not set
func (o *Option) GetValue() interface{} {
	val := o.Value
	if val == nil {
		val = o.Default
		if val == nil {
			val = ""
		}
	}
	return val
}

// String turns Option into a string
func (o *Option) String() string {
	return fmt.Sprint(o.GetValue())
}

// Set an Option from a string
func (o *Option) Set(s string) (err error) {
	newValue, err := configstruct.StringToInterface(o.GetValue(), s)
	if err != nil {
		return err
	}
	o.Value = newValue
	return nil
}

// Type of the value
func (o *Option) Type() string {
	return reflect.TypeOf(o.GetValue()).Name()
}

// FlagName for the option
func (o *Option) FlagName(prefix string) string {
	name := strings.Replace(o.Name, "_", "-", -1) // convert snake_case to kebab-case
	if !o.NoPrefix {
		name = prefix + "-" + name
	}
	return name
}

// EnvVarName for the option
func (o *Option) EnvVarName(prefix string) string {
	return OptionToEnv(prefix + "-" + o.Name)
}

// OptionExamples is a slice of examples
type OptionExamples []OptionExample

// Len is part of sort.Interface.
func (os OptionExamples) Len() int { return len(os) }

// Swap is part of sort.Interface.
func (os OptionExamples) Swap(i, j int) { os[i], os[j] = os[j], os[i] }

// Less is part of sort.Interface.
func (os OptionExamples) Less(i, j int) bool { return os[i].Help < os[j].Help }

// Sort sorts an OptionExamples
func (os OptionExamples) Sort() { sort.Sort(os) }

// OptionExample describes an example for an Option
type OptionExample struct {
	Value    string
	Help     string
	Provider string
}

// Register a filesystem
//
// Fs modules  should use this in an init() function
func Register(info *RegInfo) {
	info.Options.setValues()
	if info.Prefix == "" {
		info.Prefix = info.Name
	}
	Registry = append(Registry, info)
}

// Fs is the interface a cloud storage system must provide
type Fs interface {
	Info

	// List the objects and directories in dir into entries.  The
	// entries can be returned in any order but should be for a
	// complete directory.
	//
	// dir should be "" to list the root, and should not have
	// trailing slashes.
	//
	// This should return ErrDirNotFound if the directory isn't
	// found.
	List(ctx context.Context, dir string) (entries DirEntries, err error)

	// NewObject finds the Object at remote.  If it can't be found
	// it returns the error ErrorObjectNotFound.
	NewObject(ctx context.Context, remote string) (Object, error)

	// Put in to the remote path with the modTime given of the given size
	//
	// When called from outside an Fs by rclone, src.Size() will always be >= 0.
	// But for unknown-sized objects (indicated by src.Size() == -1), Put should either
	// return an error or upload it properly (rather than e.g. calling panic).
	//
	// May create the object even if it returns an error - if so
	// will return the object and the error, otherwise will return
	// nil and the error
	Put(ctx context.Context, in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error)

	// Mkdir makes the directory (container, bucket)
	//
	// Shouldn't return an error if it already exists
	Mkdir(ctx context.Context, dir string) error

	// Rmdir removes the directory (container, bucket) if empty
	//
	// Return an error if it doesn't exist or isn't empty
	Rmdir(ctx context.Context, dir string) error
}

// Info provides a read only interface to information about a filesystem.
type Info interface {
	// Name of the remote (as passed into NewFs)
	Name() string

	// Root of the remote (as passed into NewFs)
	Root() string

	// String returns a description of the FS
	String() string

	// Precision of the ModTimes in this Fs
	Precision() time.Duration

	// Returns the supported hash types of the filesystem
	Hashes() hash.Set

	// Features returns the optional features of this Fs
	Features() *Features
}

// Object is a filesystem like object provided by an Fs
type Object interface {
	ObjectInfo

	// SetModTime sets the metadata on the object to set the modification date
	SetModTime(ctx context.Context, t time.Time) error

	// Open opens the file for read.  Call Close() on the returned io.ReadCloser
	Open(ctx context.Context, options ...OpenOption) (io.ReadCloser, error)

	// Update in to the object with the modTime given of the given size
	//
	// When called from outside an Fs by rclone, src.Size() will always be >= 0.
	// But for unknown-sized objects (indicated by src.Size() == -1), Upload should either
	// return an error or update the object properly (rather than e.g. calling panic).
	Update(ctx context.Context, in io.Reader, src ObjectInfo, options ...OpenOption) error

	// Removes this object
	Remove(ctx context.Context) error
}

// ObjectInfo provides read only information about an object.
type ObjectInfo interface {
	DirEntry

	// Fs returns read only access to the Fs that this object is part of
	Fs() Info

	// Hash returns the selected checksum of the file
	// If no checksum is available it returns ""
	Hash(ctx context.Context, ty hash.Type) (string, error)

	// Storable says whether this object can be stored
	Storable() bool
}

// DirEntry provides read only information about the common subset of
// a Dir or Object.  These are returned from directory listings - type
// assert them into the correct type.
type DirEntry interface {
	// String returns a description of the Object
	String() string

	// Remote returns the remote path
	Remote() string

	// ModTime returns the modification date of the file
	// It should return a best guess if one isn't available
	ModTime(context.Context) time.Time

	// Size returns the size of the file
	Size() int64
}

// Directory is a filesystem like directory provided by an Fs
type Directory interface {
	DirEntry

	// Items returns the count of items in this directory or this
	// directory and subdirectories if known, -1 for unknown
	Items() int64

	// ID returns the internal ID of this directory if known, or
	// "" otherwise
	ID() string
}

// MimeTyper is an optional interface for Object
type MimeTyper interface {
	// MimeType returns the content type of the Object if
	// known, or "" if not
	MimeType(ctx context.Context) string
}

// IDer is an optional interface for Object
type IDer interface {
	// ID returns the ID of the Object if known, or "" if not
	ID() string
}

// ParentIDer is an optional interface for Object
type ParentIDer interface {
	// ParentID returns the ID of the parent directory if known or nil if not
	ParentID() string
}

// ObjectUnWrapper is an optional interface for Object
type ObjectUnWrapper interface {
	// UnWrap returns the Object that this Object is wrapping or
	// nil if it isn't wrapping anything
	UnWrap() Object
}

// SetTierer is an optional interface for Object
type SetTierer interface {
	// SetTier performs changing storage tier of the Object if
	// multiple storage classes supported
	SetTier(tier string) error
}

// GetTierer is an optional interface for Object
type GetTierer interface {
	// GetTier returns storage tier or class of the Object
	GetTier() string
}

// FullObjectInfo contains all the read-only optional interfaces
//
// Use for checking making wrapping ObjectInfos implement everything
type FullObjectInfo interface {
	ObjectInfo
	MimeTyper
	IDer
	ObjectUnWrapper
	GetTierer
}

// FullObject contains all the optional interfaces for Object
//
// Use for checking making wrapping Objects implement everything
type FullObject interface {
	Object
	MimeTyper
	IDer
	ObjectUnWrapper
	GetTierer
	SetTierer
}

// ObjectOptionalInterfaces returns the names of supported and
// unsupported optional interfaces for an Object
func ObjectOptionalInterfaces(o Object) (supported, unsupported []string) {
	store := func(ok bool, name string) {
		if ok {
			supported = append(supported, name)
		} else {
			unsupported = append(unsupported, name)
		}
	}

	_, ok := o.(MimeTyper)
	store(ok, "MimeType")

	_, ok = o.(IDer)
	store(ok, "ID")

	_, ok = o.(ObjectUnWrapper)
	store(ok, "UnWrap")

	_, ok = o.(SetTierer)
	store(ok, "SetTier")

	_, ok = o.(GetTierer)
	store(ok, "GetTier")

	return supported, unsupported
}

// ListRCallback defines a callback function for ListR to use
//
// It is called for each tranche of entries read from the listing and
// if it returns an error, the listing stops.
type ListRCallback func(entries DirEntries) error

// ListRFn is defines the call used to recursively list a directory
type ListRFn func(ctx context.Context, dir string, callback ListRCallback) error

// NewUsageValue makes a valid value
func NewUsageValue(value int64) *int64 {
	p := new(int64)
	*p = value
	return p
}

// Usage is returned by the About call
//
// If a value is nil then it isn't supported by that backend
type Usage struct {
	Total   *int64 `json:"total,omitempty"`   // quota of bytes that can be used
	Used    *int64 `json:"used,omitempty"`    // bytes in use
	Trashed *int64 `json:"trashed,omitempty"` // bytes in trash
	Other   *int64 `json:"other,omitempty"`   // other usage e.g. gmail in drive
	Free    *int64 `json:"free,omitempty"`    // bytes which can be uploaded before reaching the quota
	Objects *int64 `json:"objects,omitempty"` // objects in the storage system
}

// WriterAtCloser wraps io.WriterAt and io.Closer
type WriterAtCloser interface {
	io.WriterAt
	io.Closer
}

// Features describe the optional features of the Fs
type Features struct {
	// Feature flags, whether Fs
	CaseInsensitive         bool // has case insensitive files
	DuplicateFiles          bool // allows duplicate files
	ReadMimeType            bool // can read the mime type of objects
	WriteMimeType           bool // can set the mime type of objects
	CanHaveEmptyDirectories bool // can have empty directories
	BucketBased             bool // is bucket based (like s3, swift, etc.)
	BucketBasedRootOK       bool // is bucket based and can use from root
	SetTier                 bool // allows set tier functionality on objects
	GetTier                 bool // allows to retrieve storage tier of objects
	ServerSideAcrossConfigs bool // can server-side copy between different remotes of the same type
	IsLocal                 bool // is the local backend
	SlowModTime             bool // if calling ModTime() generally takes an extra transaction
	SlowHash                bool // if calling Hash() generally takes an extra transaction

	// Purge all files in the directory specified
	//
	// Implement this if you have a way of deleting all the files
	// quicker than just running Remove() on the result of List()
	//
	// Return an error if it doesn't exist
	Purge func(ctx context.Context, dir string) error

	// Copy src to this remote using server-side copy operations.
	//
	// This is stored with the remote path given
	//
	// It returns the destination Object and a possible error
	//
	// Will only be called if src.Fs().Name() == f.Name()
	//
	// If it isn't possible then return fs.ErrorCantCopy
	Copy func(ctx context.Context, src Object, remote string) (Object, error)

	// Move src to this remote using server-side move operations.
	//
	// This is stored with the remote path given
	//
	// It returns the destination Object and a possible error
	//
	// Will only be called if src.Fs().Name() == f.Name()
	//
	// If it isn't possible then return fs.ErrorCantMove
	Move func(ctx context.Context, src Object, remote string) (Object, error)

	// DirMove moves src, srcRemote to this remote at dstRemote
	// using server-side move operations.
	//
	// Will only be called if src.Fs().Name() == f.Name()
	//
	// If it isn't possible then return fs.ErrorCantDirMove
	//
	// If destination exists then return fs.ErrorDirExists
	DirMove func(ctx context.Context, src Fs, srcRemote, dstRemote string) error

	// ChangeNotify calls the passed function with a path
	// that has had changes. If the implementation
	// uses polling, it should adhere to the given interval.
	ChangeNotify func(context.Context, func(string, EntryType), <-chan time.Duration)

	// UnWrap returns the Fs that this Fs is wrapping
	UnWrap func() Fs

	// WrapFs returns the Fs that is wrapping this Fs
	WrapFs func() Fs

	// SetWrapper sets the Fs that is wrapping this Fs
	SetWrapper func(f Fs)

	// DirCacheFlush resets the directory cache - used in testing
	// as an optional interface
	DirCacheFlush func()

	// PublicLink generates a public link to the remote path (usually readable by anyone)
	PublicLink func(ctx context.Context, remote string, expire Duration, unlink bool) (string, error)

	// Put in to the remote path with the modTime given of the given size
	//
	// May create the object even if it returns an error - if so
	// will return the object and the error, otherwise will return
	// nil and the error
	//
	// May create duplicates or return errors if src already
	// exists.
	PutUnchecked func(ctx context.Context, in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error)

	// PutStream uploads to the remote path with the modTime given of indeterminate size
	//
	// May create the object even if it returns an error - if so
	// will return the object and the error, otherwise will return
	// nil and the error
	PutStream func(ctx context.Context, in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error)

	// MergeDirs merges the contents of all the directories passed
	// in into the first one and rmdirs the other directories.
	MergeDirs func(ctx context.Context, dirs []Directory) error

	// CleanUp the trash in the Fs
	//
	// Implement this if you have a way of emptying the trash or
	// otherwise cleaning up old versions of files.
	CleanUp func(ctx context.Context) error

	// ListR lists the objects and directories of the Fs starting
	// from dir recursively into out.
	//
	// dir should be "" to start from the root, and should not
	// have trailing slashes.
	//
	// This should return ErrDirNotFound if the directory isn't
	// found.
	//
	// It should call callback for each tranche of entries read.
	// These need not be returned in any particular order.  If
	// callback returns an error then the listing will stop
	// immediately.
	//
	// Don't implement this unless you have a more efficient way
	// of listing recursively that doing a directory traversal.
	ListR ListRFn

	// About gets quota information from the Fs
	About func(ctx context.Context) (*Usage, error)

	// OpenWriterAt opens with a handle for random access writes
	//
	// Pass in the remote desired and the size if known.
	//
	// It truncates any existing object
	OpenWriterAt func(ctx context.Context, remote string, size int64) (WriterAtCloser, error)

	// UserInfo returns info about the connected user
	UserInfo func(ctx context.Context) (map[string]string, error)

	// Disconnect the current user
	Disconnect func(ctx context.Context) error

	// Command the backend to run a named command
	//
	// The command run is name
	// args may be used to read arguments from
	// opts may be used to read optional arguments from
	//
	// The result should be capable of being JSON encoded
	// If it is a string or a []string it will be shown to the user
	// otherwise it will be JSON encoded and shown to the user like that
	Command func(ctx context.Context, name string, arg []string, opt map[string]string) (interface{}, error)

	// Shutdown the backend, closing any background tasks and any
	// cached connections.
	Shutdown func(ctx context.Context) error
}

// Disable nil's out the named feature.  If it isn't found then it
// will log a message.
func (ft *Features) Disable(name string) *Features {
	v := reflect.ValueOf(ft).Elem()
	vType := v.Type()
	for i := 0; i < v.NumField(); i++ {
		vName := vType.Field(i).Name
		field := v.Field(i)
		if strings.EqualFold(name, vName) {
			if !field.CanSet() {
				Errorf(nil, "Can't set Feature %q", name)
			} else {
				zero := reflect.Zero(field.Type())
				field.Set(zero)
				Debugf(nil, "Reset feature %q", name)
			}
		}
	}
	return ft
}

// List returns a slice of all the possible feature names
func (ft *Features) List() (out []string) {
	v := reflect.ValueOf(ft).Elem()
	vType := v.Type()
	for i := 0; i < v.NumField(); i++ {
		out = append(out, vType.Field(i).Name)
	}
	return out
}

// Enabled returns a map of features with keys showing whether they
// are enabled or not
func (ft *Features) Enabled() (features map[string]bool) {
	v := reflect.ValueOf(ft).Elem()
	vType := v.Type()
	features = make(map[string]bool, v.NumField())
	for i := 0; i < v.NumField(); i++ {
		vName := vType.Field(i).Name
		field := v.Field(i)
		if field.Kind() == reflect.Func {
			// Can't compare functions
			features[vName] = !field.IsNil()
		} else {
			zero := reflect.Zero(field.Type())
			features[vName] = field.Interface() != zero.Interface()
		}
	}
	return features
}

// DisableList nil's out the comma separated list of named features.
// If it isn't found then it will log a message.
func (ft *Features) DisableList(list []string) *Features {
	for _, feature := range list {
		ft.Disable(strings.TrimSpace(feature))
	}
	return ft
}

// Fill fills in the function pointers in the Features struct from the
// optional interfaces.  It returns the original updated Features
// struct passed in.
func (ft *Features) Fill(ctx context.Context, f Fs) *Features {
	if do, ok := f.(Purger); ok {
		ft.Purge = do.Purge
	}
	if do, ok := f.(Copier); ok {
		ft.Copy = do.Copy
	}
	if do, ok := f.(Mover); ok {
		ft.Move = do.Move
	}
	if do, ok := f.(DirMover); ok {
		ft.DirMove = do.DirMove
	}
	if do, ok := f.(ChangeNotifier); ok {
		ft.ChangeNotify = do.ChangeNotify
	}
	if do, ok := f.(UnWrapper); ok {
		ft.UnWrap = do.UnWrap
	}
	if do, ok := f.(Wrapper); ok {
		ft.WrapFs = do.WrapFs
		ft.SetWrapper = do.SetWrapper
	}
	if do, ok := f.(DirCacheFlusher); ok {
		ft.DirCacheFlush = do.DirCacheFlush
	}
	if do, ok := f.(PublicLinker); ok {
		ft.PublicLink = do.PublicLink
	}
	if do, ok := f.(PutUncheckeder); ok {
		ft.PutUnchecked = do.PutUnchecked
	}
	if do, ok := f.(PutStreamer); ok {
		ft.PutStream = do.PutStream
	}
	if do, ok := f.(MergeDirser); ok {
		ft.MergeDirs = do.MergeDirs
	}
	if do, ok := f.(CleanUpper); ok {
		ft.CleanUp = do.CleanUp
	}
	if do, ok := f.(ListRer); ok {
		ft.ListR = do.ListR
	}
	if do, ok := f.(Abouter); ok {
		ft.About = do.About
	}
	if do, ok := f.(OpenWriterAter); ok {
		ft.OpenWriterAt = do.OpenWriterAt
	}
	if do, ok := f.(UserInfoer); ok {
		ft.UserInfo = do.UserInfo
	}
	if do, ok := f.(Disconnecter); ok {
		ft.Disconnect = do.Disconnect
	}
	if do, ok := f.(Commander); ok {
		ft.Command = do.Command
	}
	if do, ok := f.(Shutdowner); ok {
		ft.Shutdown = do.Shutdown
	}
	return ft.DisableList(GetConfig(ctx).DisableFeatures)
}

// Mask the Features with the Fs passed in
//
// Only optional features which are implemented in both the original
// Fs AND the one passed in will be advertised.  Any features which
// aren't in both will be set to false/nil, except for UnWrap/Wrap which
// will be left untouched.
func (ft *Features) Mask(ctx context.Context, f Fs) *Features {
	mask := f.Features()
	ft.CaseInsensitive = ft.CaseInsensitive && mask.CaseInsensitive
	ft.DuplicateFiles = ft.DuplicateFiles && mask.DuplicateFiles
	ft.ReadMimeType = ft.ReadMimeType && mask.ReadMimeType
	ft.WriteMimeType = ft.WriteMimeType && mask.WriteMimeType
	ft.CanHaveEmptyDirectories = ft.CanHaveEmptyDirectories && mask.CanHaveEmptyDirectories
	ft.BucketBased = ft.BucketBased && mask.BucketBased
	ft.BucketBasedRootOK = ft.BucketBasedRootOK && mask.BucketBasedRootOK
	ft.SetTier = ft.SetTier && mask.SetTier
	ft.GetTier = ft.GetTier && mask.GetTier
	ft.ServerSideAcrossConfigs = ft.ServerSideAcrossConfigs && mask.ServerSideAcrossConfigs
	// ft.IsLocal = ft.IsLocal && mask.IsLocal Don't propagate IsLocal
	ft.SlowModTime = ft.SlowModTime && mask.SlowModTime
	ft.SlowHash = ft.SlowHash && mask.SlowHash

	if mask.Purge == nil {
		ft.Purge = nil
	}
	if mask.Copy == nil {
		ft.Copy = nil
	}
	if mask.Move == nil {
		ft.Move = nil
	}
	if mask.DirMove == nil {
		ft.DirMove = nil
	}
	if mask.ChangeNotify == nil {
		ft.ChangeNotify = nil
	}
	// if mask.UnWrap == nil {
	// 	ft.UnWrap = nil
	// }
	// if mask.Wrapper == nil {
	// 	ft.Wrapper = nil
	// }
	if mask.DirCacheFlush == nil {
		ft.DirCacheFlush = nil
	}
	if mask.PublicLink == nil {
		ft.PublicLink = nil
	}
	if mask.PutUnchecked == nil {
		ft.PutUnchecked = nil
	}
	if mask.PutStream == nil {
		ft.PutStream = nil
	}
	if mask.MergeDirs == nil {
		ft.MergeDirs = nil
	}
	if mask.CleanUp == nil {
		ft.CleanUp = nil
	}
	if mask.ListR == nil {
		ft.ListR = nil
	}
	if mask.About == nil {
		ft.About = nil
	}
	if mask.OpenWriterAt == nil {
		ft.OpenWriterAt = nil
	}
	if mask.UserInfo == nil {
		ft.UserInfo = nil
	}
	if mask.Disconnect == nil {
		ft.Disconnect = nil
	}
	// Command is always local so we don't mask it
	if mask.Shutdown == nil {
		ft.Shutdown = nil
	}
	return ft.DisableList(GetConfig(ctx).DisableFeatures)
}

// Wrap makes a Copy of the features passed in, overriding the UnWrap/Wrap
// method only if available in f.
func (ft *Features) Wrap(f Fs) *Features {
	ftCopy := new(Features)
	*ftCopy = *ft
	if do, ok := f.(UnWrapper); ok {
		ftCopy.UnWrap = do.UnWrap
	}
	if do, ok := f.(Wrapper); ok {
		ftCopy.WrapFs = do.WrapFs
		ftCopy.SetWrapper = do.SetWrapper
	}
	return ftCopy
}

// WrapsFs adds extra information between `f` which wraps `w`
func (ft *Features) WrapsFs(f Fs, w Fs) *Features {
	wFeatures := w.Features()
	if wFeatures.WrapFs != nil && wFeatures.SetWrapper != nil {
		wFeatures.SetWrapper(f)
	}
	return ft
}

// Purger is an optional interfaces for Fs
type Purger interface {
	// Purge all files in the directory specified
	//
	// Implement this if you have a way of deleting all the files
	// quicker than just running Remove() on the result of List()
	//
	// Return an error if it doesn't exist
	Purge(ctx context.Context, dir string) error
}

// Copier is an optional interface for Fs
type Copier interface {
	// Copy src to this remote using server-side copy operations.
	//
	// This is stored with the remote path given
	//
	// It returns the destination Object and a possible error
	//
	// Will only be called if src.Fs().Name() == f.Name()
	//
	// If it isn't possible then return fs.ErrorCantCopy
	Copy(ctx context.Context, src Object, remote string) (Object, error)
}

// Mover is an optional interface for Fs
type Mover interface {
	// Move src to this remote using server-side move operations.
	//
	// This is stored with the remote path given
	//
	// It returns the destination Object and a possible error
	//
	// Will only be called if src.Fs().Name() == f.Name()
	//
	// If it isn't possible then return fs.ErrorCantMove
	Move(ctx context.Context, src Object, remote string) (Object, error)
}

// DirMover is an optional interface for Fs
type DirMover interface {
	// DirMove moves src, srcRemote to this remote at dstRemote
	// using server-side move operations.
	//
	// Will only be called if src.Fs().Name() == f.Name()
	//
	// If it isn't possible then return fs.ErrorCantDirMove
	//
	// If destination exists then return fs.ErrorDirExists
	DirMove(ctx context.Context, src Fs, srcRemote, dstRemote string) error
}

// ChangeNotifier is an optional interface for Fs
type ChangeNotifier interface {
	// ChangeNotify calls the passed function with a path
	// that has had changes. If the implementation
	// uses polling, it should adhere to the given interval.
	// At least one value will be written to the channel,
	// specifying the initial value and updated values might
	// follow. A 0 Duration should pause the polling.
	// The ChangeNotify implementation must empty the channel
	// regularly. When the channel gets closed, the implementation
	// should stop polling and release resources.
	ChangeNotify(context.Context, func(string, EntryType), <-chan time.Duration)
}

// UnWrapper is an optional interfaces for Fs
type UnWrapper interface {
	// UnWrap returns the Fs that this Fs is wrapping
	UnWrap() Fs
}

// Wrapper is an optional interfaces for Fs
type Wrapper interface {
	// Wrap returns the Fs that is wrapping this Fs
	WrapFs() Fs
	// SetWrapper sets the Fs that is wrapping this Fs
	SetWrapper(f Fs)
}

// DirCacheFlusher is an optional interface for Fs
type DirCacheFlusher interface {
	// DirCacheFlush resets the directory cache - used in testing
	// as an optional interface
	DirCacheFlush()
}

// PutUncheckeder is an optional interface for Fs
type PutUncheckeder interface {
	// Put in to the remote path with the modTime given of the given size
	//
	// May create the object even if it returns an error - if so
	// will return the object and the error, otherwise will return
	// nil and the error
	//
	// May create duplicates or return errors if src already
	// exists.
	PutUnchecked(ctx context.Context, in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error)
}

// PutStreamer is an optional interface for Fs
type PutStreamer interface {
	// PutStream uploads to the remote path with the modTime given of indeterminate size
	//
	// May create the object even if it returns an error - if so
	// will return the object and the error, otherwise will return
	// nil and the error
	PutStream(ctx context.Context, in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error)
}

// PublicLinker is an optional interface for Fs
type PublicLinker interface {
	// PublicLink generates a public link to the remote path (usually readable by anyone)
	PublicLink(ctx context.Context, remote string, expire Duration, unlink bool) (string, error)
}

// MergeDirser is an option interface for Fs
type MergeDirser interface {
	// MergeDirs merges the contents of all the directories passed
	// in into the first one and rmdirs the other directories.
	MergeDirs(ctx context.Context, dirs []Directory) error
}

// CleanUpper is an optional interfaces for Fs
type CleanUpper interface {
	// CleanUp the trash in the Fs
	//
	// Implement this if you have a way of emptying the trash or
	// otherwise cleaning up old versions of files.
	CleanUp(ctx context.Context) error
}

// ListRer is an optional interfaces for Fs
type ListRer interface {
	// ListR lists the objects and directories of the Fs starting
	// from dir recursively into out.
	//
	// dir should be "" to start from the root, and should not
	// have trailing slashes.
	//
	// This should return ErrDirNotFound if the directory isn't
	// found.
	//
	// It should call callback for each tranche of entries read.
	// These need not be returned in any particular order.  If
	// callback returns an error then the listing will stop
	// immediately.
	//
	// Don't implement this unless you have a more efficient way
	// of listing recursively that doing a directory traversal.
	ListR(ctx context.Context, dir string, callback ListRCallback) error
}

// RangeSeeker is the interface that wraps the RangeSeek method.
//
// Some of the returns from Object.Open() may optionally implement
// this method for efficiency purposes.
type RangeSeeker interface {
	// RangeSeek behaves like a call to Seek(offset int64, whence
	// int) with the output wrapped in an io.LimitedReader
	// limiting the total length to limit.
	//
	// RangeSeek with a limit of < 0 is equivalent to a regular Seek.
	RangeSeek(ctx context.Context, offset int64, whence int, length int64) (int64, error)
}

// Abouter is an optional interface for Fs
type Abouter interface {
	// About gets quota information from the Fs
	About(ctx context.Context) (*Usage, error)
}

// OpenWriterAter is an optional interface for Fs
type OpenWriterAter interface {
	// OpenWriterAt opens with a handle for random access writes
	//
	// Pass in the remote desired and the size if known.
	//
	// It truncates any existing object
	OpenWriterAt(ctx context.Context, remote string, size int64) (WriterAtCloser, error)
}

// UserInfoer is an optional interface for Fs
type UserInfoer interface {
	// UserInfo returns info about the connected user
	UserInfo(ctx context.Context) (map[string]string, error)
}

// Disconnecter is an optional interface for Fs
type Disconnecter interface {
	// Disconnect the current user
	Disconnect(ctx context.Context) error
}

// CommandHelp describes a single backend Command
//
// These are automatically inserted in the docs
type CommandHelp struct {
	Name  string            // Name of the command, e.g. "link"
	Short string            // Single line description
	Long  string            // Long multi-line description
	Opts  map[string]string // maps option name to a single line help
}

// Commander is an interface to wrap the Command function
type Commander interface {
	// Command the backend to run a named command
	//
	// The command run is name
	// args may be used to read arguments from
	// opts may be used to read optional arguments from
	//
	// The result should be capable of being JSON encoded
	// If it is a string or a []string it will be shown to the user
	// otherwise it will be JSON encoded and shown to the user like that
	Command(ctx context.Context, name string, arg []string, opt map[string]string) (interface{}, error)
}

// Shutdowner is an interface to wrap the Shutdown function
type Shutdowner interface {
	// Shutdown the backend, closing any background tasks and any
	// cached connections.
	Shutdown(ctx context.Context) error
}

// ObjectsChan is a channel of Objects
type ObjectsChan chan Object

// Objects is a slice of Object~s
type Objects []Object

// ObjectPair is a pair of Objects used to describe a potential copy
// operation.
type ObjectPair struct {
	Src, Dst Object
}

// UnWrapFs unwraps f as much as possible and returns the base Fs
func UnWrapFs(f Fs) Fs {
	for {
		unwrap := f.Features().UnWrap
		if unwrap == nil {
			break // not a wrapped Fs, use current
		}
		next := unwrap()
		if next == nil {
			break // no base Fs found, use current
		}
		f = next
	}
	return f
}

// UnWrapObject unwraps o as much as possible and returns the base object
func UnWrapObject(o Object) Object {
	for {
		u, ok := o.(ObjectUnWrapper)
		if !ok {
			break // not a wrapped object, use current
		}
		next := u.UnWrap()
		if next == nil {
			break // no base object found, use current
		}
		o = next
	}
	return o
}

// UnWrapObjectInfo returns the underlying Object unwrapped as much as
// possible or nil.
func UnWrapObjectInfo(oi ObjectInfo) Object {
	o, ok := oi.(Object)
	if !ok {
		return nil
	}
	return UnWrapObject(o)
}

// Find looks for a RegInfo object for the name passed in.  The name
// can be either the Name or the Prefix.
//
// Services are looked up in the config file
func Find(name string) (*RegInfo, error) {
	for _, item := range Registry {
		if item.Name == name || item.Prefix == name || item.FileName() == name {
			return item, nil
		}
	}
	return nil, errors.Errorf("didn't find backend called %q", name)
}

// MustFind looks for an Info object for the type name passed in
//
// Services are looked up in the config file
//
// Exits with a fatal error if not found
func MustFind(name string) *RegInfo {
	fs, err := Find(name)
	if err != nil {
		log.Fatalf("Failed to find remote: %v", err)
	}
	return fs
}

// ParseRemote deconstructs a path into configName, fsPath, looking up
// the fsName in the config file (returning NotFoundInConfigFile if not found)
func ParseRemote(path string) (fsInfo *RegInfo, configName, fsPath string, connectionStringConfig configmap.Simple, err error) {
	parsed, err := fspath.Parse(path)
	if err != nil {
		return nil, "", "", nil, err
	}
	configName, fsPath = parsed.Name, parsed.Path
	var fsName string
	var ok bool
	if configName != "" {
		if strings.HasPrefix(configName, ":") {
			fsName = configName[1:]
		} else {
			m := ConfigMap(nil, configName, parsed.Config)
			fsName, ok = m.Get("type")
			if !ok {
				return nil, "", "", nil, ErrorNotFoundInConfigFile
			}
		}
	} else {
		fsName = "local"
		configName = "local"
	}
	fsInfo, err = Find(fsName)
	return fsInfo, configName, fsPath, parsed.Config, err
}

// A configmap.Getter to read from the environment RCLONE_CONFIG_backend_option_name
type configEnvVars string

// Get a config item from the environment variables if possible
func (configName configEnvVars) Get(key string) (value string, ok bool) {
	return os.LookupEnv(ConfigToEnv(string(configName), key))
}

// A configmap.Getter to read from the environment RCLONE_option_name
type optionEnvVars struct {
	fsInfo *RegInfo
}

// Get a config item from the option environment variables if possible
func (oev optionEnvVars) Get(key string) (value string, ok bool) {
	opt := oev.fsInfo.Options.Get(key)
	if opt == nil {
		return "", false
	}
	// For options with NoPrefix set, check without prefix too
	if opt.NoPrefix {
		value, ok = os.LookupEnv(OptionToEnv(key))
		if ok {
			return value, ok
		}
	}
	return os.LookupEnv(OptionToEnv(oev.fsInfo.Prefix + "-" + key))
}

// A configmap.Getter to read either the default value or the set
// value from the RegInfo.Options
type regInfoValues struct {
	fsInfo     *RegInfo
	useDefault bool
}

// override the values in configMap with the either the flag values or
// the default values
func (r *regInfoValues) Get(key string) (value string, ok bool) {
	opt := r.fsInfo.Options.Get(key)
	if opt != nil && (r.useDefault || opt.Value != nil) {
		return opt.String(), true
	}
	return "", false
}

// A configmap.Setter to read from the config file
type setConfigFile string

// Set a config item into the config file
func (section setConfigFile) Set(key, value string) {
	if strings.HasPrefix(string(section), ":") {
		Logf(nil, "Can't save config %q = %q for on the fly backend %q", key, value, section)
		return
	}
	Debugf(nil, "Saving config %q = %q in section %q of the config file", key, value, section)
	err := ConfigFileSet(string(section), key, value)
	if err != nil {
		Errorf(nil, "Failed saving config %q = %q in section %q of the config file: %v", key, value, section, err)
	}
}

// A configmap.Getter to read from the config file
type getConfigFile string

// Get a config item from the config file
func (section getConfigFile) Get(key string) (value string, ok bool) {
	value, ok = ConfigFileGet(string(section), key)
	// Ignore empty lines in the config file
	if value == "" {
		ok = false
	}
	return value, ok
}

// ConfigMap creates a configmap.Map from the *RegInfo and the
// configName passed in. If connectionStringConfig has any entries (it may be nil),
// then it will be added to the lookup with the highest priority.
//
// If fsInfo is nil then the returned configmap.Map should only be
// used for reading non backend specific parameters, such as "type".
func ConfigMap(fsInfo *RegInfo, configName string, connectionStringConfig configmap.Simple) (config *configmap.Map) {
	// Create the config
	config = configmap.New()

	// Read the config, more specific to least specific

	// Config from connection string
	if len(connectionStringConfig) > 0 {
		config.AddGetter(connectionStringConfig, configmap.PriorityNormal)
	}

	// flag values
	if fsInfo != nil {
		config.AddGetter(&regInfoValues{fsInfo, false}, configmap.PriorityNormal)
	}

	// remote specific environment vars
	config.AddGetter(configEnvVars(configName), configmap.PriorityNormal)

	// backend specific environment vars
	if fsInfo != nil {
		config.AddGetter(optionEnvVars{fsInfo: fsInfo}, configmap.PriorityNormal)
	}

	// config file
	config.AddGetter(getConfigFile(configName), configmap.PriorityConfig)

	// default values
	if fsInfo != nil {
		config.AddGetter(&regInfoValues{fsInfo, true}, configmap.PriorityDefault)
	}

	// Set Config
	config.AddSetter(setConfigFile(configName))
	return config
}

// ConfigFs makes the config for calling NewFs with.
//
// It parses the path which is of the form remote:path
//
// Remotes are looked up in the config file.  If the remote isn't
// found then NotFoundInConfigFile will be returned.
func ConfigFs(path string) (fsInfo *RegInfo, configName, fsPath string, config *configmap.Map, err error) {
	// Parse the remote path
	fsInfo, configName, fsPath, connectionStringConfig, err := ParseRemote(path)
	if err != nil {
		return
	}
	config = ConfigMap(fsInfo, configName, connectionStringConfig)
	return
}

// NewFs makes a new Fs object from the path
//
// The path is of the form remote:path
//
// Remotes are looked up in the config file.  If the remote isn't
// found then NotFoundInConfigFile will be returned.
//
// On Windows avoid single character remote names as they can be mixed
// up with drive letters.
func NewFs(ctx context.Context, path string) (Fs, error) {
	Debugf(nil, "Creating backend with remote %q", path)
	fsInfo, configName, fsPath, config, err := ConfigFs(path)
	if err != nil {
		return nil, err
	}
	overridden := fsInfo.Options.Overridden(config)
	if len(overridden) > 0 {
		extraConfig := overridden.String()
		//Debugf(nil, "detected overriden config %q", extraConfig)
		md5sumBinary := md5.Sum([]byte(extraConfig))
		suffix := base64.RawURLEncoding.EncodeToString(md5sumBinary[:])
		// 5 characters length is 5*6 = 30 bits of base64
		const maxLength = 5
		if len(suffix) > maxLength {
			suffix = suffix[:maxLength]
		}
		suffix = "{" + suffix + "}"
		Debugf(configName, "detected overridden config - adding %q suffix to name", suffix)
		// Add the suffix to the config name
		//
		// These need to work as filesystem names as the VFS cache will use them
		configName += suffix
	}
	return fsInfo.NewFs(ctx, configName, fsPath, config)
}

// ConfigString returns a canonical version of the config string used
// to configure the Fs as passed to fs.NewFs
func ConfigString(f Fs) string {
	name := f.Name()
	root := f.Root()
	if name == "local" && f.Features().IsLocal {
		return root
	}
	return name + ":" + root
}

// TemporaryLocalFs creates a local FS in the OS's temporary directory.
//
// No cleanup is performed, the caller must call Purge on the Fs themselves.
func TemporaryLocalFs(ctx context.Context) (Fs, error) {
	path, err := ioutil.TempDir("", "rclone-spool")
	if err == nil {
		err = os.Remove(path)
	}
	if err != nil {
		return nil, err
	}
	path = filepath.ToSlash(path)
	return NewFs(ctx, path)
}

// CheckClose is a utility function used to check the return from
// Close in a defer statement.
func CheckClose(c io.Closer, err *error) {
	cerr := c.Close()
	if *err == nil {
		*err = cerr
	}
}

// FileExists returns true if a file remote exists.
// If remote is a directory, FileExists returns false.
func FileExists(ctx context.Context, fs Fs, remote string) (bool, error) {
	_, err := fs.NewObject(ctx, remote)
	if err != nil {
		if err == ErrorObjectNotFound || err == ErrorNotAFile || err == ErrorPermissionDenied {
			return false, nil
		}
		return false, err
	}
	return true, nil
}

// GetModifyWindow calculates the maximum modify window between the given Fses
// and the Config.ModifyWindow parameter.
func GetModifyWindow(ctx context.Context, fss ...Info) time.Duration {
	window := GetConfig(ctx).ModifyWindow
	for _, f := range fss {
		if f != nil {
			precision := f.Precision()
			if precision == ModTimeNotSupported {
				return ModTimeNotSupported
			}
			if precision > window {
				window = precision
			}
		}
	}
	return window
}

// Pacer is a simple wrapper around a pacer.Pacer with logging.
type Pacer struct {
	*pacer.Pacer
}

type logCalculator struct {
	pacer.Calculator
}

// NewPacer creates a Pacer for the given Fs and Calculator.
func NewPacer(ctx context.Context, c pacer.Calculator) *Pacer {
	ci := GetConfig(ctx)
	retries := ci.LowLevelRetries
	if retries <= 0 {
		retries = 1
	}
	p := &Pacer{
		Pacer: pacer.New(
			pacer.InvokerOption(pacerInvoker),
			pacer.MaxConnectionsOption(ci.Checkers+ci.Transfers),
			pacer.RetriesOption(retries),
			pacer.CalculatorOption(c),
		),
	}
	p.SetCalculator(c)
	return p
}

func (d *logCalculator) Calculate(state pacer.State) time.Duration {
	oldSleepTime := state.SleepTime
	newSleepTime := d.Calculator.Calculate(state)
	if state.ConsecutiveRetries > 0 {
		if newSleepTime != oldSleepTime {
			Debugf("pacer", "Rate limited, increasing sleep to %v", newSleepTime)
		}
	} else {
		if newSleepTime != oldSleepTime {
			Debugf("pacer", "Reducing sleep to %v", newSleepTime)
		}
	}
	return newSleepTime
}

// SetCalculator sets the pacing algorithm. Don't modify the Calculator object
// afterwards, use the ModifyCalculator method when needed.
//
// It will choose the default algorithm if nil is passed in.
func (p *Pacer) SetCalculator(c pacer.Calculator) {
	switch c.(type) {
	case *logCalculator:
		Logf("pacer", "Invalid Calculator in fs.Pacer.SetCalculator")
	case nil:
		c = &logCalculator{pacer.NewDefault()}
	default:
		c = &logCalculator{c}
	}

	p.Pacer.SetCalculator(c)
}

// ModifyCalculator calls the given function with the currently configured
// Calculator and the Pacer lock held.
func (p *Pacer) ModifyCalculator(f func(pacer.Calculator)) {
	p.ModifyCalculator(func(c pacer.Calculator) {
		switch _c := c.(type) {
		case *logCalculator:
			f(_c.Calculator)
		default:
			Logf("pacer", "Invalid Calculator in fs.Pacer: %t", c)
			f(c)
		}
	})
}

func pacerInvoker(try, retries int, f pacer.Paced) (retry bool, err error) {
	retry, err = f()
	if retry {
		Debugf("pacer", "low level retry %d/%d (error %v)", try, retries, err)
		err = fserrors.RetryError(err)
	}
	return
}