2015-09-22 17:47:16 +00:00
|
|
|
// Package fs is a generic file system interface for rclone object storage systems
|
2013-06-27 19:13:07 +00:00
|
|
|
package fs
|
2012-12-26 12:23:58 +00:00
|
|
|
|
|
|
|
import (
|
|
|
|
"io"
|
2017-08-03 19:42:35 +00:00
|
|
|
"io/ioutil"
|
2012-12-31 16:40:34 +00:00
|
|
|
"log"
|
2016-04-21 19:06:21 +00:00
|
|
|
"math"
|
2017-08-03 19:42:35 +00:00
|
|
|
"os"
|
2015-03-02 17:04:34 +00:00
|
|
|
"path/filepath"
|
2017-08-07 20:10:03 +00:00
|
|
|
"reflect"
|
2013-06-27 19:13:07 +00:00
|
|
|
"regexp"
|
2016-02-15 18:11:53 +00:00
|
|
|
"sort"
|
2017-08-07 20:10:03 +00:00
|
|
|
"strings"
|
2012-12-26 12:23:58 +00:00
|
|
|
"time"
|
2016-06-12 14:06:02 +00:00
|
|
|
|
|
|
|
"github.com/pkg/errors"
|
2012-12-26 12:23:58 +00:00
|
|
|
)
|
|
|
|
|
2014-07-13 17:58:06 +00:00
|
|
|
// Constants
|
|
|
|
const (
|
2015-09-22 17:47:16 +00:00
|
|
|
// ModTimeNotSupported is a very large precision value to show
|
|
|
|
// mod time isn't supported on this Fs
|
2015-08-16 22:24:34 +00:00
|
|
|
ModTimeNotSupported = 100 * 365 * 24 * time.Hour
|
2016-04-21 19:06:21 +00:00
|
|
|
// MaxLevel is a sentinel representing an infinite depth for listings
|
|
|
|
MaxLevel = math.MaxInt32
|
2014-07-13 17:58:06 +00:00
|
|
|
)
|
|
|
|
|
2013-06-27 19:13:07 +00:00
|
|
|
// Globals
|
|
|
|
var (
|
|
|
|
// Filesystem registry
|
2016-02-18 11:35:25 +00:00
|
|
|
fsRegistry []*RegInfo
|
2015-09-22 17:47:16 +00:00
|
|
|
// ErrorNotFoundInConfigFile is returned by NewFs if not found in config file
|
2017-06-13 12:58:39 +00:00
|
|
|
ErrorNotFoundInConfigFile = errors.New("didn't find section in config file")
|
|
|
|
ErrorCantPurge = errors.New("can't purge directory")
|
|
|
|
ErrorCantCopy = errors.New("can't copy object - incompatible remotes")
|
|
|
|
ErrorCantMove = errors.New("can't move object - incompatible remotes")
|
|
|
|
ErrorCantDirMove = errors.New("can't move directory - incompatible remotes")
|
|
|
|
ErrorDirExists = errors.New("can't copy directory - destination already exists")
|
|
|
|
ErrorCantSetModTime = errors.New("can't set modified time")
|
|
|
|
ErrorCantSetModTimeWithoutDelete = errors.New("can't set modified time without deleting existing object")
|
|
|
|
ErrorDirNotFound = errors.New("directory not found")
|
|
|
|
ErrorObjectNotFound = errors.New("object not found")
|
|
|
|
ErrorLevelNotSupported = errors.New("level value not supported")
|
|
|
|
ErrorListAborted = errors.New("list aborted")
|
2017-06-11 21:43:31 +00:00
|
|
|
ErrorListBucketRequired = errors.New("bucket or container name is needed in remote")
|
2017-06-13 12:58:39 +00:00
|
|
|
ErrorIsFile = errors.New("is a file not a directory")
|
|
|
|
ErrorNotAFile = errors.New("is a not a regular file")
|
|
|
|
ErrorNotDeleting = errors.New("not deleting files as there were IO errors")
|
2017-08-09 20:06:39 +00:00
|
|
|
ErrorNotDeletingDirs = errors.New("not deleting directories as there were IO errors")
|
2017-06-13 12:58:39 +00:00
|
|
|
ErrorCantMoveOverlapping = errors.New("can't move files on overlapping remotes")
|
2017-07-25 14:18:13 +00:00
|
|
|
ErrorDirectoryNotEmpty = errors.New("directory not empty")
|
2017-09-02 08:29:01 +00:00
|
|
|
ErrorImmutableModified = errors.New("immutable file modified")
|
2013-06-27 19:13:07 +00:00
|
|
|
)
|
|
|
|
|
2016-02-18 11:35:25 +00:00
|
|
|
// RegInfo provides information about a filesystem
|
|
|
|
type RegInfo struct {
|
2014-05-05 18:52:52 +00:00
|
|
|
// Name of this fs
|
|
|
|
Name string
|
2016-02-15 18:11:53 +00:00
|
|
|
// Description of this fs - defaults to Name
|
|
|
|
Description string
|
2014-05-05 18:52:52 +00:00
|
|
|
// Create a new file system. If root refers to an existing
|
2016-06-21 17:01:53 +00:00
|
|
|
// object, then it should return a Fs which which points to
|
|
|
|
// the parent of that object and ErrorIsFile.
|
2017-10-14 10:50:41 +00:00
|
|
|
NewFs func(name string, root string) (Fs, error) `json:"-"`
|
2014-05-05 18:52:52 +00:00
|
|
|
// Function to call to help with config
|
2017-10-14 10:50:41 +00:00
|
|
|
Config func(string) `json:"-"`
|
2014-05-05 18:52:52 +00:00
|
|
|
// Options for the Fs configuration
|
2014-03-15 16:06:11 +00:00
|
|
|
Options []Option
|
2013-06-27 19:13:07 +00:00
|
|
|
}
|
|
|
|
|
2015-09-22 17:47:16 +00:00
|
|
|
// Option is describes an option for the config wizard
|
2014-03-15 16:06:11 +00:00
|
|
|
type Option struct {
|
2016-08-14 16:16:06 +00:00
|
|
|
Name string
|
|
|
|
Help string
|
|
|
|
Optional bool
|
|
|
|
IsPassword bool
|
2017-10-14 10:50:41 +00:00
|
|
|
Examples OptionExamples `json:",omitempty"`
|
2014-03-15 16:06:11 +00:00
|
|
|
}
|
|
|
|
|
2016-02-15 18:11:53 +00:00
|
|
|
// OptionExamples is a slice of examples
|
|
|
|
type OptionExamples []OptionExample
|
|
|
|
|
|
|
|
// Len is part of sort.Interface.
|
|
|
|
func (os OptionExamples) Len() int { return len(os) }
|
|
|
|
|
|
|
|
// Swap is part of sort.Interface.
|
|
|
|
func (os OptionExamples) Swap(i, j int) { os[i], os[j] = os[j], os[i] }
|
|
|
|
|
|
|
|
// Less is part of sort.Interface.
|
|
|
|
func (os OptionExamples) Less(i, j int) bool { return os[i].Help < os[j].Help }
|
|
|
|
|
|
|
|
// Sort sorts an OptionExamples
|
|
|
|
func (os OptionExamples) Sort() { sort.Sort(os) }
|
|
|
|
|
2015-09-22 17:47:16 +00:00
|
|
|
// OptionExample describes an example for an Option
|
2014-03-15 16:06:11 +00:00
|
|
|
type OptionExample struct {
|
|
|
|
Value string
|
|
|
|
Help string
|
|
|
|
}
|
|
|
|
|
2013-06-27 19:13:07 +00:00
|
|
|
// Register a filesystem
|
|
|
|
//
|
|
|
|
// Fs modules should use this in an init() function
|
2016-02-18 11:35:25 +00:00
|
|
|
func Register(info *RegInfo) {
|
2014-03-15 16:06:11 +00:00
|
|
|
fsRegistry = append(fsRegistry, info)
|
2013-06-27 19:13:07 +00:00
|
|
|
}
|
|
|
|
|
2017-06-30 09:41:48 +00:00
|
|
|
// Fs is the interface a cloud storage system must provide
|
|
|
|
type Fs interface {
|
|
|
|
Info
|
|
|
|
|
2017-06-11 21:43:31 +00:00
|
|
|
// List the objects and directories in dir into entries. The
|
|
|
|
// entries can be returned in any order but should be for a
|
|
|
|
// complete directory.
|
2016-04-21 19:06:21 +00:00
|
|
|
//
|
2017-06-11 21:43:31 +00:00
|
|
|
// dir should be "" to list the root, and should not have
|
|
|
|
// trailing slashes.
|
2016-06-18 13:12:47 +00:00
|
|
|
//
|
2017-06-11 21:43:31 +00:00
|
|
|
// This should return ErrDirNotFound if the directory isn't
|
|
|
|
// found.
|
|
|
|
List(dir string) (entries DirEntries, err error)
|
2016-10-07 10:39:39 +00:00
|
|
|
|
|
|
|
// NewObject finds the Object at remote. If it can't be found
|
|
|
|
// it returns the error ErrorObjectNotFound.
|
|
|
|
NewObject(remote string) (Object, error)
|
2013-01-23 22:43:20 +00:00
|
|
|
|
2013-01-18 18:54:19 +00:00
|
|
|
// Put in to the remote path with the modTime given of the given size
|
|
|
|
//
|
|
|
|
// May create the object even if it returns an error - if so
|
|
|
|
// will return the object and the error, otherwise will return
|
|
|
|
// nil and the error
|
2017-05-28 11:44:22 +00:00
|
|
|
Put(in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error)
|
2013-01-18 18:54:19 +00:00
|
|
|
|
2015-09-22 17:47:16 +00:00
|
|
|
// Mkdir makes the directory (container, bucket)
|
2014-07-24 21:50:11 +00:00
|
|
|
//
|
|
|
|
// Shouldn't return an error if it already exists
|
2016-11-25 21:52:43 +00:00
|
|
|
Mkdir(dir string) error
|
2013-01-18 18:54:19 +00:00
|
|
|
|
2015-09-22 17:47:16 +00:00
|
|
|
// Rmdir removes the directory (container, bucket) if empty
|
2014-07-24 21:50:11 +00:00
|
|
|
//
|
2014-07-28 20:02:00 +00:00
|
|
|
// Return an error if it doesn't exist or isn't empty
|
2016-11-25 21:52:43 +00:00
|
|
|
Rmdir(dir string) error
|
2016-02-18 11:35:25 +00:00
|
|
|
}
|
|
|
|
|
2017-06-30 09:41:48 +00:00
|
|
|
// Info provides a read only interface to information about a filesystem.
|
2016-02-18 11:35:25 +00:00
|
|
|
type Info interface {
|
|
|
|
// Name of the remote (as passed into NewFs)
|
|
|
|
Name() string
|
|
|
|
|
|
|
|
// Root of the remote (as passed into NewFs)
|
|
|
|
Root() string
|
|
|
|
|
|
|
|
// String returns a description of the FS
|
|
|
|
String() string
|
2013-01-18 23:21:02 +00:00
|
|
|
|
|
|
|
// Precision of the ModTimes in this Fs
|
|
|
|
Precision() time.Duration
|
2016-01-11 12:39:33 +00:00
|
|
|
|
|
|
|
// Returns the supported hash types of the filesystem
|
|
|
|
Hashes() HashSet
|
2017-01-13 17:21:47 +00:00
|
|
|
|
|
|
|
// Features returns the optional features of this Fs
|
|
|
|
Features() *Features
|
2012-12-26 12:23:58 +00:00
|
|
|
}
|
|
|
|
|
2015-09-22 17:47:16 +00:00
|
|
|
// Object is a filesystem like object provided by an Fs
|
2013-06-28 07:57:32 +00:00
|
|
|
type Object interface {
|
2016-02-18 11:35:25 +00:00
|
|
|
ObjectInfo
|
|
|
|
|
|
|
|
// SetModTime sets the metadata on the object to set the modification date
|
2016-03-22 15:07:10 +00:00
|
|
|
SetModTime(time.Time) error
|
2016-02-18 11:35:25 +00:00
|
|
|
|
|
|
|
// Open opens the file for read. Call Close() on the returned io.ReadCloser
|
2016-09-10 10:29:57 +00:00
|
|
|
Open(options ...OpenOption) (io.ReadCloser, error)
|
2016-02-18 11:35:25 +00:00
|
|
|
|
|
|
|
// Update in to the object with the modTime given of the given size
|
2017-05-28 11:44:22 +00:00
|
|
|
Update(in io.Reader, src ObjectInfo, options ...OpenOption) error
|
2016-02-18 11:35:25 +00:00
|
|
|
|
|
|
|
// Removes this object
|
|
|
|
Remove() error
|
|
|
|
}
|
|
|
|
|
2017-06-30 09:41:48 +00:00
|
|
|
// ObjectInfo provides read only information about an object.
|
2016-02-18 11:35:25 +00:00
|
|
|
type ObjectInfo interface {
|
2017-06-30 09:54:14 +00:00
|
|
|
DirEntry
|
2016-07-18 13:07:35 +00:00
|
|
|
|
2016-02-18 11:35:25 +00:00
|
|
|
// Fs returns read only access to the Fs that this object is part of
|
|
|
|
Fs() Info
|
2014-03-28 17:56:04 +00:00
|
|
|
|
2016-02-18 11:35:25 +00:00
|
|
|
// Hash returns the selected checksum of the file
|
|
|
|
// If no checksum is available it returns ""
|
2016-01-11 12:39:33 +00:00
|
|
|
Hash(HashType) (string, error)
|
2013-01-18 18:54:19 +00:00
|
|
|
|
2016-07-18 13:07:35 +00:00
|
|
|
// Storable says whether this object can be stored
|
|
|
|
Storable() bool
|
|
|
|
}
|
|
|
|
|
2017-06-30 09:54:14 +00:00
|
|
|
// DirEntry provides read only information about the common subset of
|
|
|
|
// a Dir or Object. These are returned from directory listings - type
|
|
|
|
// assert them into the correct type.
|
|
|
|
type DirEntry interface {
|
2016-10-20 16:46:51 +00:00
|
|
|
// String returns a description of the Object
|
|
|
|
String() string
|
|
|
|
|
2016-07-18 13:07:35 +00:00
|
|
|
// Remote returns the remote path
|
|
|
|
Remote() string
|
|
|
|
|
2013-01-18 18:54:19 +00:00
|
|
|
// ModTime returns the modification date of the file
|
2015-08-16 22:24:34 +00:00
|
|
|
// It should return a best guess if one isn't available
|
2013-01-02 15:21:55 +00:00
|
|
|
ModTime() time.Time
|
2013-01-18 18:54:19 +00:00
|
|
|
|
|
|
|
// Size returns the size of the file
|
2012-12-26 12:23:58 +00:00
|
|
|
Size() int64
|
|
|
|
}
|
|
|
|
|
2017-06-30 12:37:29 +00:00
|
|
|
// Directory is a filesystem like directory provided by an Fs
|
|
|
|
type Directory interface {
|
|
|
|
DirEntry
|
|
|
|
|
|
|
|
// Items returns the count of items in this directory or this
|
|
|
|
// directory and subdirectories if known, -1 for unknown
|
|
|
|
Items() int64
|
2017-08-02 15:44:36 +00:00
|
|
|
|
|
|
|
// ID returns the internal ID of this directory if known, or
|
|
|
|
// "" otherwise
|
|
|
|
ID() string
|
2017-06-30 12:37:29 +00:00
|
|
|
}
|
|
|
|
|
2016-09-21 21:13:24 +00:00
|
|
|
// MimeTyper is an optional interface for Object
|
|
|
|
type MimeTyper interface {
|
|
|
|
// MimeType returns the content type of the Object if
|
|
|
|
// known, or "" if not
|
|
|
|
MimeType() string
|
|
|
|
}
|
|
|
|
|
2017-06-11 21:43:31 +00:00
|
|
|
// ListRCallback defines a callback function for ListR to use
|
|
|
|
//
|
|
|
|
// It is called for each tranche of entries read from the listing and
|
|
|
|
// if it returns an error, the listing stops.
|
|
|
|
type ListRCallback func(entries DirEntries) error
|
|
|
|
|
|
|
|
// ListRFn is defines the call used to recursively list a directory
|
|
|
|
type ListRFn func(dir string, callback ListRCallback) error
|
|
|
|
|
2017-01-13 17:21:47 +00:00
|
|
|
// Features describe the optional features of the Fs
|
|
|
|
type Features struct {
|
2017-08-09 14:27:43 +00:00
|
|
|
// Feature flags, whether Fs
|
|
|
|
CaseInsensitive bool // has case insensitive files
|
|
|
|
DuplicateFiles bool // allows duplicate files
|
|
|
|
ReadMimeType bool // can read the mime type of objects
|
|
|
|
WriteMimeType bool // can set the mime type of objects
|
|
|
|
CanHaveEmptyDirectories bool // can have empty directories
|
|
|
|
BucketBased bool // is bucket based (like s3, swift etc)
|
2017-01-13 17:21:47 +00:00
|
|
|
|
|
|
|
// Purge all files in the root and the root directory
|
|
|
|
//
|
|
|
|
// Implement this if you have a way of deleting all the files
|
|
|
|
// quicker than just running Remove() on the result of List()
|
|
|
|
//
|
|
|
|
// Return an error if it doesn't exist
|
|
|
|
Purge func() error
|
|
|
|
|
|
|
|
// Copy src to this remote using server side copy operations.
|
|
|
|
//
|
|
|
|
// This is stored with the remote path given
|
|
|
|
//
|
|
|
|
// It returns the destination Object and a possible error
|
|
|
|
//
|
|
|
|
// Will only be called if src.Fs().Name() == f.Name()
|
|
|
|
//
|
|
|
|
// If it isn't possible then return fs.ErrorCantCopy
|
|
|
|
Copy func(src Object, remote string) (Object, error)
|
|
|
|
|
|
|
|
// Move src to this remote using server side move operations.
|
|
|
|
//
|
|
|
|
// This is stored with the remote path given
|
|
|
|
//
|
|
|
|
// It returns the destination Object and a possible error
|
|
|
|
//
|
|
|
|
// Will only be called if src.Fs().Name() == f.Name()
|
|
|
|
//
|
|
|
|
// If it isn't possible then return fs.ErrorCantMove
|
|
|
|
Move func(src Object, remote string) (Object, error)
|
|
|
|
|
2017-02-05 21:20:56 +00:00
|
|
|
// DirMove moves src, srcRemote to this remote at dstRemote
|
|
|
|
// using server side move operations.
|
2017-01-13 17:21:47 +00:00
|
|
|
//
|
|
|
|
// Will only be called if src.Fs().Name() == f.Name()
|
|
|
|
//
|
|
|
|
// If it isn't possible then return fs.ErrorCantDirMove
|
|
|
|
//
|
|
|
|
// If destination exists then return fs.ErrorDirExists
|
2017-02-05 21:20:56 +00:00
|
|
|
DirMove func(src Fs, srcRemote, dstRemote string) error
|
2017-01-13 17:21:47 +00:00
|
|
|
|
2017-05-25 21:05:49 +00:00
|
|
|
// DirChangeNotify calls the passed function with a path
|
|
|
|
// of a directory that has had changes. If the implementation
|
|
|
|
// uses polling, it should adhere to the given interval.
|
|
|
|
DirChangeNotify func(func(string), time.Duration) chan bool
|
|
|
|
|
2017-01-13 17:21:47 +00:00
|
|
|
// UnWrap returns the Fs that this Fs is wrapping
|
|
|
|
UnWrap func() Fs
|
|
|
|
|
|
|
|
// DirCacheFlush resets the directory cache - used in testing
|
|
|
|
// as an optional interface
|
|
|
|
DirCacheFlush func()
|
|
|
|
|
|
|
|
// Put in to the remote path with the modTime given of the given size
|
|
|
|
//
|
|
|
|
// May create the object even if it returns an error - if so
|
|
|
|
// will return the object and the error, otherwise will return
|
|
|
|
// nil and the error
|
|
|
|
//
|
|
|
|
// May create duplicates or return errors if src already
|
|
|
|
// exists.
|
2017-05-28 11:44:22 +00:00
|
|
|
PutUnchecked func(in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error)
|
2017-01-13 17:21:47 +00:00
|
|
|
|
2017-08-03 19:42:35 +00:00
|
|
|
// PutStream uploads to the remote path with the modTime given of indeterminate size
|
|
|
|
//
|
|
|
|
// May create the object even if it returns an error - if so
|
|
|
|
// will return the object and the error, otherwise will return
|
|
|
|
// nil and the error
|
|
|
|
PutStream func(in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error)
|
|
|
|
|
2017-08-02 15:51:24 +00:00
|
|
|
// MergeDirs merges the contents of all the directories passed
|
|
|
|
// in into the first one and rmdirs the other directories.
|
|
|
|
MergeDirs func([]Directory) error
|
|
|
|
|
2017-01-13 17:21:47 +00:00
|
|
|
// CleanUp the trash in the Fs
|
|
|
|
//
|
|
|
|
// Implement this if you have a way of emptying the trash or
|
|
|
|
// otherwise cleaning up old versions of files.
|
|
|
|
CleanUp func() error
|
2017-06-05 15:14:24 +00:00
|
|
|
|
|
|
|
// ListR lists the objects and directories of the Fs starting
|
|
|
|
// from dir recursively into out.
|
|
|
|
//
|
|
|
|
// dir should be "" to start from the root, and should not
|
|
|
|
// have trailing slashes.
|
|
|
|
//
|
2017-06-11 21:43:31 +00:00
|
|
|
// This should return ErrDirNotFound if the directory isn't
|
|
|
|
// found.
|
|
|
|
//
|
|
|
|
// It should call callback for each tranche of entries read.
|
|
|
|
// These need not be returned in any particular order. If
|
|
|
|
// callback returns an error then the listing will stop
|
|
|
|
// immediately.
|
2017-06-05 15:14:24 +00:00
|
|
|
//
|
|
|
|
// Don't implement this unless you have a more efficient way
|
|
|
|
// of listing recursively that doing a directory traversal.
|
2017-06-11 21:43:31 +00:00
|
|
|
ListR ListRFn
|
2017-01-13 17:21:47 +00:00
|
|
|
}
|
|
|
|
|
2017-08-07 20:10:03 +00:00
|
|
|
// Disable nil's out the named feature. If it isn't found then it
|
|
|
|
// will log a message.
|
|
|
|
func (ft *Features) Disable(name string) *Features {
|
|
|
|
v := reflect.ValueOf(ft).Elem()
|
|
|
|
vType := v.Type()
|
|
|
|
for i := 0; i < v.NumField(); i++ {
|
|
|
|
vName := vType.Field(i).Name
|
|
|
|
field := v.Field(i)
|
|
|
|
if strings.EqualFold(name, vName) {
|
|
|
|
if !field.CanSet() {
|
|
|
|
Errorf(nil, "Can't set Feature %q", name)
|
|
|
|
} else {
|
|
|
|
zero := reflect.Zero(field.Type())
|
|
|
|
field.Set(zero)
|
|
|
|
Debugf(nil, "Reset feature %q", name)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return ft
|
|
|
|
}
|
|
|
|
|
|
|
|
// List returns a slice of all the possible feature names
|
|
|
|
func (ft *Features) List() (out []string) {
|
|
|
|
v := reflect.ValueOf(ft).Elem()
|
|
|
|
vType := v.Type()
|
|
|
|
for i := 0; i < v.NumField(); i++ {
|
|
|
|
out = append(out, vType.Field(i).Name)
|
|
|
|
}
|
|
|
|
return out
|
|
|
|
}
|
|
|
|
|
|
|
|
// DisableList nil's out the comma separated list of named features.
|
|
|
|
// If it isn't found then it will log a message.
|
|
|
|
func (ft *Features) DisableList(list []string) *Features {
|
|
|
|
for _, feature := range list {
|
|
|
|
ft.Disable(strings.TrimSpace(feature))
|
|
|
|
}
|
|
|
|
return ft
|
|
|
|
}
|
|
|
|
|
2017-01-13 17:21:47 +00:00
|
|
|
// Fill fills in the function pointers in the Features struct from the
|
|
|
|
// optional interfaces. It returns the original updated Features
|
|
|
|
// struct passed in.
|
|
|
|
func (ft *Features) Fill(f Fs) *Features {
|
|
|
|
if do, ok := f.(Purger); ok {
|
|
|
|
ft.Purge = do.Purge
|
|
|
|
}
|
|
|
|
if do, ok := f.(Copier); ok {
|
|
|
|
ft.Copy = do.Copy
|
|
|
|
}
|
|
|
|
if do, ok := f.(Mover); ok {
|
|
|
|
ft.Move = do.Move
|
|
|
|
}
|
|
|
|
if do, ok := f.(DirMover); ok {
|
|
|
|
ft.DirMove = do.DirMove
|
|
|
|
}
|
2017-05-25 21:05:49 +00:00
|
|
|
if do, ok := f.(DirChangeNotifier); ok {
|
|
|
|
ft.DirChangeNotify = do.DirChangeNotify
|
|
|
|
}
|
2017-01-13 17:21:47 +00:00
|
|
|
if do, ok := f.(UnWrapper); ok {
|
|
|
|
ft.UnWrap = do.UnWrap
|
|
|
|
}
|
|
|
|
if do, ok := f.(DirCacheFlusher); ok {
|
|
|
|
ft.DirCacheFlush = do.DirCacheFlush
|
|
|
|
}
|
|
|
|
if do, ok := f.(PutUncheckeder); ok {
|
|
|
|
ft.PutUnchecked = do.PutUnchecked
|
|
|
|
}
|
2017-08-03 19:42:35 +00:00
|
|
|
if do, ok := f.(PutStreamer); ok {
|
|
|
|
ft.PutStream = do.PutStream
|
|
|
|
}
|
2017-08-02 15:51:24 +00:00
|
|
|
if do, ok := f.(MergeDirser); ok {
|
|
|
|
ft.MergeDirs = do.MergeDirs
|
|
|
|
}
|
2017-01-13 17:21:47 +00:00
|
|
|
if do, ok := f.(CleanUpper); ok {
|
|
|
|
ft.CleanUp = do.CleanUp
|
|
|
|
}
|
2017-06-05 15:14:24 +00:00
|
|
|
if do, ok := f.(ListRer); ok {
|
|
|
|
ft.ListR = do.ListR
|
|
|
|
}
|
2017-08-07 20:10:03 +00:00
|
|
|
return ft.DisableList(Config.DisableFeatures)
|
2017-01-13 17:21:47 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Mask the Features with the Fs passed in
|
|
|
|
//
|
|
|
|
// Only optional features which are implemented in both the original
|
|
|
|
// Fs AND the one passed in will be advertised. Any features which
|
|
|
|
// aren't in both will be set to false/nil, except for UnWrap which
|
|
|
|
// will be left untouched.
|
|
|
|
func (ft *Features) Mask(f Fs) *Features {
|
|
|
|
mask := f.Features()
|
|
|
|
ft.CaseInsensitive = ft.CaseInsensitive && mask.CaseInsensitive
|
|
|
|
ft.DuplicateFiles = ft.DuplicateFiles && mask.DuplicateFiles
|
|
|
|
ft.ReadMimeType = ft.ReadMimeType && mask.ReadMimeType
|
|
|
|
ft.WriteMimeType = ft.WriteMimeType && mask.WriteMimeType
|
2017-08-09 14:27:43 +00:00
|
|
|
ft.CanHaveEmptyDirectories = ft.CanHaveEmptyDirectories && mask.CanHaveEmptyDirectories
|
|
|
|
ft.BucketBased = ft.BucketBased && mask.BucketBased
|
2017-01-13 17:21:47 +00:00
|
|
|
if mask.Purge == nil {
|
|
|
|
ft.Purge = nil
|
|
|
|
}
|
|
|
|
if mask.Copy == nil {
|
|
|
|
ft.Copy = nil
|
|
|
|
}
|
|
|
|
if mask.Move == nil {
|
|
|
|
ft.Move = nil
|
|
|
|
}
|
|
|
|
if mask.DirMove == nil {
|
|
|
|
ft.DirMove = nil
|
|
|
|
}
|
2017-05-25 21:05:49 +00:00
|
|
|
if mask.DirChangeNotify == nil {
|
|
|
|
ft.DirChangeNotify = nil
|
|
|
|
}
|
2017-01-13 17:21:47 +00:00
|
|
|
// if mask.UnWrap == nil {
|
|
|
|
// ft.UnWrap = nil
|
|
|
|
// }
|
|
|
|
if mask.DirCacheFlush == nil {
|
|
|
|
ft.DirCacheFlush = nil
|
|
|
|
}
|
|
|
|
if mask.PutUnchecked == nil {
|
|
|
|
ft.PutUnchecked = nil
|
|
|
|
}
|
2017-08-03 19:42:35 +00:00
|
|
|
if mask.PutStream == nil {
|
|
|
|
ft.PutStream = nil
|
|
|
|
}
|
2017-08-02 15:51:24 +00:00
|
|
|
if mask.MergeDirs == nil {
|
|
|
|
ft.MergeDirs = nil
|
|
|
|
}
|
2017-01-13 17:21:47 +00:00
|
|
|
if mask.CleanUp == nil {
|
|
|
|
ft.CleanUp = nil
|
|
|
|
}
|
2017-06-05 15:14:24 +00:00
|
|
|
if mask.ListR == nil {
|
|
|
|
ft.ListR = nil
|
|
|
|
}
|
2017-08-07 20:10:03 +00:00
|
|
|
return ft.DisableList(Config.DisableFeatures)
|
2017-01-13 17:21:47 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Wrap makes a Copy of the features passed in, overriding the UnWrap
|
|
|
|
// method only if available in f.
|
|
|
|
func (ft *Features) Wrap(f Fs) *Features {
|
|
|
|
copy := new(Features)
|
|
|
|
*copy = *ft
|
|
|
|
if do, ok := f.(UnWrapper); ok {
|
|
|
|
copy.UnWrap = do.UnWrap
|
|
|
|
}
|
|
|
|
return copy
|
|
|
|
}
|
|
|
|
|
2015-09-22 17:47:16 +00:00
|
|
|
// Purger is an optional interfaces for Fs
|
2013-01-18 17:01:47 +00:00
|
|
|
type Purger interface {
|
|
|
|
// Purge all files in the root and the root directory
|
|
|
|
//
|
|
|
|
// Implement this if you have a way of deleting all the files
|
|
|
|
// quicker than just running Remove() on the result of List()
|
2014-07-28 20:02:00 +00:00
|
|
|
//
|
|
|
|
// Return an error if it doesn't exist
|
2013-01-18 17:01:47 +00:00
|
|
|
Purge() error
|
|
|
|
}
|
|
|
|
|
2015-09-22 17:47:16 +00:00
|
|
|
// Copier is an optional interface for Fs
|
2015-02-14 18:48:08 +00:00
|
|
|
type Copier interface {
|
|
|
|
// Copy src to this remote using server side copy operations.
|
|
|
|
//
|
|
|
|
// This is stored with the remote path given
|
|
|
|
//
|
|
|
|
// It returns the destination Object and a possible error
|
|
|
|
//
|
|
|
|
// Will only be called if src.Fs().Name() == f.Name()
|
|
|
|
//
|
|
|
|
// If it isn't possible then return fs.ErrorCantCopy
|
|
|
|
Copy(src Object, remote string) (Object, error)
|
|
|
|
}
|
|
|
|
|
2015-09-22 17:47:16 +00:00
|
|
|
// Mover is an optional interface for Fs
|
2015-08-24 20:42:23 +00:00
|
|
|
type Mover interface {
|
|
|
|
// Move src to this remote using server side move operations.
|
|
|
|
//
|
|
|
|
// This is stored with the remote path given
|
|
|
|
//
|
|
|
|
// It returns the destination Object and a possible error
|
|
|
|
//
|
|
|
|
// Will only be called if src.Fs().Name() == f.Name()
|
|
|
|
//
|
|
|
|
// If it isn't possible then return fs.ErrorCantMove
|
|
|
|
Move(src Object, remote string) (Object, error)
|
|
|
|
}
|
|
|
|
|
2015-09-22 17:47:16 +00:00
|
|
|
// DirMover is an optional interface for Fs
|
2015-08-24 20:42:23 +00:00
|
|
|
type DirMover interface {
|
2017-02-05 21:20:56 +00:00
|
|
|
// DirMove moves src, srcRemote to this remote at dstRemote
|
|
|
|
// using server side move operations.
|
2015-08-24 20:42:23 +00:00
|
|
|
//
|
|
|
|
// Will only be called if src.Fs().Name() == f.Name()
|
|
|
|
//
|
|
|
|
// If it isn't possible then return fs.ErrorCantDirMove
|
|
|
|
//
|
|
|
|
// If destination exists then return fs.ErrorDirExists
|
2017-02-05 21:20:56 +00:00
|
|
|
DirMove(src Fs, srcRemote, dstRemote string) error
|
2015-08-24 20:42:23 +00:00
|
|
|
}
|
|
|
|
|
2017-05-25 21:05:49 +00:00
|
|
|
// DirChangeNotifier is an optional interface for Fs
|
|
|
|
type DirChangeNotifier interface {
|
|
|
|
// DirChangeNotify calls the passed function with a path
|
|
|
|
// of a directory that has had changes. If the implementation
|
|
|
|
// uses polling, it should adhere to the given interval.
|
|
|
|
DirChangeNotify(func(string), time.Duration) chan bool
|
|
|
|
}
|
|
|
|
|
2015-11-08 15:27:55 +00:00
|
|
|
// UnWrapper is an optional interfaces for Fs
|
|
|
|
type UnWrapper interface {
|
|
|
|
// UnWrap returns the Fs that this Fs is wrapping
|
|
|
|
UnWrap() Fs
|
|
|
|
}
|
|
|
|
|
2016-12-09 15:39:29 +00:00
|
|
|
// DirCacheFlusher is an optional interface for Fs
|
|
|
|
type DirCacheFlusher interface {
|
|
|
|
// DirCacheFlush resets the directory cache - used in testing
|
|
|
|
// as an optional interface
|
|
|
|
DirCacheFlush()
|
|
|
|
}
|
|
|
|
|
2016-06-12 14:06:27 +00:00
|
|
|
// PutUncheckeder is an optional interface for Fs
|
|
|
|
type PutUncheckeder interface {
|
|
|
|
// Put in to the remote path with the modTime given of the given size
|
|
|
|
//
|
|
|
|
// May create the object even if it returns an error - if so
|
|
|
|
// will return the object and the error, otherwise will return
|
|
|
|
// nil and the error
|
|
|
|
//
|
|
|
|
// May create duplicates or return errors if src already
|
|
|
|
// exists.
|
2017-05-28 11:44:22 +00:00
|
|
|
PutUnchecked(in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error)
|
2016-06-12 14:06:27 +00:00
|
|
|
}
|
|
|
|
|
2017-08-03 19:42:35 +00:00
|
|
|
// PutStreamer is an optional interface for Fs
|
|
|
|
type PutStreamer interface {
|
|
|
|
// PutStream uploads to the remote path with the modTime given of indeterminate size
|
|
|
|
//
|
|
|
|
// May create the object even if it returns an error - if so
|
|
|
|
// will return the object and the error, otherwise will return
|
|
|
|
// nil and the error
|
|
|
|
PutStream(in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error)
|
|
|
|
}
|
|
|
|
|
2017-08-02 15:51:24 +00:00
|
|
|
// MergeDirser is an option interface for Fs
|
|
|
|
type MergeDirser interface {
|
|
|
|
// MergeDirs merges the contents of all the directories passed
|
|
|
|
// in into the first one and rmdirs the other directories.
|
|
|
|
MergeDirs([]Directory) error
|
|
|
|
}
|
|
|
|
|
2016-07-01 15:35:36 +00:00
|
|
|
// CleanUpper is an optional interfaces for Fs
|
|
|
|
type CleanUpper interface {
|
|
|
|
// CleanUp the trash in the Fs
|
|
|
|
//
|
|
|
|
// Implement this if you have a way of emptying the trash or
|
|
|
|
// otherwise cleaning up old versions of files.
|
|
|
|
CleanUp() error
|
|
|
|
}
|
|
|
|
|
2017-06-05 15:14:24 +00:00
|
|
|
// ListRer is an optional interfaces for Fs
|
|
|
|
type ListRer interface {
|
|
|
|
// ListR lists the objects and directories of the Fs starting
|
|
|
|
// from dir recursively into out.
|
|
|
|
//
|
|
|
|
// dir should be "" to start from the root, and should not
|
|
|
|
// have trailing slashes.
|
|
|
|
//
|
2017-06-11 21:43:31 +00:00
|
|
|
// This should return ErrDirNotFound if the directory isn't
|
|
|
|
// found.
|
|
|
|
//
|
|
|
|
// It should call callback for each tranche of entries read.
|
|
|
|
// These need not be returned in any particular order. If
|
|
|
|
// callback returns an error then the listing will stop
|
|
|
|
// immediately.
|
2017-06-05 15:14:24 +00:00
|
|
|
//
|
|
|
|
// Don't implement this unless you have a more efficient way
|
|
|
|
// of listing recursively that doing a directory traversal.
|
2017-06-11 21:43:31 +00:00
|
|
|
ListR(dir string, callback ListRCallback) error
|
2017-06-05 15:14:24 +00:00
|
|
|
}
|
|
|
|
|
2015-09-22 17:47:16 +00:00
|
|
|
// ObjectsChan is a channel of Objects
|
2013-06-28 07:57:32 +00:00
|
|
|
type ObjectsChan chan Object
|
2012-12-26 12:23:58 +00:00
|
|
|
|
2015-09-22 17:47:16 +00:00
|
|
|
// Objects is a slice of Object~s
|
2013-06-28 07:57:32 +00:00
|
|
|
type Objects []Object
|
2012-12-26 12:23:58 +00:00
|
|
|
|
2015-09-22 17:47:16 +00:00
|
|
|
// ObjectPair is a pair of Objects used to describe a potential copy
|
|
|
|
// operation.
|
2014-03-28 17:56:04 +00:00
|
|
|
type ObjectPair struct {
|
|
|
|
src, dst Object
|
|
|
|
}
|
|
|
|
|
2015-09-22 17:47:16 +00:00
|
|
|
// ObjectPairChan is a channel of ObjectPair
|
2014-03-28 17:56:04 +00:00
|
|
|
type ObjectPairChan chan ObjectPair
|
|
|
|
|
2015-09-22 17:47:16 +00:00
|
|
|
// Find looks for an Info object for the name passed in
|
2014-03-15 16:06:11 +00:00
|
|
|
//
|
|
|
|
// Services are looked up in the config file
|
2016-02-18 11:35:25 +00:00
|
|
|
func Find(name string) (*RegInfo, error) {
|
2014-03-15 16:06:11 +00:00
|
|
|
for _, item := range fsRegistry {
|
|
|
|
if item.Name == name {
|
|
|
|
return item, nil
|
|
|
|
}
|
|
|
|
}
|
2016-06-12 14:06:02 +00:00
|
|
|
return nil, errors.Errorf("didn't find filing system for %q", name)
|
2014-03-15 16:06:11 +00:00
|
|
|
}
|
|
|
|
|
2016-10-08 10:26:14 +00:00
|
|
|
// MustFind looks for an Info object for the type name passed in
|
|
|
|
//
|
|
|
|
// Services are looked up in the config file
|
|
|
|
//
|
|
|
|
// Exits with a fatal error if not found
|
|
|
|
func MustFind(name string) *RegInfo {
|
|
|
|
fs, err := Find(name)
|
|
|
|
if err != nil {
|
|
|
|
log.Fatalf("Failed to find remote: %v", err)
|
|
|
|
}
|
|
|
|
return fs
|
|
|
|
}
|
|
|
|
|
2015-02-07 15:49:09 +00:00
|
|
|
// Pattern to match an rclone url
|
2015-10-12 16:54:09 +00:00
|
|
|
var matcher = regexp.MustCompile(`^([\w_ -]+):(.*)$`)
|
2015-02-07 15:49:09 +00:00
|
|
|
|
2016-06-21 17:01:53 +00:00
|
|
|
// ParseRemote deconstructs a path into configName, fsPath, looking up
|
|
|
|
// the fsName in the config file (returning NotFoundInConfigFile if not found)
|
|
|
|
func ParseRemote(path string) (fsInfo *RegInfo, configName, fsPath string, err error) {
|
|
|
|
parts := matcher.FindStringSubmatch(path)
|
|
|
|
var fsName string
|
|
|
|
fsName, configName, fsPath = "local", "local", path
|
|
|
|
if parts != nil && !isDriveLetter(parts[1]) {
|
|
|
|
configName, fsPath = parts[1], parts[2]
|
2016-12-20 18:03:09 +00:00
|
|
|
fsName = ConfigFileGet(configName, "type")
|
|
|
|
if fsName == "" {
|
2016-06-21 17:01:53 +00:00
|
|
|
return nil, "", "", ErrorNotFoundInConfigFile
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// change native directory separators to / if there are any
|
|
|
|
fsPath = filepath.ToSlash(fsPath)
|
|
|
|
fsInfo, err = Find(fsName)
|
|
|
|
return fsInfo, configName, fsPath, err
|
|
|
|
}
|
|
|
|
|
2012-12-29 11:35:41 +00:00
|
|
|
// NewFs makes a new Fs object from the path
|
|
|
|
//
|
2014-07-31 20:24:52 +00:00
|
|
|
// The path is of the form remote:path
|
2014-03-15 16:06:11 +00:00
|
|
|
//
|
2014-07-31 20:24:52 +00:00
|
|
|
// Remotes are looked up in the config file. If the remote isn't
|
|
|
|
// found then NotFoundInConfigFile will be returned.
|
2015-02-07 15:49:09 +00:00
|
|
|
//
|
|
|
|
// On Windows avoid single character remote names as they can be mixed
|
|
|
|
// up with drive letters.
|
2012-12-29 11:35:41 +00:00
|
|
|
func NewFs(path string) (Fs, error) {
|
2016-06-21 17:01:53 +00:00
|
|
|
fsInfo, configName, fsPath, err := ParseRemote(path)
|
2014-03-15 16:06:11 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2016-06-21 17:01:53 +00:00
|
|
|
return fsInfo.NewFs(configName, fsPath)
|
2012-12-29 11:35:41 +00:00
|
|
|
}
|
|
|
|
|
2017-08-03 19:42:35 +00:00
|
|
|
// temporaryLocalFs creates a local FS in the OS's temporary directory.
|
|
|
|
//
|
|
|
|
// No cleanup is performed, the caller must call Purge on the Fs themselves.
|
|
|
|
func temporaryLocalFs() (Fs, error) {
|
|
|
|
path, err := ioutil.TempDir("", "rclone-spool")
|
|
|
|
if err == nil {
|
|
|
|
err = os.Remove(path)
|
|
|
|
}
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
path = filepath.ToSlash(path)
|
|
|
|
return NewFs(path)
|
|
|
|
}
|
|
|
|
|
2015-11-28 18:13:08 +00:00
|
|
|
// CheckClose is a utility function used to check the return from
|
2012-12-26 12:23:58 +00:00
|
|
|
// Close in a defer statement.
|
2015-11-28 18:13:08 +00:00
|
|
|
func CheckClose(c io.Closer, err *error) {
|
2012-12-26 12:23:58 +00:00
|
|
|
cerr := c.Close()
|
|
|
|
if *err == nil {
|
|
|
|
*err = cerr
|
|
|
|
}
|
|
|
|
}
|
2017-11-13 13:46:31 +00:00
|
|
|
|
|
|
|
// FileExists returns true if a file remote exists.
|
|
|
|
// If remote is a directory, FileExists returns false.
|
|
|
|
func FileExists(fs Fs, remote string) (bool, error) {
|
|
|
|
_, err := fs.NewObject(remote)
|
|
|
|
if err != nil {
|
|
|
|
if err == ErrorObjectNotFound || err == ErrorNotAFile {
|
|
|
|
return false, nil
|
|
|
|
}
|
|
|
|
return false, err
|
|
|
|
}
|
|
|
|
return true, nil
|
|
|
|
}
|