2015-09-22 17:47:16 +00:00
|
|
|
// Package fs is a generic file system interface for rclone object storage systems
|
2013-06-27 19:13:07 +00:00
|
|
|
package fs
|
2012-12-26 12:23:58 +00:00
|
|
|
|
|
|
|
import (
|
2012-12-31 16:40:34 +00:00
|
|
|
"fmt"
|
2012-12-26 12:23:58 +00:00
|
|
|
"io"
|
2012-12-31 16:40:34 +00:00
|
|
|
"log"
|
2016-04-21 19:06:21 +00:00
|
|
|
"math"
|
2015-03-02 17:04:34 +00:00
|
|
|
"path/filepath"
|
2013-06-27 19:13:07 +00:00
|
|
|
"regexp"
|
2016-02-15 18:11:53 +00:00
|
|
|
"sort"
|
2016-04-21 19:06:21 +00:00
|
|
|
"strings"
|
|
|
|
"sync"
|
2012-12-26 12:23:58 +00:00
|
|
|
"time"
|
|
|
|
)
|
|
|
|
|
2014-07-13 17:58:06 +00:00
|
|
|
// Constants
|
|
|
|
const (
|
2015-09-22 17:47:16 +00:00
|
|
|
// UserAgent for Fs which can set it
|
2014-07-13 17:58:06 +00:00
|
|
|
UserAgent = "rclone/" + Version
|
2015-09-22 17:47:16 +00:00
|
|
|
// ModTimeNotSupported is a very large precision value to show
|
|
|
|
// mod time isn't supported on this Fs
|
2015-08-16 22:24:34 +00:00
|
|
|
ModTimeNotSupported = 100 * 365 * 24 * time.Hour
|
2016-04-21 19:06:21 +00:00
|
|
|
// MaxLevel is a sentinel representing an infinite depth for listings
|
|
|
|
MaxLevel = math.MaxInt32
|
2014-07-13 17:58:06 +00:00
|
|
|
)
|
|
|
|
|
2013-06-27 19:13:07 +00:00
|
|
|
// Globals
|
|
|
|
var (
|
|
|
|
// Filesystem registry
|
2016-02-18 11:35:25 +00:00
|
|
|
fsRegistry []*RegInfo
|
2015-09-22 17:47:16 +00:00
|
|
|
// ErrorNotFoundInConfigFile is returned by NewFs if not found in config file
|
|
|
|
ErrorNotFoundInConfigFile = fmt.Errorf("Didn't find section in config file")
|
2015-11-08 14:16:00 +00:00
|
|
|
ErrorCantPurge = fmt.Errorf("Can't purge directory")
|
2015-09-22 17:47:16 +00:00
|
|
|
ErrorCantCopy = fmt.Errorf("Can't copy object - incompatible remotes")
|
2016-02-09 16:15:50 +00:00
|
|
|
ErrorCantMove = fmt.Errorf("Can't move object - incompatible remotes")
|
|
|
|
ErrorCantDirMove = fmt.Errorf("Can't move directory - incompatible remotes")
|
2015-09-22 17:47:16 +00:00
|
|
|
ErrorDirExists = fmt.Errorf("Can't copy directory - destination already exists")
|
2016-03-22 15:07:10 +00:00
|
|
|
ErrorCantSetModTime = fmt.Errorf("Can't set modified time")
|
2016-04-21 19:06:21 +00:00
|
|
|
ErrorDirNotFound = fmt.Errorf("Directory not found")
|
|
|
|
ErrorLevelNotSupported = fmt.Errorf("Level value not supported")
|
|
|
|
ErrorListAborted = fmt.Errorf("List aborted")
|
2016-04-23 20:46:52 +00:00
|
|
|
ErrorListOnlyRoot = fmt.Errorf("Can only list from root")
|
2013-06-27 19:13:07 +00:00
|
|
|
)
|
|
|
|
|
2016-02-18 11:35:25 +00:00
|
|
|
// RegInfo provides information about a filesystem
|
|
|
|
type RegInfo struct {
|
2014-05-05 18:52:52 +00:00
|
|
|
// Name of this fs
|
|
|
|
Name string
|
2016-02-15 18:11:53 +00:00
|
|
|
// Description of this fs - defaults to Name
|
|
|
|
Description string
|
2014-05-05 18:52:52 +00:00
|
|
|
// Create a new file system. If root refers to an existing
|
|
|
|
// object, then it should return a Fs which only returns that
|
|
|
|
// object.
|
|
|
|
NewFs func(name string, root string) (Fs, error)
|
|
|
|
// Function to call to help with config
|
|
|
|
Config func(string)
|
|
|
|
// Options for the Fs configuration
|
2014-03-15 16:06:11 +00:00
|
|
|
Options []Option
|
2013-06-27 19:13:07 +00:00
|
|
|
}
|
|
|
|
|
2015-09-22 17:47:16 +00:00
|
|
|
// Option is describes an option for the config wizard
|
2014-03-15 16:06:11 +00:00
|
|
|
type Option struct {
|
|
|
|
Name string
|
|
|
|
Help string
|
|
|
|
Optional bool
|
2016-02-15 18:11:53 +00:00
|
|
|
Examples OptionExamples
|
2014-03-15 16:06:11 +00:00
|
|
|
}
|
|
|
|
|
2016-02-15 18:11:53 +00:00
|
|
|
// OptionExamples is a slice of examples
|
|
|
|
type OptionExamples []OptionExample
|
|
|
|
|
|
|
|
// Len is part of sort.Interface.
|
|
|
|
func (os OptionExamples) Len() int { return len(os) }
|
|
|
|
|
|
|
|
// Swap is part of sort.Interface.
|
|
|
|
func (os OptionExamples) Swap(i, j int) { os[i], os[j] = os[j], os[i] }
|
|
|
|
|
|
|
|
// Less is part of sort.Interface.
|
|
|
|
func (os OptionExamples) Less(i, j int) bool { return os[i].Help < os[j].Help }
|
|
|
|
|
|
|
|
// Sort sorts an OptionExamples
|
|
|
|
func (os OptionExamples) Sort() { sort.Sort(os) }
|
|
|
|
|
2015-09-22 17:47:16 +00:00
|
|
|
// OptionExample describes an example for an Option
|
2014-03-15 16:06:11 +00:00
|
|
|
type OptionExample struct {
|
|
|
|
Value string
|
|
|
|
Help string
|
|
|
|
}
|
|
|
|
|
2013-06-27 19:13:07 +00:00
|
|
|
// Register a filesystem
|
|
|
|
//
|
|
|
|
// Fs modules should use this in an init() function
|
2016-02-18 11:35:25 +00:00
|
|
|
func Register(info *RegInfo) {
|
2014-03-15 16:06:11 +00:00
|
|
|
fsRegistry = append(fsRegistry, info)
|
2013-06-27 19:13:07 +00:00
|
|
|
}
|
|
|
|
|
2015-09-22 17:47:16 +00:00
|
|
|
// Fs is the interface a cloud storage system must provide
|
2012-12-26 12:23:58 +00:00
|
|
|
type Fs interface {
|
2016-02-18 11:35:25 +00:00
|
|
|
Info
|
2013-01-18 18:54:19 +00:00
|
|
|
|
2016-04-23 20:46:52 +00:00
|
|
|
// List the objects and directories of the Fs starting from dir
|
2016-04-21 19:06:21 +00:00
|
|
|
//
|
2016-04-23 20:46:52 +00:00
|
|
|
// dir should be "" to start from the root, and should not
|
|
|
|
// have trailing slashes.
|
|
|
|
//
|
|
|
|
// This should return ErrDirNotFound (using out.SetError())
|
|
|
|
// if the directory isn't found.
|
|
|
|
List(out ListOpts, dir string)
|
2013-01-23 22:43:20 +00:00
|
|
|
|
2015-09-22 17:47:16 +00:00
|
|
|
// NewFsObject finds the Object at remote. Returns nil if can't be found
|
2013-06-28 07:57:32 +00:00
|
|
|
NewFsObject(remote string) Object
|
2013-01-18 18:54:19 +00:00
|
|
|
|
|
|
|
// Put in to the remote path with the modTime given of the given size
|
|
|
|
//
|
|
|
|
// May create the object even if it returns an error - if so
|
|
|
|
// will return the object and the error, otherwise will return
|
|
|
|
// nil and the error
|
2016-02-18 11:35:25 +00:00
|
|
|
Put(in io.Reader, src ObjectInfo) (Object, error)
|
2013-01-18 18:54:19 +00:00
|
|
|
|
2015-09-22 17:47:16 +00:00
|
|
|
// Mkdir makes the directory (container, bucket)
|
2014-07-24 21:50:11 +00:00
|
|
|
//
|
|
|
|
// Shouldn't return an error if it already exists
|
2012-12-26 12:23:58 +00:00
|
|
|
Mkdir() error
|
2013-01-18 18:54:19 +00:00
|
|
|
|
2015-09-22 17:47:16 +00:00
|
|
|
// Rmdir removes the directory (container, bucket) if empty
|
2014-07-24 21:50:11 +00:00
|
|
|
//
|
2014-07-28 20:02:00 +00:00
|
|
|
// Return an error if it doesn't exist or isn't empty
|
2012-12-26 12:23:58 +00:00
|
|
|
Rmdir() error
|
2016-02-18 11:35:25 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Info provides an interface to reading information about a filesystem.
|
|
|
|
type Info interface {
|
|
|
|
// Name of the remote (as passed into NewFs)
|
|
|
|
Name() string
|
|
|
|
|
|
|
|
// Root of the remote (as passed into NewFs)
|
|
|
|
Root() string
|
|
|
|
|
|
|
|
// String returns a description of the FS
|
|
|
|
String() string
|
2013-01-18 23:21:02 +00:00
|
|
|
|
|
|
|
// Precision of the ModTimes in this Fs
|
|
|
|
Precision() time.Duration
|
2016-01-11 12:39:33 +00:00
|
|
|
|
|
|
|
// Returns the supported hash types of the filesystem
|
|
|
|
Hashes() HashSet
|
2012-12-26 12:23:58 +00:00
|
|
|
}
|
|
|
|
|
2015-09-22 17:47:16 +00:00
|
|
|
// Object is a filesystem like object provided by an Fs
|
2013-06-28 07:57:32 +00:00
|
|
|
type Object interface {
|
2016-02-18 11:35:25 +00:00
|
|
|
ObjectInfo
|
|
|
|
|
2014-03-28 17:56:04 +00:00
|
|
|
// String returns a description of the Object
|
|
|
|
String() string
|
|
|
|
|
2016-02-18 11:35:25 +00:00
|
|
|
// SetModTime sets the metadata on the object to set the modification date
|
2016-03-22 15:07:10 +00:00
|
|
|
SetModTime(time.Time) error
|
2016-02-18 11:35:25 +00:00
|
|
|
|
|
|
|
// Open opens the file for read. Call Close() on the returned io.ReadCloser
|
|
|
|
Open() (io.ReadCloser, error)
|
|
|
|
|
|
|
|
// Update in to the object with the modTime given of the given size
|
|
|
|
Update(in io.Reader, src ObjectInfo) error
|
|
|
|
|
|
|
|
// Removes this object
|
|
|
|
Remove() error
|
|
|
|
}
|
|
|
|
|
|
|
|
// ObjectInfo contains information about an object.
|
|
|
|
type ObjectInfo interface {
|
|
|
|
// Fs returns read only access to the Fs that this object is part of
|
|
|
|
Fs() Info
|
2014-03-28 17:56:04 +00:00
|
|
|
|
2013-01-18 18:54:19 +00:00
|
|
|
// Remote returns the remote path
|
2012-12-26 12:23:58 +00:00
|
|
|
Remote() string
|
2013-01-18 18:54:19 +00:00
|
|
|
|
2016-02-18 11:35:25 +00:00
|
|
|
// Hash returns the selected checksum of the file
|
|
|
|
// If no checksum is available it returns ""
|
2016-01-11 12:39:33 +00:00
|
|
|
Hash(HashType) (string, error)
|
2013-01-18 18:54:19 +00:00
|
|
|
|
|
|
|
// ModTime returns the modification date of the file
|
2015-08-16 22:24:34 +00:00
|
|
|
// It should return a best guess if one isn't available
|
2013-01-02 15:21:55 +00:00
|
|
|
ModTime() time.Time
|
2013-01-18 18:54:19 +00:00
|
|
|
|
|
|
|
// Size returns the size of the file
|
2012-12-26 12:23:58 +00:00
|
|
|
Size() int64
|
2013-01-18 18:54:19 +00:00
|
|
|
|
|
|
|
// Storable says whether this object can be stored
|
2012-12-26 12:23:58 +00:00
|
|
|
Storable() bool
|
|
|
|
}
|
|
|
|
|
2015-09-22 17:47:16 +00:00
|
|
|
// Purger is an optional interfaces for Fs
|
2013-01-18 17:01:47 +00:00
|
|
|
type Purger interface {
|
|
|
|
// Purge all files in the root and the root directory
|
|
|
|
//
|
|
|
|
// Implement this if you have a way of deleting all the files
|
|
|
|
// quicker than just running Remove() on the result of List()
|
2014-07-28 20:02:00 +00:00
|
|
|
//
|
|
|
|
// Return an error if it doesn't exist
|
2013-01-18 17:01:47 +00:00
|
|
|
Purge() error
|
|
|
|
}
|
|
|
|
|
2015-09-22 17:47:16 +00:00
|
|
|
// Copier is an optional interface for Fs
|
2015-02-14 18:48:08 +00:00
|
|
|
type Copier interface {
|
|
|
|
// Copy src to this remote using server side copy operations.
|
|
|
|
//
|
|
|
|
// This is stored with the remote path given
|
|
|
|
//
|
|
|
|
// It returns the destination Object and a possible error
|
|
|
|
//
|
|
|
|
// Will only be called if src.Fs().Name() == f.Name()
|
|
|
|
//
|
|
|
|
// If it isn't possible then return fs.ErrorCantCopy
|
|
|
|
Copy(src Object, remote string) (Object, error)
|
|
|
|
}
|
|
|
|
|
2015-09-22 17:47:16 +00:00
|
|
|
// Mover is an optional interface for Fs
|
2015-08-24 20:42:23 +00:00
|
|
|
type Mover interface {
|
|
|
|
// Move src to this remote using server side move operations.
|
|
|
|
//
|
|
|
|
// This is stored with the remote path given
|
|
|
|
//
|
|
|
|
// It returns the destination Object and a possible error
|
|
|
|
//
|
|
|
|
// Will only be called if src.Fs().Name() == f.Name()
|
|
|
|
//
|
|
|
|
// If it isn't possible then return fs.ErrorCantMove
|
|
|
|
Move(src Object, remote string) (Object, error)
|
|
|
|
}
|
|
|
|
|
2015-09-22 17:47:16 +00:00
|
|
|
// DirMover is an optional interface for Fs
|
2015-08-24 20:42:23 +00:00
|
|
|
type DirMover interface {
|
2015-09-22 17:47:16 +00:00
|
|
|
// DirMove moves src to this remote using server side move
|
|
|
|
// operations.
|
2015-08-24 20:42:23 +00:00
|
|
|
//
|
|
|
|
// Will only be called if src.Fs().Name() == f.Name()
|
|
|
|
//
|
|
|
|
// If it isn't possible then return fs.ErrorCantDirMove
|
|
|
|
//
|
|
|
|
// If destination exists then return fs.ErrorDirExists
|
|
|
|
DirMove(src Fs) error
|
|
|
|
}
|
|
|
|
|
2015-11-08 15:27:55 +00:00
|
|
|
// UnWrapper is an optional interfaces for Fs
|
|
|
|
type UnWrapper interface {
|
|
|
|
// UnWrap returns the Fs that this Fs is wrapping
|
|
|
|
UnWrap() Fs
|
|
|
|
}
|
|
|
|
|
2015-09-22 17:47:16 +00:00
|
|
|
// ObjectsChan is a channel of Objects
|
2013-06-28 07:57:32 +00:00
|
|
|
type ObjectsChan chan Object
|
2012-12-26 12:23:58 +00:00
|
|
|
|
2016-04-21 19:06:21 +00:00
|
|
|
// ListOpts describes the interface used for Fs.List operations
|
|
|
|
type ListOpts interface {
|
|
|
|
// Add an object to the output.
|
|
|
|
// If the function returns true, the operation has been aborted.
|
|
|
|
// Multiple goroutines can safely add objects concurrently.
|
|
|
|
Add(obj Object) (abort bool)
|
|
|
|
|
|
|
|
// Add a directory to the output.
|
|
|
|
// If the function returns true, the operation has been aborted.
|
|
|
|
// Multiple goroutines can safely add objects concurrently.
|
|
|
|
AddDir(dir *Dir) (abort bool)
|
|
|
|
|
|
|
|
// IncludeDirectory returns whether this directory should be
|
|
|
|
// included in the listing (and recursed into or not).
|
|
|
|
IncludeDirectory(remote string) bool
|
|
|
|
|
|
|
|
// SetError will set an error state, and will cause the listing to
|
|
|
|
// be aborted.
|
|
|
|
// Multiple goroutines can set the error state concurrently,
|
|
|
|
// but only the first will be returned to the caller.
|
|
|
|
SetError(err error)
|
|
|
|
|
|
|
|
// Level returns the level it should recurse to. Fses may
|
|
|
|
// ignore this in which case the listing will be less
|
|
|
|
// efficient.
|
|
|
|
Level() int
|
|
|
|
|
|
|
|
// Buffer returns the channel depth in use
|
|
|
|
Buffer() int
|
|
|
|
|
|
|
|
// Finished should be called when listing is finished
|
|
|
|
Finished()
|
|
|
|
|
|
|
|
// IsFinished returns whether Finished or SetError have been called
|
|
|
|
IsFinished() bool
|
|
|
|
}
|
|
|
|
|
|
|
|
// listerResult is returned by the lister methods
|
|
|
|
type listerResult struct {
|
|
|
|
Obj Object
|
|
|
|
Dir *Dir
|
|
|
|
Err error
|
|
|
|
}
|
|
|
|
|
|
|
|
// Lister objects are used for controlling listing of Fs objects
|
|
|
|
type Lister struct {
|
|
|
|
mu sync.RWMutex
|
|
|
|
buffer int
|
|
|
|
abort bool
|
|
|
|
results chan listerResult
|
|
|
|
finished sync.Once
|
|
|
|
level int
|
|
|
|
filter *Filter
|
|
|
|
}
|
|
|
|
|
|
|
|
// NewLister creates a Lister object.
|
|
|
|
//
|
|
|
|
// The default channel buffer size will be Config.Checkers unless
|
|
|
|
// overridden with SetBuffer. The default level will be infinite.
|
|
|
|
func NewLister() *Lister {
|
|
|
|
o := &Lister{}
|
|
|
|
return o.SetLevel(-1).SetBuffer(Config.Checkers)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Start starts a go routine listing the Fs passed in. It returns the
|
|
|
|
// same Lister that was passed in for convenience.
|
2016-04-23 20:46:52 +00:00
|
|
|
func (o *Lister) Start(f Fs, dir string) *Lister {
|
2016-04-21 19:06:21 +00:00
|
|
|
o.results = make(chan listerResult, o.buffer)
|
|
|
|
go func() {
|
2016-04-23 20:46:52 +00:00
|
|
|
f.List(o, dir)
|
2016-04-21 19:06:21 +00:00
|
|
|
}()
|
|
|
|
return o
|
|
|
|
}
|
|
|
|
|
|
|
|
// SetLevel sets the level to recurse to. It returns same Lister that
|
|
|
|
// was passed in for convenience. If Level is < 0 then it sets it to
|
|
|
|
// infinite. Must be called before Start().
|
|
|
|
func (o *Lister) SetLevel(level int) *Lister {
|
|
|
|
if level < 0 {
|
|
|
|
o.level = MaxLevel
|
|
|
|
} else {
|
|
|
|
o.level = level
|
|
|
|
}
|
|
|
|
return o
|
|
|
|
}
|
|
|
|
|
|
|
|
// SetFilter sets the Filter that is in use. It defaults to no
|
|
|
|
// filtering. Must be called before Start().
|
|
|
|
func (o *Lister) SetFilter(filter *Filter) *Lister {
|
|
|
|
o.filter = filter
|
|
|
|
return o
|
|
|
|
}
|
|
|
|
|
|
|
|
// Level gets the recursion level for this listing.
|
|
|
|
//
|
|
|
|
// Fses may ignore this, but should implement it for improved efficiency if possible.
|
|
|
|
//
|
|
|
|
// Level 1 means list just the contents of the directory
|
|
|
|
//
|
|
|
|
// Each returned item must have less than level `/`s in.
|
|
|
|
func (o *Lister) Level() int {
|
|
|
|
return o.level
|
|
|
|
}
|
|
|
|
|
|
|
|
// SetBuffer sets the channel buffer size in use. Must be called
|
|
|
|
// before Start().
|
|
|
|
func (o *Lister) SetBuffer(buffer int) *Lister {
|
|
|
|
if buffer < 1 {
|
|
|
|
buffer = 1
|
|
|
|
}
|
|
|
|
o.buffer = buffer
|
|
|
|
return o
|
|
|
|
}
|
|
|
|
|
|
|
|
// Buffer gets the channel buffer size in use
|
|
|
|
func (o *Lister) Buffer() int {
|
|
|
|
return o.buffer
|
|
|
|
}
|
|
|
|
|
|
|
|
// Add an object to the output.
|
|
|
|
// If the function returns true, the operation has been aborted.
|
|
|
|
// Multiple goroutines can safely add objects concurrently.
|
|
|
|
func (o *Lister) Add(obj Object) (abort bool) {
|
|
|
|
o.mu.RLock()
|
|
|
|
defer o.mu.RUnlock()
|
|
|
|
if o.abort {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
o.results <- listerResult{Obj: obj}
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
|
|
|
// AddDir will a directory to the output.
|
|
|
|
// If the function returns true, the operation has been aborted.
|
|
|
|
// Multiple goroutines can safely add objects concurrently.
|
|
|
|
func (o *Lister) AddDir(dir *Dir) (abort bool) {
|
|
|
|
o.mu.RLock()
|
|
|
|
defer o.mu.RUnlock()
|
|
|
|
if o.abort {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
remote := dir.Name
|
|
|
|
remote = strings.Trim(remote, "/")
|
|
|
|
dir.Name = remote
|
|
|
|
// Check the level and ignore if too high
|
|
|
|
slashes := strings.Count(remote, "/")
|
|
|
|
if slashes >= o.level {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
// Check if directory is included
|
|
|
|
if !o.IncludeDirectory(remote) {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
o.results <- listerResult{Dir: dir}
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
|
|
|
// IncludeDirectory returns whether this directory should be
|
|
|
|
// included in the listing (and recursed into or not).
|
|
|
|
func (o *Lister) IncludeDirectory(remote string) bool {
|
|
|
|
if o.filter == nil {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
return o.filter.IncludeDirectory(remote)
|
|
|
|
}
|
|
|
|
|
|
|
|
// SetError will set an error state, and will cause the listing to
|
|
|
|
// be aborted.
|
|
|
|
// Multiple goroutines can set the error state concurrently,
|
|
|
|
// but only the first will be returned to the caller.
|
|
|
|
func (o *Lister) SetError(err error) {
|
|
|
|
o.mu.RLock()
|
|
|
|
if err != nil && !o.abort {
|
|
|
|
o.results <- listerResult{Err: err}
|
|
|
|
}
|
|
|
|
o.mu.RUnlock()
|
|
|
|
o.Finished()
|
|
|
|
}
|
|
|
|
|
|
|
|
// Finished should be called when listing is finished
|
|
|
|
func (o *Lister) Finished() {
|
|
|
|
o.finished.Do(func() {
|
|
|
|
o.mu.Lock()
|
|
|
|
o.abort = true
|
|
|
|
close(o.results)
|
|
|
|
o.mu.Unlock()
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
// IsFinished returns whether the directory listing is finished or not
|
|
|
|
func (o *Lister) IsFinished() bool {
|
|
|
|
o.mu.RLock()
|
|
|
|
defer o.mu.RUnlock()
|
|
|
|
return o.abort
|
|
|
|
}
|
|
|
|
|
|
|
|
// Get an object from the listing.
|
|
|
|
// Will return either an object or a directory, never both.
|
|
|
|
// Will return (nil, nil, nil) when all objects have been returned.
|
|
|
|
func (o *Lister) Get() (Object, *Dir, error) {
|
|
|
|
select {
|
|
|
|
case r := <-o.results:
|
|
|
|
return r.Obj, r.Dir, r.Err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// GetObject will return an object from the listing.
|
|
|
|
// It will skip over any directories.
|
|
|
|
// Will return (nil, nil) when all objects have been returned.
|
|
|
|
func (o *Lister) GetObject() (Object, error) {
|
|
|
|
for {
|
|
|
|
obj, dir, err := o.Get()
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
// Check if we are finished
|
|
|
|
if dir == nil && obj == nil {
|
|
|
|
return nil, nil
|
|
|
|
}
|
|
|
|
// Ignore directories
|
|
|
|
if dir != nil {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
return obj, nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// GetObjects will return a slice of object from the listing.
|
|
|
|
// It will skip over any directories.
|
|
|
|
func (o *Lister) GetObjects() (objs []Object, err error) {
|
|
|
|
for {
|
|
|
|
obj, dir, err := o.Get()
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
// Check if we are finished
|
|
|
|
if dir == nil && obj == nil {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
if obj != nil {
|
|
|
|
objs = append(objs, obj)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return objs, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// GetDir will return a directory from the listing.
|
|
|
|
// It will skip over any objects.
|
|
|
|
// Will return (nil, nil) when all objects have been returned.
|
|
|
|
func (o *Lister) GetDir() (*Dir, error) {
|
|
|
|
for {
|
|
|
|
obj, dir, err := o.Get()
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
// Check if we are finished
|
|
|
|
if dir == nil && obj == nil {
|
|
|
|
return nil, nil
|
|
|
|
}
|
|
|
|
// Ignore objects
|
|
|
|
if obj != nil {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
return dir, nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// GetDirs will return a slice of directories from the listing.
|
|
|
|
// It will skip over any objects.
|
|
|
|
func (o *Lister) GetDirs() (dirs []*Dir, err error) {
|
|
|
|
for {
|
|
|
|
obj, dir, err := o.Get()
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
// Check if we are finished
|
|
|
|
if dir == nil && obj == nil {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
if dir != nil {
|
|
|
|
dirs = append(dirs, dir)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return dirs, nil
|
|
|
|
}
|
|
|
|
|
2015-09-22 17:47:16 +00:00
|
|
|
// Objects is a slice of Object~s
|
2013-06-28 07:57:32 +00:00
|
|
|
type Objects []Object
|
2012-12-26 12:23:58 +00:00
|
|
|
|
2015-09-22 17:47:16 +00:00
|
|
|
// ObjectPair is a pair of Objects used to describe a potential copy
|
|
|
|
// operation.
|
2014-03-28 17:56:04 +00:00
|
|
|
type ObjectPair struct {
|
|
|
|
src, dst Object
|
|
|
|
}
|
|
|
|
|
2015-09-22 17:47:16 +00:00
|
|
|
// ObjectPairChan is a channel of ObjectPair
|
2014-03-28 17:56:04 +00:00
|
|
|
type ObjectPairChan chan ObjectPair
|
|
|
|
|
2015-09-22 17:47:16 +00:00
|
|
|
// Dir describes a directory for directory/container/bucket lists
|
2013-06-28 07:57:32 +00:00
|
|
|
type Dir struct {
|
2013-01-23 22:43:20 +00:00
|
|
|
Name string // name of the directory
|
|
|
|
When time.Time // modification or creation time - IsZero for unknown
|
|
|
|
Bytes int64 // size of directory and contents -1 for unknown
|
|
|
|
Count int64 // number of objects -1 for unknown
|
|
|
|
}
|
|
|
|
|
2015-09-22 17:47:16 +00:00
|
|
|
// DirChan is a channel of Dir objects
|
2013-06-28 07:57:32 +00:00
|
|
|
type DirChan chan *Dir
|
2013-01-23 22:43:20 +00:00
|
|
|
|
2015-09-22 17:47:16 +00:00
|
|
|
// Find looks for an Info object for the name passed in
|
2014-03-15 16:06:11 +00:00
|
|
|
//
|
|
|
|
// Services are looked up in the config file
|
2016-02-18 11:35:25 +00:00
|
|
|
func Find(name string) (*RegInfo, error) {
|
2014-03-15 16:06:11 +00:00
|
|
|
for _, item := range fsRegistry {
|
|
|
|
if item.Name == name {
|
|
|
|
return item, nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil, fmt.Errorf("Didn't find filing system for %q", name)
|
|
|
|
}
|
|
|
|
|
2015-02-07 15:49:09 +00:00
|
|
|
// Pattern to match an rclone url
|
2015-10-12 16:54:09 +00:00
|
|
|
var matcher = regexp.MustCompile(`^([\w_ -]+):(.*)$`)
|
2015-02-07 15:49:09 +00:00
|
|
|
|
2012-12-29 11:35:41 +00:00
|
|
|
// NewFs makes a new Fs object from the path
|
|
|
|
//
|
2014-07-31 20:24:52 +00:00
|
|
|
// The path is of the form remote:path
|
2014-03-15 16:06:11 +00:00
|
|
|
//
|
2014-07-31 20:24:52 +00:00
|
|
|
// Remotes are looked up in the config file. If the remote isn't
|
|
|
|
// found then NotFoundInConfigFile will be returned.
|
2015-02-07 15:49:09 +00:00
|
|
|
//
|
|
|
|
// On Windows avoid single character remote names as they can be mixed
|
|
|
|
// up with drive letters.
|
2012-12-29 11:35:41 +00:00
|
|
|
func NewFs(path string) (Fs, error) {
|
2014-03-15 16:06:11 +00:00
|
|
|
parts := matcher.FindStringSubmatch(path)
|
|
|
|
fsName, configName, fsPath := "local", "local", path
|
2015-02-07 15:49:09 +00:00
|
|
|
if parts != nil && !isDriveLetter(parts[1]) {
|
2014-03-15 16:06:11 +00:00
|
|
|
configName, fsPath = parts[1], parts[2]
|
|
|
|
var err error
|
|
|
|
fsName, err = ConfigFile.GetValue(configName, "type")
|
|
|
|
if err != nil {
|
2015-09-22 17:47:16 +00:00
|
|
|
return nil, ErrorNotFoundInConfigFile
|
2013-06-27 19:13:07 +00:00
|
|
|
}
|
2013-01-14 23:38:18 +00:00
|
|
|
}
|
2014-03-15 16:06:11 +00:00
|
|
|
fs, err := Find(fsName)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2015-03-02 17:04:34 +00:00
|
|
|
// change native directory separators to / if there are any
|
|
|
|
fsPath = filepath.ToSlash(fsPath)
|
2014-03-15 16:06:11 +00:00
|
|
|
return fs.NewFs(configName, fsPath)
|
2012-12-29 11:35:41 +00:00
|
|
|
}
|
|
|
|
|
2015-09-22 17:47:16 +00:00
|
|
|
// OutputLog logs for an object
|
2014-03-28 17:56:04 +00:00
|
|
|
func OutputLog(o interface{}, text string, args ...interface{}) {
|
|
|
|
description := ""
|
2015-03-09 08:11:38 +00:00
|
|
|
if o != nil {
|
|
|
|
description = fmt.Sprintf("%v: ", o)
|
2014-03-28 17:56:04 +00:00
|
|
|
}
|
|
|
|
out := fmt.Sprintf(text, args...)
|
|
|
|
log.Print(description + out)
|
|
|
|
}
|
|
|
|
|
2015-09-22 17:47:16 +00:00
|
|
|
// Debug writes debuging output for this Object or Fs
|
2014-03-28 17:56:04 +00:00
|
|
|
func Debug(o interface{}, text string, args ...interface{}) {
|
2013-06-27 19:13:07 +00:00
|
|
|
if Config.Verbose {
|
2014-03-28 17:56:04 +00:00
|
|
|
OutputLog(o, text, args...)
|
2012-12-31 16:40:34 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-09-22 17:47:16 +00:00
|
|
|
// Log writes log output for this Object or Fs
|
2014-03-28 17:56:04 +00:00
|
|
|
func Log(o interface{}, text string, args ...interface{}) {
|
2013-06-27 19:13:07 +00:00
|
|
|
if !Config.Quiet {
|
2014-03-28 17:56:04 +00:00
|
|
|
OutputLog(o, text, args...)
|
2012-12-31 16:40:34 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-09-22 17:47:16 +00:00
|
|
|
// ErrorLog writes error log output for this Object or Fs. It
|
|
|
|
// unconditionally logs a message regardless of Config.Quiet or
|
|
|
|
// Config.Verbose.
|
2015-08-08 19:10:31 +00:00
|
|
|
func ErrorLog(o interface{}, text string, args ...interface{}) {
|
|
|
|
OutputLog(o, text, args...)
|
|
|
|
}
|
|
|
|
|
2015-11-28 18:13:08 +00:00
|
|
|
// CheckClose is a utility function used to check the return from
|
2012-12-26 12:23:58 +00:00
|
|
|
// Close in a defer statement.
|
2015-11-28 18:13:08 +00:00
|
|
|
func CheckClose(c io.Closer, err *error) {
|
2012-12-26 12:23:58 +00:00
|
|
|
cerr := c.Close()
|
|
|
|
if *err == nil {
|
|
|
|
*err = cerr
|
|
|
|
}
|
|
|
|
}
|
2016-02-18 11:35:25 +00:00
|
|
|
|
|
|
|
// NewStaticObjectInfo returns a static ObjectInfo
|
|
|
|
// If hashes is nil and fs is not nil, the hash map will be replaced with
|
|
|
|
// empty hashes of the types supported by the fs.
|
|
|
|
func NewStaticObjectInfo(remote string, modTime time.Time, size int64, storable bool, hashes map[HashType]string, fs Info) ObjectInfo {
|
|
|
|
info := &staticObjectInfo{
|
|
|
|
remote: remote,
|
|
|
|
modTime: modTime,
|
|
|
|
size: size,
|
|
|
|
storable: storable,
|
|
|
|
hashes: hashes,
|
|
|
|
fs: fs,
|
|
|
|
}
|
|
|
|
if fs != nil && hashes == nil {
|
|
|
|
set := fs.Hashes().Array()
|
|
|
|
info.hashes = make(map[HashType]string)
|
|
|
|
for _, ht := range set {
|
|
|
|
info.hashes[ht] = ""
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return info
|
|
|
|
}
|
|
|
|
|
|
|
|
type staticObjectInfo struct {
|
|
|
|
remote string
|
|
|
|
modTime time.Time
|
|
|
|
size int64
|
|
|
|
storable bool
|
|
|
|
hashes map[HashType]string
|
|
|
|
fs Info
|
|
|
|
}
|
|
|
|
|
|
|
|
func (i *staticObjectInfo) Fs() Info { return i.fs }
|
|
|
|
func (i *staticObjectInfo) Remote() string { return i.remote }
|
|
|
|
func (i *staticObjectInfo) ModTime() time.Time { return i.modTime }
|
|
|
|
func (i *staticObjectInfo) Size() int64 { return i.size }
|
|
|
|
func (i *staticObjectInfo) Storable() bool { return i.storable }
|
|
|
|
func (i *staticObjectInfo) Hash(h HashType) (string, error) {
|
|
|
|
if len(i.hashes) == 0 {
|
|
|
|
return "", ErrHashUnsupported
|
|
|
|
}
|
|
|
|
if hash, ok := i.hashes[h]; ok {
|
|
|
|
return hash, nil
|
|
|
|
}
|
|
|
|
return "", ErrHashUnsupported
|
|
|
|
}
|