2015-09-22 17:47:16 +00:00
|
|
|
// Package fs is a generic file system interface for rclone object storage systems
|
2013-06-27 19:13:07 +00:00
|
|
|
package fs
|
2012-12-26 12:23:58 +00:00
|
|
|
|
|
|
|
import (
|
2019-06-17 08:34:30 +00:00
|
|
|
"context"
|
2021-03-10 14:10:03 +00:00
|
|
|
"crypto/md5"
|
|
|
|
"encoding/base64"
|
2019-06-03 14:21:05 +00:00
|
|
|
"encoding/json"
|
2018-05-14 17:06:57 +00:00
|
|
|
"fmt"
|
2012-12-26 12:23:58 +00:00
|
|
|
"io"
|
2017-08-03 19:42:35 +00:00
|
|
|
"io/ioutil"
|
2012-12-31 16:40:34 +00:00
|
|
|
"log"
|
2016-04-21 19:06:21 +00:00
|
|
|
"math"
|
2017-08-03 19:42:35 +00:00
|
|
|
"os"
|
2015-03-02 17:04:34 +00:00
|
|
|
"path/filepath"
|
2017-08-07 20:10:03 +00:00
|
|
|
"reflect"
|
2016-02-15 18:11:53 +00:00
|
|
|
"sort"
|
2017-08-07 20:10:03 +00:00
|
|
|
"strings"
|
2012-12-26 12:23:58 +00:00
|
|
|
"time"
|
2016-06-12 14:06:02 +00:00
|
|
|
|
|
|
|
"github.com/pkg/errors"
|
2019-07-28 17:47:38 +00:00
|
|
|
"github.com/rclone/rclone/fs/config/configmap"
|
|
|
|
"github.com/rclone/rclone/fs/config/configstruct"
|
|
|
|
"github.com/rclone/rclone/fs/fserrors"
|
|
|
|
"github.com/rclone/rclone/fs/fspath"
|
|
|
|
"github.com/rclone/rclone/fs/hash"
|
|
|
|
"github.com/rclone/rclone/lib/pacer"
|
2012-12-26 12:23:58 +00:00
|
|
|
)
|
|
|
|
|
2018-03-08 20:03:34 +00:00
|
|
|
// EntryType can be associated with remote paths to identify their type
|
|
|
|
type EntryType int
|
|
|
|
|
2014-07-13 17:58:06 +00:00
|
|
|
// Constants
|
|
|
|
const (
|
2015-09-22 17:47:16 +00:00
|
|
|
// ModTimeNotSupported is a very large precision value to show
|
|
|
|
// mod time isn't supported on this Fs
|
2015-08-16 22:24:34 +00:00
|
|
|
ModTimeNotSupported = 100 * 365 * 24 * time.Hour
|
2016-04-21 19:06:21 +00:00
|
|
|
// MaxLevel is a sentinel representing an infinite depth for listings
|
|
|
|
MaxLevel = math.MaxInt32
|
2018-03-08 20:03:34 +00:00
|
|
|
// EntryDirectory should be used to classify remote paths in directories
|
|
|
|
EntryDirectory EntryType = iota // 0
|
|
|
|
// EntryObject should be used to classify remote paths in objects
|
|
|
|
EntryObject // 1
|
2014-07-13 17:58:06 +00:00
|
|
|
)
|
|
|
|
|
2013-06-27 19:13:07 +00:00
|
|
|
// Globals
|
|
|
|
var (
|
|
|
|
// Filesystem registry
|
2018-01-12 16:30:54 +00:00
|
|
|
Registry []*RegInfo
|
2015-09-22 17:47:16 +00:00
|
|
|
// ErrorNotFoundInConfigFile is returned by NewFs if not found in config file
|
2017-06-13 12:58:39 +00:00
|
|
|
ErrorNotFoundInConfigFile = errors.New("didn't find section in config file")
|
|
|
|
ErrorCantPurge = errors.New("can't purge directory")
|
|
|
|
ErrorCantCopy = errors.New("can't copy object - incompatible remotes")
|
|
|
|
ErrorCantMove = errors.New("can't move object - incompatible remotes")
|
|
|
|
ErrorCantDirMove = errors.New("can't move directory - incompatible remotes")
|
2019-06-29 01:17:53 +00:00
|
|
|
ErrorCantUploadEmptyFiles = errors.New("can't upload empty files to this remote")
|
2017-06-13 12:58:39 +00:00
|
|
|
ErrorDirExists = errors.New("can't copy directory - destination already exists")
|
|
|
|
ErrorCantSetModTime = errors.New("can't set modified time")
|
|
|
|
ErrorCantSetModTimeWithoutDelete = errors.New("can't set modified time without deleting existing object")
|
|
|
|
ErrorDirNotFound = errors.New("directory not found")
|
|
|
|
ErrorObjectNotFound = errors.New("object not found")
|
|
|
|
ErrorLevelNotSupported = errors.New("level value not supported")
|
|
|
|
ErrorListAborted = errors.New("list aborted")
|
2017-06-11 21:43:31 +00:00
|
|
|
ErrorListBucketRequired = errors.New("bucket or container name is needed in remote")
|
2017-06-13 12:58:39 +00:00
|
|
|
ErrorIsFile = errors.New("is a file not a directory")
|
2020-04-15 11:50:26 +00:00
|
|
|
ErrorNotAFile = errors.New("is not a regular file")
|
2017-06-13 12:58:39 +00:00
|
|
|
ErrorNotDeleting = errors.New("not deleting files as there were IO errors")
|
2017-08-09 20:06:39 +00:00
|
|
|
ErrorNotDeletingDirs = errors.New("not deleting directories as there were IO errors")
|
2019-02-14 12:06:26 +00:00
|
|
|
ErrorOverlapping = errors.New("can't sync or move files on overlapping remotes")
|
2017-07-25 14:18:13 +00:00
|
|
|
ErrorDirectoryNotEmpty = errors.New("directory not empty")
|
2017-09-02 08:29:01 +00:00
|
|
|
ErrorImmutableModified = errors.New("immutable file modified")
|
2018-01-16 20:00:16 +00:00
|
|
|
ErrorPermissionDenied = errors.New("permission denied")
|
2019-07-24 15:57:56 +00:00
|
|
|
ErrorCantShareDirectories = errors.New("this backend can't share directories with link")
|
2019-08-12 10:07:31 +00:00
|
|
|
ErrorNotImplemented = errors.New("optional feature not implemented")
|
2020-04-28 11:58:34 +00:00
|
|
|
ErrorCommandNotFound = errors.New("command not found")
|
2020-11-27 11:49:37 +00:00
|
|
|
ErrorFileNameTooLong = errors.New("file name too long")
|
2013-06-27 19:13:07 +00:00
|
|
|
)
|
|
|
|
|
2016-02-18 11:35:25 +00:00
|
|
|
// RegInfo provides information about a filesystem
|
|
|
|
type RegInfo struct {
|
2014-05-05 18:52:52 +00:00
|
|
|
// Name of this fs
|
|
|
|
Name string
|
2016-02-15 18:11:53 +00:00
|
|
|
// Description of this fs - defaults to Name
|
|
|
|
Description string
|
2018-05-14 17:06:57 +00:00
|
|
|
// Prefix for command line flags for this fs - defaults to Name if not set
|
|
|
|
Prefix string
|
2014-05-05 18:52:52 +00:00
|
|
|
// Create a new file system. If root refers to an existing
|
2020-05-25 06:05:53 +00:00
|
|
|
// object, then it should return an Fs which which points to
|
2016-06-21 17:01:53 +00:00
|
|
|
// the parent of that object and ErrorIsFile.
|
2020-11-05 15:18:51 +00:00
|
|
|
NewFs func(ctx context.Context, name string, root string, config configmap.Mapper) (Fs, error) `json:"-"`
|
2014-05-05 18:52:52 +00:00
|
|
|
// Function to call to help with config
|
2021-04-06 20:27:34 +00:00
|
|
|
Config func(ctx context.Context, name string, config configmap.Mapper) error `json:"-"`
|
2014-05-05 18:52:52 +00:00
|
|
|
// Options for the Fs configuration
|
2018-05-14 17:06:57 +00:00
|
|
|
Options Options
|
2020-04-28 11:58:34 +00:00
|
|
|
// The command help, if any
|
|
|
|
CommandHelp []CommandHelp
|
2013-06-27 19:13:07 +00:00
|
|
|
}
|
|
|
|
|
2018-10-01 20:05:33 +00:00
|
|
|
// FileName returns the on disk file name for this backend
|
|
|
|
func (ri *RegInfo) FileName() string {
|
|
|
|
return strings.Replace(ri.Name, " ", "", -1)
|
|
|
|
}
|
|
|
|
|
2018-05-14 17:06:57 +00:00
|
|
|
// Options is a slice of configuration Option for a backend
|
|
|
|
type Options []Option
|
|
|
|
|
|
|
|
// Set the default values for the options
|
|
|
|
func (os Options) setValues() {
|
|
|
|
for i := range os {
|
|
|
|
o := &os[i]
|
|
|
|
if o.Default == nil {
|
|
|
|
o.Default = ""
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-09-14 17:26:07 +00:00
|
|
|
// Get the Option corresponding to name or return nil if not found
|
|
|
|
func (os Options) Get(name string) *Option {
|
|
|
|
for i := range os {
|
|
|
|
opt := &os[i]
|
|
|
|
if opt.Name == name {
|
|
|
|
return opt
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2021-04-03 14:58:47 +00:00
|
|
|
// Overridden discovers which config items have been overridden in the
|
|
|
|
// configmap passed in, either by the config string, command line
|
|
|
|
// flags or environment variables
|
|
|
|
func (os Options) Overridden(m *configmap.Map) configmap.Simple {
|
|
|
|
var overridden = configmap.Simple{}
|
|
|
|
for i := range os {
|
|
|
|
opt := &os[i]
|
2021-04-04 11:41:36 +00:00
|
|
|
value, isSet := m.GetPriority(opt.Name, configmap.PriorityNormal)
|
2021-04-03 14:58:47 +00:00
|
|
|
if isSet {
|
|
|
|
overridden.Set(opt.Name, value)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return overridden
|
|
|
|
}
|
|
|
|
|
2021-04-04 13:47:57 +00:00
|
|
|
// NonDefault discovers which config values aren't at their default
|
|
|
|
func (os Options) NonDefault(m configmap.Getter) configmap.Simple {
|
|
|
|
var nonDefault = configmap.Simple{}
|
|
|
|
for i := range os {
|
|
|
|
opt := &os[i]
|
|
|
|
value, isSet := m.Get(opt.Name)
|
|
|
|
if !isSet {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
defaultValue := fmt.Sprint(opt.Default)
|
|
|
|
if value != defaultValue {
|
|
|
|
nonDefault.Set(opt.Name, value)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nonDefault
|
|
|
|
}
|
|
|
|
|
2018-05-14 17:06:57 +00:00
|
|
|
// OptionVisibility controls whether the options are visible in the
|
|
|
|
// configurator or the command line.
|
|
|
|
type OptionVisibility byte
|
|
|
|
|
|
|
|
// Constants Option.Hide
|
|
|
|
const (
|
|
|
|
OptionHideCommandLine OptionVisibility = 1 << iota
|
|
|
|
OptionHideConfigurator
|
|
|
|
OptionHideBoth = OptionHideCommandLine | OptionHideConfigurator
|
|
|
|
)
|
|
|
|
|
2015-09-22 17:47:16 +00:00
|
|
|
// Option is describes an option for the config wizard
|
2018-05-14 17:06:57 +00:00
|
|
|
//
|
|
|
|
// This also describes command line options and environment variables
|
2014-03-15 16:06:11 +00:00
|
|
|
type Option struct {
|
2018-05-14 17:06:57 +00:00
|
|
|
Name string // name of the option in snake_case
|
|
|
|
Help string // Help, the first line only is used for the command line help
|
|
|
|
Provider string // Set to filter on provider
|
|
|
|
Default interface{} // default value, nil => ""
|
|
|
|
Value interface{} // value to be set by flags
|
|
|
|
Examples OptionExamples `json:",omitempty"` // config examples
|
|
|
|
ShortOpt string // the short option for this if required
|
|
|
|
Hide OptionVisibility // set this to hide the config from the configurator or the command line
|
|
|
|
Required bool // this option is required
|
|
|
|
IsPassword bool // set if the option is a password
|
|
|
|
NoPrefix bool // set if the option for this should not use the backend prefix
|
|
|
|
Advanced bool // set if this is an advanced config option
|
|
|
|
}
|
|
|
|
|
2019-06-03 14:21:05 +00:00
|
|
|
// BaseOption is an alias for Option used internally
|
|
|
|
type BaseOption Option
|
|
|
|
|
|
|
|
// MarshalJSON turns an Option into JSON
|
|
|
|
//
|
|
|
|
// It adds some generated fields for ease of use
|
|
|
|
// - DefaultStr - a string rendering of Default
|
|
|
|
// - ValueStr - a string rendering of Value
|
|
|
|
// - Type - the type of the option
|
|
|
|
func (o *Option) MarshalJSON() ([]byte, error) {
|
|
|
|
return json.Marshal(struct {
|
|
|
|
BaseOption
|
|
|
|
DefaultStr string
|
|
|
|
ValueStr string
|
|
|
|
Type string
|
|
|
|
}{
|
|
|
|
BaseOption: BaseOption(*o),
|
|
|
|
DefaultStr: fmt.Sprint(o.Default),
|
|
|
|
ValueStr: o.String(),
|
|
|
|
Type: o.Type(),
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2018-10-01 14:52:39 +00:00
|
|
|
// GetValue gets the current current value which is the default if not set
|
|
|
|
func (o *Option) GetValue() interface{} {
|
2018-05-14 17:06:57 +00:00
|
|
|
val := o.Value
|
|
|
|
if val == nil {
|
|
|
|
val = o.Default
|
2019-09-14 17:26:07 +00:00
|
|
|
if val == nil {
|
|
|
|
val = ""
|
|
|
|
}
|
2018-05-14 17:06:57 +00:00
|
|
|
}
|
|
|
|
return val
|
|
|
|
}
|
|
|
|
|
|
|
|
// String turns Option into a string
|
|
|
|
func (o *Option) String() string {
|
2018-10-01 14:52:39 +00:00
|
|
|
return fmt.Sprint(o.GetValue())
|
2018-05-14 17:06:57 +00:00
|
|
|
}
|
|
|
|
|
2020-05-25 06:05:53 +00:00
|
|
|
// Set an Option from a string
|
2018-05-14 17:06:57 +00:00
|
|
|
func (o *Option) Set(s string) (err error) {
|
2018-10-01 14:52:39 +00:00
|
|
|
newValue, err := configstruct.StringToInterface(o.GetValue(), s)
|
2018-05-14 17:06:57 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
o.Value = newValue
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Type of the value
|
|
|
|
func (o *Option) Type() string {
|
2018-10-01 14:52:39 +00:00
|
|
|
return reflect.TypeOf(o.GetValue()).Name()
|
2014-03-15 16:06:11 +00:00
|
|
|
}
|
|
|
|
|
2018-09-24 20:27:30 +00:00
|
|
|
// FlagName for the option
|
|
|
|
func (o *Option) FlagName(prefix string) string {
|
|
|
|
name := strings.Replace(o.Name, "_", "-", -1) // convert snake_case to kebab-case
|
|
|
|
if !o.NoPrefix {
|
|
|
|
name = prefix + "-" + name
|
|
|
|
}
|
|
|
|
return name
|
|
|
|
}
|
|
|
|
|
|
|
|
// EnvVarName for the option
|
|
|
|
func (o *Option) EnvVarName(prefix string) string {
|
|
|
|
return OptionToEnv(prefix + "-" + o.Name)
|
|
|
|
}
|
|
|
|
|
2016-02-15 18:11:53 +00:00
|
|
|
// OptionExamples is a slice of examples
|
|
|
|
type OptionExamples []OptionExample
|
|
|
|
|
|
|
|
// Len is part of sort.Interface.
|
|
|
|
func (os OptionExamples) Len() int { return len(os) }
|
|
|
|
|
|
|
|
// Swap is part of sort.Interface.
|
|
|
|
func (os OptionExamples) Swap(i, j int) { os[i], os[j] = os[j], os[i] }
|
|
|
|
|
|
|
|
// Less is part of sort.Interface.
|
|
|
|
func (os OptionExamples) Less(i, j int) bool { return os[i].Help < os[j].Help }
|
|
|
|
|
|
|
|
// Sort sorts an OptionExamples
|
|
|
|
func (os OptionExamples) Sort() { sort.Sort(os) }
|
|
|
|
|
2015-09-22 17:47:16 +00:00
|
|
|
// OptionExample describes an example for an Option
|
2014-03-15 16:06:11 +00:00
|
|
|
type OptionExample struct {
|
2018-04-12 16:05:53 +00:00
|
|
|
Value string
|
|
|
|
Help string
|
|
|
|
Provider string
|
2014-03-15 16:06:11 +00:00
|
|
|
}
|
|
|
|
|
2013-06-27 19:13:07 +00:00
|
|
|
// Register a filesystem
|
|
|
|
//
|
|
|
|
// Fs modules should use this in an init() function
|
2016-02-18 11:35:25 +00:00
|
|
|
func Register(info *RegInfo) {
|
2018-05-14 17:06:57 +00:00
|
|
|
info.Options.setValues()
|
|
|
|
if info.Prefix == "" {
|
|
|
|
info.Prefix = info.Name
|
|
|
|
}
|
2018-01-12 16:30:54 +00:00
|
|
|
Registry = append(Registry, info)
|
2013-06-27 19:13:07 +00:00
|
|
|
}
|
|
|
|
|
2017-06-30 09:41:48 +00:00
|
|
|
// Fs is the interface a cloud storage system must provide
|
|
|
|
type Fs interface {
|
|
|
|
Info
|
|
|
|
|
2017-06-11 21:43:31 +00:00
|
|
|
// List the objects and directories in dir into entries. The
|
|
|
|
// entries can be returned in any order but should be for a
|
|
|
|
// complete directory.
|
2016-04-21 19:06:21 +00:00
|
|
|
//
|
2017-06-11 21:43:31 +00:00
|
|
|
// dir should be "" to list the root, and should not have
|
|
|
|
// trailing slashes.
|
2016-06-18 13:12:47 +00:00
|
|
|
//
|
2017-06-11 21:43:31 +00:00
|
|
|
// This should return ErrDirNotFound if the directory isn't
|
|
|
|
// found.
|
2019-06-17 08:34:30 +00:00
|
|
|
List(ctx context.Context, dir string) (entries DirEntries, err error)
|
2016-10-07 10:39:39 +00:00
|
|
|
|
|
|
|
// NewObject finds the Object at remote. If it can't be found
|
|
|
|
// it returns the error ErrorObjectNotFound.
|
2019-06-17 08:34:30 +00:00
|
|
|
NewObject(ctx context.Context, remote string) (Object, error)
|
2013-01-23 22:43:20 +00:00
|
|
|
|
2013-01-18 18:54:19 +00:00
|
|
|
// Put in to the remote path with the modTime given of the given size
|
|
|
|
//
|
2020-05-25 06:05:53 +00:00
|
|
|
// When called from outside an Fs by rclone, src.Size() will always be >= 0.
|
2019-01-19 13:51:34 +00:00
|
|
|
// But for unknown-sized objects (indicated by src.Size() == -1), Put should either
|
|
|
|
// return an error or upload it properly (rather than e.g. calling panic).
|
|
|
|
//
|
2013-01-18 18:54:19 +00:00
|
|
|
// May create the object even if it returns an error - if so
|
|
|
|
// will return the object and the error, otherwise will return
|
|
|
|
// nil and the error
|
2019-06-17 08:34:30 +00:00
|
|
|
Put(ctx context.Context, in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error)
|
2013-01-18 18:54:19 +00:00
|
|
|
|
2015-09-22 17:47:16 +00:00
|
|
|
// Mkdir makes the directory (container, bucket)
|
2014-07-24 21:50:11 +00:00
|
|
|
//
|
|
|
|
// Shouldn't return an error if it already exists
|
2019-06-17 08:34:30 +00:00
|
|
|
Mkdir(ctx context.Context, dir string) error
|
2013-01-18 18:54:19 +00:00
|
|
|
|
2015-09-22 17:47:16 +00:00
|
|
|
// Rmdir removes the directory (container, bucket) if empty
|
2014-07-24 21:50:11 +00:00
|
|
|
//
|
2014-07-28 20:02:00 +00:00
|
|
|
// Return an error if it doesn't exist or isn't empty
|
2019-06-17 08:34:30 +00:00
|
|
|
Rmdir(ctx context.Context, dir string) error
|
2016-02-18 11:35:25 +00:00
|
|
|
}
|
|
|
|
|
2017-06-30 09:41:48 +00:00
|
|
|
// Info provides a read only interface to information about a filesystem.
|
2016-02-18 11:35:25 +00:00
|
|
|
type Info interface {
|
|
|
|
// Name of the remote (as passed into NewFs)
|
|
|
|
Name() string
|
|
|
|
|
|
|
|
// Root of the remote (as passed into NewFs)
|
|
|
|
Root() string
|
|
|
|
|
|
|
|
// String returns a description of the FS
|
|
|
|
String() string
|
2013-01-18 23:21:02 +00:00
|
|
|
|
|
|
|
// Precision of the ModTimes in this Fs
|
|
|
|
Precision() time.Duration
|
2016-01-11 12:39:33 +00:00
|
|
|
|
|
|
|
// Returns the supported hash types of the filesystem
|
2018-01-12 16:30:54 +00:00
|
|
|
Hashes() hash.Set
|
2017-01-13 17:21:47 +00:00
|
|
|
|
|
|
|
// Features returns the optional features of this Fs
|
|
|
|
Features() *Features
|
2012-12-26 12:23:58 +00:00
|
|
|
}
|
|
|
|
|
2015-09-22 17:47:16 +00:00
|
|
|
// Object is a filesystem like object provided by an Fs
|
2013-06-28 07:57:32 +00:00
|
|
|
type Object interface {
|
2016-02-18 11:35:25 +00:00
|
|
|
ObjectInfo
|
|
|
|
|
|
|
|
// SetModTime sets the metadata on the object to set the modification date
|
2019-06-17 08:34:30 +00:00
|
|
|
SetModTime(ctx context.Context, t time.Time) error
|
2016-02-18 11:35:25 +00:00
|
|
|
|
|
|
|
// Open opens the file for read. Call Close() on the returned io.ReadCloser
|
2019-06-17 08:34:30 +00:00
|
|
|
Open(ctx context.Context, options ...OpenOption) (io.ReadCloser, error)
|
2016-02-18 11:35:25 +00:00
|
|
|
|
|
|
|
// Update in to the object with the modTime given of the given size
|
2019-01-19 13:51:34 +00:00
|
|
|
//
|
2020-05-25 06:05:53 +00:00
|
|
|
// When called from outside an Fs by rclone, src.Size() will always be >= 0.
|
2019-01-19 13:51:34 +00:00
|
|
|
// But for unknown-sized objects (indicated by src.Size() == -1), Upload should either
|
|
|
|
// return an error or update the object properly (rather than e.g. calling panic).
|
2019-06-17 08:34:30 +00:00
|
|
|
Update(ctx context.Context, in io.Reader, src ObjectInfo, options ...OpenOption) error
|
2016-02-18 11:35:25 +00:00
|
|
|
|
|
|
|
// Removes this object
|
2019-06-17 08:34:30 +00:00
|
|
|
Remove(ctx context.Context) error
|
2016-02-18 11:35:25 +00:00
|
|
|
}
|
|
|
|
|
2017-06-30 09:41:48 +00:00
|
|
|
// ObjectInfo provides read only information about an object.
|
2016-02-18 11:35:25 +00:00
|
|
|
type ObjectInfo interface {
|
2017-06-30 09:54:14 +00:00
|
|
|
DirEntry
|
2016-07-18 13:07:35 +00:00
|
|
|
|
2016-02-18 11:35:25 +00:00
|
|
|
// Fs returns read only access to the Fs that this object is part of
|
|
|
|
Fs() Info
|
2014-03-28 17:56:04 +00:00
|
|
|
|
2016-02-18 11:35:25 +00:00
|
|
|
// Hash returns the selected checksum of the file
|
|
|
|
// If no checksum is available it returns ""
|
2019-06-17 08:34:30 +00:00
|
|
|
Hash(ctx context.Context, ty hash.Type) (string, error)
|
2013-01-18 18:54:19 +00:00
|
|
|
|
2016-07-18 13:07:35 +00:00
|
|
|
// Storable says whether this object can be stored
|
|
|
|
Storable() bool
|
|
|
|
}
|
|
|
|
|
2017-06-30 09:54:14 +00:00
|
|
|
// DirEntry provides read only information about the common subset of
|
|
|
|
// a Dir or Object. These are returned from directory listings - type
|
|
|
|
// assert them into the correct type.
|
|
|
|
type DirEntry interface {
|
2016-10-20 16:46:51 +00:00
|
|
|
// String returns a description of the Object
|
|
|
|
String() string
|
|
|
|
|
2016-07-18 13:07:35 +00:00
|
|
|
// Remote returns the remote path
|
|
|
|
Remote() string
|
|
|
|
|
2013-01-18 18:54:19 +00:00
|
|
|
// ModTime returns the modification date of the file
|
2015-08-16 22:24:34 +00:00
|
|
|
// It should return a best guess if one isn't available
|
2019-06-17 08:34:30 +00:00
|
|
|
ModTime(context.Context) time.Time
|
2013-01-18 18:54:19 +00:00
|
|
|
|
|
|
|
// Size returns the size of the file
|
2012-12-26 12:23:58 +00:00
|
|
|
Size() int64
|
|
|
|
}
|
|
|
|
|
2017-06-30 12:37:29 +00:00
|
|
|
// Directory is a filesystem like directory provided by an Fs
|
|
|
|
type Directory interface {
|
|
|
|
DirEntry
|
|
|
|
|
|
|
|
// Items returns the count of items in this directory or this
|
|
|
|
// directory and subdirectories if known, -1 for unknown
|
|
|
|
Items() int64
|
2017-08-02 15:44:36 +00:00
|
|
|
|
|
|
|
// ID returns the internal ID of this directory if known, or
|
|
|
|
// "" otherwise
|
|
|
|
ID() string
|
2017-06-30 12:37:29 +00:00
|
|
|
}
|
|
|
|
|
2016-09-21 21:13:24 +00:00
|
|
|
// MimeTyper is an optional interface for Object
|
|
|
|
type MimeTyper interface {
|
|
|
|
// MimeType returns the content type of the Object if
|
|
|
|
// known, or "" if not
|
2019-06-17 08:34:30 +00:00
|
|
|
MimeType(ctx context.Context) string
|
2016-09-21 21:13:24 +00:00
|
|
|
}
|
|
|
|
|
2018-05-13 08:16:56 +00:00
|
|
|
// IDer is an optional interface for Object
|
|
|
|
type IDer interface {
|
|
|
|
// ID returns the ID of the Object if known, or "" if not
|
|
|
|
ID() string
|
|
|
|
}
|
|
|
|
|
2021-03-11 17:40:29 +00:00
|
|
|
// ParentIDer is an optional interface for Object
|
|
|
|
type ParentIDer interface {
|
|
|
|
// ParentID returns the ID of the parent directory if known or nil if not
|
|
|
|
ParentID() string
|
|
|
|
}
|
|
|
|
|
2018-01-31 14:22:49 +00:00
|
|
|
// ObjectUnWrapper is an optional interface for Object
|
|
|
|
type ObjectUnWrapper interface {
|
|
|
|
// UnWrap returns the Object that this Object is wrapping or
|
|
|
|
// nil if it isn't wrapping anything
|
|
|
|
UnWrap() Object
|
|
|
|
}
|
|
|
|
|
2018-09-11 01:53:32 +00:00
|
|
|
// SetTierer is an optional interface for Object
|
|
|
|
type SetTierer interface {
|
|
|
|
// SetTier performs changing storage tier of the Object if
|
|
|
|
// multiple storage classes supported
|
|
|
|
SetTier(tier string) error
|
|
|
|
}
|
|
|
|
|
|
|
|
// GetTierer is an optional interface for Object
|
|
|
|
type GetTierer interface {
|
|
|
|
// GetTier returns storage tier or class of the Object
|
|
|
|
GetTier() string
|
|
|
|
}
|
|
|
|
|
2019-07-04 11:24:58 +00:00
|
|
|
// FullObjectInfo contains all the read-only optional interfaces
|
|
|
|
//
|
|
|
|
// Use for checking making wrapping ObjectInfos implement everything
|
|
|
|
type FullObjectInfo interface {
|
|
|
|
ObjectInfo
|
|
|
|
MimeTyper
|
|
|
|
IDer
|
|
|
|
ObjectUnWrapper
|
|
|
|
GetTierer
|
|
|
|
}
|
|
|
|
|
|
|
|
// FullObject contains all the optional interfaces for Object
|
|
|
|
//
|
|
|
|
// Use for checking making wrapping Objects implement everything
|
|
|
|
type FullObject interface {
|
|
|
|
Object
|
|
|
|
MimeTyper
|
|
|
|
IDer
|
|
|
|
ObjectUnWrapper
|
|
|
|
GetTierer
|
|
|
|
SetTierer
|
|
|
|
}
|
|
|
|
|
2019-01-11 12:28:41 +00:00
|
|
|
// ObjectOptionalInterfaces returns the names of supported and
|
|
|
|
// unsupported optional interfaces for an Object
|
|
|
|
func ObjectOptionalInterfaces(o Object) (supported, unsupported []string) {
|
|
|
|
store := func(ok bool, name string) {
|
|
|
|
if ok {
|
|
|
|
supported = append(supported, name)
|
|
|
|
} else {
|
|
|
|
unsupported = append(unsupported, name)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
_, ok := o.(MimeTyper)
|
|
|
|
store(ok, "MimeType")
|
|
|
|
|
|
|
|
_, ok = o.(IDer)
|
|
|
|
store(ok, "ID")
|
|
|
|
|
|
|
|
_, ok = o.(ObjectUnWrapper)
|
|
|
|
store(ok, "UnWrap")
|
|
|
|
|
|
|
|
_, ok = o.(SetTierer)
|
|
|
|
store(ok, "SetTier")
|
|
|
|
|
|
|
|
_, ok = o.(GetTierer)
|
|
|
|
store(ok, "GetTier")
|
|
|
|
|
|
|
|
return supported, unsupported
|
|
|
|
}
|
|
|
|
|
2017-06-11 21:43:31 +00:00
|
|
|
// ListRCallback defines a callback function for ListR to use
|
|
|
|
//
|
|
|
|
// It is called for each tranche of entries read from the listing and
|
|
|
|
// if it returns an error, the listing stops.
|
|
|
|
type ListRCallback func(entries DirEntries) error
|
|
|
|
|
|
|
|
// ListRFn is defines the call used to recursively list a directory
|
2019-06-17 08:34:30 +00:00
|
|
|
type ListRFn func(ctx context.Context, dir string, callback ListRCallback) error
|
2017-06-11 21:43:31 +00:00
|
|
|
|
2018-04-16 21:19:25 +00:00
|
|
|
// NewUsageValue makes a valid value
|
|
|
|
func NewUsageValue(value int64) *int64 {
|
|
|
|
p := new(int64)
|
|
|
|
*p = value
|
|
|
|
return p
|
|
|
|
}
|
|
|
|
|
|
|
|
// Usage is returned by the About call
|
|
|
|
//
|
|
|
|
// If a value is nil then it isn't supported by that backend
|
|
|
|
type Usage struct {
|
|
|
|
Total *int64 `json:"total,omitempty"` // quota of bytes that can be used
|
|
|
|
Used *int64 `json:"used,omitempty"` // bytes in use
|
|
|
|
Trashed *int64 `json:"trashed,omitempty"` // bytes in trash
|
2020-10-13 21:49:58 +00:00
|
|
|
Other *int64 `json:"other,omitempty"` // other usage e.g. gmail in drive
|
2018-04-16 21:19:25 +00:00
|
|
|
Free *int64 `json:"free,omitempty"` // bytes which can be uploaded before reaching the quota
|
|
|
|
Objects *int64 `json:"objects,omitempty"` // objects in the storage system
|
|
|
|
}
|
|
|
|
|
2019-04-22 18:22:42 +00:00
|
|
|
// WriterAtCloser wraps io.WriterAt and io.Closer
|
|
|
|
type WriterAtCloser interface {
|
|
|
|
io.WriterAt
|
|
|
|
io.Closer
|
|
|
|
}
|
|
|
|
|
2017-01-13 17:21:47 +00:00
|
|
|
// Features describe the optional features of the Fs
|
|
|
|
type Features struct {
|
2017-08-09 14:27:43 +00:00
|
|
|
// Feature flags, whether Fs
|
|
|
|
CaseInsensitive bool // has case insensitive files
|
|
|
|
DuplicateFiles bool // allows duplicate files
|
|
|
|
ReadMimeType bool // can read the mime type of objects
|
|
|
|
WriteMimeType bool // can set the mime type of objects
|
|
|
|
CanHaveEmptyDirectories bool // can have empty directories
|
2020-10-13 22:07:12 +00:00
|
|
|
BucketBased bool // is bucket based (like s3, swift, etc.)
|
2019-08-08 15:49:34 +00:00
|
|
|
BucketBasedRootOK bool // is bucket based and can use from root
|
2018-09-11 01:53:32 +00:00
|
|
|
SetTier bool // allows set tier functionality on objects
|
|
|
|
GetTier bool // allows to retrieve storage tier of objects
|
2020-10-13 21:43:40 +00:00
|
|
|
ServerSideAcrossConfigs bool // can server-side copy between different remotes of the same type
|
2019-08-13 11:35:55 +00:00
|
|
|
IsLocal bool // is the local backend
|
2020-06-19 09:28:34 +00:00
|
|
|
SlowModTime bool // if calling ModTime() generally takes an extra transaction
|
|
|
|
SlowHash bool // if calling Hash() generally takes an extra transaction
|
2017-01-13 17:21:47 +00:00
|
|
|
|
2020-06-04 21:25:14 +00:00
|
|
|
// Purge all files in the directory specified
|
2017-01-13 17:21:47 +00:00
|
|
|
//
|
|
|
|
// Implement this if you have a way of deleting all the files
|
|
|
|
// quicker than just running Remove() on the result of List()
|
|
|
|
//
|
|
|
|
// Return an error if it doesn't exist
|
2020-06-04 21:25:14 +00:00
|
|
|
Purge func(ctx context.Context, dir string) error
|
2017-01-13 17:21:47 +00:00
|
|
|
|
2020-10-13 21:43:40 +00:00
|
|
|
// Copy src to this remote using server-side copy operations.
|
2017-01-13 17:21:47 +00:00
|
|
|
//
|
|
|
|
// This is stored with the remote path given
|
|
|
|
//
|
|
|
|
// It returns the destination Object and a possible error
|
|
|
|
//
|
|
|
|
// Will only be called if src.Fs().Name() == f.Name()
|
|
|
|
//
|
|
|
|
// If it isn't possible then return fs.ErrorCantCopy
|
2019-06-17 08:34:30 +00:00
|
|
|
Copy func(ctx context.Context, src Object, remote string) (Object, error)
|
2017-01-13 17:21:47 +00:00
|
|
|
|
2020-10-13 21:43:40 +00:00
|
|
|
// Move src to this remote using server-side move operations.
|
2017-01-13 17:21:47 +00:00
|
|
|
//
|
|
|
|
// This is stored with the remote path given
|
|
|
|
//
|
|
|
|
// It returns the destination Object and a possible error
|
|
|
|
//
|
|
|
|
// Will only be called if src.Fs().Name() == f.Name()
|
|
|
|
//
|
|
|
|
// If it isn't possible then return fs.ErrorCantMove
|
2019-06-17 08:34:30 +00:00
|
|
|
Move func(ctx context.Context, src Object, remote string) (Object, error)
|
2017-01-13 17:21:47 +00:00
|
|
|
|
2017-02-05 21:20:56 +00:00
|
|
|
// DirMove moves src, srcRemote to this remote at dstRemote
|
2020-10-13 21:43:40 +00:00
|
|
|
// using server-side move operations.
|
2017-01-13 17:21:47 +00:00
|
|
|
//
|
|
|
|
// Will only be called if src.Fs().Name() == f.Name()
|
|
|
|
//
|
|
|
|
// If it isn't possible then return fs.ErrorCantDirMove
|
|
|
|
//
|
|
|
|
// If destination exists then return fs.ErrorDirExists
|
2019-06-17 08:34:30 +00:00
|
|
|
DirMove func(ctx context.Context, src Fs, srcRemote, dstRemote string) error
|
2017-01-13 17:21:47 +00:00
|
|
|
|
2018-03-08 20:03:34 +00:00
|
|
|
// ChangeNotify calls the passed function with a path
|
|
|
|
// that has had changes. If the implementation
|
2017-05-25 21:05:49 +00:00
|
|
|
// uses polling, it should adhere to the given interval.
|
2019-06-17 08:34:30 +00:00
|
|
|
ChangeNotify func(context.Context, func(string, EntryType), <-chan time.Duration)
|
2017-05-25 21:05:49 +00:00
|
|
|
|
2017-01-13 17:21:47 +00:00
|
|
|
// UnWrap returns the Fs that this Fs is wrapping
|
|
|
|
UnWrap func() Fs
|
|
|
|
|
2017-12-06 15:14:34 +00:00
|
|
|
// WrapFs returns the Fs that is wrapping this Fs
|
|
|
|
WrapFs func() Fs
|
|
|
|
|
|
|
|
// SetWrapper sets the Fs that is wrapping this Fs
|
|
|
|
SetWrapper func(f Fs)
|
|
|
|
|
2017-01-13 17:21:47 +00:00
|
|
|
// DirCacheFlush resets the directory cache - used in testing
|
|
|
|
// as an optional interface
|
|
|
|
DirCacheFlush func()
|
|
|
|
|
2018-03-29 07:10:19 +00:00
|
|
|
// PublicLink generates a public link to the remote path (usually readable by anyone)
|
2020-05-31 21:18:01 +00:00
|
|
|
PublicLink func(ctx context.Context, remote string, expire Duration, unlink bool) (string, error)
|
2018-03-29 07:10:19 +00:00
|
|
|
|
2017-01-13 17:21:47 +00:00
|
|
|
// Put in to the remote path with the modTime given of the given size
|
|
|
|
//
|
|
|
|
// May create the object even if it returns an error - if so
|
|
|
|
// will return the object and the error, otherwise will return
|
|
|
|
// nil and the error
|
|
|
|
//
|
|
|
|
// May create duplicates or return errors if src already
|
|
|
|
// exists.
|
2019-06-17 08:34:30 +00:00
|
|
|
PutUnchecked func(ctx context.Context, in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error)
|
2017-01-13 17:21:47 +00:00
|
|
|
|
2017-08-03 19:42:35 +00:00
|
|
|
// PutStream uploads to the remote path with the modTime given of indeterminate size
|
|
|
|
//
|
|
|
|
// May create the object even if it returns an error - if so
|
|
|
|
// will return the object and the error, otherwise will return
|
|
|
|
// nil and the error
|
2019-06-17 08:34:30 +00:00
|
|
|
PutStream func(ctx context.Context, in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error)
|
2017-08-03 19:42:35 +00:00
|
|
|
|
2017-08-02 15:51:24 +00:00
|
|
|
// MergeDirs merges the contents of all the directories passed
|
|
|
|
// in into the first one and rmdirs the other directories.
|
2019-06-17 08:34:30 +00:00
|
|
|
MergeDirs func(ctx context.Context, dirs []Directory) error
|
2017-08-02 15:51:24 +00:00
|
|
|
|
2017-01-13 17:21:47 +00:00
|
|
|
// CleanUp the trash in the Fs
|
|
|
|
//
|
|
|
|
// Implement this if you have a way of emptying the trash or
|
|
|
|
// otherwise cleaning up old versions of files.
|
2019-06-17 08:34:30 +00:00
|
|
|
CleanUp func(ctx context.Context) error
|
2017-06-05 15:14:24 +00:00
|
|
|
|
|
|
|
// ListR lists the objects and directories of the Fs starting
|
|
|
|
// from dir recursively into out.
|
|
|
|
//
|
|
|
|
// dir should be "" to start from the root, and should not
|
|
|
|
// have trailing slashes.
|
|
|
|
//
|
2017-06-11 21:43:31 +00:00
|
|
|
// This should return ErrDirNotFound if the directory isn't
|
|
|
|
// found.
|
|
|
|
//
|
|
|
|
// It should call callback for each tranche of entries read.
|
|
|
|
// These need not be returned in any particular order. If
|
|
|
|
// callback returns an error then the listing will stop
|
|
|
|
// immediately.
|
2017-06-05 15:14:24 +00:00
|
|
|
//
|
|
|
|
// Don't implement this unless you have a more efficient way
|
|
|
|
// of listing recursively that doing a directory traversal.
|
2017-06-11 21:43:31 +00:00
|
|
|
ListR ListRFn
|
2018-02-09 20:48:32 +00:00
|
|
|
|
2018-04-16 21:19:25 +00:00
|
|
|
// About gets quota information from the Fs
|
2019-06-17 08:34:30 +00:00
|
|
|
About func(ctx context.Context) (*Usage, error)
|
2019-04-22 18:22:42 +00:00
|
|
|
|
|
|
|
// OpenWriterAt opens with a handle for random access writes
|
|
|
|
//
|
|
|
|
// Pass in the remote desired and the size if known.
|
|
|
|
//
|
|
|
|
// It truncates any existing object
|
2019-06-17 08:34:30 +00:00
|
|
|
OpenWriterAt func(ctx context.Context, remote string, size int64) (WriterAtCloser, error)
|
2019-08-12 10:07:31 +00:00
|
|
|
|
|
|
|
// UserInfo returns info about the connected user
|
|
|
|
UserInfo func(ctx context.Context) (map[string]string, error)
|
|
|
|
|
|
|
|
// Disconnect the current user
|
|
|
|
Disconnect func(ctx context.Context) error
|
2020-04-28 11:58:34 +00:00
|
|
|
|
|
|
|
// Command the backend to run a named command
|
|
|
|
//
|
|
|
|
// The command run is name
|
|
|
|
// args may be used to read arguments from
|
|
|
|
// opts may be used to read optional arguments from
|
|
|
|
//
|
|
|
|
// The result should be capable of being JSON encoded
|
|
|
|
// If it is a string or a []string it will be shown to the user
|
|
|
|
// otherwise it will be JSON encoded and shown to the user like that
|
|
|
|
Command func(ctx context.Context, name string, arg []string, opt map[string]string) (interface{}, error)
|
2020-11-27 17:02:00 +00:00
|
|
|
|
|
|
|
// Shutdown the backend, closing any background tasks and any
|
|
|
|
// cached connections.
|
|
|
|
Shutdown func(ctx context.Context) error
|
2017-01-13 17:21:47 +00:00
|
|
|
}
|
|
|
|
|
2017-08-07 20:10:03 +00:00
|
|
|
// Disable nil's out the named feature. If it isn't found then it
|
|
|
|
// will log a message.
|
|
|
|
func (ft *Features) Disable(name string) *Features {
|
|
|
|
v := reflect.ValueOf(ft).Elem()
|
|
|
|
vType := v.Type()
|
|
|
|
for i := 0; i < v.NumField(); i++ {
|
|
|
|
vName := vType.Field(i).Name
|
|
|
|
field := v.Field(i)
|
|
|
|
if strings.EqualFold(name, vName) {
|
|
|
|
if !field.CanSet() {
|
|
|
|
Errorf(nil, "Can't set Feature %q", name)
|
|
|
|
} else {
|
|
|
|
zero := reflect.Zero(field.Type())
|
|
|
|
field.Set(zero)
|
|
|
|
Debugf(nil, "Reset feature %q", name)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return ft
|
|
|
|
}
|
|
|
|
|
|
|
|
// List returns a slice of all the possible feature names
|
|
|
|
func (ft *Features) List() (out []string) {
|
|
|
|
v := reflect.ValueOf(ft).Elem()
|
|
|
|
vType := v.Type()
|
|
|
|
for i := 0; i < v.NumField(); i++ {
|
|
|
|
out = append(out, vType.Field(i).Name)
|
|
|
|
}
|
|
|
|
return out
|
|
|
|
}
|
|
|
|
|
2019-06-08 07:46:53 +00:00
|
|
|
// Enabled returns a map of features with keys showing whether they
|
|
|
|
// are enabled or not
|
|
|
|
func (ft *Features) Enabled() (features map[string]bool) {
|
|
|
|
v := reflect.ValueOf(ft).Elem()
|
|
|
|
vType := v.Type()
|
|
|
|
features = make(map[string]bool, v.NumField())
|
|
|
|
for i := 0; i < v.NumField(); i++ {
|
|
|
|
vName := vType.Field(i).Name
|
|
|
|
field := v.Field(i)
|
|
|
|
if field.Kind() == reflect.Func {
|
|
|
|
// Can't compare functions
|
|
|
|
features[vName] = !field.IsNil()
|
|
|
|
} else {
|
|
|
|
zero := reflect.Zero(field.Type())
|
|
|
|
features[vName] = field.Interface() != zero.Interface()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return features
|
|
|
|
}
|
|
|
|
|
2017-08-07 20:10:03 +00:00
|
|
|
// DisableList nil's out the comma separated list of named features.
|
|
|
|
// If it isn't found then it will log a message.
|
|
|
|
func (ft *Features) DisableList(list []string) *Features {
|
|
|
|
for _, feature := range list {
|
|
|
|
ft.Disable(strings.TrimSpace(feature))
|
|
|
|
}
|
|
|
|
return ft
|
|
|
|
}
|
|
|
|
|
2017-01-13 17:21:47 +00:00
|
|
|
// Fill fills in the function pointers in the Features struct from the
|
|
|
|
// optional interfaces. It returns the original updated Features
|
|
|
|
// struct passed in.
|
2020-11-05 16:00:40 +00:00
|
|
|
func (ft *Features) Fill(ctx context.Context, f Fs) *Features {
|
2017-01-13 17:21:47 +00:00
|
|
|
if do, ok := f.(Purger); ok {
|
|
|
|
ft.Purge = do.Purge
|
|
|
|
}
|
|
|
|
if do, ok := f.(Copier); ok {
|
|
|
|
ft.Copy = do.Copy
|
|
|
|
}
|
|
|
|
if do, ok := f.(Mover); ok {
|
|
|
|
ft.Move = do.Move
|
|
|
|
}
|
|
|
|
if do, ok := f.(DirMover); ok {
|
|
|
|
ft.DirMove = do.DirMove
|
|
|
|
}
|
2018-03-08 20:03:34 +00:00
|
|
|
if do, ok := f.(ChangeNotifier); ok {
|
|
|
|
ft.ChangeNotify = do.ChangeNotify
|
2017-05-25 21:05:49 +00:00
|
|
|
}
|
2017-01-13 17:21:47 +00:00
|
|
|
if do, ok := f.(UnWrapper); ok {
|
|
|
|
ft.UnWrap = do.UnWrap
|
|
|
|
}
|
2017-12-06 15:14:34 +00:00
|
|
|
if do, ok := f.(Wrapper); ok {
|
|
|
|
ft.WrapFs = do.WrapFs
|
|
|
|
ft.SetWrapper = do.SetWrapper
|
|
|
|
}
|
2017-01-13 17:21:47 +00:00
|
|
|
if do, ok := f.(DirCacheFlusher); ok {
|
|
|
|
ft.DirCacheFlush = do.DirCacheFlush
|
|
|
|
}
|
2018-03-29 07:10:19 +00:00
|
|
|
if do, ok := f.(PublicLinker); ok {
|
|
|
|
ft.PublicLink = do.PublicLink
|
|
|
|
}
|
2017-01-13 17:21:47 +00:00
|
|
|
if do, ok := f.(PutUncheckeder); ok {
|
|
|
|
ft.PutUnchecked = do.PutUnchecked
|
|
|
|
}
|
2017-08-03 19:42:35 +00:00
|
|
|
if do, ok := f.(PutStreamer); ok {
|
|
|
|
ft.PutStream = do.PutStream
|
|
|
|
}
|
2017-08-02 15:51:24 +00:00
|
|
|
if do, ok := f.(MergeDirser); ok {
|
|
|
|
ft.MergeDirs = do.MergeDirs
|
|
|
|
}
|
2017-01-13 17:21:47 +00:00
|
|
|
if do, ok := f.(CleanUpper); ok {
|
|
|
|
ft.CleanUp = do.CleanUp
|
|
|
|
}
|
2017-06-05 15:14:24 +00:00
|
|
|
if do, ok := f.(ListRer); ok {
|
|
|
|
ft.ListR = do.ListR
|
|
|
|
}
|
2018-02-09 20:48:32 +00:00
|
|
|
if do, ok := f.(Abouter); ok {
|
|
|
|
ft.About = do.About
|
|
|
|
}
|
2019-04-22 18:22:42 +00:00
|
|
|
if do, ok := f.(OpenWriterAter); ok {
|
|
|
|
ft.OpenWriterAt = do.OpenWriterAt
|
|
|
|
}
|
2019-08-12 10:07:31 +00:00
|
|
|
if do, ok := f.(UserInfoer); ok {
|
|
|
|
ft.UserInfo = do.UserInfo
|
|
|
|
}
|
|
|
|
if do, ok := f.(Disconnecter); ok {
|
|
|
|
ft.Disconnect = do.Disconnect
|
|
|
|
}
|
2020-04-28 11:58:34 +00:00
|
|
|
if do, ok := f.(Commander); ok {
|
|
|
|
ft.Command = do.Command
|
|
|
|
}
|
2020-11-27 17:02:00 +00:00
|
|
|
if do, ok := f.(Shutdowner); ok {
|
|
|
|
ft.Shutdown = do.Shutdown
|
|
|
|
}
|
2020-11-05 11:33:32 +00:00
|
|
|
return ft.DisableList(GetConfig(ctx).DisableFeatures)
|
2017-01-13 17:21:47 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Mask the Features with the Fs passed in
|
|
|
|
//
|
|
|
|
// Only optional features which are implemented in both the original
|
|
|
|
// Fs AND the one passed in will be advertised. Any features which
|
2017-12-06 15:14:34 +00:00
|
|
|
// aren't in both will be set to false/nil, except for UnWrap/Wrap which
|
2017-01-13 17:21:47 +00:00
|
|
|
// will be left untouched.
|
2020-11-05 16:00:40 +00:00
|
|
|
func (ft *Features) Mask(ctx context.Context, f Fs) *Features {
|
2017-01-13 17:21:47 +00:00
|
|
|
mask := f.Features()
|
|
|
|
ft.CaseInsensitive = ft.CaseInsensitive && mask.CaseInsensitive
|
|
|
|
ft.DuplicateFiles = ft.DuplicateFiles && mask.DuplicateFiles
|
|
|
|
ft.ReadMimeType = ft.ReadMimeType && mask.ReadMimeType
|
|
|
|
ft.WriteMimeType = ft.WriteMimeType && mask.WriteMimeType
|
2017-08-09 14:27:43 +00:00
|
|
|
ft.CanHaveEmptyDirectories = ft.CanHaveEmptyDirectories && mask.CanHaveEmptyDirectories
|
|
|
|
ft.BucketBased = ft.BucketBased && mask.BucketBased
|
2019-08-08 15:49:34 +00:00
|
|
|
ft.BucketBasedRootOK = ft.BucketBasedRootOK && mask.BucketBasedRootOK
|
2018-09-11 01:53:32 +00:00
|
|
|
ft.SetTier = ft.SetTier && mask.SetTier
|
|
|
|
ft.GetTier = ft.GetTier && mask.GetTier
|
2020-06-19 09:28:34 +00:00
|
|
|
ft.ServerSideAcrossConfigs = ft.ServerSideAcrossConfigs && mask.ServerSideAcrossConfigs
|
|
|
|
// ft.IsLocal = ft.IsLocal && mask.IsLocal Don't propagate IsLocal
|
|
|
|
ft.SlowModTime = ft.SlowModTime && mask.SlowModTime
|
|
|
|
ft.SlowHash = ft.SlowHash && mask.SlowHash
|
2018-09-11 01:53:32 +00:00
|
|
|
|
2017-01-13 17:21:47 +00:00
|
|
|
if mask.Purge == nil {
|
|
|
|
ft.Purge = nil
|
|
|
|
}
|
|
|
|
if mask.Copy == nil {
|
|
|
|
ft.Copy = nil
|
|
|
|
}
|
|
|
|
if mask.Move == nil {
|
|
|
|
ft.Move = nil
|
|
|
|
}
|
|
|
|
if mask.DirMove == nil {
|
|
|
|
ft.DirMove = nil
|
|
|
|
}
|
2018-03-08 20:03:34 +00:00
|
|
|
if mask.ChangeNotify == nil {
|
|
|
|
ft.ChangeNotify = nil
|
2017-05-25 21:05:49 +00:00
|
|
|
}
|
2017-01-13 17:21:47 +00:00
|
|
|
// if mask.UnWrap == nil {
|
|
|
|
// ft.UnWrap = nil
|
|
|
|
// }
|
2019-03-20 11:59:00 +00:00
|
|
|
// if mask.Wrapper == nil {
|
|
|
|
// ft.Wrapper = nil
|
|
|
|
// }
|
2017-01-13 17:21:47 +00:00
|
|
|
if mask.DirCacheFlush == nil {
|
|
|
|
ft.DirCacheFlush = nil
|
|
|
|
}
|
2019-03-20 11:59:00 +00:00
|
|
|
if mask.PublicLink == nil {
|
|
|
|
ft.PublicLink = nil
|
|
|
|
}
|
2017-01-13 17:21:47 +00:00
|
|
|
if mask.PutUnchecked == nil {
|
|
|
|
ft.PutUnchecked = nil
|
|
|
|
}
|
2017-08-03 19:42:35 +00:00
|
|
|
if mask.PutStream == nil {
|
|
|
|
ft.PutStream = nil
|
|
|
|
}
|
2017-08-02 15:51:24 +00:00
|
|
|
if mask.MergeDirs == nil {
|
|
|
|
ft.MergeDirs = nil
|
|
|
|
}
|
2017-01-13 17:21:47 +00:00
|
|
|
if mask.CleanUp == nil {
|
|
|
|
ft.CleanUp = nil
|
|
|
|
}
|
2017-06-05 15:14:24 +00:00
|
|
|
if mask.ListR == nil {
|
|
|
|
ft.ListR = nil
|
|
|
|
}
|
2018-02-09 20:48:32 +00:00
|
|
|
if mask.About == nil {
|
|
|
|
ft.About = nil
|
|
|
|
}
|
2019-04-22 18:22:42 +00:00
|
|
|
if mask.OpenWriterAt == nil {
|
|
|
|
ft.OpenWriterAt = nil
|
|
|
|
}
|
2019-08-12 10:07:31 +00:00
|
|
|
if mask.UserInfo == nil {
|
|
|
|
ft.UserInfo = nil
|
|
|
|
}
|
|
|
|
if mask.Disconnect == nil {
|
|
|
|
ft.Disconnect = nil
|
|
|
|
}
|
2020-04-28 11:58:34 +00:00
|
|
|
// Command is always local so we don't mask it
|
2020-11-27 17:02:00 +00:00
|
|
|
if mask.Shutdown == nil {
|
|
|
|
ft.Shutdown = nil
|
|
|
|
}
|
2020-11-05 11:33:32 +00:00
|
|
|
return ft.DisableList(GetConfig(ctx).DisableFeatures)
|
2017-01-13 17:21:47 +00:00
|
|
|
}
|
|
|
|
|
2017-12-06 15:14:34 +00:00
|
|
|
// Wrap makes a Copy of the features passed in, overriding the UnWrap/Wrap
|
2017-01-13 17:21:47 +00:00
|
|
|
// method only if available in f.
|
|
|
|
func (ft *Features) Wrap(f Fs) *Features {
|
2018-08-04 10:16:43 +00:00
|
|
|
ftCopy := new(Features)
|
|
|
|
*ftCopy = *ft
|
2017-01-13 17:21:47 +00:00
|
|
|
if do, ok := f.(UnWrapper); ok {
|
2018-08-04 10:16:43 +00:00
|
|
|
ftCopy.UnWrap = do.UnWrap
|
2017-01-13 17:21:47 +00:00
|
|
|
}
|
2017-12-06 15:14:34 +00:00
|
|
|
if do, ok := f.(Wrapper); ok {
|
2018-08-04 10:16:43 +00:00
|
|
|
ftCopy.WrapFs = do.WrapFs
|
|
|
|
ftCopy.SetWrapper = do.SetWrapper
|
2017-12-06 15:14:34 +00:00
|
|
|
}
|
2018-08-04 10:16:43 +00:00
|
|
|
return ftCopy
|
2017-01-13 17:21:47 +00:00
|
|
|
}
|
|
|
|
|
2017-12-06 15:14:34 +00:00
|
|
|
// WrapsFs adds extra information between `f` which wraps `w`
|
|
|
|
func (ft *Features) WrapsFs(f Fs, w Fs) *Features {
|
|
|
|
wFeatures := w.Features()
|
|
|
|
if wFeatures.WrapFs != nil && wFeatures.SetWrapper != nil {
|
|
|
|
wFeatures.SetWrapper(f)
|
|
|
|
}
|
|
|
|
return ft
|
|
|
|
}
|
|
|
|
|
2015-09-22 17:47:16 +00:00
|
|
|
// Purger is an optional interfaces for Fs
|
2013-01-18 17:01:47 +00:00
|
|
|
type Purger interface {
|
2020-06-04 21:25:14 +00:00
|
|
|
// Purge all files in the directory specified
|
2013-01-18 17:01:47 +00:00
|
|
|
//
|
|
|
|
// Implement this if you have a way of deleting all the files
|
|
|
|
// quicker than just running Remove() on the result of List()
|
2014-07-28 20:02:00 +00:00
|
|
|
//
|
|
|
|
// Return an error if it doesn't exist
|
2020-06-04 21:25:14 +00:00
|
|
|
Purge(ctx context.Context, dir string) error
|
2013-01-18 17:01:47 +00:00
|
|
|
}
|
|
|
|
|
2015-09-22 17:47:16 +00:00
|
|
|
// Copier is an optional interface for Fs
|
2015-02-14 18:48:08 +00:00
|
|
|
type Copier interface {
|
2020-10-13 21:43:40 +00:00
|
|
|
// Copy src to this remote using server-side copy operations.
|
2015-02-14 18:48:08 +00:00
|
|
|
//
|
|
|
|
// This is stored with the remote path given
|
|
|
|
//
|
|
|
|
// It returns the destination Object and a possible error
|
|
|
|
//
|
|
|
|
// Will only be called if src.Fs().Name() == f.Name()
|
|
|
|
//
|
|
|
|
// If it isn't possible then return fs.ErrorCantCopy
|
2019-06-17 08:34:30 +00:00
|
|
|
Copy(ctx context.Context, src Object, remote string) (Object, error)
|
2015-02-14 18:48:08 +00:00
|
|
|
}
|
|
|
|
|
2015-09-22 17:47:16 +00:00
|
|
|
// Mover is an optional interface for Fs
|
2015-08-24 20:42:23 +00:00
|
|
|
type Mover interface {
|
2020-10-13 21:43:40 +00:00
|
|
|
// Move src to this remote using server-side move operations.
|
2015-08-24 20:42:23 +00:00
|
|
|
//
|
|
|
|
// This is stored with the remote path given
|
|
|
|
//
|
|
|
|
// It returns the destination Object and a possible error
|
|
|
|
//
|
|
|
|
// Will only be called if src.Fs().Name() == f.Name()
|
|
|
|
//
|
|
|
|
// If it isn't possible then return fs.ErrorCantMove
|
2019-06-17 08:34:30 +00:00
|
|
|
Move(ctx context.Context, src Object, remote string) (Object, error)
|
2015-08-24 20:42:23 +00:00
|
|
|
}
|
|
|
|
|
2015-09-22 17:47:16 +00:00
|
|
|
// DirMover is an optional interface for Fs
|
2015-08-24 20:42:23 +00:00
|
|
|
type DirMover interface {
|
2017-02-05 21:20:56 +00:00
|
|
|
// DirMove moves src, srcRemote to this remote at dstRemote
|
2020-10-13 21:43:40 +00:00
|
|
|
// using server-side move operations.
|
2015-08-24 20:42:23 +00:00
|
|
|
//
|
|
|
|
// Will only be called if src.Fs().Name() == f.Name()
|
|
|
|
//
|
|
|
|
// If it isn't possible then return fs.ErrorCantDirMove
|
|
|
|
//
|
|
|
|
// If destination exists then return fs.ErrorDirExists
|
2019-06-17 08:34:30 +00:00
|
|
|
DirMove(ctx context.Context, src Fs, srcRemote, dstRemote string) error
|
2015-08-24 20:42:23 +00:00
|
|
|
}
|
|
|
|
|
2018-03-08 20:03:34 +00:00
|
|
|
// ChangeNotifier is an optional interface for Fs
|
|
|
|
type ChangeNotifier interface {
|
|
|
|
// ChangeNotify calls the passed function with a path
|
|
|
|
// that has had changes. If the implementation
|
2017-05-25 21:05:49 +00:00
|
|
|
// uses polling, it should adhere to the given interval.
|
2018-08-25 19:28:57 +00:00
|
|
|
// At least one value will be written to the channel,
|
|
|
|
// specifying the initial value and updated values might
|
|
|
|
// follow. A 0 Duration should pause the polling.
|
2019-04-30 12:06:24 +00:00
|
|
|
// The ChangeNotify implementation must empty the channel
|
|
|
|
// regularly. When the channel gets closed, the implementation
|
2018-08-25 19:28:57 +00:00
|
|
|
// should stop polling and release resources.
|
2019-06-17 08:34:30 +00:00
|
|
|
ChangeNotify(context.Context, func(string, EntryType), <-chan time.Duration)
|
2017-05-25 21:05:49 +00:00
|
|
|
}
|
|
|
|
|
2015-11-08 15:27:55 +00:00
|
|
|
// UnWrapper is an optional interfaces for Fs
|
|
|
|
type UnWrapper interface {
|
|
|
|
// UnWrap returns the Fs that this Fs is wrapping
|
|
|
|
UnWrap() Fs
|
|
|
|
}
|
|
|
|
|
2017-12-06 15:14:34 +00:00
|
|
|
// Wrapper is an optional interfaces for Fs
|
|
|
|
type Wrapper interface {
|
|
|
|
// Wrap returns the Fs that is wrapping this Fs
|
|
|
|
WrapFs() Fs
|
|
|
|
// SetWrapper sets the Fs that is wrapping this Fs
|
|
|
|
SetWrapper(f Fs)
|
|
|
|
}
|
|
|
|
|
2016-12-09 15:39:29 +00:00
|
|
|
// DirCacheFlusher is an optional interface for Fs
|
|
|
|
type DirCacheFlusher interface {
|
|
|
|
// DirCacheFlush resets the directory cache - used in testing
|
|
|
|
// as an optional interface
|
|
|
|
DirCacheFlush()
|
|
|
|
}
|
|
|
|
|
2016-06-12 14:06:27 +00:00
|
|
|
// PutUncheckeder is an optional interface for Fs
|
|
|
|
type PutUncheckeder interface {
|
|
|
|
// Put in to the remote path with the modTime given of the given size
|
|
|
|
//
|
|
|
|
// May create the object even if it returns an error - if so
|
|
|
|
// will return the object and the error, otherwise will return
|
|
|
|
// nil and the error
|
|
|
|
//
|
|
|
|
// May create duplicates or return errors if src already
|
|
|
|
// exists.
|
2019-06-17 08:34:30 +00:00
|
|
|
PutUnchecked(ctx context.Context, in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error)
|
2016-06-12 14:06:27 +00:00
|
|
|
}
|
|
|
|
|
2017-08-03 19:42:35 +00:00
|
|
|
// PutStreamer is an optional interface for Fs
|
|
|
|
type PutStreamer interface {
|
|
|
|
// PutStream uploads to the remote path with the modTime given of indeterminate size
|
|
|
|
//
|
|
|
|
// May create the object even if it returns an error - if so
|
|
|
|
// will return the object and the error, otherwise will return
|
|
|
|
// nil and the error
|
2019-06-17 08:34:30 +00:00
|
|
|
PutStream(ctx context.Context, in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error)
|
2017-08-03 19:42:35 +00:00
|
|
|
}
|
|
|
|
|
2018-03-29 07:10:19 +00:00
|
|
|
// PublicLinker is an optional interface for Fs
|
|
|
|
type PublicLinker interface {
|
|
|
|
// PublicLink generates a public link to the remote path (usually readable by anyone)
|
2020-05-31 21:18:01 +00:00
|
|
|
PublicLink(ctx context.Context, remote string, expire Duration, unlink bool) (string, error)
|
2018-03-29 07:10:19 +00:00
|
|
|
}
|
|
|
|
|
2017-08-02 15:51:24 +00:00
|
|
|
// MergeDirser is an option interface for Fs
|
|
|
|
type MergeDirser interface {
|
|
|
|
// MergeDirs merges the contents of all the directories passed
|
|
|
|
// in into the first one and rmdirs the other directories.
|
2019-06-17 08:34:30 +00:00
|
|
|
MergeDirs(ctx context.Context, dirs []Directory) error
|
2017-08-02 15:51:24 +00:00
|
|
|
}
|
|
|
|
|
2016-07-01 15:35:36 +00:00
|
|
|
// CleanUpper is an optional interfaces for Fs
|
|
|
|
type CleanUpper interface {
|
|
|
|
// CleanUp the trash in the Fs
|
|
|
|
//
|
|
|
|
// Implement this if you have a way of emptying the trash or
|
|
|
|
// otherwise cleaning up old versions of files.
|
2019-06-17 08:34:30 +00:00
|
|
|
CleanUp(ctx context.Context) error
|
2016-07-01 15:35:36 +00:00
|
|
|
}
|
|
|
|
|
2017-06-05 15:14:24 +00:00
|
|
|
// ListRer is an optional interfaces for Fs
|
|
|
|
type ListRer interface {
|
|
|
|
// ListR lists the objects and directories of the Fs starting
|
|
|
|
// from dir recursively into out.
|
|
|
|
//
|
|
|
|
// dir should be "" to start from the root, and should not
|
|
|
|
// have trailing slashes.
|
|
|
|
//
|
2017-06-11 21:43:31 +00:00
|
|
|
// This should return ErrDirNotFound if the directory isn't
|
|
|
|
// found.
|
|
|
|
//
|
|
|
|
// It should call callback for each tranche of entries read.
|
|
|
|
// These need not be returned in any particular order. If
|
|
|
|
// callback returns an error then the listing will stop
|
|
|
|
// immediately.
|
2017-06-05 15:14:24 +00:00
|
|
|
//
|
|
|
|
// Don't implement this unless you have a more efficient way
|
|
|
|
// of listing recursively that doing a directory traversal.
|
2019-06-17 08:34:30 +00:00
|
|
|
ListR(ctx context.Context, dir string, callback ListRCallback) error
|
2017-06-05 15:14:24 +00:00
|
|
|
}
|
|
|
|
|
2018-01-21 17:33:58 +00:00
|
|
|
// RangeSeeker is the interface that wraps the RangeSeek method.
|
|
|
|
//
|
|
|
|
// Some of the returns from Object.Open() may optionally implement
|
|
|
|
// this method for efficiency purposes.
|
|
|
|
type RangeSeeker interface {
|
|
|
|
// RangeSeek behaves like a call to Seek(offset int64, whence
|
|
|
|
// int) with the output wrapped in an io.LimitedReader
|
|
|
|
// limiting the total length to limit.
|
|
|
|
//
|
|
|
|
// RangeSeek with a limit of < 0 is equivalent to a regular Seek.
|
2019-06-17 08:34:30 +00:00
|
|
|
RangeSeek(ctx context.Context, offset int64, whence int, length int64) (int64, error)
|
2018-01-21 17:33:58 +00:00
|
|
|
}
|
|
|
|
|
2018-02-09 20:48:32 +00:00
|
|
|
// Abouter is an optional interface for Fs
|
|
|
|
type Abouter interface {
|
|
|
|
// About gets quota information from the Fs
|
2019-06-17 08:34:30 +00:00
|
|
|
About(ctx context.Context) (*Usage, error)
|
2018-02-09 20:48:32 +00:00
|
|
|
}
|
|
|
|
|
2019-04-22 18:22:42 +00:00
|
|
|
// OpenWriterAter is an optional interface for Fs
|
|
|
|
type OpenWriterAter interface {
|
|
|
|
// OpenWriterAt opens with a handle for random access writes
|
|
|
|
//
|
|
|
|
// Pass in the remote desired and the size if known.
|
|
|
|
//
|
|
|
|
// It truncates any existing object
|
2019-06-17 08:34:30 +00:00
|
|
|
OpenWriterAt(ctx context.Context, remote string, size int64) (WriterAtCloser, error)
|
2019-04-22 18:22:42 +00:00
|
|
|
}
|
|
|
|
|
2019-08-12 10:07:31 +00:00
|
|
|
// UserInfoer is an optional interface for Fs
|
|
|
|
type UserInfoer interface {
|
|
|
|
// UserInfo returns info about the connected user
|
|
|
|
UserInfo(ctx context.Context) (map[string]string, error)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Disconnecter is an optional interface for Fs
|
|
|
|
type Disconnecter interface {
|
|
|
|
// Disconnect the current user
|
|
|
|
Disconnect(ctx context.Context) error
|
|
|
|
}
|
|
|
|
|
2020-04-28 11:58:34 +00:00
|
|
|
// CommandHelp describes a single backend Command
|
|
|
|
//
|
|
|
|
// These are automatically inserted in the docs
|
|
|
|
type CommandHelp struct {
|
2020-10-13 21:49:58 +00:00
|
|
|
Name string // Name of the command, e.g. "link"
|
2020-04-28 11:58:34 +00:00
|
|
|
Short string // Single line description
|
|
|
|
Long string // Long multi-line description
|
|
|
|
Opts map[string]string // maps option name to a single line help
|
|
|
|
}
|
|
|
|
|
2020-05-20 10:39:20 +00:00
|
|
|
// Commander is an interface to wrap the Command function
|
2020-04-28 11:58:34 +00:00
|
|
|
type Commander interface {
|
|
|
|
// Command the backend to run a named command
|
|
|
|
//
|
|
|
|
// The command run is name
|
|
|
|
// args may be used to read arguments from
|
|
|
|
// opts may be used to read optional arguments from
|
|
|
|
//
|
|
|
|
// The result should be capable of being JSON encoded
|
|
|
|
// If it is a string or a []string it will be shown to the user
|
|
|
|
// otherwise it will be JSON encoded and shown to the user like that
|
|
|
|
Command(ctx context.Context, name string, arg []string, opt map[string]string) (interface{}, error)
|
|
|
|
}
|
|
|
|
|
2020-11-27 17:02:00 +00:00
|
|
|
// Shutdowner is an interface to wrap the Shutdown function
|
|
|
|
type Shutdowner interface {
|
|
|
|
// Shutdown the backend, closing any background tasks and any
|
|
|
|
// cached connections.
|
|
|
|
Shutdown(ctx context.Context) error
|
|
|
|
}
|
|
|
|
|
2015-09-22 17:47:16 +00:00
|
|
|
// ObjectsChan is a channel of Objects
|
2013-06-28 07:57:32 +00:00
|
|
|
type ObjectsChan chan Object
|
2012-12-26 12:23:58 +00:00
|
|
|
|
2015-09-22 17:47:16 +00:00
|
|
|
// Objects is a slice of Object~s
|
2013-06-28 07:57:32 +00:00
|
|
|
type Objects []Object
|
2012-12-26 12:23:58 +00:00
|
|
|
|
2015-09-22 17:47:16 +00:00
|
|
|
// ObjectPair is a pair of Objects used to describe a potential copy
|
|
|
|
// operation.
|
2014-03-28 17:56:04 +00:00
|
|
|
type ObjectPair struct {
|
2018-01-12 16:30:54 +00:00
|
|
|
Src, Dst Object
|
2014-03-28 17:56:04 +00:00
|
|
|
}
|
|
|
|
|
2019-08-12 21:03:53 +00:00
|
|
|
// UnWrapFs unwraps f as much as possible and returns the base Fs
|
|
|
|
func UnWrapFs(f Fs) Fs {
|
|
|
|
for {
|
|
|
|
unwrap := f.Features().UnWrap
|
|
|
|
if unwrap == nil {
|
|
|
|
break // not a wrapped Fs, use current
|
|
|
|
}
|
|
|
|
next := unwrap()
|
|
|
|
if next == nil {
|
|
|
|
break // no base Fs found, use current
|
|
|
|
}
|
|
|
|
f = next
|
|
|
|
}
|
|
|
|
return f
|
|
|
|
}
|
|
|
|
|
|
|
|
// UnWrapObject unwraps o as much as possible and returns the base object
|
|
|
|
func UnWrapObject(o Object) Object {
|
|
|
|
for {
|
|
|
|
u, ok := o.(ObjectUnWrapper)
|
|
|
|
if !ok {
|
|
|
|
break // not a wrapped object, use current
|
|
|
|
}
|
|
|
|
next := u.UnWrap()
|
|
|
|
if next == nil {
|
|
|
|
break // no base object found, use current
|
|
|
|
}
|
|
|
|
o = next
|
|
|
|
}
|
|
|
|
return o
|
|
|
|
}
|
|
|
|
|
2020-09-17 10:46:32 +00:00
|
|
|
// UnWrapObjectInfo returns the underlying Object unwrapped as much as
|
|
|
|
// possible or nil.
|
|
|
|
func UnWrapObjectInfo(oi ObjectInfo) Object {
|
|
|
|
o, ok := oi.(Object)
|
|
|
|
if !ok {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
return UnWrapObject(o)
|
|
|
|
}
|
|
|
|
|
2020-05-20 10:39:20 +00:00
|
|
|
// Find looks for a RegInfo object for the name passed in. The name
|
2018-08-08 08:23:03 +00:00
|
|
|
// can be either the Name or the Prefix.
|
2014-03-15 16:06:11 +00:00
|
|
|
//
|
|
|
|
// Services are looked up in the config file
|
2016-02-18 11:35:25 +00:00
|
|
|
func Find(name string) (*RegInfo, error) {
|
2018-01-12 16:30:54 +00:00
|
|
|
for _, item := range Registry {
|
2018-10-01 20:05:33 +00:00
|
|
|
if item.Name == name || item.Prefix == name || item.FileName() == name {
|
2014-03-15 16:06:11 +00:00
|
|
|
return item, nil
|
|
|
|
}
|
|
|
|
}
|
2018-08-08 08:23:03 +00:00
|
|
|
return nil, errors.Errorf("didn't find backend called %q", name)
|
2014-03-15 16:06:11 +00:00
|
|
|
}
|
|
|
|
|
2016-10-08 10:26:14 +00:00
|
|
|
// MustFind looks for an Info object for the type name passed in
|
|
|
|
//
|
|
|
|
// Services are looked up in the config file
|
|
|
|
//
|
|
|
|
// Exits with a fatal error if not found
|
|
|
|
func MustFind(name string) *RegInfo {
|
|
|
|
fs, err := Find(name)
|
|
|
|
if err != nil {
|
|
|
|
log.Fatalf("Failed to find remote: %v", err)
|
|
|
|
}
|
|
|
|
return fs
|
|
|
|
}
|
|
|
|
|
2016-06-21 17:01:53 +00:00
|
|
|
// ParseRemote deconstructs a path into configName, fsPath, looking up
|
|
|
|
// the fsName in the config file (returning NotFoundInConfigFile if not found)
|
2021-02-10 14:40:36 +00:00
|
|
|
func ParseRemote(path string) (fsInfo *RegInfo, configName, fsPath string, connectionStringConfig configmap.Simple, err error) {
|
2021-02-09 09:30:40 +00:00
|
|
|
parsed, err := fspath.Parse(path)
|
2019-09-05 10:01:04 +00:00
|
|
|
if err != nil {
|
2021-02-10 14:40:36 +00:00
|
|
|
return nil, "", "", nil, err
|
2019-09-05 10:01:04 +00:00
|
|
|
}
|
2021-02-09 09:30:40 +00:00
|
|
|
configName, fsPath = parsed.Name, parsed.Path
|
2016-06-21 17:01:53 +00:00
|
|
|
var fsName string
|
2018-05-21 13:53:45 +00:00
|
|
|
var ok bool
|
2018-06-18 09:39:00 +00:00
|
|
|
if configName != "" {
|
2018-08-07 21:51:06 +00:00
|
|
|
if strings.HasPrefix(configName, ":") {
|
|
|
|
fsName = configName[1:]
|
|
|
|
} else {
|
2021-02-10 14:40:36 +00:00
|
|
|
m := ConfigMap(nil, configName, parsed.Config)
|
2018-08-07 21:51:06 +00:00
|
|
|
fsName, ok = m.Get("type")
|
|
|
|
if !ok {
|
2021-02-10 14:40:36 +00:00
|
|
|
return nil, "", "", nil, ErrorNotFoundInConfigFile
|
2018-08-07 21:51:06 +00:00
|
|
|
}
|
2016-06-21 17:01:53 +00:00
|
|
|
}
|
2018-06-18 09:39:00 +00:00
|
|
|
} else {
|
|
|
|
fsName = "local"
|
|
|
|
configName = "local"
|
2016-06-21 17:01:53 +00:00
|
|
|
}
|
|
|
|
fsInfo, err = Find(fsName)
|
2021-02-10 14:40:36 +00:00
|
|
|
return fsInfo, configName, fsPath, parsed.Config, err
|
2016-06-21 17:01:53 +00:00
|
|
|
}
|
|
|
|
|
2018-05-14 17:06:57 +00:00
|
|
|
// A configmap.Getter to read from the environment RCLONE_CONFIG_backend_option_name
|
|
|
|
type configEnvVars string
|
|
|
|
|
|
|
|
// Get a config item from the environment variables if possible
|
|
|
|
func (configName configEnvVars) Get(key string) (value string, ok bool) {
|
|
|
|
return os.LookupEnv(ConfigToEnv(string(configName), key))
|
|
|
|
}
|
|
|
|
|
|
|
|
// A configmap.Getter to read from the environment RCLONE_option_name
|
2019-09-14 17:26:07 +00:00
|
|
|
type optionEnvVars struct {
|
|
|
|
fsInfo *RegInfo
|
|
|
|
}
|
2018-05-14 17:06:57 +00:00
|
|
|
|
|
|
|
// Get a config item from the option environment variables if possible
|
2019-09-14 17:26:07 +00:00
|
|
|
func (oev optionEnvVars) Get(key string) (value string, ok bool) {
|
|
|
|
opt := oev.fsInfo.Options.Get(key)
|
|
|
|
if opt == nil {
|
|
|
|
return "", false
|
|
|
|
}
|
|
|
|
// For options with NoPrefix set, check without prefix too
|
|
|
|
if opt.NoPrefix {
|
|
|
|
value, ok = os.LookupEnv(OptionToEnv(key))
|
|
|
|
if ok {
|
|
|
|
return value, ok
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return os.LookupEnv(OptionToEnv(oev.fsInfo.Prefix + "-" + key))
|
2018-05-14 17:06:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// A configmap.Getter to read either the default value or the set
|
|
|
|
// value from the RegInfo.Options
|
|
|
|
type regInfoValues struct {
|
|
|
|
fsInfo *RegInfo
|
|
|
|
useDefault bool
|
|
|
|
}
|
|
|
|
|
|
|
|
// override the values in configMap with the either the flag values or
|
|
|
|
// the default values
|
|
|
|
func (r *regInfoValues) Get(key string) (value string, ok bool) {
|
2019-09-14 17:26:07 +00:00
|
|
|
opt := r.fsInfo.Options.Get(key)
|
|
|
|
if opt != nil && (r.useDefault || opt.Value != nil) {
|
|
|
|
return opt.String(), true
|
2018-05-14 17:06:57 +00:00
|
|
|
}
|
|
|
|
return "", false
|
|
|
|
}
|
|
|
|
|
|
|
|
// A configmap.Setter to read from the config file
|
|
|
|
type setConfigFile string
|
|
|
|
|
|
|
|
// Set a config item into the config file
|
|
|
|
func (section setConfigFile) Set(key, value string) {
|
2021-03-10 14:09:30 +00:00
|
|
|
if strings.HasPrefix(string(section), ":") {
|
2021-04-08 15:49:47 +00:00
|
|
|
Logf(nil, "Can't save config %q = %q for on the fly backend %q", key, value, section)
|
|
|
|
return
|
2021-03-10 14:09:30 +00:00
|
|
|
}
|
2018-05-14 17:06:57 +00:00
|
|
|
Debugf(nil, "Saving config %q = %q in section %q of the config file", key, value, section)
|
2019-08-14 16:06:13 +00:00
|
|
|
err := ConfigFileSet(string(section), key, value)
|
|
|
|
if err != nil {
|
|
|
|
Errorf(nil, "Failed saving config %q = %q in section %q of the config file: %v", key, value, section, err)
|
|
|
|
}
|
2018-05-14 17:06:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// A configmap.Getter to read from the config file
|
|
|
|
type getConfigFile string
|
|
|
|
|
|
|
|
// Get a config item from the config file
|
|
|
|
func (section getConfigFile) Get(key string) (value string, ok bool) {
|
|
|
|
value, ok = ConfigFileGet(string(section), key)
|
|
|
|
// Ignore empty lines in the config file
|
|
|
|
if value == "" {
|
|
|
|
ok = false
|
|
|
|
}
|
|
|
|
return value, ok
|
|
|
|
}
|
|
|
|
|
|
|
|
// ConfigMap creates a configmap.Map from the *RegInfo and the
|
2021-02-10 14:40:36 +00:00
|
|
|
// configName passed in. If connectionStringConfig has any entries (it may be nil),
|
|
|
|
// then it will be added to the lookup with the highest priority.
|
2018-05-14 17:06:57 +00:00
|
|
|
//
|
|
|
|
// If fsInfo is nil then the returned configmap.Map should only be
|
|
|
|
// used for reading non backend specific parameters, such as "type".
|
2021-02-10 14:40:36 +00:00
|
|
|
func ConfigMap(fsInfo *RegInfo, configName string, connectionStringConfig configmap.Simple) (config *configmap.Map) {
|
2018-05-14 17:06:57 +00:00
|
|
|
// Create the config
|
|
|
|
config = configmap.New()
|
|
|
|
|
|
|
|
// Read the config, more specific to least specific
|
|
|
|
|
2021-02-10 14:40:36 +00:00
|
|
|
// Config from connection string
|
|
|
|
if len(connectionStringConfig) > 0 {
|
2021-04-04 11:41:36 +00:00
|
|
|
config.AddGetter(connectionStringConfig, configmap.PriorityNormal)
|
2021-02-10 14:40:36 +00:00
|
|
|
}
|
|
|
|
|
2018-05-14 17:06:57 +00:00
|
|
|
// flag values
|
|
|
|
if fsInfo != nil {
|
2021-04-04 11:41:36 +00:00
|
|
|
config.AddGetter(®InfoValues{fsInfo, false}, configmap.PriorityNormal)
|
2018-05-14 17:06:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// remote specific environment vars
|
2021-04-04 11:41:36 +00:00
|
|
|
config.AddGetter(configEnvVars(configName), configmap.PriorityNormal)
|
2018-05-14 17:06:57 +00:00
|
|
|
|
|
|
|
// backend specific environment vars
|
|
|
|
if fsInfo != nil {
|
2021-04-04 11:41:36 +00:00
|
|
|
config.AddGetter(optionEnvVars{fsInfo: fsInfo}, configmap.PriorityNormal)
|
2018-05-14 17:06:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// config file
|
2021-04-04 11:41:36 +00:00
|
|
|
config.AddGetter(getConfigFile(configName), configmap.PriorityConfig)
|
2018-05-14 17:06:57 +00:00
|
|
|
|
|
|
|
// default values
|
|
|
|
if fsInfo != nil {
|
2021-04-04 11:41:36 +00:00
|
|
|
config.AddGetter(®InfoValues{fsInfo, true}, configmap.PriorityDefault)
|
2018-05-14 17:06:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Set Config
|
|
|
|
config.AddSetter(setConfigFile(configName))
|
|
|
|
return config
|
|
|
|
}
|
|
|
|
|
|
|
|
// ConfigFs makes the config for calling NewFs with.
|
|
|
|
//
|
|
|
|
// It parses the path which is of the form remote:path
|
|
|
|
//
|
|
|
|
// Remotes are looked up in the config file. If the remote isn't
|
|
|
|
// found then NotFoundInConfigFile will be returned.
|
|
|
|
func ConfigFs(path string) (fsInfo *RegInfo, configName, fsPath string, config *configmap.Map, err error) {
|
|
|
|
// Parse the remote path
|
2021-02-10 14:40:36 +00:00
|
|
|
fsInfo, configName, fsPath, connectionStringConfig, err := ParseRemote(path)
|
2018-05-14 17:06:57 +00:00
|
|
|
if err != nil {
|
|
|
|
return
|
|
|
|
}
|
2021-02-10 14:40:36 +00:00
|
|
|
config = ConfigMap(fsInfo, configName, connectionStringConfig)
|
2018-05-14 17:06:57 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2012-12-29 11:35:41 +00:00
|
|
|
// NewFs makes a new Fs object from the path
|
|
|
|
//
|
2014-07-31 20:24:52 +00:00
|
|
|
// The path is of the form remote:path
|
2014-03-15 16:06:11 +00:00
|
|
|
//
|
2014-07-31 20:24:52 +00:00
|
|
|
// Remotes are looked up in the config file. If the remote isn't
|
|
|
|
// found then NotFoundInConfigFile will be returned.
|
2015-02-07 15:49:09 +00:00
|
|
|
//
|
|
|
|
// On Windows avoid single character remote names as they can be mixed
|
|
|
|
// up with drive letters.
|
2020-11-05 15:18:51 +00:00
|
|
|
func NewFs(ctx context.Context, path string) (Fs, error) {
|
2020-08-31 13:51:06 +00:00
|
|
|
Debugf(nil, "Creating backend with remote %q", path)
|
2018-05-14 17:06:57 +00:00
|
|
|
fsInfo, configName, fsPath, config, err := ConfigFs(path)
|
2014-03-15 16:06:11 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2021-04-03 14:58:47 +00:00
|
|
|
overridden := fsInfo.Options.Overridden(config)
|
2021-03-10 14:10:03 +00:00
|
|
|
if len(overridden) > 0 {
|
|
|
|
extraConfig := overridden.String()
|
|
|
|
//Debugf(nil, "detected overriden config %q", extraConfig)
|
|
|
|
md5sumBinary := md5.Sum([]byte(extraConfig))
|
|
|
|
suffix := base64.RawStdEncoding.EncodeToString(md5sumBinary[:])
|
|
|
|
// 5 characters length is 5*6 = 30 bits of base64
|
|
|
|
const maxLength = 5
|
|
|
|
if len(suffix) > maxLength {
|
|
|
|
suffix = suffix[:maxLength]
|
|
|
|
}
|
|
|
|
suffix = "{" + suffix + "}"
|
|
|
|
Debugf(configName, "detected overridden config - adding %q suffix to name", suffix)
|
|
|
|
// Add the suffix to the config name
|
|
|
|
//
|
|
|
|
// These need to work as filesystem names as the VFS cache will use them
|
|
|
|
configName += suffix
|
|
|
|
}
|
2020-11-05 15:18:51 +00:00
|
|
|
return fsInfo.NewFs(ctx, configName, fsPath, config)
|
2012-12-29 11:35:41 +00:00
|
|
|
}
|
|
|
|
|
2020-05-01 10:51:38 +00:00
|
|
|
// ConfigString returns a canonical version of the config string used
|
|
|
|
// to configure the Fs as passed to fs.NewFs
|
|
|
|
func ConfigString(f Fs) string {
|
|
|
|
name := f.Name()
|
|
|
|
root := f.Root()
|
|
|
|
if name == "local" && f.Features().IsLocal {
|
|
|
|
return root
|
|
|
|
}
|
|
|
|
return name + ":" + root
|
|
|
|
}
|
|
|
|
|
2018-01-12 16:30:54 +00:00
|
|
|
// TemporaryLocalFs creates a local FS in the OS's temporary directory.
|
2017-08-03 19:42:35 +00:00
|
|
|
//
|
|
|
|
// No cleanup is performed, the caller must call Purge on the Fs themselves.
|
2020-11-05 15:18:51 +00:00
|
|
|
func TemporaryLocalFs(ctx context.Context) (Fs, error) {
|
2017-08-03 19:42:35 +00:00
|
|
|
path, err := ioutil.TempDir("", "rclone-spool")
|
|
|
|
if err == nil {
|
|
|
|
err = os.Remove(path)
|
|
|
|
}
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
path = filepath.ToSlash(path)
|
2020-11-05 15:18:51 +00:00
|
|
|
return NewFs(ctx, path)
|
2017-08-03 19:42:35 +00:00
|
|
|
}
|
|
|
|
|
2015-11-28 18:13:08 +00:00
|
|
|
// CheckClose is a utility function used to check the return from
|
2012-12-26 12:23:58 +00:00
|
|
|
// Close in a defer statement.
|
2015-11-28 18:13:08 +00:00
|
|
|
func CheckClose(c io.Closer, err *error) {
|
2012-12-26 12:23:58 +00:00
|
|
|
cerr := c.Close()
|
|
|
|
if *err == nil {
|
|
|
|
*err = cerr
|
|
|
|
}
|
|
|
|
}
|
2017-11-13 13:46:31 +00:00
|
|
|
|
|
|
|
// FileExists returns true if a file remote exists.
|
|
|
|
// If remote is a directory, FileExists returns false.
|
2019-06-17 08:34:30 +00:00
|
|
|
func FileExists(ctx context.Context, fs Fs, remote string) (bool, error) {
|
|
|
|
_, err := fs.NewObject(ctx, remote)
|
2017-11-13 13:46:31 +00:00
|
|
|
if err != nil {
|
2018-01-16 20:00:16 +00:00
|
|
|
if err == ErrorObjectNotFound || err == ErrorNotAFile || err == ErrorPermissionDenied {
|
2017-11-13 13:46:31 +00:00
|
|
|
return false, nil
|
|
|
|
}
|
|
|
|
return false, err
|
|
|
|
}
|
|
|
|
return true, nil
|
|
|
|
}
|
2018-01-12 16:30:54 +00:00
|
|
|
|
2018-06-03 18:45:34 +00:00
|
|
|
// GetModifyWindow calculates the maximum modify window between the given Fses
|
|
|
|
// and the Config.ModifyWindow parameter.
|
2020-11-05 16:27:01 +00:00
|
|
|
func GetModifyWindow(ctx context.Context, fss ...Info) time.Duration {
|
2020-11-05 11:33:32 +00:00
|
|
|
window := GetConfig(ctx).ModifyWindow
|
2018-01-12 16:30:54 +00:00
|
|
|
for _, f := range fss {
|
|
|
|
if f != nil {
|
|
|
|
precision := f.Precision()
|
|
|
|
if precision == ModTimeNotSupported {
|
2018-06-03 18:45:34 +00:00
|
|
|
return ModTimeNotSupported
|
|
|
|
}
|
|
|
|
if precision > window {
|
|
|
|
window = precision
|
2018-01-12 16:30:54 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2018-06-03 18:45:34 +00:00
|
|
|
return window
|
2018-01-12 16:30:54 +00:00
|
|
|
}
|
2019-02-09 20:52:15 +00:00
|
|
|
|
|
|
|
// Pacer is a simple wrapper around a pacer.Pacer with logging.
|
|
|
|
type Pacer struct {
|
|
|
|
*pacer.Pacer
|
|
|
|
}
|
|
|
|
|
|
|
|
type logCalculator struct {
|
|
|
|
pacer.Calculator
|
|
|
|
}
|
|
|
|
|
|
|
|
// NewPacer creates a Pacer for the given Fs and Calculator.
|
2020-11-05 11:33:32 +00:00
|
|
|
func NewPacer(ctx context.Context, c pacer.Calculator) *Pacer {
|
2021-02-16 17:53:44 +00:00
|
|
|
ci := GetConfig(ctx)
|
|
|
|
retries := ci.LowLevelRetries
|
|
|
|
if retries <= 0 {
|
|
|
|
retries = 1
|
|
|
|
}
|
2019-02-09 20:52:15 +00:00
|
|
|
p := &Pacer{
|
|
|
|
Pacer: pacer.New(
|
|
|
|
pacer.InvokerOption(pacerInvoker),
|
2021-02-16 17:53:44 +00:00
|
|
|
pacer.MaxConnectionsOption(ci.Checkers+ci.Transfers),
|
|
|
|
pacer.RetriesOption(retries),
|
2019-02-09 20:52:15 +00:00
|
|
|
pacer.CalculatorOption(c),
|
|
|
|
),
|
|
|
|
}
|
|
|
|
p.SetCalculator(c)
|
|
|
|
return p
|
|
|
|
}
|
|
|
|
|
|
|
|
func (d *logCalculator) Calculate(state pacer.State) time.Duration {
|
|
|
|
oldSleepTime := state.SleepTime
|
|
|
|
newSleepTime := d.Calculator.Calculate(state)
|
|
|
|
if state.ConsecutiveRetries > 0 {
|
|
|
|
if newSleepTime != oldSleepTime {
|
|
|
|
Debugf("pacer", "Rate limited, increasing sleep to %v", newSleepTime)
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
if newSleepTime != oldSleepTime {
|
|
|
|
Debugf("pacer", "Reducing sleep to %v", newSleepTime)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return newSleepTime
|
|
|
|
}
|
|
|
|
|
|
|
|
// SetCalculator sets the pacing algorithm. Don't modify the Calculator object
|
|
|
|
// afterwards, use the ModifyCalculator method when needed.
|
|
|
|
//
|
|
|
|
// It will choose the default algorithm if nil is passed in.
|
|
|
|
func (p *Pacer) SetCalculator(c pacer.Calculator) {
|
|
|
|
switch c.(type) {
|
|
|
|
case *logCalculator:
|
|
|
|
Logf("pacer", "Invalid Calculator in fs.Pacer.SetCalculator")
|
|
|
|
case nil:
|
|
|
|
c = &logCalculator{pacer.NewDefault()}
|
|
|
|
default:
|
|
|
|
c = &logCalculator{c}
|
|
|
|
}
|
|
|
|
|
|
|
|
p.Pacer.SetCalculator(c)
|
|
|
|
}
|
|
|
|
|
|
|
|
// ModifyCalculator calls the given function with the currently configured
|
|
|
|
// Calculator and the Pacer lock held.
|
|
|
|
func (p *Pacer) ModifyCalculator(f func(pacer.Calculator)) {
|
|
|
|
p.ModifyCalculator(func(c pacer.Calculator) {
|
|
|
|
switch _c := c.(type) {
|
|
|
|
case *logCalculator:
|
|
|
|
f(_c.Calculator)
|
|
|
|
default:
|
|
|
|
Logf("pacer", "Invalid Calculator in fs.Pacer: %t", c)
|
|
|
|
f(c)
|
|
|
|
}
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
func pacerInvoker(try, retries int, f pacer.Paced) (retry bool, err error) {
|
|
|
|
retry, err = f()
|
|
|
|
if retry {
|
|
|
|
Debugf("pacer", "low level retry %d/%d (error %v)", try, retries, err)
|
|
|
|
err = fserrors.RetryError(err)
|
|
|
|
}
|
|
|
|
return
|
|
|
|
}
|