2db0e23584
Before this change the concurrency used for an upload was rather inconsistent. - if size below `--backend-upload-cutoff` (default 200M) do single part upload. - if size below `--multi-thread-cutoff` (default 256M) or using streaming uploads (eg `rclone rcat) do multipart upload using `--backend-upload-concurrency` to set the concurrency used by the uploader. - otherwise do multipart upload using `--multi-thread-streams` to set the concurrency. This change makes the default for the concurrency used be the `--backend-upload-concurrency`. If `--multi-thread-streams` is set and larger than the `--backend-upload-concurrency` then that will be used instead. This means that if the user sets `--backend-upload-concurrency` then it will be obeyed for all multipart/multi-thread transfers and the user can override them all with `--multi-thread-streams`. See: #7056
768 lines
24 KiB
Go
768 lines
24 KiB
Go
// Filesystem features and optional interfaces
|
|
|
|
package fs
|
|
|
|
import (
|
|
"context"
|
|
"io"
|
|
"reflect"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// Features describe the optional features of the Fs
|
|
type Features struct {
|
|
// Feature flags, whether Fs
|
|
CaseInsensitive bool // has case insensitive files
|
|
DuplicateFiles bool // allows duplicate files
|
|
ReadMimeType bool // can read the mime type of objects
|
|
WriteMimeType bool // can set the mime type of objects
|
|
CanHaveEmptyDirectories bool // can have empty directories
|
|
BucketBased bool // is bucket based (like s3, swift, etc.)
|
|
BucketBasedRootOK bool // is bucket based and can use from root
|
|
SetTier bool // allows set tier functionality on objects
|
|
GetTier bool // allows to retrieve storage tier of objects
|
|
ServerSideAcrossConfigs bool // can server-side copy between different remotes of the same type
|
|
IsLocal bool // is the local backend
|
|
SlowModTime bool // if calling ModTime() generally takes an extra transaction
|
|
SlowHash bool // if calling Hash() generally takes an extra transaction
|
|
ReadMetadata bool // can read metadata from objects
|
|
WriteMetadata bool // can write metadata to objects
|
|
UserMetadata bool // can read/write general purpose metadata
|
|
FilterAware bool // can make use of filters if provided for listing
|
|
PartialUploads bool // uploaded file can appear incomplete on the fs while it's being uploaded
|
|
NoMultiThreading bool // set if can't have multiplethreads on one download open
|
|
Overlay bool // this wraps one or more backends to add functionality
|
|
|
|
// Purge all files in the directory specified
|
|
//
|
|
// Implement this if you have a way of deleting all the files
|
|
// quicker than just running Remove() on the result of List()
|
|
//
|
|
// Return an error if it doesn't exist
|
|
Purge func(ctx context.Context, dir string) error
|
|
|
|
// Copy src to this remote using server-side copy operations.
|
|
//
|
|
// This is stored with the remote path given
|
|
//
|
|
// It returns the destination Object and a possible error
|
|
//
|
|
// Will only be called if src.Fs().Name() == f.Name()
|
|
//
|
|
// If it isn't possible then return fs.ErrorCantCopy
|
|
Copy func(ctx context.Context, src Object, remote string) (Object, error)
|
|
|
|
// Move src to this remote using server-side move operations.
|
|
//
|
|
// This is stored with the remote path given
|
|
//
|
|
// It returns the destination Object and a possible error
|
|
//
|
|
// Will only be called if src.Fs().Name() == f.Name()
|
|
//
|
|
// If it isn't possible then return fs.ErrorCantMove
|
|
Move func(ctx context.Context, src Object, remote string) (Object, error)
|
|
|
|
// DirMove moves src, srcRemote to this remote at dstRemote
|
|
// using server-side move operations.
|
|
//
|
|
// Will only be called if src.Fs().Name() == f.Name()
|
|
//
|
|
// If it isn't possible then return fs.ErrorCantDirMove
|
|
//
|
|
// If destination exists then return fs.ErrorDirExists
|
|
DirMove func(ctx context.Context, src Fs, srcRemote, dstRemote string) error
|
|
|
|
// ChangeNotify calls the passed function with a path
|
|
// that has had changes. If the implementation
|
|
// uses polling, it should adhere to the given interval.
|
|
ChangeNotify func(context.Context, func(string, EntryType), <-chan time.Duration)
|
|
|
|
// UnWrap returns the Fs that this Fs is wrapping
|
|
UnWrap func() Fs
|
|
|
|
// WrapFs returns the Fs that is wrapping this Fs
|
|
WrapFs func() Fs
|
|
|
|
// SetWrapper sets the Fs that is wrapping this Fs
|
|
SetWrapper func(f Fs)
|
|
|
|
// DirCacheFlush resets the directory cache - used in testing
|
|
// as an optional interface
|
|
DirCacheFlush func()
|
|
|
|
// PublicLink generates a public link to the remote path (usually readable by anyone)
|
|
PublicLink func(ctx context.Context, remote string, expire Duration, unlink bool) (string, error)
|
|
|
|
// Put in to the remote path with the modTime given of the given size
|
|
//
|
|
// May create the object even if it returns an error - if so
|
|
// will return the object and the error, otherwise will return
|
|
// nil and the error
|
|
//
|
|
// May create duplicates or return errors if src already
|
|
// exists.
|
|
PutUnchecked func(ctx context.Context, in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error)
|
|
|
|
// PutStream uploads to the remote path with the modTime given of indeterminate size
|
|
//
|
|
// May create the object even if it returns an error - if so
|
|
// will return the object and the error, otherwise will return
|
|
// nil and the error
|
|
PutStream func(ctx context.Context, in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error)
|
|
|
|
// MergeDirs merges the contents of all the directories passed
|
|
// in into the first one and rmdirs the other directories.
|
|
MergeDirs func(ctx context.Context, dirs []Directory) error
|
|
|
|
// CleanUp the trash in the Fs
|
|
//
|
|
// Implement this if you have a way of emptying the trash or
|
|
// otherwise cleaning up old versions of files.
|
|
CleanUp func(ctx context.Context) error
|
|
|
|
// ListR lists the objects and directories of the Fs starting
|
|
// from dir recursively into out.
|
|
//
|
|
// dir should be "" to start from the root, and should not
|
|
// have trailing slashes.
|
|
//
|
|
// This should return ErrDirNotFound if the directory isn't
|
|
// found.
|
|
//
|
|
// It should call callback for each tranche of entries read.
|
|
// These need not be returned in any particular order. If
|
|
// callback returns an error then the listing will stop
|
|
// immediately.
|
|
//
|
|
// Don't implement this unless you have a more efficient way
|
|
// of listing recursively that doing a directory traversal.
|
|
ListR ListRFn
|
|
|
|
// About gets quota information from the Fs
|
|
About func(ctx context.Context) (*Usage, error)
|
|
|
|
// OpenWriterAt opens with a handle for random access writes
|
|
//
|
|
// Pass in the remote desired and the size if known.
|
|
//
|
|
// It truncates any existing object
|
|
OpenWriterAt func(ctx context.Context, remote string, size int64) (WriterAtCloser, error)
|
|
|
|
// OpenChunkWriter returns the chunk size and a ChunkWriter
|
|
//
|
|
// Pass in the remote and the src object
|
|
// You can also use options to hint at the desired chunk size
|
|
//
|
|
OpenChunkWriter func(ctx context.Context, remote string, src ObjectInfo, options ...OpenOption) (info ChunkWriterInfo, writer ChunkWriter, err error)
|
|
|
|
// UserInfo returns info about the connected user
|
|
UserInfo func(ctx context.Context) (map[string]string, error)
|
|
|
|
// Disconnect the current user
|
|
Disconnect func(ctx context.Context) error
|
|
|
|
// Command the backend to run a named command
|
|
//
|
|
// The command run is name
|
|
// args may be used to read arguments from
|
|
// opts may be used to read optional arguments from
|
|
//
|
|
// The result should be capable of being JSON encoded
|
|
// If it is a string or a []string it will be shown to the user
|
|
// otherwise it will be JSON encoded and shown to the user like that
|
|
Command func(ctx context.Context, name string, arg []string, opt map[string]string) (interface{}, error)
|
|
|
|
// Shutdown the backend, closing any background tasks and any
|
|
// cached connections.
|
|
Shutdown func(ctx context.Context) error
|
|
}
|
|
|
|
// Disable nil's out the named feature. If it isn't found then it
|
|
// will log a message.
|
|
func (ft *Features) Disable(name string) *Features {
|
|
// Prefix boolean values with ! to set the feature
|
|
invert := false
|
|
if strings.HasPrefix(name, "!") {
|
|
name = name[1:]
|
|
invert = true
|
|
}
|
|
v := reflect.ValueOf(ft).Elem()
|
|
vType := v.Type()
|
|
for i := 0; i < v.NumField(); i++ {
|
|
vName := vType.Field(i).Name
|
|
field := v.Field(i)
|
|
if strings.EqualFold(name, vName) {
|
|
if !field.CanSet() {
|
|
Errorf(nil, "Can't set Feature %q", name)
|
|
} else {
|
|
if invert {
|
|
if field.Type().Kind() == reflect.Bool {
|
|
field.Set(reflect.ValueOf(true))
|
|
Debugf(nil, "Set feature %q", name)
|
|
} else {
|
|
Errorf(nil, "Can't set Feature %q to true", name)
|
|
}
|
|
} else {
|
|
zero := reflect.Zero(field.Type())
|
|
field.Set(zero)
|
|
Debugf(nil, "Reset feature %q", name)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return ft
|
|
}
|
|
|
|
// List returns a slice of all the possible feature names
|
|
func (ft *Features) List() (out []string) {
|
|
v := reflect.ValueOf(ft).Elem()
|
|
vType := v.Type()
|
|
for i := 0; i < v.NumField(); i++ {
|
|
out = append(out, vType.Field(i).Name)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Enabled returns a map of features with keys showing whether they
|
|
// are enabled or not
|
|
func (ft *Features) Enabled() (features map[string]bool) {
|
|
v := reflect.ValueOf(ft).Elem()
|
|
vType := v.Type()
|
|
features = make(map[string]bool, v.NumField())
|
|
for i := 0; i < v.NumField(); i++ {
|
|
vName := vType.Field(i).Name
|
|
field := v.Field(i)
|
|
if field.Kind() == reflect.Func {
|
|
// Can't compare functions
|
|
features[vName] = !field.IsNil()
|
|
} else {
|
|
zero := reflect.Zero(field.Type())
|
|
features[vName] = field.Interface() != zero.Interface()
|
|
}
|
|
}
|
|
return features
|
|
}
|
|
|
|
// DisableList nil's out the comma separated list of named features.
|
|
// If it isn't found then it will log a message.
|
|
func (ft *Features) DisableList(list []string) *Features {
|
|
for _, feature := range list {
|
|
ft.Disable(strings.TrimSpace(feature))
|
|
}
|
|
return ft
|
|
}
|
|
|
|
// Fill fills in the function pointers in the Features struct from the
|
|
// optional interfaces. It returns the original updated Features
|
|
// struct passed in.
|
|
func (ft *Features) Fill(ctx context.Context, f Fs) *Features {
|
|
if do, ok := f.(Purger); ok {
|
|
ft.Purge = do.Purge
|
|
}
|
|
if do, ok := f.(Copier); ok {
|
|
ft.Copy = do.Copy
|
|
}
|
|
if do, ok := f.(Mover); ok {
|
|
ft.Move = do.Move
|
|
}
|
|
if do, ok := f.(DirMover); ok {
|
|
ft.DirMove = do.DirMove
|
|
}
|
|
if do, ok := f.(ChangeNotifier); ok {
|
|
ft.ChangeNotify = do.ChangeNotify
|
|
}
|
|
if do, ok := f.(UnWrapper); ok {
|
|
ft.UnWrap = do.UnWrap
|
|
}
|
|
if do, ok := f.(Wrapper); ok {
|
|
ft.WrapFs = do.WrapFs
|
|
ft.SetWrapper = do.SetWrapper
|
|
ft.Overlay = true // if implement UnWrap then must be an Overlay
|
|
}
|
|
if do, ok := f.(DirCacheFlusher); ok {
|
|
ft.DirCacheFlush = do.DirCacheFlush
|
|
}
|
|
if do, ok := f.(PublicLinker); ok {
|
|
ft.PublicLink = do.PublicLink
|
|
}
|
|
if do, ok := f.(PutUncheckeder); ok {
|
|
ft.PutUnchecked = do.PutUnchecked
|
|
}
|
|
if do, ok := f.(PutStreamer); ok {
|
|
ft.PutStream = do.PutStream
|
|
}
|
|
if do, ok := f.(MergeDirser); ok {
|
|
ft.MergeDirs = do.MergeDirs
|
|
}
|
|
if do, ok := f.(CleanUpper); ok {
|
|
ft.CleanUp = do.CleanUp
|
|
}
|
|
if do, ok := f.(ListRer); ok {
|
|
ft.ListR = do.ListR
|
|
}
|
|
if do, ok := f.(Abouter); ok {
|
|
ft.About = do.About
|
|
}
|
|
if do, ok := f.(OpenWriterAter); ok {
|
|
ft.OpenWriterAt = do.OpenWriterAt
|
|
}
|
|
if do, ok := f.(OpenChunkWriter); ok {
|
|
ft.OpenChunkWriter = do.OpenChunkWriter
|
|
}
|
|
if do, ok := f.(UserInfoer); ok {
|
|
ft.UserInfo = do.UserInfo
|
|
}
|
|
if do, ok := f.(Disconnecter); ok {
|
|
ft.Disconnect = do.Disconnect
|
|
}
|
|
if do, ok := f.(Commander); ok {
|
|
ft.Command = do.Command
|
|
}
|
|
if do, ok := f.(Shutdowner); ok {
|
|
ft.Shutdown = do.Shutdown
|
|
}
|
|
return ft.DisableList(GetConfig(ctx).DisableFeatures)
|
|
}
|
|
|
|
// Mask the Features with the Fs passed in
|
|
//
|
|
// Only optional features which are implemented in both the original
|
|
// Fs AND the one passed in will be advertised. Any features which
|
|
// aren't in both will be set to false/nil, except for UnWrap/Wrap which
|
|
// will be left untouched.
|
|
func (ft *Features) Mask(ctx context.Context, f Fs) *Features {
|
|
mask := f.Features()
|
|
ft.CaseInsensitive = ft.CaseInsensitive && mask.CaseInsensitive
|
|
ft.DuplicateFiles = ft.DuplicateFiles && mask.DuplicateFiles
|
|
ft.ReadMimeType = ft.ReadMimeType && mask.ReadMimeType
|
|
ft.WriteMimeType = ft.WriteMimeType && mask.WriteMimeType
|
|
ft.ReadMetadata = ft.ReadMetadata && mask.ReadMetadata
|
|
ft.WriteMetadata = ft.WriteMetadata && mask.WriteMetadata
|
|
ft.UserMetadata = ft.UserMetadata && mask.UserMetadata
|
|
ft.CanHaveEmptyDirectories = ft.CanHaveEmptyDirectories && mask.CanHaveEmptyDirectories
|
|
ft.BucketBased = ft.BucketBased && mask.BucketBased
|
|
ft.BucketBasedRootOK = ft.BucketBasedRootOK && mask.BucketBasedRootOK
|
|
ft.SetTier = ft.SetTier && mask.SetTier
|
|
ft.GetTier = ft.GetTier && mask.GetTier
|
|
ft.ServerSideAcrossConfigs = ft.ServerSideAcrossConfigs && mask.ServerSideAcrossConfigs
|
|
// ft.IsLocal = ft.IsLocal && mask.IsLocal Don't propagate IsLocal
|
|
ft.SlowModTime = ft.SlowModTime && mask.SlowModTime
|
|
ft.SlowHash = ft.SlowHash && mask.SlowHash
|
|
ft.FilterAware = ft.FilterAware && mask.FilterAware
|
|
ft.PartialUploads = ft.PartialUploads && mask.PartialUploads
|
|
ft.NoMultiThreading = ft.NoMultiThreading && mask.NoMultiThreading
|
|
// ft.Overlay = ft.Overlay && mask.Overlay don't propagate Overlay
|
|
|
|
if mask.Purge == nil {
|
|
ft.Purge = nil
|
|
}
|
|
if mask.Copy == nil {
|
|
ft.Copy = nil
|
|
}
|
|
if mask.Move == nil {
|
|
ft.Move = nil
|
|
}
|
|
if mask.DirMove == nil {
|
|
ft.DirMove = nil
|
|
}
|
|
if mask.ChangeNotify == nil {
|
|
ft.ChangeNotify = nil
|
|
}
|
|
// if mask.UnWrap == nil {
|
|
// ft.UnWrap = nil
|
|
// }
|
|
// if mask.Wrapper == nil {
|
|
// ft.Wrapper = nil
|
|
// }
|
|
if mask.DirCacheFlush == nil {
|
|
ft.DirCacheFlush = nil
|
|
}
|
|
if mask.PublicLink == nil {
|
|
ft.PublicLink = nil
|
|
}
|
|
if mask.PutUnchecked == nil {
|
|
ft.PutUnchecked = nil
|
|
}
|
|
if mask.PutStream == nil {
|
|
ft.PutStream = nil
|
|
}
|
|
if mask.MergeDirs == nil {
|
|
ft.MergeDirs = nil
|
|
}
|
|
if mask.CleanUp == nil {
|
|
ft.CleanUp = nil
|
|
}
|
|
if mask.ListR == nil {
|
|
ft.ListR = nil
|
|
}
|
|
if mask.About == nil {
|
|
ft.About = nil
|
|
}
|
|
if mask.OpenWriterAt == nil {
|
|
ft.OpenWriterAt = nil
|
|
}
|
|
if mask.OpenChunkWriter == nil {
|
|
ft.OpenChunkWriter = nil
|
|
}
|
|
if mask.UserInfo == nil {
|
|
ft.UserInfo = nil
|
|
}
|
|
if mask.Disconnect == nil {
|
|
ft.Disconnect = nil
|
|
}
|
|
// Command is always local so we don't mask it
|
|
if mask.Shutdown == nil {
|
|
ft.Shutdown = nil
|
|
}
|
|
return ft.DisableList(GetConfig(ctx).DisableFeatures)
|
|
}
|
|
|
|
// Wrap makes a Copy of the features passed in, overriding the UnWrap/Wrap
|
|
// method only if available in f.
|
|
func (ft *Features) Wrap(f Fs) *Features {
|
|
ftCopy := new(Features)
|
|
*ftCopy = *ft
|
|
if do, ok := f.(UnWrapper); ok {
|
|
ftCopy.UnWrap = do.UnWrap
|
|
}
|
|
if do, ok := f.(Wrapper); ok {
|
|
ftCopy.WrapFs = do.WrapFs
|
|
ftCopy.SetWrapper = do.SetWrapper
|
|
}
|
|
return ftCopy
|
|
}
|
|
|
|
// WrapsFs adds extra information between `f` which wraps `w`
|
|
func (ft *Features) WrapsFs(f Fs, w Fs) *Features {
|
|
wFeatures := w.Features()
|
|
if wFeatures.WrapFs != nil && wFeatures.SetWrapper != nil {
|
|
wFeatures.SetWrapper(f)
|
|
}
|
|
return ft
|
|
}
|
|
|
|
// Purger is an optional interfaces for Fs
|
|
type Purger interface {
|
|
// Purge all files in the directory specified
|
|
//
|
|
// Implement this if you have a way of deleting all the files
|
|
// quicker than just running Remove() on the result of List()
|
|
//
|
|
// Return an error if it doesn't exist
|
|
Purge(ctx context.Context, dir string) error
|
|
}
|
|
|
|
// Copier is an optional interface for Fs
|
|
type Copier interface {
|
|
// Copy src to this remote using server-side copy operations.
|
|
//
|
|
// This is stored with the remote path given
|
|
//
|
|
// It returns the destination Object and a possible error
|
|
//
|
|
// Will only be called if src.Fs().Name() == f.Name()
|
|
//
|
|
// If it isn't possible then return fs.ErrorCantCopy
|
|
Copy(ctx context.Context, src Object, remote string) (Object, error)
|
|
}
|
|
|
|
// Mover is an optional interface for Fs
|
|
type Mover interface {
|
|
// Move src to this remote using server-side move operations.
|
|
//
|
|
// This is stored with the remote path given
|
|
//
|
|
// It returns the destination Object and a possible error
|
|
//
|
|
// Will only be called if src.Fs().Name() == f.Name()
|
|
//
|
|
// If it isn't possible then return fs.ErrorCantMove
|
|
Move(ctx context.Context, src Object, remote string) (Object, error)
|
|
}
|
|
|
|
// DirMover is an optional interface for Fs
|
|
type DirMover interface {
|
|
// DirMove moves src, srcRemote to this remote at dstRemote
|
|
// using server-side move operations.
|
|
//
|
|
// Will only be called if src.Fs().Name() == f.Name()
|
|
//
|
|
// If it isn't possible then return fs.ErrorCantDirMove
|
|
//
|
|
// If destination exists then return fs.ErrorDirExists
|
|
DirMove(ctx context.Context, src Fs, srcRemote, dstRemote string) error
|
|
}
|
|
|
|
// ChangeNotifier is an optional interface for Fs
|
|
type ChangeNotifier interface {
|
|
// ChangeNotify calls the passed function with a path
|
|
// that has had changes. If the implementation
|
|
// uses polling, it should adhere to the given interval.
|
|
// At least one value will be written to the channel,
|
|
// specifying the initial value and updated values might
|
|
// follow. A 0 Duration should pause the polling.
|
|
// The ChangeNotify implementation must empty the channel
|
|
// regularly. When the channel gets closed, the implementation
|
|
// should stop polling and release resources.
|
|
ChangeNotify(context.Context, func(string, EntryType), <-chan time.Duration)
|
|
}
|
|
|
|
// EntryType can be associated with remote paths to identify their type
|
|
type EntryType int
|
|
|
|
// Constants
|
|
const (
|
|
// EntryDirectory should be used to classify remote paths in directories
|
|
EntryDirectory EntryType = iota // 0
|
|
// EntryObject should be used to classify remote paths in objects
|
|
EntryObject // 1
|
|
)
|
|
|
|
// UnWrapper is an optional interfaces for Fs
|
|
type UnWrapper interface {
|
|
// UnWrap returns the Fs that this Fs is wrapping
|
|
UnWrap() Fs
|
|
}
|
|
|
|
// Wrapper is an optional interfaces for Fs
|
|
type Wrapper interface {
|
|
// Wrap returns the Fs that is wrapping this Fs
|
|
WrapFs() Fs
|
|
// SetWrapper sets the Fs that is wrapping this Fs
|
|
SetWrapper(f Fs)
|
|
}
|
|
|
|
// DirCacheFlusher is an optional interface for Fs
|
|
type DirCacheFlusher interface {
|
|
// DirCacheFlush resets the directory cache - used in testing
|
|
// as an optional interface
|
|
DirCacheFlush()
|
|
}
|
|
|
|
// PutUncheckeder is an optional interface for Fs
|
|
type PutUncheckeder interface {
|
|
// Put in to the remote path with the modTime given of the given size
|
|
//
|
|
// May create the object even if it returns an error - if so
|
|
// will return the object and the error, otherwise will return
|
|
// nil and the error
|
|
//
|
|
// May create duplicates or return errors if src already
|
|
// exists.
|
|
PutUnchecked(ctx context.Context, in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error)
|
|
}
|
|
|
|
// PutStreamer is an optional interface for Fs
|
|
type PutStreamer interface {
|
|
// PutStream uploads to the remote path with the modTime given of indeterminate size
|
|
//
|
|
// May create the object even if it returns an error - if so
|
|
// will return the object and the error, otherwise will return
|
|
// nil and the error
|
|
PutStream(ctx context.Context, in io.Reader, src ObjectInfo, options ...OpenOption) (Object, error)
|
|
}
|
|
|
|
// PublicLinker is an optional interface for Fs
|
|
type PublicLinker interface {
|
|
// PublicLink generates a public link to the remote path (usually readable by anyone)
|
|
PublicLink(ctx context.Context, remote string, expire Duration, unlink bool) (string, error)
|
|
}
|
|
|
|
// MergeDirser is an option interface for Fs
|
|
type MergeDirser interface {
|
|
// MergeDirs merges the contents of all the directories passed
|
|
// in into the first one and rmdirs the other directories.
|
|
MergeDirs(ctx context.Context, dirs []Directory) error
|
|
}
|
|
|
|
// CleanUpper is an optional interfaces for Fs
|
|
type CleanUpper interface {
|
|
// CleanUp the trash in the Fs
|
|
//
|
|
// Implement this if you have a way of emptying the trash or
|
|
// otherwise cleaning up old versions of files.
|
|
CleanUp(ctx context.Context) error
|
|
}
|
|
|
|
// ListRer is an optional interfaces for Fs
|
|
type ListRer interface {
|
|
// ListR lists the objects and directories of the Fs starting
|
|
// from dir recursively into out.
|
|
//
|
|
// dir should be "" to start from the root, and should not
|
|
// have trailing slashes.
|
|
//
|
|
// This should return ErrDirNotFound if the directory isn't
|
|
// found.
|
|
//
|
|
// It should call callback for each tranche of entries read.
|
|
// These need not be returned in any particular order. If
|
|
// callback returns an error then the listing will stop
|
|
// immediately.
|
|
//
|
|
// Don't implement this unless you have a more efficient way
|
|
// of listing recursively that doing a directory traversal.
|
|
ListR(ctx context.Context, dir string, callback ListRCallback) error
|
|
}
|
|
|
|
// RangeSeeker is the interface that wraps the RangeSeek method.
|
|
//
|
|
// Some of the returns from Object.Open() may optionally implement
|
|
// this method for efficiency purposes.
|
|
type RangeSeeker interface {
|
|
// RangeSeek behaves like a call to Seek(offset int64, whence
|
|
// int) with the output wrapped in an io.LimitedReader
|
|
// limiting the total length to limit.
|
|
//
|
|
// RangeSeek with a limit of < 0 is equivalent to a regular Seek.
|
|
RangeSeek(ctx context.Context, offset int64, whence int, length int64) (int64, error)
|
|
}
|
|
|
|
// Abouter is an optional interface for Fs
|
|
type Abouter interface {
|
|
// About gets quota information from the Fs
|
|
About(ctx context.Context) (*Usage, error)
|
|
}
|
|
|
|
// OpenWriterAter is an optional interface for Fs
|
|
type OpenWriterAter interface {
|
|
// OpenWriterAt opens with a handle for random access writes
|
|
//
|
|
// Pass in the remote desired and the size if known.
|
|
//
|
|
// It truncates any existing object
|
|
OpenWriterAt(ctx context.Context, remote string, size int64) (WriterAtCloser, error)
|
|
}
|
|
|
|
// OpenWriterAtFn describes the OpenWriterAt function pointer
|
|
type OpenWriterAtFn func(ctx context.Context, remote string, size int64) (WriterAtCloser, error)
|
|
|
|
// ChunkWriterInfo describes how a backend would like ChunkWriter called
|
|
type ChunkWriterInfo struct {
|
|
ChunkSize int64 // preferred chunk size
|
|
Concurrency int // how many chunks to write at once
|
|
LeavePartsOnError bool // if set don't delete parts uploaded so far on error
|
|
}
|
|
|
|
// OpenChunkWriter is an option interface for Fs to implement chunked writing
|
|
type OpenChunkWriter interface {
|
|
// OpenChunkWriter returns the chunk size and a ChunkWriter
|
|
//
|
|
// Pass in the remote and the src object
|
|
// You can also use options to hint at the desired chunk size
|
|
OpenChunkWriter(ctx context.Context, remote string, src ObjectInfo, options ...OpenOption) (info ChunkWriterInfo, writer ChunkWriter, err error)
|
|
}
|
|
|
|
// OpenChunkWriterFn describes the OpenChunkWriter function pointer
|
|
type OpenChunkWriterFn func(ctx context.Context, remote string, src ObjectInfo, options ...OpenOption) (info ChunkWriterInfo, writer ChunkWriter, err error)
|
|
|
|
// ChunkWriter is returned by OpenChunkWriter to implement chunked writing
|
|
type ChunkWriter interface {
|
|
// WriteChunk will write chunk number with reader bytes, where chunk number >= 0
|
|
WriteChunk(ctx context.Context, chunkNumber int, reader io.ReadSeeker) (bytesWritten int64, err error)
|
|
|
|
// Close complete chunked writer
|
|
Close(ctx context.Context) error
|
|
|
|
// Abort chunk write
|
|
Abort(ctx context.Context) error
|
|
}
|
|
|
|
// UserInfoer is an optional interface for Fs
|
|
type UserInfoer interface {
|
|
// UserInfo returns info about the connected user
|
|
UserInfo(ctx context.Context) (map[string]string, error)
|
|
}
|
|
|
|
// Disconnecter is an optional interface for Fs
|
|
type Disconnecter interface {
|
|
// Disconnect the current user
|
|
Disconnect(ctx context.Context) error
|
|
}
|
|
|
|
// CommandHelp describes a single backend Command
|
|
//
|
|
// These are automatically inserted in the docs
|
|
type CommandHelp struct {
|
|
Name string // Name of the command, e.g. "link"
|
|
Short string // Single line description
|
|
Long string // Long multi-line description
|
|
Opts map[string]string // maps option name to a single line help
|
|
}
|
|
|
|
// Commander is an interface to wrap the Command function
|
|
type Commander interface {
|
|
// Command the backend to run a named command
|
|
//
|
|
// The command run is name
|
|
// args may be used to read arguments from
|
|
// opts may be used to read optional arguments from
|
|
//
|
|
// The result should be capable of being JSON encoded
|
|
// If it is a string or a []string it will be shown to the user
|
|
// otherwise it will be JSON encoded and shown to the user like that
|
|
Command(ctx context.Context, name string, arg []string, opt map[string]string) (interface{}, error)
|
|
}
|
|
|
|
// Shutdowner is an interface to wrap the Shutdown function
|
|
type Shutdowner interface {
|
|
// Shutdown the backend, closing any background tasks and any
|
|
// cached connections.
|
|
Shutdown(ctx context.Context) error
|
|
}
|
|
|
|
// ObjectsChan is a channel of Objects
|
|
type ObjectsChan chan Object
|
|
|
|
// Objects is a slice of Object~s
|
|
type Objects []Object
|
|
|
|
// ObjectPair is a pair of Objects used to describe a potential copy
|
|
// operation.
|
|
type ObjectPair struct {
|
|
Src, Dst Object
|
|
}
|
|
|
|
// UnWrapFs unwraps f as much as possible and returns the base Fs
|
|
func UnWrapFs(f Fs) Fs {
|
|
for {
|
|
unwrap := f.Features().UnWrap
|
|
if unwrap == nil {
|
|
break // not a wrapped Fs, use current
|
|
}
|
|
next := unwrap()
|
|
if next == nil {
|
|
break // no base Fs found, use current
|
|
}
|
|
f = next
|
|
}
|
|
return f
|
|
}
|
|
|
|
// UnWrapObject unwraps o as much as possible and returns the base object
|
|
func UnWrapObject(o Object) Object {
|
|
for {
|
|
u, ok := o.(ObjectUnWrapper)
|
|
if !ok {
|
|
break // not a wrapped object, use current
|
|
}
|
|
next := u.UnWrap()
|
|
if next == nil {
|
|
break // no base object found, use current
|
|
}
|
|
o = next
|
|
}
|
|
return o
|
|
}
|
|
|
|
// UnWrapObjectInfo returns the underlying Object unwrapped as much as
|
|
// possible or nil.
|
|
func UnWrapObjectInfo(oi ObjectInfo) Object {
|
|
o, ok := oi.(Object)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
return UnWrapObject(o)
|
|
}
|