913 lines
23 KiB
Go
913 lines
23 KiB
Go
|
// Package filescom provides an interface to the Files.com
|
||
|
// object storage system.
|
||
|
package filescom
|
||
|
|
||
|
import (
|
||
|
"context"
|
||
|
"errors"
|
||
|
"fmt"
|
||
|
"io"
|
||
|
"net/http"
|
||
|
"net/url"
|
||
|
"path"
|
||
|
"strings"
|
||
|
"time"
|
||
|
|
||
|
files_sdk "github.com/Files-com/files-sdk-go/v3"
|
||
|
"github.com/Files-com/files-sdk-go/v3/bundle"
|
||
|
"github.com/Files-com/files-sdk-go/v3/file"
|
||
|
file_migration "github.com/Files-com/files-sdk-go/v3/filemigration"
|
||
|
"github.com/Files-com/files-sdk-go/v3/folder"
|
||
|
"github.com/Files-com/files-sdk-go/v3/session"
|
||
|
"github.com/rclone/rclone/fs"
|
||
|
"github.com/rclone/rclone/fs/config"
|
||
|
"github.com/rclone/rclone/fs/config/configmap"
|
||
|
"github.com/rclone/rclone/fs/config/configstruct"
|
||
|
"github.com/rclone/rclone/fs/config/obscure"
|
||
|
"github.com/rclone/rclone/fs/fserrors"
|
||
|
"github.com/rclone/rclone/fs/fshttp"
|
||
|
"github.com/rclone/rclone/fs/hash"
|
||
|
"github.com/rclone/rclone/lib/encoder"
|
||
|
"github.com/rclone/rclone/lib/pacer"
|
||
|
)
|
||
|
|
||
|
/*
|
||
|
Run of rclone info
|
||
|
stringNeedsEscaping = []rune{
|
||
|
'/', '\x00'
|
||
|
}
|
||
|
maxFileLength = 512 // for 1 byte unicode characters
|
||
|
maxFileLength = 512 // for 2 byte unicode characters
|
||
|
maxFileLength = 512 // for 3 byte unicode characters
|
||
|
maxFileLength = 512 // for 4 byte unicode characters
|
||
|
canWriteUnnormalized = true
|
||
|
canReadUnnormalized = true
|
||
|
canReadRenormalized = true
|
||
|
canStream = true
|
||
|
*/
|
||
|
|
||
|
const (
|
||
|
minSleep = 10 * time.Millisecond
|
||
|
maxSleep = 2 * time.Second
|
||
|
decayConstant = 2 // bigger for slower decay, exponential
|
||
|
|
||
|
folderNotEmpty = "processing-failure/folder-not-empty"
|
||
|
)
|
||
|
|
||
|
// Register with Fs
|
||
|
func init() {
|
||
|
fs.Register(&fs.RegInfo{
|
||
|
Name: "filescom",
|
||
|
Description: "Files.com",
|
||
|
NewFs: NewFs,
|
||
|
Options: []fs.Option{
|
||
|
{
|
||
|
Name: "site",
|
||
|
Help: "Your site subdomain (e.g. mysite) or custom domain (e.g. myfiles.customdomain.com).",
|
||
|
}, {
|
||
|
Name: "username",
|
||
|
Help: "The username used to authenticate with Files.com.",
|
||
|
}, {
|
||
|
Name: "password",
|
||
|
Help: "The password used to authenticate with Files.com.",
|
||
|
IsPassword: true,
|
||
|
}, {
|
||
|
Name: "api_key",
|
||
|
Help: "The API key used to authenticate with Files.com.",
|
||
|
Advanced: true,
|
||
|
Sensitive: true,
|
||
|
}, {
|
||
|
Name: config.ConfigEncoding,
|
||
|
Help: config.ConfigEncodingHelp,
|
||
|
Advanced: true,
|
||
|
Default: (encoder.Display |
|
||
|
encoder.EncodeBackSlash |
|
||
|
encoder.EncodeRightSpace |
|
||
|
encoder.EncodeRightCrLfHtVt |
|
||
|
encoder.EncodeInvalidUtf8),
|
||
|
}},
|
||
|
})
|
||
|
}
|
||
|
|
||
|
// Options defines the configuration for this backend
|
||
|
type Options struct {
|
||
|
Site string `config:"site"`
|
||
|
Username string `config:"username"`
|
||
|
Password string `config:"password"`
|
||
|
APIKey string `config:"api_key"`
|
||
|
Enc encoder.MultiEncoder `config:"encoding"`
|
||
|
}
|
||
|
|
||
|
// Fs represents a remote files.com server
|
||
|
type Fs struct {
|
||
|
name string // name of this remote
|
||
|
root string // the path we are working on
|
||
|
opt Options // parsed options
|
||
|
features *fs.Features // optional features
|
||
|
fileClient *file.Client // the connection to the file API
|
||
|
folderClient *folder.Client // the connection to the folder API
|
||
|
migrationClient *file_migration.Client // the connection to the file migration API
|
||
|
bundleClient *bundle.Client // the connection to the bundle API
|
||
|
pacer *fs.Pacer // pacer for API calls
|
||
|
}
|
||
|
|
||
|
// Object describes a files object
|
||
|
//
|
||
|
// Will definitely have info but maybe not meta
|
||
|
type Object struct {
|
||
|
fs *Fs // what this object is part of
|
||
|
remote string // The remote path
|
||
|
size int64 // size of the object
|
||
|
crc32 string // CRC32 of the object content
|
||
|
md5 string // MD5 of the object content
|
||
|
mimeType string // Content-Type of the object
|
||
|
modTime time.Time // modification time of the object
|
||
|
}
|
||
|
|
||
|
// ------------------------------------------------------------
|
||
|
|
||
|
// Name of the remote (as passed into NewFs)
|
||
|
func (f *Fs) Name() string {
|
||
|
return f.name
|
||
|
}
|
||
|
|
||
|
// Root of the remote (as passed into NewFs)
|
||
|
func (f *Fs) Root() string {
|
||
|
return f.root
|
||
|
}
|
||
|
|
||
|
// String converts this Fs to a string
|
||
|
func (f *Fs) String() string {
|
||
|
return fmt.Sprintf("files root '%s'", f.root)
|
||
|
}
|
||
|
|
||
|
// Features returns the optional features of this Fs
|
||
|
func (f *Fs) Features() *fs.Features {
|
||
|
return f.features
|
||
|
}
|
||
|
|
||
|
// Encode remote and turn it into an absolute path in the share
|
||
|
func (f *Fs) absPath(remote string) string {
|
||
|
return f.opt.Enc.FromStandardPath(path.Join(f.root, remote))
|
||
|
}
|
||
|
|
||
|
// retryErrorCodes is a slice of error codes that we will retry
|
||
|
var retryErrorCodes = []int{
|
||
|
429, // Too Many Requests.
|
||
|
500, // Internal Server Error
|
||
|
502, // Bad Gateway
|
||
|
503, // Service Unavailable
|
||
|
504, // Gateway Timeout
|
||
|
509, // Bandwidth Limit Exceeded
|
||
|
}
|
||
|
|
||
|
// shouldRetry returns a boolean as to whether this err deserves to be
|
||
|
// retried. It returns the err as a convenience
|
||
|
func shouldRetry(ctx context.Context, err error) (bool, error) {
|
||
|
if fserrors.ContextError(ctx, &err) {
|
||
|
return false, err
|
||
|
}
|
||
|
|
||
|
if apiErr, ok := err.(files_sdk.ResponseError); ok {
|
||
|
for _, e := range retryErrorCodes {
|
||
|
if apiErr.HttpCode == e {
|
||
|
fs.Debugf(nil, "Retrying API error %v", err)
|
||
|
return true, err
|
||
|
}
|
||
|
}
|
||
|
}
|
||
|
|
||
|
return fserrors.ShouldRetry(err), err
|
||
|
}
|
||
|
|
||
|
// readMetaDataForPath reads the metadata from the path
|
||
|
func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *files_sdk.File, err error) {
|
||
|
params := files_sdk.FileFindParams{
|
||
|
Path: f.absPath(path),
|
||
|
}
|
||
|
|
||
|
var file files_sdk.File
|
||
|
err = f.pacer.Call(func() (bool, error) {
|
||
|
file, err = f.fileClient.Find(params, files_sdk.WithContext(ctx))
|
||
|
return shouldRetry(ctx, err)
|
||
|
})
|
||
|
if err != nil {
|
||
|
return nil, err
|
||
|
}
|
||
|
|
||
|
return &file, nil
|
||
|
}
|
||
|
|
||
|
// NewFs constructs an Fs from the path, container:path
|
||
|
func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||
|
// Parse config into Options struct
|
||
|
opt := new(Options)
|
||
|
err := configstruct.Set(m, opt)
|
||
|
if err != nil {
|
||
|
return nil, err
|
||
|
}
|
||
|
|
||
|
root = strings.Trim(root, "/")
|
||
|
|
||
|
config, err := newClientConfig(ctx, opt)
|
||
|
if err != nil {
|
||
|
return nil, err
|
||
|
}
|
||
|
|
||
|
f := &Fs{
|
||
|
name: name,
|
||
|
root: root,
|
||
|
opt: *opt,
|
||
|
fileClient: &file.Client{Config: config},
|
||
|
folderClient: &folder.Client{Config: config},
|
||
|
migrationClient: &file_migration.Client{Config: config},
|
||
|
bundleClient: &bundle.Client{Config: config},
|
||
|
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
|
||
|
}
|
||
|
f.features = (&fs.Features{
|
||
|
CaseInsensitive: true,
|
||
|
CanHaveEmptyDirectories: true,
|
||
|
ReadMimeType: true,
|
||
|
DirModTimeUpdatesOnWrite: true,
|
||
|
}).Fill(ctx, f)
|
||
|
|
||
|
if f.root != "" {
|
||
|
info, err := f.readMetaDataForPath(ctx, "")
|
||
|
if err == nil && !info.IsDir() {
|
||
|
f.root = path.Dir(f.root)
|
||
|
if f.root == "." {
|
||
|
f.root = ""
|
||
|
}
|
||
|
return f, fs.ErrorIsFile
|
||
|
}
|
||
|
}
|
||
|
|
||
|
return f, err
|
||
|
}
|
||
|
|
||
|
func newClientConfig(ctx context.Context, opt *Options) (config files_sdk.Config, err error) {
|
||
|
if opt.Site != "" {
|
||
|
config.Subdomain = opt.Site
|
||
|
|
||
|
_, err = url.Parse(config.Endpoint())
|
||
|
if err != nil {
|
||
|
config.Subdomain = ""
|
||
|
config.EndpointOverride = opt.Site
|
||
|
|
||
|
_, err = url.Parse(config.Endpoint())
|
||
|
if err != nil {
|
||
|
err = fmt.Errorf("invalid domain or subdomain: %v", opt.Site)
|
||
|
return
|
||
|
}
|
||
|
}
|
||
|
}
|
||
|
|
||
|
config = config.Init().SetCustomClient(fshttp.NewClient(ctx))
|
||
|
|
||
|
if opt.APIKey != "" {
|
||
|
config.APIKey = opt.APIKey
|
||
|
return
|
||
|
}
|
||
|
|
||
|
if opt.Username == "" {
|
||
|
err = errors.New("username not found")
|
||
|
return
|
||
|
}
|
||
|
if opt.Password == "" {
|
||
|
err = errors.New("password not found")
|
||
|
return
|
||
|
}
|
||
|
opt.Password, err = obscure.Reveal(opt.Password)
|
||
|
if err != nil {
|
||
|
return
|
||
|
}
|
||
|
|
||
|
sessionClient := session.Client{Config: config}
|
||
|
params := files_sdk.SessionCreateParams{
|
||
|
Username: opt.Username,
|
||
|
Password: opt.Password,
|
||
|
}
|
||
|
|
||
|
thisSession, err := sessionClient.Create(params, files_sdk.WithContext(ctx))
|
||
|
if err != nil {
|
||
|
err = fmt.Errorf("couldn't create session: %w", err)
|
||
|
return
|
||
|
}
|
||
|
|
||
|
config.SessionId = thisSession.Id
|
||
|
return
|
||
|
}
|
||
|
|
||
|
// Return an Object from a path
|
||
|
//
|
||
|
// If it can't be found it returns the error fs.ErrorObjectNotFound.
|
||
|
func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, file *files_sdk.File) (fs.Object, error) {
|
||
|
o := &Object{
|
||
|
fs: f,
|
||
|
remote: remote,
|
||
|
}
|
||
|
var err error
|
||
|
if file != nil {
|
||
|
err = o.setMetaData(file)
|
||
|
} else {
|
||
|
err = o.readMetaData(ctx) // reads info and meta, returning an error
|
||
|
}
|
||
|
if err != nil {
|
||
|
return nil, err
|
||
|
}
|
||
|
return o, nil
|
||
|
}
|
||
|
|
||
|
// NewObject finds the Object at remote. If it can't be found
|
||
|
// it returns the error fs.ErrorObjectNotFound.
|
||
|
func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
|
||
|
return f.newObjectWithInfo(ctx, remote, nil)
|
||
|
}
|
||
|
|
||
|
// List the objects and directories in dir into entries. The
|
||
|
// entries can be returned in any order but should be for a
|
||
|
// complete directory.
|
||
|
//
|
||
|
// dir should be "" to list the root, and should not have
|
||
|
// trailing slashes.
|
||
|
//
|
||
|
// This should return ErrDirNotFound if the directory isn't
|
||
|
// found.
|
||
|
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
|
||
|
var it *folder.Iter
|
||
|
params := files_sdk.FolderListForParams{
|
||
|
Path: f.absPath(dir),
|
||
|
}
|
||
|
|
||
|
err = f.pacer.Call(func() (bool, error) {
|
||
|
it, err = f.folderClient.ListFor(params, files_sdk.WithContext(ctx))
|
||
|
return shouldRetry(ctx, err)
|
||
|
})
|
||
|
if err != nil {
|
||
|
return nil, fmt.Errorf("couldn't list files: %w", err)
|
||
|
}
|
||
|
|
||
|
for it.Next() {
|
||
|
item := ptr(it.File())
|
||
|
remote := f.opt.Enc.ToStandardPath(item.DisplayName)
|
||
|
remote = path.Join(dir, remote)
|
||
|
if remote == dir {
|
||
|
continue
|
||
|
}
|
||
|
|
||
|
item, err = f.readMetaDataForPath(ctx, remote)
|
||
|
if err != nil {
|
||
|
if files_sdk.IsNotExist(err) {
|
||
|
continue
|
||
|
}
|
||
|
|
||
|
return nil, err
|
||
|
}
|
||
|
|
||
|
if item.IsDir() {
|
||
|
d := fs.NewDir(remote, item.ModTime())
|
||
|
entries = append(entries, d)
|
||
|
} else {
|
||
|
o, err := f.newObjectWithInfo(ctx, remote, item)
|
||
|
if err != nil {
|
||
|
return nil, err
|
||
|
}
|
||
|
entries = append(entries, o)
|
||
|
}
|
||
|
}
|
||
|
err = it.Err()
|
||
|
if files_sdk.IsNotExist(err) {
|
||
|
return nil, fs.ErrorDirNotFound
|
||
|
}
|
||
|
return
|
||
|
}
|
||
|
|
||
|
// Creates from the parameters passed in a half finished Object which
|
||
|
// must have setMetaData called on it
|
||
|
//
|
||
|
// Returns the object and error.
|
||
|
//
|
||
|
// Used to create new objects
|
||
|
func (f *Fs) createObject(ctx context.Context, remote string) (o *Object, err error) {
|
||
|
// Create the directory for the object if it doesn't exist
|
||
|
err = f.mkParentDir(ctx, remote)
|
||
|
if err != nil {
|
||
|
return
|
||
|
}
|
||
|
// Temporary Object under construction
|
||
|
o = &Object{
|
||
|
fs: f,
|
||
|
remote: remote,
|
||
|
}
|
||
|
return o, nil
|
||
|
}
|
||
|
|
||
|
// Put the object
|
||
|
//
|
||
|
// Copy the reader in to the new object which is returned.
|
||
|
//
|
||
|
// The new object may have been created if an error is returned
|
||
|
func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
||
|
// Temporary Object under construction
|
||
|
fs := &Object{
|
||
|
fs: f,
|
||
|
remote: src.Remote(),
|
||
|
}
|
||
|
return fs, fs.Update(ctx, in, src, options...)
|
||
|
}
|
||
|
|
||
|
// PutStream uploads to the remote path with the modTime given of indeterminate size
|
||
|
func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
||
|
return f.Put(ctx, in, src, options...)
|
||
|
}
|
||
|
|
||
|
func (f *Fs) mkdir(ctx context.Context, path string) error {
|
||
|
if path == "" || path == "." {
|
||
|
return nil
|
||
|
}
|
||
|
|
||
|
params := files_sdk.FolderCreateParams{
|
||
|
Path: path,
|
||
|
MkdirParents: ptr(true),
|
||
|
}
|
||
|
|
||
|
err := f.pacer.Call(func() (bool, error) {
|
||
|
_, err := f.folderClient.Create(params, files_sdk.WithContext(ctx))
|
||
|
return shouldRetry(ctx, err)
|
||
|
})
|
||
|
if files_sdk.IsExist(err) {
|
||
|
return nil
|
||
|
}
|
||
|
return err
|
||
|
}
|
||
|
|
||
|
// Make the parent directory of remote
|
||
|
func (f *Fs) mkParentDir(ctx context.Context, remote string) error {
|
||
|
return f.mkdir(ctx, path.Dir(f.absPath(remote)))
|
||
|
}
|
||
|
|
||
|
// Mkdir creates the container if it doesn't exist
|
||
|
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
||
|
return f.mkdir(ctx, f.absPath(dir))
|
||
|
}
|
||
|
|
||
|
// DirSetModTime sets the directory modtime for dir
|
||
|
func (f *Fs) DirSetModTime(ctx context.Context, dir string, modTime time.Time) error {
|
||
|
o := Object{
|
||
|
fs: f,
|
||
|
remote: dir,
|
||
|
}
|
||
|
return o.SetModTime(ctx, modTime)
|
||
|
}
|
||
|
|
||
|
// purgeCheck removes the root directory, if check is set then it
|
||
|
// refuses to do so if it has anything in
|
||
|
func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
|
||
|
path := f.absPath(dir)
|
||
|
if path == "" || path == "." {
|
||
|
return errors.New("can't purge root directory")
|
||
|
}
|
||
|
|
||
|
params := files_sdk.FileDeleteParams{
|
||
|
Path: path,
|
||
|
Recursive: ptr(!check),
|
||
|
}
|
||
|
|
||
|
err := f.pacer.Call(func() (bool, error) {
|
||
|
err := f.fileClient.Delete(params, files_sdk.WithContext(ctx))
|
||
|
// Allow for eventual consistency deletion of child objects.
|
||
|
if isFolderNotEmpty(err) {
|
||
|
return true, err
|
||
|
}
|
||
|
return shouldRetry(ctx, err)
|
||
|
})
|
||
|
if err != nil {
|
||
|
if files_sdk.IsNotExist(err) {
|
||
|
return fs.ErrorDirNotFound
|
||
|
} else if isFolderNotEmpty(err) {
|
||
|
return fs.ErrorDirectoryNotEmpty
|
||
|
}
|
||
|
|
||
|
return fmt.Errorf("rmdir failed: %w", err)
|
||
|
}
|
||
|
return nil
|
||
|
}
|
||
|
|
||
|
// Rmdir deletes the root folder
|
||
|
//
|
||
|
// Returns an error if it isn't empty
|
||
|
func (f *Fs) Rmdir(ctx context.Context, dir string) error {
|
||
|
return f.purgeCheck(ctx, dir, true)
|
||
|
}
|
||
|
|
||
|
// Precision return the precision of this Fs
|
||
|
func (f *Fs) Precision() time.Duration {
|
||
|
return time.Second
|
||
|
}
|
||
|
|
||
|
// Copy src to this remote using server-side copy operations.
|
||
|
//
|
||
|
// This is stored with the remote path given.
|
||
|
//
|
||
|
// It returns the destination Object and a possible error.
|
||
|
//
|
||
|
// Will only be called if src.Fs().Name() == f.Name()
|
||
|
//
|
||
|
// If it isn't possible then return fs.ErrorCantCopy
|
||
|
func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dstObj fs.Object, err error) {
|
||
|
srcObj, ok := src.(*Object)
|
||
|
if !ok {
|
||
|
fs.Debugf(src, "Can't copy - not same remote type")
|
||
|
return nil, fs.ErrorCantCopy
|
||
|
}
|
||
|
err = srcObj.readMetaData(ctx)
|
||
|
if err != nil {
|
||
|
return
|
||
|
}
|
||
|
|
||
|
srcPath := srcObj.fs.absPath(srcObj.remote)
|
||
|
dstPath := f.absPath(remote)
|
||
|
if strings.EqualFold(srcPath, dstPath) {
|
||
|
return nil, fmt.Errorf("can't copy %q -> %q as are same name when lowercase", srcPath, dstPath)
|
||
|
}
|
||
|
|
||
|
// Create temporary object
|
||
|
dstObj, err = f.createObject(ctx, remote)
|
||
|
if err != nil {
|
||
|
return
|
||
|
}
|
||
|
|
||
|
// Copy the object
|
||
|
params := files_sdk.FileCopyParams{
|
||
|
Path: srcPath,
|
||
|
Destination: dstPath,
|
||
|
Overwrite: ptr(true),
|
||
|
}
|
||
|
|
||
|
var action files_sdk.FileAction
|
||
|
err = f.pacer.Call(func() (bool, error) {
|
||
|
action, err = f.fileClient.Copy(params, files_sdk.WithContext(ctx))
|
||
|
return shouldRetry(ctx, err)
|
||
|
})
|
||
|
if err != nil {
|
||
|
return
|
||
|
}
|
||
|
|
||
|
err = f.waitForAction(ctx, action, "copy")
|
||
|
if err != nil {
|
||
|
return
|
||
|
}
|
||
|
|
||
|
err = dstObj.SetModTime(ctx, srcObj.modTime)
|
||
|
return
|
||
|
}
|
||
|
|
||
|
// Purge deletes all the files and the container
|
||
|
//
|
||
|
// Optional interface: Only implement this if you have a way of
|
||
|
// deleting all the files quicker than just running Remove() on the
|
||
|
// result of List()
|
||
|
func (f *Fs) Purge(ctx context.Context, dir string) error {
|
||
|
return f.purgeCheck(ctx, dir, false)
|
||
|
}
|
||
|
|
||
|
// move a file or folder
|
||
|
func (f *Fs) move(ctx context.Context, src *Fs, srcRemote string, dstRemote string) (info *files_sdk.File, err error) {
|
||
|
// Move the object
|
||
|
params := files_sdk.FileMoveParams{
|
||
|
Path: src.absPath(srcRemote),
|
||
|
Destination: f.absPath(dstRemote),
|
||
|
}
|
||
|
|
||
|
var action files_sdk.FileAction
|
||
|
err = f.pacer.Call(func() (bool, error) {
|
||
|
action, err = f.fileClient.Move(params, files_sdk.WithContext(ctx))
|
||
|
return shouldRetry(ctx, err)
|
||
|
})
|
||
|
if err != nil {
|
||
|
return nil, err
|
||
|
}
|
||
|
|
||
|
err = f.waitForAction(ctx, action, "move")
|
||
|
if err != nil {
|
||
|
return nil, err
|
||
|
}
|
||
|
|
||
|
info, err = f.readMetaDataForPath(ctx, dstRemote)
|
||
|
return
|
||
|
}
|
||
|
|
||
|
func (f *Fs) waitForAction(ctx context.Context, action files_sdk.FileAction, operation string) (err error) {
|
||
|
var migration files_sdk.FileMigration
|
||
|
err = f.pacer.Call(func() (bool, error) {
|
||
|
migration, err = f.migrationClient.Wait(action, func(migration files_sdk.FileMigration) {
|
||
|
// noop
|
||
|
}, files_sdk.WithContext(ctx))
|
||
|
return shouldRetry(ctx, err)
|
||
|
})
|
||
|
if err == nil && migration.Status != "completed" {
|
||
|
return fmt.Errorf("%v did not complete successfully: %v", operation, migration.Status)
|
||
|
}
|
||
|
return
|
||
|
}
|
||
|
|
||
|
// Move src to this remote using server-side move operations.
|
||
|
//
|
||
|
// This is stored with the remote path given.
|
||
|
//
|
||
|
// It returns the destination Object and a possible error.
|
||
|
//
|
||
|
// Will only be called if src.Fs().Name() == f.Name()
|
||
|
//
|
||
|
// If it isn't possible then return fs.ErrorCantMove
|
||
|
func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
|
||
|
srcObj, ok := src.(*Object)
|
||
|
if !ok {
|
||
|
fs.Debugf(src, "Can't move - not same remote type")
|
||
|
return nil, fs.ErrorCantMove
|
||
|
}
|
||
|
|
||
|
// Create temporary object
|
||
|
dstObj, err := f.createObject(ctx, remote)
|
||
|
if err != nil {
|
||
|
return nil, err
|
||
|
}
|
||
|
|
||
|
// Do the move
|
||
|
info, err := f.move(ctx, srcObj.fs, srcObj.remote, dstObj.remote)
|
||
|
if err != nil {
|
||
|
return nil, err
|
||
|
}
|
||
|
|
||
|
err = dstObj.setMetaData(info)
|
||
|
if err != nil {
|
||
|
return nil, err
|
||
|
}
|
||
|
return dstObj, nil
|
||
|
}
|
||
|
|
||
|
// DirMove moves src, srcRemote to this remote at dstRemote
|
||
|
// using server-side move operations.
|
||
|
//
|
||
|
// Will only be called if src.Fs().Name() == f.Name()
|
||
|
//
|
||
|
// If it isn't possible then return fs.ErrorCantDirMove
|
||
|
//
|
||
|
// If destination exists then return fs.ErrorDirExists
|
||
|
func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) (err error) {
|
||
|
srcFs, ok := src.(*Fs)
|
||
|
if !ok {
|
||
|
fs.Debugf(srcFs, "Can't move directory - not same remote type")
|
||
|
return fs.ErrorCantDirMove
|
||
|
}
|
||
|
|
||
|
// Check if destination exists
|
||
|
_, err = f.readMetaDataForPath(ctx, dstRemote)
|
||
|
if err == nil {
|
||
|
return fs.ErrorDirExists
|
||
|
}
|
||
|
|
||
|
// Create temporary object
|
||
|
dstObj, err := f.createObject(ctx, dstRemote)
|
||
|
if err != nil {
|
||
|
return
|
||
|
}
|
||
|
|
||
|
// Do the move
|
||
|
_, err = f.move(ctx, srcFs, srcRemote, dstObj.remote)
|
||
|
return
|
||
|
}
|
||
|
|
||
|
// PublicLink adds a "readable by anyone with link" permission on the given file or folder.
|
||
|
func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (url string, err error) {
|
||
|
params := files_sdk.BundleCreateParams{
|
||
|
Paths: []string{f.absPath(remote)},
|
||
|
}
|
||
|
if expire < fs.DurationOff {
|
||
|
params.ExpiresAt = ptr(time.Now().Add(time.Duration(expire)))
|
||
|
}
|
||
|
|
||
|
var bundle files_sdk.Bundle
|
||
|
err = f.pacer.Call(func() (bool, error) {
|
||
|
bundle, err = f.bundleClient.Create(params, files_sdk.WithContext(ctx))
|
||
|
return shouldRetry(ctx, err)
|
||
|
})
|
||
|
|
||
|
url = bundle.Url
|
||
|
return
|
||
|
}
|
||
|
|
||
|
// Hashes returns the supported hash sets.
|
||
|
func (f *Fs) Hashes() hash.Set {
|
||
|
return hash.NewHashSet(hash.CRC32, hash.MD5)
|
||
|
}
|
||
|
|
||
|
// ------------------------------------------------------------
|
||
|
|
||
|
// Fs returns the parent Fs
|
||
|
func (o *Object) Fs() fs.Info {
|
||
|
return o.fs
|
||
|
}
|
||
|
|
||
|
// Return a string version
|
||
|
func (o *Object) String() string {
|
||
|
if o == nil {
|
||
|
return "<nil>"
|
||
|
}
|
||
|
return o.remote
|
||
|
}
|
||
|
|
||
|
// Remote returns the remote path
|
||
|
func (o *Object) Remote() string {
|
||
|
return o.remote
|
||
|
}
|
||
|
|
||
|
// Hash returns the MD5 of an object returning a lowercase hex string
|
||
|
func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
|
||
|
switch t {
|
||
|
case hash.CRC32:
|
||
|
if o.crc32 == "" {
|
||
|
return "", nil
|
||
|
}
|
||
|
return fmt.Sprintf("%08s", o.crc32), nil
|
||
|
case hash.MD5:
|
||
|
return o.md5, nil
|
||
|
}
|
||
|
return "", hash.ErrUnsupported
|
||
|
}
|
||
|
|
||
|
// Size returns the size of an object in bytes
|
||
|
func (o *Object) Size() int64 {
|
||
|
return o.size
|
||
|
}
|
||
|
|
||
|
// setMetaData sets the metadata from info
|
||
|
func (o *Object) setMetaData(file *files_sdk.File) error {
|
||
|
o.modTime = file.ModTime()
|
||
|
|
||
|
if !file.IsDir() {
|
||
|
o.size = file.Size
|
||
|
o.crc32 = file.Crc32
|
||
|
o.md5 = file.Md5
|
||
|
o.mimeType = file.MimeType
|
||
|
}
|
||
|
|
||
|
return nil
|
||
|
}
|
||
|
|
||
|
// readMetaData gets the metadata if it hasn't already been fetched
|
||
|
//
|
||
|
// it also sets the info
|
||
|
func (o *Object) readMetaData(ctx context.Context) (err error) {
|
||
|
file, err := o.fs.readMetaDataForPath(ctx, o.remote)
|
||
|
if err != nil {
|
||
|
if files_sdk.IsNotExist(err) {
|
||
|
return fs.ErrorObjectNotFound
|
||
|
}
|
||
|
return err
|
||
|
}
|
||
|
if file.IsDir() {
|
||
|
return fs.ErrorIsDir
|
||
|
}
|
||
|
return o.setMetaData(file)
|
||
|
}
|
||
|
|
||
|
// ModTime returns the modification time of the object
|
||
|
//
|
||
|
// It attempts to read the objects mtime and if that isn't present the
|
||
|
// LastModified returned in the http headers
|
||
|
func (o *Object) ModTime(ctx context.Context) time.Time {
|
||
|
return o.modTime
|
||
|
}
|
||
|
|
||
|
// SetModTime sets the modification time of the local fs object
|
||
|
func (o *Object) SetModTime(ctx context.Context, modTime time.Time) (err error) {
|
||
|
params := files_sdk.FileUpdateParams{
|
||
|
Path: o.fs.absPath(o.remote),
|
||
|
ProvidedMtime: &modTime,
|
||
|
}
|
||
|
|
||
|
var file files_sdk.File
|
||
|
err = o.fs.pacer.Call(func() (bool, error) {
|
||
|
file, err = o.fs.fileClient.Update(params, files_sdk.WithContext(ctx))
|
||
|
return shouldRetry(ctx, err)
|
||
|
})
|
||
|
if err != nil {
|
||
|
return err
|
||
|
}
|
||
|
return o.setMetaData(&file)
|
||
|
}
|
||
|
|
||
|
// Storable returns a boolean showing whether this object storable
|
||
|
func (o *Object) Storable() bool {
|
||
|
return true
|
||
|
}
|
||
|
|
||
|
// Open an object for read
|
||
|
func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
|
||
|
// Offset and Count for range download
|
||
|
var offset, count int64
|
||
|
fs.FixRangeOption(options, o.size)
|
||
|
for _, option := range options {
|
||
|
switch x := option.(type) {
|
||
|
case *fs.RangeOption:
|
||
|
offset, count = x.Decode(o.size)
|
||
|
if count < 0 {
|
||
|
count = o.size - offset
|
||
|
}
|
||
|
case *fs.SeekOption:
|
||
|
offset = x.Offset
|
||
|
count = o.size - offset
|
||
|
default:
|
||
|
if option.Mandatory() {
|
||
|
fs.Logf(o, "Unsupported mandatory option: %v", option)
|
||
|
}
|
||
|
}
|
||
|
}
|
||
|
|
||
|
params := files_sdk.FileDownloadParams{
|
||
|
Path: o.fs.absPath(o.remote),
|
||
|
}
|
||
|
|
||
|
headers := &http.Header{}
|
||
|
headers.Set("Range", fmt.Sprintf("bytes=%v-%v", offset, offset+count-1))
|
||
|
err = o.fs.pacer.Call(func() (bool, error) {
|
||
|
_, err = o.fs.fileClient.Download(
|
||
|
params,
|
||
|
files_sdk.WithContext(ctx),
|
||
|
files_sdk.RequestHeadersOption(headers),
|
||
|
files_sdk.ResponseBodyOption(func(closer io.ReadCloser) error {
|
||
|
in = closer
|
||
|
return err
|
||
|
}),
|
||
|
)
|
||
|
return shouldRetry(ctx, err)
|
||
|
})
|
||
|
return
|
||
|
}
|
||
|
|
||
|
// Returns a pointer to t - useful for returning pointers to constants
|
||
|
func ptr[T any](t T) *T {
|
||
|
return &t
|
||
|
}
|
||
|
|
||
|
func isFolderNotEmpty(err error) bool {
|
||
|
var re files_sdk.ResponseError
|
||
|
ok := errors.As(err, &re)
|
||
|
return ok && re.Type == folderNotEmpty
|
||
|
}
|
||
|
|
||
|
// Update the object with the contents of the io.Reader, modTime and size
|
||
|
//
|
||
|
// If existing is set then it updates the object rather than creating a new one.
|
||
|
//
|
||
|
// The new object may have been created if an error is returned.
|
||
|
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
|
||
|
uploadOpts := []file.UploadOption{
|
||
|
file.UploadWithContext(ctx),
|
||
|
file.UploadWithReader(in),
|
||
|
file.UploadWithDestinationPath(o.fs.absPath(o.remote)),
|
||
|
file.UploadWithProvidedMtime(src.ModTime(ctx)),
|
||
|
}
|
||
|
|
||
|
err := o.fs.pacer.Call(func() (bool, error) {
|
||
|
err := o.fs.fileClient.Upload(uploadOpts...)
|
||
|
return shouldRetry(ctx, err)
|
||
|
})
|
||
|
if err != nil {
|
||
|
return err
|
||
|
}
|
||
|
|
||
|
return o.readMetaData(ctx)
|
||
|
}
|
||
|
|
||
|
// Remove an object
|
||
|
func (o *Object) Remove(ctx context.Context) error {
|
||
|
params := files_sdk.FileDeleteParams{
|
||
|
Path: o.fs.absPath(o.remote),
|
||
|
}
|
||
|
|
||
|
return o.fs.pacer.Call(func() (bool, error) {
|
||
|
err := o.fs.fileClient.Delete(params, files_sdk.WithContext(ctx))
|
||
|
return shouldRetry(ctx, err)
|
||
|
})
|
||
|
}
|
||
|
|
||
|
// MimeType of an Object if known, "" otherwise
|
||
|
func (o *Object) MimeType(ctx context.Context) string {
|
||
|
return o.mimeType
|
||
|
}
|
||
|
|
||
|
// Check the interfaces are satisfied
|
||
|
var (
|
||
|
_ fs.Fs = (*Fs)(nil)
|
||
|
_ fs.Purger = (*Fs)(nil)
|
||
|
_ fs.PutStreamer = (*Fs)(nil)
|
||
|
_ fs.Copier = (*Fs)(nil)
|
||
|
_ fs.Mover = (*Fs)(nil)
|
||
|
_ fs.DirMover = (*Fs)(nil)
|
||
|
_ fs.PublicLinker = (*Fs)(nil)
|
||
|
_ fs.Object = (*Object)(nil)
|
||
|
_ fs.MimeTyper = (*Object)(nil)
|
||
|
)
|