KurlesHS/rclone
1
0
Fork 0
forked from TrueCloudLab/rclone
Code Pull requests Activity

rest: add context propagation to rest library #3257

Browse source 
This fixes up the calling and propagates the contexts for the backends
which use lib/rest.
This commit is contained in:
Nick Craig-Wood 2019-09-04 20:00:37 +01:00
parent ba1daea072
commit 58a531a203
19 changed files with 448 additions and 439 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

108
backend/b2/b2.go
View file

@ -273,11 +273,11 @@ func (f *Fs) shouldRetryNoReauth(resp *http.Response, err error) (bool, error) {
// shouldRetry returns a boolean as to whether this resp and err // shouldRetry returns a boolean as to whether this resp and err
// deserve to be retried. It returns the err as a convenience // deserve to be retried. It returns the err as a convenience
func (f *Fs) shouldRetry(resp *http.Response, err error) (bool, error) { func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
if resp != nil && resp.StatusCode == 401 { if resp != nil && resp.StatusCode == 401 {
fs.Debugf(f, "Unauthorized: %v", err) fs.Debugf(f, "Unauthorized: %v", err)
// Reauth // Reauth
authErr := f.authorizeAccount() authErr := f.authorizeAccount(ctx)
if authErr != nil { if authErr != nil {
err = authErr err = authErr
} }
@ -393,7 +393,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
fs.Debugf(f, "Setting test header \"%s: %s\"", testModeHeader, testMode) fs.Debugf(f, "Setting test header \"%s: %s\"", testModeHeader, testMode)
} }
f.fillBufferTokens() f.fillBufferTokens()
err = f.authorizeAccount() err = f.authorizeAccount(ctx)
if err != nil { if err != nil {
return nil, errors.Wrap(err, "failed to authorize account") return nil, errors.Wrap(err, "failed to authorize account")
} }
@ -431,7 +431,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
// authorizeAccount gets the API endpoint and auth token. Can be used // authorizeAccount gets the API endpoint and auth token. Can be used
// for reauthentication too. // for reauthentication too.
func (f *Fs) authorizeAccount() error { func (f *Fs) authorizeAccount(ctx context.Context) error {
f.authMu.Lock() f.authMu.Lock()
defer f.authMu.Unlock() defer f.authMu.Unlock()
opts := rest.Opts{ opts := rest.Opts{
@ -443,7 +443,7 @@ func (f *Fs) authorizeAccount() error {
ExtraHeaders: map[string]string{"Authorization": ""}, // unset the Authorization for this request ExtraHeaders: map[string]string{"Authorization": ""}, // unset the Authorization for this request
} }
err := f.pacer.Call(func() (bool, error) { err := f.pacer.Call(func() (bool, error) {
resp, err := f.srv.CallJSON(&opts, nil, &f.info) resp, err := f.srv.CallJSON(ctx, &opts, nil, &f.info)
return f.shouldRetryNoReauth(resp, err) return f.shouldRetryNoReauth(resp, err)
}) })
if err != nil { if err != nil {
@ -466,10 +466,10 @@ func (f *Fs) hasPermission(permission string) bool {
// getUploadURL returns the upload info with the UploadURL and the AuthorizationToken // getUploadURL returns the upload info with the UploadURL and the AuthorizationToken
// //
// This should be returned with returnUploadURL when finished // This should be returned with returnUploadURL when finished
func (f *Fs) getUploadURL(bucket string) (upload *api.GetUploadURLResponse, err error) { func (f *Fs) getUploadURL(ctx context.Context, bucket string) (upload *api.GetUploadURLResponse, err error) {
f.uploadMu.Lock() f.uploadMu.Lock()
defer f.uploadMu.Unlock() defer f.uploadMu.Unlock()
bucketID, err := f.getBucketID(bucket) bucketID, err := f.getBucketID(ctx, bucket)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -489,8 +489,8 @@ func (f *Fs) getUploadURL(bucket string) (upload *api.GetUploadURLResponse, err
BucketID: bucketID, BucketID: bucketID,
} }
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err := f.srv.CallJSON(&opts, &request, &upload) resp, err := f.srv.CallJSON(ctx, &opts, &request, &upload)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, errors.Wrap(err, "failed to get upload URL") return nil, errors.Wrap(err, "failed to get upload URL")
@ -609,7 +609,7 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
if !recurse { if !recurse {
delimiter = "/" delimiter = "/"
} }
bucketID, err := f.getBucketID(bucket) bucketID, err := f.getBucketID(ctx, bucket)
if err != nil { if err != nil {
return err return err
} }
@ -636,8 +636,8 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
for { for {
var response api.ListFileNamesResponse var response api.ListFileNamesResponse
err := f.pacer.Call(func() (bool, error) { err := f.pacer.Call(func() (bool, error) {
resp, err := f.srv.CallJSON(&opts, &request, &response) resp, err := f.srv.CallJSON(ctx, &opts, &request, &response)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return err return err
@ -727,7 +727,7 @@ func (f *Fs) listDir(ctx context.Context, bucket, directory, prefix string, addB
// listBuckets returns all the buckets to out // listBuckets returns all the buckets to out
func (f *Fs) listBuckets(ctx context.Context) (entries fs.DirEntries, err error) { func (f *Fs) listBuckets(ctx context.Context) (entries fs.DirEntries, err error) {
err = f.listBucketsToFn(func(bucket *api.Bucket) error { err = f.listBucketsToFn(ctx, func(bucket *api.Bucket) error {
d := fs.NewDir(bucket.Name, time.Time{}) d := fs.NewDir(bucket.Name, time.Time{})
entries = append(entries, d) entries = append(entries, d)
return nil return nil
@ -820,7 +820,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
type listBucketFn func(*api.Bucket) error type listBucketFn func(*api.Bucket) error
// listBucketsToFn lists the buckets to the function supplied // listBucketsToFn lists the buckets to the function supplied
func (f *Fs) listBucketsToFn(fn listBucketFn) error { func (f *Fs) listBucketsToFn(ctx context.Context, fn listBucketFn) error {
var account = api.ListBucketsRequest{ var account = api.ListBucketsRequest{
AccountID: f.info.AccountID, AccountID: f.info.AccountID,
BucketID: f.info.Allowed.BucketID, BucketID: f.info.Allowed.BucketID,
@ -832,8 +832,8 @@ func (f *Fs) listBucketsToFn(fn listBucketFn) error {
Path: "/b2_list_buckets", Path: "/b2_list_buckets",
} }
err := f.pacer.Call(func() (bool, error) { err := f.pacer.Call(func() (bool, error) {
resp, err := f.srv.CallJSON(&opts, &account, &response) resp, err := f.srv.CallJSON(ctx, &opts, &account, &response)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return err return err
@ -862,14 +862,14 @@ func (f *Fs) listBucketsToFn(fn listBucketFn) error {
// getbucketType finds the bucketType for the current bucket name // getbucketType finds the bucketType for the current bucket name
// can be one of allPublic. allPrivate, or snapshot // can be one of allPublic. allPrivate, or snapshot
func (f *Fs) getbucketType(bucket string) (bucketType string, err error) { func (f *Fs) getbucketType(ctx context.Context, bucket string) (bucketType string, err error) {
f.bucketTypeMutex.Lock() f.bucketTypeMutex.Lock()
bucketType = f._bucketType[bucket] bucketType = f._bucketType[bucket]
f.bucketTypeMutex.Unlock() f.bucketTypeMutex.Unlock()
if bucketType != "" { if bucketType != "" {
return bucketType, nil return bucketType, nil
} }
err = f.listBucketsToFn(func(bucket *api.Bucket) error { err = f.listBucketsToFn(ctx, func(bucket *api.Bucket) error {
// listBucketsToFn reads bucket Types // listBucketsToFn reads bucket Types
return nil return nil
}) })
@ -897,14 +897,14 @@ func (f *Fs) clearBucketType(bucket string) {
} }
// getBucketID finds the ID for the current bucket name // getBucketID finds the ID for the current bucket name
func (f *Fs) getBucketID(bucket string) (bucketID string, err error) { func (f *Fs) getBucketID(ctx context.Context, bucket string) (bucketID string, err error) {
f.bucketIDMutex.Lock() f.bucketIDMutex.Lock()
bucketID = f._bucketID[bucket] bucketID = f._bucketID[bucket]
f.bucketIDMutex.Unlock() f.bucketIDMutex.Unlock()
if bucketID != "" { if bucketID != "" {
return bucketID, nil return bucketID, nil
} }
err = f.listBucketsToFn(func(bucket *api.Bucket) error { err = f.listBucketsToFn(ctx, func(bucket *api.Bucket) error {
// listBucketsToFn sets IDs // listBucketsToFn sets IDs
return nil return nil
}) })
@ -970,15 +970,15 @@ func (f *Fs) makeBucket(ctx context.Context, bucket string) error {
} }
var response api.Bucket var response api.Bucket
err := f.pacer.Call(func() (bool, error) { err := f.pacer.Call(func() (bool, error) {
resp, err := f.srv.CallJSON(&opts, &request, &response) resp, err := f.srv.CallJSON(ctx, &opts, &request, &response)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
if apiErr, ok := err.(*api.Error); ok { if apiErr, ok := err.(*api.Error); ok {
if apiErr.Code == "duplicate_bucket_name" { if apiErr.Code == "duplicate_bucket_name" {
// Check this is our bucket - buckets are globally unique and this // Check this is our bucket - buckets are globally unique and this
// might be someone elses. // might be someone elses.
_, getBucketErr := f.getBucketID(bucket) _, getBucketErr := f.getBucketID(ctx, bucket)
if getBucketErr == nil { if getBucketErr == nil {
// found so it is our bucket // found so it is our bucket
return nil return nil
@ -1009,7 +1009,7 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
Method: "POST", Method: "POST",
Path: "/b2_delete_bucket", Path: "/b2_delete_bucket",
} }
bucketID, err := f.getBucketID(bucket) bucketID, err := f.getBucketID(ctx, bucket)
if err != nil { if err != nil {
return err return err
} }
@ -1019,8 +1019,8 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
} }
var response api.Bucket var response api.Bucket
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err := f.srv.CallJSON(&opts, &request, &response) resp, err := f.srv.CallJSON(ctx, &opts, &request, &response)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return errors.Wrap(err, "failed to delete bucket") return errors.Wrap(err, "failed to delete bucket")
@ -1038,8 +1038,8 @@ func (f *Fs) Precision() time.Duration {
} }
// hide hides a file on the remote // hide hides a file on the remote
func (f *Fs) hide(bucket, bucketPath string) error { func (f *Fs) hide(ctx context.Context, bucket, bucketPath string) error {
bucketID, err := f.getBucketID(bucket) bucketID, err := f.getBucketID(ctx, bucket)
if err != nil { if err != nil {
return err return err
} }
@ -1053,8 +1053,8 @@ func (f *Fs) hide(bucket, bucketPath string) error {
} }
var response api.File var response api.File
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err := f.srv.CallJSON(&opts, &request, &response) resp, err := f.srv.CallJSON(ctx, &opts, &request, &response)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
if apiErr, ok := err.(*api.Error); ok { if apiErr, ok := err.(*api.Error); ok {
@ -1070,7 +1070,7 @@ func (f *Fs) hide(bucket, bucketPath string) error {
} }
// deleteByID deletes a file version given Name and ID // deleteByID deletes a file version given Name and ID
func (f *Fs) deleteByID(ID, Name string) error { func (f *Fs) deleteByID(ctx context.Context, ID, Name string) error {
opts := rest.Opts{ opts := rest.Opts{
Method: "POST", Method: "POST",
Path: "/b2_delete_file_version", Path: "/b2_delete_file_version",
@ -1081,8 +1081,8 @@ func (f *Fs) deleteByID(ID, Name string) error {
} }
var response api.File var response api.File
err := f.pacer.Call(func() (bool, error) { err := f.pacer.Call(func() (bool, error) {
resp, err := f.srv.CallJSON(&opts, &request, &response) resp, err := f.srv.CallJSON(ctx, &opts, &request, &response)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return errors.Wrapf(err, "failed to delete %q", Name) return errors.Wrapf(err, "failed to delete %q", Name)
@ -1132,7 +1132,7 @@ func (f *Fs) purge(ctx context.Context, bucket, directory string, oldOnly bool)
continue continue
} }
tr := accounting.Stats(ctx).NewCheckingTransfer(oi) tr := accounting.Stats(ctx).NewCheckingTransfer(oi)
err = f.deleteByID(object.ID, object.Name) err = f.deleteByID(ctx, object.ID, object.Name)
checkErr(err) checkErr(err)
tr.Done(err) tr.Done(err)
} }
@ -1205,7 +1205,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
fs.Debugf(src, "Can't copy - not same remote type") fs.Debugf(src, "Can't copy - not same remote type")
return nil, fs.ErrorCantCopy return nil, fs.ErrorCantCopy
} }
destBucketID, err := f.getBucketID(dstBucket) destBucketID, err := f.getBucketID(ctx, dstBucket)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -1221,8 +1221,8 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
} }
var response api.FileInfo var response api.FileInfo
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err := f.srv.CallJSON(&opts, &request, &response) resp, err := f.srv.CallJSON(ctx, &opts, &request, &response)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@ -1245,7 +1245,7 @@ func (f *Fs) Hashes() hash.Set {
// getDownloadAuthorization returns authorization token for downloading // getDownloadAuthorization returns authorization token for downloading
// without account. // without account.
func (f *Fs) getDownloadAuthorization(bucket, remote string) (authorization string, err error) { func (f *Fs) getDownloadAuthorization(ctx context.Context, bucket, remote string) (authorization string, err error) {
validDurationInSeconds := time.Duration(f.opt.DownloadAuthorizationDuration).Nanoseconds() / 1e9 validDurationInSeconds := time.Duration(f.opt.DownloadAuthorizationDuration).Nanoseconds() / 1e9
if validDurationInSeconds <= 0 || validDurationInSeconds > 604800 { if validDurationInSeconds <= 0 || validDurationInSeconds > 604800 {
return "", errors.New("--b2-download-auth-duration must be between 1 sec and 1 week") return "", errors.New("--b2-download-auth-duration must be between 1 sec and 1 week")
@ -1253,7 +1253,7 @@ func (f *Fs) getDownloadAuthorization(bucket, remote string) (authorization stri
if !f.hasPermission("shareFiles") { if !f.hasPermission("shareFiles") {
return "", errors.New("sharing a file link requires the shareFiles permission") return "", errors.New("sharing a file link requires the shareFiles permission")
} }
bucketID, err := f.getBucketID(bucket) bucketID, err := f.getBucketID(ctx, bucket)
if err != nil { if err != nil {
return "", err return "", err
} }
@ -1268,8 +1268,8 @@ func (f *Fs) getDownloadAuthorization(bucket, remote string) (authorization stri
} }
var response api.GetDownloadAuthorizationResponse var response api.GetDownloadAuthorizationResponse
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err := f.srv.CallJSON(&opts, &request, &response) resp, err := f.srv.CallJSON(ctx, &opts, &request, &response)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return "", errors.Wrap(err, "failed to get download authorization") return "", errors.Wrap(err, "failed to get download authorization")
@ -1301,12 +1301,12 @@ func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err er
} }
absPath := "/" + bucketPath absPath := "/" + bucketPath
link = RootURL + "/file/" + urlEncode(bucket) + absPath link = RootURL + "/file/" + urlEncode(bucket) + absPath
bucketType, err := f.getbucketType(bucket) bucketType, err := f.getbucketType(ctx, bucket)
if err != nil { if err != nil {
return "", err return "", err
} }
if bucketType == "allPrivate" || bucketType == "snapshot" { if bucketType == "allPrivate" || bucketType == "snapshot" {
AuthorizationToken, err := f.getDownloadAuthorization(bucket, remote) AuthorizationToken, err := f.getDownloadAuthorization(ctx, bucket, remote)
if err != nil { if err != nil {
return "", err return "", err
} }
@ -1505,8 +1505,8 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
} }
var response api.FileInfo var response api.FileInfo
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err := o.fs.srv.CallJSON(&opts, &request, &response) resp, err := o.fs.srv.CallJSON(ctx, &opts, &request, &response)
return o.fs.shouldRetry(resp, err) return o.fs.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return err return err
@ -1604,8 +1604,8 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
} }
var resp *http.Response var resp *http.Response
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.Call(&opts) resp, err = o.fs.srv.Call(ctx, &opts)
return o.fs.shouldRetry(resp, err) return o.fs.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, errors.Wrap(err, "failed to open for download") return nil, errors.Wrap(err, "failed to open for download")
@ -1701,7 +1701,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
o.fs.putUploadBlock(buf) o.fs.putUploadBlock(buf)
return err return err
} }
return up.Stream(buf) return up.Stream(ctx, buf)
} else if err == io.EOF || err == io.ErrUnexpectedEOF { } else if err == io.EOF || err == io.ErrUnexpectedEOF {
fs.Debugf(o, "File has %d bytes, which makes only one chunk. Using direct upload.", n) fs.Debugf(o, "File has %d bytes, which makes only one chunk. Using direct upload.", n)
defer o.fs.putUploadBlock(buf) defer o.fs.putUploadBlock(buf)
@ -1715,7 +1715,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
if err != nil { if err != nil {
return err return err
} }
return up.Upload() return up.Upload(ctx)
} }
modTime := src.ModTime(ctx) modTime := src.ModTime(ctx)
@ -1729,7 +1729,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
} }
// Get upload URL // Get upload URL
upload, err := o.fs.getUploadURL(bucket) upload, err := o.fs.getUploadURL(ctx, bucket)
if err != nil { if err != nil {
return err return err
} }
@ -1807,8 +1807,8 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
var response api.FileInfo var response api.FileInfo
// Don't retry, return a retry error instead // Don't retry, return a retry error instead
err = o.fs.pacer.CallNoRetry(func() (bool, error) { err = o.fs.pacer.CallNoRetry(func() (bool, error) {
resp, err := o.fs.srv.CallJSON(&opts, nil, &response) resp, err := o.fs.srv.CallJSON(ctx, &opts, nil, &response)
retry, err := o.fs.shouldRetry(resp, err) retry, err := o.fs.shouldRetry(ctx, resp, err)
// On retryable error clear UploadURL // On retryable error clear UploadURL
if retry { if retry {
fs.Debugf(o, "Clearing upload URL because of error: %v", err) fs.Debugf(o, "Clearing upload URL because of error: %v", err)
@ -1829,9 +1829,9 @@ func (o *Object) Remove(ctx context.Context) error {
return errNotWithVersions return errNotWithVersions
} }
if o.fs.opt.HardDelete { if o.fs.opt.HardDelete {
return o.fs.deleteByID(o.id, bucketPath) return o.fs.deleteByID(ctx, o.id, bucketPath)
} }
return o.fs.hide(bucket, bucketPath) return o.fs.hide(ctx, bucket, bucketPath)
} }
// MimeType of an Object if known, "" otherwise // MimeType of an Object if known, "" otherwise

56
backend/b2/upload.go
View file

@ -105,7 +105,7 @@ func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs
Path: "/b2_start_large_file", Path: "/b2_start_large_file",
} }
bucket, bucketPath := o.split() bucket, bucketPath := o.split()
bucketID, err := f.getBucketID(bucket) bucketID, err := f.getBucketID(ctx, bucket)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -125,8 +125,8 @@ func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs
} }
var response api.StartLargeFileResponse var response api.StartLargeFileResponse
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err := f.srv.CallJSON(&opts, &request, &response) resp, err := f.srv.CallJSON(ctx, &opts, &request, &response)
return f.shouldRetry(resp, err) return f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@ -150,7 +150,7 @@ func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs
// getUploadURL returns the upload info with the UploadURL and the AuthorizationToken // getUploadURL returns the upload info with the UploadURL and the AuthorizationToken
// //
// This should be returned with returnUploadURL when finished // This should be returned with returnUploadURL when finished
func (up *largeUpload) getUploadURL() (upload *api.GetUploadPartURLResponse, err error) { func (up *largeUpload) getUploadURL(ctx context.Context) (upload *api.GetUploadPartURLResponse, err error) {
up.uploadMu.Lock() up.uploadMu.Lock()
defer up.uploadMu.Unlock() defer up.uploadMu.Unlock()
if len(up.uploads) == 0 { if len(up.uploads) == 0 {
@ -162,8 +162,8 @@ func (up *largeUpload) getUploadURL() (upload *api.GetUploadPartURLResponse, err
ID: up.id, ID: up.id,
} }
err := up.f.pacer.Call(func() (bool, error) { err := up.f.pacer.Call(func() (bool, error) {
resp, err := up.f.srv.CallJSON(&opts, &request, &upload) resp, err := up.f.srv.CallJSON(ctx, &opts, &request, &upload)
return up.f.shouldRetry(resp, err) return up.f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return nil, errors.Wrap(err, "failed to get upload URL") return nil, errors.Wrap(err, "failed to get upload URL")
@ -192,12 +192,12 @@ func (up *largeUpload) clearUploadURL() {
} }
// Transfer a chunk // Transfer a chunk
func (up *largeUpload) transferChunk(part int64, body []byte) error { func (up *largeUpload) transferChunk(ctx context.Context, part int64, body []byte) error {
err := up.f.pacer.Call(func() (bool, error) { err := up.f.pacer.Call(func() (bool, error) {
fs.Debugf(up.o, "Sending chunk %d length %d", part, len(body)) fs.Debugf(up.o, "Sending chunk %d length %d", part, len(body))
// Get upload URL // Get upload URL
upload, err := up.getUploadURL() upload, err := up.getUploadURL(ctx)
if err != nil { if err != nil {
return false, err return false, err
} }
@ -241,8 +241,8 @@ func (up *largeUpload) transferChunk(part int64, body []byte) error {
var response api.UploadPartResponse var response api.UploadPartResponse
resp, err := up.f.srv.CallJSON(&opts, nil, &response) resp, err := up.f.srv.CallJSON(ctx, &opts, nil, &response)
retry, err := up.f.shouldRetry(resp, err) retry, err := up.f.shouldRetry(ctx, resp, err)
if err != nil { if err != nil {
fs.Debugf(up.o, "Error sending chunk %d (retry=%v): %v: %#v", part, retry, err, err) fs.Debugf(up.o, "Error sending chunk %d (retry=%v): %v: %#v", part, retry, err, err)
} }
@ -264,7 +264,7 @@ func (up *largeUpload) transferChunk(part int64, body []byte) error {
} }
// finish closes off the large upload // finish closes off the large upload
func (up *largeUpload) finish() error { func (up *largeUpload) finish(ctx context.Context) error {
fs.Debugf(up.o, "Finishing large file upload with %d parts", up.parts) fs.Debugf(up.o, "Finishing large file upload with %d parts", up.parts)
opts := rest.Opts{ opts := rest.Opts{
Method: "POST", Method: "POST",
@ -276,8 +276,8 @@ func (up *largeUpload) finish() error {
} }
var response api.FileInfo var response api.FileInfo
err := up.f.pacer.Call(func() (bool, error) { err := up.f.pacer.Call(func() (bool, error) {
resp, err := up.f.srv.CallJSON(&opts, &request, &response) resp, err := up.f.srv.CallJSON(ctx, &opts, &request, &response)
return up.f.shouldRetry(resp, err) return up.f.shouldRetry(ctx, resp, err)
}) })
if err != nil { if err != nil {
return err return err
@ -286,7 +286,7 @@ func (up *largeUpload) finish() error {
} }
// cancel aborts the large upload // cancel aborts the large upload
func (up *largeUpload) cancel() error { func (up *largeUpload) cancel(ctx context.Context) error {
opts := rest.Opts{ opts := rest.Opts{
Method: "POST", Method: "POST",
Path: "/b2_cancel_large_file", Path: "/b2_cancel_large_file",
@ -296,18 +296,18 @@ func (up *largeUpload) cancel() error {
} }
var response api.CancelLargeFileResponse var response api.CancelLargeFileResponse
err := up.f.pacer.Call(func() (bool, error) { err := up.f.pacer.Call(func() (bool, error) {
resp, err := up.f.srv.CallJSON(&opts, &request, &response) resp, err := up.f.srv.CallJSON(ctx, &opts, &request, &response)
return up.f.shouldRetry(resp, err) return up.f.shouldRetry(ctx, resp, err)
}) })
return err return err
} }
func (up *largeUpload) managedTransferChunk(wg *sync.WaitGroup, errs chan error, part int64, buf []byte) { func (up *largeUpload) managedTransferChunk(ctx context.Context, wg *sync.WaitGroup, errs chan error, part int64, buf []byte) {
wg.Add(1) wg.Add(1)
go func(part int64, buf []byte) { go func(part int64, buf []byte) {
defer wg.Done() defer wg.Done()
defer up.f.putUploadBlock(buf) defer up.f.putUploadBlock(buf)
err := up.transferChunk(part, buf) err := up.transferChunk(ctx, part, buf)
if err != nil { if err != nil {
select { select {
case errs <- err: case errs <- err:
@ -317,7 +317,7 @@ func (up *largeUpload) managedTransferChunk(wg *sync.WaitGroup, errs chan error,
}(part, buf) }(part, buf)
} }
func (up *largeUpload) finishOrCancelOnError(err error, errs chan error) error { func (up *largeUpload) finishOrCancelOnError(ctx context.Context, err error, errs chan error) error {
if err == nil { if err == nil {
select { select {
case err = <-errs: case err = <-errs:
@ -326,19 +326,19 @@ func (up *largeUpload) finishOrCancelOnError(err error, errs chan error) error {
} }
if err != nil { if err != nil {
fs.Debugf(up.o, "Cancelling large file upload due to error: %v", err) fs.Debugf(up.o, "Cancelling large file upload due to error: %v", err)
cancelErr := up.cancel() cancelErr := up.cancel(ctx)
if cancelErr != nil { if cancelErr != nil {
fs.Errorf(up.o, "Failed to cancel large file upload: %v", cancelErr) fs.Errorf(up.o, "Failed to cancel large file upload: %v", cancelErr)
} }
return err return err
} }
return up.finish() return up.finish(ctx)
} }
// Stream uploads the chunks from the input, starting with a required initial // Stream uploads the chunks from the input, starting with a required initial
// chunk. Assumes the file size is unknown and will upload until the input // chunk. Assumes the file size is unknown and will upload until the input
// reaches EOF. // reaches EOF.
func (up *largeUpload) Stream(initialUploadBlock []byte) (err error) { func (up *largeUpload) Stream(ctx context.Context, initialUploadBlock []byte) (err error) {
fs.Debugf(up.o, "Starting streaming of large file (id %q)", up.id) fs.Debugf(up.o, "Starting streaming of large file (id %q)", up.id)
errs := make(chan error, 1) errs := make(chan error, 1)
hasMoreParts := true hasMoreParts := true
@ -346,7 +346,7 @@ func (up *largeUpload) Stream(initialUploadBlock []byte) (err error) {
// Transfer initial chunk // Transfer initial chunk
up.size = int64(len(initialUploadBlock)) up.size = int64(len(initialUploadBlock))
up.managedTransferChunk(&wg, errs, 1, initialUploadBlock) up.managedTransferChunk(ctx, &wg, errs, 1, initialUploadBlock)
outer: outer:
for part := int64(2); hasMoreParts; part++ { for part := int64(2); hasMoreParts; part++ {
@ -388,16 +388,16 @@ outer:
} }
// Transfer the chunk // Transfer the chunk
up.managedTransferChunk(&wg, errs, part, buf) up.managedTransferChunk(ctx, &wg, errs, part, buf)
} }
wg.Wait() wg.Wait()
up.sha1s = up.sha1s[:up.parts] up.sha1s = up.sha1s[:up.parts]
return up.finishOrCancelOnError(err, errs) return up.finishOrCancelOnError(ctx, err, errs)
} }
// Upload uploads the chunks from the input // Upload uploads the chunks from the input
func (up *largeUpload) Upload() error { func (up *largeUpload) Upload(ctx context.Context) error {
fs.Debugf(up.o, "Starting upload of large file in %d chunks (id %q)", up.parts, up.id) fs.Debugf(up.o, "Starting upload of large file in %d chunks (id %q)", up.parts, up.id)
remaining := up.size remaining := up.size
errs := make(chan error, 1) errs := make(chan error, 1)
@ -428,10 +428,10 @@ outer:
} }
// Transfer the chunk // Transfer the chunk
up.managedTransferChunk(&wg, errs, part, buf) up.managedTransferChunk(ctx, &wg, errs, part, buf)
remaining -= reqSize remaining -= reqSize
} }
wg.Wait() wg.Wait()
return up.finishOrCancelOnError(err, errs) return up.finishOrCancelOnError(ctx, err, errs)
} }

44
backend/box/box.go
View file

@ -204,7 +204,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.It
return nil, err return nil, err
} }
found, err := f.listAll(directoryID, false, true, func(item *api.Item) bool { found, err := f.listAll(ctx, directoryID, false, true, func(item *api.Item) bool {
if item.Name == leaf { if item.Name == leaf {
info = item info = item
return true return true
@ -352,7 +352,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
// FindLeaf finds a directory of name leaf in the folder with ID pathID // FindLeaf finds a directory of name leaf in the folder with ID pathID
func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) { func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) {
// Find the leaf in pathID // Find the leaf in pathID
found, err = f.listAll(pathID, true, false, func(item *api.Item) bool { found, err = f.listAll(ctx, pathID, true, false, func(item *api.Item) bool {
if item.Name == leaf { if item.Name == leaf {
pathIDOut = item.ID pathIDOut = item.ID
return true return true
@ -386,7 +386,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
}, },
} }
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(&opts, &mkdir, &info) resp, err = f.srv.CallJSON(ctx, &opts, &mkdir, &info)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -408,7 +408,7 @@ type listAllFn func(*api.Item) bool
// Lists the directory required calling the user function on each item found // Lists the directory required calling the user function on each item found
// //
// If the user fn ever returns true then it early exits with found = true // If the user fn ever returns true then it early exits with found = true
func (f *Fs) listAll(dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) { func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) {
opts := rest.Opts{ opts := rest.Opts{
Method: "GET", Method: "GET",
Path: "/folders/" + dirID + "/items", Path: "/folders/" + dirID + "/items",
@ -423,7 +423,7 @@ OUTER:
var result api.FolderItems var result api.FolderItems
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(&opts, nil, &result) resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -479,7 +479,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
return nil, err return nil, err
} }
var iErr error var iErr error
_, err = f.listAll(directoryID, false, false, func(info *api.Item) bool { _, err = f.listAll(ctx, directoryID, false, false, func(info *api.Item) bool {
remote := path.Join(dir, info.Name) remote := path.Join(dir, info.Name)
if info.Type == api.ItemTypeFolder { if info.Type == api.ItemTypeFolder {
// cache the directory ID for later lookups // cache the directory ID for later lookups
@ -581,14 +581,14 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
} }
// deleteObject removes an object by ID // deleteObject removes an object by ID
func (f *Fs) deleteObject(id string) error { func (f *Fs) deleteObject(ctx context.Context, id string) error {
opts := rest.Opts{ opts := rest.Opts{
Method: "DELETE", Method: "DELETE",
Path: "/files/" + id, Path: "/files/" + id,
NoResponse: true, NoResponse: true,
} }
return f.pacer.Call(func() (bool, error) { return f.pacer.Call(func() (bool, error) {
resp, err := f.srv.Call(&opts) resp, err := f.srv.Call(ctx, &opts)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
} }
@ -619,7 +619,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
opts.Parameters.Set("recursive", strconv.FormatBool(!check)) opts.Parameters.Set("recursive", strconv.FormatBool(!check))
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.Call(&opts) resp, err = f.srv.Call(ctx, &opts)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -692,7 +692,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
var resp *http.Response var resp *http.Response
var info *api.Item var info *api.Item
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(&opts, &copyFile, &info) resp, err = f.srv.CallJSON(ctx, &opts, &copyFile, &info)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -715,7 +715,7 @@ func (f *Fs) Purge(ctx context.Context) error {
} }
// move a file or folder // move a file or folder
func (f *Fs) move(endpoint, id, leaf, directoryID string) (info *api.Item, err error) { func (f *Fs) move(ctx context.Context, endpoint, id, leaf, directoryID string) (info *api.Item, err error) {
// Move the object // Move the object
opts := rest.Opts{ opts := rest.Opts{
Method: "PUT", Method: "PUT",
@ -730,7 +730,7 @@ func (f *Fs) move(endpoint, id, leaf, directoryID string) (info *api.Item, err e
} }
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(&opts, &move, &info) resp, err = f.srv.CallJSON(ctx, &opts, &move, &info)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -762,7 +762,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
} }
// Do the move // Do the move
info, err := f.move("/files/", srcObj.id, leaf, directoryID) info, err := f.move(ctx, "/files/", srcObj.id, leaf, directoryID)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -845,7 +845,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
} }
// Do the move // Do the move
_, err = f.move("/folders/", srcID, leaf, directoryID) _, err = f.move(ctx, "/folders/", srcID, leaf, directoryID)
if err != nil { if err != nil {
return err return err
} }
@ -887,7 +887,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string) (string, error) {
var info api.Item var info api.Item
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(&opts, &shareLink, &info) resp, err = f.srv.CallJSON(ctx, &opts, &shareLink, &info)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
return info.SharedLink.URL, err return info.SharedLink.URL, err
@ -1006,7 +1006,7 @@ func (o *Object) setModTime(ctx context.Context, modTime time.Time) (*api.Item,
} }
var info *api.Item var info *api.Item
err := o.fs.pacer.Call(func() (bool, error) { err := o.fs.pacer.Call(func() (bool, error) {
resp, err := o.fs.srv.CallJSON(&opts, &update, &info) resp, err := o.fs.srv.CallJSON(ctx, &opts, &update, &info)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
return info, err return info, err
@ -1039,7 +1039,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
Options: options, Options: options,
} }
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.Call(&opts) resp, err = o.fs.srv.Call(ctx, &opts)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -1051,7 +1051,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
// upload does a single non-multipart upload // upload does a single non-multipart upload
// //
// This is recommended for less than 50 MB of content // This is recommended for less than 50 MB of content
func (o *Object) upload(in io.Reader, leaf, directoryID string, modTime time.Time) (err error) { func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID string, modTime time.Time) (err error) {
upload := api.UploadFile{ upload := api.UploadFile{
Name: replaceReservedChars(leaf), Name: replaceReservedChars(leaf),
ContentModifiedAt: api.Time(modTime), ContentModifiedAt: api.Time(modTime),
@ -1078,7 +1078,7 @@ func (o *Object) upload(in io.Reader, leaf, directoryID string, modTime time.Tim
opts.Path = "/files/content" opts.Path = "/files/content"
} }
err = o.fs.pacer.CallNoRetry(func() (bool, error) { err = o.fs.pacer.CallNoRetry(func() (bool, error) {
resp, err = o.fs.srv.CallJSON(&opts, &upload, &result) resp, err = o.fs.srv.CallJSON(ctx, &opts, &upload, &result)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -1111,16 +1111,16 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
// Upload with simple or multipart // Upload with simple or multipart
if size <= int64(o.fs.opt.UploadCutoff) { if size <= int64(o.fs.opt.UploadCutoff) {
err = o.upload(in, leaf, directoryID, modTime) err = o.upload(ctx, in, leaf, directoryID, modTime)
} else { } else {
err = o.uploadMultipart(in, leaf, directoryID, size, modTime) err = o.uploadMultipart(ctx, in, leaf, directoryID, size, modTime)
} }
return err return err
} }
// Remove an object // Remove an object
func (o *Object) Remove(ctx context.Context) error { func (o *Object) Remove(ctx context.Context) error {
return o.fs.deleteObject(o.id) return o.fs.deleteObject(ctx, o.id)
} }
// ID returns the ID of the Object if known, or "" if not // ID returns the ID of the Object if known, or "" if not

27
backend/box/upload.go
View file

@ -4,6 +4,7 @@ package box
import ( import (
"bytes" "bytes"
"context"
"crypto/sha1" "crypto/sha1"
"encoding/base64" "encoding/base64"
"encoding/json" "encoding/json"
@ -22,7 +23,7 @@ import (
) )
// createUploadSession creates an upload session for the object // createUploadSession creates an upload session for the object
func (o *Object) createUploadSession(leaf, directoryID string, size int64) (response *api.UploadSessionResponse, err error) { func (o *Object) createUploadSession(ctx context.Context, leaf, directoryID string, size int64) (response *api.UploadSessionResponse, err error) {
opts := rest.Opts{ opts := rest.Opts{
Method: "POST", Method: "POST",
Path: "/files/upload_sessions", Path: "/files/upload_sessions",
@ -41,7 +42,7 @@ func (o *Object) createUploadSession(leaf, directoryID string, size int64) (resp
} }
var resp *http.Response var resp *http.Response
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.CallJSON(&opts, &request, &response) resp, err = o.fs.srv.CallJSON(ctx, &opts, &request, &response)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
return return
@ -53,7 +54,7 @@ func sha1Digest(digest []byte) string {
} }
// uploadPart uploads a part in an upload session // uploadPart uploads a part in an upload session
func (o *Object) uploadPart(SessionID string, offset, totalSize int64, chunk []byte, wrap accounting.WrapFn) (response *api.UploadPartResponse, err error) { func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, totalSize int64, chunk []byte, wrap accounting.WrapFn) (response *api.UploadPartResponse, err error) {
chunkSize := int64(len(chunk)) chunkSize := int64(len(chunk))
sha1sum := sha1.Sum(chunk) sha1sum := sha1.Sum(chunk)
opts := rest.Opts{ opts := rest.Opts{
@ -70,7 +71,7 @@ func (o *Object) uploadPart(SessionID string, offset, totalSize int64, chunk []b
var resp *http.Response var resp *http.Response
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
opts.Body = wrap(bytes.NewReader(chunk)) opts.Body = wrap(bytes.NewReader(chunk))
resp, err = o.fs.srv.CallJSON(&opts, nil, &response) resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &response)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -80,7 +81,7 @@ func (o *Object) uploadPart(SessionID string, offset, totalSize int64, chunk []b
} }
// commitUpload finishes an upload session // commitUpload finishes an upload session
func (o *Object) commitUpload(SessionID string, parts []api.Part, modTime time.Time, sha1sum []byte) (result *api.FolderItems, err error) { func (o *Object) commitUpload(ctx context.Context, SessionID string, parts []api.Part, modTime time.Time, sha1sum []byte) (result *api.FolderItems, err error) {
opts := rest.Opts{ opts := rest.Opts{
Method: "POST", Method: "POST",
Path: "/files/upload_sessions/" + SessionID + "/commit", Path: "/files/upload_sessions/" + SessionID + "/commit",
@ -104,7 +105,7 @@ func (o *Object) commitUpload(SessionID string, parts []api.Part, modTime time.T
outer: outer:
for tries = 0; tries < maxTries; tries++ { for tries = 0; tries < maxTries; tries++ {
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.CallJSON(&opts, &request, nil) resp, err = o.fs.srv.CallJSON(ctx, &opts, &request, nil)
if err != nil { if err != nil {
return shouldRetry(resp, err) return shouldRetry(resp, err)
} }
@ -154,7 +155,7 @@ outer:
} }
// abortUpload cancels an upload session // abortUpload cancels an upload session
func (o *Object) abortUpload(SessionID string) (err error) { func (o *Object) abortUpload(ctx context.Context, SessionID string) (err error) {
opts := rest.Opts{ opts := rest.Opts{
Method: "DELETE", Method: "DELETE",
Path: "/files/upload_sessions/" + SessionID, Path: "/files/upload_sessions/" + SessionID,
@ -163,16 +164,16 @@ func (o *Object) abortUpload(SessionID string) (err error) {
} }
var resp *http.Response var resp *http.Response
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.Call(&opts) resp, err = o.fs.srv.Call(ctx, &opts)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
return err return err
} }
// uploadMultipart uploads a file using multipart upload // uploadMultipart uploads a file using multipart upload
func (o *Object) uploadMultipart(in io.Reader, leaf, directoryID string, size int64, modTime time.Time) (err error) { func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, leaf, directoryID string, size int64, modTime time.Time) (err error) {
// Create upload session // Create upload session
session, err := o.createUploadSession(leaf, directoryID, size) session, err := o.createUploadSession(ctx, leaf, directoryID, size)
if err != nil { if err != nil {
return errors.Wrap(err, "multipart upload create session failed") return errors.Wrap(err, "multipart upload create session failed")
} }
@ -183,7 +184,7 @@ func (o *Object) uploadMultipart(in io.Reader, leaf, directoryID string, size in
defer func() { defer func() {
if err != nil { if err != nil {
fs.Debugf(o, "Cancelling multipart upload: %v", err) fs.Debugf(o, "Cancelling multipart upload: %v", err)
cancelErr := o.abortUpload(session.ID) cancelErr := o.abortUpload(ctx, session.ID)
if cancelErr != nil { if cancelErr != nil {
fs.Logf(o, "Failed to cancel multipart upload: %v", err) fs.Logf(o, "Failed to cancel multipart upload: %v", err)
} }
@ -235,7 +236,7 @@ outer:
defer wg.Done() defer wg.Done()
defer o.fs.uploadToken.Put() defer o.fs.uploadToken.Put()
fs.Debugf(o, "Uploading part %d/%d offset %v/%v part size %v", part+1, session.TotalParts, fs.SizeSuffix(position), fs.SizeSuffix(size), fs.SizeSuffix(chunkSize)) fs.Debugf(o, "Uploading part %d/%d offset %v/%v part size %v", part+1, session.TotalParts, fs.SizeSuffix(position), fs.SizeSuffix(size), fs.SizeSuffix(chunkSize))
partResponse, err := o.uploadPart(session.ID, position, size, buf, wrap) partResponse, err := o.uploadPart(ctx, session.ID, position, size, buf, wrap)
if err != nil { if err != nil {
err = errors.Wrap(err, "multipart upload failed to upload part") err = errors.Wrap(err, "multipart upload failed to upload part")
select { select {
@ -263,7 +264,7 @@ outer:
} }
// Finalise the upload session // Finalise the upload session
result, err := o.commitUpload(session.ID, parts, modTime, hash.Sum(nil)) result, err := o.commitUpload(ctx, session.ID, parts, modTime, hash.Sum(nil))
if err != nil { if err != nil {
return errors.Wrap(err, "multipart upload failed to finalize") return errors.Wrap(err, "multipart upload failed to finalize")
} }

42
backend/fichier/api.go
View file

@ -32,7 +32,7 @@ func shouldRetry(resp *http.Response, err error) (bool, error) {
var isAlphaNumeric = regexp.MustCompile(`^[a-zA-Z0-9]+$`).MatchString var isAlphaNumeric = regexp.MustCompile(`^[a-zA-Z0-9]+$`).MatchString
func (f *Fs) getDownloadToken(url string) (*GetTokenResponse, error) { func (f *Fs) getDownloadToken(ctx context.Context, url string) (*GetTokenResponse, error) {
request := DownloadRequest{ request := DownloadRequest{
URL: url, URL: url,
Single: 1, Single: 1,
@ -44,7 +44,7 @@ func (f *Fs) getDownloadToken(url string) (*GetTokenResponse, error) {
var token GetTokenResponse var token GetTokenResponse
err := f.pacer.Call(func() (bool, error) { err := f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(&opts, &request, &token) resp, err := f.rest.CallJSON(ctx, &opts, &request, &token)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -72,7 +72,7 @@ func (f *Fs) listSharedFiles(ctx context.Context, id string) (entries fs.DirEntr
var sharedFiles SharedFolderResponse var sharedFiles SharedFolderResponse
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(&opts, nil, &sharedFiles) resp, err := f.rest.CallJSON(ctx, &opts, nil, &sharedFiles)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -88,7 +88,7 @@ func (f *Fs) listSharedFiles(ctx context.Context, id string) (entries fs.DirEntr
return entries, nil return entries, nil
} }
func (f *Fs) listFiles(directoryID int) (filesList *FilesList, err error) { func (f *Fs) listFiles(ctx context.Context, directoryID int) (filesList *FilesList, err error) {
// fs.Debugf(f, "Requesting files for dir `%s`", directoryID) // fs.Debugf(f, "Requesting files for dir `%s`", directoryID)
request := ListFilesRequest{ request := ListFilesRequest{
FolderID: directoryID, FolderID: directoryID,
@ -101,7 +101,7 @@ func (f *Fs) listFiles(directoryID int) (filesList *FilesList, err error) {
filesList = &FilesList{} filesList = &FilesList{}
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(&opts, &request, filesList) resp, err := f.rest.CallJSON(ctx, &opts, &request, filesList)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -111,7 +111,7 @@ func (f *Fs) listFiles(directoryID int) (filesList *FilesList, err error) {
return filesList, nil return filesList, nil
} }
func (f *Fs) listFolders(directoryID int) (foldersList *FoldersList, err error) { func (f *Fs) listFolders(ctx context.Context, directoryID int) (foldersList *FoldersList, err error) {
// fs.Debugf(f, "Requesting folders for id `%s`", directoryID) // fs.Debugf(f, "Requesting folders for id `%s`", directoryID)
request := ListFolderRequest{ request := ListFolderRequest{
@ -125,7 +125,7 @@ func (f *Fs) listFolders(directoryID int) (foldersList *FoldersList, err error)
foldersList = &FoldersList{} foldersList = &FoldersList{}
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(&opts, &request, foldersList) resp, err := f.rest.CallJSON(ctx, &opts, &request, foldersList)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -153,12 +153,12 @@ func (f *Fs) listDir(ctx context.Context, dir string) (entries fs.DirEntries, er
return nil, err return nil, err
} }
files, err := f.listFiles(folderID) files, err := f.listFiles(ctx, folderID)
if err != nil { if err != nil {
return nil, err return nil, err
} }
folders, err := f.listFolders(folderID) folders, err := f.listFolders(ctx, folderID)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -205,7 +205,7 @@ func getRemote(dir, fileName string) string {
return dir + "/" + fileName return dir + "/" + fileName
} }
func (f *Fs) makeFolder(leaf string, folderID int) (response *MakeFolderResponse, err error) { func (f *Fs) makeFolder(ctx context.Context, leaf string, folderID int) (response *MakeFolderResponse, err error) {
name := replaceReservedChars(leaf) name := replaceReservedChars(leaf)
// fs.Debugf(f, "Creating folder `%s` in id `%s`", name, directoryID) // fs.Debugf(f, "Creating folder `%s` in id `%s`", name, directoryID)
@ -221,7 +221,7 @@ func (f *Fs) makeFolder(leaf string, folderID int) (response *MakeFolderResponse
response = &MakeFolderResponse{} response = &MakeFolderResponse{}
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(&opts, &request, response) resp, err := f.rest.CallJSON(ctx, &opts, &request, response)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -233,7 +233,7 @@ func (f *Fs) makeFolder(leaf string, folderID int) (response *MakeFolderResponse
return response, err return response, err
} }
func (f *Fs) removeFolder(name string, folderID int) (response *GenericOKResponse, err error) { func (f *Fs) removeFolder(ctx context.Context, name string, folderID int) (response *GenericOKResponse, err error) {
// fs.Debugf(f, "Removing folder with id `%s`", directoryID) // fs.Debugf(f, "Removing folder with id `%s`", directoryID)
request := &RemoveFolderRequest{ request := &RemoveFolderRequest{
@ -248,7 +248,7 @@ func (f *Fs) removeFolder(name string, folderID int) (response *GenericOKRespons
response = &GenericOKResponse{} response = &GenericOKResponse{}
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.rest.CallJSON(&opts, request, response) resp, err = f.rest.CallJSON(ctx, &opts, request, response)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -263,7 +263,7 @@ func (f *Fs) removeFolder(name string, folderID int) (response *GenericOKRespons
return response, nil return response, nil
} }
func (f *Fs) deleteFile(url string) (response *GenericOKResponse, err error) { func (f *Fs) deleteFile(ctx context.Context, url string) (response *GenericOKResponse, err error) {
request := &RemoveFileRequest{ request := &RemoveFileRequest{
Files: []RmFile{ Files: []RmFile{
{url}, {url},
@ -277,7 +277,7 @@ func (f *Fs) deleteFile(url string) (response *GenericOKResponse, err error) {
response = &GenericOKResponse{} response = &GenericOKResponse{}
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(&opts, request, response) resp, err := f.rest.CallJSON(ctx, &opts, request, response)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
@ -290,7 +290,7 @@ func (f *Fs) deleteFile(url string) (response *GenericOKResponse, err error) {
return response, nil return response, nil
} }
func (f *Fs) getUploadNode() (response *GetUploadNodeResponse, err error) { func (f *Fs) getUploadNode(ctx context.Context) (response *GetUploadNodeResponse, err error) {
// fs.Debugf(f, "Requesting Upload node") // fs.Debugf(f, "Requesting Upload node")
opts := rest.Opts{ opts := rest.Opts{
@ -301,7 +301,7 @@ func (f *Fs) getUploadNode() (response *GetUploadNodeResponse, err error) {
response = &GetUploadNodeResponse{} response = &GetUploadNodeResponse{}
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(&opts, nil, response) resp, err := f.rest.CallJSON(ctx, &opts, nil, response)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -313,7 +313,7 @@ func (f *Fs) getUploadNode() (response *GetUploadNodeResponse, err error) {
return response, err return response, err
} }
func (f *Fs) uploadFile(in io.Reader, size int64, fileName, folderID, uploadID, node string) (response *http.Response, err error) { func (f *Fs) uploadFile(ctx context.Context, in io.Reader, size int64, fileName, folderID, uploadID, node string) (response *http.Response, err error) {
// fs.Debugf(f, "Uploading File `%s`", fileName) // fs.Debugf(f, "Uploading File `%s`", fileName)
fileName = replaceReservedChars(fileName) fileName = replaceReservedChars(fileName)
@ -343,7 +343,7 @@ func (f *Fs) uploadFile(in io.Reader, size int64, fileName, folderID, uploadID,
} }
err = f.pacer.CallNoRetry(func() (bool, error) { err = f.pacer.CallNoRetry(func() (bool, error) {
resp, err := f.rest.CallJSON(&opts, nil, nil) resp, err := f.rest.CallJSON(ctx, &opts, nil, nil)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
@ -356,7 +356,7 @@ func (f *Fs) uploadFile(in io.Reader, size int64, fileName, folderID, uploadID,
return response, err return response, err
} }
func (f *Fs) endUpload(uploadID string, nodeurl string) (response *EndFileUploadResponse, err error) { func (f *Fs) endUpload(ctx context.Context, uploadID string, nodeurl string) (response *EndFileUploadResponse, err error) {
// fs.Debugf(f, "Ending File Upload `%s`", uploadID) // fs.Debugf(f, "Ending File Upload `%s`", uploadID)
if len(uploadID) > 10 || !isAlphaNumeric(uploadID) { if len(uploadID) > 10 || !isAlphaNumeric(uploadID) {
@ -377,7 +377,7 @@ func (f *Fs) endUpload(uploadID string, nodeurl string) (response *EndFileUpload
response = &EndFileUploadResponse{} response = &EndFileUploadResponse{}
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(&opts, nil, response) resp, err := f.rest.CallJSON(ctx, &opts, nil, response)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })

14
backend/fichier/fichier.go
View file

@ -74,7 +74,7 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin
if err != nil { if err != nil {
return "", false, err return "", false, err
} }
folders, err := f.listFolders(folderID) folders, err := f.listFolders(ctx, folderID)
if err != nil { if err != nil {
return "", false, err return "", false, err
} }
@ -95,7 +95,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
if err != nil { if err != nil {
return "", err return "", err
} }
resp, err := f.makeFolder(leaf, folderID) resp, err := f.makeFolder(ctx, leaf, folderID)
if err != nil { if err != nil {
return "", err return "", err
} }
@ -251,7 +251,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
files, err := f.listFiles(folderID) files, err := f.listFiles(ctx, folderID)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -304,7 +304,7 @@ func (f *Fs) putUnchecked(ctx context.Context, in io.Reader, remote string, size
return nil, fs.ErrorCantUploadEmptyFiles return nil, fs.ErrorCantUploadEmptyFiles
} }
nodeResponse, err := f.getUploadNode() nodeResponse, err := f.getUploadNode(ctx)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -314,12 +314,12 @@ func (f *Fs) putUnchecked(ctx context.Context, in io.Reader, remote string, size
return nil, err return nil, err
} }
_, err = f.uploadFile(in, size, leaf, directoryID, nodeResponse.ID, nodeResponse.URL) _, err = f.uploadFile(ctx, in, size, leaf, directoryID, nodeResponse.ID, nodeResponse.URL)
if err != nil { if err != nil {
return nil, err return nil, err
} }
fileUploadResponse, err := f.endUpload(nodeResponse.ID, nodeResponse.URL) fileUploadResponse, err := f.endUpload(ctx, nodeResponse.ID, nodeResponse.URL)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -393,7 +393,7 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
return err return err
} }
_, err = f.removeFolder(dir, folderID) _, err = f.removeFolder(ctx, dir, folderID)
if err != nil { if err != nil {
return err return err
} }

6
backend/fichier/object.go
View file

@ -75,7 +75,7 @@ func (o *Object) SetModTime(context.Context, time.Time) error {
// Open opens the file for read. Call Close() on the returned io.ReadCloser // Open opens the file for read. Call Close() on the returned io.ReadCloser
func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) { func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
fs.FixRangeOption(options, int64(o.file.Size)) fs.FixRangeOption(options, int64(o.file.Size))
downloadToken, err := o.fs.getDownloadToken(o.file.URL) downloadToken, err := o.fs.getDownloadToken(ctx, o.file.URL)
if err != nil { if err != nil {
return nil, err return nil, err
@ -89,7 +89,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
} }
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.rest.Call(&opts) resp, err = o.fs.rest.Call(ctx, &opts)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
@ -131,7 +131,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
func (o *Object) Remove(ctx context.Context) error { func (o *Object) Remove(ctx context.Context) error {
// fs.Debugf(f, "Removing file `%s` with url `%s`", o.file.Filename, o.file.URL) // fs.Debugf(f, "Removing file `%s` with url `%s`", o.file.Filename, o.file.URL)
_, err := o.fs.deleteFile(o.file.URL) _, err := o.fs.deleteFile(ctx, o.file.URL)
if err != nil { if err != nil {
return err return err

40
backend/googlephotos/googlephotos.go
View file

@ -290,7 +290,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
} }
// fetchEndpoint gets the openid endpoint named from the Google config // fetchEndpoint gets the openid endpoint named from the Google config
func (f *Fs) fetchEndpoint(name string) (endpoint string, err error) { func (f *Fs) fetchEndpoint(ctx context.Context, name string) (endpoint string, err error) {
// Get openID config without auth // Get openID config without auth
opts := rest.Opts{ opts := rest.Opts{
Method: "GET", Method: "GET",
@ -298,7 +298,7 @@ func (f *Fs) fetchEndpoint(name string) (endpoint string, err error) {
} }
var openIDconfig map[string]interface{} var openIDconfig map[string]interface{}
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err := f.unAuth.CallJSON(&opts, nil, &openIDconfig) resp, err := f.unAuth.CallJSON(ctx, &opts, nil, &openIDconfig)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -316,7 +316,7 @@ func (f *Fs) fetchEndpoint(name string) (endpoint string, err error) {
// UserInfo fetches info about the current user with oauth2 // UserInfo fetches info about the current user with oauth2
func (f *Fs) UserInfo(ctx context.Context) (userInfo map[string]string, err error) { func (f *Fs) UserInfo(ctx context.Context) (userInfo map[string]string, err error) {
endpoint, err := f.fetchEndpoint("userinfo_endpoint") endpoint, err := f.fetchEndpoint(ctx, "userinfo_endpoint")
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -327,7 +327,7 @@ func (f *Fs) UserInfo(ctx context.Context) (userInfo map[string]string, err erro
RootURL: endpoint, RootURL: endpoint,
} }
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err := f.srv.CallJSON(&opts, nil, &userInfo) resp, err := f.srv.CallJSON(ctx, &opts, nil, &userInfo)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -338,7 +338,7 @@ func (f *Fs) UserInfo(ctx context.Context) (userInfo map[string]string, err erro
// Disconnect kills the token and refresh token // Disconnect kills the token and refresh token
func (f *Fs) Disconnect(ctx context.Context) (err error) { func (f *Fs) Disconnect(ctx context.Context) (err error) {
endpoint, err := f.fetchEndpoint("revocation_endpoint") endpoint, err := f.fetchEndpoint(ctx, "revocation_endpoint")
if err != nil { if err != nil {
return err return err
} }
@ -358,7 +358,7 @@ func (f *Fs) Disconnect(ctx context.Context) (err error) {
} }
var res interface{} var res interface{}
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err := f.srv.CallJSON(&opts, nil, &res) resp, err := f.srv.CallJSON(ctx, &opts, nil, &res)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -423,7 +423,7 @@ func findID(name string) string {
// list the albums into an internal cache // list the albums into an internal cache
// FIXME cache invalidation // FIXME cache invalidation
func (f *Fs) listAlbums(shared bool) (all *albums, err error) { func (f *Fs) listAlbums(ctx context.Context, shared bool) (all *albums, err error) {
f.albumsMu.Lock() f.albumsMu.Lock()
defer f.albumsMu.Unlock() defer f.albumsMu.Unlock()
all, ok := f.albums[shared] all, ok := f.albums[shared]
@ -445,7 +445,7 @@ func (f *Fs) listAlbums(shared bool) (all *albums, err error) {
var result api.ListAlbums var result api.ListAlbums
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(&opts, nil, &result) resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -482,7 +482,7 @@ type listFn func(remote string, object *api.MediaItem, isDirectory bool) error
// dir is the starting directory, "" for root // dir is the starting directory, "" for root
// //
// Set recurse to read sub directories // Set recurse to read sub directories
func (f *Fs) list(filter api.SearchFilter, fn listFn) (err error) { func (f *Fs) list(ctx context.Context, filter api.SearchFilter, fn listFn) (err error) {
opts := rest.Opts{ opts := rest.Opts{
Method: "POST", Method: "POST",
Path: "/mediaItems:search", Path: "/mediaItems:search",
@ -494,7 +494,7 @@ func (f *Fs) list(filter api.SearchFilter, fn listFn) (err error) {
var result api.MediaItems var result api.MediaItems
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(&opts, &filter, &result) resp, err = f.srv.CallJSON(ctx, &opts, &filter, &result)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -543,7 +543,7 @@ func (f *Fs) itemToDirEntry(ctx context.Context, remote string, item *api.MediaI
// listDir lists a single directory // listDir lists a single directory
func (f *Fs) listDir(ctx context.Context, prefix string, filter api.SearchFilter) (entries fs.DirEntries, err error) { func (f *Fs) listDir(ctx context.Context, prefix string, filter api.SearchFilter) (entries fs.DirEntries, err error) {
// List the objects // List the objects
err = f.list(filter, func(remote string, item *api.MediaItem, isDirectory bool) error { err = f.list(ctx, filter, func(remote string, item *api.MediaItem, isDirectory bool) error {
entry, err := f.itemToDirEntry(ctx, prefix+remote, item, isDirectory) entry, err := f.itemToDirEntry(ctx, prefix+remote, item, isDirectory)
if err != nil { if err != nil {
return err return err
@ -638,7 +638,7 @@ func (f *Fs) createAlbum(ctx context.Context, albumTitle string) (album *api.Alb
var result api.Album var result api.Album
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(&opts, request, &result) resp, err = f.srv.CallJSON(ctx, &opts, request, &result)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -654,7 +654,7 @@ func (f *Fs) createAlbum(ctx context.Context, albumTitle string) (album *api.Alb
func (f *Fs) getOrCreateAlbum(ctx context.Context, albumTitle string) (album *api.Album, err error) { func (f *Fs) getOrCreateAlbum(ctx context.Context, albumTitle string) (album *api.Album, err error) {
f.createMu.Lock() f.createMu.Lock()
defer f.createMu.Unlock() defer f.createMu.Unlock()
albums, err := f.listAlbums(false) albums, err := f.listAlbums(ctx, false)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -708,7 +708,7 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) (err error) {
return err return err
} }
albumTitle := match[1] albumTitle := match[1]
allAlbums, err := f.listAlbums(false) allAlbums, err := f.listAlbums(ctx, false)
if err != nil { if err != nil {
return err return err
} }
@ -773,7 +773,7 @@ func (o *Object) Size() int64 {
RootURL: o.downloadURL(), RootURL: o.downloadURL(),
} }
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.Call(&opts) resp, err = o.fs.srv.Call(ctx, &opts)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -824,7 +824,7 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
var item api.MediaItem var item api.MediaItem
var resp *http.Response var resp *http.Response
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.CallJSON(&opts, nil, &item) resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &item)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -901,7 +901,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
Options: options, Options: options,
} }
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.Call(&opts) resp, err = o.fs.srv.Call(ctx, &opts)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -954,7 +954,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
var token []byte var token []byte
var resp *http.Response var resp *http.Response
err = o.fs.pacer.CallNoRetry(func() (bool, error) { err = o.fs.pacer.CallNoRetry(func() (bool, error) {
resp, err = o.fs.srv.Call(&opts) resp, err = o.fs.srv.Call(ctx, &opts)
if err != nil { if err != nil {
return shouldRetry(resp, err) return shouldRetry(resp, err)
} }
@ -986,7 +986,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
} }
var result api.BatchCreateResponse var result api.BatchCreateResponse
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.CallJSON(&opts, request, &result) resp, err = o.fs.srv.CallJSON(ctx, &opts, request, &result)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -1029,7 +1029,7 @@ func (o *Object) Remove(ctx context.Context) (err error) {
} }
var resp *http.Response var resp *http.Response
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.CallJSON(&opts, &request, nil) resp, err = o.fs.srv.CallJSON(ctx, &opts, &request, nil)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {

4
backend/googlephotos/pattern.go
View file

@ -20,7 +20,7 @@ import (
// file pattern parsing // file pattern parsing
type lister interface { type lister interface {
listDir(ctx context.Context, prefix string, filter api.SearchFilter) (entries fs.DirEntries, err error) listDir(ctx context.Context, prefix string, filter api.SearchFilter) (entries fs.DirEntries, err error)
listAlbums(shared bool) (all *albums, err error) listAlbums(ctx context.Context, shared bool) (all *albums, err error)
listUploads(ctx context.Context, dir string) (entries fs.DirEntries, err error) listUploads(ctx context.Context, dir string) (entries fs.DirEntries, err error)
dirTime() time.Time dirTime() time.Time
} }
@ -296,7 +296,7 @@ func yearMonthDayFilter(ctx context.Context, f lister, match []string) (sf api.S
// is a prefix of another album, or actual files, or a combination of // is a prefix of another album, or actual files, or a combination of
// the two. // the two.
func albumsToEntries(ctx context.Context, f lister, shared bool, prefix string, albumPath string) (entries fs.DirEntries, err error) { func albumsToEntries(ctx context.Context, f lister, shared bool, prefix string, albumPath string) (entries fs.DirEntries, err error) {
albums, err := f.listAlbums(shared) albums, err := f.listAlbums(ctx, shared)
if err != nil { if err != nil {
return nil, err return nil, err
} }

2
backend/googlephotos/pattern_test.go
View file

@ -44,7 +44,7 @@ func (f *testLister) listDir(ctx context.Context, prefix string, filter api.Sear
} }
// mock listAlbums for testing // mock listAlbums for testing
func (f *testLister) listAlbums(shared bool) (all *albums, err error) { func (f *testLister) listAlbums(ctx context.Context, shared bool) (all *albums, err error) {
return f.albums, nil return f.albums, nil
} }

109
backend/jottacloud/jottacloud.go
View file

@ -77,6 +77,7 @@ func init() {
Description: "JottaCloud", Description: "JottaCloud",
NewFs: NewFs, NewFs: NewFs,
Config: func(name string, m configmap.Mapper) { Config: func(name string, m configmap.Mapper) {
ctx := context.TODO()
tokenString, ok := m.Get("token") tokenString, ok := m.Get("token")
if ok && tokenString != "" { if ok && tokenString != "" {
fmt.Printf("Already have a token - refresh?\n") fmt.Printf("Already have a token - refresh?\n")
@ -88,7 +89,7 @@ func init() {
srv := rest.NewClient(fshttp.NewClient(fs.Config)) srv := rest.NewClient(fshttp.NewClient(fs.Config))
fmt.Printf("\nDo you want to create a machine specific API key?\n\nRclone has it's own Jottacloud API KEY which works fine as long as one only uses rclone on a single machine. When you want to use rclone with this account on more than one machine it's recommended to create a machine specific API key. These keys can NOT be shared between machines.\n\n") fmt.Printf("\nDo you want to create a machine specific API key?\n\nRclone has it's own Jottacloud API KEY which works fine as long as one only uses rclone on a single machine. When you want to use rclone with this account on more than one machine it's recommended to create a machine specific API key. These keys can NOT be shared between machines.\n\n")
if config.Confirm() { if config.Confirm() {
deviceRegistration, err := registerDevice(srv) deviceRegistration, err := registerDevice(ctx, srv)
if err != nil { if err != nil {
log.Fatalf("Failed to register device: %v", err) log.Fatalf("Failed to register device: %v", err)
} }
@ -113,7 +114,7 @@ func init() {
username := config.ReadLine() username := config.ReadLine()
password := config.GetPassword("Your Jottacloud password is only required during setup and will not be stored.") password := config.GetPassword("Your Jottacloud password is only required during setup and will not be stored.")
token, err := doAuth(srv, username, password) token, err := doAuth(ctx, srv, username, password)
if err != nil { if err != nil {
log.Fatalf("Failed to get oauth token: %s", err) log.Fatalf("Failed to get oauth token: %s", err)
} }
@ -132,7 +133,7 @@ func init() {
srv = rest.NewClient(oAuthClient).SetRoot(rootURL) srv = rest.NewClient(oAuthClient).SetRoot(rootURL)
apiSrv := rest.NewClient(oAuthClient).SetRoot(apiURL) apiSrv := rest.NewClient(oAuthClient).SetRoot(apiURL)
device, mountpoint, err := setupMountpoint(srv, apiSrv) device, mountpoint, err := setupMountpoint(ctx, srv, apiSrv)
if err != nil { if err != nil {
log.Fatalf("Failed to setup mountpoint: %s", err) log.Fatalf("Failed to setup mountpoint: %s", err)
} }
@ -246,7 +247,7 @@ func shouldRetry(resp *http.Response, err error) (bool, error) {
} }
// registerDevice register a new device for use with the jottacloud API // registerDevice register a new device for use with the jottacloud API
func registerDevice(srv *rest.Client) (reg *api.DeviceRegistrationResponse, err error) { func registerDevice(ctx context.Context, srv *rest.Client) (reg *api.DeviceRegistrationResponse, err error) {
// random generator to generate random device names // random generator to generate random device names
seededRand := rand.New(rand.NewSource(time.Now().UnixNano())) seededRand := rand.New(rand.NewSource(time.Now().UnixNano()))
randonDeviceNamePartLength := 21 randonDeviceNamePartLength := 21
@ -269,12 +270,12 @@ func registerDevice(srv *rest.Client) (reg *api.DeviceRegistrationResponse, err
} }
var deviceRegistration *api.DeviceRegistrationResponse var deviceRegistration *api.DeviceRegistrationResponse
_, err = srv.CallJSON(&opts, nil, &deviceRegistration) _, err = srv.CallJSON(ctx, &opts, nil, &deviceRegistration)
return deviceRegistration, err return deviceRegistration, err
} }
// doAuth runs the actual token request // doAuth runs the actual token request
func doAuth(srv *rest.Client, username, password string) (token oauth2.Token, err error) { func doAuth(ctx context.Context, srv *rest.Client, username, password string) (token oauth2.Token, err error) {
// prepare out token request with username and password // prepare out token request with username and password
values := url.Values{} values := url.Values{}
values.Set("grant_type", "PASSWORD") values.Set("grant_type", "PASSWORD")
@ -291,7 +292,7 @@ func doAuth(srv *rest.Client, username, password string) (token oauth2.Token, er
// do the first request // do the first request
var jsonToken api.TokenJSON var jsonToken api.TokenJSON
resp, err := srv.CallJSON(&opts, nil, &jsonToken) resp, err := srv.CallJSON(ctx, &opts, nil, &jsonToken)
if err != nil { if err != nil {
// if 2fa is enabled the first request is expected to fail. We will do another request with the 2fa code as an additional http header // if 2fa is enabled the first request is expected to fail. We will do another request with the 2fa code as an additional http header
if resp != nil { if resp != nil {
@ -303,7 +304,7 @@ func doAuth(srv *rest.Client, username, password string) (token oauth2.Token, er
authCode = strings.Replace(authCode, "-", "", -1) // remove any "-" contained in the code so we have a 6 digit number authCode = strings.Replace(authCode, "-", "", -1) // remove any "-" contained in the code so we have a 6 digit number
opts.ExtraHeaders = make(map[string]string) opts.ExtraHeaders = make(map[string]string)
opts.ExtraHeaders["X-Jottacloud-Otp"] = authCode opts.ExtraHeaders["X-Jottacloud-Otp"] = authCode
resp, err = srv.CallJSON(&opts, nil, &jsonToken) resp, err = srv.CallJSON(ctx, &opts, nil, &jsonToken)
} }
} }
} }
@ -316,13 +317,13 @@ func doAuth(srv *rest.Client, username, password string) (token oauth2.Token, er
} }
// setupMountpoint sets up a custom device and mountpoint if desired by the user // setupMountpoint sets up a custom device and mountpoint if desired by the user
func setupMountpoint(srv *rest.Client, apiSrv *rest.Client) (device, mountpoint string, err error) { func setupMountpoint(ctx context.Context, srv *rest.Client, apiSrv *rest.Client) (device, mountpoint string, err error) {
cust, err := getCustomerInfo(apiSrv) cust, err := getCustomerInfo(ctx, apiSrv)
if err != nil { if err != nil {
return "", "", err return "", "", err
} }
acc, err := getDriveInfo(srv, cust.Username) acc, err := getDriveInfo(ctx, srv, cust.Username)
if err != nil { if err != nil {
return "", "", err return "", "", err
} }
@ -333,7 +334,7 @@ func setupMountpoint(srv *rest.Client, apiSrv *rest.Client) (device, mountpoint
fmt.Printf("Please select the device to use. Normally this will be Jotta\n") fmt.Printf("Please select the device to use. Normally this will be Jotta\n")
device = config.Choose("Devices", deviceNames, nil, false) device = config.Choose("Devices", deviceNames, nil, false)
dev, err := getDeviceInfo(srv, path.Join(cust.Username, device)) dev, err := getDeviceInfo(ctx, srv, path.Join(cust.Username, device))
if err != nil { if err != nil {
return "", "", err return "", "", err
} }
@ -351,13 +352,13 @@ func setupMountpoint(srv *rest.Client, apiSrv *rest.Client) (device, mountpoint
} }
// getCustomerInfo queries general information about the account // getCustomerInfo queries general information about the account
func getCustomerInfo(srv *rest.Client) (info *api.CustomerInfo, err error) { func getCustomerInfo(ctx context.Context, srv *rest.Client) (info *api.CustomerInfo, err error) {
opts := rest.Opts{ opts := rest.Opts{
Method: "GET", Method: "GET",
Path: "account/v1/customer", Path: "account/v1/customer",
} }
_, err = srv.CallJSON(&opts, nil, &info) _, err = srv.CallJSON(ctx, &opts, nil, &info)
if err != nil { if err != nil {
return nil, errors.Wrap(err, "couldn't get customer info") return nil, errors.Wrap(err, "couldn't get customer info")
} }
@ -366,13 +367,13 @@ func getCustomerInfo(srv *rest.Client) (info *api.CustomerInfo, err error) {
} }
// getDriveInfo queries general information about the account and the available devices and mountpoints. // getDriveInfo queries general information about the account and the available devices and mountpoints.
func getDriveInfo(srv *rest.Client, username string) (info *api.DriveInfo, err error) { func getDriveInfo(ctx context.Context, srv *rest.Client, username string) (info *api.DriveInfo, err error) {
opts := rest.Opts{ opts := rest.Opts{
Method: "GET", Method: "GET",
Path: username, Path: username,
} }
_, err = srv.CallXML(&opts, nil, &info) _, err = srv.CallXML(ctx, &opts, nil, &info)
if err != nil { if err != nil {
return nil, errors.Wrap(err, "couldn't get drive info") return nil, errors.Wrap(err, "couldn't get drive info")
} }
@ -381,13 +382,13 @@ func getDriveInfo(srv *rest.Client, username string) (info *api.DriveInfo, err e
} }
// getDeviceInfo queries Information about a jottacloud device // getDeviceInfo queries Information about a jottacloud device
func getDeviceInfo(srv *rest.Client, path string) (info *api.JottaDevice, err error) { func getDeviceInfo(ctx context.Context, srv *rest.Client, path string) (info *api.JottaDevice, err error) {
opts := rest.Opts{ opts := rest.Opts{
Method: "GET", Method: "GET",
Path: urlPathEscape(path), Path: urlPathEscape(path),
} }
_, err = srv.CallXML(&opts, nil, &info) _, err = srv.CallXML(ctx, &opts, nil, &info)
if err != nil { if err != nil {
return nil, errors.Wrap(err, "couldn't get device info") return nil, errors.Wrap(err, "couldn't get device info")
} }
@ -407,7 +408,7 @@ func (f *Fs) setEndpointURL() {
} }
// readMetaDataForPath reads the metadata from the path // readMetaDataForPath reads the metadata from the path
func (f *Fs) readMetaDataForPath(path string) (info *api.JottaFile, err error) { func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.JottaFile, err error) {
opts := rest.Opts{ opts := rest.Opts{
Method: "GET", Method: "GET",
Path: f.filePath(path), Path: f.filePath(path),
@ -415,7 +416,7 @@ func (f *Fs) readMetaDataForPath(path string) (info *api.JottaFile, err error) {
var result api.JottaFile var result api.JottaFile
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallXML(&opts, nil, &result) resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
@ -492,6 +493,7 @@ func grantTypeFilter(req *http.Request) {
// NewFs constructs an Fs from the path, container:path // NewFs constructs an Fs from the path, container:path
func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
ctx := context.TODO()
// Parse config into Options struct // Parse config into Options struct
opt := new(Options) opt := new(Options)
err := configstruct.Set(m, opt) err := configstruct.Set(m, opt)
@ -546,11 +548,11 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
// Renew the token in the background // Renew the token in the background
f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error { f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error {
_, err := f.readMetaDataForPath("") _, err := f.readMetaDataForPath(ctx, "")
return err return err
}) })
cust, err := getCustomerInfo(f.apiSrv) cust, err := getCustomerInfo(ctx, f.apiSrv)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -582,7 +584,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
// Return an Object from a path // Return an Object from a path
// //
// If it can't be found it returns the error fs.ErrorObjectNotFound. // If it can't be found it returns the error fs.ErrorObjectNotFound.
func (f *Fs) newObjectWithInfo(remote string, info *api.JottaFile) (fs.Object, error) { func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *api.JottaFile) (fs.Object, error) {
o := &Object{ o := &Object{
fs: f, fs: f,
remote: remote, remote: remote,
@ -592,7 +594,7 @@ func (f *Fs) newObjectWithInfo(remote string, info *api.JottaFile) (fs.Object, e
// Set info // Set info
err = o.setMetaData(info) err = o.setMetaData(info)
} else { } else {
err = o.readMetaData(false) // reads info and meta, returning an error err = o.readMetaData(ctx, false) // reads info and meta, returning an error
} }
if err != nil { if err != nil {
return nil, err return nil, err
@ -603,11 +605,11 @@ func (f *Fs) newObjectWithInfo(remote string, info *api.JottaFile) (fs.Object, e
// NewObject finds the Object at remote. If it can't be found // NewObject finds the Object at remote. If it can't be found
// it returns the error fs.ErrorObjectNotFound. // it returns the error fs.ErrorObjectNotFound.
func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
return f.newObjectWithInfo(remote, nil) return f.newObjectWithInfo(ctx, remote, nil)
} }
// CreateDir makes a directory // CreateDir makes a directory
func (f *Fs) CreateDir(path string) (jf *api.JottaFolder, err error) { func (f *Fs) CreateDir(ctx context.Context, path string) (jf *api.JottaFolder, err error) {
// fs.Debugf(f, "CreateDir(%q, %q)\n", pathID, leaf) // fs.Debugf(f, "CreateDir(%q, %q)\n", pathID, leaf)
var resp *http.Response var resp *http.Response
opts := rest.Opts{ opts := rest.Opts{
@ -619,7 +621,7 @@ func (f *Fs) CreateDir(path string) (jf *api.JottaFolder, err error) {
opts.Parameters.Set("mkDir", "true") opts.Parameters.Set("mkDir", "true")
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallXML(&opts, nil, &jf) resp, err = f.srv.CallXML(ctx, &opts, nil, &jf)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -648,7 +650,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
var resp *http.Response var resp *http.Response
var result api.JottaFolder var result api.JottaFolder
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallXML(&opts, nil, &result) resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
@ -682,7 +684,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
continue continue
} }
remote := path.Join(dir, restoreReservedChars(item.Name)) remote := path.Join(dir, restoreReservedChars(item.Name))
o, err := f.newObjectWithInfo(remote, item) o, err := f.newObjectWithInfo(ctx, remote, item)
if err != nil { if err != nil {
continue continue
} }
@ -696,7 +698,7 @@ type listFileDirFn func(fs.DirEntry) error
// List the objects and directories into entries, from a // List the objects and directories into entries, from a
// special kind of JottaFolder representing a FileDirLis // special kind of JottaFolder representing a FileDirLis
func (f *Fs) listFileDir(remoteStartPath string, startFolder *api.JottaFolder, fn listFileDirFn) error { func (f *Fs) listFileDir(ctx context.Context, remoteStartPath string, startFolder *api.JottaFolder, fn listFileDirFn) error {
pathPrefix := "/" + f.filePathRaw("") // Non-escaped prefix of API paths to be cut off, to be left with the remote path including the remoteStartPath pathPrefix := "/" + f.filePathRaw("") // Non-escaped prefix of API paths to be cut off, to be left with the remote path including the remoteStartPath
pathPrefixLength := len(pathPrefix) pathPrefixLength := len(pathPrefix)
startPath := path.Join(pathPrefix, remoteStartPath) // Non-escaped API path up to and including remoteStartPath, to decide if it should be created as a new dir object startPath := path.Join(pathPrefix, remoteStartPath) // Non-escaped API path up to and including remoteStartPath, to decide if it should be created as a new dir object
@ -725,7 +727,7 @@ func (f *Fs) listFileDir(remoteStartPath string, startFolder *api.JottaFolder, f
continue continue
} }
remoteFile := path.Join(remoteDir, restoreReservedChars(file.Name)) remoteFile := path.Join(remoteDir, restoreReservedChars(file.Name))
o, err := f.newObjectWithInfo(remoteFile, file) o, err := f.newObjectWithInfo(ctx, remoteFile, file)
if err != nil { if err != nil {
return err return err
} }
@ -754,7 +756,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
var resp *http.Response var resp *http.Response
var result api.JottaFolder // Could be JottaFileDirList, but JottaFolder is close enough var result api.JottaFolder // Could be JottaFileDirList, but JottaFolder is close enough
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallXML(&opts, nil, &result) resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -767,7 +769,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
return errors.Wrap(err, "couldn't list files") return errors.Wrap(err, "couldn't list files")
} }
list := walk.NewListRHelper(callback) list := walk.NewListRHelper(callback)
err = f.listFileDir(dir, &result, func(entry fs.DirEntry) error { err = f.listFileDir(ctx, dir, &result, func(entry fs.DirEntry) error {
return list.Add(entry) return list.Add(entry)
}) })
if err != nil { if err != nil {
@ -821,7 +823,7 @@ func (f *Fs) mkParentDir(ctx context.Context, dirPath string) error {
// Mkdir creates the container if it doesn't exist // Mkdir creates the container if it doesn't exist
func (f *Fs) Mkdir(ctx context.Context, dir string) error { func (f *Fs) Mkdir(ctx context.Context, dir string) error {
_, err := f.CreateDir(dir) _, err := f.CreateDir(ctx, dir)
return err return err
} }
@ -860,7 +862,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.Call(&opts) resp, err = f.srv.Call(ctx, &opts)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -888,7 +890,7 @@ func (f *Fs) Purge(ctx context.Context) error {
} }
// copyOrMoves copies or moves directories or files depending on the method parameter // copyOrMoves copies or moves directories or files depending on the method parameter
func (f *Fs) copyOrMove(method, src, dest string) (info *api.JottaFile, err error) { func (f *Fs) copyOrMove(ctx context.Context, method, src, dest string) (info *api.JottaFile, err error) {
opts := rest.Opts{ opts := rest.Opts{
Method: "POST", Method: "POST",
Path: src, Path: src,
@ -899,7 +901,7 @@ func (f *Fs) copyOrMove(method, src, dest string) (info *api.JottaFile, err erro
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallXML(&opts, nil, &info) resp, err = f.srv.CallXML(ctx, &opts, nil, &info)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -928,13 +930,13 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
if err != nil { if err != nil {
return nil, err return nil, err
} }
info, err := f.copyOrMove("cp", srcObj.filePath(), remote) info, err := f.copyOrMove(ctx, "cp", srcObj.filePath(), remote)
if err != nil { if err != nil {
return nil, errors.Wrap(err, "couldn't copy file") return nil, errors.Wrap(err, "couldn't copy file")
} }
return f.newObjectWithInfo(remote, info) return f.newObjectWithInfo(ctx, remote, info)
//return f.newObjectWithInfo(remote, &result) //return f.newObjectWithInfo(remote, &result)
} }
@ -958,13 +960,13 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
if err != nil { if err != nil {
return nil, err return nil, err
} }
info, err := f.copyOrMove("mv", srcObj.filePath(), remote) info, err := f.copyOrMove(ctx, "mv", srcObj.filePath(), remote)
if err != nil { if err != nil {
return nil, errors.Wrap(err, "couldn't move file") return nil, errors.Wrap(err, "couldn't move file")
} }
return f.newObjectWithInfo(remote, info) return f.newObjectWithInfo(ctx, remote, info)
//return f.newObjectWithInfo(remote, result) //return f.newObjectWithInfo(remote, result)
} }
@ -1002,7 +1004,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
return fs.ErrorDirExists return fs.ErrorDirExists
} }
_, err = f.copyOrMove("mvDir", path.Join(f.endpointURL, replaceReservedChars(srcPath))+"/", dstRemote) _, err = f.copyOrMove(ctx, "mvDir", path.Join(f.endpointURL, replaceReservedChars(srcPath))+"/", dstRemote)
if err != nil { if err != nil {
return errors.Wrap(err, "couldn't move directory") return errors.Wrap(err, "couldn't move directory")
@ -1027,7 +1029,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err er
var resp *http.Response var resp *http.Response
var result api.JottaFile var result api.JottaFile
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallXML(&opts, nil, &result) resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
@ -1058,7 +1060,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err er
// About gets quota information // About gets quota information
func (f *Fs) About(ctx context.Context) (*fs.Usage, error) { func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
info, err := getDriveInfo(f.srv, f.user) info, err := getDriveInfo(ctx, f.srv, f.user)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -1113,7 +1115,8 @@ func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
// Size returns the size of an object in bytes // Size returns the size of an object in bytes
func (o *Object) Size() int64 { func (o *Object) Size() int64 {
err := o.readMetaData(false) ctx := context.TODO()
err := o.readMetaData(ctx, false)
if err != nil { if err != nil {
fs.Logf(o, "Failed to read metadata: %v", err) fs.Logf(o, "Failed to read metadata: %v", err)
return 0 return 0
@ -1137,11 +1140,11 @@ func (o *Object) setMetaData(info *api.JottaFile) (err error) {
} }
// readMetaData reads and updates the metadata for an object // readMetaData reads and updates the metadata for an object
func (o *Object) readMetaData(force bool) (err error) { func (o *Object) readMetaData(ctx context.Context, force bool) (err error) {
if o.hasMetaData && !force { if o.hasMetaData && !force {
return nil return nil
} }
info, err := o.fs.readMetaDataForPath(o.remote) info, err := o.fs.readMetaDataForPath(ctx, o.remote)
if err != nil { if err != nil {
return err return err
} }
@ -1156,7 +1159,7 @@ func (o *Object) readMetaData(force bool) (err error) {
// It attempts to read the objects mtime and if that isn't present the // It attempts to read the objects mtime and if that isn't present the
// LastModified returned in the http headers // LastModified returned in the http headers
func (o *Object) ModTime(ctx context.Context) time.Time { func (o *Object) ModTime(ctx context.Context) time.Time {
err := o.readMetaData(false) err := o.readMetaData(ctx, false)
if err != nil { if err != nil {
fs.Logf(o, "Failed to read metadata: %v", err) fs.Logf(o, "Failed to read metadata: %v", err)
return time.Now() return time.Now()
@ -1188,7 +1191,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
opts.Parameters.Set("mode", "bin") opts.Parameters.Set("mode", "bin")
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.Call(&opts) resp, err = o.fs.srv.Call(ctx, &opts)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -1298,7 +1301,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
// send it // send it
var response api.AllocateFileResponse var response api.AllocateFileResponse
err = o.fs.pacer.CallNoRetry(func() (bool, error) { err = o.fs.pacer.CallNoRetry(func() (bool, error) {
resp, err = o.fs.apiSrv.CallJSON(&opts, &request, &response) resp, err = o.fs.apiSrv.CallJSON(ctx, &opts, &request, &response)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -1329,7 +1332,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
} }
// send the remaining bytes // send the remaining bytes
resp, err = o.fs.apiSrv.CallJSON(&opts, nil, &result) resp, err = o.fs.apiSrv.CallJSON(ctx, &opts, nil, &result)
if err != nil { if err != nil {
return err return err
} }
@ -1341,7 +1344,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
o.modTime = time.Unix(result.Modified/1000, 0) o.modTime = time.Unix(result.Modified/1000, 0)
} else { } else {
// If the file state is COMPLETE we don't need to upload it because the file was already found but we still ned to update our metadata // If the file state is COMPLETE we don't need to upload it because the file was already found but we still ned to update our metadata
return o.readMetaData(true) return o.readMetaData(ctx, true)
} }
return nil return nil
@ -1363,7 +1366,7 @@ func (o *Object) Remove(ctx context.Context) error {
} }
return o.fs.pacer.Call(func() (bool, error) { return o.fs.pacer.Call(func() (bool, error) {
resp, err := o.fs.srv.CallXML(&opts, nil, nil) resp, err := o.fs.srv.CallXML(ctx, &opts, nil, nil)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
} }

54
backend/mailru/mailru.go
View file

@ -535,7 +535,7 @@ func (f *Fs) relPath(absPath string) (string, error) {
} }
// metaServer ... // metaServer ...
func (f *Fs) metaServer() (string, error) { func (f *Fs) metaServer(ctx context.Context) (string, error) {
f.metaMu.Lock() f.metaMu.Lock()
defer f.metaMu.Unlock() defer f.metaMu.Unlock()
@ -555,7 +555,7 @@ func (f *Fs) metaServer() (string, error) {
err error err error
) )
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
res, err = f.srv.Call(&opts) res, err = f.srv.Call(ctx, &opts)
if err == nil { if err == nil {
url, err = readBodyWord(res) url, err = readBodyWord(res)
} }
@ -608,7 +608,7 @@ func (f *Fs) readItemMetaData(ctx context.Context, path string) (entry fs.DirEnt
var info api.ItemInfoResponse var info api.ItemInfoResponse
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
res, err := f.srv.CallJSON(&opts, nil, &info) res, err := f.srv.CallJSON(ctx, &opts, nil, &info)
return shouldRetry(res, err, f, &opts) return shouldRetry(res, err, f, &opts)
}) })
@ -716,7 +716,7 @@ func (f *Fs) listM1(ctx context.Context, dirPath string, offset int, limit int)
res *http.Response res *http.Response
) )
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
res, err = f.srv.CallJSON(&opts, nil, &info) res, err = f.srv.CallJSON(ctx, &opts, nil, &info)
return shouldRetry(res, err, f, &opts) return shouldRetry(res, err, f, &opts)
}) })
@ -758,7 +758,7 @@ func (f *Fs) listBin(ctx context.Context, dirPath string, depth int) (entries fs
if err != nil { if err != nil {
return nil, err return nil, err
} }
metaURL, err := f.metaServer() metaURL, err := f.metaServer(ctx)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -776,7 +776,7 @@ func (f *Fs) listBin(ctx context.Context, dirPath string, depth int) (entries fs
var res *http.Response var res *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
res, err = f.srv.Call(&opts) res, err = f.srv.Call(ctx, &opts)
return shouldRetry(res, err, f, &opts) return shouldRetry(res, err, f, &opts)
}) })
if err != nil { if err != nil {
@ -1031,7 +1031,7 @@ func (f *Fs) CreateDir(ctx context.Context, path string) error {
if err != nil { if err != nil {
return err return err
} }
metaURL, err := f.metaServer() metaURL, err := f.metaServer(ctx)
if err != nil { if err != nil {
return err return err
} }
@ -1049,7 +1049,7 @@ func (f *Fs) CreateDir(ctx context.Context, path string) error {
var res *http.Response var res *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
res, err = f.srv.Call(&opts) res, err = f.srv.Call(ctx, &opts)
return shouldRetry(res, err, f, &opts) return shouldRetry(res, err, f, &opts)
}) })
if err != nil { if err != nil {
@ -1192,7 +1192,7 @@ func (f *Fs) delete(ctx context.Context, path string, hardDelete bool) error {
var response api.GenericResponse var response api.GenericResponse
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
res, err := f.srv.CallJSON(&opts, nil, &response) res, err := f.srv.CallJSON(ctx, &opts, nil, &response)
return shouldRetry(res, err, f, &opts) return shouldRetry(res, err, f, &opts)
}) })
@ -1264,7 +1264,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
var response api.GenericBodyResponse var response api.GenericBodyResponse
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
res, err := f.srv.CallJSON(&opts, nil, &response) res, err := f.srv.CallJSON(ctx, &opts, nil, &response)
return shouldRetry(res, err, f, &opts) return shouldRetry(res, err, f, &opts)
}) })
@ -1342,7 +1342,7 @@ func (f *Fs) moveItemBin(ctx context.Context, srcPath, dstPath, opName string) e
if err != nil { if err != nil {
return err return err
} }
metaURL, err := f.metaServer() metaURL, err := f.metaServer(ctx)
if err != nil { if err != nil {
return err return err
} }
@ -1368,7 +1368,7 @@ func (f *Fs) moveItemBin(ctx context.Context, srcPath, dstPath, opName string) e
var res *http.Response var res *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
res, err = f.srv.Call(&opts) res, err = f.srv.Call(ctx, &opts)
return shouldRetry(res, err, f, &opts) return shouldRetry(res, err, f, &opts)
}) })
if err != nil { if err != nil {
@ -1459,7 +1459,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err er
var response api.GenericBodyResponse var response api.GenericBodyResponse
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
res, err := f.srv.CallJSON(&opts, nil, &response) res, err := f.srv.CallJSON(ctx, &opts, nil, &response)
return shouldRetry(res, err, f, &opts) return shouldRetry(res, err, f, &opts)
}) })
@ -1500,7 +1500,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
var response api.CleanupResponse var response api.CleanupResponse
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
res, err := f.srv.CallJSON(&opts, nil, &response) res, err := f.srv.CallJSON(ctx, &opts, nil, &response)
return shouldRetry(res, err, f, &opts) return shouldRetry(res, err, f, &opts)
}) })
if err != nil { if err != nil {
@ -1533,7 +1533,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
var info api.UserInfoResponse var info api.UserInfoResponse
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
res, err := f.srv.CallJSON(&opts, nil, &info) res, err := f.srv.CallJSON(ctx, &opts, nil, &info)
return shouldRetry(res, err, f, &opts) return shouldRetry(res, err, f, &opts)
}) })
if err != nil { if err != nil {
@ -1664,7 +1664,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
hasher = mrhash.New() hasher = mrhash.New()
wrapIn = io.TeeReader(wrapIn, hasher) wrapIn = io.TeeReader(wrapIn, hasher)
} }
newHash, err = o.upload(wrapIn, size, options...) newHash, err = o.upload(ctx, wrapIn, size, options...)
if fileHash == nil && err == nil { if fileHash == nil && err == nil {
fileHash = hasher.Sum(nil) fileHash = hasher.Sum(nil)
} }
@ -1783,12 +1783,12 @@ func makeTempFile(ctx context.Context, tmpFs fs.Fs, wrapIn io.Reader, src fs.Obj
return return
} }
func (o *Object) upload(in io.Reader, size int64, options ...fs.OpenOption) ([]byte, error) { func (o *Object) upload(ctx context.Context, in io.Reader, size int64, options ...fs.OpenOption) ([]byte, error) {
token, err := o.fs.accessToken() token, err := o.fs.accessToken()
if err != nil { if err != nil {
return nil, err return nil, err
} }
shardURL, err := o.fs.uploadShard() shardURL, err := o.fs.uploadShard(ctx)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -1813,7 +1813,7 @@ func (o *Object) upload(in io.Reader, size int64, options ...fs.OpenOption) ([]b
strHash string strHash string
) )
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
res, err = o.fs.srv.Call(&opts) res, err = o.fs.srv.Call(ctx, &opts)
if err == nil { if err == nil {
strHash, err = readBodyWord(res) strHash, err = readBodyWord(res)
} }
@ -1832,7 +1832,7 @@ func (o *Object) upload(in io.Reader, size int64, options ...fs.OpenOption) ([]b
} }
} }
func (f *Fs) uploadShard() (string, error) { func (f *Fs) uploadShard(ctx context.Context) (string, error) {
f.shardMu.Lock() f.shardMu.Lock()
defer f.shardMu.Unlock() defer f.shardMu.Unlock()
@ -1856,7 +1856,7 @@ func (f *Fs) uploadShard() (string, error) {
var info api.ShardInfoResponse var info api.ShardInfoResponse
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
res, err := f.srv.CallJSON(&opts, nil, &info) res, err := f.srv.CallJSON(ctx, &opts, nil, &info)
return shouldRetry(res, err, f, &opts) return shouldRetry(res, err, f, &opts)
}) })
if err != nil { if err != nil {
@ -1995,7 +1995,7 @@ func (o *Object) addFileMetaData(ctx context.Context, overwrite bool) error {
if err != nil { if err != nil {
return err return err
} }
metaURL, err := o.fs.metaServer() metaURL, err := o.fs.metaServer(ctx)
if err != nil { if err != nil {
return err return err
} }
@ -2032,7 +2032,7 @@ func (o *Object) addFileMetaData(ctx context.Context, overwrite bool) error {
var res *http.Response var res *http.Response
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
res, err = o.fs.srv.Call(&opts) res, err = o.fs.srv.Call(ctx, &opts)
return shouldRetry(res, err, o.fs, &opts) return shouldRetry(res, err, o.fs, &opts)
}) })
if err != nil { if err != nil {
@ -2115,12 +2115,12 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
var res *http.Response var res *http.Response
server := "" server := ""
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
server, err = o.fs.fileServers.Dispatch(server) server, err = o.fs.fileServers.Dispatch(ctx, server)
if err != nil { if err != nil {
return false, err return false, err
} }
opts.RootURL = server opts.RootURL = server
res, err = o.fs.srv.Call(&opts) res, err = o.fs.srv.Call(ctx, &opts)
return shouldRetry(res, err, o.fs, &opts) return shouldRetry(res, err, o.fs, &opts)
}) })
if err != nil { if err != nil {
@ -2213,7 +2213,7 @@ type pendingServer struct {
// Dispatch dispatches next download server. // Dispatch dispatches next download server.
// It prefers switching and tries to avoid current server // It prefers switching and tries to avoid current server
// in use by caller because it may be overloaded or slow. // in use by caller because it may be overloaded or slow.
func (p *serverPool) Dispatch(current string) (string, error) { func (p *serverPool) Dispatch(ctx context.Context, current string) (string, error) {
now := time.Now() now := time.Now()
url := p.getServer(current, now) url := p.getServer(current, now)
if url != "" { if url != "" {
@ -2231,7 +2231,7 @@ func (p *serverPool) Dispatch(current string) (string, error) {
err error err error
) )
err = p.fs.pacer.Call(func() (bool, error) { err = p.fs.pacer.Call(func() (bool, error) {
res, err = p.fs.srv.Call(&opts) res, err = p.fs.srv.Call(ctx, &opts)
if err != nil { if err != nil {
return fserrors.ShouldRetry(err), err return fserrors.ShouldRetry(err), err
} }

67
backend/onedrive/onedrive.go
View file

@ -72,6 +72,7 @@ func init() {
Description: "Microsoft OneDrive", Description: "Microsoft OneDrive",
NewFs: NewFs, NewFs: NewFs,
Config: func(name string, m configmap.Mapper) { Config: func(name string, m configmap.Mapper) {
ctx := context.TODO()
err := oauthutil.Config("onedrive", name, m, oauthConfig) err := oauthutil.Config("onedrive", name, m, oauthConfig)
if err != nil { if err != nil {
log.Fatalf("Failed to configure token: %v", err) log.Fatalf("Failed to configure token: %v", err)
@ -143,7 +144,7 @@ func init() {
} }
sites := siteResponse{} sites := siteResponse{}
_, err := srv.CallJSON(&opts, nil, &sites) _, err := srv.CallJSON(ctx, &opts, nil, &sites)
if err != nil { if err != nil {
log.Fatalf("Failed to query available sites: %v", err) log.Fatalf("Failed to query available sites: %v", err)
} }
@ -172,7 +173,7 @@ func init() {
// query Microsoft Graph // query Microsoft Graph
if finalDriveID == "" { if finalDriveID == "" {
drives := drivesResponse{} drives := drivesResponse{}
_, err := srv.CallJSON(&opts, nil, &drives) _, err := srv.CallJSON(ctx, &opts, nil, &drives)
if err != nil { if err != nil {
log.Fatalf("Failed to query available drives: %v", err) log.Fatalf("Failed to query available drives: %v", err)
} }
@ -194,7 +195,7 @@ func init() {
RootURL: graphURL, RootURL: graphURL,
Path: "/drives/" + finalDriveID + "/root"} Path: "/drives/" + finalDriveID + "/root"}
var rootItem api.Item var rootItem api.Item
_, err = srv.CallJSON(&opts, nil, &rootItem) _, err = srv.CallJSON(ctx, &opts, nil, &rootItem)
if err != nil { if err != nil {
log.Fatalf("Failed to query root for drive %s: %v", finalDriveID, err) log.Fatalf("Failed to query root for drive %s: %v", finalDriveID, err)
} }
@ -343,10 +344,10 @@ func shouldRetry(resp *http.Response, err error) (bool, error) {
// instead of simply using `drives/driveID/root:/itemPath` because it works for // instead of simply using `drives/driveID/root:/itemPath` because it works for
// "shared with me" folders in OneDrive Personal (See #2536, #2778) // "shared with me" folders in OneDrive Personal (See #2536, #2778)
// This path pattern comes from https://github.com/OneDrive/onedrive-api-docs/issues/908#issuecomment-417488480 // This path pattern comes from https://github.com/OneDrive/onedrive-api-docs/issues/908#issuecomment-417488480
func (f *Fs) readMetaDataForPathRelativeToID(normalizedID string, relPath string) (info *api.Item, resp *http.Response, err error) { func (f *Fs) readMetaDataForPathRelativeToID(ctx context.Context, normalizedID string, relPath string) (info *api.Item, resp *http.Response, err error) {
opts := newOptsCall(normalizedID, "GET", ":/"+withTrailingColon(rest.URLPathEscape(replaceReservedChars(relPath)))) opts := newOptsCall(normalizedID, "GET", ":/"+withTrailingColon(rest.URLPathEscape(replaceReservedChars(relPath))))
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(&opts, nil, &info) resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
@ -371,7 +372,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.It
} }
} }
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(&opts, nil, &info) resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
return info, resp, err return info, resp, err
@ -426,7 +427,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.It
} }
} }
return f.readMetaDataForPathRelativeToID(baseNormalizedID, relPath) return f.readMetaDataForPathRelativeToID(ctx, baseNormalizedID, relPath)
} }
// errorHandler parses a non 2xx error response into an error // errorHandler parses a non 2xx error response into an error
@ -592,7 +593,7 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin
if !ok { if !ok {
return "", false, errors.New("couldn't find parent ID") return "", false, errors.New("couldn't find parent ID")
} }
info, resp, err := f.readMetaDataForPathRelativeToID(pathID, leaf) info, resp, err := f.readMetaDataForPathRelativeToID(ctx, pathID, leaf)
if err != nil { if err != nil {
if resp != nil && resp.StatusCode == http.StatusNotFound { if resp != nil && resp.StatusCode == http.StatusNotFound {
return "", false, nil return "", false, nil
@ -619,7 +620,7 @@ func (f *Fs) CreateDir(ctx context.Context, dirID, leaf string) (newID string, e
ConflictBehavior: "fail", ConflictBehavior: "fail",
} }
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(&opts, &mkdir, &info) resp, err = f.srv.CallJSON(ctx, &opts, &mkdir, &info)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -642,7 +643,7 @@ type listAllFn func(*api.Item) bool
// Lists the directory required calling the user function on each item found // Lists the directory required calling the user function on each item found
// //
// If the user fn ever returns true then it early exits with found = true // If the user fn ever returns true then it early exits with found = true
func (f *Fs) listAll(dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) { func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) {
// Top parameter asks for bigger pages of data // Top parameter asks for bigger pages of data
// https://dev.onedrive.com/odata/optional-query-parameters.htm // https://dev.onedrive.com/odata/optional-query-parameters.htm
opts := newOptsCall(dirID, "GET", "/children?$top=1000") opts := newOptsCall(dirID, "GET", "/children?$top=1000")
@ -651,7 +652,7 @@ OUTER:
var result api.ListChildrenResponse var result api.ListChildrenResponse
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(&opts, nil, &result) resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -709,7 +710,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
return nil, err return nil, err
} }
var iErr error var iErr error
_, err = f.listAll(directoryID, false, false, func(info *api.Item) bool { _, err = f.listAll(ctx, directoryID, false, false, func(info *api.Item) bool {
if !f.opt.ExposeOneNoteFiles && info.GetPackageType() == api.PackageTypeOneNote { if !f.opt.ExposeOneNoteFiles && info.GetPackageType() == api.PackageTypeOneNote {
fs.Debugf(info.Name, "OneNote file not shown in directory listing") fs.Debugf(info.Name, "OneNote file not shown in directory listing")
return false return false
@ -793,12 +794,12 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
} }
// deleteObject removes an object by ID // deleteObject removes an object by ID
func (f *Fs) deleteObject(id string) error { func (f *Fs) deleteObject(ctx context.Context, id string) error {
opts := newOptsCall(id, "DELETE", "") opts := newOptsCall(id, "DELETE", "")
opts.NoResponse = true opts.NoResponse = true
return f.pacer.Call(func() (bool, error) { return f.pacer.Call(func() (bool, error) {
resp, err := f.srv.Call(&opts) resp, err := f.srv.Call(ctx, &opts)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
} }
@ -821,7 +822,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
} }
if check { if check {
// check to see if there are any items // check to see if there are any items
found, err := f.listAll(rootID, false, false, func(item *api.Item) bool { found, err := f.listAll(ctx, rootID, false, false, func(item *api.Item) bool {
return true return true
}) })
if err != nil { if err != nil {
@ -831,7 +832,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
return fs.ErrorDirectoryNotEmpty return fs.ErrorDirectoryNotEmpty
} }
} }
err = f.deleteObject(rootID) err = f.deleteObject(ctx, rootID)
if err != nil { if err != nil {
return err return err
} }
@ -941,7 +942,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
} }
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(&opts, &copyReq, nil) resp, err = f.srv.CallJSON(ctx, &opts, &copyReq, nil)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -1029,7 +1030,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
var resp *http.Response var resp *http.Response
var info api.Item var info api.Item
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(&opts, &move, &info) resp, err = f.srv.CallJSON(ctx, &opts, &move, &info)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -1122,7 +1123,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
} }
// Get timestamps of src so they can be preserved // Get timestamps of src so they can be preserved
srcInfo, _, err := srcFs.readMetaDataForPathRelativeToID(srcID, "") srcInfo, _, err := srcFs.readMetaDataForPathRelativeToID(ctx, srcID, "")
if err != nil { if err != nil {
return err return err
} }
@ -1144,7 +1145,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
var resp *http.Response var resp *http.Response
var info api.Item var info api.Item
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(&opts, &move, &info) resp, err = f.srv.CallJSON(ctx, &opts, &move, &info)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -1170,7 +1171,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
} }
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(&opts, nil, &drive) resp, err = f.srv.CallJSON(ctx, &opts, nil, &drive)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -1210,7 +1211,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err er
var resp *http.Response var resp *http.Response
var result api.CreateShareLinkResponse var result api.CreateShareLinkResponse
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(&opts, &share, &result) resp, err = f.srv.CallJSON(ctx, &opts, &share, &result)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -1370,7 +1371,7 @@ func (o *Object) setModTime(ctx context.Context, modTime time.Time) (*api.Item,
} }
var info *api.Item var info *api.Item
err := o.fs.pacer.Call(func() (bool, error) { err := o.fs.pacer.Call(func() (bool, error) {
resp, err := o.fs.srv.CallJSON(&opts, &update, &info) resp, err := o.fs.srv.CallJSON(ctx, &opts, &update, &info)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
return info, err return info, err
@ -1405,7 +1406,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
opts.Options = options opts.Options = options
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.Call(&opts) resp, err = o.fs.srv.Call(ctx, &opts)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -1441,7 +1442,7 @@ func (o *Object) createUploadSession(ctx context.Context, modTime time.Time) (re
createRequest.Item.FileSystemInfo.LastModifiedDateTime = api.Timestamp(modTime) createRequest.Item.FileSystemInfo.LastModifiedDateTime = api.Timestamp(modTime)
var resp *http.Response var resp *http.Response
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.CallJSON(&opts, &createRequest, &response) resp, err = o.fs.srv.CallJSON(ctx, &opts, &createRequest, &response)
if apiErr, ok := err.(*api.Error); ok { if apiErr, ok := err.(*api.Error); ok {
if apiErr.ErrorInfo.Code == "nameAlreadyExists" { if apiErr.ErrorInfo.Code == "nameAlreadyExists" {
// Make the error more user-friendly // Make the error more user-friendly
@ -1454,7 +1455,7 @@ func (o *Object) createUploadSession(ctx context.Context, modTime time.Time) (re
} }
// uploadFragment uploads a part // uploadFragment uploads a part
func (o *Object) uploadFragment(url string, start int64, totalSize int64, chunk io.ReadSeeker, chunkSize int64) (info *api.Item, err error) { func (o *Object) uploadFragment(ctx context.Context, url string, start int64, totalSize int64, chunk io.ReadSeeker, chunkSize int64) (info *api.Item, err error) {
opts := rest.Opts{ opts := rest.Opts{
Method: "PUT", Method: "PUT",
RootURL: url, RootURL: url,
@ -1467,7 +1468,7 @@ func (o *Object) uploadFragment(url string, start int64, totalSize int64, chunk
var body []byte var body []byte
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
_, _ = chunk.Seek(0, io.SeekStart) _, _ = chunk.Seek(0, io.SeekStart)
resp, err = o.fs.srv.Call(&opts) resp, err = o.fs.srv.Call(ctx, &opts)
if err != nil { if err != nil {
return shouldRetry(resp, err) return shouldRetry(resp, err)
} }
@ -1487,7 +1488,7 @@ func (o *Object) uploadFragment(url string, start int64, totalSize int64, chunk
} }
// cancelUploadSession cancels an upload session // cancelUploadSession cancels an upload session
func (o *Object) cancelUploadSession(url string) (err error) { func (o *Object) cancelUploadSession(ctx context.Context, url string) (err error) {
opts := rest.Opts{ opts := rest.Opts{
Method: "DELETE", Method: "DELETE",
RootURL: url, RootURL: url,
@ -1495,7 +1496,7 @@ func (o *Object) cancelUploadSession(url string) (err error) {
} }
var resp *http.Response var resp *http.Response
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.Call(&opts) resp, err = o.fs.srv.Call(ctx, &opts)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
return return
@ -1517,7 +1518,7 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, size int64,
} }
fs.Debugf(o, "Cancelling multipart upload") fs.Debugf(o, "Cancelling multipart upload")
cancelErr := o.cancelUploadSession(uploadURL) cancelErr := o.cancelUploadSession(ctx, uploadURL)
if cancelErr != nil { if cancelErr != nil {
fs.Logf(o, "Failed to cancel multipart upload: %v", cancelErr) fs.Logf(o, "Failed to cancel multipart upload: %v", cancelErr)
} }
@ -1553,7 +1554,7 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, size int64,
} }
seg := readers.NewRepeatableReader(io.LimitReader(in, n)) seg := readers.NewRepeatableReader(io.LimitReader(in, n))
fs.Debugf(o, "Uploading segment %d/%d size %d", position, size, n) fs.Debugf(o, "Uploading segment %d/%d size %d", position, size, n)
info, err = o.uploadFragment(uploadURL, position, size, seg, n) info, err = o.uploadFragment(ctx, uploadURL, position, size, seg, n)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -1594,7 +1595,7 @@ func (o *Object) uploadSinglepart(ctx context.Context, in io.Reader, size int64,
} }
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.CallJSON(&opts, nil, &info) resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &info)
if apiErr, ok := err.(*api.Error); ok { if apiErr, ok := err.(*api.Error); ok {
if apiErr.ErrorInfo.Code == "nameAlreadyExists" { if apiErr.ErrorInfo.Code == "nameAlreadyExists" {
// Make the error more user-friendly // Make the error more user-friendly
@ -1646,7 +1647,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
// Remove an object // Remove an object
func (o *Object) Remove(ctx context.Context) error { func (o *Object) Remove(ctx context.Context) error {
return o.fs.deleteObject(o.id) return o.fs.deleteObject(ctx, o.id)
} }
// MimeType of an Object if known, "" otherwise // MimeType of an Object if known, "" otherwise

44
backend/opendrive/opendrive.go
View file

@ -161,7 +161,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
Method: "POST", Method: "POST",
Path: "/session/login.json", Path: "/session/login.json",
} }
resp, err = f.srv.CallJSON(&opts, &account, &f.session) resp, err = f.srv.CallJSON(ctx, &opts, &account, &f.session)
return f.shouldRetry(resp, err) return f.shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -246,7 +246,7 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
} }
// deleteObject removes an object by ID // deleteObject removes an object by ID
func (f *Fs) deleteObject(id string) error { func (f *Fs) deleteObject(ctx context.Context, id string) error {
return f.pacer.Call(func() (bool, error) { return f.pacer.Call(func() (bool, error) {
removeDirData := removeFolder{SessionID: f.session.SessionID, FolderID: id} removeDirData := removeFolder{SessionID: f.session.SessionID, FolderID: id}
opts := rest.Opts{ opts := rest.Opts{
@ -254,7 +254,7 @@ func (f *Fs) deleteObject(id string) error {
NoResponse: true, NoResponse: true,
Path: "/folder/remove.json", Path: "/folder/remove.json",
} }
resp, err := f.srv.CallJSON(&opts, &removeDirData, nil) resp, err := f.srv.CallJSON(ctx, &opts, &removeDirData, nil)
return f.shouldRetry(resp, err) return f.shouldRetry(resp, err)
}) })
} }
@ -275,14 +275,14 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
if err != nil { if err != nil {
return err return err
} }
item, err := f.readMetaDataForFolderID(rootID) item, err := f.readMetaDataForFolderID(ctx, rootID)
if err != nil { if err != nil {
return err return err
} }
if check && len(item.Files) != 0 { if check && len(item.Files) != 0 {
return errors.New("folder not empty") return errors.New("folder not empty")
} }
err = f.deleteObject(rootID) err = f.deleteObject(ctx, rootID)
if err != nil { if err != nil {
return err return err
} }
@ -353,7 +353,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
Method: "POST", Method: "POST",
Path: "/file/move_copy.json", Path: "/file/move_copy.json",
} }
resp, err = f.srv.CallJSON(&opts, &copyFileData, &response) resp, err = f.srv.CallJSON(ctx, &opts, &copyFileData, &response)
return f.shouldRetry(resp, err) return f.shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -410,7 +410,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
Method: "POST", Method: "POST",
Path: "/file/move_copy.json", Path: "/file/move_copy.json",
} }
resp, err = f.srv.CallJSON(&opts, &copyFileData, &response) resp, err = f.srv.CallJSON(ctx, &opts, &copyFileData, &response)
return f.shouldRetry(resp, err) return f.shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -509,7 +509,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
Method: "POST", Method: "POST",
Path: "/folder/move_copy.json", Path: "/folder/move_copy.json",
} }
resp, err = f.srv.CallJSON(&opts, &moveFolderData, &response) resp, err = f.srv.CallJSON(ctx, &opts, &moveFolderData, &response)
return f.shouldRetry(resp, err) return f.shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -589,14 +589,14 @@ func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time,
} }
// readMetaDataForPath reads the metadata from the path // readMetaDataForPath reads the metadata from the path
func (f *Fs) readMetaDataForFolderID(id string) (info *FolderList, err error) { func (f *Fs) readMetaDataForFolderID(ctx context.Context, id string) (info *FolderList, err error) {
var resp *http.Response var resp *http.Response
opts := rest.Opts{ opts := rest.Opts{
Method: "GET", Method: "GET",
Path: "/folder/list.json/" + f.session.SessionID + "/" + id, Path: "/folder/list.json/" + f.session.SessionID + "/" + id,
} }
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(&opts, nil, &info) resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
return f.shouldRetry(resp, err) return f.shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -641,7 +641,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
Method: "POST", Method: "POST",
Path: "/upload/create_file.json", Path: "/upload/create_file.json",
} }
resp, err = o.fs.srv.CallJSON(&opts, &createFileData, &response) resp, err = o.fs.srv.CallJSON(ctx, &opts, &createFileData, &response)
return o.fs.shouldRetry(resp, err) return o.fs.shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -694,7 +694,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
Method: "POST", Method: "POST",
Path: "/folder.json", Path: "/folder.json",
} }
resp, err = f.srv.CallJSON(&opts, &createDirData, &response) resp, err = f.srv.CallJSON(ctx, &opts, &createDirData, &response)
return f.shouldRetry(resp, err) return f.shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -722,7 +722,7 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin
Method: "GET", Method: "GET",
Path: "/folder/list.json/" + f.session.SessionID + "/" + pathID, Path: "/folder/list.json/" + f.session.SessionID + "/" + pathID,
} }
resp, err = f.srv.CallJSON(&opts, nil, &folderList) resp, err = f.srv.CallJSON(ctx, &opts, nil, &folderList)
return f.shouldRetry(resp, err) return f.shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -769,7 +769,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
} }
folderList := FolderList{} folderList := FolderList{}
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(&opts, nil, &folderList) resp, err = f.srv.CallJSON(ctx, &opts, nil, &folderList)
return f.shouldRetry(resp, err) return f.shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -853,7 +853,7 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
} }
update := modTimeFile{SessionID: o.fs.session.SessionID, FileID: o.id, FileModificationTime: strconv.FormatInt(modTime.Unix(), 10)} update := modTimeFile{SessionID: o.fs.session.SessionID, FileID: o.id, FileModificationTime: strconv.FormatInt(modTime.Unix(), 10)}
err := o.fs.pacer.Call(func() (bool, error) { err := o.fs.pacer.Call(func() (bool, error) {
resp, err := o.fs.srv.CallJSON(&opts, &update, nil) resp, err := o.fs.srv.CallJSON(ctx, &opts, &update, nil)
return o.fs.shouldRetry(resp, err) return o.fs.shouldRetry(resp, err)
}) })
@ -873,7 +873,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
} }
var resp *http.Response var resp *http.Response
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.Call(&opts) resp, err = o.fs.srv.Call(ctx, &opts)
return o.fs.shouldRetry(resp, err) return o.fs.shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -892,7 +892,7 @@ func (o *Object) Remove(ctx context.Context) error {
NoResponse: true, NoResponse: true,
Path: "/file.json/" + o.fs.session.SessionID + "/" + o.id, Path: "/file.json/" + o.fs.session.SessionID + "/" + o.id,
} }
resp, err := o.fs.srv.Call(&opts) resp, err := o.fs.srv.Call(ctx, &opts)
return o.fs.shouldRetry(resp, err) return o.fs.shouldRetry(resp, err)
}) })
} }
@ -920,7 +920,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
Method: "POST", Method: "POST",
Path: "/upload/open_file_upload.json", Path: "/upload/open_file_upload.json",
} }
resp, err := o.fs.srv.CallJSON(&opts, &openUploadData, &openResponse) resp, err := o.fs.srv.CallJSON(ctx, &opts, &openUploadData, &openResponse)
return o.fs.shouldRetry(resp, err) return o.fs.shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -966,7 +966,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
MultipartFileName: o.remote, // ..name of the file for the attached file MultipartFileName: o.remote, // ..name of the file for the attached file
} }
resp, err = o.fs.srv.CallJSON(&opts, nil, &reply) resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &reply)
return o.fs.shouldRetry(resp, err) return o.fs.shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -989,7 +989,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
Method: "POST", Method: "POST",
Path: "/upload/close_file_upload.json", Path: "/upload/close_file_upload.json",
} }
resp, err = o.fs.srv.CallJSON(&opts, &closeUploadData, &closeResponse) resp, err = o.fs.srv.CallJSON(ctx, &opts, &closeUploadData, &closeResponse)
return o.fs.shouldRetry(resp, err) return o.fs.shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -1015,7 +1015,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
NoResponse: true, NoResponse: true,
Path: "/file/access.json", Path: "/file/access.json",
} }
resp, err = o.fs.srv.CallJSON(&opts, &update, nil) resp, err = o.fs.srv.CallJSON(ctx, &opts, &update, nil)
return o.fs.shouldRetry(resp, err) return o.fs.shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -1040,7 +1040,7 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
Method: "GET", Method: "GET",
Path: "/folder/itembyname.json/" + o.fs.session.SessionID + "/" + directoryID + "?name=" + url.QueryEscape(replaceReservedChars(leaf)), Path: "/folder/itembyname.json/" + o.fs.session.SessionID + "/" + directoryID + "?name=" + url.QueryEscape(replaceReservedChars(leaf)),
} }
resp, err = o.fs.srv.CallJSON(&opts, nil, &folderList) resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &folderList)
return o.fs.shouldRetry(resp, err) return o.fs.shouldRetry(resp, err)
}) })
if err != nil { if err != nil {

38
backend/pcloud/pcloud.go
View file

@ -201,7 +201,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.It
return nil, err return nil, err
} }
found, err := f.listAll(directoryID, false, true, func(item *api.Item) bool { found, err := f.listAll(ctx, directoryID, false, true, func(item *api.Item) bool {
if item.Name == leaf { if item.Name == leaf {
info = item info = item
return true return true
@ -334,7 +334,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
// FindLeaf finds a directory of name leaf in the folder with ID pathID // FindLeaf finds a directory of name leaf in the folder with ID pathID
func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) { func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) {
// Find the leaf in pathID // Find the leaf in pathID
found, err = f.listAll(pathID, true, false, func(item *api.Item) bool { found, err = f.listAll(ctx, pathID, true, false, func(item *api.Item) bool {
if item.Name == leaf { if item.Name == leaf {
pathIDOut = item.ID pathIDOut = item.ID
return true return true
@ -357,7 +357,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
opts.Parameters.Set("name", replaceReservedChars(leaf)) opts.Parameters.Set("name", replaceReservedChars(leaf))
opts.Parameters.Set("folderid", dirIDtoNumber(pathID)) opts.Parameters.Set("folderid", dirIDtoNumber(pathID))
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(&opts, nil, &result) resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
err = result.Error.Update(err) err = result.Error.Update(err)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
@ -400,7 +400,7 @@ type listAllFn func(*api.Item) bool
// Lists the directory required calling the user function on each item found // Lists the directory required calling the user function on each item found
// //
// If the user fn ever returns true then it early exits with found = true // If the user fn ever returns true then it early exits with found = true
func (f *Fs) listAll(dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) { func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) {
opts := rest.Opts{ opts := rest.Opts{
Method: "GET", Method: "GET",
Path: "/listfolder", Path: "/listfolder",
@ -412,7 +412,7 @@ func (f *Fs) listAll(dirID string, directoriesOnly bool, filesOnly bool, fn list
var result api.ItemResult var result api.ItemResult
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(&opts, nil, &result) resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
err = result.Error.Update(err) err = result.Error.Update(err)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
@ -458,7 +458,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
return nil, err return nil, err
} }
var iErr error var iErr error
_, err = f.listAll(directoryID, false, false, func(info *api.Item) bool { _, err = f.listAll(ctx, directoryID, false, false, func(info *api.Item) bool {
remote := path.Join(dir, info.Name) remote := path.Join(dir, info.Name)
if info.IsFolder { if info.IsFolder {
// cache the directory ID for later lookups // cache the directory ID for later lookups
@ -563,7 +563,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
var resp *http.Response var resp *http.Response
var result api.ItemResult var result api.ItemResult
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(&opts, nil, &result) resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
err = result.Error.Update(err) err = result.Error.Update(err)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
@ -628,7 +628,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
var resp *http.Response var resp *http.Response
var result api.ItemResult var result api.ItemResult
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(&opts, nil, &result) resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
err = result.Error.Update(err) err = result.Error.Update(err)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
@ -666,7 +666,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
var resp *http.Response var resp *http.Response
var result api.Error var result api.Error
return f.pacer.Call(func() (bool, error) { return f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(&opts, nil, &result) resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
err = result.Update(err) err = result.Update(err)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
@ -706,7 +706,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
var resp *http.Response var resp *http.Response
var result api.ItemResult var result api.ItemResult
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(&opts, nil, &result) resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
err = result.Error.Update(err) err = result.Error.Update(err)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
@ -803,7 +803,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
var resp *http.Response var resp *http.Response
var result api.ItemResult var result api.ItemResult
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(&opts, nil, &result) resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
err = result.Error.Update(err) err = result.Error.Update(err)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
@ -830,7 +830,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
var resp *http.Response var resp *http.Response
var q api.UserInfo var q api.UserInfo
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(&opts, nil, &q) resp, err = f.srv.CallJSON(ctx, &opts, nil, &q)
err = q.Error.Update(err) err = q.Error.Update(err)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
@ -881,7 +881,7 @@ func (o *Object) getHashes(ctx context.Context) (err error) {
} }
opts.Parameters.Set("fileid", fileIDtoNumber(o.id)) opts.Parameters.Set("fileid", fileIDtoNumber(o.id))
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.CallJSON(&opts, nil, &result) resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &result)
err = result.Error.Update(err) err = result.Error.Update(err)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
@ -984,7 +984,7 @@ func (o *Object) Storable() bool {
} }
// downloadURL fetches the download link // downloadURL fetches the download link
func (o *Object) downloadURL() (URL string, err error) { func (o *Object) downloadURL(ctx context.Context) (URL string, err error) {
if o.id == "" { if o.id == "" {
return "", errors.New("can't download - no id") return "", errors.New("can't download - no id")
} }
@ -1000,7 +1000,7 @@ func (o *Object) downloadURL() (URL string, err error) {
} }
opts.Parameters.Set("fileid", fileIDtoNumber(o.id)) opts.Parameters.Set("fileid", fileIDtoNumber(o.id))
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.CallJSON(&opts, nil, &result) resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &result)
err = result.Error.Update(err) err = result.Error.Update(err)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
@ -1016,7 +1016,7 @@ func (o *Object) downloadURL() (URL string, err error) {
// Open an object for read // Open an object for read
func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) { func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
url, err := o.downloadURL() url, err := o.downloadURL(ctx)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -1027,7 +1027,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
Options: options, Options: options,
} }
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.Call(&opts) resp, err = o.fs.srv.Call(ctx, &opts)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -1104,7 +1104,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
} }
err = o.fs.pacer.CallNoRetry(func() (bool, error) { err = o.fs.pacer.CallNoRetry(func() (bool, error) {
resp, err = o.fs.srv.CallJSON(&opts, nil, &result) resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &result)
err = result.Error.Update(err) err = result.Error.Update(err)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
@ -1134,7 +1134,7 @@ func (o *Object) Remove(ctx context.Context) error {
var result api.ItemResult var result api.ItemResult
opts.Parameters.Set("fileid", fileIDtoNumber(o.id)) opts.Parameters.Set("fileid", fileIDtoNumber(o.id))
return o.fs.pacer.Call(func() (bool, error) { return o.fs.pacer.Call(func() (bool, error) {
resp, err := o.fs.srv.CallJSON(&opts, nil, &result) resp, err := o.fs.srv.CallJSON(ctx, &opts, nil, &result)
err = result.Error.Update(err) err = result.Error.Update(err)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })

30
backend/premiumizeme/premiumizeme.go
View file

@ -200,7 +200,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string, directoriesOn
} }
lcLeaf := strings.ToLower(leaf) lcLeaf := strings.ToLower(leaf)
found, err := f.listAll(directoryID, directoriesOnly, filesOnly, func(item *api.Item) bool { found, err := f.listAll(ctx, directoryID, directoriesOnly, filesOnly, func(item *api.Item) bool {
if strings.ToLower(item.Name) == lcLeaf { if strings.ToLower(item.Name) == lcLeaf {
info = item info = item
return true return true
@ -361,7 +361,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
// FindLeaf finds a directory of name leaf in the folder with ID pathID // FindLeaf finds a directory of name leaf in the folder with ID pathID
func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) { func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) {
// Find the leaf in pathID // Find the leaf in pathID
found, err = f.listAll(pathID, true, false, func(item *api.Item) bool { found, err = f.listAll(ctx, pathID, true, false, func(item *api.Item) bool {
if item.Name == leaf { if item.Name == leaf {
pathIDOut = item.ID pathIDOut = item.ID
return true return true
@ -386,7 +386,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
}, },
} }
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(&opts, nil, &info) resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -411,7 +411,7 @@ type listAllFn func(*api.Item) bool
// Lists the directory required calling the user function on each item found // Lists the directory required calling the user function on each item found
// //
// If the user fn ever returns true then it early exits with found = true // If the user fn ever returns true then it early exits with found = true
func (f *Fs) listAll(dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) { func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) {
opts := rest.Opts{ opts := rest.Opts{
Method: "GET", Method: "GET",
Path: "/folder/list", Path: "/folder/list",
@ -423,7 +423,7 @@ func (f *Fs) listAll(dirID string, directoriesOnly bool, filesOnly bool, fn list
var result api.FolderListResponse var result api.FolderListResponse
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(&opts, nil, &result) resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -475,7 +475,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
return nil, err return nil, err
} }
var iErr error var iErr error
_, err = f.listAll(directoryID, false, false, func(info *api.Item) bool { _, err = f.listAll(ctx, directoryID, false, false, func(info *api.Item) bool {
remote := path.Join(dir, info.Name) remote := path.Join(dir, info.Name)
if info.Type == api.ItemTypeFolder { if info.Type == api.ItemTypeFolder {
// cache the directory ID for later lookups // cache the directory ID for later lookups
@ -589,7 +589,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
// need to check if empty as it will delete recursively by default // need to check if empty as it will delete recursively by default
if check { if check {
found, err := f.listAll(rootID, false, false, func(item *api.Item) bool { found, err := f.listAll(ctx, rootID, false, false, func(item *api.Item) bool {
return true return true
}) })
if err != nil { if err != nil {
@ -611,7 +611,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
var resp *http.Response var resp *http.Response
var result api.Response var result api.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(&opts, nil, &result) resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -690,7 +690,7 @@ func (f *Fs) move(ctx context.Context, isFile bool, id, oldLeaf, newLeaf, oldDir
var resp *http.Response var resp *http.Response
var result api.Response var result api.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(&opts, nil, &result) resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -860,7 +860,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
Parameters: f.baseParams(), Parameters: f.baseParams(),
} }
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(&opts, nil, &info) resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -992,7 +992,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
Options: options, Options: options,
} }
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.Call(&opts) resp, err = o.fs.srv.Call(ctx, &opts)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -1036,7 +1036,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
}, },
} }
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.CallJSON(&opts, nil, &info) resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &info)
if err != nil { if err != nil {
return shouldRetry(resp, err) return shouldRetry(resp, err)
} }
@ -1096,7 +1096,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
} }
var result api.Response var result api.Response
err = o.fs.pacer.CallNoRetry(func() (bool, error) { err = o.fs.pacer.CallNoRetry(func() (bool, error) {
resp, err = o.fs.srv.CallJSON(&opts, nil, &result) resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &result)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -1138,7 +1138,7 @@ func (f *Fs) renameLeaf(ctx context.Context, isFile bool, id string, newLeaf str
var resp *http.Response var resp *http.Response
var result api.Response var result api.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(&opts, nil, &result) resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -1163,7 +1163,7 @@ func (f *Fs) remove(ctx context.Context, id string) (err error) {
var resp *http.Response var resp *http.Response
var result api.Response var result api.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(&opts, nil, &result) resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {

69
backend/webdav/webdav.go
View file

@ -205,7 +205,7 @@ func itemIsDir(item *api.Response) bool {
} }
// readMetaDataForPath reads the metadata from the path // readMetaDataForPath reads the metadata from the path
func (f *Fs) readMetaDataForPath(path string, depth string) (info *api.Prop, err error) { func (f *Fs) readMetaDataForPath(ctx context.Context, path string, depth string) (info *api.Prop, err error) {
// FIXME how do we read back additional properties? // FIXME how do we read back additional properties?
opts := rest.Opts{ opts := rest.Opts{
Method: "PROPFIND", Method: "PROPFIND",
@ -221,7 +221,7 @@ func (f *Fs) readMetaDataForPath(path string, depth string) (info *api.Prop, err
var result api.Multistatus var result api.Multistatus
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallXML(&opts, nil, &result) resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
return f.shouldRetry(resp, err) return f.shouldRetry(resp, err)
}) })
if apiErr, ok := err.(*api.Error); ok { if apiErr, ok := err.(*api.Error); ok {
@ -229,7 +229,7 @@ func (f *Fs) readMetaDataForPath(path string, depth string) (info *api.Prop, err
switch apiErr.StatusCode { switch apiErr.StatusCode {
case http.StatusNotFound: case http.StatusNotFound:
if f.retryWithZeroDepth && depth != "0" { if f.retryWithZeroDepth && depth != "0" {
return f.readMetaDataForPath(path, "0") return f.readMetaDataForPath(ctx, path, "0")
} }
return nil, fs.ErrorObjectNotFound return nil, fs.ErrorObjectNotFound
case http.StatusMovedPermanently, http.StatusFound, http.StatusSeeOther: case http.StatusMovedPermanently, http.StatusFound, http.StatusSeeOther:
@ -477,7 +477,7 @@ func (f *Fs) setQuirks(vendor string) error {
// Return an Object from a path // Return an Object from a path
// //
// If it can't be found it returns the error fs.ErrorObjectNotFound. // If it can't be found it returns the error fs.ErrorObjectNotFound.
func (f *Fs) newObjectWithInfo(remote string, info *api.Prop) (fs.Object, error) { func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *api.Prop) (fs.Object, error) {
o := &Object{ o := &Object{
fs: f, fs: f,
remote: remote, remote: remote,
@ -487,7 +487,7 @@ func (f *Fs) newObjectWithInfo(remote string, info *api.Prop) (fs.Object, error)
// Set info // Set info
err = o.setMetaData(info) err = o.setMetaData(info)
} else { } else {
err = o.readMetaData() // reads info and meta, returning an error err = o.readMetaData(ctx) // reads info and meta, returning an error
} }
if err != nil { if err != nil {
return nil, err return nil, err
@ -498,7 +498,7 @@ func (f *Fs) newObjectWithInfo(remote string, info *api.Prop) (fs.Object, error)
// NewObject finds the Object at remote. If it can't be found // NewObject finds the Object at remote. If it can't be found
// it returns the error fs.ErrorObjectNotFound. // it returns the error fs.ErrorObjectNotFound.
func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
return f.newObjectWithInfo(remote, nil) return f.newObjectWithInfo(ctx, remote, nil)
} }
// Read the normal props, plus the checksums // Read the normal props, plus the checksums
@ -528,7 +528,7 @@ type listAllFn func(string, bool, *api.Prop) bool
// Lists the directory required calling the user function on each item found // Lists the directory required calling the user function on each item found
// //
// If the user fn ever returns true then it early exits with found = true // If the user fn ever returns true then it early exits with found = true
func (f *Fs) listAll(dir string, directoriesOnly bool, filesOnly bool, depth string, fn listAllFn) (found bool, err error) { func (f *Fs) listAll(ctx context.Context, dir string, directoriesOnly bool, filesOnly bool, depth string, fn listAllFn) (found bool, err error) {
opts := rest.Opts{ opts := rest.Opts{
Method: "PROPFIND", Method: "PROPFIND",
Path: f.dirPath(dir), // FIXME Should not start with / Path: f.dirPath(dir), // FIXME Should not start with /
@ -542,7 +542,7 @@ func (f *Fs) listAll(dir string, directoriesOnly bool, filesOnly bool, depth str
var result api.Multistatus var result api.Multistatus
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallXML(&opts, nil, &result) resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
return f.shouldRetry(resp, err) return f.shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -550,7 +550,7 @@ func (f *Fs) listAll(dir string, directoriesOnly bool, filesOnly bool, depth str
// does not exist // does not exist
if apiErr.StatusCode == http.StatusNotFound { if apiErr.StatusCode == http.StatusNotFound {
if f.retryWithZeroDepth && depth != "0" { if f.retryWithZeroDepth && depth != "0" {
return f.listAll(dir, directoriesOnly, filesOnly, "0", fn) return f.listAll(ctx, dir, directoriesOnly, filesOnly, "0", fn)
} }
return found, fs.ErrorDirNotFound return found, fs.ErrorDirNotFound
} }
@ -625,14 +625,14 @@ func (f *Fs) listAll(dir string, directoriesOnly bool, filesOnly bool, depth str
// found. // found.
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) { func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
var iErr error var iErr error
_, err = f.listAll(dir, false, false, defaultDepth, func(remote string, isDir bool, info *api.Prop) bool { _, err = f.listAll(ctx, dir, false, false, defaultDepth, func(remote string, isDir bool, info *api.Prop) bool {
if isDir { if isDir {
d := fs.NewDir(remote, time.Time(info.Modified)) d := fs.NewDir(remote, time.Time(info.Modified))
// .SetID(info.ID) // .SetID(info.ID)
// FIXME more info from dir? can set size, items? // FIXME more info from dir? can set size, items?
entries = append(entries, d) entries = append(entries, d)
} else { } else {
o, err := f.newObjectWithInfo(remote, info) o, err := f.newObjectWithInfo(ctx, remote, info)
if err != nil { if err != nil {
iErr = err iErr = err
return true return true
@ -696,7 +696,7 @@ func (f *Fs) mkParentDir(ctx context.Context, dirPath string) error {
} }
// low level mkdir, only makes the directory, doesn't attempt to create parents // low level mkdir, only makes the directory, doesn't attempt to create parents
func (f *Fs) _mkdir(dirPath string) error { func (f *Fs) _mkdir(ctx context.Context, dirPath string) error {
// We assume the root is already created // We assume the root is already created
if dirPath == "" { if dirPath == "" {
return nil return nil
@ -711,7 +711,7 @@ func (f *Fs) _mkdir(dirPath string) error {
NoResponse: true, NoResponse: true,
} }
err := f.pacer.Call(func() (bool, error) { err := f.pacer.Call(func() (bool, error) {
resp, err := f.srv.Call(&opts) resp, err := f.srv.Call(ctx, &opts)
return f.shouldRetry(resp, err) return f.shouldRetry(resp, err)
}) })
if apiErr, ok := err.(*api.Error); ok { if apiErr, ok := err.(*api.Error); ok {
@ -727,13 +727,13 @@ func (f *Fs) _mkdir(dirPath string) error {
// mkdir makes the directory and parents using native paths // mkdir makes the directory and parents using native paths
func (f *Fs) mkdir(ctx context.Context, dirPath string) error { func (f *Fs) mkdir(ctx context.Context, dirPath string) error {
// defer log.Trace(dirPath, "")("") // defer log.Trace(dirPath, "")("")
err := f._mkdir(dirPath) err := f._mkdir(ctx, dirPath)
if apiErr, ok := err.(*api.Error); ok { if apiErr, ok := err.(*api.Error); ok {
// parent does not exist so create it first then try again // parent does not exist so create it first then try again
if apiErr.StatusCode == http.StatusConflict { if apiErr.StatusCode == http.StatusConflict {
err = f.mkParentDir(ctx, dirPath) err = f.mkParentDir(ctx, dirPath)
if err == nil { if err == nil {
err = f._mkdir(dirPath) err = f._mkdir(ctx, dirPath)
} }
} }
} }
@ -749,17 +749,17 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
// dirNotEmpty returns true if the directory exists and is not Empty // dirNotEmpty returns true if the directory exists and is not Empty
// //
// if the directory does not exist then err will be ErrorDirNotFound // if the directory does not exist then err will be ErrorDirNotFound
func (f *Fs) dirNotEmpty(dir string) (found bool, err error) { func (f *Fs) dirNotEmpty(ctx context.Context, dir string) (found bool, err error) {
return f.listAll(dir, false, false, defaultDepth, func(remote string, isDir bool, info *api.Prop) bool { return f.listAll(ctx, dir, false, false, defaultDepth, func(remote string, isDir bool, info *api.Prop) bool {
return true return true
}) })
} }
// purgeCheck removes the root directory, if check is set then it // purgeCheck removes the root directory, if check is set then it
// refuses to do so if it has anything in // refuses to do so if it has anything in
func (f *Fs) purgeCheck(dir string, check bool) error { func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
if check { if check {
notEmpty, err := f.dirNotEmpty(dir) notEmpty, err := f.dirNotEmpty(ctx, dir)
if err != nil { if err != nil {
return err return err
} }
@ -775,7 +775,7 @@ func (f *Fs) purgeCheck(dir string, check bool) error {
var resp *http.Response var resp *http.Response
var err error var err error
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallXML(&opts, nil, nil) resp, err = f.srv.CallXML(ctx, &opts, nil, nil)
return f.shouldRetry(resp, err) return f.shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -789,7 +789,7 @@ func (f *Fs) purgeCheck(dir string, check bool) error {
// //
// Returns an error if it isn't empty // Returns an error if it isn't empty
func (f *Fs) Rmdir(ctx context.Context, dir string) error { func (f *Fs) Rmdir(ctx context.Context, dir string) error {
return f.purgeCheck(dir, true) return f.purgeCheck(ctx, dir, true)
} }
// Precision return the precision of this Fs // Precision return the precision of this Fs
@ -838,7 +838,7 @@ func (f *Fs) copyOrMove(ctx context.Context, src fs.Object, remote string, metho
opts.ExtraHeaders["X-OC-Mtime"] = fmt.Sprintf("%f", float64(src.ModTime(ctx).UnixNano())/1e9) opts.ExtraHeaders["X-OC-Mtime"] = fmt.Sprintf("%f", float64(src.ModTime(ctx).UnixNano())/1e9)
} }
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.Call(&opts) resp, err = f.srv.Call(ctx, &opts)
return f.shouldRetry(resp, err) return f.shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -870,7 +870,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
// deleting all the files quicker than just running Remove() on the // deleting all the files quicker than just running Remove() on the
// result of List() // result of List()
func (f *Fs) Purge(ctx context.Context) error { func (f *Fs) Purge(ctx context.Context) error {
return f.purgeCheck("", false) return f.purgeCheck(ctx, "", false)
} }
// Move src to this remote using server side move operations. // Move src to this remote using server side move operations.
@ -904,7 +904,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
dstPath := f.filePath(dstRemote) dstPath := f.filePath(dstRemote)
// Check if destination exists // Check if destination exists
_, err := f.dirNotEmpty(dstRemote) _, err := f.dirNotEmpty(ctx, dstRemote)
if err == nil { if err == nil {
return fs.ErrorDirExists return fs.ErrorDirExists
} }
@ -934,7 +934,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
}, },
} }
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.Call(&opts) resp, err = f.srv.Call(ctx, &opts)
return f.shouldRetry(resp, err) return f.shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -975,7 +975,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
var resp *http.Response var resp *http.Response
var err error var err error
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallXML(&opts, nil, &q) resp, err = f.srv.CallXML(ctx, &opts, nil, &q)
return f.shouldRetry(resp, err) return f.shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -1028,7 +1028,8 @@ func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
// Size returns the size of an object in bytes // Size returns the size of an object in bytes
func (o *Object) Size() int64 { func (o *Object) Size() int64 {
err := o.readMetaData() ctx := context.TODO()
err := o.readMetaData(ctx)
if err != nil { if err != nil {
fs.Logf(o, "Failed to read metadata: %v", err) fs.Logf(o, "Failed to read metadata: %v", err)
return 0 return 0
@ -1052,11 +1053,11 @@ func (o *Object) setMetaData(info *api.Prop) (err error) {
// readMetaData gets the metadata if it hasn't already been fetched // readMetaData gets the metadata if it hasn't already been fetched
// //
// it also sets the info // it also sets the info
func (o *Object) readMetaData() (err error) { func (o *Object) readMetaData(ctx context.Context) (err error) {
if o.hasMetaData { if o.hasMetaData {
return nil return nil
} }
info, err := o.fs.readMetaDataForPath(o.remote, defaultDepth) info, err := o.fs.readMetaDataForPath(ctx, o.remote, defaultDepth)
if err != nil { if err != nil {
return err return err
} }
@ -1068,7 +1069,7 @@ func (o *Object) readMetaData() (err error) {
// It attempts to read the objects mtime and if that isn't present the // It attempts to read the objects mtime and if that isn't present the
// LastModified returned in the http headers // LastModified returned in the http headers
func (o *Object) ModTime(ctx context.Context) time.Time { func (o *Object) ModTime(ctx context.Context) time.Time {
err := o.readMetaData() err := o.readMetaData(ctx)
if err != nil { if err != nil {
fs.Logf(o, "Failed to read metadata: %v", err) fs.Logf(o, "Failed to read metadata: %v", err)
return time.Now() return time.Now()
@ -1095,7 +1096,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
Options: options, Options: options,
} }
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.Call(&opts) resp, err = o.fs.srv.Call(ctx, &opts)
return o.fs.shouldRetry(resp, err) return o.fs.shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -1143,7 +1144,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
} }
} }
err = o.fs.pacer.CallNoRetry(func() (bool, error) { err = o.fs.pacer.CallNoRetry(func() (bool, error) {
resp, err = o.fs.srv.Call(&opts) resp, err = o.fs.srv.Call(ctx, &opts)
return o.fs.shouldRetry(resp, err) return o.fs.shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -1159,7 +1160,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
} }
// read metadata from remote // read metadata from remote
o.hasMetaData = false o.hasMetaData = false
return o.readMetaData() return o.readMetaData(ctx)
} }
// Remove an object // Remove an object
@ -1170,7 +1171,7 @@ func (o *Object) Remove(ctx context.Context) error {
NoResponse: true, NoResponse: true,
} }
return o.fs.pacer.Call(func() (bool, error) { return o.fs.pacer.Call(func() (bool, error) {
resp, err := o.fs.srv.Call(&opts) resp, err := o.fs.srv.Call(ctx, &opts)
return o.fs.shouldRetry(resp, err) return o.fs.shouldRetry(resp, err)
}) })
} }

116
backend/yandex/yandex.go
View file

@ -200,7 +200,7 @@ func (f *Fs) dirPath(file string) string {
return path.Join(f.diskRoot, file) + "/" return path.Join(f.diskRoot, file) + "/"
} }
func (f *Fs) readMetaDataForPath(path string, options *api.ResourceInfoRequestOptions) (*api.ResourceInfoResponse, error) { func (f *Fs) readMetaDataForPath(ctx context.Context, path string, options *api.ResourceInfoRequestOptions) (*api.ResourceInfoResponse, error) {
opts := rest.Opts{ opts := rest.Opts{
Method: "GET", Method: "GET",
Path: "/resources", Path: "/resources",
@ -226,7 +226,7 @@ func (f *Fs) readMetaDataForPath(path string, options *api.ResourceInfoRequestOp
var info api.ResourceInfoResponse var info api.ResourceInfoResponse
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(&opts, nil, &info) resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
@ -239,6 +239,7 @@ func (f *Fs) readMetaDataForPath(path string, options *api.ResourceInfoRequestOp
// NewFs constructs an Fs from the path, container:path // NewFs constructs an Fs from the path, container:path
func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
ctx := context.TODO()
// Parse config into Options struct // Parse config into Options struct
opt := new(Options) opt := new(Options)
err := configstruct.Set(m, opt) err := configstruct.Set(m, opt)
@ -284,7 +285,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
//request object meta info //request object meta info
// Check to see if the object exists and is a file // Check to see if the object exists and is a file
//request object meta info //request object meta info
if info, err := f.readMetaDataForPath(f.diskRoot, &api.ResourceInfoRequestOptions{}); err != nil { if info, err := f.readMetaDataForPath(ctx, f.diskRoot, &api.ResourceInfoRequestOptions{}); err != nil {
} else { } else {
if info.ResourceType == "file" { if info.ResourceType == "file" {
@ -301,7 +302,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
} }
// Convert a list item into a DirEntry // Convert a list item into a DirEntry
func (f *Fs) itemToDirEntry(remote string, object *api.ResourceInfoResponse) (fs.DirEntry, error) { func (f *Fs) itemToDirEntry(ctx context.Context, remote string, object *api.ResourceInfoResponse) (fs.DirEntry, error) {
switch object.ResourceType { switch object.ResourceType {
case "dir": case "dir":
t, err := time.Parse(time.RFC3339Nano, object.Modified) t, err := time.Parse(time.RFC3339Nano, object.Modified)
@ -311,7 +312,7 @@ func (f *Fs) itemToDirEntry(remote string, object *api.ResourceInfoResponse) (fs
d := fs.NewDir(remote, t).SetSize(object.Size) d := fs.NewDir(remote, t).SetSize(object.Size)
return d, nil return d, nil
case "file": case "file":
o, err := f.newObjectWithInfo(remote, object) o, err := f.newObjectWithInfo(ctx, remote, object)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -343,7 +344,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
Limit: limit, Limit: limit,
Offset: offset, Offset: offset,
} }
info, err := f.readMetaDataForPath(root, opts) info, err := f.readMetaDataForPath(ctx, root, opts)
if err != nil { if err != nil {
if apiErr, ok := err.(*api.ErrorResponse); ok { if apiErr, ok := err.(*api.ErrorResponse); ok {
@ -360,7 +361,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
//list all subdirs //list all subdirs
for _, element := range info.Embedded.Items { for _, element := range info.Embedded.Items {
remote := path.Join(dir, element.Name) remote := path.Join(dir, element.Name)
entry, err := f.itemToDirEntry(remote, &element) entry, err := f.itemToDirEntry(ctx, remote, &element)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -386,7 +387,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
// Return an Object from a path // Return an Object from a path
// //
// If it can't be found it returns the error fs.ErrorObjectNotFound. // If it can't be found it returns the error fs.ErrorObjectNotFound.
func (f *Fs) newObjectWithInfo(remote string, info *api.ResourceInfoResponse) (fs.Object, error) { func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *api.ResourceInfoResponse) (fs.Object, error) {
o := &Object{ o := &Object{
fs: f, fs: f,
remote: remote, remote: remote,
@ -395,7 +396,7 @@ func (f *Fs) newObjectWithInfo(remote string, info *api.ResourceInfoResponse) (f
if info != nil { if info != nil {
err = o.setMetaData(info) err = o.setMetaData(info)
} else { } else {
err = o.readMetaData() err = o.readMetaData(ctx)
if apiErr, ok := err.(*api.ErrorResponse); ok { if apiErr, ok := err.(*api.ErrorResponse); ok {
// does not exist // does not exist
if apiErr.ErrorName == "DiskNotFoundError" { if apiErr.ErrorName == "DiskNotFoundError" {
@ -412,7 +413,7 @@ func (f *Fs) newObjectWithInfo(remote string, info *api.ResourceInfoResponse) (f
// NewObject finds the Object at remote. If it can't be found it // NewObject finds the Object at remote. If it can't be found it
// returns the error fs.ErrorObjectNotFound. // returns the error fs.ErrorObjectNotFound.
func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
return f.newObjectWithInfo(remote, nil) return f.newObjectWithInfo(ctx, remote, nil)
} }
// Creates from the parameters passed in a half finished Object which // Creates from the parameters passed in a half finished Object which
@ -446,7 +447,7 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
} }
// CreateDir makes a directory // CreateDir makes a directory
func (f *Fs) CreateDir(path string) (err error) { func (f *Fs) CreateDir(ctx context.Context, path string) (err error) {
//fmt.Printf("CreateDir: %s\n", path) //fmt.Printf("CreateDir: %s\n", path)
var resp *http.Response var resp *http.Response
@ -460,7 +461,7 @@ func (f *Fs) CreateDir(path string) (err error) {
opts.Parameters.Set("path", path) opts.Parameters.Set("path", path)
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.Call(&opts) resp, err = f.srv.Call(ctx, &opts)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -474,7 +475,7 @@ func (f *Fs) CreateDir(path string) (err error) {
// This really needs improvement and especially proper error checking // This really needs improvement and especially proper error checking
// but Yandex does not publish a List of possible errors and when they're // but Yandex does not publish a List of possible errors and when they're
// expected to occur. // expected to occur.
func (f *Fs) mkDirs(path string) (err error) { func (f *Fs) mkDirs(ctx context.Context, path string) (err error) {
//trim filename from path //trim filename from path
//dirString := strings.TrimSuffix(path, filepath.Base(path)) //dirString := strings.TrimSuffix(path, filepath.Base(path))
//trim "disk:" from path //trim "disk:" from path
@ -483,7 +484,7 @@ func (f *Fs) mkDirs(path string) (err error) {
return nil return nil
} }
if err = f.CreateDir(dirString); err != nil { if err = f.CreateDir(ctx, dirString); err != nil {
if apiErr, ok := err.(*api.ErrorResponse); ok { if apiErr, ok := err.(*api.ErrorResponse); ok {
// allready exists // allready exists
if apiErr.ErrorName != "DiskPathPointsToExistentDirectoryError" { if apiErr.ErrorName != "DiskPathPointsToExistentDirectoryError" {
@ -493,7 +494,7 @@ func (f *Fs) mkDirs(path string) (err error) {
for _, element := range dirs { for _, element := range dirs {
if element != "" { if element != "" {
mkdirpath += element + "/" //path separator / mkdirpath += element + "/" //path separator /
if err = f.CreateDir(mkdirpath); err != nil { if err = f.CreateDir(ctx, mkdirpath); err != nil {
// ignore errors while creating dirs // ignore errors while creating dirs
} }
} }
@ -505,7 +506,7 @@ func (f *Fs) mkDirs(path string) (err error) {
return err return err
} }
func (f *Fs) mkParentDirs(resPath string) error { func (f *Fs) mkParentDirs(ctx context.Context, resPath string) error {
// defer log.Trace(dirPath, "")("") // defer log.Trace(dirPath, "")("")
// chop off trailing / if it exists // chop off trailing / if it exists
if strings.HasSuffix(resPath, "/") { if strings.HasSuffix(resPath, "/") {
@ -515,17 +516,17 @@ func (f *Fs) mkParentDirs(resPath string) error {
if parent == "." { if parent == "." {
parent = "" parent = ""
} }
return f.mkDirs(parent) return f.mkDirs(ctx, parent)
} }
// Mkdir creates the container if it doesn't exist // Mkdir creates the container if it doesn't exist
func (f *Fs) Mkdir(ctx context.Context, dir string) error { func (f *Fs) Mkdir(ctx context.Context, dir string) error {
path := f.filePath(dir) path := f.filePath(dir)
return f.mkDirs(path) return f.mkDirs(ctx, path)
} }
// waitForJob waits for the job with status in url to complete // waitForJob waits for the job with status in url to complete
func (f *Fs) waitForJob(location string) (err error) { func (f *Fs) waitForJob(ctx context.Context, location string) (err error) {
opts := rest.Opts{ opts := rest.Opts{
RootURL: location, RootURL: location,
Method: "GET", Method: "GET",
@ -535,7 +536,7 @@ func (f *Fs) waitForJob(location string) (err error) {
var resp *http.Response var resp *http.Response
var body []byte var body []byte
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.Call(&opts) resp, err = f.srv.Call(ctx, &opts)
if err != nil { if err != nil {
return fserrors.ShouldRetry(err), err return fserrors.ShouldRetry(err), err
} }
@ -564,7 +565,7 @@ func (f *Fs) waitForJob(location string) (err error) {
return errors.Errorf("async operation didn't complete after %v", fs.Config.Timeout) return errors.Errorf("async operation didn't complete after %v", fs.Config.Timeout)
} }
func (f *Fs) delete(path string, hardDelete bool) (err error) { func (f *Fs) delete(ctx context.Context, path string, hardDelete bool) (err error) {
opts := rest.Opts{ opts := rest.Opts{
Method: "DELETE", Method: "DELETE",
Path: "/resources", Path: "/resources",
@ -577,7 +578,7 @@ func (f *Fs) delete(path string, hardDelete bool) (err error) {
var resp *http.Response var resp *http.Response
var body []byte var body []byte
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.Call(&opts) resp, err = f.srv.Call(ctx, &opts)
if err != nil { if err != nil {
return fserrors.ShouldRetry(err), err return fserrors.ShouldRetry(err), err
} }
@ -595,19 +596,19 @@ func (f *Fs) delete(path string, hardDelete bool) (err error) {
if err != nil { if err != nil {
return errors.Wrapf(err, "async info result not JSON: %q", body) return errors.Wrapf(err, "async info result not JSON: %q", body)
} }
return f.waitForJob(info.HRef) return f.waitForJob(ctx, info.HRef)
} }
return nil return nil
} }
// purgeCheck remotes the root directory, if check is set then it // purgeCheck remotes the root directory, if check is set then it
// refuses to do so if it has anything in // refuses to do so if it has anything in
func (f *Fs) purgeCheck(dir string, check bool) error { func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
root := f.filePath(dir) root := f.filePath(dir)
if check { if check {
//to comply with rclone logic we check if the directory is empty before delete. //to comply with rclone logic we check if the directory is empty before delete.
//send request to get list of objects in this directory. //send request to get list of objects in this directory.
info, err := f.readMetaDataForPath(root, &api.ResourceInfoRequestOptions{}) info, err := f.readMetaDataForPath(ctx, root, &api.ResourceInfoRequestOptions{})
if err != nil { if err != nil {
return errors.Wrap(err, "rmdir failed") return errors.Wrap(err, "rmdir failed")
} }
@ -616,14 +617,14 @@ func (f *Fs) purgeCheck(dir string, check bool) error {
} }
} }
//delete directory //delete directory
return f.delete(root, false) return f.delete(ctx, root, false)
} }
// Rmdir deletes the container // Rmdir deletes the container
// //
// Returns an error if it isn't empty // Returns an error if it isn't empty
func (f *Fs) Rmdir(ctx context.Context, dir string) error { func (f *Fs) Rmdir(ctx context.Context, dir string) error {
return f.purgeCheck(dir, true) return f.purgeCheck(ctx, dir, true)
} }
// Purge deletes all the files and the container // Purge deletes all the files and the container
@ -632,11 +633,11 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
// deleting all the files quicker than just running Remove() on the // deleting all the files quicker than just running Remove() on the
// result of List() // result of List()
func (f *Fs) Purge(ctx context.Context) error { func (f *Fs) Purge(ctx context.Context) error {
return f.purgeCheck("", false) return f.purgeCheck(ctx, "", false)
} }
// copyOrMoves copies or moves directories or files depending on the method parameter // copyOrMoves copies or moves directories or files depending on the method parameter
func (f *Fs) copyOrMove(method, src, dst string, overwrite bool) (err error) { func (f *Fs) copyOrMove(ctx context.Context, method, src, dst string, overwrite bool) (err error) {
opts := rest.Opts{ opts := rest.Opts{
Method: "POST", Method: "POST",
Path: "/resources/" + method, Path: "/resources/" + method,
@ -650,7 +651,7 @@ func (f *Fs) copyOrMove(method, src, dst string, overwrite bool) (err error) {
var resp *http.Response var resp *http.Response
var body []byte var body []byte
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.Call(&opts) resp, err = f.srv.Call(ctx, &opts)
if err != nil { if err != nil {
return fserrors.ShouldRetry(err), err return fserrors.ShouldRetry(err), err
} }
@ -668,7 +669,7 @@ func (f *Fs) copyOrMove(method, src, dst string, overwrite bool) (err error) {
if err != nil { if err != nil {
return errors.Wrapf(err, "async info result not JSON: %q", body) return errors.Wrapf(err, "async info result not JSON: %q", body)
} }
return f.waitForJob(info.HRef) return f.waitForJob(ctx, info.HRef)
} }
return nil return nil
} }
@ -690,11 +691,11 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
} }
dstPath := f.filePath(remote) dstPath := f.filePath(remote)
err := f.mkParentDirs(dstPath) err := f.mkParentDirs(ctx, dstPath)
if err != nil { if err != nil {
return nil, err return nil, err
} }
err = f.copyOrMove("copy", srcObj.filePath(), dstPath, false) err = f.copyOrMove(ctx, "copy", srcObj.filePath(), dstPath, false)
if err != nil { if err != nil {
return nil, errors.Wrap(err, "couldn't copy file") return nil, errors.Wrap(err, "couldn't copy file")
@ -720,11 +721,11 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
} }
dstPath := f.filePath(remote) dstPath := f.filePath(remote)
err := f.mkParentDirs(dstPath) err := f.mkParentDirs(ctx, dstPath)
if err != nil { if err != nil {
return nil, err return nil, err
} }
err = f.copyOrMove("move", srcObj.filePath(), dstPath, false) err = f.copyOrMove(ctx, "move", srcObj.filePath(), dstPath, false)
if err != nil { if err != nil {
return nil, errors.Wrap(err, "couldn't move file") return nil, errors.Wrap(err, "couldn't move file")
@ -758,12 +759,12 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
return errors.New("can't move root directory") return errors.New("can't move root directory")
} }
err := f.mkParentDirs(dstPath) err := f.mkParentDirs(ctx, dstPath)
if err != nil { if err != nil {
return err return err
} }
_, err = f.readMetaDataForPath(dstPath, &api.ResourceInfoRequestOptions{}) _, err = f.readMetaDataForPath(ctx, dstPath, &api.ResourceInfoRequestOptions{})
if apiErr, ok := err.(*api.ErrorResponse); ok { if apiErr, ok := err.(*api.ErrorResponse); ok {
// does not exist // does not exist
if apiErr.ErrorName == "DiskNotFoundError" { if apiErr.ErrorName == "DiskNotFoundError" {
@ -775,7 +776,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
return fs.ErrorDirExists return fs.ErrorDirExists
} }
err = f.copyOrMove("move", srcPath, dstPath, false) err = f.copyOrMove(ctx, "move", srcPath, dstPath, false)
if err != nil { if err != nil {
return errors.Wrap(err, "couldn't move directory") return errors.Wrap(err, "couldn't move directory")
@ -802,7 +803,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err er
var resp *http.Response var resp *http.Response
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.Call(&opts) resp, err = f.srv.Call(ctx, &opts)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
@ -819,7 +820,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err er
return "", errors.Wrap(err, "couldn't create public link") return "", errors.Wrap(err, "couldn't create public link")
} }
info, err := f.readMetaDataForPath(f.filePath(remote), &api.ResourceInfoRequestOptions{}) info, err := f.readMetaDataForPath(ctx, f.filePath(remote), &api.ResourceInfoRequestOptions{})
if err != nil { if err != nil {
return "", err return "", err
} }
@ -840,7 +841,7 @@ func (f *Fs) CleanUp(ctx context.Context) (err error) {
} }
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.Call(&opts) resp, err = f.srv.Call(ctx, &opts)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
return err return err
@ -857,7 +858,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
var info api.DiskInfo var info api.DiskInfo
var err error var err error
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(&opts, nil, &info) resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
@ -924,11 +925,11 @@ func (o *Object) setMetaData(info *api.ResourceInfoResponse) (err error) {
} }
// readMetaData reads ands sets the new metadata for a storage.Object // readMetaData reads ands sets the new metadata for a storage.Object
func (o *Object) readMetaData() (err error) { func (o *Object) readMetaData(ctx context.Context) (err error) {
if o.hasMetaData { if o.hasMetaData {
return nil return nil
} }
info, err := o.fs.readMetaDataForPath(o.filePath(), &api.ResourceInfoRequestOptions{}) info, err := o.fs.readMetaDataForPath(ctx, o.filePath(), &api.ResourceInfoRequestOptions{})
if err != nil { if err != nil {
return err return err
} }
@ -943,7 +944,7 @@ func (o *Object) readMetaData() (err error) {
// It attempts to read the objects mtime and if that isn't present the // It attempts to read the objects mtime and if that isn't present the
// LastModified returned in the http headers // LastModified returned in the http headers
func (o *Object) ModTime(ctx context.Context) time.Time { func (o *Object) ModTime(ctx context.Context) time.Time {
err := o.readMetaData() err := o.readMetaData(ctx)
if err != nil { if err != nil {
fs.Logf(o, "Failed to read metadata: %v", err) fs.Logf(o, "Failed to read metadata: %v", err)
return time.Now() return time.Now()
@ -953,7 +954,8 @@ func (o *Object) ModTime(ctx context.Context) time.Time {
// Size returns the size of an object in bytes // Size returns the size of an object in bytes
func (o *Object) Size() int64 { func (o *Object) Size() int64 {
err := o.readMetaData() ctx := context.TODO()
err := o.readMetaData(ctx)
if err != nil { if err != nil {
fs.Logf(o, "Failed to read metadata: %v", err) fs.Logf(o, "Failed to read metadata: %v", err)
return 0 return 0
@ -974,7 +976,7 @@ func (o *Object) Storable() bool {
return true return true
} }
func (o *Object) setCustomProperty(property string, value string) (err error) { func (o *Object) setCustomProperty(ctx context.Context, property string, value string) (err error) {
var resp *http.Response var resp *http.Response
opts := rest.Opts{ opts := rest.Opts{
Method: "PATCH", Method: "PATCH",
@ -990,7 +992,7 @@ func (o *Object) setCustomProperty(property string, value string) (err error) {
cpr := api.CustomPropertyResponse{CustomProperties: rcm} cpr := api.CustomPropertyResponse{CustomProperties: rcm}
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.CallJSON(&opts, &cpr, nil) resp, err = o.fs.srv.CallJSON(ctx, &opts, &cpr, nil)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
return err return err
@ -1001,7 +1003,7 @@ func (o *Object) setCustomProperty(property string, value string) (err error) {
// Commits the datastore // Commits the datastore
func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error { func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
// set custom_property 'rclone_modified' of object to modTime // set custom_property 'rclone_modified' of object to modTime
err := o.setCustomProperty("rclone_modified", modTime.Format(time.RFC3339Nano)) err := o.setCustomProperty(ctx, "rclone_modified", modTime.Format(time.RFC3339Nano))
if err != nil { if err != nil {
return err return err
} }
@ -1023,7 +1025,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
opts.Parameters.Set("path", o.filePath()) opts.Parameters.Set("path", o.filePath())
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.CallJSON(&opts, nil, &dl) resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &dl)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
@ -1038,7 +1040,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
Options: options, Options: options,
} }
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.Call(&opts) resp, err = o.fs.srv.Call(ctx, &opts)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
if err != nil { if err != nil {
@ -1047,7 +1049,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
return resp.Body, err return resp.Body, err
} }
func (o *Object) upload(in io.Reader, overwrite bool, mimeType string) (err error) { func (o *Object) upload(ctx context.Context, in io.Reader, overwrite bool, mimeType string) (err error) {
// prepare upload // prepare upload
var resp *http.Response var resp *http.Response
var ur api.AsyncInfo var ur api.AsyncInfo
@ -1061,7 +1063,7 @@ func (o *Object) upload(in io.Reader, overwrite bool, mimeType string) (err erro
opts.Parameters.Set("overwrite", strconv.FormatBool(overwrite)) opts.Parameters.Set("overwrite", strconv.FormatBool(overwrite))
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.CallJSON(&opts, nil, &ur) resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &ur)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
@ -1079,7 +1081,7 @@ func (o *Object) upload(in io.Reader, overwrite bool, mimeType string) (err erro
} }
err = o.fs.pacer.Call(func() (bool, error) { err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.Call(&opts) resp, err = o.fs.srv.Call(ctx, &opts)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
@ -1097,13 +1099,13 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
remote := o.filePath() remote := o.filePath()
//create full path to file before upload. //create full path to file before upload.
err := o.fs.mkParentDirs(remote) err := o.fs.mkParentDirs(ctx, remote)
if err != nil { if err != nil {
return err return err
} }
//upload file //upload file
err = o.upload(in1, true, fs.MimeType(ctx, src)) err = o.upload(ctx, in1, true, fs.MimeType(ctx, src))
if err != nil { if err != nil {
return err return err
} }
@ -1120,7 +1122,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
// Remove an object // Remove an object
func (o *Object) Remove(ctx context.Context) error { func (o *Object) Remove(ctx context.Context) error {
return o.fs.delete(o.filePath(), false) return o.fs.delete(ctx, o.filePath(), false)
} }
// MimeType of an Object if known, "" otherwise // MimeType of an Object if known, "" otherwise

17
lib/rest/rest.go
View file

@ -5,6 +5,7 @@ package rest
import ( import (
"bytes" "bytes"
"context"
"encoding/json" "encoding/json"
"encoding/xml" "encoding/xml"
"io" "io"
@ -184,7 +185,7 @@ func ClientWithNoRedirects(c *http.Client) *http.Client {
// if err != nil then resp.Body will have been closed // if err != nil then resp.Body will have been closed
// //
// it will return resp if at all possible, even if err is set // it will return resp if at all possible, even if err is set
func (api *Client) Call(opts *Opts) (resp *http.Response, err error) { func (api *Client) Call(ctx context.Context, opts *Opts) (resp *http.Response, err error) {
api.mu.RLock() api.mu.RLock()
defer api.mu.RUnlock() defer api.mu.RUnlock()
if opts == nil { if opts == nil {
@ -211,7 +212,7 @@ func (api *Client) Call(opts *Opts) (resp *http.Response, err error) {
if opts.ContentLength != nil && *opts.ContentLength == 0 { if opts.ContentLength != nil && *opts.ContentLength == 0 {
body = nil body = nil
} }
req, err := http.NewRequest(opts.Method, url, body) req, err := http.NewRequestWithContext(ctx, opts.Method, url, body)
if err != nil { if err != nil {
return return
} }
@ -391,8 +392,8 @@ func MultipartUpload(in io.Reader, params url.Values, contentName, fileName stri
// parameter name MultipartMetadataName. // parameter name MultipartMetadataName.
// //
// It will return resp if at all possible, even if err is set // It will return resp if at all possible, even if err is set
func (api *Client) CallJSON(opts *Opts, request interface{}, response interface{}) (resp *http.Response, err error) { func (api *Client) CallJSON(ctx context.Context, opts *Opts, request interface{}, response interface{}) (resp *http.Response, err error) {
return api.callCodec(opts, request, response, json.Marshal, DecodeJSON, "application/json") return api.callCodec(ctx, opts, request, response, json.Marshal, DecodeJSON, "application/json")
} }
// CallXML runs Call and decodes the body as a XML object into response (if not nil) // CallXML runs Call and decodes the body as a XML object into response (if not nil)
@ -408,14 +409,14 @@ func (api *Client) CallJSON(opts *Opts, request interface{}, response interface{
// See CallJSON for a description of MultipartParams and related opts // See CallJSON for a description of MultipartParams and related opts
// //
// It will return resp if at all possible, even if err is set // It will return resp if at all possible, even if err is set
func (api *Client) CallXML(opts *Opts, request interface{}, response interface{}) (resp *http.Response, err error) { func (api *Client) CallXML(ctx context.Context, opts *Opts, request interface{}, response interface{}) (resp *http.Response, err error) {
return api.callCodec(opts, request, response, xml.Marshal, DecodeXML, "application/xml") return api.callCodec(ctx, opts, request, response, xml.Marshal, DecodeXML, "application/xml")
} }
type marshalFn func(v interface{}) ([]byte, error) type marshalFn func(v interface{}) ([]byte, error)
type decodeFn func(resp *http.Response, result interface{}) (err error) type decodeFn func(resp *http.Response, result interface{}) (err error)
func (api *Client) callCodec(opts *Opts, request interface{}, response interface{}, marshal marshalFn, decode decodeFn, contentType string) (resp *http.Response, err error) { func (api *Client) callCodec(ctx context.Context, opts *Opts, request interface{}, response interface{}, marshal marshalFn, decode decodeFn, contentType string) (resp *http.Response, err error) {
var requestBody []byte var requestBody []byte
// Marshal the request if given // Marshal the request if given
if request != nil { if request != nil {
@ -446,7 +447,7 @@ func (api *Client) callCodec(opts *Opts, request interface{}, response interface
*opts.ContentLength += overhead *opts.ContentLength += overhead
} }
} }
resp, err = api.Call(opts) resp, err = api.Call(ctx, opts)
if err != nil { if err != nil {
return resp, err return resp, err
} }