From 58a531a203abe7cb6dac91832e16e81f64c8e0ab Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 4 Sep 2019 20:00:37 +0100 Subject: [PATCH] rest: add context propagation to rest library #3257 This fixes up the calling and propagates the contexts for the backends which use lib/rest. --- backend/b2/b2.go | 108 ++++++++++++------------- backend/b2/upload.go | 56 ++++++------- backend/box/box.go | 44 +++++----- backend/box/upload.go | 27 ++++--- backend/fichier/api.go | 42 +++++----- backend/fichier/fichier.go | 14 ++-- backend/fichier/object.go | 6 +- backend/googlephotos/googlephotos.go | 40 ++++----- backend/googlephotos/pattern.go | 4 +- backend/googlephotos/pattern_test.go | 2 +- backend/jottacloud/jottacloud.go | 109 +++++++++++++------------ backend/mailru/mailru.go | 54 ++++++------- backend/onedrive/onedrive.go | 67 ++++++++-------- backend/opendrive/opendrive.go | 44 +++++----- backend/pcloud/pcloud.go | 38 ++++----- backend/premiumizeme/premiumizeme.go | 30 +++---- backend/webdav/webdav.go | 69 ++++++++-------- backend/yandex/yandex.go | 116 ++++++++++++++------------- lib/rest/rest.go | 17 ++-- 19 files changed, 448 insertions(+), 439 deletions(-) diff --git a/backend/b2/b2.go b/backend/b2/b2.go index ee05cd02a..7c5b32772 100644 --- a/backend/b2/b2.go +++ b/backend/b2/b2.go @@ -273,11 +273,11 @@ func (f *Fs) shouldRetryNoReauth(resp *http.Response, err error) (bool, error) { // shouldRetry returns a boolean as to whether this resp and err // deserve to be retried. It returns the err as a convenience -func (f *Fs) shouldRetry(resp *http.Response, err error) (bool, error) { +func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) { if resp != nil && resp.StatusCode == 401 { fs.Debugf(f, "Unauthorized: %v", err) // Reauth - authErr := f.authorizeAccount() + authErr := f.authorizeAccount(ctx) if authErr != nil { err = authErr } @@ -393,7 +393,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { fs.Debugf(f, "Setting test header \"%s: %s\"", testModeHeader, testMode) } f.fillBufferTokens() - err = f.authorizeAccount() + err = f.authorizeAccount(ctx) if err != nil { return nil, errors.Wrap(err, "failed to authorize account") } @@ -431,7 +431,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { // authorizeAccount gets the API endpoint and auth token. Can be used // for reauthentication too. -func (f *Fs) authorizeAccount() error { +func (f *Fs) authorizeAccount(ctx context.Context) error { f.authMu.Lock() defer f.authMu.Unlock() opts := rest.Opts{ @@ -443,7 +443,7 @@ func (f *Fs) authorizeAccount() error { ExtraHeaders: map[string]string{"Authorization": ""}, // unset the Authorization for this request } err := f.pacer.Call(func() (bool, error) { - resp, err := f.srv.CallJSON(&opts, nil, &f.info) + resp, err := f.srv.CallJSON(ctx, &opts, nil, &f.info) return f.shouldRetryNoReauth(resp, err) }) if err != nil { @@ -466,10 +466,10 @@ func (f *Fs) hasPermission(permission string) bool { // getUploadURL returns the upload info with the UploadURL and the AuthorizationToken // // This should be returned with returnUploadURL when finished -func (f *Fs) getUploadURL(bucket string) (upload *api.GetUploadURLResponse, err error) { +func (f *Fs) getUploadURL(ctx context.Context, bucket string) (upload *api.GetUploadURLResponse, err error) { f.uploadMu.Lock() defer f.uploadMu.Unlock() - bucketID, err := f.getBucketID(bucket) + bucketID, err := f.getBucketID(ctx, bucket) if err != nil { return nil, err } @@ -489,8 +489,8 @@ func (f *Fs) getUploadURL(bucket string) (upload *api.GetUploadURLResponse, err BucketID: bucketID, } err = f.pacer.Call(func() (bool, error) { - resp, err := f.srv.CallJSON(&opts, &request, &upload) - return f.shouldRetry(resp, err) + resp, err := f.srv.CallJSON(ctx, &opts, &request, &upload) + return f.shouldRetry(ctx, resp, err) }) if err != nil { return nil, errors.Wrap(err, "failed to get upload URL") @@ -609,7 +609,7 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck if !recurse { delimiter = "/" } - bucketID, err := f.getBucketID(bucket) + bucketID, err := f.getBucketID(ctx, bucket) if err != nil { return err } @@ -636,8 +636,8 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck for { var response api.ListFileNamesResponse err := f.pacer.Call(func() (bool, error) { - resp, err := f.srv.CallJSON(&opts, &request, &response) - return f.shouldRetry(resp, err) + resp, err := f.srv.CallJSON(ctx, &opts, &request, &response) + return f.shouldRetry(ctx, resp, err) }) if err != nil { return err @@ -727,7 +727,7 @@ func (f *Fs) listDir(ctx context.Context, bucket, directory, prefix string, addB // listBuckets returns all the buckets to out func (f *Fs) listBuckets(ctx context.Context) (entries fs.DirEntries, err error) { - err = f.listBucketsToFn(func(bucket *api.Bucket) error { + err = f.listBucketsToFn(ctx, func(bucket *api.Bucket) error { d := fs.NewDir(bucket.Name, time.Time{}) entries = append(entries, d) return nil @@ -820,7 +820,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) ( type listBucketFn func(*api.Bucket) error // listBucketsToFn lists the buckets to the function supplied -func (f *Fs) listBucketsToFn(fn listBucketFn) error { +func (f *Fs) listBucketsToFn(ctx context.Context, fn listBucketFn) error { var account = api.ListBucketsRequest{ AccountID: f.info.AccountID, BucketID: f.info.Allowed.BucketID, @@ -832,8 +832,8 @@ func (f *Fs) listBucketsToFn(fn listBucketFn) error { Path: "/b2_list_buckets", } err := f.pacer.Call(func() (bool, error) { - resp, err := f.srv.CallJSON(&opts, &account, &response) - return f.shouldRetry(resp, err) + resp, err := f.srv.CallJSON(ctx, &opts, &account, &response) + return f.shouldRetry(ctx, resp, err) }) if err != nil { return err @@ -862,14 +862,14 @@ func (f *Fs) listBucketsToFn(fn listBucketFn) error { // getbucketType finds the bucketType for the current bucket name // can be one of allPublic. allPrivate, or snapshot -func (f *Fs) getbucketType(bucket string) (bucketType string, err error) { +func (f *Fs) getbucketType(ctx context.Context, bucket string) (bucketType string, err error) { f.bucketTypeMutex.Lock() bucketType = f._bucketType[bucket] f.bucketTypeMutex.Unlock() if bucketType != "" { return bucketType, nil } - err = f.listBucketsToFn(func(bucket *api.Bucket) error { + err = f.listBucketsToFn(ctx, func(bucket *api.Bucket) error { // listBucketsToFn reads bucket Types return nil }) @@ -897,14 +897,14 @@ func (f *Fs) clearBucketType(bucket string) { } // getBucketID finds the ID for the current bucket name -func (f *Fs) getBucketID(bucket string) (bucketID string, err error) { +func (f *Fs) getBucketID(ctx context.Context, bucket string) (bucketID string, err error) { f.bucketIDMutex.Lock() bucketID = f._bucketID[bucket] f.bucketIDMutex.Unlock() if bucketID != "" { return bucketID, nil } - err = f.listBucketsToFn(func(bucket *api.Bucket) error { + err = f.listBucketsToFn(ctx, func(bucket *api.Bucket) error { // listBucketsToFn sets IDs return nil }) @@ -970,15 +970,15 @@ func (f *Fs) makeBucket(ctx context.Context, bucket string) error { } var response api.Bucket err := f.pacer.Call(func() (bool, error) { - resp, err := f.srv.CallJSON(&opts, &request, &response) - return f.shouldRetry(resp, err) + resp, err := f.srv.CallJSON(ctx, &opts, &request, &response) + return f.shouldRetry(ctx, resp, err) }) if err != nil { if apiErr, ok := err.(*api.Error); ok { if apiErr.Code == "duplicate_bucket_name" { // Check this is our bucket - buckets are globally unique and this // might be someone elses. - _, getBucketErr := f.getBucketID(bucket) + _, getBucketErr := f.getBucketID(ctx, bucket) if getBucketErr == nil { // found so it is our bucket return nil @@ -1009,7 +1009,7 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error { Method: "POST", Path: "/b2_delete_bucket", } - bucketID, err := f.getBucketID(bucket) + bucketID, err := f.getBucketID(ctx, bucket) if err != nil { return err } @@ -1019,8 +1019,8 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error { } var response api.Bucket err = f.pacer.Call(func() (bool, error) { - resp, err := f.srv.CallJSON(&opts, &request, &response) - return f.shouldRetry(resp, err) + resp, err := f.srv.CallJSON(ctx, &opts, &request, &response) + return f.shouldRetry(ctx, resp, err) }) if err != nil { return errors.Wrap(err, "failed to delete bucket") @@ -1038,8 +1038,8 @@ func (f *Fs) Precision() time.Duration { } // hide hides a file on the remote -func (f *Fs) hide(bucket, bucketPath string) error { - bucketID, err := f.getBucketID(bucket) +func (f *Fs) hide(ctx context.Context, bucket, bucketPath string) error { + bucketID, err := f.getBucketID(ctx, bucket) if err != nil { return err } @@ -1053,8 +1053,8 @@ func (f *Fs) hide(bucket, bucketPath string) error { } var response api.File err = f.pacer.Call(func() (bool, error) { - resp, err := f.srv.CallJSON(&opts, &request, &response) - return f.shouldRetry(resp, err) + resp, err := f.srv.CallJSON(ctx, &opts, &request, &response) + return f.shouldRetry(ctx, resp, err) }) if err != nil { if apiErr, ok := err.(*api.Error); ok { @@ -1070,7 +1070,7 @@ func (f *Fs) hide(bucket, bucketPath string) error { } // deleteByID deletes a file version given Name and ID -func (f *Fs) deleteByID(ID, Name string) error { +func (f *Fs) deleteByID(ctx context.Context, ID, Name string) error { opts := rest.Opts{ Method: "POST", Path: "/b2_delete_file_version", @@ -1081,8 +1081,8 @@ func (f *Fs) deleteByID(ID, Name string) error { } var response api.File err := f.pacer.Call(func() (bool, error) { - resp, err := f.srv.CallJSON(&opts, &request, &response) - return f.shouldRetry(resp, err) + resp, err := f.srv.CallJSON(ctx, &opts, &request, &response) + return f.shouldRetry(ctx, resp, err) }) if err != nil { return errors.Wrapf(err, "failed to delete %q", Name) @@ -1132,7 +1132,7 @@ func (f *Fs) purge(ctx context.Context, bucket, directory string, oldOnly bool) continue } tr := accounting.Stats(ctx).NewCheckingTransfer(oi) - err = f.deleteByID(object.ID, object.Name) + err = f.deleteByID(ctx, object.ID, object.Name) checkErr(err) tr.Done(err) } @@ -1205,7 +1205,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, fs.Debugf(src, "Can't copy - not same remote type") return nil, fs.ErrorCantCopy } - destBucketID, err := f.getBucketID(dstBucket) + destBucketID, err := f.getBucketID(ctx, dstBucket) if err != nil { return nil, err } @@ -1221,8 +1221,8 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, } var response api.FileInfo err = f.pacer.Call(func() (bool, error) { - resp, err := f.srv.CallJSON(&opts, &request, &response) - return f.shouldRetry(resp, err) + resp, err := f.srv.CallJSON(ctx, &opts, &request, &response) + return f.shouldRetry(ctx, resp, err) }) if err != nil { return nil, err @@ -1245,7 +1245,7 @@ func (f *Fs) Hashes() hash.Set { // getDownloadAuthorization returns authorization token for downloading // without account. -func (f *Fs) getDownloadAuthorization(bucket, remote string) (authorization string, err error) { +func (f *Fs) getDownloadAuthorization(ctx context.Context, bucket, remote string) (authorization string, err error) { validDurationInSeconds := time.Duration(f.opt.DownloadAuthorizationDuration).Nanoseconds() / 1e9 if validDurationInSeconds <= 0 || validDurationInSeconds > 604800 { return "", errors.New("--b2-download-auth-duration must be between 1 sec and 1 week") @@ -1253,7 +1253,7 @@ func (f *Fs) getDownloadAuthorization(bucket, remote string) (authorization stri if !f.hasPermission("shareFiles") { return "", errors.New("sharing a file link requires the shareFiles permission") } - bucketID, err := f.getBucketID(bucket) + bucketID, err := f.getBucketID(ctx, bucket) if err != nil { return "", err } @@ -1268,8 +1268,8 @@ func (f *Fs) getDownloadAuthorization(bucket, remote string) (authorization stri } var response api.GetDownloadAuthorizationResponse err = f.pacer.Call(func() (bool, error) { - resp, err := f.srv.CallJSON(&opts, &request, &response) - return f.shouldRetry(resp, err) + resp, err := f.srv.CallJSON(ctx, &opts, &request, &response) + return f.shouldRetry(ctx, resp, err) }) if err != nil { return "", errors.Wrap(err, "failed to get download authorization") @@ -1301,12 +1301,12 @@ func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err er } absPath := "/" + bucketPath link = RootURL + "/file/" + urlEncode(bucket) + absPath - bucketType, err := f.getbucketType(bucket) + bucketType, err := f.getbucketType(ctx, bucket) if err != nil { return "", err } if bucketType == "allPrivate" || bucketType == "snapshot" { - AuthorizationToken, err := f.getDownloadAuthorization(bucket, remote) + AuthorizationToken, err := f.getDownloadAuthorization(ctx, bucket, remote) if err != nil { return "", err } @@ -1505,8 +1505,8 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error { } var response api.FileInfo err = o.fs.pacer.Call(func() (bool, error) { - resp, err := o.fs.srv.CallJSON(&opts, &request, &response) - return o.fs.shouldRetry(resp, err) + resp, err := o.fs.srv.CallJSON(ctx, &opts, &request, &response) + return o.fs.shouldRetry(ctx, resp, err) }) if err != nil { return err @@ -1604,8 +1604,8 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read } var resp *http.Response err = o.fs.pacer.Call(func() (bool, error) { - resp, err = o.fs.srv.Call(&opts) - return o.fs.shouldRetry(resp, err) + resp, err = o.fs.srv.Call(ctx, &opts) + return o.fs.shouldRetry(ctx, resp, err) }) if err != nil { return nil, errors.Wrap(err, "failed to open for download") @@ -1701,7 +1701,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op o.fs.putUploadBlock(buf) return err } - return up.Stream(buf) + return up.Stream(ctx, buf) } else if err == io.EOF || err == io.ErrUnexpectedEOF { fs.Debugf(o, "File has %d bytes, which makes only one chunk. Using direct upload.", n) defer o.fs.putUploadBlock(buf) @@ -1715,7 +1715,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op if err != nil { return err } - return up.Upload() + return up.Upload(ctx) } modTime := src.ModTime(ctx) @@ -1729,7 +1729,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op } // Get upload URL - upload, err := o.fs.getUploadURL(bucket) + upload, err := o.fs.getUploadURL(ctx, bucket) if err != nil { return err } @@ -1807,8 +1807,8 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op var response api.FileInfo // Don't retry, return a retry error instead err = o.fs.pacer.CallNoRetry(func() (bool, error) { - resp, err := o.fs.srv.CallJSON(&opts, nil, &response) - retry, err := o.fs.shouldRetry(resp, err) + resp, err := o.fs.srv.CallJSON(ctx, &opts, nil, &response) + retry, err := o.fs.shouldRetry(ctx, resp, err) // On retryable error clear UploadURL if retry { fs.Debugf(o, "Clearing upload URL because of error: %v", err) @@ -1829,9 +1829,9 @@ func (o *Object) Remove(ctx context.Context) error { return errNotWithVersions } if o.fs.opt.HardDelete { - return o.fs.deleteByID(o.id, bucketPath) + return o.fs.deleteByID(ctx, o.id, bucketPath) } - return o.fs.hide(bucket, bucketPath) + return o.fs.hide(ctx, bucket, bucketPath) } // MimeType of an Object if known, "" otherwise diff --git a/backend/b2/upload.go b/backend/b2/upload.go index f891a5561..f25c5b046 100644 --- a/backend/b2/upload.go +++ b/backend/b2/upload.go @@ -105,7 +105,7 @@ func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs Path: "/b2_start_large_file", } bucket, bucketPath := o.split() - bucketID, err := f.getBucketID(bucket) + bucketID, err := f.getBucketID(ctx, bucket) if err != nil { return nil, err } @@ -125,8 +125,8 @@ func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs } var response api.StartLargeFileResponse err = f.pacer.Call(func() (bool, error) { - resp, err := f.srv.CallJSON(&opts, &request, &response) - return f.shouldRetry(resp, err) + resp, err := f.srv.CallJSON(ctx, &opts, &request, &response) + return f.shouldRetry(ctx, resp, err) }) if err != nil { return nil, err @@ -150,7 +150,7 @@ func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs // getUploadURL returns the upload info with the UploadURL and the AuthorizationToken // // This should be returned with returnUploadURL when finished -func (up *largeUpload) getUploadURL() (upload *api.GetUploadPartURLResponse, err error) { +func (up *largeUpload) getUploadURL(ctx context.Context) (upload *api.GetUploadPartURLResponse, err error) { up.uploadMu.Lock() defer up.uploadMu.Unlock() if len(up.uploads) == 0 { @@ -162,8 +162,8 @@ func (up *largeUpload) getUploadURL() (upload *api.GetUploadPartURLResponse, err ID: up.id, } err := up.f.pacer.Call(func() (bool, error) { - resp, err := up.f.srv.CallJSON(&opts, &request, &upload) - return up.f.shouldRetry(resp, err) + resp, err := up.f.srv.CallJSON(ctx, &opts, &request, &upload) + return up.f.shouldRetry(ctx, resp, err) }) if err != nil { return nil, errors.Wrap(err, "failed to get upload URL") @@ -192,12 +192,12 @@ func (up *largeUpload) clearUploadURL() { } // Transfer a chunk -func (up *largeUpload) transferChunk(part int64, body []byte) error { +func (up *largeUpload) transferChunk(ctx context.Context, part int64, body []byte) error { err := up.f.pacer.Call(func() (bool, error) { fs.Debugf(up.o, "Sending chunk %d length %d", part, len(body)) // Get upload URL - upload, err := up.getUploadURL() + upload, err := up.getUploadURL(ctx) if err != nil { return false, err } @@ -241,8 +241,8 @@ func (up *largeUpload) transferChunk(part int64, body []byte) error { var response api.UploadPartResponse - resp, err := up.f.srv.CallJSON(&opts, nil, &response) - retry, err := up.f.shouldRetry(resp, err) + resp, err := up.f.srv.CallJSON(ctx, &opts, nil, &response) + retry, err := up.f.shouldRetry(ctx, resp, err) if err != nil { fs.Debugf(up.o, "Error sending chunk %d (retry=%v): %v: %#v", part, retry, err, err) } @@ -264,7 +264,7 @@ func (up *largeUpload) transferChunk(part int64, body []byte) error { } // finish closes off the large upload -func (up *largeUpload) finish() error { +func (up *largeUpload) finish(ctx context.Context) error { fs.Debugf(up.o, "Finishing large file upload with %d parts", up.parts) opts := rest.Opts{ Method: "POST", @@ -276,8 +276,8 @@ func (up *largeUpload) finish() error { } var response api.FileInfo err := up.f.pacer.Call(func() (bool, error) { - resp, err := up.f.srv.CallJSON(&opts, &request, &response) - return up.f.shouldRetry(resp, err) + resp, err := up.f.srv.CallJSON(ctx, &opts, &request, &response) + return up.f.shouldRetry(ctx, resp, err) }) if err != nil { return err @@ -286,7 +286,7 @@ func (up *largeUpload) finish() error { } // cancel aborts the large upload -func (up *largeUpload) cancel() error { +func (up *largeUpload) cancel(ctx context.Context) error { opts := rest.Opts{ Method: "POST", Path: "/b2_cancel_large_file", @@ -296,18 +296,18 @@ func (up *largeUpload) cancel() error { } var response api.CancelLargeFileResponse err := up.f.pacer.Call(func() (bool, error) { - resp, err := up.f.srv.CallJSON(&opts, &request, &response) - return up.f.shouldRetry(resp, err) + resp, err := up.f.srv.CallJSON(ctx, &opts, &request, &response) + return up.f.shouldRetry(ctx, resp, err) }) return err } -func (up *largeUpload) managedTransferChunk(wg *sync.WaitGroup, errs chan error, part int64, buf []byte) { +func (up *largeUpload) managedTransferChunk(ctx context.Context, wg *sync.WaitGroup, errs chan error, part int64, buf []byte) { wg.Add(1) go func(part int64, buf []byte) { defer wg.Done() defer up.f.putUploadBlock(buf) - err := up.transferChunk(part, buf) + err := up.transferChunk(ctx, part, buf) if err != nil { select { case errs <- err: @@ -317,7 +317,7 @@ func (up *largeUpload) managedTransferChunk(wg *sync.WaitGroup, errs chan error, }(part, buf) } -func (up *largeUpload) finishOrCancelOnError(err error, errs chan error) error { +func (up *largeUpload) finishOrCancelOnError(ctx context.Context, err error, errs chan error) error { if err == nil { select { case err = <-errs: @@ -326,19 +326,19 @@ func (up *largeUpload) finishOrCancelOnError(err error, errs chan error) error { } if err != nil { fs.Debugf(up.o, "Cancelling large file upload due to error: %v", err) - cancelErr := up.cancel() + cancelErr := up.cancel(ctx) if cancelErr != nil { fs.Errorf(up.o, "Failed to cancel large file upload: %v", cancelErr) } return err } - return up.finish() + return up.finish(ctx) } // Stream uploads the chunks from the input, starting with a required initial // chunk. Assumes the file size is unknown and will upload until the input // reaches EOF. -func (up *largeUpload) Stream(initialUploadBlock []byte) (err error) { +func (up *largeUpload) Stream(ctx context.Context, initialUploadBlock []byte) (err error) { fs.Debugf(up.o, "Starting streaming of large file (id %q)", up.id) errs := make(chan error, 1) hasMoreParts := true @@ -346,7 +346,7 @@ func (up *largeUpload) Stream(initialUploadBlock []byte) (err error) { // Transfer initial chunk up.size = int64(len(initialUploadBlock)) - up.managedTransferChunk(&wg, errs, 1, initialUploadBlock) + up.managedTransferChunk(ctx, &wg, errs, 1, initialUploadBlock) outer: for part := int64(2); hasMoreParts; part++ { @@ -388,16 +388,16 @@ outer: } // Transfer the chunk - up.managedTransferChunk(&wg, errs, part, buf) + up.managedTransferChunk(ctx, &wg, errs, part, buf) } wg.Wait() up.sha1s = up.sha1s[:up.parts] - return up.finishOrCancelOnError(err, errs) + return up.finishOrCancelOnError(ctx, err, errs) } // Upload uploads the chunks from the input -func (up *largeUpload) Upload() error { +func (up *largeUpload) Upload(ctx context.Context) error { fs.Debugf(up.o, "Starting upload of large file in %d chunks (id %q)", up.parts, up.id) remaining := up.size errs := make(chan error, 1) @@ -428,10 +428,10 @@ outer: } // Transfer the chunk - up.managedTransferChunk(&wg, errs, part, buf) + up.managedTransferChunk(ctx, &wg, errs, part, buf) remaining -= reqSize } wg.Wait() - return up.finishOrCancelOnError(err, errs) + return up.finishOrCancelOnError(ctx, err, errs) } diff --git a/backend/box/box.go b/backend/box/box.go index 1a88fc057..1c307edae 100644 --- a/backend/box/box.go +++ b/backend/box/box.go @@ -204,7 +204,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.It return nil, err } - found, err := f.listAll(directoryID, false, true, func(item *api.Item) bool { + found, err := f.listAll(ctx, directoryID, false, true, func(item *api.Item) bool { if item.Name == leaf { info = item return true @@ -352,7 +352,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { // FindLeaf finds a directory of name leaf in the folder with ID pathID func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) { // Find the leaf in pathID - found, err = f.listAll(pathID, true, false, func(item *api.Item) bool { + found, err = f.listAll(ctx, pathID, true, false, func(item *api.Item) bool { if item.Name == leaf { pathIDOut = item.ID return true @@ -386,7 +386,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, }, } err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallJSON(&opts, &mkdir, &info) + resp, err = f.srv.CallJSON(ctx, &opts, &mkdir, &info) return shouldRetry(resp, err) }) if err != nil { @@ -408,7 +408,7 @@ type listAllFn func(*api.Item) bool // Lists the directory required calling the user function on each item found // // If the user fn ever returns true then it early exits with found = true -func (f *Fs) listAll(dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) { +func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) { opts := rest.Opts{ Method: "GET", Path: "/folders/" + dirID + "/items", @@ -423,7 +423,7 @@ OUTER: var result api.FolderItems var resp *http.Response err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallJSON(&opts, nil, &result) + resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) return shouldRetry(resp, err) }) if err != nil { @@ -479,7 +479,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e return nil, err } var iErr error - _, err = f.listAll(directoryID, false, false, func(info *api.Item) bool { + _, err = f.listAll(ctx, directoryID, false, false, func(info *api.Item) bool { remote := path.Join(dir, info.Name) if info.Type == api.ItemTypeFolder { // cache the directory ID for later lookups @@ -581,14 +581,14 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error { } // deleteObject removes an object by ID -func (f *Fs) deleteObject(id string) error { +func (f *Fs) deleteObject(ctx context.Context, id string) error { opts := rest.Opts{ Method: "DELETE", Path: "/files/" + id, NoResponse: true, } return f.pacer.Call(func() (bool, error) { - resp, err := f.srv.Call(&opts) + resp, err := f.srv.Call(ctx, &opts) return shouldRetry(resp, err) }) } @@ -619,7 +619,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error { opts.Parameters.Set("recursive", strconv.FormatBool(!check)) var resp *http.Response err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.Call(&opts) + resp, err = f.srv.Call(ctx, &opts) return shouldRetry(resp, err) }) if err != nil { @@ -692,7 +692,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, var resp *http.Response var info *api.Item err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallJSON(&opts, ©File, &info) + resp, err = f.srv.CallJSON(ctx, &opts, ©File, &info) return shouldRetry(resp, err) }) if err != nil { @@ -715,7 +715,7 @@ func (f *Fs) Purge(ctx context.Context) error { } // move a file or folder -func (f *Fs) move(endpoint, id, leaf, directoryID string) (info *api.Item, err error) { +func (f *Fs) move(ctx context.Context, endpoint, id, leaf, directoryID string) (info *api.Item, err error) { // Move the object opts := rest.Opts{ Method: "PUT", @@ -730,7 +730,7 @@ func (f *Fs) move(endpoint, id, leaf, directoryID string) (info *api.Item, err e } var resp *http.Response err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallJSON(&opts, &move, &info) + resp, err = f.srv.CallJSON(ctx, &opts, &move, &info) return shouldRetry(resp, err) }) if err != nil { @@ -762,7 +762,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, } // Do the move - info, err := f.move("/files/", srcObj.id, leaf, directoryID) + info, err := f.move(ctx, "/files/", srcObj.id, leaf, directoryID) if err != nil { return nil, err } @@ -845,7 +845,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string } // Do the move - _, err = f.move("/folders/", srcID, leaf, directoryID) + _, err = f.move(ctx, "/folders/", srcID, leaf, directoryID) if err != nil { return err } @@ -887,7 +887,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string) (string, error) { var info api.Item var resp *http.Response err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallJSON(&opts, &shareLink, &info) + resp, err = f.srv.CallJSON(ctx, &opts, &shareLink, &info) return shouldRetry(resp, err) }) return info.SharedLink.URL, err @@ -1006,7 +1006,7 @@ func (o *Object) setModTime(ctx context.Context, modTime time.Time) (*api.Item, } var info *api.Item err := o.fs.pacer.Call(func() (bool, error) { - resp, err := o.fs.srv.CallJSON(&opts, &update, &info) + resp, err := o.fs.srv.CallJSON(ctx, &opts, &update, &info) return shouldRetry(resp, err) }) return info, err @@ -1039,7 +1039,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read Options: options, } err = o.fs.pacer.Call(func() (bool, error) { - resp, err = o.fs.srv.Call(&opts) + resp, err = o.fs.srv.Call(ctx, &opts) return shouldRetry(resp, err) }) if err != nil { @@ -1051,7 +1051,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read // upload does a single non-multipart upload // // This is recommended for less than 50 MB of content -func (o *Object) upload(in io.Reader, leaf, directoryID string, modTime time.Time) (err error) { +func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID string, modTime time.Time) (err error) { upload := api.UploadFile{ Name: replaceReservedChars(leaf), ContentModifiedAt: api.Time(modTime), @@ -1078,7 +1078,7 @@ func (o *Object) upload(in io.Reader, leaf, directoryID string, modTime time.Tim opts.Path = "/files/content" } err = o.fs.pacer.CallNoRetry(func() (bool, error) { - resp, err = o.fs.srv.CallJSON(&opts, &upload, &result) + resp, err = o.fs.srv.CallJSON(ctx, &opts, &upload, &result) return shouldRetry(resp, err) }) if err != nil { @@ -1111,16 +1111,16 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op // Upload with simple or multipart if size <= int64(o.fs.opt.UploadCutoff) { - err = o.upload(in, leaf, directoryID, modTime) + err = o.upload(ctx, in, leaf, directoryID, modTime) } else { - err = o.uploadMultipart(in, leaf, directoryID, size, modTime) + err = o.uploadMultipart(ctx, in, leaf, directoryID, size, modTime) } return err } // Remove an object func (o *Object) Remove(ctx context.Context) error { - return o.fs.deleteObject(o.id) + return o.fs.deleteObject(ctx, o.id) } // ID returns the ID of the Object if known, or "" if not diff --git a/backend/box/upload.go b/backend/box/upload.go index 6e187635f..d413a8483 100644 --- a/backend/box/upload.go +++ b/backend/box/upload.go @@ -4,6 +4,7 @@ package box import ( "bytes" + "context" "crypto/sha1" "encoding/base64" "encoding/json" @@ -22,7 +23,7 @@ import ( ) // createUploadSession creates an upload session for the object -func (o *Object) createUploadSession(leaf, directoryID string, size int64) (response *api.UploadSessionResponse, err error) { +func (o *Object) createUploadSession(ctx context.Context, leaf, directoryID string, size int64) (response *api.UploadSessionResponse, err error) { opts := rest.Opts{ Method: "POST", Path: "/files/upload_sessions", @@ -41,7 +42,7 @@ func (o *Object) createUploadSession(leaf, directoryID string, size int64) (resp } var resp *http.Response err = o.fs.pacer.Call(func() (bool, error) { - resp, err = o.fs.srv.CallJSON(&opts, &request, &response) + resp, err = o.fs.srv.CallJSON(ctx, &opts, &request, &response) return shouldRetry(resp, err) }) return @@ -53,7 +54,7 @@ func sha1Digest(digest []byte) string { } // uploadPart uploads a part in an upload session -func (o *Object) uploadPart(SessionID string, offset, totalSize int64, chunk []byte, wrap accounting.WrapFn) (response *api.UploadPartResponse, err error) { +func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, totalSize int64, chunk []byte, wrap accounting.WrapFn) (response *api.UploadPartResponse, err error) { chunkSize := int64(len(chunk)) sha1sum := sha1.Sum(chunk) opts := rest.Opts{ @@ -70,7 +71,7 @@ func (o *Object) uploadPart(SessionID string, offset, totalSize int64, chunk []b var resp *http.Response err = o.fs.pacer.Call(func() (bool, error) { opts.Body = wrap(bytes.NewReader(chunk)) - resp, err = o.fs.srv.CallJSON(&opts, nil, &response) + resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &response) return shouldRetry(resp, err) }) if err != nil { @@ -80,7 +81,7 @@ func (o *Object) uploadPart(SessionID string, offset, totalSize int64, chunk []b } // commitUpload finishes an upload session -func (o *Object) commitUpload(SessionID string, parts []api.Part, modTime time.Time, sha1sum []byte) (result *api.FolderItems, err error) { +func (o *Object) commitUpload(ctx context.Context, SessionID string, parts []api.Part, modTime time.Time, sha1sum []byte) (result *api.FolderItems, err error) { opts := rest.Opts{ Method: "POST", Path: "/files/upload_sessions/" + SessionID + "/commit", @@ -104,7 +105,7 @@ func (o *Object) commitUpload(SessionID string, parts []api.Part, modTime time.T outer: for tries = 0; tries < maxTries; tries++ { err = o.fs.pacer.Call(func() (bool, error) { - resp, err = o.fs.srv.CallJSON(&opts, &request, nil) + resp, err = o.fs.srv.CallJSON(ctx, &opts, &request, nil) if err != nil { return shouldRetry(resp, err) } @@ -154,7 +155,7 @@ outer: } // abortUpload cancels an upload session -func (o *Object) abortUpload(SessionID string) (err error) { +func (o *Object) abortUpload(ctx context.Context, SessionID string) (err error) { opts := rest.Opts{ Method: "DELETE", Path: "/files/upload_sessions/" + SessionID, @@ -163,16 +164,16 @@ func (o *Object) abortUpload(SessionID string) (err error) { } var resp *http.Response err = o.fs.pacer.Call(func() (bool, error) { - resp, err = o.fs.srv.Call(&opts) + resp, err = o.fs.srv.Call(ctx, &opts) return shouldRetry(resp, err) }) return err } // uploadMultipart uploads a file using multipart upload -func (o *Object) uploadMultipart(in io.Reader, leaf, directoryID string, size int64, modTime time.Time) (err error) { +func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, leaf, directoryID string, size int64, modTime time.Time) (err error) { // Create upload session - session, err := o.createUploadSession(leaf, directoryID, size) + session, err := o.createUploadSession(ctx, leaf, directoryID, size) if err != nil { return errors.Wrap(err, "multipart upload create session failed") } @@ -183,7 +184,7 @@ func (o *Object) uploadMultipart(in io.Reader, leaf, directoryID string, size in defer func() { if err != nil { fs.Debugf(o, "Cancelling multipart upload: %v", err) - cancelErr := o.abortUpload(session.ID) + cancelErr := o.abortUpload(ctx, session.ID) if cancelErr != nil { fs.Logf(o, "Failed to cancel multipart upload: %v", err) } @@ -235,7 +236,7 @@ outer: defer wg.Done() defer o.fs.uploadToken.Put() fs.Debugf(o, "Uploading part %d/%d offset %v/%v part size %v", part+1, session.TotalParts, fs.SizeSuffix(position), fs.SizeSuffix(size), fs.SizeSuffix(chunkSize)) - partResponse, err := o.uploadPart(session.ID, position, size, buf, wrap) + partResponse, err := o.uploadPart(ctx, session.ID, position, size, buf, wrap) if err != nil { err = errors.Wrap(err, "multipart upload failed to upload part") select { @@ -263,7 +264,7 @@ outer: } // Finalise the upload session - result, err := o.commitUpload(session.ID, parts, modTime, hash.Sum(nil)) + result, err := o.commitUpload(ctx, session.ID, parts, modTime, hash.Sum(nil)) if err != nil { return errors.Wrap(err, "multipart upload failed to finalize") } diff --git a/backend/fichier/api.go b/backend/fichier/api.go index d8b82a0e2..b111e37f7 100644 --- a/backend/fichier/api.go +++ b/backend/fichier/api.go @@ -32,7 +32,7 @@ func shouldRetry(resp *http.Response, err error) (bool, error) { var isAlphaNumeric = regexp.MustCompile(`^[a-zA-Z0-9]+$`).MatchString -func (f *Fs) getDownloadToken(url string) (*GetTokenResponse, error) { +func (f *Fs) getDownloadToken(ctx context.Context, url string) (*GetTokenResponse, error) { request := DownloadRequest{ URL: url, Single: 1, @@ -44,7 +44,7 @@ func (f *Fs) getDownloadToken(url string) (*GetTokenResponse, error) { var token GetTokenResponse err := f.pacer.Call(func() (bool, error) { - resp, err := f.rest.CallJSON(&opts, &request, &token) + resp, err := f.rest.CallJSON(ctx, &opts, &request, &token) return shouldRetry(resp, err) }) if err != nil { @@ -72,7 +72,7 @@ func (f *Fs) listSharedFiles(ctx context.Context, id string) (entries fs.DirEntr var sharedFiles SharedFolderResponse err = f.pacer.Call(func() (bool, error) { - resp, err := f.rest.CallJSON(&opts, nil, &sharedFiles) + resp, err := f.rest.CallJSON(ctx, &opts, nil, &sharedFiles) return shouldRetry(resp, err) }) if err != nil { @@ -88,7 +88,7 @@ func (f *Fs) listSharedFiles(ctx context.Context, id string) (entries fs.DirEntr return entries, nil } -func (f *Fs) listFiles(directoryID int) (filesList *FilesList, err error) { +func (f *Fs) listFiles(ctx context.Context, directoryID int) (filesList *FilesList, err error) { // fs.Debugf(f, "Requesting files for dir `%s`", directoryID) request := ListFilesRequest{ FolderID: directoryID, @@ -101,7 +101,7 @@ func (f *Fs) listFiles(directoryID int) (filesList *FilesList, err error) { filesList = &FilesList{} err = f.pacer.Call(func() (bool, error) { - resp, err := f.rest.CallJSON(&opts, &request, filesList) + resp, err := f.rest.CallJSON(ctx, &opts, &request, filesList) return shouldRetry(resp, err) }) if err != nil { @@ -111,7 +111,7 @@ func (f *Fs) listFiles(directoryID int) (filesList *FilesList, err error) { return filesList, nil } -func (f *Fs) listFolders(directoryID int) (foldersList *FoldersList, err error) { +func (f *Fs) listFolders(ctx context.Context, directoryID int) (foldersList *FoldersList, err error) { // fs.Debugf(f, "Requesting folders for id `%s`", directoryID) request := ListFolderRequest{ @@ -125,7 +125,7 @@ func (f *Fs) listFolders(directoryID int) (foldersList *FoldersList, err error) foldersList = &FoldersList{} err = f.pacer.Call(func() (bool, error) { - resp, err := f.rest.CallJSON(&opts, &request, foldersList) + resp, err := f.rest.CallJSON(ctx, &opts, &request, foldersList) return shouldRetry(resp, err) }) if err != nil { @@ -153,12 +153,12 @@ func (f *Fs) listDir(ctx context.Context, dir string) (entries fs.DirEntries, er return nil, err } - files, err := f.listFiles(folderID) + files, err := f.listFiles(ctx, folderID) if err != nil { return nil, err } - folders, err := f.listFolders(folderID) + folders, err := f.listFolders(ctx, folderID) if err != nil { return nil, err } @@ -205,7 +205,7 @@ func getRemote(dir, fileName string) string { return dir + "/" + fileName } -func (f *Fs) makeFolder(leaf string, folderID int) (response *MakeFolderResponse, err error) { +func (f *Fs) makeFolder(ctx context.Context, leaf string, folderID int) (response *MakeFolderResponse, err error) { name := replaceReservedChars(leaf) // fs.Debugf(f, "Creating folder `%s` in id `%s`", name, directoryID) @@ -221,7 +221,7 @@ func (f *Fs) makeFolder(leaf string, folderID int) (response *MakeFolderResponse response = &MakeFolderResponse{} err = f.pacer.Call(func() (bool, error) { - resp, err := f.rest.CallJSON(&opts, &request, response) + resp, err := f.rest.CallJSON(ctx, &opts, &request, response) return shouldRetry(resp, err) }) if err != nil { @@ -233,7 +233,7 @@ func (f *Fs) makeFolder(leaf string, folderID int) (response *MakeFolderResponse return response, err } -func (f *Fs) removeFolder(name string, folderID int) (response *GenericOKResponse, err error) { +func (f *Fs) removeFolder(ctx context.Context, name string, folderID int) (response *GenericOKResponse, err error) { // fs.Debugf(f, "Removing folder with id `%s`", directoryID) request := &RemoveFolderRequest{ @@ -248,7 +248,7 @@ func (f *Fs) removeFolder(name string, folderID int) (response *GenericOKRespons response = &GenericOKResponse{} var resp *http.Response err = f.pacer.Call(func() (bool, error) { - resp, err = f.rest.CallJSON(&opts, request, response) + resp, err = f.rest.CallJSON(ctx, &opts, request, response) return shouldRetry(resp, err) }) if err != nil { @@ -263,7 +263,7 @@ func (f *Fs) removeFolder(name string, folderID int) (response *GenericOKRespons return response, nil } -func (f *Fs) deleteFile(url string) (response *GenericOKResponse, err error) { +func (f *Fs) deleteFile(ctx context.Context, url string) (response *GenericOKResponse, err error) { request := &RemoveFileRequest{ Files: []RmFile{ {url}, @@ -277,7 +277,7 @@ func (f *Fs) deleteFile(url string) (response *GenericOKResponse, err error) { response = &GenericOKResponse{} err = f.pacer.Call(func() (bool, error) { - resp, err := f.rest.CallJSON(&opts, request, response) + resp, err := f.rest.CallJSON(ctx, &opts, request, response) return shouldRetry(resp, err) }) @@ -290,7 +290,7 @@ func (f *Fs) deleteFile(url string) (response *GenericOKResponse, err error) { return response, nil } -func (f *Fs) getUploadNode() (response *GetUploadNodeResponse, err error) { +func (f *Fs) getUploadNode(ctx context.Context) (response *GetUploadNodeResponse, err error) { // fs.Debugf(f, "Requesting Upload node") opts := rest.Opts{ @@ -301,7 +301,7 @@ func (f *Fs) getUploadNode() (response *GetUploadNodeResponse, err error) { response = &GetUploadNodeResponse{} err = f.pacer.Call(func() (bool, error) { - resp, err := f.rest.CallJSON(&opts, nil, response) + resp, err := f.rest.CallJSON(ctx, &opts, nil, response) return shouldRetry(resp, err) }) if err != nil { @@ -313,7 +313,7 @@ func (f *Fs) getUploadNode() (response *GetUploadNodeResponse, err error) { return response, err } -func (f *Fs) uploadFile(in io.Reader, size int64, fileName, folderID, uploadID, node string) (response *http.Response, err error) { +func (f *Fs) uploadFile(ctx context.Context, in io.Reader, size int64, fileName, folderID, uploadID, node string) (response *http.Response, err error) { // fs.Debugf(f, "Uploading File `%s`", fileName) fileName = replaceReservedChars(fileName) @@ -343,7 +343,7 @@ func (f *Fs) uploadFile(in io.Reader, size int64, fileName, folderID, uploadID, } err = f.pacer.CallNoRetry(func() (bool, error) { - resp, err := f.rest.CallJSON(&opts, nil, nil) + resp, err := f.rest.CallJSON(ctx, &opts, nil, nil) return shouldRetry(resp, err) }) @@ -356,7 +356,7 @@ func (f *Fs) uploadFile(in io.Reader, size int64, fileName, folderID, uploadID, return response, err } -func (f *Fs) endUpload(uploadID string, nodeurl string) (response *EndFileUploadResponse, err error) { +func (f *Fs) endUpload(ctx context.Context, uploadID string, nodeurl string) (response *EndFileUploadResponse, err error) { // fs.Debugf(f, "Ending File Upload `%s`", uploadID) if len(uploadID) > 10 || !isAlphaNumeric(uploadID) { @@ -377,7 +377,7 @@ func (f *Fs) endUpload(uploadID string, nodeurl string) (response *EndFileUpload response = &EndFileUploadResponse{} err = f.pacer.Call(func() (bool, error) { - resp, err := f.rest.CallJSON(&opts, nil, response) + resp, err := f.rest.CallJSON(ctx, &opts, nil, response) return shouldRetry(resp, err) }) diff --git a/backend/fichier/fichier.go b/backend/fichier/fichier.go index 96153d6b5..2802d79d4 100644 --- a/backend/fichier/fichier.go +++ b/backend/fichier/fichier.go @@ -74,7 +74,7 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin if err != nil { return "", false, err } - folders, err := f.listFolders(folderID) + folders, err := f.listFolders(ctx, folderID) if err != nil { return "", false, err } @@ -95,7 +95,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, if err != nil { return "", err } - resp, err := f.makeFolder(leaf, folderID) + resp, err := f.makeFolder(ctx, leaf, folderID) if err != nil { return "", err } @@ -251,7 +251,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { if err != nil { return nil, err } - files, err := f.listFiles(folderID) + files, err := f.listFiles(ctx, folderID) if err != nil { return nil, err } @@ -304,7 +304,7 @@ func (f *Fs) putUnchecked(ctx context.Context, in io.Reader, remote string, size return nil, fs.ErrorCantUploadEmptyFiles } - nodeResponse, err := f.getUploadNode() + nodeResponse, err := f.getUploadNode(ctx) if err != nil { return nil, err } @@ -314,12 +314,12 @@ func (f *Fs) putUnchecked(ctx context.Context, in io.Reader, remote string, size return nil, err } - _, err = f.uploadFile(in, size, leaf, directoryID, nodeResponse.ID, nodeResponse.URL) + _, err = f.uploadFile(ctx, in, size, leaf, directoryID, nodeResponse.ID, nodeResponse.URL) if err != nil { return nil, err } - fileUploadResponse, err := f.endUpload(nodeResponse.ID, nodeResponse.URL) + fileUploadResponse, err := f.endUpload(ctx, nodeResponse.ID, nodeResponse.URL) if err != nil { return nil, err } @@ -393,7 +393,7 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error { return err } - _, err = f.removeFolder(dir, folderID) + _, err = f.removeFolder(ctx, dir, folderID) if err != nil { return err } diff --git a/backend/fichier/object.go b/backend/fichier/object.go index 5b4eb5933..f8e918722 100644 --- a/backend/fichier/object.go +++ b/backend/fichier/object.go @@ -75,7 +75,7 @@ func (o *Object) SetModTime(context.Context, time.Time) error { // Open opens the file for read. Call Close() on the returned io.ReadCloser func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) { fs.FixRangeOption(options, int64(o.file.Size)) - downloadToken, err := o.fs.getDownloadToken(o.file.URL) + downloadToken, err := o.fs.getDownloadToken(ctx, o.file.URL) if err != nil { return nil, err @@ -89,7 +89,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo } err = o.fs.pacer.Call(func() (bool, error) { - resp, err = o.fs.rest.Call(&opts) + resp, err = o.fs.rest.Call(ctx, &opts) return shouldRetry(resp, err) }) @@ -131,7 +131,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op func (o *Object) Remove(ctx context.Context) error { // fs.Debugf(f, "Removing file `%s` with url `%s`", o.file.Filename, o.file.URL) - _, err := o.fs.deleteFile(o.file.URL) + _, err := o.fs.deleteFile(ctx, o.file.URL) if err != nil { return err diff --git a/backend/googlephotos/googlephotos.go b/backend/googlephotos/googlephotos.go index 0bb996253..3a8775222 100644 --- a/backend/googlephotos/googlephotos.go +++ b/backend/googlephotos/googlephotos.go @@ -290,7 +290,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { } // fetchEndpoint gets the openid endpoint named from the Google config -func (f *Fs) fetchEndpoint(name string) (endpoint string, err error) { +func (f *Fs) fetchEndpoint(ctx context.Context, name string) (endpoint string, err error) { // Get openID config without auth opts := rest.Opts{ Method: "GET", @@ -298,7 +298,7 @@ func (f *Fs) fetchEndpoint(name string) (endpoint string, err error) { } var openIDconfig map[string]interface{} err = f.pacer.Call(func() (bool, error) { - resp, err := f.unAuth.CallJSON(&opts, nil, &openIDconfig) + resp, err := f.unAuth.CallJSON(ctx, &opts, nil, &openIDconfig) return shouldRetry(resp, err) }) if err != nil { @@ -316,7 +316,7 @@ func (f *Fs) fetchEndpoint(name string) (endpoint string, err error) { // UserInfo fetches info about the current user with oauth2 func (f *Fs) UserInfo(ctx context.Context) (userInfo map[string]string, err error) { - endpoint, err := f.fetchEndpoint("userinfo_endpoint") + endpoint, err := f.fetchEndpoint(ctx, "userinfo_endpoint") if err != nil { return nil, err } @@ -327,7 +327,7 @@ func (f *Fs) UserInfo(ctx context.Context) (userInfo map[string]string, err erro RootURL: endpoint, } err = f.pacer.Call(func() (bool, error) { - resp, err := f.srv.CallJSON(&opts, nil, &userInfo) + resp, err := f.srv.CallJSON(ctx, &opts, nil, &userInfo) return shouldRetry(resp, err) }) if err != nil { @@ -338,7 +338,7 @@ func (f *Fs) UserInfo(ctx context.Context) (userInfo map[string]string, err erro // Disconnect kills the token and refresh token func (f *Fs) Disconnect(ctx context.Context) (err error) { - endpoint, err := f.fetchEndpoint("revocation_endpoint") + endpoint, err := f.fetchEndpoint(ctx, "revocation_endpoint") if err != nil { return err } @@ -358,7 +358,7 @@ func (f *Fs) Disconnect(ctx context.Context) (err error) { } var res interface{} err = f.pacer.Call(func() (bool, error) { - resp, err := f.srv.CallJSON(&opts, nil, &res) + resp, err := f.srv.CallJSON(ctx, &opts, nil, &res) return shouldRetry(resp, err) }) if err != nil { @@ -423,7 +423,7 @@ func findID(name string) string { // list the albums into an internal cache // FIXME cache invalidation -func (f *Fs) listAlbums(shared bool) (all *albums, err error) { +func (f *Fs) listAlbums(ctx context.Context, shared bool) (all *albums, err error) { f.albumsMu.Lock() defer f.albumsMu.Unlock() all, ok := f.albums[shared] @@ -445,7 +445,7 @@ func (f *Fs) listAlbums(shared bool) (all *albums, err error) { var result api.ListAlbums var resp *http.Response err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallJSON(&opts, nil, &result) + resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) return shouldRetry(resp, err) }) if err != nil { @@ -482,7 +482,7 @@ type listFn func(remote string, object *api.MediaItem, isDirectory bool) error // dir is the starting directory, "" for root // // Set recurse to read sub directories -func (f *Fs) list(filter api.SearchFilter, fn listFn) (err error) { +func (f *Fs) list(ctx context.Context, filter api.SearchFilter, fn listFn) (err error) { opts := rest.Opts{ Method: "POST", Path: "/mediaItems:search", @@ -494,7 +494,7 @@ func (f *Fs) list(filter api.SearchFilter, fn listFn) (err error) { var result api.MediaItems var resp *http.Response err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallJSON(&opts, &filter, &result) + resp, err = f.srv.CallJSON(ctx, &opts, &filter, &result) return shouldRetry(resp, err) }) if err != nil { @@ -543,7 +543,7 @@ func (f *Fs) itemToDirEntry(ctx context.Context, remote string, item *api.MediaI // listDir lists a single directory func (f *Fs) listDir(ctx context.Context, prefix string, filter api.SearchFilter) (entries fs.DirEntries, err error) { // List the objects - err = f.list(filter, func(remote string, item *api.MediaItem, isDirectory bool) error { + err = f.list(ctx, filter, func(remote string, item *api.MediaItem, isDirectory bool) error { entry, err := f.itemToDirEntry(ctx, prefix+remote, item, isDirectory) if err != nil { return err @@ -638,7 +638,7 @@ func (f *Fs) createAlbum(ctx context.Context, albumTitle string) (album *api.Alb var result api.Album var resp *http.Response err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallJSON(&opts, request, &result) + resp, err = f.srv.CallJSON(ctx, &opts, request, &result) return shouldRetry(resp, err) }) if err != nil { @@ -654,7 +654,7 @@ func (f *Fs) createAlbum(ctx context.Context, albumTitle string) (album *api.Alb func (f *Fs) getOrCreateAlbum(ctx context.Context, albumTitle string) (album *api.Album, err error) { f.createMu.Lock() defer f.createMu.Unlock() - albums, err := f.listAlbums(false) + albums, err := f.listAlbums(ctx, false) if err != nil { return nil, err } @@ -708,7 +708,7 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) (err error) { return err } albumTitle := match[1] - allAlbums, err := f.listAlbums(false) + allAlbums, err := f.listAlbums(ctx, false) if err != nil { return err } @@ -773,7 +773,7 @@ func (o *Object) Size() int64 { RootURL: o.downloadURL(), } err = o.fs.pacer.Call(func() (bool, error) { - resp, err = o.fs.srv.Call(&opts) + resp, err = o.fs.srv.Call(ctx, &opts) return shouldRetry(resp, err) }) if err != nil { @@ -824,7 +824,7 @@ func (o *Object) readMetaData(ctx context.Context) (err error) { var item api.MediaItem var resp *http.Response err = o.fs.pacer.Call(func() (bool, error) { - resp, err = o.fs.srv.CallJSON(&opts, nil, &item) + resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &item) return shouldRetry(resp, err) }) if err != nil { @@ -901,7 +901,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read Options: options, } err = o.fs.pacer.Call(func() (bool, error) { - resp, err = o.fs.srv.Call(&opts) + resp, err = o.fs.srv.Call(ctx, &opts) return shouldRetry(resp, err) }) if err != nil { @@ -954,7 +954,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op var token []byte var resp *http.Response err = o.fs.pacer.CallNoRetry(func() (bool, error) { - resp, err = o.fs.srv.Call(&opts) + resp, err = o.fs.srv.Call(ctx, &opts) if err != nil { return shouldRetry(resp, err) } @@ -986,7 +986,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op } var result api.BatchCreateResponse err = o.fs.pacer.Call(func() (bool, error) { - resp, err = o.fs.srv.CallJSON(&opts, request, &result) + resp, err = o.fs.srv.CallJSON(ctx, &opts, request, &result) return shouldRetry(resp, err) }) if err != nil { @@ -1029,7 +1029,7 @@ func (o *Object) Remove(ctx context.Context) (err error) { } var resp *http.Response err = o.fs.pacer.Call(func() (bool, error) { - resp, err = o.fs.srv.CallJSON(&opts, &request, nil) + resp, err = o.fs.srv.CallJSON(ctx, &opts, &request, nil) return shouldRetry(resp, err) }) if err != nil { diff --git a/backend/googlephotos/pattern.go b/backend/googlephotos/pattern.go index 2dafede39..198a82c06 100644 --- a/backend/googlephotos/pattern.go +++ b/backend/googlephotos/pattern.go @@ -20,7 +20,7 @@ import ( // file pattern parsing type lister interface { listDir(ctx context.Context, prefix string, filter api.SearchFilter) (entries fs.DirEntries, err error) - listAlbums(shared bool) (all *albums, err error) + listAlbums(ctx context.Context, shared bool) (all *albums, err error) listUploads(ctx context.Context, dir string) (entries fs.DirEntries, err error) dirTime() time.Time } @@ -296,7 +296,7 @@ func yearMonthDayFilter(ctx context.Context, f lister, match []string) (sf api.S // is a prefix of another album, or actual files, or a combination of // the two. func albumsToEntries(ctx context.Context, f lister, shared bool, prefix string, albumPath string) (entries fs.DirEntries, err error) { - albums, err := f.listAlbums(shared) + albums, err := f.listAlbums(ctx, shared) if err != nil { return nil, err } diff --git a/backend/googlephotos/pattern_test.go b/backend/googlephotos/pattern_test.go index 2bdd7c28a..2fc2648e1 100644 --- a/backend/googlephotos/pattern_test.go +++ b/backend/googlephotos/pattern_test.go @@ -44,7 +44,7 @@ func (f *testLister) listDir(ctx context.Context, prefix string, filter api.Sear } // mock listAlbums for testing -func (f *testLister) listAlbums(shared bool) (all *albums, err error) { +func (f *testLister) listAlbums(ctx context.Context, shared bool) (all *albums, err error) { return f.albums, nil } diff --git a/backend/jottacloud/jottacloud.go b/backend/jottacloud/jottacloud.go index cb3c726b0..0d365230b 100644 --- a/backend/jottacloud/jottacloud.go +++ b/backend/jottacloud/jottacloud.go @@ -77,6 +77,7 @@ func init() { Description: "JottaCloud", NewFs: NewFs, Config: func(name string, m configmap.Mapper) { + ctx := context.TODO() tokenString, ok := m.Get("token") if ok && tokenString != "" { fmt.Printf("Already have a token - refresh?\n") @@ -88,7 +89,7 @@ func init() { srv := rest.NewClient(fshttp.NewClient(fs.Config)) fmt.Printf("\nDo you want to create a machine specific API key?\n\nRclone has it's own Jottacloud API KEY which works fine as long as one only uses rclone on a single machine. When you want to use rclone with this account on more than one machine it's recommended to create a machine specific API key. These keys can NOT be shared between machines.\n\n") if config.Confirm() { - deviceRegistration, err := registerDevice(srv) + deviceRegistration, err := registerDevice(ctx, srv) if err != nil { log.Fatalf("Failed to register device: %v", err) } @@ -113,7 +114,7 @@ func init() { username := config.ReadLine() password := config.GetPassword("Your Jottacloud password is only required during setup and will not be stored.") - token, err := doAuth(srv, username, password) + token, err := doAuth(ctx, srv, username, password) if err != nil { log.Fatalf("Failed to get oauth token: %s", err) } @@ -132,7 +133,7 @@ func init() { srv = rest.NewClient(oAuthClient).SetRoot(rootURL) apiSrv := rest.NewClient(oAuthClient).SetRoot(apiURL) - device, mountpoint, err := setupMountpoint(srv, apiSrv) + device, mountpoint, err := setupMountpoint(ctx, srv, apiSrv) if err != nil { log.Fatalf("Failed to setup mountpoint: %s", err) } @@ -246,7 +247,7 @@ func shouldRetry(resp *http.Response, err error) (bool, error) { } // registerDevice register a new device for use with the jottacloud API -func registerDevice(srv *rest.Client) (reg *api.DeviceRegistrationResponse, err error) { +func registerDevice(ctx context.Context, srv *rest.Client) (reg *api.DeviceRegistrationResponse, err error) { // random generator to generate random device names seededRand := rand.New(rand.NewSource(time.Now().UnixNano())) randonDeviceNamePartLength := 21 @@ -269,12 +270,12 @@ func registerDevice(srv *rest.Client) (reg *api.DeviceRegistrationResponse, err } var deviceRegistration *api.DeviceRegistrationResponse - _, err = srv.CallJSON(&opts, nil, &deviceRegistration) + _, err = srv.CallJSON(ctx, &opts, nil, &deviceRegistration) return deviceRegistration, err } // doAuth runs the actual token request -func doAuth(srv *rest.Client, username, password string) (token oauth2.Token, err error) { +func doAuth(ctx context.Context, srv *rest.Client, username, password string) (token oauth2.Token, err error) { // prepare out token request with username and password values := url.Values{} values.Set("grant_type", "PASSWORD") @@ -291,7 +292,7 @@ func doAuth(srv *rest.Client, username, password string) (token oauth2.Token, er // do the first request var jsonToken api.TokenJSON - resp, err := srv.CallJSON(&opts, nil, &jsonToken) + resp, err := srv.CallJSON(ctx, &opts, nil, &jsonToken) if err != nil { // if 2fa is enabled the first request is expected to fail. We will do another request with the 2fa code as an additional http header if resp != nil { @@ -303,7 +304,7 @@ func doAuth(srv *rest.Client, username, password string) (token oauth2.Token, er authCode = strings.Replace(authCode, "-", "", -1) // remove any "-" contained in the code so we have a 6 digit number opts.ExtraHeaders = make(map[string]string) opts.ExtraHeaders["X-Jottacloud-Otp"] = authCode - resp, err = srv.CallJSON(&opts, nil, &jsonToken) + resp, err = srv.CallJSON(ctx, &opts, nil, &jsonToken) } } } @@ -316,13 +317,13 @@ func doAuth(srv *rest.Client, username, password string) (token oauth2.Token, er } // setupMountpoint sets up a custom device and mountpoint if desired by the user -func setupMountpoint(srv *rest.Client, apiSrv *rest.Client) (device, mountpoint string, err error) { - cust, err := getCustomerInfo(apiSrv) +func setupMountpoint(ctx context.Context, srv *rest.Client, apiSrv *rest.Client) (device, mountpoint string, err error) { + cust, err := getCustomerInfo(ctx, apiSrv) if err != nil { return "", "", err } - acc, err := getDriveInfo(srv, cust.Username) + acc, err := getDriveInfo(ctx, srv, cust.Username) if err != nil { return "", "", err } @@ -333,7 +334,7 @@ func setupMountpoint(srv *rest.Client, apiSrv *rest.Client) (device, mountpoint fmt.Printf("Please select the device to use. Normally this will be Jotta\n") device = config.Choose("Devices", deviceNames, nil, false) - dev, err := getDeviceInfo(srv, path.Join(cust.Username, device)) + dev, err := getDeviceInfo(ctx, srv, path.Join(cust.Username, device)) if err != nil { return "", "", err } @@ -351,13 +352,13 @@ func setupMountpoint(srv *rest.Client, apiSrv *rest.Client) (device, mountpoint } // getCustomerInfo queries general information about the account -func getCustomerInfo(srv *rest.Client) (info *api.CustomerInfo, err error) { +func getCustomerInfo(ctx context.Context, srv *rest.Client) (info *api.CustomerInfo, err error) { opts := rest.Opts{ Method: "GET", Path: "account/v1/customer", } - _, err = srv.CallJSON(&opts, nil, &info) + _, err = srv.CallJSON(ctx, &opts, nil, &info) if err != nil { return nil, errors.Wrap(err, "couldn't get customer info") } @@ -366,13 +367,13 @@ func getCustomerInfo(srv *rest.Client) (info *api.CustomerInfo, err error) { } // getDriveInfo queries general information about the account and the available devices and mountpoints. -func getDriveInfo(srv *rest.Client, username string) (info *api.DriveInfo, err error) { +func getDriveInfo(ctx context.Context, srv *rest.Client, username string) (info *api.DriveInfo, err error) { opts := rest.Opts{ Method: "GET", Path: username, } - _, err = srv.CallXML(&opts, nil, &info) + _, err = srv.CallXML(ctx, &opts, nil, &info) if err != nil { return nil, errors.Wrap(err, "couldn't get drive info") } @@ -381,13 +382,13 @@ func getDriveInfo(srv *rest.Client, username string) (info *api.DriveInfo, err e } // getDeviceInfo queries Information about a jottacloud device -func getDeviceInfo(srv *rest.Client, path string) (info *api.JottaDevice, err error) { +func getDeviceInfo(ctx context.Context, srv *rest.Client, path string) (info *api.JottaDevice, err error) { opts := rest.Opts{ Method: "GET", Path: urlPathEscape(path), } - _, err = srv.CallXML(&opts, nil, &info) + _, err = srv.CallXML(ctx, &opts, nil, &info) if err != nil { return nil, errors.Wrap(err, "couldn't get device info") } @@ -407,7 +408,7 @@ func (f *Fs) setEndpointURL() { } // readMetaDataForPath reads the metadata from the path -func (f *Fs) readMetaDataForPath(path string) (info *api.JottaFile, err error) { +func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.JottaFile, err error) { opts := rest.Opts{ Method: "GET", Path: f.filePath(path), @@ -415,7 +416,7 @@ func (f *Fs) readMetaDataForPath(path string) (info *api.JottaFile, err error) { var result api.JottaFile var resp *http.Response err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallXML(&opts, nil, &result) + resp, err = f.srv.CallXML(ctx, &opts, nil, &result) return shouldRetry(resp, err) }) @@ -492,6 +493,7 @@ func grantTypeFilter(req *http.Request) { // NewFs constructs an Fs from the path, container:path func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { + ctx := context.TODO() // Parse config into Options struct opt := new(Options) err := configstruct.Set(m, opt) @@ -546,11 +548,11 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { // Renew the token in the background f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error { - _, err := f.readMetaDataForPath("") + _, err := f.readMetaDataForPath(ctx, "") return err }) - cust, err := getCustomerInfo(f.apiSrv) + cust, err := getCustomerInfo(ctx, f.apiSrv) if err != nil { return nil, err } @@ -582,7 +584,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { // Return an Object from a path // // If it can't be found it returns the error fs.ErrorObjectNotFound. -func (f *Fs) newObjectWithInfo(remote string, info *api.JottaFile) (fs.Object, error) { +func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *api.JottaFile) (fs.Object, error) { o := &Object{ fs: f, remote: remote, @@ -592,7 +594,7 @@ func (f *Fs) newObjectWithInfo(remote string, info *api.JottaFile) (fs.Object, e // Set info err = o.setMetaData(info) } else { - err = o.readMetaData(false) // reads info and meta, returning an error + err = o.readMetaData(ctx, false) // reads info and meta, returning an error } if err != nil { return nil, err @@ -603,11 +605,11 @@ func (f *Fs) newObjectWithInfo(remote string, info *api.JottaFile) (fs.Object, e // NewObject finds the Object at remote. If it can't be found // it returns the error fs.ErrorObjectNotFound. func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { - return f.newObjectWithInfo(remote, nil) + return f.newObjectWithInfo(ctx, remote, nil) } // CreateDir makes a directory -func (f *Fs) CreateDir(path string) (jf *api.JottaFolder, err error) { +func (f *Fs) CreateDir(ctx context.Context, path string) (jf *api.JottaFolder, err error) { // fs.Debugf(f, "CreateDir(%q, %q)\n", pathID, leaf) var resp *http.Response opts := rest.Opts{ @@ -619,7 +621,7 @@ func (f *Fs) CreateDir(path string) (jf *api.JottaFolder, err error) { opts.Parameters.Set("mkDir", "true") err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallXML(&opts, nil, &jf) + resp, err = f.srv.CallXML(ctx, &opts, nil, &jf) return shouldRetry(resp, err) }) if err != nil { @@ -648,7 +650,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e var resp *http.Response var result api.JottaFolder err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallXML(&opts, nil, &result) + resp, err = f.srv.CallXML(ctx, &opts, nil, &result) return shouldRetry(resp, err) }) @@ -682,7 +684,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e continue } remote := path.Join(dir, restoreReservedChars(item.Name)) - o, err := f.newObjectWithInfo(remote, item) + o, err := f.newObjectWithInfo(ctx, remote, item) if err != nil { continue } @@ -696,7 +698,7 @@ type listFileDirFn func(fs.DirEntry) error // List the objects and directories into entries, from a // special kind of JottaFolder representing a FileDirLis -func (f *Fs) listFileDir(remoteStartPath string, startFolder *api.JottaFolder, fn listFileDirFn) error { +func (f *Fs) listFileDir(ctx context.Context, remoteStartPath string, startFolder *api.JottaFolder, fn listFileDirFn) error { pathPrefix := "/" + f.filePathRaw("") // Non-escaped prefix of API paths to be cut off, to be left with the remote path including the remoteStartPath pathPrefixLength := len(pathPrefix) startPath := path.Join(pathPrefix, remoteStartPath) // Non-escaped API path up to and including remoteStartPath, to decide if it should be created as a new dir object @@ -725,7 +727,7 @@ func (f *Fs) listFileDir(remoteStartPath string, startFolder *api.JottaFolder, f continue } remoteFile := path.Join(remoteDir, restoreReservedChars(file.Name)) - o, err := f.newObjectWithInfo(remoteFile, file) + o, err := f.newObjectWithInfo(ctx, remoteFile, file) if err != nil { return err } @@ -754,7 +756,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) ( var resp *http.Response var result api.JottaFolder // Could be JottaFileDirList, but JottaFolder is close enough err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallXML(&opts, nil, &result) + resp, err = f.srv.CallXML(ctx, &opts, nil, &result) return shouldRetry(resp, err) }) if err != nil { @@ -767,7 +769,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) ( return errors.Wrap(err, "couldn't list files") } list := walk.NewListRHelper(callback) - err = f.listFileDir(dir, &result, func(entry fs.DirEntry) error { + err = f.listFileDir(ctx, dir, &result, func(entry fs.DirEntry) error { return list.Add(entry) }) if err != nil { @@ -821,7 +823,7 @@ func (f *Fs) mkParentDir(ctx context.Context, dirPath string) error { // Mkdir creates the container if it doesn't exist func (f *Fs) Mkdir(ctx context.Context, dir string) error { - _, err := f.CreateDir(dir) + _, err := f.CreateDir(ctx, dir) return err } @@ -860,7 +862,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error) var resp *http.Response err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.Call(&opts) + resp, err = f.srv.Call(ctx, &opts) return shouldRetry(resp, err) }) if err != nil { @@ -888,7 +890,7 @@ func (f *Fs) Purge(ctx context.Context) error { } // copyOrMoves copies or moves directories or files depending on the method parameter -func (f *Fs) copyOrMove(method, src, dest string) (info *api.JottaFile, err error) { +func (f *Fs) copyOrMove(ctx context.Context, method, src, dest string) (info *api.JottaFile, err error) { opts := rest.Opts{ Method: "POST", Path: src, @@ -899,7 +901,7 @@ func (f *Fs) copyOrMove(method, src, dest string) (info *api.JottaFile, err erro var resp *http.Response err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallXML(&opts, nil, &info) + resp, err = f.srv.CallXML(ctx, &opts, nil, &info) return shouldRetry(resp, err) }) if err != nil { @@ -928,13 +930,13 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, if err != nil { return nil, err } - info, err := f.copyOrMove("cp", srcObj.filePath(), remote) + info, err := f.copyOrMove(ctx, "cp", srcObj.filePath(), remote) if err != nil { return nil, errors.Wrap(err, "couldn't copy file") } - return f.newObjectWithInfo(remote, info) + return f.newObjectWithInfo(ctx, remote, info) //return f.newObjectWithInfo(remote, &result) } @@ -958,13 +960,13 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, if err != nil { return nil, err } - info, err := f.copyOrMove("mv", srcObj.filePath(), remote) + info, err := f.copyOrMove(ctx, "mv", srcObj.filePath(), remote) if err != nil { return nil, errors.Wrap(err, "couldn't move file") } - return f.newObjectWithInfo(remote, info) + return f.newObjectWithInfo(ctx, remote, info) //return f.newObjectWithInfo(remote, result) } @@ -1002,7 +1004,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string return fs.ErrorDirExists } - _, err = f.copyOrMove("mvDir", path.Join(f.endpointURL, replaceReservedChars(srcPath))+"/", dstRemote) + _, err = f.copyOrMove(ctx, "mvDir", path.Join(f.endpointURL, replaceReservedChars(srcPath))+"/", dstRemote) if err != nil { return errors.Wrap(err, "couldn't move directory") @@ -1027,7 +1029,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err er var resp *http.Response var result api.JottaFile err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallXML(&opts, nil, &result) + resp, err = f.srv.CallXML(ctx, &opts, nil, &result) return shouldRetry(resp, err) }) @@ -1058,7 +1060,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err er // About gets quota information func (f *Fs) About(ctx context.Context) (*fs.Usage, error) { - info, err := getDriveInfo(f.srv, f.user) + info, err := getDriveInfo(ctx, f.srv, f.user) if err != nil { return nil, err } @@ -1113,7 +1115,8 @@ func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) { // Size returns the size of an object in bytes func (o *Object) Size() int64 { - err := o.readMetaData(false) + ctx := context.TODO() + err := o.readMetaData(ctx, false) if err != nil { fs.Logf(o, "Failed to read metadata: %v", err) return 0 @@ -1137,11 +1140,11 @@ func (o *Object) setMetaData(info *api.JottaFile) (err error) { } // readMetaData reads and updates the metadata for an object -func (o *Object) readMetaData(force bool) (err error) { +func (o *Object) readMetaData(ctx context.Context, force bool) (err error) { if o.hasMetaData && !force { return nil } - info, err := o.fs.readMetaDataForPath(o.remote) + info, err := o.fs.readMetaDataForPath(ctx, o.remote) if err != nil { return err } @@ -1156,7 +1159,7 @@ func (o *Object) readMetaData(force bool) (err error) { // It attempts to read the objects mtime and if that isn't present the // LastModified returned in the http headers func (o *Object) ModTime(ctx context.Context) time.Time { - err := o.readMetaData(false) + err := o.readMetaData(ctx, false) if err != nil { fs.Logf(o, "Failed to read metadata: %v", err) return time.Now() @@ -1188,7 +1191,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read opts.Parameters.Set("mode", "bin") err = o.fs.pacer.Call(func() (bool, error) { - resp, err = o.fs.srv.Call(&opts) + resp, err = o.fs.srv.Call(ctx, &opts) return shouldRetry(resp, err) }) if err != nil { @@ -1298,7 +1301,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op // send it var response api.AllocateFileResponse err = o.fs.pacer.CallNoRetry(func() (bool, error) { - resp, err = o.fs.apiSrv.CallJSON(&opts, &request, &response) + resp, err = o.fs.apiSrv.CallJSON(ctx, &opts, &request, &response) return shouldRetry(resp, err) }) if err != nil { @@ -1329,7 +1332,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op } // send the remaining bytes - resp, err = o.fs.apiSrv.CallJSON(&opts, nil, &result) + resp, err = o.fs.apiSrv.CallJSON(ctx, &opts, nil, &result) if err != nil { return err } @@ -1341,7 +1344,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op o.modTime = time.Unix(result.Modified/1000, 0) } else { // If the file state is COMPLETE we don't need to upload it because the file was already found but we still ned to update our metadata - return o.readMetaData(true) + return o.readMetaData(ctx, true) } return nil @@ -1363,7 +1366,7 @@ func (o *Object) Remove(ctx context.Context) error { } return o.fs.pacer.Call(func() (bool, error) { - resp, err := o.fs.srv.CallXML(&opts, nil, nil) + resp, err := o.fs.srv.CallXML(ctx, &opts, nil, nil) return shouldRetry(resp, err) }) } diff --git a/backend/mailru/mailru.go b/backend/mailru/mailru.go index 594ab952a..e61206b01 100644 --- a/backend/mailru/mailru.go +++ b/backend/mailru/mailru.go @@ -535,7 +535,7 @@ func (f *Fs) relPath(absPath string) (string, error) { } // metaServer ... -func (f *Fs) metaServer() (string, error) { +func (f *Fs) metaServer(ctx context.Context) (string, error) { f.metaMu.Lock() defer f.metaMu.Unlock() @@ -555,7 +555,7 @@ func (f *Fs) metaServer() (string, error) { err error ) err = f.pacer.Call(func() (bool, error) { - res, err = f.srv.Call(&opts) + res, err = f.srv.Call(ctx, &opts) if err == nil { url, err = readBodyWord(res) } @@ -608,7 +608,7 @@ func (f *Fs) readItemMetaData(ctx context.Context, path string) (entry fs.DirEnt var info api.ItemInfoResponse err = f.pacer.Call(func() (bool, error) { - res, err := f.srv.CallJSON(&opts, nil, &info) + res, err := f.srv.CallJSON(ctx, &opts, nil, &info) return shouldRetry(res, err, f, &opts) }) @@ -716,7 +716,7 @@ func (f *Fs) listM1(ctx context.Context, dirPath string, offset int, limit int) res *http.Response ) err = f.pacer.Call(func() (bool, error) { - res, err = f.srv.CallJSON(&opts, nil, &info) + res, err = f.srv.CallJSON(ctx, &opts, nil, &info) return shouldRetry(res, err, f, &opts) }) @@ -758,7 +758,7 @@ func (f *Fs) listBin(ctx context.Context, dirPath string, depth int) (entries fs if err != nil { return nil, err } - metaURL, err := f.metaServer() + metaURL, err := f.metaServer(ctx) if err != nil { return nil, err } @@ -776,7 +776,7 @@ func (f *Fs) listBin(ctx context.Context, dirPath string, depth int) (entries fs var res *http.Response err = f.pacer.Call(func() (bool, error) { - res, err = f.srv.Call(&opts) + res, err = f.srv.Call(ctx, &opts) return shouldRetry(res, err, f, &opts) }) if err != nil { @@ -1031,7 +1031,7 @@ func (f *Fs) CreateDir(ctx context.Context, path string) error { if err != nil { return err } - metaURL, err := f.metaServer() + metaURL, err := f.metaServer(ctx) if err != nil { return err } @@ -1049,7 +1049,7 @@ func (f *Fs) CreateDir(ctx context.Context, path string) error { var res *http.Response err = f.pacer.Call(func() (bool, error) { - res, err = f.srv.Call(&opts) + res, err = f.srv.Call(ctx, &opts) return shouldRetry(res, err, f, &opts) }) if err != nil { @@ -1192,7 +1192,7 @@ func (f *Fs) delete(ctx context.Context, path string, hardDelete bool) error { var response api.GenericResponse err = f.pacer.Call(func() (bool, error) { - res, err := f.srv.CallJSON(&opts, nil, &response) + res, err := f.srv.CallJSON(ctx, &opts, nil, &response) return shouldRetry(res, err, f, &opts) }) @@ -1264,7 +1264,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, var response api.GenericBodyResponse err = f.pacer.Call(func() (bool, error) { - res, err := f.srv.CallJSON(&opts, nil, &response) + res, err := f.srv.CallJSON(ctx, &opts, nil, &response) return shouldRetry(res, err, f, &opts) }) @@ -1342,7 +1342,7 @@ func (f *Fs) moveItemBin(ctx context.Context, srcPath, dstPath, opName string) e if err != nil { return err } - metaURL, err := f.metaServer() + metaURL, err := f.metaServer(ctx) if err != nil { return err } @@ -1368,7 +1368,7 @@ func (f *Fs) moveItemBin(ctx context.Context, srcPath, dstPath, opName string) e var res *http.Response err = f.pacer.Call(func() (bool, error) { - res, err = f.srv.Call(&opts) + res, err = f.srv.Call(ctx, &opts) return shouldRetry(res, err, f, &opts) }) if err != nil { @@ -1459,7 +1459,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err er var response api.GenericBodyResponse err = f.pacer.Call(func() (bool, error) { - res, err := f.srv.CallJSON(&opts, nil, &response) + res, err := f.srv.CallJSON(ctx, &opts, nil, &response) return shouldRetry(res, err, f, &opts) }) @@ -1500,7 +1500,7 @@ func (f *Fs) CleanUp(ctx context.Context) error { var response api.CleanupResponse err = f.pacer.Call(func() (bool, error) { - res, err := f.srv.CallJSON(&opts, nil, &response) + res, err := f.srv.CallJSON(ctx, &opts, nil, &response) return shouldRetry(res, err, f, &opts) }) if err != nil { @@ -1533,7 +1533,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) { var info api.UserInfoResponse err = f.pacer.Call(func() (bool, error) { - res, err := f.srv.CallJSON(&opts, nil, &info) + res, err := f.srv.CallJSON(ctx, &opts, nil, &info) return shouldRetry(res, err, f, &opts) }) if err != nil { @@ -1664,7 +1664,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op hasher = mrhash.New() wrapIn = io.TeeReader(wrapIn, hasher) } - newHash, err = o.upload(wrapIn, size, options...) + newHash, err = o.upload(ctx, wrapIn, size, options...) if fileHash == nil && err == nil { fileHash = hasher.Sum(nil) } @@ -1783,12 +1783,12 @@ func makeTempFile(ctx context.Context, tmpFs fs.Fs, wrapIn io.Reader, src fs.Obj return } -func (o *Object) upload(in io.Reader, size int64, options ...fs.OpenOption) ([]byte, error) { +func (o *Object) upload(ctx context.Context, in io.Reader, size int64, options ...fs.OpenOption) ([]byte, error) { token, err := o.fs.accessToken() if err != nil { return nil, err } - shardURL, err := o.fs.uploadShard() + shardURL, err := o.fs.uploadShard(ctx) if err != nil { return nil, err } @@ -1813,7 +1813,7 @@ func (o *Object) upload(in io.Reader, size int64, options ...fs.OpenOption) ([]b strHash string ) err = o.fs.pacer.Call(func() (bool, error) { - res, err = o.fs.srv.Call(&opts) + res, err = o.fs.srv.Call(ctx, &opts) if err == nil { strHash, err = readBodyWord(res) } @@ -1832,7 +1832,7 @@ func (o *Object) upload(in io.Reader, size int64, options ...fs.OpenOption) ([]b } } -func (f *Fs) uploadShard() (string, error) { +func (f *Fs) uploadShard(ctx context.Context) (string, error) { f.shardMu.Lock() defer f.shardMu.Unlock() @@ -1856,7 +1856,7 @@ func (f *Fs) uploadShard() (string, error) { var info api.ShardInfoResponse err = f.pacer.Call(func() (bool, error) { - res, err := f.srv.CallJSON(&opts, nil, &info) + res, err := f.srv.CallJSON(ctx, &opts, nil, &info) return shouldRetry(res, err, f, &opts) }) if err != nil { @@ -1995,7 +1995,7 @@ func (o *Object) addFileMetaData(ctx context.Context, overwrite bool) error { if err != nil { return err } - metaURL, err := o.fs.metaServer() + metaURL, err := o.fs.metaServer(ctx) if err != nil { return err } @@ -2032,7 +2032,7 @@ func (o *Object) addFileMetaData(ctx context.Context, overwrite bool) error { var res *http.Response err = o.fs.pacer.Call(func() (bool, error) { - res, err = o.fs.srv.Call(&opts) + res, err = o.fs.srv.Call(ctx, &opts) return shouldRetry(res, err, o.fs, &opts) }) if err != nil { @@ -2115,12 +2115,12 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read var res *http.Response server := "" err = o.fs.pacer.Call(func() (bool, error) { - server, err = o.fs.fileServers.Dispatch(server) + server, err = o.fs.fileServers.Dispatch(ctx, server) if err != nil { return false, err } opts.RootURL = server - res, err = o.fs.srv.Call(&opts) + res, err = o.fs.srv.Call(ctx, &opts) return shouldRetry(res, err, o.fs, &opts) }) if err != nil { @@ -2213,7 +2213,7 @@ type pendingServer struct { // Dispatch dispatches next download server. // It prefers switching and tries to avoid current server // in use by caller because it may be overloaded or slow. -func (p *serverPool) Dispatch(current string) (string, error) { +func (p *serverPool) Dispatch(ctx context.Context, current string) (string, error) { now := time.Now() url := p.getServer(current, now) if url != "" { @@ -2231,7 +2231,7 @@ func (p *serverPool) Dispatch(current string) (string, error) { err error ) err = p.fs.pacer.Call(func() (bool, error) { - res, err = p.fs.srv.Call(&opts) + res, err = p.fs.srv.Call(ctx, &opts) if err != nil { return fserrors.ShouldRetry(err), err } diff --git a/backend/onedrive/onedrive.go b/backend/onedrive/onedrive.go index 82b9bba5f..45c98a435 100644 --- a/backend/onedrive/onedrive.go +++ b/backend/onedrive/onedrive.go @@ -72,6 +72,7 @@ func init() { Description: "Microsoft OneDrive", NewFs: NewFs, Config: func(name string, m configmap.Mapper) { + ctx := context.TODO() err := oauthutil.Config("onedrive", name, m, oauthConfig) if err != nil { log.Fatalf("Failed to configure token: %v", err) @@ -143,7 +144,7 @@ func init() { } sites := siteResponse{} - _, err := srv.CallJSON(&opts, nil, &sites) + _, err := srv.CallJSON(ctx, &opts, nil, &sites) if err != nil { log.Fatalf("Failed to query available sites: %v", err) } @@ -172,7 +173,7 @@ func init() { // query Microsoft Graph if finalDriveID == "" { drives := drivesResponse{} - _, err := srv.CallJSON(&opts, nil, &drives) + _, err := srv.CallJSON(ctx, &opts, nil, &drives) if err != nil { log.Fatalf("Failed to query available drives: %v", err) } @@ -194,7 +195,7 @@ func init() { RootURL: graphURL, Path: "/drives/" + finalDriveID + "/root"} var rootItem api.Item - _, err = srv.CallJSON(&opts, nil, &rootItem) + _, err = srv.CallJSON(ctx, &opts, nil, &rootItem) if err != nil { log.Fatalf("Failed to query root for drive %s: %v", finalDriveID, err) } @@ -343,10 +344,10 @@ func shouldRetry(resp *http.Response, err error) (bool, error) { // instead of simply using `drives/driveID/root:/itemPath` because it works for // "shared with me" folders in OneDrive Personal (See #2536, #2778) // This path pattern comes from https://github.com/OneDrive/onedrive-api-docs/issues/908#issuecomment-417488480 -func (f *Fs) readMetaDataForPathRelativeToID(normalizedID string, relPath string) (info *api.Item, resp *http.Response, err error) { +func (f *Fs) readMetaDataForPathRelativeToID(ctx context.Context, normalizedID string, relPath string) (info *api.Item, resp *http.Response, err error) { opts := newOptsCall(normalizedID, "GET", ":/"+withTrailingColon(rest.URLPathEscape(replaceReservedChars(relPath)))) err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallJSON(&opts, nil, &info) + resp, err = f.srv.CallJSON(ctx, &opts, nil, &info) return shouldRetry(resp, err) }) @@ -371,7 +372,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.It } } err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallJSON(&opts, nil, &info) + resp, err = f.srv.CallJSON(ctx, &opts, nil, &info) return shouldRetry(resp, err) }) return info, resp, err @@ -426,7 +427,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.It } } - return f.readMetaDataForPathRelativeToID(baseNormalizedID, relPath) + return f.readMetaDataForPathRelativeToID(ctx, baseNormalizedID, relPath) } // errorHandler parses a non 2xx error response into an error @@ -592,7 +593,7 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin if !ok { return "", false, errors.New("couldn't find parent ID") } - info, resp, err := f.readMetaDataForPathRelativeToID(pathID, leaf) + info, resp, err := f.readMetaDataForPathRelativeToID(ctx, pathID, leaf) if err != nil { if resp != nil && resp.StatusCode == http.StatusNotFound { return "", false, nil @@ -619,7 +620,7 @@ func (f *Fs) CreateDir(ctx context.Context, dirID, leaf string) (newID string, e ConflictBehavior: "fail", } err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallJSON(&opts, &mkdir, &info) + resp, err = f.srv.CallJSON(ctx, &opts, &mkdir, &info) return shouldRetry(resp, err) }) if err != nil { @@ -642,7 +643,7 @@ type listAllFn func(*api.Item) bool // Lists the directory required calling the user function on each item found // // If the user fn ever returns true then it early exits with found = true -func (f *Fs) listAll(dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) { +func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) { // Top parameter asks for bigger pages of data // https://dev.onedrive.com/odata/optional-query-parameters.htm opts := newOptsCall(dirID, "GET", "/children?$top=1000") @@ -651,7 +652,7 @@ OUTER: var result api.ListChildrenResponse var resp *http.Response err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallJSON(&opts, nil, &result) + resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) return shouldRetry(resp, err) }) if err != nil { @@ -709,7 +710,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e return nil, err } var iErr error - _, err = f.listAll(directoryID, false, false, func(info *api.Item) bool { + _, err = f.listAll(ctx, directoryID, false, false, func(info *api.Item) bool { if !f.opt.ExposeOneNoteFiles && info.GetPackageType() == api.PackageTypeOneNote { fs.Debugf(info.Name, "OneNote file not shown in directory listing") return false @@ -793,12 +794,12 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error { } // deleteObject removes an object by ID -func (f *Fs) deleteObject(id string) error { +func (f *Fs) deleteObject(ctx context.Context, id string) error { opts := newOptsCall(id, "DELETE", "") opts.NoResponse = true return f.pacer.Call(func() (bool, error) { - resp, err := f.srv.Call(&opts) + resp, err := f.srv.Call(ctx, &opts) return shouldRetry(resp, err) }) } @@ -821,7 +822,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error { } if check { // check to see if there are any items - found, err := f.listAll(rootID, false, false, func(item *api.Item) bool { + found, err := f.listAll(ctx, rootID, false, false, func(item *api.Item) bool { return true }) if err != nil { @@ -831,7 +832,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error { return fs.ErrorDirectoryNotEmpty } } - err = f.deleteObject(rootID) + err = f.deleteObject(ctx, rootID) if err != nil { return err } @@ -941,7 +942,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, } var resp *http.Response err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallJSON(&opts, ©Req, nil) + resp, err = f.srv.CallJSON(ctx, &opts, ©Req, nil) return shouldRetry(resp, err) }) if err != nil { @@ -1029,7 +1030,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, var resp *http.Response var info api.Item err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallJSON(&opts, &move, &info) + resp, err = f.srv.CallJSON(ctx, &opts, &move, &info) return shouldRetry(resp, err) }) if err != nil { @@ -1122,7 +1123,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string } // Get timestamps of src so they can be preserved - srcInfo, _, err := srcFs.readMetaDataForPathRelativeToID(srcID, "") + srcInfo, _, err := srcFs.readMetaDataForPathRelativeToID(ctx, srcID, "") if err != nil { return err } @@ -1144,7 +1145,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string var resp *http.Response var info api.Item err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallJSON(&opts, &move, &info) + resp, err = f.srv.CallJSON(ctx, &opts, &move, &info) return shouldRetry(resp, err) }) if err != nil { @@ -1170,7 +1171,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) { } var resp *http.Response err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallJSON(&opts, nil, &drive) + resp, err = f.srv.CallJSON(ctx, &opts, nil, &drive) return shouldRetry(resp, err) }) if err != nil { @@ -1210,7 +1211,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err er var resp *http.Response var result api.CreateShareLinkResponse err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallJSON(&opts, &share, &result) + resp, err = f.srv.CallJSON(ctx, &opts, &share, &result) return shouldRetry(resp, err) }) if err != nil { @@ -1370,7 +1371,7 @@ func (o *Object) setModTime(ctx context.Context, modTime time.Time) (*api.Item, } var info *api.Item err := o.fs.pacer.Call(func() (bool, error) { - resp, err := o.fs.srv.CallJSON(&opts, &update, &info) + resp, err := o.fs.srv.CallJSON(ctx, &opts, &update, &info) return shouldRetry(resp, err) }) return info, err @@ -1405,7 +1406,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read opts.Options = options err = o.fs.pacer.Call(func() (bool, error) { - resp, err = o.fs.srv.Call(&opts) + resp, err = o.fs.srv.Call(ctx, &opts) return shouldRetry(resp, err) }) if err != nil { @@ -1441,7 +1442,7 @@ func (o *Object) createUploadSession(ctx context.Context, modTime time.Time) (re createRequest.Item.FileSystemInfo.LastModifiedDateTime = api.Timestamp(modTime) var resp *http.Response err = o.fs.pacer.Call(func() (bool, error) { - resp, err = o.fs.srv.CallJSON(&opts, &createRequest, &response) + resp, err = o.fs.srv.CallJSON(ctx, &opts, &createRequest, &response) if apiErr, ok := err.(*api.Error); ok { if apiErr.ErrorInfo.Code == "nameAlreadyExists" { // Make the error more user-friendly @@ -1454,7 +1455,7 @@ func (o *Object) createUploadSession(ctx context.Context, modTime time.Time) (re } // uploadFragment uploads a part -func (o *Object) uploadFragment(url string, start int64, totalSize int64, chunk io.ReadSeeker, chunkSize int64) (info *api.Item, err error) { +func (o *Object) uploadFragment(ctx context.Context, url string, start int64, totalSize int64, chunk io.ReadSeeker, chunkSize int64) (info *api.Item, err error) { opts := rest.Opts{ Method: "PUT", RootURL: url, @@ -1467,7 +1468,7 @@ func (o *Object) uploadFragment(url string, start int64, totalSize int64, chunk var body []byte err = o.fs.pacer.Call(func() (bool, error) { _, _ = chunk.Seek(0, io.SeekStart) - resp, err = o.fs.srv.Call(&opts) + resp, err = o.fs.srv.Call(ctx, &opts) if err != nil { return shouldRetry(resp, err) } @@ -1487,7 +1488,7 @@ func (o *Object) uploadFragment(url string, start int64, totalSize int64, chunk } // cancelUploadSession cancels an upload session -func (o *Object) cancelUploadSession(url string) (err error) { +func (o *Object) cancelUploadSession(ctx context.Context, url string) (err error) { opts := rest.Opts{ Method: "DELETE", RootURL: url, @@ -1495,7 +1496,7 @@ func (o *Object) cancelUploadSession(url string) (err error) { } var resp *http.Response err = o.fs.pacer.Call(func() (bool, error) { - resp, err = o.fs.srv.Call(&opts) + resp, err = o.fs.srv.Call(ctx, &opts) return shouldRetry(resp, err) }) return @@ -1517,7 +1518,7 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, size int64, } fs.Debugf(o, "Cancelling multipart upload") - cancelErr := o.cancelUploadSession(uploadURL) + cancelErr := o.cancelUploadSession(ctx, uploadURL) if cancelErr != nil { fs.Logf(o, "Failed to cancel multipart upload: %v", cancelErr) } @@ -1553,7 +1554,7 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, size int64, } seg := readers.NewRepeatableReader(io.LimitReader(in, n)) fs.Debugf(o, "Uploading segment %d/%d size %d", position, size, n) - info, err = o.uploadFragment(uploadURL, position, size, seg, n) + info, err = o.uploadFragment(ctx, uploadURL, position, size, seg, n) if err != nil { return nil, err } @@ -1594,7 +1595,7 @@ func (o *Object) uploadSinglepart(ctx context.Context, in io.Reader, size int64, } err = o.fs.pacer.Call(func() (bool, error) { - resp, err = o.fs.srv.CallJSON(&opts, nil, &info) + resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &info) if apiErr, ok := err.(*api.Error); ok { if apiErr.ErrorInfo.Code == "nameAlreadyExists" { // Make the error more user-friendly @@ -1646,7 +1647,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op // Remove an object func (o *Object) Remove(ctx context.Context) error { - return o.fs.deleteObject(o.id) + return o.fs.deleteObject(ctx, o.id) } // MimeType of an Object if known, "" otherwise diff --git a/backend/opendrive/opendrive.go b/backend/opendrive/opendrive.go index 6143ec570..e1fd43113 100644 --- a/backend/opendrive/opendrive.go +++ b/backend/opendrive/opendrive.go @@ -161,7 +161,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { Method: "POST", Path: "/session/login.json", } - resp, err = f.srv.CallJSON(&opts, &account, &f.session) + resp, err = f.srv.CallJSON(ctx, &opts, &account, &f.session) return f.shouldRetry(resp, err) }) if err != nil { @@ -246,7 +246,7 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error { } // deleteObject removes an object by ID -func (f *Fs) deleteObject(id string) error { +func (f *Fs) deleteObject(ctx context.Context, id string) error { return f.pacer.Call(func() (bool, error) { removeDirData := removeFolder{SessionID: f.session.SessionID, FolderID: id} opts := rest.Opts{ @@ -254,7 +254,7 @@ func (f *Fs) deleteObject(id string) error { NoResponse: true, Path: "/folder/remove.json", } - resp, err := f.srv.CallJSON(&opts, &removeDirData, nil) + resp, err := f.srv.CallJSON(ctx, &opts, &removeDirData, nil) return f.shouldRetry(resp, err) }) } @@ -275,14 +275,14 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error { if err != nil { return err } - item, err := f.readMetaDataForFolderID(rootID) + item, err := f.readMetaDataForFolderID(ctx, rootID) if err != nil { return err } if check && len(item.Files) != 0 { return errors.New("folder not empty") } - err = f.deleteObject(rootID) + err = f.deleteObject(ctx, rootID) if err != nil { return err } @@ -353,7 +353,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, Method: "POST", Path: "/file/move_copy.json", } - resp, err = f.srv.CallJSON(&opts, ©FileData, &response) + resp, err = f.srv.CallJSON(ctx, &opts, ©FileData, &response) return f.shouldRetry(resp, err) }) if err != nil { @@ -410,7 +410,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, Method: "POST", Path: "/file/move_copy.json", } - resp, err = f.srv.CallJSON(&opts, ©FileData, &response) + resp, err = f.srv.CallJSON(ctx, &opts, ©FileData, &response) return f.shouldRetry(resp, err) }) if err != nil { @@ -509,7 +509,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string Method: "POST", Path: "/folder/move_copy.json", } - resp, err = f.srv.CallJSON(&opts, &moveFolderData, &response) + resp, err = f.srv.CallJSON(ctx, &opts, &moveFolderData, &response) return f.shouldRetry(resp, err) }) if err != nil { @@ -589,14 +589,14 @@ func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, } // readMetaDataForPath reads the metadata from the path -func (f *Fs) readMetaDataForFolderID(id string) (info *FolderList, err error) { +func (f *Fs) readMetaDataForFolderID(ctx context.Context, id string) (info *FolderList, err error) { var resp *http.Response opts := rest.Opts{ Method: "GET", Path: "/folder/list.json/" + f.session.SessionID + "/" + id, } err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallJSON(&opts, nil, &info) + resp, err = f.srv.CallJSON(ctx, &opts, nil, &info) return f.shouldRetry(resp, err) }) if err != nil { @@ -641,7 +641,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options . Method: "POST", Path: "/upload/create_file.json", } - resp, err = o.fs.srv.CallJSON(&opts, &createFileData, &response) + resp, err = o.fs.srv.CallJSON(ctx, &opts, &createFileData, &response) return o.fs.shouldRetry(resp, err) }) if err != nil { @@ -694,7 +694,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, Method: "POST", Path: "/folder.json", } - resp, err = f.srv.CallJSON(&opts, &createDirData, &response) + resp, err = f.srv.CallJSON(ctx, &opts, &createDirData, &response) return f.shouldRetry(resp, err) }) if err != nil { @@ -722,7 +722,7 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin Method: "GET", Path: "/folder/list.json/" + f.session.SessionID + "/" + pathID, } - resp, err = f.srv.CallJSON(&opts, nil, &folderList) + resp, err = f.srv.CallJSON(ctx, &opts, nil, &folderList) return f.shouldRetry(resp, err) }) if err != nil { @@ -769,7 +769,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e } folderList := FolderList{} err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallJSON(&opts, nil, &folderList) + resp, err = f.srv.CallJSON(ctx, &opts, nil, &folderList) return f.shouldRetry(resp, err) }) if err != nil { @@ -853,7 +853,7 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error { } update := modTimeFile{SessionID: o.fs.session.SessionID, FileID: o.id, FileModificationTime: strconv.FormatInt(modTime.Unix(), 10)} err := o.fs.pacer.Call(func() (bool, error) { - resp, err := o.fs.srv.CallJSON(&opts, &update, nil) + resp, err := o.fs.srv.CallJSON(ctx, &opts, &update, nil) return o.fs.shouldRetry(resp, err) }) @@ -873,7 +873,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read } var resp *http.Response err = o.fs.pacer.Call(func() (bool, error) { - resp, err = o.fs.srv.Call(&opts) + resp, err = o.fs.srv.Call(ctx, &opts) return o.fs.shouldRetry(resp, err) }) if err != nil { @@ -892,7 +892,7 @@ func (o *Object) Remove(ctx context.Context) error { NoResponse: true, Path: "/file.json/" + o.fs.session.SessionID + "/" + o.id, } - resp, err := o.fs.srv.Call(&opts) + resp, err := o.fs.srv.Call(ctx, &opts) return o.fs.shouldRetry(resp, err) }) } @@ -920,7 +920,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op Method: "POST", Path: "/upload/open_file_upload.json", } - resp, err := o.fs.srv.CallJSON(&opts, &openUploadData, &openResponse) + resp, err := o.fs.srv.CallJSON(ctx, &opts, &openUploadData, &openResponse) return o.fs.shouldRetry(resp, err) }) if err != nil { @@ -966,7 +966,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op MultipartFileName: o.remote, // ..name of the file for the attached file } - resp, err = o.fs.srv.CallJSON(&opts, nil, &reply) + resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &reply) return o.fs.shouldRetry(resp, err) }) if err != nil { @@ -989,7 +989,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op Method: "POST", Path: "/upload/close_file_upload.json", } - resp, err = o.fs.srv.CallJSON(&opts, &closeUploadData, &closeResponse) + resp, err = o.fs.srv.CallJSON(ctx, &opts, &closeUploadData, &closeResponse) return o.fs.shouldRetry(resp, err) }) if err != nil { @@ -1015,7 +1015,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op NoResponse: true, Path: "/file/access.json", } - resp, err = o.fs.srv.CallJSON(&opts, &update, nil) + resp, err = o.fs.srv.CallJSON(ctx, &opts, &update, nil) return o.fs.shouldRetry(resp, err) }) if err != nil { @@ -1040,7 +1040,7 @@ func (o *Object) readMetaData(ctx context.Context) (err error) { Method: "GET", Path: "/folder/itembyname.json/" + o.fs.session.SessionID + "/" + directoryID + "?name=" + url.QueryEscape(replaceReservedChars(leaf)), } - resp, err = o.fs.srv.CallJSON(&opts, nil, &folderList) + resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &folderList) return o.fs.shouldRetry(resp, err) }) if err != nil { diff --git a/backend/pcloud/pcloud.go b/backend/pcloud/pcloud.go index b40f8ba8f..9761bfe66 100644 --- a/backend/pcloud/pcloud.go +++ b/backend/pcloud/pcloud.go @@ -201,7 +201,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.It return nil, err } - found, err := f.listAll(directoryID, false, true, func(item *api.Item) bool { + found, err := f.listAll(ctx, directoryID, false, true, func(item *api.Item) bool { if item.Name == leaf { info = item return true @@ -334,7 +334,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { // FindLeaf finds a directory of name leaf in the folder with ID pathID func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) { // Find the leaf in pathID - found, err = f.listAll(pathID, true, false, func(item *api.Item) bool { + found, err = f.listAll(ctx, pathID, true, false, func(item *api.Item) bool { if item.Name == leaf { pathIDOut = item.ID return true @@ -357,7 +357,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, opts.Parameters.Set("name", replaceReservedChars(leaf)) opts.Parameters.Set("folderid", dirIDtoNumber(pathID)) err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallJSON(&opts, nil, &result) + resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) err = result.Error.Update(err) return shouldRetry(resp, err) }) @@ -400,7 +400,7 @@ type listAllFn func(*api.Item) bool // Lists the directory required calling the user function on each item found // // If the user fn ever returns true then it early exits with found = true -func (f *Fs) listAll(dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) { +func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) { opts := rest.Opts{ Method: "GET", Path: "/listfolder", @@ -412,7 +412,7 @@ func (f *Fs) listAll(dirID string, directoriesOnly bool, filesOnly bool, fn list var result api.ItemResult var resp *http.Response err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallJSON(&opts, nil, &result) + resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) err = result.Error.Update(err) return shouldRetry(resp, err) }) @@ -458,7 +458,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e return nil, err } var iErr error - _, err = f.listAll(directoryID, false, false, func(info *api.Item) bool { + _, err = f.listAll(ctx, directoryID, false, false, func(info *api.Item) bool { remote := path.Join(dir, info.Name) if info.IsFolder { // cache the directory ID for later lookups @@ -563,7 +563,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error { var resp *http.Response var result api.ItemResult err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallJSON(&opts, nil, &result) + resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) err = result.Error.Update(err) return shouldRetry(resp, err) }) @@ -628,7 +628,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, var resp *http.Response var result api.ItemResult err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallJSON(&opts, nil, &result) + resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) err = result.Error.Update(err) return shouldRetry(resp, err) }) @@ -666,7 +666,7 @@ func (f *Fs) CleanUp(ctx context.Context) error { var resp *http.Response var result api.Error return f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallJSON(&opts, nil, &result) + resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) err = result.Update(err) return shouldRetry(resp, err) }) @@ -706,7 +706,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, var resp *http.Response var result api.ItemResult err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallJSON(&opts, nil, &result) + resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) err = result.Error.Update(err) return shouldRetry(resp, err) }) @@ -803,7 +803,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string var resp *http.Response var result api.ItemResult err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallJSON(&opts, nil, &result) + resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) err = result.Error.Update(err) return shouldRetry(resp, err) }) @@ -830,7 +830,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) { var resp *http.Response var q api.UserInfo err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallJSON(&opts, nil, &q) + resp, err = f.srv.CallJSON(ctx, &opts, nil, &q) err = q.Error.Update(err) return shouldRetry(resp, err) }) @@ -881,7 +881,7 @@ func (o *Object) getHashes(ctx context.Context) (err error) { } opts.Parameters.Set("fileid", fileIDtoNumber(o.id)) err = o.fs.pacer.Call(func() (bool, error) { - resp, err = o.fs.srv.CallJSON(&opts, nil, &result) + resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &result) err = result.Error.Update(err) return shouldRetry(resp, err) }) @@ -984,7 +984,7 @@ func (o *Object) Storable() bool { } // downloadURL fetches the download link -func (o *Object) downloadURL() (URL string, err error) { +func (o *Object) downloadURL(ctx context.Context) (URL string, err error) { if o.id == "" { return "", errors.New("can't download - no id") } @@ -1000,7 +1000,7 @@ func (o *Object) downloadURL() (URL string, err error) { } opts.Parameters.Set("fileid", fileIDtoNumber(o.id)) err = o.fs.pacer.Call(func() (bool, error) { - resp, err = o.fs.srv.CallJSON(&opts, nil, &result) + resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &result) err = result.Error.Update(err) return shouldRetry(resp, err) }) @@ -1016,7 +1016,7 @@ func (o *Object) downloadURL() (URL string, err error) { // Open an object for read func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) { - url, err := o.downloadURL() + url, err := o.downloadURL(ctx) if err != nil { return nil, err } @@ -1027,7 +1027,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read Options: options, } err = o.fs.pacer.Call(func() (bool, error) { - resp, err = o.fs.srv.Call(&opts) + resp, err = o.fs.srv.Call(ctx, &opts) return shouldRetry(resp, err) }) if err != nil { @@ -1104,7 +1104,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op } err = o.fs.pacer.CallNoRetry(func() (bool, error) { - resp, err = o.fs.srv.CallJSON(&opts, nil, &result) + resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &result) err = result.Error.Update(err) return shouldRetry(resp, err) }) @@ -1134,7 +1134,7 @@ func (o *Object) Remove(ctx context.Context) error { var result api.ItemResult opts.Parameters.Set("fileid", fileIDtoNumber(o.id)) return o.fs.pacer.Call(func() (bool, error) { - resp, err := o.fs.srv.CallJSON(&opts, nil, &result) + resp, err := o.fs.srv.CallJSON(ctx, &opts, nil, &result) err = result.Error.Update(err) return shouldRetry(resp, err) }) diff --git a/backend/premiumizeme/premiumizeme.go b/backend/premiumizeme/premiumizeme.go index 1aee001ca..4a2d5b18c 100644 --- a/backend/premiumizeme/premiumizeme.go +++ b/backend/premiumizeme/premiumizeme.go @@ -200,7 +200,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string, directoriesOn } lcLeaf := strings.ToLower(leaf) - found, err := f.listAll(directoryID, directoriesOnly, filesOnly, func(item *api.Item) bool { + found, err := f.listAll(ctx, directoryID, directoriesOnly, filesOnly, func(item *api.Item) bool { if strings.ToLower(item.Name) == lcLeaf { info = item return true @@ -361,7 +361,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { // FindLeaf finds a directory of name leaf in the folder with ID pathID func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) { // Find the leaf in pathID - found, err = f.listAll(pathID, true, false, func(item *api.Item) bool { + found, err = f.listAll(ctx, pathID, true, false, func(item *api.Item) bool { if item.Name == leaf { pathIDOut = item.ID return true @@ -386,7 +386,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, }, } err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallJSON(&opts, nil, &info) + resp, err = f.srv.CallJSON(ctx, &opts, nil, &info) return shouldRetry(resp, err) }) if err != nil { @@ -411,7 +411,7 @@ type listAllFn func(*api.Item) bool // Lists the directory required calling the user function on each item found // // If the user fn ever returns true then it early exits with found = true -func (f *Fs) listAll(dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) { +func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) { opts := rest.Opts{ Method: "GET", Path: "/folder/list", @@ -423,7 +423,7 @@ func (f *Fs) listAll(dirID string, directoriesOnly bool, filesOnly bool, fn list var result api.FolderListResponse var resp *http.Response err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallJSON(&opts, nil, &result) + resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) return shouldRetry(resp, err) }) if err != nil { @@ -475,7 +475,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e return nil, err } var iErr error - _, err = f.listAll(directoryID, false, false, func(info *api.Item) bool { + _, err = f.listAll(ctx, directoryID, false, false, func(info *api.Item) bool { remote := path.Join(dir, info.Name) if info.Type == api.ItemTypeFolder { // cache the directory ID for later lookups @@ -589,7 +589,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error { // need to check if empty as it will delete recursively by default if check { - found, err := f.listAll(rootID, false, false, func(item *api.Item) bool { + found, err := f.listAll(ctx, rootID, false, false, func(item *api.Item) bool { return true }) if err != nil { @@ -611,7 +611,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error { var resp *http.Response var result api.Response err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallJSON(&opts, nil, &result) + resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) return shouldRetry(resp, err) }) if err != nil { @@ -690,7 +690,7 @@ func (f *Fs) move(ctx context.Context, isFile bool, id, oldLeaf, newLeaf, oldDir var resp *http.Response var result api.Response err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallJSON(&opts, nil, &result) + resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) return shouldRetry(resp, err) }) if err != nil { @@ -860,7 +860,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) { Parameters: f.baseParams(), } err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallJSON(&opts, nil, &info) + resp, err = f.srv.CallJSON(ctx, &opts, nil, &info) return shouldRetry(resp, err) }) if err != nil { @@ -992,7 +992,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read Options: options, } err = o.fs.pacer.Call(func() (bool, error) { - resp, err = o.fs.srv.Call(&opts) + resp, err = o.fs.srv.Call(ctx, &opts) return shouldRetry(resp, err) }) if err != nil { @@ -1036,7 +1036,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op }, } err = o.fs.pacer.Call(func() (bool, error) { - resp, err = o.fs.srv.CallJSON(&opts, nil, &info) + resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &info) if err != nil { return shouldRetry(resp, err) } @@ -1096,7 +1096,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op } var result api.Response err = o.fs.pacer.CallNoRetry(func() (bool, error) { - resp, err = o.fs.srv.CallJSON(&opts, nil, &result) + resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &result) return shouldRetry(resp, err) }) if err != nil { @@ -1138,7 +1138,7 @@ func (f *Fs) renameLeaf(ctx context.Context, isFile bool, id string, newLeaf str var resp *http.Response var result api.Response err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallJSON(&opts, nil, &result) + resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) return shouldRetry(resp, err) }) if err != nil { @@ -1163,7 +1163,7 @@ func (f *Fs) remove(ctx context.Context, id string) (err error) { var resp *http.Response var result api.Response err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallJSON(&opts, nil, &result) + resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) return shouldRetry(resp, err) }) if err != nil { diff --git a/backend/webdav/webdav.go b/backend/webdav/webdav.go index 8683fccde..cd2eef089 100644 --- a/backend/webdav/webdav.go +++ b/backend/webdav/webdav.go @@ -205,7 +205,7 @@ func itemIsDir(item *api.Response) bool { } // readMetaDataForPath reads the metadata from the path -func (f *Fs) readMetaDataForPath(path string, depth string) (info *api.Prop, err error) { +func (f *Fs) readMetaDataForPath(ctx context.Context, path string, depth string) (info *api.Prop, err error) { // FIXME how do we read back additional properties? opts := rest.Opts{ Method: "PROPFIND", @@ -221,7 +221,7 @@ func (f *Fs) readMetaDataForPath(path string, depth string) (info *api.Prop, err var result api.Multistatus var resp *http.Response err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallXML(&opts, nil, &result) + resp, err = f.srv.CallXML(ctx, &opts, nil, &result) return f.shouldRetry(resp, err) }) if apiErr, ok := err.(*api.Error); ok { @@ -229,7 +229,7 @@ func (f *Fs) readMetaDataForPath(path string, depth string) (info *api.Prop, err switch apiErr.StatusCode { case http.StatusNotFound: if f.retryWithZeroDepth && depth != "0" { - return f.readMetaDataForPath(path, "0") + return f.readMetaDataForPath(ctx, path, "0") } return nil, fs.ErrorObjectNotFound case http.StatusMovedPermanently, http.StatusFound, http.StatusSeeOther: @@ -477,7 +477,7 @@ func (f *Fs) setQuirks(vendor string) error { // Return an Object from a path // // If it can't be found it returns the error fs.ErrorObjectNotFound. -func (f *Fs) newObjectWithInfo(remote string, info *api.Prop) (fs.Object, error) { +func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *api.Prop) (fs.Object, error) { o := &Object{ fs: f, remote: remote, @@ -487,7 +487,7 @@ func (f *Fs) newObjectWithInfo(remote string, info *api.Prop) (fs.Object, error) // Set info err = o.setMetaData(info) } else { - err = o.readMetaData() // reads info and meta, returning an error + err = o.readMetaData(ctx) // reads info and meta, returning an error } if err != nil { return nil, err @@ -498,7 +498,7 @@ func (f *Fs) newObjectWithInfo(remote string, info *api.Prop) (fs.Object, error) // NewObject finds the Object at remote. If it can't be found // it returns the error fs.ErrorObjectNotFound. func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { - return f.newObjectWithInfo(remote, nil) + return f.newObjectWithInfo(ctx, remote, nil) } // Read the normal props, plus the checksums @@ -528,7 +528,7 @@ type listAllFn func(string, bool, *api.Prop) bool // Lists the directory required calling the user function on each item found // // If the user fn ever returns true then it early exits with found = true -func (f *Fs) listAll(dir string, directoriesOnly bool, filesOnly bool, depth string, fn listAllFn) (found bool, err error) { +func (f *Fs) listAll(ctx context.Context, dir string, directoriesOnly bool, filesOnly bool, depth string, fn listAllFn) (found bool, err error) { opts := rest.Opts{ Method: "PROPFIND", Path: f.dirPath(dir), // FIXME Should not start with / @@ -542,7 +542,7 @@ func (f *Fs) listAll(dir string, directoriesOnly bool, filesOnly bool, depth str var result api.Multistatus var resp *http.Response err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallXML(&opts, nil, &result) + resp, err = f.srv.CallXML(ctx, &opts, nil, &result) return f.shouldRetry(resp, err) }) if err != nil { @@ -550,7 +550,7 @@ func (f *Fs) listAll(dir string, directoriesOnly bool, filesOnly bool, depth str // does not exist if apiErr.StatusCode == http.StatusNotFound { if f.retryWithZeroDepth && depth != "0" { - return f.listAll(dir, directoriesOnly, filesOnly, "0", fn) + return f.listAll(ctx, dir, directoriesOnly, filesOnly, "0", fn) } return found, fs.ErrorDirNotFound } @@ -625,14 +625,14 @@ func (f *Fs) listAll(dir string, directoriesOnly bool, filesOnly bool, depth str // found. func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) { var iErr error - _, err = f.listAll(dir, false, false, defaultDepth, func(remote string, isDir bool, info *api.Prop) bool { + _, err = f.listAll(ctx, dir, false, false, defaultDepth, func(remote string, isDir bool, info *api.Prop) bool { if isDir { d := fs.NewDir(remote, time.Time(info.Modified)) // .SetID(info.ID) // FIXME more info from dir? can set size, items? entries = append(entries, d) } else { - o, err := f.newObjectWithInfo(remote, info) + o, err := f.newObjectWithInfo(ctx, remote, info) if err != nil { iErr = err return true @@ -696,7 +696,7 @@ func (f *Fs) mkParentDir(ctx context.Context, dirPath string) error { } // low level mkdir, only makes the directory, doesn't attempt to create parents -func (f *Fs) _mkdir(dirPath string) error { +func (f *Fs) _mkdir(ctx context.Context, dirPath string) error { // We assume the root is already created if dirPath == "" { return nil @@ -711,7 +711,7 @@ func (f *Fs) _mkdir(dirPath string) error { NoResponse: true, } err := f.pacer.Call(func() (bool, error) { - resp, err := f.srv.Call(&opts) + resp, err := f.srv.Call(ctx, &opts) return f.shouldRetry(resp, err) }) if apiErr, ok := err.(*api.Error); ok { @@ -727,13 +727,13 @@ func (f *Fs) _mkdir(dirPath string) error { // mkdir makes the directory and parents using native paths func (f *Fs) mkdir(ctx context.Context, dirPath string) error { // defer log.Trace(dirPath, "")("") - err := f._mkdir(dirPath) + err := f._mkdir(ctx, dirPath) if apiErr, ok := err.(*api.Error); ok { // parent does not exist so create it first then try again if apiErr.StatusCode == http.StatusConflict { err = f.mkParentDir(ctx, dirPath) if err == nil { - err = f._mkdir(dirPath) + err = f._mkdir(ctx, dirPath) } } } @@ -749,17 +749,17 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error { // dirNotEmpty returns true if the directory exists and is not Empty // // if the directory does not exist then err will be ErrorDirNotFound -func (f *Fs) dirNotEmpty(dir string) (found bool, err error) { - return f.listAll(dir, false, false, defaultDepth, func(remote string, isDir bool, info *api.Prop) bool { +func (f *Fs) dirNotEmpty(ctx context.Context, dir string) (found bool, err error) { + return f.listAll(ctx, dir, false, false, defaultDepth, func(remote string, isDir bool, info *api.Prop) bool { return true }) } // purgeCheck removes the root directory, if check is set then it // refuses to do so if it has anything in -func (f *Fs) purgeCheck(dir string, check bool) error { +func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error { if check { - notEmpty, err := f.dirNotEmpty(dir) + notEmpty, err := f.dirNotEmpty(ctx, dir) if err != nil { return err } @@ -775,7 +775,7 @@ func (f *Fs) purgeCheck(dir string, check bool) error { var resp *http.Response var err error err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallXML(&opts, nil, nil) + resp, err = f.srv.CallXML(ctx, &opts, nil, nil) return f.shouldRetry(resp, err) }) if err != nil { @@ -789,7 +789,7 @@ func (f *Fs) purgeCheck(dir string, check bool) error { // // Returns an error if it isn't empty func (f *Fs) Rmdir(ctx context.Context, dir string) error { - return f.purgeCheck(dir, true) + return f.purgeCheck(ctx, dir, true) } // Precision return the precision of this Fs @@ -838,7 +838,7 @@ func (f *Fs) copyOrMove(ctx context.Context, src fs.Object, remote string, metho opts.ExtraHeaders["X-OC-Mtime"] = fmt.Sprintf("%f", float64(src.ModTime(ctx).UnixNano())/1e9) } err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.Call(&opts) + resp, err = f.srv.Call(ctx, &opts) return f.shouldRetry(resp, err) }) if err != nil { @@ -870,7 +870,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, // deleting all the files quicker than just running Remove() on the // result of List() func (f *Fs) Purge(ctx context.Context) error { - return f.purgeCheck("", false) + return f.purgeCheck(ctx, "", false) } // Move src to this remote using server side move operations. @@ -904,7 +904,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string dstPath := f.filePath(dstRemote) // Check if destination exists - _, err := f.dirNotEmpty(dstRemote) + _, err := f.dirNotEmpty(ctx, dstRemote) if err == nil { return fs.ErrorDirExists } @@ -934,7 +934,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string }, } err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.Call(&opts) + resp, err = f.srv.Call(ctx, &opts) return f.shouldRetry(resp, err) }) if err != nil { @@ -975,7 +975,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) { var resp *http.Response var err error err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallXML(&opts, nil, &q) + resp, err = f.srv.CallXML(ctx, &opts, nil, &q) return f.shouldRetry(resp, err) }) if err != nil { @@ -1028,7 +1028,8 @@ func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) { // Size returns the size of an object in bytes func (o *Object) Size() int64 { - err := o.readMetaData() + ctx := context.TODO() + err := o.readMetaData(ctx) if err != nil { fs.Logf(o, "Failed to read metadata: %v", err) return 0 @@ -1052,11 +1053,11 @@ func (o *Object) setMetaData(info *api.Prop) (err error) { // readMetaData gets the metadata if it hasn't already been fetched // // it also sets the info -func (o *Object) readMetaData() (err error) { +func (o *Object) readMetaData(ctx context.Context) (err error) { if o.hasMetaData { return nil } - info, err := o.fs.readMetaDataForPath(o.remote, defaultDepth) + info, err := o.fs.readMetaDataForPath(ctx, o.remote, defaultDepth) if err != nil { return err } @@ -1068,7 +1069,7 @@ func (o *Object) readMetaData() (err error) { // It attempts to read the objects mtime and if that isn't present the // LastModified returned in the http headers func (o *Object) ModTime(ctx context.Context) time.Time { - err := o.readMetaData() + err := o.readMetaData(ctx) if err != nil { fs.Logf(o, "Failed to read metadata: %v", err) return time.Now() @@ -1095,7 +1096,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read Options: options, } err = o.fs.pacer.Call(func() (bool, error) { - resp, err = o.fs.srv.Call(&opts) + resp, err = o.fs.srv.Call(ctx, &opts) return o.fs.shouldRetry(resp, err) }) if err != nil { @@ -1143,7 +1144,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op } } err = o.fs.pacer.CallNoRetry(func() (bool, error) { - resp, err = o.fs.srv.Call(&opts) + resp, err = o.fs.srv.Call(ctx, &opts) return o.fs.shouldRetry(resp, err) }) if err != nil { @@ -1159,7 +1160,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op } // read metadata from remote o.hasMetaData = false - return o.readMetaData() + return o.readMetaData(ctx) } // Remove an object @@ -1170,7 +1171,7 @@ func (o *Object) Remove(ctx context.Context) error { NoResponse: true, } return o.fs.pacer.Call(func() (bool, error) { - resp, err := o.fs.srv.Call(&opts) + resp, err := o.fs.srv.Call(ctx, &opts) return o.fs.shouldRetry(resp, err) }) } diff --git a/backend/yandex/yandex.go b/backend/yandex/yandex.go index 71d9e5896..3a4ba0ee3 100644 --- a/backend/yandex/yandex.go +++ b/backend/yandex/yandex.go @@ -200,7 +200,7 @@ func (f *Fs) dirPath(file string) string { return path.Join(f.diskRoot, file) + "/" } -func (f *Fs) readMetaDataForPath(path string, options *api.ResourceInfoRequestOptions) (*api.ResourceInfoResponse, error) { +func (f *Fs) readMetaDataForPath(ctx context.Context, path string, options *api.ResourceInfoRequestOptions) (*api.ResourceInfoResponse, error) { opts := rest.Opts{ Method: "GET", Path: "/resources", @@ -226,7 +226,7 @@ func (f *Fs) readMetaDataForPath(path string, options *api.ResourceInfoRequestOp var info api.ResourceInfoResponse var resp *http.Response err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallJSON(&opts, nil, &info) + resp, err = f.srv.CallJSON(ctx, &opts, nil, &info) return shouldRetry(resp, err) }) @@ -239,6 +239,7 @@ func (f *Fs) readMetaDataForPath(path string, options *api.ResourceInfoRequestOp // NewFs constructs an Fs from the path, container:path func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { + ctx := context.TODO() // Parse config into Options struct opt := new(Options) err := configstruct.Set(m, opt) @@ -284,7 +285,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { //request object meta info // Check to see if the object exists and is a file //request object meta info - if info, err := f.readMetaDataForPath(f.diskRoot, &api.ResourceInfoRequestOptions{}); err != nil { + if info, err := f.readMetaDataForPath(ctx, f.diskRoot, &api.ResourceInfoRequestOptions{}); err != nil { } else { if info.ResourceType == "file" { @@ -301,7 +302,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { } // Convert a list item into a DirEntry -func (f *Fs) itemToDirEntry(remote string, object *api.ResourceInfoResponse) (fs.DirEntry, error) { +func (f *Fs) itemToDirEntry(ctx context.Context, remote string, object *api.ResourceInfoResponse) (fs.DirEntry, error) { switch object.ResourceType { case "dir": t, err := time.Parse(time.RFC3339Nano, object.Modified) @@ -311,7 +312,7 @@ func (f *Fs) itemToDirEntry(remote string, object *api.ResourceInfoResponse) (fs d := fs.NewDir(remote, t).SetSize(object.Size) return d, nil case "file": - o, err := f.newObjectWithInfo(remote, object) + o, err := f.newObjectWithInfo(ctx, remote, object) if err != nil { return nil, err } @@ -343,7 +344,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e Limit: limit, Offset: offset, } - info, err := f.readMetaDataForPath(root, opts) + info, err := f.readMetaDataForPath(ctx, root, opts) if err != nil { if apiErr, ok := err.(*api.ErrorResponse); ok { @@ -360,7 +361,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e //list all subdirs for _, element := range info.Embedded.Items { remote := path.Join(dir, element.Name) - entry, err := f.itemToDirEntry(remote, &element) + entry, err := f.itemToDirEntry(ctx, remote, &element) if err != nil { return nil, err } @@ -386,7 +387,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e // Return an Object from a path // // If it can't be found it returns the error fs.ErrorObjectNotFound. -func (f *Fs) newObjectWithInfo(remote string, info *api.ResourceInfoResponse) (fs.Object, error) { +func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *api.ResourceInfoResponse) (fs.Object, error) { o := &Object{ fs: f, remote: remote, @@ -395,7 +396,7 @@ func (f *Fs) newObjectWithInfo(remote string, info *api.ResourceInfoResponse) (f if info != nil { err = o.setMetaData(info) } else { - err = o.readMetaData() + err = o.readMetaData(ctx) if apiErr, ok := err.(*api.ErrorResponse); ok { // does not exist if apiErr.ErrorName == "DiskNotFoundError" { @@ -412,7 +413,7 @@ func (f *Fs) newObjectWithInfo(remote string, info *api.ResourceInfoResponse) (f // NewObject finds the Object at remote. If it can't be found it // returns the error fs.ErrorObjectNotFound. func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { - return f.newObjectWithInfo(remote, nil) + return f.newObjectWithInfo(ctx, remote, nil) } // Creates from the parameters passed in a half finished Object which @@ -446,7 +447,7 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt } // CreateDir makes a directory -func (f *Fs) CreateDir(path string) (err error) { +func (f *Fs) CreateDir(ctx context.Context, path string) (err error) { //fmt.Printf("CreateDir: %s\n", path) var resp *http.Response @@ -460,7 +461,7 @@ func (f *Fs) CreateDir(path string) (err error) { opts.Parameters.Set("path", path) err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.Call(&opts) + resp, err = f.srv.Call(ctx, &opts) return shouldRetry(resp, err) }) if err != nil { @@ -474,7 +475,7 @@ func (f *Fs) CreateDir(path string) (err error) { // This really needs improvement and especially proper error checking // but Yandex does not publish a List of possible errors and when they're // expected to occur. -func (f *Fs) mkDirs(path string) (err error) { +func (f *Fs) mkDirs(ctx context.Context, path string) (err error) { //trim filename from path //dirString := strings.TrimSuffix(path, filepath.Base(path)) //trim "disk:" from path @@ -483,7 +484,7 @@ func (f *Fs) mkDirs(path string) (err error) { return nil } - if err = f.CreateDir(dirString); err != nil { + if err = f.CreateDir(ctx, dirString); err != nil { if apiErr, ok := err.(*api.ErrorResponse); ok { // allready exists if apiErr.ErrorName != "DiskPathPointsToExistentDirectoryError" { @@ -493,7 +494,7 @@ func (f *Fs) mkDirs(path string) (err error) { for _, element := range dirs { if element != "" { mkdirpath += element + "/" //path separator / - if err = f.CreateDir(mkdirpath); err != nil { + if err = f.CreateDir(ctx, mkdirpath); err != nil { // ignore errors while creating dirs } } @@ -505,7 +506,7 @@ func (f *Fs) mkDirs(path string) (err error) { return err } -func (f *Fs) mkParentDirs(resPath string) error { +func (f *Fs) mkParentDirs(ctx context.Context, resPath string) error { // defer log.Trace(dirPath, "")("") // chop off trailing / if it exists if strings.HasSuffix(resPath, "/") { @@ -515,17 +516,17 @@ func (f *Fs) mkParentDirs(resPath string) error { if parent == "." { parent = "" } - return f.mkDirs(parent) + return f.mkDirs(ctx, parent) } // Mkdir creates the container if it doesn't exist func (f *Fs) Mkdir(ctx context.Context, dir string) error { path := f.filePath(dir) - return f.mkDirs(path) + return f.mkDirs(ctx, path) } // waitForJob waits for the job with status in url to complete -func (f *Fs) waitForJob(location string) (err error) { +func (f *Fs) waitForJob(ctx context.Context, location string) (err error) { opts := rest.Opts{ RootURL: location, Method: "GET", @@ -535,7 +536,7 @@ func (f *Fs) waitForJob(location string) (err error) { var resp *http.Response var body []byte err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.Call(&opts) + resp, err = f.srv.Call(ctx, &opts) if err != nil { return fserrors.ShouldRetry(err), err } @@ -564,7 +565,7 @@ func (f *Fs) waitForJob(location string) (err error) { return errors.Errorf("async operation didn't complete after %v", fs.Config.Timeout) } -func (f *Fs) delete(path string, hardDelete bool) (err error) { +func (f *Fs) delete(ctx context.Context, path string, hardDelete bool) (err error) { opts := rest.Opts{ Method: "DELETE", Path: "/resources", @@ -577,7 +578,7 @@ func (f *Fs) delete(path string, hardDelete bool) (err error) { var resp *http.Response var body []byte err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.Call(&opts) + resp, err = f.srv.Call(ctx, &opts) if err != nil { return fserrors.ShouldRetry(err), err } @@ -595,19 +596,19 @@ func (f *Fs) delete(path string, hardDelete bool) (err error) { if err != nil { return errors.Wrapf(err, "async info result not JSON: %q", body) } - return f.waitForJob(info.HRef) + return f.waitForJob(ctx, info.HRef) } return nil } // purgeCheck remotes the root directory, if check is set then it // refuses to do so if it has anything in -func (f *Fs) purgeCheck(dir string, check bool) error { +func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error { root := f.filePath(dir) if check { //to comply with rclone logic we check if the directory is empty before delete. //send request to get list of objects in this directory. - info, err := f.readMetaDataForPath(root, &api.ResourceInfoRequestOptions{}) + info, err := f.readMetaDataForPath(ctx, root, &api.ResourceInfoRequestOptions{}) if err != nil { return errors.Wrap(err, "rmdir failed") } @@ -616,14 +617,14 @@ func (f *Fs) purgeCheck(dir string, check bool) error { } } //delete directory - return f.delete(root, false) + return f.delete(ctx, root, false) } // Rmdir deletes the container // // Returns an error if it isn't empty func (f *Fs) Rmdir(ctx context.Context, dir string) error { - return f.purgeCheck(dir, true) + return f.purgeCheck(ctx, dir, true) } // Purge deletes all the files and the container @@ -632,11 +633,11 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error { // deleting all the files quicker than just running Remove() on the // result of List() func (f *Fs) Purge(ctx context.Context) error { - return f.purgeCheck("", false) + return f.purgeCheck(ctx, "", false) } // copyOrMoves copies or moves directories or files depending on the method parameter -func (f *Fs) copyOrMove(method, src, dst string, overwrite bool) (err error) { +func (f *Fs) copyOrMove(ctx context.Context, method, src, dst string, overwrite bool) (err error) { opts := rest.Opts{ Method: "POST", Path: "/resources/" + method, @@ -650,7 +651,7 @@ func (f *Fs) copyOrMove(method, src, dst string, overwrite bool) (err error) { var resp *http.Response var body []byte err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.Call(&opts) + resp, err = f.srv.Call(ctx, &opts) if err != nil { return fserrors.ShouldRetry(err), err } @@ -668,7 +669,7 @@ func (f *Fs) copyOrMove(method, src, dst string, overwrite bool) (err error) { if err != nil { return errors.Wrapf(err, "async info result not JSON: %q", body) } - return f.waitForJob(info.HRef) + return f.waitForJob(ctx, info.HRef) } return nil } @@ -690,11 +691,11 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, } dstPath := f.filePath(remote) - err := f.mkParentDirs(dstPath) + err := f.mkParentDirs(ctx, dstPath) if err != nil { return nil, err } - err = f.copyOrMove("copy", srcObj.filePath(), dstPath, false) + err = f.copyOrMove(ctx, "copy", srcObj.filePath(), dstPath, false) if err != nil { return nil, errors.Wrap(err, "couldn't copy file") @@ -720,11 +721,11 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, } dstPath := f.filePath(remote) - err := f.mkParentDirs(dstPath) + err := f.mkParentDirs(ctx, dstPath) if err != nil { return nil, err } - err = f.copyOrMove("move", srcObj.filePath(), dstPath, false) + err = f.copyOrMove(ctx, "move", srcObj.filePath(), dstPath, false) if err != nil { return nil, errors.Wrap(err, "couldn't move file") @@ -758,12 +759,12 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string return errors.New("can't move root directory") } - err := f.mkParentDirs(dstPath) + err := f.mkParentDirs(ctx, dstPath) if err != nil { return err } - _, err = f.readMetaDataForPath(dstPath, &api.ResourceInfoRequestOptions{}) + _, err = f.readMetaDataForPath(ctx, dstPath, &api.ResourceInfoRequestOptions{}) if apiErr, ok := err.(*api.ErrorResponse); ok { // does not exist if apiErr.ErrorName == "DiskNotFoundError" { @@ -775,7 +776,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string return fs.ErrorDirExists } - err = f.copyOrMove("move", srcPath, dstPath, false) + err = f.copyOrMove(ctx, "move", srcPath, dstPath, false) if err != nil { return errors.Wrap(err, "couldn't move directory") @@ -802,7 +803,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err er var resp *http.Response err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.Call(&opts) + resp, err = f.srv.Call(ctx, &opts) return shouldRetry(resp, err) }) @@ -819,7 +820,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err er return "", errors.Wrap(err, "couldn't create public link") } - info, err := f.readMetaDataForPath(f.filePath(remote), &api.ResourceInfoRequestOptions{}) + info, err := f.readMetaDataForPath(ctx, f.filePath(remote), &api.ResourceInfoRequestOptions{}) if err != nil { return "", err } @@ -840,7 +841,7 @@ func (f *Fs) CleanUp(ctx context.Context) (err error) { } err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.Call(&opts) + resp, err = f.srv.Call(ctx, &opts) return shouldRetry(resp, err) }) return err @@ -857,7 +858,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) { var info api.DiskInfo var err error err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallJSON(&opts, nil, &info) + resp, err = f.srv.CallJSON(ctx, &opts, nil, &info) return shouldRetry(resp, err) }) @@ -924,11 +925,11 @@ func (o *Object) setMetaData(info *api.ResourceInfoResponse) (err error) { } // readMetaData reads ands sets the new metadata for a storage.Object -func (o *Object) readMetaData() (err error) { +func (o *Object) readMetaData(ctx context.Context) (err error) { if o.hasMetaData { return nil } - info, err := o.fs.readMetaDataForPath(o.filePath(), &api.ResourceInfoRequestOptions{}) + info, err := o.fs.readMetaDataForPath(ctx, o.filePath(), &api.ResourceInfoRequestOptions{}) if err != nil { return err } @@ -943,7 +944,7 @@ func (o *Object) readMetaData() (err error) { // It attempts to read the objects mtime and if that isn't present the // LastModified returned in the http headers func (o *Object) ModTime(ctx context.Context) time.Time { - err := o.readMetaData() + err := o.readMetaData(ctx) if err != nil { fs.Logf(o, "Failed to read metadata: %v", err) return time.Now() @@ -953,7 +954,8 @@ func (o *Object) ModTime(ctx context.Context) time.Time { // Size returns the size of an object in bytes func (o *Object) Size() int64 { - err := o.readMetaData() + ctx := context.TODO() + err := o.readMetaData(ctx) if err != nil { fs.Logf(o, "Failed to read metadata: %v", err) return 0 @@ -974,7 +976,7 @@ func (o *Object) Storable() bool { return true } -func (o *Object) setCustomProperty(property string, value string) (err error) { +func (o *Object) setCustomProperty(ctx context.Context, property string, value string) (err error) { var resp *http.Response opts := rest.Opts{ Method: "PATCH", @@ -990,7 +992,7 @@ func (o *Object) setCustomProperty(property string, value string) (err error) { cpr := api.CustomPropertyResponse{CustomProperties: rcm} err = o.fs.pacer.Call(func() (bool, error) { - resp, err = o.fs.srv.CallJSON(&opts, &cpr, nil) + resp, err = o.fs.srv.CallJSON(ctx, &opts, &cpr, nil) return shouldRetry(resp, err) }) return err @@ -1001,7 +1003,7 @@ func (o *Object) setCustomProperty(property string, value string) (err error) { // Commits the datastore func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error { // set custom_property 'rclone_modified' of object to modTime - err := o.setCustomProperty("rclone_modified", modTime.Format(time.RFC3339Nano)) + err := o.setCustomProperty(ctx, "rclone_modified", modTime.Format(time.RFC3339Nano)) if err != nil { return err } @@ -1023,7 +1025,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read opts.Parameters.Set("path", o.filePath()) err = o.fs.pacer.Call(func() (bool, error) { - resp, err = o.fs.srv.CallJSON(&opts, nil, &dl) + resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &dl) return shouldRetry(resp, err) }) @@ -1038,7 +1040,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read Options: options, } err = o.fs.pacer.Call(func() (bool, error) { - resp, err = o.fs.srv.Call(&opts) + resp, err = o.fs.srv.Call(ctx, &opts) return shouldRetry(resp, err) }) if err != nil { @@ -1047,7 +1049,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read return resp.Body, err } -func (o *Object) upload(in io.Reader, overwrite bool, mimeType string) (err error) { +func (o *Object) upload(ctx context.Context, in io.Reader, overwrite bool, mimeType string) (err error) { // prepare upload var resp *http.Response var ur api.AsyncInfo @@ -1061,7 +1063,7 @@ func (o *Object) upload(in io.Reader, overwrite bool, mimeType string) (err erro opts.Parameters.Set("overwrite", strconv.FormatBool(overwrite)) err = o.fs.pacer.Call(func() (bool, error) { - resp, err = o.fs.srv.CallJSON(&opts, nil, &ur) + resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &ur) return shouldRetry(resp, err) }) @@ -1079,7 +1081,7 @@ func (o *Object) upload(in io.Reader, overwrite bool, mimeType string) (err erro } err = o.fs.pacer.Call(func() (bool, error) { - resp, err = o.fs.srv.Call(&opts) + resp, err = o.fs.srv.Call(ctx, &opts) return shouldRetry(resp, err) }) @@ -1097,13 +1099,13 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op remote := o.filePath() //create full path to file before upload. - err := o.fs.mkParentDirs(remote) + err := o.fs.mkParentDirs(ctx, remote) if err != nil { return err } //upload file - err = o.upload(in1, true, fs.MimeType(ctx, src)) + err = o.upload(ctx, in1, true, fs.MimeType(ctx, src)) if err != nil { return err } @@ -1120,7 +1122,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op // Remove an object func (o *Object) Remove(ctx context.Context) error { - return o.fs.delete(o.filePath(), false) + return o.fs.delete(ctx, o.filePath(), false) } // MimeType of an Object if known, "" otherwise diff --git a/lib/rest/rest.go b/lib/rest/rest.go index 39dbe1807..3626060cf 100644 --- a/lib/rest/rest.go +++ b/lib/rest/rest.go @@ -5,6 +5,7 @@ package rest import ( "bytes" + "context" "encoding/json" "encoding/xml" "io" @@ -184,7 +185,7 @@ func ClientWithNoRedirects(c *http.Client) *http.Client { // if err != nil then resp.Body will have been closed // // it will return resp if at all possible, even if err is set -func (api *Client) Call(opts *Opts) (resp *http.Response, err error) { +func (api *Client) Call(ctx context.Context, opts *Opts) (resp *http.Response, err error) { api.mu.RLock() defer api.mu.RUnlock() if opts == nil { @@ -211,7 +212,7 @@ func (api *Client) Call(opts *Opts) (resp *http.Response, err error) { if opts.ContentLength != nil && *opts.ContentLength == 0 { body = nil } - req, err := http.NewRequest(opts.Method, url, body) + req, err := http.NewRequestWithContext(ctx, opts.Method, url, body) if err != nil { return } @@ -391,8 +392,8 @@ func MultipartUpload(in io.Reader, params url.Values, contentName, fileName stri // parameter name MultipartMetadataName. // // It will return resp if at all possible, even if err is set -func (api *Client) CallJSON(opts *Opts, request interface{}, response interface{}) (resp *http.Response, err error) { - return api.callCodec(opts, request, response, json.Marshal, DecodeJSON, "application/json") +func (api *Client) CallJSON(ctx context.Context, opts *Opts, request interface{}, response interface{}) (resp *http.Response, err error) { + return api.callCodec(ctx, opts, request, response, json.Marshal, DecodeJSON, "application/json") } // CallXML runs Call and decodes the body as a XML object into response (if not nil) @@ -408,14 +409,14 @@ func (api *Client) CallJSON(opts *Opts, request interface{}, response interface{ // See CallJSON for a description of MultipartParams and related opts // // It will return resp if at all possible, even if err is set -func (api *Client) CallXML(opts *Opts, request interface{}, response interface{}) (resp *http.Response, err error) { - return api.callCodec(opts, request, response, xml.Marshal, DecodeXML, "application/xml") +func (api *Client) CallXML(ctx context.Context, opts *Opts, request interface{}, response interface{}) (resp *http.Response, err error) { + return api.callCodec(ctx, opts, request, response, xml.Marshal, DecodeXML, "application/xml") } type marshalFn func(v interface{}) ([]byte, error) type decodeFn func(resp *http.Response, result interface{}) (err error) -func (api *Client) callCodec(opts *Opts, request interface{}, response interface{}, marshal marshalFn, decode decodeFn, contentType string) (resp *http.Response, err error) { +func (api *Client) callCodec(ctx context.Context, opts *Opts, request interface{}, response interface{}, marshal marshalFn, decode decodeFn, contentType string) (resp *http.Response, err error) { var requestBody []byte // Marshal the request if given if request != nil { @@ -446,7 +447,7 @@ func (api *Client) callCodec(opts *Opts, request interface{}, response interface *opts.ContentLength += overhead } } - resp, err = api.Call(opts) + resp, err = api.Call(ctx, opts) if err != nil { return resp, err }