KurlesHS/rclone
1
0
Fork 0
forked from TrueCloudLab/rclone
Code Pull requests Activity

s3: cancel in progress multipart uploads and copies on rclone exit #4300

Browse source
This commit is contained in:
Nick Craig-Wood 2020-06-04 11:09:27 +01:00
parent 54fda3422e
commit 5f75444ef6
1 changed files with 26 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

56
backend/s3/s3.go
View file

@ -53,6 +53,7 @@ import (
"github.com/rclone/rclone/fs/fshttp" "github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fs/walk" "github.com/rclone/rclone/fs/walk"
"github.com/rclone/rclone/lib/atexit"
"github.com/rclone/rclone/lib/bucket" "github.com/rclone/rclone/lib/bucket"
"github.com/rclone/rclone/lib/encoder" "github.com/rclone/rclone/lib/encoder"
"github.com/rclone/rclone/lib/pacer" "github.com/rclone/rclone/lib/pacer"
@ -1900,21 +1901,19 @@ func (f *Fs) copyMultipart(ctx context.Context, req *s3.CopyObjectInput, dstBuck
} }
uid := cout.UploadId uid := cout.UploadId
defer func() { defer atexit.OnError(&err, func() {
if err != nil { // Try to abort the upload, but ignore the error.
// We can try to abort the upload, but ignore the error. fs.Debugf(nil, "Cancelling multipart copy")
fs.Debugf(nil, "Cancelling multipart copy") _ = f.pacer.Call(func() (bool, error) {
_ = f.pacer.Call(func() (bool, error) { _, err := f.c.AbortMultipartUploadWithContext(context.Background(), &s3.AbortMultipartUploadInput{
_, err := f.c.AbortMultipartUploadWithContext(context.Background(), &s3.AbortMultipartUploadInput{ Bucket: &dstBucket,
Bucket: &dstBucket, Key: &dstPath,
Key: &dstPath, UploadId: uid,
UploadId: uid, RequestPayer: req.RequestPayer,
RequestPayer: req.RequestPayer,
})
return f.shouldRetry(err)
}) })
} return f.shouldRetry(err)
}() })
})()
partSize := int64(f.opt.CopyCutoff) partSize := int64(f.opt.CopyCutoff)
numParts := (srcSize-1)/partSize + 1 numParts := (srcSize-1)/partSize + 1
@ -2306,27 +2305,24 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
} }
uid := cout.UploadId uid := cout.UploadId
defer func() { defer atexit.OnError(&err, func() {
if o.fs.opt.LeavePartsOnError { if o.fs.opt.LeavePartsOnError {
return return
} }
if err != nil { fs.Debugf(o, "Cancelling multipart upload")
// We can try to abort the upload, but ignore the error. errCancel := f.pacer.Call(func() (bool, error) {
fs.Debugf(o, "Cancelling multipart upload") _, err := f.c.AbortMultipartUploadWithContext(context.Background(), &s3.AbortMultipartUploadInput{
errCancel := f.pacer.Call(func() (bool, error) { Bucket: req.Bucket,
_, err := f.c.AbortMultipartUploadWithContext(context.Background(), &s3.AbortMultipartUploadInput{ Key: req.Key,
Bucket: req.Bucket, UploadId: uid,
Key: req.Key, RequestPayer: req.RequestPayer,
UploadId: uid,
RequestPayer: req.RequestPayer,
})
return f.shouldRetry(err)
}) })
if errCancel != nil { return f.shouldRetry(err)
fs.Debugf(o, "Failed to cancel multipart upload: %v", errCancel) })
} if errCancel != nil {
fs.Debugf(o, "Failed to cancel multipart upload: %v", errCancel)
} }
}() })()
var ( var (
g, gCtx = errgroup.WithContext(ctx) g, gCtx = errgroup.WithContext(ctx)