TrueCloudLab/rclone
16
0
Fork 2
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions 1

s3: cancel in progress multipart uploads and copies on rclone exit #4300

Browse source
This commit is contained in:
Nick Craig-Wood 2020-06-04 11:09:27 +01:00
parent 54fda3422e
commit 5f75444ef6
1 changed files with 26 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
backend/s3/s3.go
View file

@ -53,6 +53,7 @@ import (
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fs/walk"
"github.com/rclone/rclone/lib/atexit"
"github.com/rclone/rclone/lib/bucket"
"github.com/rclone/rclone/lib/encoder"
"github.com/rclone/rclone/lib/pacer"
@ -1900,9 +1901,8 @@ func (f *Fs) copyMultipart(ctx context.Context, req *s3.CopyObjectInput, dstBuck
}
uid := cout.UploadId
defer func() {
if err != nil {
// We can try to abort the upload, but ignore the error.
defer atexit.OnError(&err, func() {
// Try to abort the upload, but ignore the error.
fs.Debugf(nil, "Cancelling multipart copy")
_ = f.pacer.Call(func() (bool, error) {
_, err := f.c.AbortMultipartUploadWithContext(context.Background(), &s3.AbortMultipartUploadInput{
@ -1913,8 +1913,7 @@ func (f *Fs) copyMultipart(ctx context.Context, req *s3.CopyObjectInput, dstBuck
})
return f.shouldRetry(err)
})
}
}()
})()
partSize := int64(f.opt.CopyCutoff)
numParts := (srcSize-1)/partSize + 1
@ -2306,12 +2305,10 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
}
uid := cout.UploadId
defer func() {
defer atexit.OnError(&err, func() {
if o.fs.opt.LeavePartsOnError {
return
}
if err != nil {
// We can try to abort the upload, but ignore the error.
fs.Debugf(o, "Cancelling multipart upload")
errCancel := f.pacer.Call(func() (bool, error) {
_, err := f.c.AbortMultipartUploadWithContext(context.Background(), &s3.AbortMultipartUploadInput{
@ -2325,8 +2322,7 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
if errCancel != nil {
fs.Debugf(o, "Failed to cancel multipart upload: %v", errCancel)
}
}
}()
})()
var (
g, gCtx = errgroup.WithContext(ctx)