From 6b55b8b133cb5508d62793925016ed0566b55978 Mon Sep 17 00:00:00 2001 From: Aleksandar Jankovic Date: Wed, 24 Jul 2019 11:03:38 +0200 Subject: [PATCH] s3: add option for multipart failiure behaviour This is needed for resuming uploads across different sessions. --- backend/s3/s3.go | 14 +++++++++++++- docs/content/s3.md | 14 ++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 3072cdddc..bd2de938e 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -753,6 +753,17 @@ Use this only if v4 signatures don't work, eg pre Jewel/v10 CEPH.`, See: [AWS S3 Transfer acceleration](https://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration-examples.html)`, Default: false, Advanced: true, + }, { + Name: "leave_parts_on_error", + Provider: "AWS", + Help: `If true avoid calling abort upload on a failure, leaving all successfully uploaded parts on S3 for manual recovery. + +It should be set to true for resuming uploads across different sessions. + +WARNING: Storing parts of an incomplete multipart upload counts towards space usage on S3 and will add additional costs if not cleaned up. +`, + Default: false, + Advanced: true, }}, }) } @@ -793,6 +804,7 @@ type Options struct { ForcePathStyle bool `config:"force_path_style"` V2Auth bool `config:"v2_auth"` UseAccelerateEndpoint bool `config:"use_accelerate_endpoint"` + LeavePartsOnError bool `config:"leave_parts_on_error"` } // Fs represents a remote s3 server @@ -1880,7 +1892,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op if multipart { uploader = s3manager.NewUploader(o.fs.ses, func(u *s3manager.Uploader) { u.Concurrency = o.fs.opt.UploadConcurrency - u.LeavePartsOnError = false + u.LeavePartsOnError = o.fs.opt.LeavePartsOnError u.S3 = o.fs.c u.PartSize = int64(o.fs.opt.ChunkSize) diff --git a/docs/content/s3.md b/docs/content/s3.md index 934c23bdd..b543f2a2e 100644 --- a/docs/content/s3.md +++ b/docs/content/s3.md @@ -1109,6 +1109,20 @@ See: [AWS S3 Transfer acceleration](https://docs.aws.amazon.com/AmazonS3/latest/ - Type: bool - Default: false +#### --s3-leave-parts-on-error + +If true avoid calling abort upload on a failure, leaving all successfully uploaded parts on S3 for manual recovery. + +It should be set to true for resuming uploads across different sessions. + +WARNING: Storing parts of an incomplete multipart upload counts towards space usage on S3 and will add additional costs if not cleaned up. + + +- Config: leave_parts_on_error +- Env Var: RCLONE_S3_LEAVE_PARTS_ON_ERROR +- Type: bool +- Default: false + ### Anonymous access to public buckets ###