diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 7643e48d3..ceccd0e9c 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -1294,6 +1294,7 @@ type Fs struct { pacer *fs.Pacer // To pace the API calls srv *http.Client // a plain http client pool *pool.Pool // memory pool + etagIsNotMD5 bool // if set ETags are not MD5s } // Object describes a s3 object @@ -1618,7 +1619,16 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e opt.MemoryPoolUseMmap, ), } - + if opt.ServerSideEncryption == "aws:kms" || opt.SSECustomerAlgorithm != "" { + // From: https://docs.aws.amazon.com/AmazonS3/latest/API/RESTCommonResponseHeaders.html + // + // Objects encrypted by SSE-S3 or plaintext have ETags that are an MD5 + // digest of their data. + // + // Objects encrypted by SSE-C or SSE-KMS have ETags that are not an + // MD5 digest of their object data. + f.etagIsNotMD5 = true + } f.setRoot(root) f.features = (&fs.Features{ ReadMimeType: true, @@ -1666,7 +1676,11 @@ func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *s3.Obje } else { o.lastModified = *info.LastModified } - o.etag = aws.StringValue(info.ETag) + if o.fs.etagIsNotMD5 { + o.etag = "" + } else { + o.etag = aws.StringValue(info.ETag) + } o.bytes = aws.Int64Value(info.Size) o.storageClass = aws.StringValue(info.StorageClass) } else { @@ -2759,7 +2773,11 @@ func (o *Object) readMetaData(ctx context.Context) (err error) { if resp.ContentLength != nil { size = *resp.ContentLength } - o.etag = aws.StringValue(resp.ETag) + if o.fs.etagIsNotMD5 { + o.etag = "" + } else { + o.etag = aws.StringValue(resp.ETag) + } o.bytes = size o.meta = resp.Metadata if o.meta == nil { diff --git a/docs/content/s3.md b/docs/content/s3.md index b3b4da56c..b3421904f 100644 --- a/docs/content/s3.md +++ b/docs/content/s3.md @@ -433,11 +433,10 @@ that will generate one or more buckets that will work with `rclone sync`. ### Key Management System (KMS) ### -If you are using server-side encryption with KMS then you will find -you can't transfer small objects. As a work-around you can use the -`--ignore-checksum` flag. - -A proper fix is being worked on in [issue #1824](https://github.com/rclone/rclone/issues/1824). +If you are using server-side encryption with KMS then you must make +sure rclone is configured with `server_side_encryption = aws:kms` +otherwise you will find you can't transfer small objects - these will +create checksum errors. ### Glacier and Glacier Deep Archive ###