From ee28856f1d0a6b1e5e2eb61cd9e00a8dae42a587 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 16 Oct 2018 21:09:51 +0100 Subject: [PATCH] gcs: Allow compressed files to be downloaded - fixes #2658 Before this change, the go runtime would automatically decompress compressed objects leading to length mismatches. After this change rclone will download the compressed object which will match in length and checksum. --- backend/googlecloudstorage/googlecloudstorage.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/backend/googlecloudstorage/googlecloudstorage.go b/backend/googlecloudstorage/googlecloudstorage.go index 1ba2a945c..555b88962 100644 --- a/backend/googlecloudstorage/googlecloudstorage.go +++ b/backend/googlecloudstorage/googlecloudstorage.go @@ -298,6 +298,7 @@ type Object struct { bytes int64 // Bytes in the object modTime time.Time // Modified time of the object mimeType string + gzipped bool // set if object has Content-Encoding: gzip } // ------------------------------------------------------------ @@ -899,6 +900,7 @@ func (o *Object) setMetaData(info *storage.Object) { o.url = info.MediaLink o.bytes = int64(info.Size) o.mimeType = info.ContentType + o.gzipped = info.ContentEncoding == "gzip" // Read md5sum md5sumData, err := base64.StdEncoding.DecodeString(info.Md5Hash) @@ -1026,6 +1028,15 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read } req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext fs.FixRangeOption(options, o.bytes) + if o.gzipped { + // Allow files which are stored on the cloud storage system + // compressed to be downloaded without being decompressed. Note + // that setting this here overrides the automatic decompression + // in the Transport. + // + // See: https://cloud.google.com/storage/docs/transcoding + req.Header.Set("Accept-Encoding", "gzip") + } fs.OpenOptionAddHTTPHeaders(req.Header, options) var res *http.Response err = o.fs.pacer.Call(func() (bool, error) {