From 1d52366d2c76ad1219ba8ef666bddd03b6cb8dd3 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Tue, 16 Mar 2021 09:12:03 -0700 Subject: [PATCH 1/2] Merge pull request #2815 from bainsy88/issue_2814 Add code to handle pagination of parts. Fixes max layer size of 10GB bug Signed-off-by: David van der Spek --- registry/storage/driver/s3-aws/s3.go | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/registry/storage/driver/s3-aws/s3.go b/registry/storage/driver/s3-aws/s3.go index d9c1a826..e59c1767 100644 --- a/registry/storage/driver/s3-aws/s3.go +++ b/registry/storage/driver/s3-aws/s3.go @@ -549,9 +549,9 @@ func (d *driver) Reader(ctx context.Context, path string, offset int64) (io.Read // Writer returns a FileWriter which will store the content written to it // at the location designated by "path" after the call to Commit. -func (d *driver) Writer(ctx context.Context, path string, append bool) (storagedriver.FileWriter, error) { +func (d *driver) Writer(ctx context.Context, path string, appendParam bool) (storagedriver.FileWriter, error) { key := d.s3Path(path) - if !append { + if !appendParam { // TODO (brianbland): cancel other uploads at this path resp, err := d.S3.CreateMultipartUpload(&s3.CreateMultipartUploadInput{ Bucket: aws.String(d.Bucket), @@ -574,7 +574,7 @@ func (d *driver) Writer(ctx context.Context, path string, append bool) (storaged if err != nil { return nil, parseError(path, err) } - + var allParts []*s3.Part for _, multi := range resp.Uploads { if key != *multi.Key { continue @@ -587,11 +587,20 @@ func (d *driver) Writer(ctx context.Context, path string, append bool) (storaged if err != nil { return nil, parseError(path, err) } - var multiSize int64 - for _, part := range resp.Parts { - multiSize += *part.Size + allParts = append(allParts, resp.Parts...) + for *resp.IsTruncated { + resp, err = d.S3.ListParts(&s3.ListPartsInput{ + Bucket: aws.String(d.Bucket), + Key: aws.String(key), + UploadId: multi.UploadId, + PartNumberMarker: resp.NextPartNumberMarker, + }) + if err != nil { + return nil, parseError(path, err) + } + allParts = append(allParts, resp.Parts...) } - return d.newWriter(key, *multi.UploadId, resp.Parts), nil + return d.newWriter(key, *multi.UploadId, allParts), nil } return nil, storagedriver.PathNotFoundError{Path: path} } From 22a805033aa861ad171f4a0b560d241a7c57ac86 Mon Sep 17 00:00:00 2001 From: David van der Spek Date: Mon, 8 May 2023 13:28:28 -0400 Subject: [PATCH 2/2] fix(ci): use go install instead of go get Signed-off-by: David van der Spek --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4064d1c8..aeaf4d5f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,7 +31,7 @@ jobs: run: | sudo apt-get -q update sudo -E apt-get -yq --no-install-suggests --no-install-recommends install python2-minimal - cd /tmp && go get -u github.com/vbatts/git-validation + cd /tmp && go install github.com/vbatts/git-validation@latest - name: Build working-directory: ./src/github.com/docker/distribution @@ -47,4 +47,4 @@ jobs: - uses: codecov/codecov-action@v1 with: - directory: ./src/github.com/docker/distribution \ No newline at end of file + directory: ./src/github.com/docker/distribution