From 8b31a894bd44048dc8b88b255d68fa134184f819 Mon Sep 17 00:00:00 2001 From: Damien Mathieu Date: Tue, 21 May 2019 10:14:59 +0200 Subject: [PATCH 1/2] deduce blob UUID from location if it wasn't provided in the headers Some registries (ECR) don't provide a `Docker-Upload-UUID` when creating a blob. So we can't rely on that header. Fallback to reading it from the URL. Signed-off-by: Damien Mathieu --- registry/client/repository.go | 5 ++ registry/client/repository_test.go | 106 +++++++++++++++++++++++++++++ 2 files changed, 111 insertions(+) diff --git a/registry/client/repository.go b/registry/client/repository.go index aa442e65..cc784a31 100644 --- a/registry/client/repository.go +++ b/registry/client/repository.go @@ -752,6 +752,11 @@ func (bs *blobs) Create(ctx context.Context, options ...distribution.BlobCreateO case http.StatusAccepted: // TODO(dmcgowan): Check for invalid UUID uuid := resp.Header.Get("Docker-Upload-UUID") + if uuid == "" { + parts := strings.Split(resp.Header.Get("Location"), "/") + uuid = parts[len(parts)-1] + } + location, err := sanitizeLocation(resp.Header.Get("Location"), u) if err != nil { return nil, err diff --git a/registry/client/repository_test.go b/registry/client/repository_test.go index 746821b0..1b343d50 100644 --- a/registry/client/repository_test.go +++ b/registry/client/repository_test.go @@ -473,6 +473,112 @@ func TestBlobUploadMonolithic(t *testing.T) { } } +func TestBlobUploadMonolithicNoDockerUploadUUID(t *testing.T) { + dgst, b1 := newRandomBlob(1024) + var m testutil.RequestResponseMap + repo, _ := reference.WithName("test.example.com/uploadrepo") + uploadID := uuid.Generate().String() + m = append(m, testutil.RequestResponseMapping{ + Request: testutil.Request{ + Method: "POST", + Route: "/v2/" + repo.Name() + "/blobs/uploads/", + }, + Response: testutil.Response{ + StatusCode: http.StatusAccepted, + Headers: http.Header(map[string][]string{ + "Content-Length": {"0"}, + "Location": {"/v2/" + repo.Name() + "/blobs/uploads/" + uploadID}, + "Range": {"0-0"}, + }), + }, + }) + m = append(m, testutil.RequestResponseMapping{ + Request: testutil.Request{ + Method: "PATCH", + Route: "/v2/" + repo.Name() + "/blobs/uploads/" + uploadID, + Body: b1, + }, + Response: testutil.Response{ + StatusCode: http.StatusAccepted, + Headers: http.Header(map[string][]string{ + "Location": {"/v2/" + repo.Name() + "/blobs/uploads/" + uploadID}, + "Content-Length": {"0"}, + "Docker-Content-Digest": {dgst.String()}, + "Range": {fmt.Sprintf("0-%d", len(b1)-1)}, + }), + }, + }) + m = append(m, testutil.RequestResponseMapping{ + Request: testutil.Request{ + Method: "PUT", + Route: "/v2/" + repo.Name() + "/blobs/uploads/" + uploadID, + QueryParams: map[string][]string{ + "digest": {dgst.String()}, + }, + }, + Response: testutil.Response{ + StatusCode: http.StatusCreated, + Headers: http.Header(map[string][]string{ + "Content-Length": {"0"}, + "Docker-Content-Digest": {dgst.String()}, + "Content-Range": {fmt.Sprintf("0-%d", len(b1)-1)}, + }), + }, + }) + m = append(m, testutil.RequestResponseMapping{ + Request: testutil.Request{ + Method: "HEAD", + Route: "/v2/" + repo.Name() + "/blobs/" + dgst.String(), + }, + Response: testutil.Response{ + StatusCode: http.StatusOK, + Headers: http.Header(map[string][]string{ + "Content-Length": {fmt.Sprint(len(b1))}, + "Last-Modified": {time.Now().Add(-1 * time.Second).Format(time.ANSIC)}, + }), + }, + }) + + e, c := testServer(m) + defer c() + + ctx := context.Background() + r, err := NewRepository(repo, e, nil) + if err != nil { + t.Fatal(err) + } + l := r.Blobs(ctx) + + upload, err := l.Create(ctx) + if err != nil { + t.Fatal(err) + } + + if upload.ID() != uploadID { + log.Fatalf("Unexpected UUID %s; expected %s", upload.ID(), uploadID) + } + + n, err := upload.ReadFrom(bytes.NewReader(b1)) + if err != nil { + t.Fatal(err) + } + if n != int64(len(b1)) { + t.Fatalf("Unexpected ReadFrom length: %d; expected: %d", n, len(b1)) + } + + blob, err := upload.Commit(ctx, distribution.Descriptor{ + Digest: dgst, + Size: int64(len(b1)), + }) + if err != nil { + t.Fatal(err) + } + + if blob.Size != int64(len(b1)) { + t.Fatalf("Unexpected blob size: %d; expected: %d", blob.Size, len(b1)) + } +} + func TestBlobMount(t *testing.T) { dgst, content := newRandomBlob(1024) var m testutil.RequestResponseMap From a45e5cb13f9e2ecd3ec9408af674fe77b050d81e Mon Sep 17 00:00:00 2001 From: Damien Mathieu Date: Tue, 21 May 2019 10:22:29 +0200 Subject: [PATCH 2/2] handle create blob if the uuid couldn't be retrieved from headers or URL Signed-off-by: Damien Mathieu --- registry/client/repository.go | 3 + registry/client/repository_test.go | 88 +++++++++++++++++++++++++++++- 2 files changed, 90 insertions(+), 1 deletion(-) diff --git a/registry/client/repository.go b/registry/client/repository.go index cc784a31..55e19a4d 100644 --- a/registry/client/repository.go +++ b/registry/client/repository.go @@ -756,6 +756,9 @@ func (bs *blobs) Create(ctx context.Context, options ...distribution.BlobCreateO parts := strings.Split(resp.Header.Get("Location"), "/") uuid = parts[len(parts)-1] } + if uuid == "" { + return nil, errors.New("cannot retrieve docker upload UUID") + } location, err := sanitizeLocation(resp.Header.Get("Location"), u) if err != nil { diff --git a/registry/client/repository_test.go b/registry/client/repository_test.go index 1b343d50..4905c503 100644 --- a/registry/client/repository_test.go +++ b/registry/client/repository_test.go @@ -473,7 +473,7 @@ func TestBlobUploadMonolithic(t *testing.T) { } } -func TestBlobUploadMonolithicNoDockerUploadUUID(t *testing.T) { +func TestBlobUploadMonolithicDockerUploadUUIDFromURL(t *testing.T) { dgst, b1 := newRandomBlob(1024) var m testutil.RequestResponseMap repo, _ := reference.WithName("test.example.com/uploadrepo") @@ -579,6 +579,92 @@ func TestBlobUploadMonolithicNoDockerUploadUUID(t *testing.T) { } } +func TestBlobUploadMonolithicNoDockerUploadUUID(t *testing.T) { + dgst, b1 := newRandomBlob(1024) + var m testutil.RequestResponseMap + repo, _ := reference.WithName("test.example.com/uploadrepo") + m = append(m, testutil.RequestResponseMapping{ + Request: testutil.Request{ + Method: "POST", + Route: "/v2/" + repo.Name() + "/blobs/uploads/", + }, + Response: testutil.Response{ + StatusCode: http.StatusAccepted, + Headers: http.Header(map[string][]string{ + "Content-Length": {"0"}, + "Location": {"/v2/" + repo.Name() + "/blobs/uploads/"}, + "Range": {"0-0"}, + }), + }, + }) + m = append(m, testutil.RequestResponseMapping{ + Request: testutil.Request{ + Method: "PATCH", + Route: "/v2/" + repo.Name() + "/blobs/uploads/", + Body: b1, + }, + Response: testutil.Response{ + StatusCode: http.StatusAccepted, + Headers: http.Header(map[string][]string{ + "Location": {"/v2/" + repo.Name() + "/blobs/uploads/"}, + "Content-Length": {"0"}, + "Docker-Content-Digest": {dgst.String()}, + "Range": {fmt.Sprintf("0-%d", len(b1)-1)}, + }), + }, + }) + m = append(m, testutil.RequestResponseMapping{ + Request: testutil.Request{ + Method: "PUT", + Route: "/v2/" + repo.Name() + "/blobs/uploads/", + QueryParams: map[string][]string{ + "digest": {dgst.String()}, + }, + }, + Response: testutil.Response{ + StatusCode: http.StatusCreated, + Headers: http.Header(map[string][]string{ + "Content-Length": {"0"}, + "Docker-Content-Digest": {dgst.String()}, + "Content-Range": {fmt.Sprintf("0-%d", len(b1)-1)}, + }), + }, + }) + m = append(m, testutil.RequestResponseMapping{ + Request: testutil.Request{ + Method: "HEAD", + Route: "/v2/" + repo.Name() + "/blobs/" + dgst.String(), + }, + Response: testutil.Response{ + StatusCode: http.StatusOK, + Headers: http.Header(map[string][]string{ + "Content-Length": {fmt.Sprint(len(b1))}, + "Last-Modified": {time.Now().Add(-1 * time.Second).Format(time.ANSIC)}, + }), + }, + }) + + e, c := testServer(m) + defer c() + + ctx := context.Background() + r, err := NewRepository(repo, e, nil) + if err != nil { + t.Fatal(err) + } + l := r.Blobs(ctx) + + upload, err := l.Create(ctx) + + if err.Error() != "cannot retrieve docker upload UUID" { + log.Fatalf("expected rejection to retrieve docker upload UUID error. Got %q", err) + } + + if upload != nil { + log.Fatal("Expected upload to be nil") + } +} + func TestBlobMount(t *testing.T) { dgst, content := newRandomBlob(1024) var m testutil.RequestResponseMap