From cf771137959253c7461275630c56a751fe24a231 Mon Sep 17 00:00:00 2001 From: Manish Tomar Date: Thu, 1 Nov 2018 18:24:16 -0700 Subject: [PATCH 1/4] add pathspec for repo _layers directory Signed-off-by: Guillaume Rose --- registry/storage/paths.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/registry/storage/paths.go b/registry/storage/paths.go index 2d1f5132f..16e71583a 100644 --- a/registry/storage/paths.go +++ b/registry/storage/paths.go @@ -87,6 +87,7 @@ const ( // Blobs: // // layerLinkPathSpec: /v2/repositories//_layers///link +// layersPathSpec: /v2/repositories//_layers // // Uploads: // @@ -206,6 +207,8 @@ func pathFor(spec pathSpec) (string, error) { blobLinkPathComponents := append(repoPrefix, v.name, "_layers") return path.Join(path.Join(append(blobLinkPathComponents, components...)...), "link"), nil + case layersPathSpec: + return append(repoPrefix, v.repo, "_layers") case blobsPathSpec: blobsPathPrefix := append(rootPrefix, "blobs") return path.Join(blobsPathPrefix...), nil @@ -335,6 +338,13 @@ type manifestTagIndexEntryLinkPathSpec struct { func (manifestTagIndexEntryLinkPathSpec) pathSpec() {} +// layersPathSpec contains the path for the layers inside a repo +type layersPathSpec struct { + repo string +} + +func (layersPathSpec) pathSpec() {} + // blobLinkPathSpec specifies a path for a blob link, which is a file with a // blob id. The blob link will contain a content addressable blob id reference // into the blob store. The format of the contents is as follows: From fa7d94940821080f25fb5442cebbdc01f6a691ba Mon Sep 17 00:00:00 2001 From: Manish Tomar Date: Thu, 1 Nov 2018 18:24:36 -0700 Subject: [PATCH 2/4] allow Repository.BlobStore to enumerate over blobs Signed-off-by: Guillaume Rose --- registry/storage/registry.go | 1 + 1 file changed, 1 insertion(+) diff --git a/registry/storage/registry.go b/registry/storage/registry.go index 7e932ab9a..4809f153a 100644 --- a/registry/storage/registry.go +++ b/registry/storage/registry.go @@ -330,6 +330,7 @@ func (repo *repository) Blobs(ctx context.Context) distribution.BlobStore { // TODO(stevvooe): linkPath limits this blob store to only layers. // This instance cannot be used for manifest checks. linkPathFns: []linkPathFunc{blobLinkPath}, + linkDirectoryPathSpec: layersPathSpec, deleteEnabled: repo.registry.deleteEnabled, resumableDigestEnabled: repo.resumableDigestEnabled, } From 5538da49234f2dbdcf2495a90ea55006508a4b70 Mon Sep 17 00:00:00 2001 From: Manish Tomar Date: Thu, 1 Nov 2018 18:35:32 -0700 Subject: [PATCH 3/4] fixes to make layersPathSpec work Signed-off-by: Guillaume Rose --- registry/storage/paths.go | 4 ++-- registry/storage/registry.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/registry/storage/paths.go b/registry/storage/paths.go index 16e71583a..4e9ad0856 100644 --- a/registry/storage/paths.go +++ b/registry/storage/paths.go @@ -208,7 +208,7 @@ func pathFor(spec pathSpec) (string, error) { return path.Join(path.Join(append(blobLinkPathComponents, components...)...), "link"), nil case layersPathSpec: - return append(repoPrefix, v.repo, "_layers") + return path.Join(append(repoPrefix, v.name, "_layers")...), nil case blobsPathSpec: blobsPathPrefix := append(rootPrefix, "blobs") return path.Join(blobsPathPrefix...), nil @@ -340,7 +340,7 @@ func (manifestTagIndexEntryLinkPathSpec) pathSpec() {} // layersPathSpec contains the path for the layers inside a repo type layersPathSpec struct { - repo string + name string } func (layersPathSpec) pathSpec() {} diff --git a/registry/storage/registry.go b/registry/storage/registry.go index 4809f153a..d74403f50 100644 --- a/registry/storage/registry.go +++ b/registry/storage/registry.go @@ -330,7 +330,7 @@ func (repo *repository) Blobs(ctx context.Context) distribution.BlobStore { // TODO(stevvooe): linkPath limits this blob store to only layers. // This instance cannot be used for manifest checks. linkPathFns: []linkPathFunc{blobLinkPath}, - linkDirectoryPathSpec: layersPathSpec, + linkDirectoryPathSpec: layersPathSpec{name: repo.name.Name()}, deleteEnabled: repo.registry.deleteEnabled, resumableDigestEnabled: repo.resumableDigestEnabled, } From c9c33243002bc0a91aedb89d022bd5a6fd79ebe6 Mon Sep 17 00:00:00 2001 From: Guillaume Rose Date: Fri, 6 Dec 2019 08:52:55 +0100 Subject: [PATCH 4/4] Add unit tests for BlobEnumerator Signed-off-by: Guillaume Rose --- registry/storage/linkedblobstore_test.go | 49 ++++++++++++++++++++++++ registry/storage/paths_test.go | 4 ++ 2 files changed, 53 insertions(+) diff --git a/registry/storage/linkedblobstore_test.go b/registry/storage/linkedblobstore_test.go index 7682b45ca..55dccd13f 100644 --- a/registry/storage/linkedblobstore_test.go +++ b/registry/storage/linkedblobstore_test.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "reflect" + "sort" "strconv" "testing" @@ -14,6 +15,54 @@ import ( "github.com/opencontainers/go-digest" ) +func TestLinkedBlobStoreEnumerator(t *testing.T) { + fooRepoName, _ := reference.WithName("nm/foo") + fooEnv := newManifestStoreTestEnv(t, fooRepoName, "thetag") + ctx := context.Background() + + var expected []string + for i := 0; i < 2; i++ { + rs, dgst, err := testutil.CreateRandomTarFile() + if err != nil { + t.Fatalf("unexpected error generating test layer file") + } + + expected = append(expected, dgst.String()) + + wr, err := fooEnv.repository.Blobs(fooEnv.ctx).Create(fooEnv.ctx) + if err != nil { + t.Fatalf("unexpected error creating test upload: %v", err) + } + + if _, err := io.Copy(wr, rs); err != nil { + t.Fatalf("unexpected error copying to upload: %v", err) + } + + if _, err := wr.Commit(fooEnv.ctx, distribution.Descriptor{Digest: dgst}); err != nil { + t.Fatalf("unexpected error finishing upload: %v", err) + } + } + + enumerator, ok := fooEnv.repository.Blobs(fooEnv.ctx).(distribution.BlobEnumerator) + if !ok { + t.Fatalf("Blobs is not a BlobEnumerator") + } + + var actual []string + if err := enumerator.Enumerate(ctx, func(dgst digest.Digest) error { + actual = append(actual, dgst.String()) + return nil + }); err != nil { + t.Fatalf("cannot enumerate on repository: %v", err) + } + + sort.Strings(actual) + sort.Strings(expected) + if !reflect.DeepEqual(expected, actual) { + t.Fatalf("unexpected array difference (expected: %v actual: %v)", expected, actual) + } +} + func TestLinkedBlobStoreCreateWithMountFrom(t *testing.T) { fooRepoName, _ := reference.WithName("nm/foo") fooEnv := newManifestStoreTestEnv(t, fooRepoName, "thetag") diff --git a/registry/storage/paths_test.go b/registry/storage/paths_test.go index ab3b8445a..68fca59e5 100644 --- a/registry/storage/paths_test.go +++ b/registry/storage/paths_test.go @@ -83,6 +83,10 @@ func TestPathMapper(t *testing.T) { }, expected: "/docker/registry/v2/repositories/foo/bar/_uploads/asdf-asdf-asdf-adsf/startedat", }, + { + spec: layersPathSpec{name: "foo/bar"}, + expected: "/docker/registry/v2/repositories/foo/bar/_layers", + }, } { p, err := pathFor(testcase.spec) if err != nil {