From 9ebf151ac2f72eaa597728390e10714cb99661a4 Mon Sep 17 00:00:00 2001 From: Manish Tomar Date: Mon, 29 Oct 2018 18:19:05 -0700 Subject: [PATCH 1/2] API to retrive tag's digests Add an interface alongside TagStore that provides API to retreive digests of all manifests that a tag historically pointed to. It also includes currently linked tag. Signed-off-by: Manish Tomar --- registry/storage/tagstore.go | 34 ++++++++++++ registry/storage/tagstore_test.go | 86 ++++++++++++++++++++++++++++++- tags.go | 9 ++++ 3 files changed, 128 insertions(+), 1 deletion(-) diff --git a/registry/storage/tagstore.go b/registry/storage/tagstore.go index f80b96282..e4b1ed15a 100644 --- a/registry/storage/tagstore.go +++ b/registry/storage/tagstore.go @@ -196,3 +196,37 @@ func (ts *tagStore) Lookup(ctx context.Context, desc distribution.Descriptor) ([ return tags, nil } + +func (ts *tagStore) Indexes(ctx context.Context, tag string) ([]digest.Digest, error) { + var tagLinkPath = func(name string, dgst digest.Digest) (string, error) { + return pathFor(manifestTagIndexEntryLinkPathSpec{ + name: name, + tag: tag, + revision: dgst, + }) + } + lbs := &linkedBlobStore{ + blobStore: ts.blobStore, + blobAccessController: &linkedBlobStatter{ + blobStore: ts.blobStore, + repository: ts.repository, + linkPathFns: []linkPathFunc{manifestRevisionLinkPath}, + }, + repository: ts.repository, + ctx: ctx, + linkPathFns: []linkPathFunc{tagLinkPath}, + linkDirectoryPathSpec: manifestTagIndexPathSpec{ + name: ts.repository.Named().Name(), + tag: tag, + }, + } + var dgsts []digest.Digest + err := lbs.Enumerate(ctx, func(dgst digest.Digest) error { + dgsts = append(dgsts, dgst) + return nil + }) + if err != nil { + return nil, err + } + return dgsts, nil +} diff --git a/registry/storage/tagstore_test.go b/registry/storage/tagstore_test.go index 314fa433c..938aa69d8 100644 --- a/registry/storage/tagstore_test.go +++ b/registry/storage/tagstore_test.go @@ -2,15 +2,22 @@ package storage import ( "context" + "reflect" "testing" "github.com/docker/distribution" + "github.com/docker/distribution/manifest" + "github.com/docker/distribution/manifest/schema2" "github.com/docker/distribution/reference" "github.com/docker/distribution/registry/storage/driver/inmemory" + digest "github.com/opencontainers/go-digest" ) type tagsTestEnv struct { ts distribution.TagService + bs distribution.BlobStore + ms distribution.ManifestService + gbs distribution.BlobStatter ctx context.Context } @@ -27,10 +34,17 @@ func testTagStore(t *testing.T) *tagsTestEnv { if err != nil { t.Fatal(err) } + ms, err := repo.Manifests(ctx) + if err != nil { + t.Fatal(err) + } return &tagsTestEnv{ ctx: ctx, ts: repo.Tags(ctx), + bs: repo.Blobs(ctx), + gbs: reg.BlobStatter(), + ms: ms, } } @@ -205,5 +219,75 @@ func TestTagLookup(t *testing.T) { if len(tags) != 2 { t.Errorf("Lookup of descB returned %d tags, expected 2", len(tags)) } - +} + +func TestTagIndexes(t *testing.T) { + env := testTagStore(t) + tagStore := env.ts + ctx := env.ctx + + indexes, ok := tagStore.(distribution.TagIndexes) + if !ok { + t.Fatal("tagStore does not implement TagIndexes interface") + } + + conf, err := env.bs.Put(ctx, "application/octet-stream", []byte{0}) + if err != nil { + t.Fatal(err) + } + + dgstsSet := make(map[digest.Digest]bool) + for i := 0; i < 3; i++ { + layer, err := env.bs.Put(ctx, "application/octet-stream", []byte{byte(i + 1)}) + if err != nil { + t.Fatal(err) + } + m := schema2.Manifest{ + Versioned: manifest.Versioned{ + SchemaVersion: 2, + MediaType: schema2.MediaTypeManifest, + }, + Config: distribution.Descriptor{ + Digest: conf.Digest, + Size: 1, + MediaType: schema2.MediaTypeImageConfig, + }, + Layers: []distribution.Descriptor{ + { + Digest: layer.Digest, + Size: 1, + MediaType: schema2.MediaTypeLayer, + }, + }, + } + dm, err := schema2.FromStruct(m) + if err != nil { + t.Fatal(err) + } + dgst, err := env.ms.Put(ctx, dm) + if err != nil { + t.Fatal(err) + } + desc, err := env.gbs.Stat(ctx, dgst) + if err != nil { + t.Fatal(err) + } + err = tagStore.Tag(ctx, "t", desc) + if err != nil { + t.Fatal(err) + } + dgstsSet[dgst] = true + } + + gotDgsts, err := indexes.Indexes(ctx, "t") + if err != nil { + t.Fatal(err) + } + gotDgstsSet := make(map[digest.Digest]bool) + for _, dgst := range gotDgsts { + gotDgstsSet[dgst] = true + } + if !reflect.DeepEqual(dgstsSet, gotDgstsSet) { + t.Fatalf("Expected digests: %v but got digests: %v", dgstsSet, gotDgstsSet) + } } diff --git a/tags.go b/tags.go index f22df2b85..60b96b27f 100644 --- a/tags.go +++ b/tags.go @@ -2,6 +2,8 @@ package distribution import ( "context" + + digest "github.com/opencontainers/go-digest" ) // TagService provides access to information about tagged objects. @@ -25,3 +27,10 @@ type TagService interface { // Lookup returns the set of tags referencing the given digest. Lookup(ctx context.Context, digest Descriptor) ([]string, error) } + +// TagIndexes proves method to retreive all the digests that a tag historically pointed to +type TagIndexes interface { + // Indexes returns set of digests that this tag historically pointed to. This also includes + // currently linked digest. There is no ordering guaranteed + Indexes(ctx context.Context, tag string) ([]digest.Digest, error) +} From 1251e51ad0d8bf24517010a259b8f56c31a60fb6 Mon Sep 17 00:00:00 2001 From: Manish Tomar Date: Thu, 1 Nov 2018 10:31:08 -0700 Subject: [PATCH 2/2] better name and updated tests - use ManifestDigests name instead of Indexes - update tests to validate against multiple tags Signed-off-by: Manish Tomar --- registry/storage/tagstore.go | 2 +- registry/storage/tagstore_test.go | 51 ++++++++++++++++++++++--------- tags.go | 11 ++++--- 3 files changed, 44 insertions(+), 20 deletions(-) diff --git a/registry/storage/tagstore.go b/registry/storage/tagstore.go index e4b1ed15a..2e7aaca6e 100644 --- a/registry/storage/tagstore.go +++ b/registry/storage/tagstore.go @@ -197,7 +197,7 @@ func (ts *tagStore) Lookup(ctx context.Context, desc distribution.Descriptor) ([ return tags, nil } -func (ts *tagStore) Indexes(ctx context.Context, tag string) ([]digest.Digest, error) { +func (ts *tagStore) ManifestDigests(ctx context.Context, tag string) ([]digest.Digest, error) { var tagLinkPath = func(name string, dgst digest.Digest) (string, error) { return pathFor(manifestTagIndexEntryLinkPathSpec{ name: name, diff --git a/registry/storage/tagstore_test.go b/registry/storage/tagstore_test.go index 938aa69d8..6f4c6c915 100644 --- a/registry/storage/tagstore_test.go +++ b/registry/storage/tagstore_test.go @@ -226,9 +226,9 @@ func TestTagIndexes(t *testing.T) { tagStore := env.ts ctx := env.ctx - indexes, ok := tagStore.(distribution.TagIndexes) + md, ok := tagStore.(distribution.TagManifestsProvider) if !ok { - t.Fatal("tagStore does not implement TagIndexes interface") + t.Fatal("tagStore does not implement TagManifestDigests interface") } conf, err := env.bs.Put(ctx, "application/octet-stream", []byte{0}) @@ -236,8 +236,9 @@ func TestTagIndexes(t *testing.T) { t.Fatal(err) } - dgstsSet := make(map[digest.Digest]bool) - for i := 0; i < 3; i++ { + t1Dgsts := make(map[digest.Digest]struct{}) + t2Dgsts := make(map[digest.Digest]struct{}) + for i := 0; i < 5; i++ { layer, err := env.bs.Put(ctx, "application/octet-stream", []byte{byte(i + 1)}) if err != nil { t.Fatal(err) @@ -272,22 +273,44 @@ func TestTagIndexes(t *testing.T) { if err != nil { t.Fatal(err) } - err = tagStore.Tag(ctx, "t", desc) - if err != nil { - t.Fatal(err) + if i < 3 { + // tag first 3 manifests as "t1" + err = tagStore.Tag(ctx, "t1", desc) + if err != nil { + t.Fatal(err) + } + t1Dgsts[dgst] = struct{}{} + } else { + // the last two under "t2" + err = tagStore.Tag(ctx, "t2", desc) + if err != nil { + t.Fatal(err) + } + t2Dgsts[dgst] = struct{}{} } - dgstsSet[dgst] = true } - gotDgsts, err := indexes.Indexes(ctx, "t") + gotT1Dgsts, err := md.ManifestDigests(ctx, "t1") if err != nil { t.Fatal(err) } - gotDgstsSet := make(map[digest.Digest]bool) - for _, dgst := range gotDgsts { - gotDgstsSet[dgst] = true + if !reflect.DeepEqual(t1Dgsts, digestMap(gotT1Dgsts)) { + t.Fatalf("Expected digests: %v but got digests: %v", t1Dgsts, digestMap(gotT1Dgsts)) } - if !reflect.DeepEqual(dgstsSet, gotDgstsSet) { - t.Fatalf("Expected digests: %v but got digests: %v", dgstsSet, gotDgstsSet) + + gotT2Dgsts, err := md.ManifestDigests(ctx, "t2") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(t2Dgsts, digestMap(gotT2Dgsts)) { + t.Fatalf("Expected digests: %v but got digests: %v", t2Dgsts, digestMap(gotT2Dgsts)) } } + +func digestMap(dgsts []digest.Digest) map[digest.Digest]struct{} { + set := make(map[digest.Digest]struct{}) + for _, dgst := range dgsts { + set[dgst] = struct{}{} + } + return set +} diff --git a/tags.go b/tags.go index 60b96b27f..f0e0bea3b 100644 --- a/tags.go +++ b/tags.go @@ -28,9 +28,10 @@ type TagService interface { Lookup(ctx context.Context, digest Descriptor) ([]string, error) } -// TagIndexes proves method to retreive all the digests that a tag historically pointed to -type TagIndexes interface { - // Indexes returns set of digests that this tag historically pointed to. This also includes - // currently linked digest. There is no ordering guaranteed - Indexes(ctx context.Context, tag string) ([]digest.Digest, error) +// TagManifestsProvider provides method to retreive the digests of manifests that a tag historically +// pointed to +type TagManifestsProvider interface { + // ManifestDigests returns set of digests that this tag historically pointed to. This also + // includes currently linked digest. There is no ordering guaranteed + ManifestDigests(ctx context.Context, tag string) ([]digest.Digest, error) }