Implement immutable manifest reference support
This changeset implements immutable manifest references via the HTTP API. Most of the changes follow from modifications to ManifestService. Once updates were made across the repo to implement these changes, the http handlers were change accordingly. The new methods on ManifestService will be broken out into a tagging service in a later PR. Unfortunately, due to complexities around managing the manifest tag index in an eventually consistent manner, direct deletes of manifests have been disabled. Signed-off-by: Stephen J Day <stephen.day@docker.com>
This commit is contained in:
Stephen J Day
parent
f46a1b73e8
commit
008236cfef
12 changed files with 325 additions and 99 deletions
|
|
@ -5,6 +5,7 @@ import (
|
|||
|
||||
"github.com/docker/distribution"
|
||||
ctxu "github.com/docker/distribution/context"
|
||||
"github.com/docker/distribution/digest"
|
||||
"github.com/docker/distribution/manifest"
|
||||
"github.com/docker/libtrust"
|
||||
)
|
||||
|
|
@ -18,31 +19,17 @@ type manifestStore struct {
|
|||
|
||||
var _ distribution.ManifestService = &manifestStore{}
|
||||
|
||||
// func (ms *manifestStore) Repository() Repository {
|
||||
// return ms.repository
|
||||
// }
|
||||
|
||||
func (ms *manifestStore) Tags() ([]string, error) {
|
||||
ctxu.GetLogger(ms.repository.ctx).Debug("(*manifestStore).Tags")
|
||||
return ms.tagStore.tags()
|
||||
}
|
||||
|
||||
func (ms *manifestStore) Exists(tag string) (bool, error) {
|
||||
func (ms *manifestStore) Exists(dgst digest.Digest) (bool, error) {
|
||||
ctxu.GetLogger(ms.repository.ctx).Debug("(*manifestStore).Exists")
|
||||
return ms.tagStore.exists(tag)
|
||||
return ms.revisionStore.exists(dgst)
|
||||
}
|
||||
|
||||
func (ms *manifestStore) Get(tag string) (*manifest.SignedManifest, error) {
|
||||
func (ms *manifestStore) Get(dgst digest.Digest) (*manifest.SignedManifest, error) {
|
||||
ctxu.GetLogger(ms.repository.ctx).Debug("(*manifestStore).Get")
|
||||
dgst, err := ms.tagStore.resolve(tag)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return ms.revisionStore.get(dgst)
|
||||
}
|
||||
|
||||
func (ms *manifestStore) Put(tag string, manifest *manifest.SignedManifest) error {
|
||||
func (ms *manifestStore) Put(manifest *manifest.SignedManifest) error {
|
||||
ctxu.GetLogger(ms.repository.ctx).Debug("(*manifestStore).Put")
|
||||
|
||||
// TODO(stevvooe): Add check here to see if the revision is already
|
||||
|
|
@ -51,7 +38,7 @@ func (ms *manifestStore) Put(tag string, manifest *manifest.SignedManifest) erro
|
|||
// indicating what happened.
|
||||
|
||||
// Verify the manifest.
|
||||
if err := ms.verifyManifest(tag, manifest); err != nil {
|
||||
if err := ms.verifyManifest(manifest); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
|
|
@ -62,46 +49,46 @@ func (ms *manifestStore) Put(tag string, manifest *manifest.SignedManifest) erro
|
|||
}
|
||||
|
||||
// Now, tag the manifest
|
||||
return ms.tagStore.tag(tag, revision)
|
||||
return ms.tagStore.tag(manifest.Tag, revision)
|
||||
}
|
||||
|
||||
// Delete removes all revisions of the given tag. We may want to change these
|
||||
// semantics in the future, but this will maintain consistency. The underlying
|
||||
// blobs are left alone.
|
||||
func (ms *manifestStore) Delete(tag string) error {
|
||||
ctxu.GetLogger(ms.repository.ctx).Debug("(*manifestStore).Delete")
|
||||
// Delete removes the revision of the specified manfiest.
|
||||
func (ms *manifestStore) Delete(dgst digest.Digest) error {
|
||||
ctxu.GetLogger(ms.repository.ctx).Debug("(*manifestStore).Delete - unsupported")
|
||||
return fmt.Errorf("deletion of manifests not supported")
|
||||
}
|
||||
|
||||
revisions, err := ms.tagStore.revisions(tag)
|
||||
func (ms *manifestStore) Tags() ([]string, error) {
|
||||
ctxu.GetLogger(ms.repository.ctx).Debug("(*manifestStore).Tags")
|
||||
return ms.tagStore.tags()
|
||||
}
|
||||
|
||||
func (ms *manifestStore) ExistsByTag(tag string) (bool, error) {
|
||||
ctxu.GetLogger(ms.repository.ctx).Debug("(*manifestStore).ExistsByTag")
|
||||
return ms.tagStore.exists(tag)
|
||||
}
|
||||
|
||||
func (ms *manifestStore) GetByTag(tag string) (*manifest.SignedManifest, error) {
|
||||
ctxu.GetLogger(ms.repository.ctx).Debug("(*manifestStore).GetByTag")
|
||||
dgst, err := ms.tagStore.resolve(tag)
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, revision := range revisions {
|
||||
if err := ms.revisionStore.delete(revision); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return ms.tagStore.delete(tag)
|
||||
return ms.revisionStore.get(dgst)
|
||||
}
|
||||
|
||||
// verifyManifest ensures that the manifest content is valid from the
|
||||
// perspective of the registry. It ensures that the name and tag match and
|
||||
// that the signature is valid for the enclosed payload. As a policy, the
|
||||
// registry only tries to store valid content, leaving trust policies of that
|
||||
// content up to consumers.
|
||||
func (ms *manifestStore) verifyManifest(tag string, mnfst *manifest.SignedManifest) error {
|
||||
// perspective of the registry. It ensures that the signature is valid for the
|
||||
// enclosed payload. As a policy, the registry only tries to store valid
|
||||
// content, leaving trust policies of that content up to consumers.
|
||||
func (ms *manifestStore) verifyManifest(mnfst *manifest.SignedManifest) error {
|
||||
var errs distribution.ErrManifestVerification
|
||||
if mnfst.Name != ms.repository.Name() {
|
||||
// TODO(stevvooe): This needs to be an exported error
|
||||
errs = append(errs, fmt.Errorf("repository name does not match manifest name"))
|
||||
}
|
||||
|
||||
if mnfst.Tag != tag {
|
||||
// TODO(stevvooe): This needs to be an exported error.
|
||||
errs = append(errs, fmt.Errorf("tag does not match manifest tag"))
|
||||
}
|
||||
|
||||
if _, err := manifest.Verify(mnfst); err != nil {
|
||||
switch err {
|
||||
case libtrust.ErrMissingSignatureKey, libtrust.ErrInvalidJSONContent, libtrust.ErrMissingSignatureKey:
|
||||
|
|
|
|||
|
|
@ -9,25 +9,47 @@ import (
|
|||
"github.com/docker/distribution"
|
||||
"github.com/docker/distribution/digest"
|
||||
"github.com/docker/distribution/manifest"
|
||||
"github.com/docker/distribution/registry/storage/driver"
|
||||
"github.com/docker/distribution/registry/storage/driver/inmemory"
|
||||
"github.com/docker/distribution/testutil"
|
||||
"github.com/docker/libtrust"
|
||||
"golang.org/x/net/context"
|
||||
)
|
||||
|
||||
func TestManifestStorage(t *testing.T) {
|
||||
type manifestStoreTestEnv struct {
|
||||
ctx context.Context
|
||||
driver driver.StorageDriver
|
||||
registry distribution.Registry
|
||||
repository distribution.Repository
|
||||
name string
|
||||
tag string
|
||||
}
|
||||
|
||||
func newManifestStoreTestEnv(t *testing.T, name, tag string) *manifestStoreTestEnv {
|
||||
ctx := context.Background()
|
||||
name := "foo/bar"
|
||||
tag := "thetag"
|
||||
driver := inmemory.New()
|
||||
registry := NewRegistryWithDriver(driver)
|
||||
|
||||
repo, err := registry.Repository(ctx, name)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error getting repo: %v", err)
|
||||
}
|
||||
ms := repo.Manifests()
|
||||
|
||||
exists, err := ms.Exists(tag)
|
||||
return &manifestStoreTestEnv{
|
||||
ctx: ctx,
|
||||
driver: driver,
|
||||
registry: registry,
|
||||
repository: repo,
|
||||
name: name,
|
||||
tag: tag,
|
||||
}
|
||||
}
|
||||
|
||||
func TestManifestStorage(t *testing.T) {
|
||||
env := newManifestStoreTestEnv(t, "foo/bar", "thetag")
|
||||
ms := env.repository.Manifests()
|
||||
|
||||
exists, err := ms.ExistsByTag(env.tag)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error checking manifest existence: %v", err)
|
||||
}
|
||||
|
|
@ -36,7 +58,7 @@ func TestManifestStorage(t *testing.T) {
|
|||
t.Fatalf("manifest should not exist")
|
||||
}
|
||||
|
||||
if _, err := ms.Get(tag); true {
|
||||
if _, err := ms.GetByTag(env.tag); true {
|
||||
switch err.(type) {
|
||||
case distribution.ErrManifestUnknown:
|
||||
break
|
||||
|
|
@ -49,8 +71,8 @@ func TestManifestStorage(t *testing.T) {
|
|||
Versioned: manifest.Versioned{
|
||||
SchemaVersion: 1,
|
||||
},
|
||||
Name: name,
|
||||
Tag: tag,
|
||||
Name: env.name,
|
||||
Tag: env.tag,
|
||||
}
|
||||
|
||||
// Build up some test layers and add them to the manifest, saving the
|
||||
|
|
@ -79,7 +101,7 @@ func TestManifestStorage(t *testing.T) {
|
|||
t.Fatalf("error signing manifest: %v", err)
|
||||
}
|
||||
|
||||
err = ms.Put(tag, sm)
|
||||
err = ms.Put(sm)
|
||||
if err == nil {
|
||||
t.Fatalf("expected errors putting manifest")
|
||||
}
|
||||
|
|
@ -88,7 +110,7 @@ func TestManifestStorage(t *testing.T) {
|
|||
|
||||
// Now, upload the layers that were missing!
|
||||
for dgst, rs := range testLayers {
|
||||
upload, err := repo.Layers().Upload()
|
||||
upload, err := env.repository.Layers().Upload()
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error creating test upload: %v", err)
|
||||
}
|
||||
|
|
@ -102,11 +124,11 @@ func TestManifestStorage(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
if err = ms.Put(tag, sm); err != nil {
|
||||
if err = ms.Put(sm); err != nil {
|
||||
t.Fatalf("unexpected error putting manifest: %v", err)
|
||||
}
|
||||
|
||||
exists, err = ms.Exists(tag)
|
||||
exists, err = ms.ExistsByTag(env.tag)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error checking manifest existence: %v", err)
|
||||
}
|
||||
|
|
@ -115,7 +137,7 @@ func TestManifestStorage(t *testing.T) {
|
|||
t.Fatalf("manifest should exist")
|
||||
}
|
||||
|
||||
fetchedManifest, err := ms.Get(tag)
|
||||
fetchedManifest, err := ms.GetByTag(env.tag)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error fetching manifest: %v", err)
|
||||
}
|
||||
|
|
@ -134,6 +156,31 @@ func TestManifestStorage(t *testing.T) {
|
|||
t.Fatalf("unexpected error extracting payload: %v", err)
|
||||
}
|
||||
|
||||
// Now that we have a payload, take a moment to check that the manifest is
|
||||
// return by the payload digest.
|
||||
dgst, err := digest.FromBytes(payload)
|
||||
if err != nil {
|
||||
t.Fatalf("error getting manifest digest: %v", err)
|
||||
}
|
||||
|
||||
exists, err = ms.Exists(dgst)
|
||||
if err != nil {
|
||||
t.Fatalf("error checking manifest existence by digest: %v", err)
|
||||
}
|
||||
|
||||
if !exists {
|
||||
t.Fatalf("manifest %s should exist", dgst)
|
||||
}
|
||||
|
||||
fetchedByDigest, err := ms.Get(dgst)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error fetching manifest by digest: %v", err)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(fetchedByDigest, fetchedManifest) {
|
||||
t.Fatalf("fetched manifest not equal: %#v != %#v", fetchedByDigest, fetchedManifest)
|
||||
}
|
||||
|
||||
sigs, err := fetchedJWS.Signatures()
|
||||
if err != nil {
|
||||
t.Fatalf("unable to extract signatures: %v", err)
|
||||
|
|
@ -153,8 +200,8 @@ func TestManifestStorage(t *testing.T) {
|
|||
t.Fatalf("unexpected tags returned: %v", tags)
|
||||
}
|
||||
|
||||
if tags[0] != tag {
|
||||
t.Fatalf("unexpected tag found in tags: %v != %v", tags, []string{tag})
|
||||
if tags[0] != env.tag {
|
||||
t.Fatalf("unexpected tag found in tags: %v != %v", tags, []string{env.tag})
|
||||
}
|
||||
|
||||
// Now, push the same manifest with a different key
|
||||
|
|
@ -182,11 +229,11 @@ func TestManifestStorage(t *testing.T) {
|
|||
t.Fatalf("unexpected number of signatures: %d != %d", len(sigs2), 1)
|
||||
}
|
||||
|
||||
if err = ms.Put(tag, sm2); err != nil {
|
||||
if err = ms.Put(sm2); err != nil {
|
||||
t.Fatalf("unexpected error putting manifest: %v", err)
|
||||
}
|
||||
|
||||
fetched, err := ms.Get(tag)
|
||||
fetched, err := ms.GetByTag(env.tag)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error fetching manifest: %v", err)
|
||||
}
|
||||
|
|
@ -231,7 +278,11 @@ func TestManifestStorage(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
if err := ms.Delete(tag); err != nil {
|
||||
t.Fatalf("unexpected error deleting manifest: %v", err)
|
||||
// TODO(stevvooe): Currently, deletes are not supported due to some
|
||||
// complexity around managing tag indexes. We'll add this support back in
|
||||
// when the manifest format has settled. For now, we expect an error for
|
||||
// all deletes.
|
||||
if err := ms.Delete(dgst); err == nil {
|
||||
t.Fatalf("unexpected an error deleting manifest by digest: %v", err)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -72,11 +72,12 @@ const storagePathVersion = "v2"
|
|||
//
|
||||
// Tags:
|
||||
//
|
||||
// manifestTagsPathSpec: <root>/v2/repositories/<name>/_manifests/tags/
|
||||
// manifestTagPathSpec: <root>/v2/repositories/<name>/_manifests/tags/<tag>/
|
||||
// manifestTagCurrentPathSpec: <root>/v2/repositories/<name>/_manifests/tags/<tag>/current/link
|
||||
// manifestTagIndexPathSpec: <root>/v2/repositories/<name>/_manifests/tags/<tag>/index/
|
||||
// manifestTagIndexEntryPathSpec: <root>/v2/repositories/<name>/_manifests/tags/<tag>/index/<algorithm>/<hex digest>/link
|
||||
// manifestTagsPathSpec: <root>/v2/repositories/<name>/_manifests/tags/
|
||||
// manifestTagPathSpec: <root>/v2/repositories/<name>/_manifests/tags/<tag>/
|
||||
// manifestTagCurrentPathSpec: <root>/v2/repositories/<name>/_manifests/tags/<tag>/current/link
|
||||
// manifestTagIndexPathSpec: <root>/v2/repositories/<name>/_manifests/tags/<tag>/index/
|
||||
// manifestTagIndexEntryPathSpec: <root>/v2/repositories/<name>/_manifests/tags/<tag>/index/<algorithm>/<hex digest>/
|
||||
// manifestTagIndexEntryLinkPathSpec: <root>/v2/repositories/<name>/_manifests/tags/<tag>/index/<algorithm>/<hex digest>/link
|
||||
//
|
||||
// Layers:
|
||||
//
|
||||
|
|
@ -199,6 +200,17 @@ func (pm *pathMapper) path(spec pathSpec) (string, error) {
|
|||
}
|
||||
|
||||
return path.Join(root, "index"), nil
|
||||
case manifestTagIndexEntryLinkPathSpec:
|
||||
root, err := pm.path(manifestTagIndexEntryPathSpec{
|
||||
name: v.name,
|
||||
tag: v.tag,
|
||||
revision: v.revision,
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return path.Join(root, "link"), nil
|
||||
case manifestTagIndexEntryPathSpec:
|
||||
root, err := pm.path(manifestTagIndexPathSpec{
|
||||
name: v.name,
|
||||
|
|
@ -213,7 +225,7 @@ func (pm *pathMapper) path(spec pathSpec) (string, error) {
|
|||
return "", err
|
||||
}
|
||||
|
||||
return path.Join(root, path.Join(append(components, "link")...)), nil
|
||||
return path.Join(root, path.Join(components...)), nil
|
||||
case layerLinkPathSpec:
|
||||
components, err := digestPathComponents(v.digest, false)
|
||||
if err != nil {
|
||||
|
|
@ -332,8 +344,7 @@ type manifestTagIndexPathSpec struct {
|
|||
|
||||
func (manifestTagIndexPathSpec) pathSpec() {}
|
||||
|
||||
// manifestTagIndexEntryPathSpec describes the link to a revisions of a
|
||||
// manifest with given tag within the index.
|
||||
// manifestTagIndexEntryPathSpec contains the entries of the index by revision.
|
||||
type manifestTagIndexEntryPathSpec struct {
|
||||
name string
|
||||
tag string
|
||||
|
|
@ -342,6 +353,16 @@ type manifestTagIndexEntryPathSpec struct {
|
|||
|
||||
func (manifestTagIndexEntryPathSpec) pathSpec() {}
|
||||
|
||||
// manifestTagIndexEntryLinkPathSpec describes the link to a revisions of a
|
||||
// manifest with given tag within the index.
|
||||
type manifestTagIndexEntryLinkPathSpec struct {
|
||||
name string
|
||||
tag string
|
||||
revision digest.Digest
|
||||
}
|
||||
|
||||
func (manifestTagIndexEntryLinkPathSpec) pathSpec() {}
|
||||
|
||||
// layerLink specifies a path for a layer link, which is a file with a blob
|
||||
// id. The layer link will contain a content addressable blob id reference
|
||||
// into the blob store. The format of the contents is as follows:
|
||||
|
|
|
|||
|
|
@ -78,6 +78,14 @@ func TestPathMapper(t *testing.T) {
|
|||
tag: "thetag",
|
||||
revision: "sha256:abcdef0123456789",
|
||||
},
|
||||
expected: "/pathmapper-test/repositories/foo/bar/_manifests/tags/thetag/index/sha256/abcdef0123456789",
|
||||
},
|
||||
{
|
||||
spec: manifestTagIndexEntryLinkPathSpec{
|
||||
name: "foo/bar",
|
||||
tag: "thetag",
|
||||
revision: "sha256:abcdef0123456789",
|
||||
},
|
||||
expected: "/pathmapper-test/repositories/foo/bar/_manifests/tags/thetag/index/sha256/abcdef0123456789/link",
|
||||
},
|
||||
{
|
||||
|
|
|
|||
|
|
@ -63,7 +63,7 @@ func (ts *tagStore) exists(tag string) (bool, error) {
|
|||
// tag tags the digest with the given tag, updating the the store to point at
|
||||
// the current tag. The digest must point to a manifest.
|
||||
func (ts *tagStore) tag(tag string, revision digest.Digest) error {
|
||||
indexEntryPath, err := ts.pm.path(manifestTagIndexEntryPathSpec{
|
||||
indexEntryPath, err := ts.pm.path(manifestTagIndexEntryLinkPathSpec{
|
||||
name: ts.Name(),
|
||||
tag: tag,
|
||||
revision: revision,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue