diff --git a/docs/spec/api.md b/docs/spec/api.md index b7baa3397..cdf225aef 100644 --- a/docs/spec/api.md +++ b/docs/spec/api.md @@ -402,6 +402,28 @@ for details): The client should verify the returned manifest signature for authenticity before fetching layers. +##### Existing Manifests + +The image manifest can be checked for existence with the following url: + +``` +HEAD /v2//manifests/ +``` + +The `name` and `reference` parameter identify the image and are required. The +reference may include a tag or digest. + +A `404 Not Found` response will be returned if the image is unknown to the +registry. If the image exists and the response is successful the response will +be as follows: + +``` +200 OK +Content-Length: +Docker-Content-Digest: +``` + + #### Pulling a Layer Layers are stored in the blob portion of the registry, keyed by digest. diff --git a/docs/spec/api.md.tmpl b/docs/spec/api.md.tmpl index 367a96c91..da778acf3 100644 --- a/docs/spec/api.md.tmpl +++ b/docs/spec/api.md.tmpl @@ -402,6 +402,28 @@ for details): The client should verify the returned manifest signature for authenticity before fetching layers. +##### Existing Manifests + +The image manifest can be checked for existence with the following url: + +``` +HEAD /v2//manifests/ +``` + +The `name` and `reference` parameter identify the image and are required. The +reference may include a tag or digest. + +A `404 Not Found` response will be returned if the image is unknown to the +registry. If the image exists and the response is successful the response will +be as follows: + +``` +200 OK +Content-Length: +Docker-Content-Digest: +``` + + #### Pulling a Layer Layers are stored in the blob portion of the registry, keyed by digest. diff --git a/errors.go b/errors.go index 7bf720e03..77bd096ec 100644 --- a/errors.go +++ b/errors.go @@ -16,6 +16,15 @@ var ErrManifestNotModified = errors.New("manifest not modified") // performed var ErrUnsupported = errors.New("operation unsupported") +// ErrTagUnknown is returned if the given tag is not known by the tag service +type ErrTagUnknown struct { + Tag string +} + +func (err ErrTagUnknown) Error() string { + return fmt.Sprintf("unknown tag=%s", err.Tag) +} + // ErrRepositoryUnknown is returned if the named repository is not known by // the registry. type ErrRepositoryUnknown struct { diff --git a/manifest/schema1/builder.go b/manifest/schema1/builder.go new file mode 100644 index 000000000..461f15aa5 --- /dev/null +++ b/manifest/schema1/builder.go @@ -0,0 +1,91 @@ +package schema1 + +import ( + "fmt" + + "errors" + "github.com/docker/distribution" + "github.com/docker/distribution/digest" + "github.com/docker/distribution/manifest" + "github.com/docker/libtrust" +) + +// ManifestBuilder is a type for constructing manifests +type manifestBuilder struct { + Manifest + pk libtrust.PrivateKey +} + +// NewManifestBuilder is used to build new manifests for the current schema +// version. +func NewManifestBuilder(pk libtrust.PrivateKey, name, tag, architecture string) distribution.ManifestBuilder { + return &manifestBuilder{ + Manifest: Manifest{ + Versioned: manifest.Versioned{ + SchemaVersion: 1, + }, + Name: name, + Tag: tag, + Architecture: architecture, + }, + pk: pk, + } +} + +// Build produces a final manifest from the given references +func (mb *manifestBuilder) Build() (distribution.Manifest, error) { + m := mb.Manifest + if len(m.FSLayers) == 0 { + return nil, errors.New("cannot build manifest with zero layers or history") + } + + m.FSLayers = make([]FSLayer, len(mb.Manifest.FSLayers)) + m.History = make([]History, len(mb.Manifest.History)) + copy(m.FSLayers, mb.Manifest.FSLayers) + copy(m.History, mb.Manifest.History) + + return Sign(&m, mb.pk) +} + +// AppendReference adds a reference to the current manifestBuilder +func (mb *manifestBuilder) AppendReference(d distribution.Describable) error { + r, ok := d.(Reference) + if !ok { + return fmt.Errorf("Unable to add non-reference type to v1 builder") + } + + // Entries need to be prepended + mb.Manifest.FSLayers = append([]FSLayer{{BlobSum: r.Digest}}, mb.Manifest.FSLayers...) + mb.Manifest.History = append([]History{r.History}, mb.Manifest.History...) + return nil + +} + +// References returns the current references added to this builder +func (mb *manifestBuilder) References() []distribution.Descriptor { + refs := make([]distribution.Descriptor, len(mb.Manifest.FSLayers)) + for i := range mb.Manifest.FSLayers { + layerDigest := mb.Manifest.FSLayers[i].BlobSum + history := mb.Manifest.History[i] + ref := Reference{layerDigest, 0, history} + refs[i] = ref.Descriptor() + } + return refs +} + +// Reference describes a manifest v2, schema version 1 dependency. +// An FSLayer associated with a history entry. +type Reference struct { + Digest digest.Digest + Size int64 // if we know it, set it for the descriptor. + History History +} + +// Descriptor describes a reference +func (r Reference) Descriptor() distribution.Descriptor { + return distribution.Descriptor{ + MediaType: MediaTypeManifestLayer, + Digest: r.Digest, + Size: r.Size, + } +} diff --git a/manifest/schema1/builder_test.go b/manifest/schema1/builder_test.go new file mode 100644 index 000000000..a5c1cea21 --- /dev/null +++ b/manifest/schema1/builder_test.go @@ -0,0 +1,97 @@ +package schema1 + +import ( + "testing" + + "github.com/docker/distribution/digest" + "github.com/docker/distribution/manifest" + "github.com/docker/libtrust" +) + +func makeSignedManifest(t *testing.T, pk libtrust.PrivateKey, refs []Reference) *SignedManifest { + u := &Manifest{ + Versioned: manifest.Versioned{ + SchemaVersion: 1, + }, + Name: "foo/bar", + Tag: "latest", + Architecture: "amd64", + } + + for i := len(refs) - 1; i >= 0; i-- { + u.FSLayers = append(u.FSLayers, FSLayer{ + BlobSum: refs[i].Digest, + }) + u.History = append(u.History, History{ + V1Compatibility: refs[i].History.V1Compatibility, + }) + } + + signedManifest, err := Sign(u, pk) + if err != nil { + t.Fatalf("unexpected error signing manifest: %v", err) + } + return signedManifest +} + +func TestBuilder(t *testing.T) { + pk, err := libtrust.GenerateECP256PrivateKey() + if err != nil { + t.Fatalf("unexpected error generating private key: %v", err) + } + + r1 := Reference{ + Digest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Size: 1, + History: History{V1Compatibility: "{\"a\" : 1 }"}, + } + r2 := Reference{ + Digest: "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + Size: 2, + History: History{V1Compatibility: "{\"\a\" : 2 }"}, + } + + handCrafted := makeSignedManifest(t, pk, []Reference{r1, r2}) + + b := NewManifestBuilder(pk, handCrafted.Manifest.Name, handCrafted.Manifest.Tag, handCrafted.Manifest.Architecture) + _, err = b.Build() + if err == nil { + t.Fatal("Expected error building zero length manifest") + } + + err = b.AppendReference(r1) + if err != nil { + t.Fatal(err) + } + + err = b.AppendReference(r2) + if err != nil { + t.Fatal(err) + } + + refs := b.References() + if len(refs) != 2 { + t.Fatalf("Unexpected reference count : %d != %d", 2, len(refs)) + } + + // Ensure ordering + if refs[0].Digest != r2.Digest { + t.Fatalf("Unexpected reference : %v", refs[0]) + } + + m, err := b.Build() + if err != nil { + t.Fatal(err) + } + + built, ok := m.(*SignedManifest) + if !ok { + t.Fatalf("unexpected type from Build() : %T", built) + } + + d1 := digest.FromBytes(built.Canonical) + d2 := digest.FromBytes(handCrafted.Canonical) + if d1 != d2 { + t.Errorf("mismatching canonical JSON") + } +} diff --git a/manifest/schema1/manifest.go b/manifest/schema1/manifest.go index 7783a55dc..98a7d8170 100644 --- a/manifest/schema1/manifest.go +++ b/manifest/schema1/manifest.go @@ -2,20 +2,22 @@ package schema1 import ( "encoding/json" + "fmt" + "github.com/docker/distribution" "github.com/docker/distribution/digest" "github.com/docker/distribution/manifest" "github.com/docker/libtrust" ) -// TODO(stevvooe): When we rev the manifest format, the contents of this -// package should be moved to manifest/v1. - const ( - // ManifestMediaType specifies the mediaType for the current version. Note - // that for schema version 1, the the media is optionally - // "application/json". - ManifestMediaType = "application/vnd.docker.distribution.manifest.v1+json" + // MediaTypeManifest specifies the mediaType for the current version. Note + // that for schema version 1, the the media is optionally "application/json". + MediaTypeManifest = "application/vnd.docker.distribution.manifest.v1+json" + // MediaTypeSignedManifest specifies the mediatype for current SignedManifest version + MediaTypeSignedManifest = "application/vnd.docker.distribution.manifest.v1+prettyjws" + // MediaTypeManifestLayer specifies the media type for manifest layers + MediaTypeManifestLayer = "application/vnd.docker.container.image.rootfs.diff+x-gtar" ) var ( @@ -26,6 +28,47 @@ var ( } ) +func init() { + schema1Func := func(b []byte) (distribution.Manifest, distribution.Descriptor, error) { + sm := new(SignedManifest) + err := sm.UnmarshalJSON(b) + if err != nil { + return nil, distribution.Descriptor{}, err + } + + desc := distribution.Descriptor{ + Digest: digest.FromBytes(sm.Canonical), + Size: int64(len(sm.Canonical)), + MediaType: MediaTypeManifest, + } + return sm, desc, err + } + err := distribution.RegisterManifestSchema(MediaTypeManifest, schema1Func) + if err != nil { + panic(fmt.Sprintf("Unable to register manifest: %s", err)) + } + err = distribution.RegisterManifestSchema("", schema1Func) + if err != nil { + panic(fmt.Sprintf("Unable to register manifest: %s", err)) + } + err = distribution.RegisterManifestSchema("application/json; charset=utf-8", schema1Func) + if err != nil { + panic(fmt.Sprintf("Unable to register manifest: %s", err)) + } +} + +// FSLayer is a container struct for BlobSums defined in an image manifest +type FSLayer struct { + // BlobSum is the tarsum of the referenced filesystem image layer + BlobSum digest.Digest `json:"blobSum"` +} + +// History stores unstructured v1 compatibility information +type History struct { + // V1Compatibility is the raw v1 compatibility information + V1Compatibility string `json:"v1Compatibility"` +} + // Manifest provides the base accessible fields for working with V2 image // format in the registry. type Manifest struct { @@ -49,59 +92,64 @@ type Manifest struct { } // SignedManifest provides an envelope for a signed image manifest, including -// the format sensitive raw bytes. It contains fields to +// the format sensitive raw bytes. type SignedManifest struct { Manifest - // Raw is the byte representation of the ImageManifest, used for signature - // verification. The value of Raw must be used directly during - // serialization, or the signature check will fail. The manifest byte + // Canonical is the canonical byte representation of the ImageManifest, + // without any attached signatures. The manifest byte // representation cannot change or it will have to be re-signed. - Raw []byte `json:"-"` + Canonical []byte `json:"-"` + + // all contains the byte representation of the Manifest including signatures + // and is retuend by Payload() + all []byte } -// UnmarshalJSON populates a new ImageManifest struct from JSON data. +// UnmarshalJSON populates a new SignedManifest struct from JSON data. func (sm *SignedManifest) UnmarshalJSON(b []byte) error { - sm.Raw = make([]byte, len(b), len(b)) - copy(sm.Raw, b) + sm.all = make([]byte, len(b), len(b)) + // store manifest and signatures in all + copy(sm.all, b) - p, err := sm.Payload() + jsig, err := libtrust.ParsePrettySignature(b, "signatures") if err != nil { return err } + // Resolve the payload in the manifest. + bytes, err := jsig.Payload() + if err != nil { + return err + } + + // sm.Canonical stores the canonical manifest JSON + sm.Canonical = make([]byte, len(bytes), len(bytes)) + copy(sm.Canonical, bytes) + + // Unmarshal canonical JSON into Manifest object var manifest Manifest - if err := json.Unmarshal(p, &manifest); err != nil { + if err := json.Unmarshal(sm.Canonical, &manifest); err != nil { return err } sm.Manifest = manifest + return nil } -// Payload returns the raw, signed content of the signed manifest. The -// contents can be used to calculate the content identifier. -func (sm *SignedManifest) Payload() ([]byte, error) { - jsig, err := libtrust.ParsePrettySignature(sm.Raw, "signatures") - if err != nil { - return nil, err +// References returnes the descriptors of this manifests references +func (sm SignedManifest) References() []distribution.Descriptor { + dependencies := make([]distribution.Descriptor, len(sm.FSLayers)) + for i, fsLayer := range sm.FSLayers { + dependencies[i] = distribution.Descriptor{ + MediaType: "application/vnd.docker.container.image.rootfs.diff+x-gtar", + Digest: fsLayer.BlobSum, + } } - // Resolve the payload in the manifest. - return jsig.Payload() -} + return dependencies -// Signatures returns the signatures as provided by -// (*libtrust.JSONSignature).Signatures. The byte slices are opaque jws -// signatures. -func (sm *SignedManifest) Signatures() ([][]byte, error) { - jsig, err := libtrust.ParsePrettySignature(sm.Raw, "signatures") - if err != nil { - return nil, err - } - - // Resolve the payload in the manifest. - return jsig.Signatures() } // MarshalJSON returns the contents of raw. If Raw is nil, marshals the inner @@ -109,22 +157,28 @@ func (sm *SignedManifest) Signatures() ([][]byte, error) { // use Raw directly, since the the content produced by json.Marshal will be // compacted and will fail signature checks. func (sm *SignedManifest) MarshalJSON() ([]byte, error) { - if len(sm.Raw) > 0 { - return sm.Raw, nil + if len(sm.all) > 0 { + return sm.all, nil } // If the raw data is not available, just dump the inner content. return json.Marshal(&sm.Manifest) } -// FSLayer is a container struct for BlobSums defined in an image manifest -type FSLayer struct { - // BlobSum is the digest of the referenced filesystem image layer - BlobSum digest.Digest `json:"blobSum"` +// Payload returns the signed content of the signed manifest. +func (sm SignedManifest) Payload() (string, []byte, error) { + return MediaTypeManifest, sm.all, nil } -// History stores unstructured v1 compatibility information -type History struct { - // V1Compatibility is the raw v1 compatibility information - V1Compatibility string `json:"v1Compatibility"` +// Signatures returns the signatures as provided by +// (*libtrust.JSONSignature).Signatures. The byte slices are opaque jws +// signatures. +func (sm *SignedManifest) Signatures() ([][]byte, error) { + jsig, err := libtrust.ParsePrettySignature(sm.all, "signatures") + if err != nil { + return nil, err + } + + // Resolve the payload in the manifest. + return jsig.Signatures() } diff --git a/manifest/schema1/manifest_test.go b/manifest/schema1/manifest_test.go index 7d0d382db..05bb8ec57 100644 --- a/manifest/schema1/manifest_test.go +++ b/manifest/schema1/manifest_test.go @@ -19,15 +19,15 @@ type testEnv struct { func TestManifestMarshaling(t *testing.T) { env := genEnv(t) - // Check that the Raw field is the same as json.MarshalIndent with these + // Check that the all field is the same as json.MarshalIndent with these // parameters. p, err := json.MarshalIndent(env.signed, "", " ") if err != nil { t.Fatalf("error marshaling manifest: %v", err) } - if !bytes.Equal(p, env.signed.Raw) { - t.Fatalf("manifest bytes not equal: %q != %q", string(env.signed.Raw), string(p)) + if !bytes.Equal(p, env.signed.all) { + t.Fatalf("manifest bytes not equal: %q != %q", string(env.signed.all), string(p)) } } @@ -35,7 +35,7 @@ func TestManifestUnmarshaling(t *testing.T) { env := genEnv(t) var signed SignedManifest - if err := json.Unmarshal(env.signed.Raw, &signed); err != nil { + if err := json.Unmarshal(env.signed.all, &signed); err != nil { t.Fatalf("error unmarshaling signed manifest: %v", err) } diff --git a/manifest/schema1/sign.go b/manifest/schema1/sign.go index 1b7b674a0..c862dd812 100644 --- a/manifest/schema1/sign.go +++ b/manifest/schema1/sign.go @@ -31,8 +31,9 @@ func Sign(m *Manifest, pk libtrust.PrivateKey) (*SignedManifest, error) { } return &SignedManifest{ - Manifest: *m, - Raw: pretty, + Manifest: *m, + all: pretty, + Canonical: p, }, nil } @@ -60,7 +61,8 @@ func SignWithChain(m *Manifest, key libtrust.PrivateKey, chain []*x509.Certifica } return &SignedManifest{ - Manifest: *m, - Raw: pretty, + Manifest: *m, + all: pretty, + Canonical: p, }, nil } diff --git a/manifest/schema1/verify.go b/manifest/schema1/verify.go index 60f8cda07..fa8daa56f 100644 --- a/manifest/schema1/verify.go +++ b/manifest/schema1/verify.go @@ -10,7 +10,7 @@ import ( // Verify verifies the signature of the signed manifest returning the public // keys used during signing. func Verify(sm *SignedManifest) ([]libtrust.PublicKey, error) { - js, err := libtrust.ParsePrettySignature(sm.Raw, "signatures") + js, err := libtrust.ParsePrettySignature(sm.all, "signatures") if err != nil { logrus.WithField("err", err).Debugf("(*SignedManifest).Verify") return nil, err @@ -23,7 +23,7 @@ func Verify(sm *SignedManifest) ([]libtrust.PublicKey, error) { // certificate pool returning the list of verified chains. Signatures without // an x509 chain are not checked. func VerifyChains(sm *SignedManifest, ca *x509.CertPool) ([][]*x509.Certificate, error) { - js, err := libtrust.ParsePrettySignature(sm.Raw, "signatures") + js, err := libtrust.ParsePrettySignature(sm.all, "signatures") if err != nil { return nil, err } diff --git a/manifests.go b/manifests.go new file mode 100644 index 000000000..7cb91ab82 --- /dev/null +++ b/manifests.go @@ -0,0 +1,100 @@ +package distribution + +import ( + "fmt" + + "github.com/docker/distribution/context" + "github.com/docker/distribution/digest" +) + +// Manifest represents a registry object specifying a set of +// references and an optional target +type Manifest interface { + // References returns a list of objects which make up this manifest. + // The references are strictly ordered from base to head. A reference + // is anything which can be represented by a distribution.Descriptor + References() []Descriptor + + // Payload provides the serialized format of the manifest, in addition to + // the mediatype. + Payload() (mediatype string, payload []byte, err error) +} + +// ManifestBuilder creates a manifest allowing one to include dependencies. +// Instances can be obtained from a version-specific manifest package. Manifest +// specific data is passed into the function which creates the builder. +type ManifestBuilder interface { + // Build creates the manifest from his builder. + Build() (Manifest, error) + + // References returns a list of objects which have been added to this + // builder. The dependencies are returned in the order they were added, + // which should be from base to head. + References() []Descriptor + + // AppendReference includes the given object in the manifest after any + // existing dependencies. If the add fails, such as when adding an + // unsupported dependency, an error may be returned. + AppendReference(dependency Describable) error +} + +// ManifestService describes operations on image manifests. +type ManifestService interface { + // Exists returns true if the manifest exists. + Exists(ctx context.Context, dgst digest.Digest) (bool, error) + + // Get retrieves the manifest specified by the given digest + Get(ctx context.Context, dgst digest.Digest, options ...ManifestServiceOption) (Manifest, error) + + // Put creates or updates the given manifest returning the manifest digest + Put(ctx context.Context, manifest Manifest, options ...ManifestServiceOption) (digest.Digest, error) + + // Delete removes the manifest specified by the given digest. Deleting + // a manifest that doesn't exist will return ErrManifestNotFound + Delete(ctx context.Context, dgst digest.Digest) error + + // Enumerate fills 'manifests' with the manifests in this service up + // to the size of 'manifests' and returns 'n' for the number of entries + // which were filled. 'last' contains an offset in the manifest set + // and can be used to resume iteration. + //Enumerate(ctx context.Context, manifests []Manifest, last Manifest) (n int, err error) +} + +// Describable is an interface for descriptors +type Describable interface { + Descriptor() Descriptor +} + +// ManifestMediaTypes returns the supported media types for manifests. +func ManifestMediaTypes() (mediaTypes []string) { + for t := range mappings { + mediaTypes = append(mediaTypes, t) + } + return +} + +// UnmarshalFunc implements manifest unmarshalling a given MediaType +type UnmarshalFunc func([]byte) (Manifest, Descriptor, error) + +var mappings = make(map[string]UnmarshalFunc, 0) + +// UnmarshalManifest looks up manifest unmarshall functions based on +// MediaType +func UnmarshalManifest(mediatype string, p []byte) (Manifest, Descriptor, error) { + unmarshalFunc, ok := mappings[mediatype] + if !ok { + return nil, Descriptor{}, fmt.Errorf("unsupported manifest mediatype: %s", mediatype) + } + + return unmarshalFunc(p) +} + +// RegisterManifestSchema registers an UnmarshalFunc for a given schema type. This +// should be called from specific +func RegisterManifestSchema(mediatype string, u UnmarshalFunc) error { + if _, ok := mappings[mediatype]; ok { + return fmt.Errorf("manifest mediatype registration would overwrite existing: %s", mediatype) + } + mappings[mediatype] = u + return nil +} diff --git a/notifications/bridge.go b/notifications/bridge.go index c33a9ff18..93a2362a6 100644 --- a/notifications/bridge.go +++ b/notifications/bridge.go @@ -7,7 +7,6 @@ import ( "github.com/docker/distribution" "github.com/docker/distribution/context" "github.com/docker/distribution/digest" - "github.com/docker/distribution/manifest/schema1" "github.com/docker/distribution/uuid" ) @@ -53,15 +52,15 @@ func NewRequestRecord(id string, r *http.Request) RequestRecord { } } -func (b *bridge) ManifestPushed(repo string, sm *schema1.SignedManifest) error { +func (b *bridge) ManifestPushed(repo string, sm distribution.Manifest) error { return b.createManifestEventAndWrite(EventActionPush, repo, sm) } -func (b *bridge) ManifestPulled(repo string, sm *schema1.SignedManifest) error { +func (b *bridge) ManifestPulled(repo string, sm distribution.Manifest) error { return b.createManifestEventAndWrite(EventActionPull, repo, sm) } -func (b *bridge) ManifestDeleted(repo string, sm *schema1.SignedManifest) error { +func (b *bridge) ManifestDeleted(repo string, sm distribution.Manifest) error { return b.createManifestEventAndWrite(EventActionDelete, repo, sm) } @@ -77,7 +76,7 @@ func (b *bridge) BlobDeleted(repo string, desc distribution.Descriptor) error { return b.createBlobEventAndWrite(EventActionDelete, repo, desc) } -func (b *bridge) createManifestEventAndWrite(action string, repo string, sm *schema1.SignedManifest) error { +func (b *bridge) createManifestEventAndWrite(action string, repo string, sm distribution.Manifest) error { manifestEvent, err := b.createManifestEvent(action, repo, sm) if err != nil { return err @@ -86,21 +85,21 @@ func (b *bridge) createManifestEventAndWrite(action string, repo string, sm *sch return b.sink.Write(*manifestEvent) } -func (b *bridge) createManifestEvent(action string, repo string, sm *schema1.SignedManifest) (*Event, error) { +func (b *bridge) createManifestEvent(action string, repo string, sm distribution.Manifest) (*Event, error) { event := b.createEvent(action) - event.Target.MediaType = schema1.ManifestMediaType event.Target.Repository = repo - p, err := sm.Payload() + mt, p, err := sm.Payload() if err != nil { return nil, err } + event.Target.MediaType = mt event.Target.Length = int64(len(p)) event.Target.Size = int64(len(p)) event.Target.Digest = digest.FromBytes(p) - event.Target.URL, err = b.ub.BuildManifestURL(sm.Name, event.Target.Digest.String()) + event.Target.URL, err = b.ub.BuildManifestURL(repo, event.Target.Digest.String()) if err != nil { return nil, err } diff --git a/notifications/bridge_test.go b/notifications/bridge_test.go index 1c82d3170..ec10b1f4d 100644 --- a/notifications/bridge_test.go +++ b/notifications/bridge_test.go @@ -85,7 +85,7 @@ func createTestEnv(t *testing.T, fn testSinkFn) Listener { t.Fatalf("error signing manifest: %v", err) } - payload, err = sm.Payload() + _, payload, err = sm.Payload() if err != nil { t.Fatalf("error getting manifest payload: %v", err) } @@ -109,7 +109,7 @@ func checkCommonManifest(t *testing.T, action string, events ...Event) { } if event.Target.URL != u { - t.Fatalf("incorrect url passed: %q != %q", event.Target.URL, u) + t.Fatalf("incorrect url passed: \n%q != \n%q", event.Target.URL, u) } } diff --git a/notifications/event_test.go b/notifications/event_test.go index 8126c7048..a1b7cb971 100644 --- a/notifications/event_test.go +++ b/notifications/event_test.go @@ -120,7 +120,7 @@ func TestEventEnvelopeJSONFormat(t *testing.T) { manifestPush.Target.Digest = "sha256:0123456789abcdef0" manifestPush.Target.Length = 1 manifestPush.Target.Size = 1 - manifestPush.Target.MediaType = schema1.ManifestMediaType + manifestPush.Target.MediaType = schema1.MediaTypeManifest manifestPush.Target.Repository = "library/test" manifestPush.Target.URL = "http://example.com/v2/library/test/manifests/latest" diff --git a/notifications/http_test.go b/notifications/http_test.go index 6e10c6221..db3bc2e2c 100644 --- a/notifications/http_test.go +++ b/notifications/http_test.go @@ -75,12 +75,12 @@ func TestHTTPSink(t *testing.T) { { statusCode: http.StatusOK, events: []Event{ - createTestEvent("push", "library/test", schema1.ManifestMediaType)}, + createTestEvent("push", "library/test", schema1.MediaTypeManifest)}, }, { statusCode: http.StatusOK, events: []Event{ - createTestEvent("push", "library/test", schema1.ManifestMediaType), + createTestEvent("push", "library/test", schema1.MediaTypeManifest), createTestEvent("push", "library/test", layerMediaType), createTestEvent("push", "library/test", layerMediaType), }, diff --git a/notifications/listener.go b/notifications/listener.go index 6c558a471..baecbdbd3 100644 --- a/notifications/listener.go +++ b/notifications/listener.go @@ -7,18 +7,17 @@ import ( "github.com/docker/distribution" "github.com/docker/distribution/context" "github.com/docker/distribution/digest" - "github.com/docker/distribution/manifest/schema1" ) // ManifestListener describes a set of methods for listening to events related to manifests. type ManifestListener interface { - ManifestPushed(repo string, sm *schema1.SignedManifest) error - ManifestPulled(repo string, sm *schema1.SignedManifest) error + ManifestPushed(repo string, sm distribution.Manifest) error + ManifestPulled(repo string, sm distribution.Manifest) error // TODO(stevvooe): Please note that delete support is still a little shaky // and we'll need to propagate these in the future. - ManifestDeleted(repo string, sm *schema1.SignedManifest) error + ManifestDeleted(repo string, sm distribution.Manifest) error } // BlobListener describes a listener that can respond to layer related events. @@ -74,8 +73,8 @@ type manifestServiceListener struct { parent *repositoryListener } -func (msl *manifestServiceListener) Get(dgst digest.Digest) (*schema1.SignedManifest, error) { - sm, err := msl.ManifestService.Get(dgst) +func (msl *manifestServiceListener) Get(ctx context.Context, dgst digest.Digest, options ...distribution.ManifestServiceOption) (distribution.Manifest, error) { + sm, err := msl.ManifestService.Get(ctx, dgst) if err == nil { if err := msl.parent.listener.ManifestPulled(msl.parent.Repository.Name(), sm); err != nil { logrus.Errorf("error dispatching manifest pull to listener: %v", err) @@ -85,8 +84,8 @@ func (msl *manifestServiceListener) Get(dgst digest.Digest) (*schema1.SignedMani return sm, err } -func (msl *manifestServiceListener) Put(sm *schema1.SignedManifest) error { - err := msl.ManifestService.Put(sm) +func (msl *manifestServiceListener) Put(ctx context.Context, sm distribution.Manifest, options ...distribution.ManifestServiceOption) (digest.Digest, error) { + dgst, err := msl.ManifestService.Put(ctx, sm, options...) if err == nil { if err := msl.parent.listener.ManifestPushed(msl.parent.Repository.Name(), sm); err != nil { @@ -94,18 +93,7 @@ func (msl *manifestServiceListener) Put(sm *schema1.SignedManifest) error { } } - return err -} - -func (msl *manifestServiceListener) GetByTag(tag string, options ...distribution.ManifestServiceOption) (*schema1.SignedManifest, error) { - sm, err := msl.ManifestService.GetByTag(tag, options...) - if err == nil { - if err := msl.parent.listener.ManifestPulled(msl.parent.Repository.Name(), sm); err != nil { - logrus.Errorf("error dispatching manifest pull to listener: %v", err) - } - } - - return sm, err + return dgst, err } type blobServiceListener struct { diff --git a/notifications/listener_test.go b/notifications/listener_test.go index d8804fcb1..319406c31 100644 --- a/notifications/listener_test.go +++ b/notifications/listener_test.go @@ -38,7 +38,7 @@ func TestListener(t *testing.T) { expectedOps := map[string]int{ "manifest:push": 1, - "manifest:pull": 2, + "manifest:pull": 1, // "manifest:delete": 0, // deletes not supported for now "layer:push": 2, "layer:pull": 2, @@ -55,18 +55,18 @@ type testListener struct { ops map[string]int } -func (tl *testListener) ManifestPushed(repo string, sm *schema1.SignedManifest) error { +func (tl *testListener) ManifestPushed(repo string, m distribution.Manifest) error { tl.ops["manifest:push"]++ return nil } -func (tl *testListener) ManifestPulled(repo string, sm *schema1.SignedManifest) error { +func (tl *testListener) ManifestPulled(repo string, m distribution.Manifest) error { tl.ops["manifest:pull"]++ return nil } -func (tl *testListener) ManifestDeleted(repo string, sm *schema1.SignedManifest) error { +func (tl *testListener) ManifestDeleted(repo string, m distribution.Manifest) error { tl.ops["manifest:delete"]++ return nil } @@ -93,8 +93,11 @@ func checkExerciseRepository(t *testing.T, repository distribution.Repository) { // takes the registry through a common set of operations. This could be // used to make cross-cutting updates by changing internals that affect // update counts. Basically, it would make writing tests a lot easier. + ctx := context.Background() tag := "thetag" + // todo: change this to use Builder + m := schema1.Manifest{ Versioned: manifest.Versioned{ SchemaVersion: 1, @@ -158,31 +161,19 @@ func checkExerciseRepository(t *testing.T, repository distribution.Repository) { t.Fatal(err.Error()) } - if err = manifests.Put(sm); err != nil { + var digestPut digest.Digest + if digestPut, err = manifests.Put(ctx, sm); err != nil { t.Fatalf("unexpected error putting the manifest: %v", err) } - p, err := sm.Payload() - if err != nil { - t.Fatalf("unexpected error getting manifest payload: %v", err) + dgst := digest.FromBytes(sm.Canonical) + if dgst != digestPut { + t.Fatalf("mismatching digest from payload and put") } - dgst := digest.FromBytes(p) - fetchedByManifest, err := manifests.Get(dgst) + _, err = manifests.Get(ctx, dgst) if err != nil { t.Fatalf("unexpected error fetching manifest: %v", err) } - if fetchedByManifest.Tag != sm.Tag { - t.Fatalf("retrieved unexpected manifest: %v", err) - } - - fetched, err := manifests.GetByTag(tag) - if err != nil { - t.Fatalf("unexpected error fetching manifest: %v", err) - } - - if fetched.Tag != fetchedByManifest.Tag { - t.Fatalf("retrieved unexpected manifest: %v", err) - } } diff --git a/registry.go b/registry.go index 001776f87..ce5d77792 100644 --- a/registry.go +++ b/registry.go @@ -2,8 +2,6 @@ package distribution import ( "github.com/docker/distribution/context" - "github.com/docker/distribution/digest" - "github.com/docker/distribution/manifest/schema1" ) // Scope defines the set of items that match a namespace. @@ -44,7 +42,9 @@ type Namespace interface { } // ManifestServiceOption is a function argument for Manifest Service methods -type ManifestServiceOption func(ManifestService) error +type ManifestServiceOption interface { + Apply(ManifestService) error +} // Repository is a named collection of manifests and layers. type Repository interface { @@ -62,59 +62,10 @@ type Repository interface { // be a BlobService for use with clients. This will allow such // implementations to avoid implementing ServeBlob. - // Signatures returns a reference to this repository's signatures service. - Signatures() SignatureService + // Tags returns a reference to this repositories tag service + Tags(ctx context.Context) TagService } // TODO(stevvooe): Must add close methods to all these. May want to change the // way instances are created to better reflect internal dependency // relationships. - -// ManifestService provides operations on image manifests. -type ManifestService interface { - // Exists returns true if the manifest exists. - Exists(dgst digest.Digest) (bool, error) - - // Get retrieves the identified by the digest, if it exists. - Get(dgst digest.Digest) (*schema1.SignedManifest, error) - - // Delete removes the manifest, if it exists. - Delete(dgst digest.Digest) error - - // Put creates or updates the manifest. - Put(manifest *schema1.SignedManifest) error - - // TODO(stevvooe): The methods after this message should be moved to a - // discrete TagService, per active proposals. - - // Tags lists the tags under the named repository. - Tags() ([]string, error) - - // ExistsByTag returns true if the manifest exists. - ExistsByTag(tag string) (bool, error) - - // GetByTag retrieves the named manifest, if it exists. - GetByTag(tag string, options ...ManifestServiceOption) (*schema1.SignedManifest, error) - - // TODO(stevvooe): There are several changes that need to be done to this - // interface: - // - // 1. Allow explicit tagging with Tag(digest digest.Digest, tag string) - // 2. Support reading tags with a re-entrant reader to avoid large - // allocations in the registry. - // 3. Long-term: Provide All() method that lets one scroll through all of - // the manifest entries. - // 4. Long-term: break out concept of signing from manifests. This is - // really a part of the distribution sprint. - // 5. Long-term: Manifest should be an interface. This code shouldn't - // really be concerned with the storage format. -} - -// SignatureService provides operations on signatures. -type SignatureService interface { - // Get retrieves all of the signature blobs for the specified digest. - Get(dgst digest.Digest) ([][]byte, error) - - // Put stores the signature for the provided digest. - Put(dgst digest.Digest, signatures ...[]byte) error -} diff --git a/registry/api/v2/descriptors.go b/registry/api/v2/descriptors.go index 7eba362af..52c725dc2 100644 --- a/registry/api/v2/descriptors.go +++ b/registry/api/v2/descriptors.go @@ -495,7 +495,7 @@ var routeDescriptors = []RouteDescriptor{ Methods: []MethodDescriptor{ { Method: "GET", - Description: "Fetch the manifest identified by `name` and `reference` where `reference` can be a tag or digest.", + Description: "Fetch the manifest identified by `name` and `reference` where `reference` can be a tag or digest. A `HEAD` request can also be issued to this endpoint to obtain resource information without receiving all data.", Requests: []RequestDescriptor{ { Headers: []ParameterDescriptor{ diff --git a/registry/client/repository.go b/registry/client/repository.go index 8f525f0d3..9d489dd58 100644 --- a/registry/client/repository.go +++ b/registry/client/repository.go @@ -3,6 +3,7 @@ package client import ( "bytes" "encoding/json" + "errors" "fmt" "io" "io/ioutil" @@ -14,7 +15,6 @@ import ( "github.com/docker/distribution" "github.com/docker/distribution/context" "github.com/docker/distribution/digest" - "github.com/docker/distribution/manifest/schema1" "github.com/docker/distribution/reference" "github.com/docker/distribution/registry/api/v2" "github.com/docker/distribution/registry/client/transport" @@ -156,26 +156,139 @@ func (r *repository) Manifests(ctx context.Context, options ...distribution.Mani }, nil } -func (r *repository) Signatures() distribution.SignatureService { - ms, _ := r.Manifests(r.context) - return &signatures{ - manifests: ms, +func (r *repository) Tags(ctx context.Context) distribution.TagService { + return &tags{ + client: r.client, + ub: r.ub, + context: r.context, + name: r.Name(), } } -type signatures struct { - manifests distribution.ManifestService +// tags implements remote tagging operations. +type tags struct { + client *http.Client + ub *v2.URLBuilder + context context.Context + name string } -func (s *signatures) Get(dgst digest.Digest) ([][]byte, error) { - m, err := s.manifests.Get(dgst) +// All returns all tags +func (t *tags) All(ctx context.Context) ([]string, error) { + var tags []string + + u, err := t.ub.BuildTagsURL(t.name) if err != nil { - return nil, err + return tags, err } - return m.Signatures() + + resp, err := t.client.Get(u) + if err != nil { + return tags, err + } + defer resp.Body.Close() + + if SuccessStatus(resp.StatusCode) { + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return tags, err + } + + tagsResponse := struct { + Tags []string `json:"tags"` + }{} + if err := json.Unmarshal(b, &tagsResponse); err != nil { + return tags, err + } + tags = tagsResponse.Tags + return tags, nil + } + return tags, handleErrorResponse(resp) } -func (s *signatures) Put(dgst digest.Digest, signatures ...[]byte) error { +func descriptorFromResponse(response *http.Response) (distribution.Descriptor, error) { + desc := distribution.Descriptor{} + headers := response.Header + + ctHeader := headers.Get("Content-Type") + if ctHeader == "" { + return distribution.Descriptor{}, errors.New("missing or empty Content-Type header") + } + desc.MediaType = ctHeader + + digestHeader := headers.Get("Docker-Content-Digest") + if digestHeader == "" { + bytes, err := ioutil.ReadAll(response.Body) + if err != nil { + return distribution.Descriptor{}, err + } + _, desc, err := distribution.UnmarshalManifest(ctHeader, bytes) + if err != nil { + return distribution.Descriptor{}, err + } + return desc, nil + } + + dgst, err := digest.ParseDigest(digestHeader) + if err != nil { + return distribution.Descriptor{}, err + } + desc.Digest = dgst + + lengthHeader := headers.Get("Content-Length") + if lengthHeader == "" { + return distribution.Descriptor{}, errors.New("missing or empty Content-Length header") + } + length, err := strconv.ParseInt(lengthHeader, 10, 64) + if err != nil { + return distribution.Descriptor{}, err + } + desc.Size = length + + return desc, nil + +} + +// Get issues a HEAD request for a Manifest against its named endpoint in order +// to construct a descriptor for the tag. If the registry doesn't support HEADing +// a manifest, fallback to GET. +func (t *tags) Get(ctx context.Context, tag string) (distribution.Descriptor, error) { + u, err := t.ub.BuildManifestURL(t.name, tag) + if err != nil { + return distribution.Descriptor{}, err + } + var attempts int + resp, err := t.client.Head(u) + +check: + if err != nil { + return distribution.Descriptor{}, err + } + + switch { + case resp.StatusCode >= 200 && resp.StatusCode < 400: + return descriptorFromResponse(resp) + case resp.StatusCode == http.StatusMethodNotAllowed: + resp, err = t.client.Get(u) + attempts++ + if attempts > 1 { + return distribution.Descriptor{}, err + } + goto check + default: + return distribution.Descriptor{}, handleErrorResponse(resp) + } +} + +func (t *tags) Lookup(ctx context.Context, digest distribution.Descriptor) ([]string, error) { + panic("not implemented") +} + +func (t *tags) Tag(ctx context.Context, tag string, desc distribution.Descriptor) error { + panic("not implemented") +} + +func (t *tags) Untag(ctx context.Context, tag string) error { panic("not implemented") } @@ -186,44 +299,8 @@ type manifests struct { etags map[string]string } -func (ms *manifests) Tags() ([]string, error) { - u, err := ms.ub.BuildTagsURL(ms.name) - if err != nil { - return nil, err - } - - resp, err := ms.client.Get(u) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - if SuccessStatus(resp.StatusCode) { - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, err - } - - tagsResponse := struct { - Tags []string `json:"tags"` - }{} - if err := json.Unmarshal(b, &tagsResponse); err != nil { - return nil, err - } - - return tagsResponse.Tags, nil - } - return nil, handleErrorResponse(resp) -} - -func (ms *manifests) Exists(dgst digest.Digest) (bool, error) { - // Call by Tag endpoint since the API uses the same - // URL endpoint for tags and digests. - return ms.ExistsByTag(dgst.String()) -} - -func (ms *manifests) ExistsByTag(tag string) (bool, error) { - u, err := ms.ub.BuildManifestURL(ms.name, tag) +func (ms *manifests) Exists(ctx context.Context, dgst digest.Digest) (bool, error) { + u, err := ms.ub.BuildManifestURL(ms.name, dgst.String()) if err != nil { return false, err } @@ -241,46 +318,63 @@ func (ms *manifests) ExistsByTag(tag string) (bool, error) { return false, handleErrorResponse(resp) } -func (ms *manifests) Get(dgst digest.Digest) (*schema1.SignedManifest, error) { - // Call by Tag endpoint since the API uses the same - // URL endpoint for tags and digests. - return ms.GetByTag(dgst.String()) -} - -// AddEtagToTag allows a client to supply an eTag to GetByTag which will be +// AddEtagToTag allows a client to supply an eTag to Get which will be // used for a conditional HTTP request. If the eTag matches, a nil manifest -// and nil error will be returned. etag is automatically quoted when added to -// this map. +// and ErrManifestNotModified error will be returned. etag is automatically +// quoted when added to this map. func AddEtagToTag(tag, etag string) distribution.ManifestServiceOption { - return func(ms distribution.ManifestService) error { - if ms, ok := ms.(*manifests); ok { - ms.etags[tag] = fmt.Sprintf(`"%s"`, etag) - return nil - } - return fmt.Errorf("etag options is a client-only option") - } + return etagOption{tag, etag} } -func (ms *manifests) GetByTag(tag string, options ...distribution.ManifestServiceOption) (*schema1.SignedManifest, error) { +type etagOption struct{ tag, etag string } + +func (o etagOption) Apply(ms distribution.ManifestService) error { + if ms, ok := ms.(*manifests); ok { + ms.etags[o.tag] = fmt.Sprintf(`"%s"`, o.etag) + return nil + } + return fmt.Errorf("etag options is a client-only option") +} + +func (ms *manifests) Get(ctx context.Context, dgst digest.Digest, options ...distribution.ManifestServiceOption) (distribution.Manifest, error) { + + var tag string for _, option := range options { - err := option(ms) - if err != nil { - return nil, err + if opt, ok := option.(withTagOption); ok { + tag = opt.tag + } else { + err := option.Apply(ms) + if err != nil { + return nil, err + } } } - u, err := ms.ub.BuildManifestURL(ms.name, tag) + var ref string + if tag != "" { + ref = tag + } else { + ref = dgst.String() + } + + u, err := ms.ub.BuildManifestURL(ms.name, ref) if err != nil { return nil, err } + req, err := http.NewRequest("GET", u, nil) if err != nil { return nil, err } - if _, ok := ms.etags[tag]; ok { - req.Header.Set("If-None-Match", ms.etags[tag]) + for _, t := range distribution.ManifestMediaTypes() { + req.Header.Add("Accept", t) } + + if _, ok := ms.etags[ref]; ok { + req.Header.Set("If-None-Match", ms.etags[ref]) + } + resp, err := ms.client.Do(req) if err != nil { return nil, err @@ -289,44 +383,89 @@ func (ms *manifests) GetByTag(tag string, options ...distribution.ManifestServic if resp.StatusCode == http.StatusNotModified { return nil, distribution.ErrManifestNotModified } else if SuccessStatus(resp.StatusCode) { - var sm schema1.SignedManifest - decoder := json.NewDecoder(resp.Body) + mt := resp.Header.Get("Content-Type") + body, err := ioutil.ReadAll(resp.Body) - if err := decoder.Decode(&sm); err != nil { + if err != nil { return nil, err } - return &sm, nil + m, _, err := distribution.UnmarshalManifest(mt, body) + if err != nil { + return nil, err + } + return m, nil } return nil, handleErrorResponse(resp) } -func (ms *manifests) Put(m *schema1.SignedManifest) error { - manifestURL, err := ms.ub.BuildManifestURL(ms.name, m.Tag) - if err != nil { - return err +// WithTag allows a tag to be passed into Put which enables the client +// to build a correct URL. +func WithTag(tag string) distribution.ManifestServiceOption { + return withTagOption{tag} +} + +type withTagOption struct{ tag string } + +func (o withTagOption) Apply(m distribution.ManifestService) error { + if _, ok := m.(*manifests); ok { + return nil + } + return fmt.Errorf("withTagOption is a client-only option") +} + +// Put puts a manifest. A tag can be specified using an options parameter which uses some shared state to hold the +// tag name in order to build the correct upload URL. This state is written and read under a lock. +func (ms *manifests) Put(ctx context.Context, m distribution.Manifest, options ...distribution.ManifestServiceOption) (digest.Digest, error) { + var tag string + + for _, option := range options { + if opt, ok := option.(withTagOption); ok { + tag = opt.tag + } else { + err := option.Apply(ms) + if err != nil { + return "", err + } + } } - // todo(richardscothern): do something with options here when they become applicable - - putRequest, err := http.NewRequest("PUT", manifestURL, bytes.NewReader(m.Raw)) + manifestURL, err := ms.ub.BuildManifestURL(ms.name, tag) if err != nil { - return err + return "", err } + mediaType, p, err := m.Payload() + if err != nil { + return "", err + } + + putRequest, err := http.NewRequest("PUT", manifestURL, bytes.NewReader(p)) + if err != nil { + return "", err + } + + putRequest.Header.Set("Content-Type", mediaType) + resp, err := ms.client.Do(putRequest) if err != nil { - return err + return "", err } defer resp.Body.Close() if SuccessStatus(resp.StatusCode) { - // TODO(dmcgowan): make use of digest header - return nil + dgstHeader := resp.Header.Get("Docker-Content-Digest") + dgst, err := digest.ParseDigest(dgstHeader) + if err != nil { + return "", err + } + + return dgst, nil } - return handleErrorResponse(resp) + + return "", handleErrorResponse(resp) } -func (ms *manifests) Delete(dgst digest.Digest) error { +func (ms *manifests) Delete(ctx context.Context, dgst digest.Digest) error { u, err := ms.ub.BuildManifestURL(ms.name, dgst.String()) if err != nil { return err @@ -348,6 +487,11 @@ func (ms *manifests) Delete(dgst digest.Digest) error { return handleErrorResponse(resp) } +// todo(richardscothern): Restore interface and implementation with merge of #1050 +/*func (ms *manifests) Enumerate(ctx context.Context, manifests []distribution.Manifest, last distribution.Manifest) (n int, err error) { + panic("not supported") +}*/ + type blobs struct { name string ub *v2.URLBuilder diff --git a/registry/client/repository_test.go b/registry/client/repository_test.go index a001b62f3..c1032ec15 100644 --- a/registry/client/repository_test.go +++ b/registry/client/repository_test.go @@ -42,7 +42,6 @@ func newRandomBlob(size int) (digest.Digest, []byte) { } func addTestFetch(repo string, dgst digest.Digest, content []byte, m *testutil.RequestResponseMap) { - *m = append(*m, testutil.RequestResponseMapping{ Request: testutil.Request{ Method: "GET", @@ -499,12 +498,7 @@ func newRandomSchemaV1Manifest(name, tag string, blobCount int) (*schema1.Signed panic(err) } - p, err := sm.Payload() - if err != nil { - panic(err) - } - - return sm, digest.FromBytes(p), p + return sm, digest.FromBytes(sm.Canonical), sm.Canonical } func addTestManifestWithEtag(repo, reference string, content []byte, m *testutil.RequestResponseMap, dgst string) { @@ -525,6 +519,7 @@ func addTestManifestWithEtag(repo, reference string, content []byte, m *testutil Headers: http.Header(map[string][]string{ "Content-Length": {"0"}, "Last-Modified": {time.Now().Add(-1 * time.Second).Format(time.ANSIC)}, + "Content-Type": {schema1.MediaTypeManifest}, }), } } else { @@ -534,6 +529,7 @@ func addTestManifestWithEtag(repo, reference string, content []byte, m *testutil Headers: http.Header(map[string][]string{ "Content-Length": {fmt.Sprint(len(content))}, "Last-Modified": {time.Now().Add(-1 * time.Second).Format(time.ANSIC)}, + "Content-Type": {schema1.MediaTypeManifest}, }), } @@ -553,6 +549,7 @@ func addTestManifest(repo, reference string, content []byte, m *testutil.Request Headers: http.Header(map[string][]string{ "Content-Length": {fmt.Sprint(len(content))}, "Last-Modified": {time.Now().Add(-1 * time.Second).Format(time.ANSIC)}, + "Content-Type": {schema1.MediaTypeManifest}, }), }, }) @@ -566,6 +563,7 @@ func addTestManifest(repo, reference string, content []byte, m *testutil.Request Headers: http.Header(map[string][]string{ "Content-Length": {fmt.Sprint(len(content))}, "Last-Modified": {time.Now().Add(-1 * time.Second).Format(time.ANSIC)}, + "Content-Type": {schema1.MediaTypeManifest}, }), }, }) @@ -598,12 +596,17 @@ func checkEqualManifest(m1, m2 *schema1.SignedManifest) error { return nil } -func TestManifestFetch(t *testing.T) { +func TestV1ManifestFetch(t *testing.T) { ctx := context.Background() repo := "test.example.com/repo" m1, dgst, _ := newRandomSchemaV1Manifest(repo, "latest", 6) var m testutil.RequestResponseMap - addTestManifest(repo, dgst.String(), m1.Raw, &m) + _, pl, err := m1.Payload() + if err != nil { + t.Fatal(err) + } + addTestManifest(repo, dgst.String(), pl, &m) + addTestManifest(repo, "latest", pl, &m) e, c := testServer(m) defer c() @@ -617,7 +620,7 @@ func TestManifestFetch(t *testing.T) { t.Fatal(err) } - ok, err := ms.Exists(dgst) + ok, err := ms.Exists(ctx, dgst) if err != nil { t.Fatal(err) } @@ -625,11 +628,29 @@ func TestManifestFetch(t *testing.T) { t.Fatal("Manifest does not exist") } - manifest, err := ms.Get(dgst) + manifest, err := ms.Get(ctx, dgst) if err != nil { t.Fatal(err) } - if err := checkEqualManifest(manifest, m1); err != nil { + v1manifest, ok := manifest.(*schema1.SignedManifest) + if !ok { + t.Fatalf("Unexpected manifest type from Get: %T", manifest) + } + + if err := checkEqualManifest(v1manifest, m1); err != nil { + t.Fatal(err) + } + + manifest, err = ms.Get(ctx, dgst, WithTag("latest")) + if err != nil { + t.Fatal(err) + } + v1manifest, ok = manifest.(*schema1.SignedManifest) + if !ok { + t.Fatalf("Unexpected manifest type from Get: %T", manifest) + } + + if err = checkEqualManifest(v1manifest, m1); err != nil { t.Fatal(err) } } @@ -643,17 +664,22 @@ func TestManifestFetchWithEtag(t *testing.T) { e, c := testServer(m) defer c() - r, err := NewRepository(context.Background(), repo, e, nil) + ctx := context.Background() + r, err := NewRepository(ctx, repo, e, nil) if err != nil { t.Fatal(err) } - ctx := context.Background() + ms, err := r.Manifests(ctx) if err != nil { t.Fatal(err) } - _, err = ms.GetByTag("latest", AddEtagToTag("latest", d1.String())) + clientManifestService, ok := ms.(*manifests) + if !ok { + panic("wrong type for client manifest service") + } + _, err = clientManifestService.Get(ctx, d1, WithTag("latest"), AddEtagToTag("latest", d1.String())) if err != distribution.ErrManifestNotModified { t.Fatal(err) } @@ -690,10 +716,10 @@ func TestManifestDelete(t *testing.T) { t.Fatal(err) } - if err := ms.Delete(dgst1); err != nil { + if err := ms.Delete(ctx, dgst1); err != nil { t.Fatal(err) } - if err := ms.Delete(dgst2); err == nil { + if err := ms.Delete(ctx, dgst2); err == nil { t.Fatal("Expected error deleting unknown manifest") } // TODO(dmcgowan): Check for specific unknown error @@ -702,12 +728,17 @@ func TestManifestDelete(t *testing.T) { func TestManifestPut(t *testing.T) { repo := "test.example.com/repo/delete" m1, dgst, _ := newRandomSchemaV1Manifest(repo, "other", 6) + + _, payload, err := m1.Payload() + if err != nil { + t.Fatal(err) + } var m testutil.RequestResponseMap m = append(m, testutil.RequestResponseMapping{ Request: testutil.Request{ Method: "PUT", Route: "/v2/" + repo + "/manifests/other", - Body: m1.Raw, + Body: payload, }, Response: testutil.Response{ StatusCode: http.StatusAccepted, @@ -731,7 +762,7 @@ func TestManifestPut(t *testing.T) { t.Fatal(err) } - if err := ms.Put(m1); err != nil { + if _, err := ms.Put(ctx, m1, WithTag(m1.Tag)); err != nil { t.Fatal(err) } @@ -751,21 +782,22 @@ func TestManifestTags(t *testing.T) { } `)) var m testutil.RequestResponseMap - m = append(m, testutil.RequestResponseMapping{ - Request: testutil.Request{ - Method: "GET", - Route: "/v2/" + repo + "/tags/list", - }, - Response: testutil.Response{ - StatusCode: http.StatusOK, - Body: tagsList, - Headers: http.Header(map[string][]string{ - "Content-Length": {fmt.Sprint(len(tagsList))}, - "Last-Modified": {time.Now().Add(-1 * time.Second).Format(time.ANSIC)}, - }), - }, - }) - + for i := 0; i < 3; i++ { + m = append(m, testutil.RequestResponseMapping{ + Request: testutil.Request{ + Method: "GET", + Route: "/v2/" + repo + "/tags/list", + }, + Response: testutil.Response{ + StatusCode: http.StatusOK, + Body: tagsList, + Headers: http.Header(map[string][]string{ + "Content-Length": {fmt.Sprint(len(tagsList))}, + "Last-Modified": {time.Now().Add(-1 * time.Second).Format(time.ANSIC)}, + }), + }, + }) + } e, c := testServer(m) defer c() @@ -773,22 +805,29 @@ func TestManifestTags(t *testing.T) { if err != nil { t.Fatal(err) } + ctx := context.Background() - ms, err := r.Manifests(ctx) + tagService := r.Tags(ctx) + + tags, err := tagService.All(ctx) if err != nil { t.Fatal(err) } - - tags, err := ms.Tags() - if err != nil { - t.Fatal(err) - } - if len(tags) != 3 { t.Fatalf("Wrong number of tags returned: %d, expected 3", len(tags)) } - // TODO(dmcgowan): Check array + expected := map[string]struct{}{ + "tag1": {}, + "tag2": {}, + "funtag": {}, + } + for _, t := range tags { + delete(expected, t) + } + if len(expected) != 0 { + t.Fatalf("unexpected tags returned: %v", expected) + } // TODO(dmcgowan): Check for error cases } @@ -821,7 +860,7 @@ func TestManifestUnauthorized(t *testing.T) { t.Fatal(err) } - _, err = ms.Get(dgst) + _, err = ms.Get(ctx, dgst) if err == nil { t.Fatal("Expected error fetching manifest") } diff --git a/registry/handlers/api_test.go b/registry/handlers/api_test.go index 7b7c3c0d2..2672b77bc 100644 --- a/registry/handlers/api_test.go +++ b/registry/handlers/api_test.go @@ -871,19 +871,15 @@ func testManifestAPI(t *testing.T, env *testEnv, args manifestArgs) (*testEnv, m t.Fatalf("unexpected error signing manifest: %v", err) } - payload, err := signedManifest.Payload() - checkErr(t, err, "getting manifest payload") - - dgst := digest.FromBytes(payload) - + dgst := digest.FromBytes(signedManifest.Canonical) args.signedManifest = signedManifest args.dgst = dgst manifestDigestURL, err := env.builder.BuildManifestURL(imageName, dgst.String()) checkErr(t, err, "building manifest url") - resp = putManifest(t, "putting signed manifest", manifestURL, signedManifest) - checkResponse(t, "putting signed manifest", resp, http.StatusCreated) + resp = putManifest(t, "putting signed manifest no error", manifestURL, signedManifest) + checkResponse(t, "putting signed manifest no error", resp, http.StatusCreated) checkHeaders(t, resp, http.Header{ "Location": []string{manifestDigestURL}, "Docker-Content-Digest": []string{dgst.String()}, @@ -914,11 +910,12 @@ func testManifestAPI(t *testing.T, env *testEnv, args manifestArgs) (*testEnv, m var fetchedManifest schema1.SignedManifest dec := json.NewDecoder(resp.Body) + if err := dec.Decode(&fetchedManifest); err != nil { t.Fatalf("error decoding fetched manifest: %v", err) } - if !bytes.Equal(fetchedManifest.Raw, signedManifest.Raw) { + if !bytes.Equal(fetchedManifest.Canonical, signedManifest.Canonical) { t.Fatalf("manifests do not match") } @@ -940,10 +937,55 @@ func testManifestAPI(t *testing.T, env *testEnv, args manifestArgs) (*testEnv, m t.Fatalf("error decoding fetched manifest: %v", err) } - if !bytes.Equal(fetchedManifestByDigest.Raw, signedManifest.Raw) { + if !bytes.Equal(fetchedManifestByDigest.Canonical, signedManifest.Canonical) { t.Fatalf("manifests do not match") } + // check signature was roundtripped + signatures, err := fetchedManifestByDigest.Signatures() + if err != nil { + t.Fatal(err) + } + + if len(signatures) != 1 { + t.Fatalf("expected 1 signature from manifest, got: %d", len(signatures)) + } + + // Re-sign, push and pull the same digest + sm2, err := schema1.Sign(&fetchedManifestByDigest.Manifest, env.pk) + if err != nil { + t.Fatal(err) + + } + + resp = putManifest(t, "re-putting signed manifest", manifestDigestURL, sm2) + checkResponse(t, "re-putting signed manifest", resp, http.StatusCreated) + + resp, err = http.Get(manifestDigestURL) + checkErr(t, err, "re-fetching manifest by digest") + defer resp.Body.Close() + + checkResponse(t, "re-fetching uploaded manifest", resp, http.StatusOK) + checkHeaders(t, resp, http.Header{ + "Docker-Content-Digest": []string{dgst.String()}, + "ETag": []string{fmt.Sprintf(`"%s"`, dgst)}, + }) + + dec = json.NewDecoder(resp.Body) + if err := dec.Decode(&fetchedManifestByDigest); err != nil { + t.Fatalf("error decoding fetched manifest: %v", err) + } + + // check two signatures were roundtripped + signatures, err = fetchedManifestByDigest.Signatures() + if err != nil { + t.Fatal(err) + } + + if len(signatures) != 2 { + t.Fatalf("expected 2 signature from manifest, got: %d", len(signatures)) + } + // Get by name with etag, gives 304 etag := resp.Header.Get("Etag") req, err := http.NewRequest("GET", manifestURL, nil) @@ -956,7 +998,7 @@ func testManifestAPI(t *testing.T, env *testEnv, args manifestArgs) (*testEnv, m t.Fatalf("Error constructing request: %s", err) } - checkResponse(t, "fetching layer with etag", resp, http.StatusNotModified) + checkResponse(t, "fetching manifest by name with etag", resp, http.StatusNotModified) // Get by digest with etag, gives 304 req, err = http.NewRequest("GET", manifestDigestURL, nil) @@ -969,7 +1011,7 @@ func testManifestAPI(t *testing.T, env *testEnv, args manifestArgs) (*testEnv, m t.Fatalf("Error constructing request: %s", err) } - checkResponse(t, "fetching layer with etag", resp, http.StatusNotModified) + checkResponse(t, "fetching manifest by dgst with etag", resp, http.StatusNotModified) // Ensure that the tag is listed. resp, err = http.Get(tagsURL) @@ -1143,8 +1185,13 @@ func newTestEnvWithConfig(t *testing.T, config *configuration.Configuration) *te func putManifest(t *testing.T, msg, url string, v interface{}) *http.Response { var body []byte + if sm, ok := v.(*schema1.SignedManifest); ok { - body = sm.Raw + _, pl, err := sm.Payload() + if err != nil { + t.Fatalf("error getting payload: %v", err) + } + body = pl } else { var err error body, err = json.MarshalIndent(v, "", " ") @@ -1435,7 +1482,7 @@ func checkErr(t *testing.T, err error, msg string) { } } -func createRepository(env *testEnv, t *testing.T, imageName string, tag string) { +func createRepository(env *testEnv, t *testing.T, imageName string, tag string) digest.Digest { unsignedManifest := &schema1.Manifest{ Versioned: manifest.Versioned{ SchemaVersion: 1, @@ -1459,7 +1506,6 @@ func createRepository(env *testEnv, t *testing.T, imageName string, tag string) for i := range unsignedManifest.FSLayers { rs, dgstStr, err := testutil.CreateRandomTarFile() - if err != nil { t.Fatalf("error creating random layer %d: %v", i, err) } @@ -1477,20 +1523,22 @@ func createRepository(env *testEnv, t *testing.T, imageName string, tag string) t.Fatalf("unexpected error signing manifest: %v", err) } - payload, err := signedManifest.Payload() - checkErr(t, err, "getting manifest payload") + dgst := digest.FromBytes(signedManifest.Canonical) - dgst := digest.FromBytes(payload) - - manifestDigestURL, err := env.builder.BuildManifestURL(imageName, dgst.String()) + // Create this repository by tag to ensure the tag mapping is made in the registry + manifestDigestURL, err := env.builder.BuildManifestURL(imageName, tag) checkErr(t, err, "building manifest url") + location, err := env.builder.BuildManifestURL(imageName, dgst.String()) + checkErr(t, err, "building location URL") + resp := putManifest(t, "putting signed manifest", manifestDigestURL, signedManifest) checkResponse(t, "putting signed manifest", resp, http.StatusCreated) checkHeaders(t, resp, http.Header{ - "Location": []string{manifestDigestURL}, + "Location": []string{location}, "Docker-Content-Digest": []string{dgst.String()}, }) + return dgst } // Test mutation operations on a registry configured as a cache. Ensure that they return @@ -1577,3 +1625,64 @@ func TestCheckContextNotifier(t *testing.T) { t.Fatalf("wrong status code - expected 200, got %d", resp.StatusCode) } } + +func TestProxyManifestGetByTag(t *testing.T) { + truthConfig := configuration.Configuration{ + Storage: configuration.Storage{ + "inmemory": configuration.Parameters{}, + }, + } + truthConfig.HTTP.Headers = headerConfig + + imageName := "foo/bar" + tag := "latest" + + truthEnv := newTestEnvWithConfig(t, &truthConfig) + // create a repository in the truth registry + dgst := createRepository(truthEnv, t, imageName, tag) + + proxyConfig := configuration.Configuration{ + Storage: configuration.Storage{ + "inmemory": configuration.Parameters{}, + }, + Proxy: configuration.Proxy{ + RemoteURL: truthEnv.server.URL, + }, + } + proxyConfig.HTTP.Headers = headerConfig + + proxyEnv := newTestEnvWithConfig(t, &proxyConfig) + + manifestDigestURL, err := proxyEnv.builder.BuildManifestURL(imageName, dgst.String()) + checkErr(t, err, "building manifest url") + + resp, err := http.Get(manifestDigestURL) + checkErr(t, err, "fetching manifest from proxy by digest") + defer resp.Body.Close() + + manifestTagURL, err := proxyEnv.builder.BuildManifestURL(imageName, tag) + checkErr(t, err, "building manifest url") + + resp, err = http.Get(manifestTagURL) + checkErr(t, err, "fetching manifest from proxy by tag") + defer resp.Body.Close() + checkResponse(t, "fetching manifest from proxy by tag", resp, http.StatusOK) + checkHeaders(t, resp, http.Header{ + "Docker-Content-Digest": []string{dgst.String()}, + }) + + // Create another manifest in the remote with the same image/tag pair + newDigest := createRepository(truthEnv, t, imageName, tag) + if dgst == newDigest { + t.Fatalf("non-random test data") + } + + // fetch it with the same proxy URL as before. Ensure the updated content is at the same tag + resp, err = http.Get(manifestTagURL) + checkErr(t, err, "fetching manifest from proxy by tag") + defer resp.Body.Close() + checkResponse(t, "fetching manifest from proxy by tag", resp, http.StatusOK) + checkHeaders(t, resp, http.Header{ + "Docker-Content-Digest": []string{newDigest.String()}, + }) +} diff --git a/registry/handlers/images.go b/registry/handlers/images.go index 2ec51b994..be14b00a0 100644 --- a/registry/handlers/images.go +++ b/registry/handlers/images.go @@ -2,19 +2,15 @@ package handlers import ( "bytes" - "encoding/json" "fmt" "net/http" - "strings" "github.com/docker/distribution" ctxu "github.com/docker/distribution/context" "github.com/docker/distribution/digest" - "github.com/docker/distribution/manifest/schema1" "github.com/docker/distribution/registry/api/errcode" "github.com/docker/distribution/registry/api/v2" "github.com/gorilla/handlers" - "golang.org/x/net/context" ) // imageManifestDispatcher takes the request context and builds the @@ -33,7 +29,8 @@ func imageManifestDispatcher(ctx *Context, r *http.Request) http.Handler { } mhandler := handlers.MethodHandler{ - "GET": http.HandlerFunc(imageManifestHandler.GetImageManifest), + "GET": http.HandlerFunc(imageManifestHandler.GetImageManifest), + "HEAD": http.HandlerFunc(imageManifestHandler.GetImageManifest), } if !ctx.readOnly { @@ -54,6 +51,8 @@ type imageManifestHandler struct { } // GetImageManifest fetches the image manifest from the storage backend, if it exists. +// todo(richardscothern): this assumes v2 schema 1 manifests for now but in the future +// get the version from the Accept HTTP header func (imh *imageManifestHandler) GetImageManifest(w http.ResponseWriter, r *http.Request) { ctxu.GetLogger(imh).Debug("GetImageManifest") manifests, err := imh.Repository.Manifests(imh) @@ -62,42 +61,38 @@ func (imh *imageManifestHandler) GetImageManifest(w http.ResponseWriter, r *http return } - var sm *schema1.SignedManifest + var manifest distribution.Manifest if imh.Tag != "" { - sm, err = manifests.GetByTag(imh.Tag) - } else { - if etagMatch(r, imh.Digest.String()) { - w.WriteHeader(http.StatusNotModified) + tags := imh.Repository.Tags(imh) + desc, err := tags.Get(imh, imh.Tag) + if err != nil { + imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithDetail(err)) return } - sm, err = manifests.Get(imh.Digest) + imh.Digest = desc.Digest } + if etagMatch(r, imh.Digest.String()) { + w.WriteHeader(http.StatusNotModified) + return + } + + manifest, err = manifests.Get(imh, imh.Digest) if err != nil { imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithDetail(err)) return } - // Get the digest, if we don't already have it. - if imh.Digest == "" { - dgst, err := digestManifest(imh, sm) - if err != nil { - imh.Errors = append(imh.Errors, v2.ErrorCodeDigestInvalid.WithDetail(err)) - return - } - if etagMatch(r, dgst.String()) { - w.WriteHeader(http.StatusNotModified) - return - } - - imh.Digest = dgst + ct, p, err := manifest.Payload() + if err != nil { + return } - w.Header().Set("Content-Type", "application/json; charset=utf-8") - w.Header().Set("Content-Length", fmt.Sprint(len(sm.Raw))) + w.Header().Set("Content-Type", ct) + w.Header().Set("Content-Length", fmt.Sprint(len(p))) w.Header().Set("Docker-Content-Digest", imh.Digest.String()) w.Header().Set("Etag", fmt.Sprintf(`"%s"`, imh.Digest)) - w.Write(sm.Raw) + w.Write(p) } func etagMatch(r *http.Request, etag string) bool { @@ -109,7 +104,7 @@ func etagMatch(r *http.Request, etag string) bool { return false } -// PutImageManifest validates and stores and image in the registry. +// PutImageManifest validates and stores an image in the registry. func (imh *imageManifestHandler) PutImageManifest(w http.ResponseWriter, r *http.Request) { ctxu.GetLogger(imh).Debug("PutImageManifest") manifests, err := imh.Repository.Manifests(imh) @@ -124,39 +119,28 @@ func (imh *imageManifestHandler) PutImageManifest(w http.ResponseWriter, r *http return } - var manifest schema1.SignedManifest - if err := json.Unmarshal(jsonBuf.Bytes(), &manifest); err != nil { + mediaType := r.Header.Get("Content-Type") + manifest, desc, err := distribution.UnmarshalManifest(mediaType, jsonBuf.Bytes()) + if err != nil { imh.Errors = append(imh.Errors, v2.ErrorCodeManifestInvalid.WithDetail(err)) return } - dgst, err := digestManifest(imh, &manifest) - if err != nil { - imh.Errors = append(imh.Errors, v2.ErrorCodeDigestInvalid.WithDetail(err)) - return - } - - // Validate manifest tag or digest matches payload - if imh.Tag != "" { - if manifest.Tag != imh.Tag { - ctxu.GetLogger(imh).Errorf("invalid tag on manifest payload: %q != %q", manifest.Tag, imh.Tag) - imh.Errors = append(imh.Errors, v2.ErrorCodeTagInvalid) - return - } - - imh.Digest = dgst - } else if imh.Digest != "" { - if dgst != imh.Digest { - ctxu.GetLogger(imh).Errorf("payload digest does match: %q != %q", dgst, imh.Digest) + if imh.Digest != "" { + if desc.Digest != imh.Digest { + ctxu.GetLogger(imh).Errorf("payload digest does match: %q != %q", desc.Digest, imh.Digest) imh.Errors = append(imh.Errors, v2.ErrorCodeDigestInvalid) return } + } else if imh.Tag != "" { + imh.Digest = desc.Digest } else { imh.Errors = append(imh.Errors, v2.ErrorCodeTagInvalid.WithDetail("no tag or digest specified")) return } - if err := manifests.Put(&manifest); err != nil { + _, err = manifests.Put(imh, manifest) + if err != nil { // TODO(stevvooe): These error handling switches really need to be // handled by an app global mapper. if err == distribution.ErrUnsupported { @@ -188,6 +172,17 @@ func (imh *imageManifestHandler) PutImageManifest(w http.ResponseWriter, r *http return } + // Tag this manifest + if imh.Tag != "" { + tags := imh.Repository.Tags(imh) + err = tags.Tag(imh, imh.Tag, desc) + if err != nil { + imh.Errors = append(imh.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) + return + } + + } + // Construct a canonical url for the uploaded manifest. location, err := imh.urlBuilder.BuildManifestURL(imh.Repository.Name(), imh.Digest.String()) if err != nil { @@ -212,7 +207,7 @@ func (imh *imageManifestHandler) DeleteImageManifest(w http.ResponseWriter, r *h return } - err = manifests.Delete(imh.Digest) + err = manifests.Delete(imh, imh.Digest) if err != nil { switch err { case digest.ErrDigestUnsupported: @@ -233,22 +228,3 @@ func (imh *imageManifestHandler) DeleteImageManifest(w http.ResponseWriter, r *h w.WriteHeader(http.StatusAccepted) } - -// digestManifest takes a digest of the given manifest. This belongs somewhere -// better but we'll wait for a refactoring cycle to find that real somewhere. -func digestManifest(ctx context.Context, sm *schema1.SignedManifest) (digest.Digest, error) { - p, err := sm.Payload() - if err != nil { - if !strings.Contains(err.Error(), "missing signature key") { - ctxu.GetLogger(ctx).Errorf("error getting manifest payload: %v", err) - return "", err - } - - // NOTE(stevvooe): There are no signatures but we still have a - // payload. The request will fail later but this is not the - // responsibility of this part of the code. - p = sm.Raw - } - - return digest.FromBytes(p), nil -} diff --git a/registry/handlers/tags.go b/registry/handlers/tags.go index 547255857..d9f0106c9 100644 --- a/registry/handlers/tags.go +++ b/registry/handlers/tags.go @@ -34,13 +34,9 @@ type tagsAPIResponse struct { // GetTags returns a json list of tags for a specific image name. func (th *tagsHandler) GetTags(w http.ResponseWriter, r *http.Request) { defer r.Body.Close() - manifests, err := th.Repository.Manifests(th) - if err != nil { - th.Errors = append(th.Errors, err) - return - } - tags, err := manifests.Tags() + tagService := th.Repository.Tags(th) + tags, err := tagService.All(th) if err != nil { switch err := err.(type) { case distribution.ErrRepositoryUnknown: diff --git a/registry/proxy/proxymanifeststore.go b/registry/proxy/proxymanifeststore.go index 1e9e24de0..13cb5f6b9 100644 --- a/registry/proxy/proxymanifeststore.go +++ b/registry/proxy/proxymanifeststore.go @@ -6,8 +6,6 @@ import ( "github.com/docker/distribution" "github.com/docker/distribution/context" "github.com/docker/distribution/digest" - "github.com/docker/distribution/manifest/schema1" - "github.com/docker/distribution/registry/client" "github.com/docker/distribution/registry/proxy/scheduler" ) @@ -24,8 +22,8 @@ type proxyManifestStore struct { var _ distribution.ManifestService = &proxyManifestStore{} -func (pms proxyManifestStore) Exists(dgst digest.Digest) (bool, error) { - exists, err := pms.localManifests.Exists(dgst) +func (pms proxyManifestStore) Exists(ctx context.Context, dgst digest.Digest) (bool, error) { + exists, err := pms.localManifests.Exists(ctx, dgst) if err != nil { return false, err } @@ -33,117 +31,56 @@ func (pms proxyManifestStore) Exists(dgst digest.Digest) (bool, error) { return true, nil } - return pms.remoteManifests.Exists(dgst) + return pms.remoteManifests.Exists(ctx, dgst) } -func (pms proxyManifestStore) Get(dgst digest.Digest) (*schema1.SignedManifest, error) { - sm, err := pms.localManifests.Get(dgst) - if err == nil { - proxyMetrics.ManifestPush(uint64(len(sm.Raw))) - return sm, err +func (pms proxyManifestStore) Get(ctx context.Context, dgst digest.Digest, options ...distribution.ManifestServiceOption) (distribution.Manifest, error) { + // At this point `dgst` was either specified explicitly, or returned by the + // tagstore with the most recent association. + var fromRemote bool + manifest, err := pms.localManifests.Get(ctx, dgst, options...) + if err != nil { + manifest, err = pms.remoteManifests.Get(ctx, dgst, options...) + if err != nil { + return nil, err + } + fromRemote = true } - sm, err = pms.remoteManifests.Get(dgst) + _, payload, err := manifest.Payload() if err != nil { return nil, err } - proxyMetrics.ManifestPull(uint64(len(sm.Raw))) - err = pms.localManifests.Put(sm) - if err != nil { - return nil, err + proxyMetrics.ManifestPush(uint64(len(payload))) + if fromRemote { + proxyMetrics.ManifestPull(uint64(len(payload))) + + _, err = pms.localManifests.Put(ctx, manifest) + if err != nil { + return nil, err + } + + // Schedule the repo for removal + pms.scheduler.AddManifest(pms.repositoryName, repositoryTTL) + + // Ensure the manifest blob is cleaned up + pms.scheduler.AddBlob(dgst.String(), repositoryTTL) } - // Schedule the repo for removal - pms.scheduler.AddManifest(pms.repositoryName, repositoryTTL) - - // Ensure the manifest blob is cleaned up - pms.scheduler.AddBlob(dgst.String(), repositoryTTL) - - proxyMetrics.ManifestPush(uint64(len(sm.Raw))) - - return sm, err + return manifest, err } -func (pms proxyManifestStore) Tags() ([]string, error) { - return pms.localManifests.Tags() +func (pms proxyManifestStore) Put(ctx context.Context, manifest distribution.Manifest, options ...distribution.ManifestServiceOption) (digest.Digest, error) { + var d digest.Digest + return d, distribution.ErrUnsupported } -func (pms proxyManifestStore) ExistsByTag(tag string) (bool, error) { - exists, err := pms.localManifests.ExistsByTag(tag) - if err != nil { - return false, err - } - if exists { - return true, nil - } - - return pms.remoteManifests.ExistsByTag(tag) -} - -func (pms proxyManifestStore) GetByTag(tag string, options ...distribution.ManifestServiceOption) (*schema1.SignedManifest, error) { - var localDigest digest.Digest - - localManifest, err := pms.localManifests.GetByTag(tag, options...) - switch err.(type) { - case distribution.ErrManifestUnknown, distribution.ErrManifestUnknownRevision: - goto fromremote - case nil: - break - default: - return nil, err - } - - localDigest, err = manifestDigest(localManifest) - if err != nil { - return nil, err - } - -fromremote: - var sm *schema1.SignedManifest - sm, err = pms.remoteManifests.GetByTag(tag, client.AddEtagToTag(tag, localDigest.String())) - if err != nil && err != distribution.ErrManifestNotModified { - return nil, err - } - - if err == distribution.ErrManifestNotModified { - context.GetLogger(pms.ctx).Debugf("Local manifest for %q is latest, dgst=%s", tag, localDigest.String()) - return localManifest, nil - } - context.GetLogger(pms.ctx).Debugf("Updated manifest for %q, dgst=%s", tag, localDigest.String()) - - err = pms.localManifests.Put(sm) - if err != nil { - return nil, err - } - - dgst, err := manifestDigest(sm) - if err != nil { - return nil, err - } - pms.scheduler.AddBlob(dgst.String(), repositoryTTL) - pms.scheduler.AddManifest(pms.repositoryName, repositoryTTL) - - proxyMetrics.ManifestPull(uint64(len(sm.Raw))) - proxyMetrics.ManifestPush(uint64(len(sm.Raw))) - - return sm, err -} - -func manifestDigest(sm *schema1.SignedManifest) (digest.Digest, error) { - payload, err := sm.Payload() - if err != nil { - return "", err - - } - - return digest.FromBytes(payload), nil -} - -func (pms proxyManifestStore) Put(manifest *schema1.SignedManifest) error { +func (pms proxyManifestStore) Delete(ctx context.Context, dgst digest.Digest) error { return distribution.ErrUnsupported } -func (pms proxyManifestStore) Delete(dgst digest.Digest) error { - return distribution.ErrUnsupported +/*func (pms proxyManifestStore) Enumerate(ctx context.Context, manifests []distribution.Manifest, last distribution.Manifest) (n int, err error) { + return 0, distribution.ErrUnsupported } +*/ diff --git a/registry/proxy/proxymanifeststore_test.go b/registry/proxy/proxymanifeststore_test.go index a5a0a21b4..aeecae10a 100644 --- a/registry/proxy/proxymanifeststore_test.go +++ b/registry/proxy/proxymanifeststore_test.go @@ -37,40 +37,31 @@ func (te manifestStoreTestEnv) RemoteStats() *map[string]int { return &rs } -func (sm statsManifest) Delete(dgst digest.Digest) error { +func (sm statsManifest) Delete(ctx context.Context, dgst digest.Digest) error { sm.stats["delete"]++ - return sm.manifests.Delete(dgst) + return sm.manifests.Delete(ctx, dgst) } -func (sm statsManifest) Exists(dgst digest.Digest) (bool, error) { +func (sm statsManifest) Exists(ctx context.Context, dgst digest.Digest) (bool, error) { sm.stats["exists"]++ - return sm.manifests.Exists(dgst) + return sm.manifests.Exists(ctx, dgst) } -func (sm statsManifest) ExistsByTag(tag string) (bool, error) { - sm.stats["existbytag"]++ - return sm.manifests.ExistsByTag(tag) -} - -func (sm statsManifest) Get(dgst digest.Digest) (*schema1.SignedManifest, error) { +func (sm statsManifest) Get(ctx context.Context, dgst digest.Digest, options ...distribution.ManifestServiceOption) (distribution.Manifest, error) { sm.stats["get"]++ - return sm.manifests.Get(dgst) + return sm.manifests.Get(ctx, dgst) } -func (sm statsManifest) GetByTag(tag string, options ...distribution.ManifestServiceOption) (*schema1.SignedManifest, error) { - sm.stats["getbytag"]++ - return sm.manifests.GetByTag(tag, options...) -} - -func (sm statsManifest) Put(manifest *schema1.SignedManifest) error { +func (sm statsManifest) Put(ctx context.Context, manifest distribution.Manifest, options ...distribution.ManifestServiceOption) (digest.Digest, error) { sm.stats["put"]++ - return sm.manifests.Put(manifest) + return sm.manifests.Put(ctx, manifest) } -func (sm statsManifest) Tags() ([]string, error) { - sm.stats["tags"]++ - return sm.manifests.Tags() +/*func (sm statsManifest) Enumerate(ctx context.Context, manifests []distribution.Manifest, last distribution.Manifest) (n int, err error) { + sm.stats["enumerate"]++ + return sm.manifests.Enumerate(ctx, manifests, last) } +*/ func newManifestStoreTestEnv(t *testing.T, name, tag string) *manifestStoreTestEnv { ctx := context.Background() @@ -169,15 +160,12 @@ func populateRepo(t *testing.T, ctx context.Context, repository distribution.Rep if err != nil { t.Fatalf(err.Error()) } - ms.Put(sm) + dgst, err := ms.Put(ctx, sm) if err != nil { t.Fatalf("unexpected errors putting manifest: %v", err) } - pl, err := sm.Payload() - if err != nil { - t.Fatal(err) - } - return digest.FromBytes(pl), nil + + return dgst, nil } // TestProxyManifests contains basic acceptance tests @@ -189,8 +177,9 @@ func TestProxyManifests(t *testing.T) { localStats := env.LocalStats() remoteStats := env.RemoteStats() + ctx := context.Background() // Stat - must check local and remote - exists, err := env.manifests.ExistsByTag("latest") + exists, err := env.manifests.Exists(ctx, env.manifestDigest) if err != nil { t.Fatalf("Error checking existance") } @@ -198,15 +187,16 @@ func TestProxyManifests(t *testing.T) { t.Errorf("Unexpected non-existant manifest") } - if (*localStats)["existbytag"] != 1 && (*remoteStats)["existbytag"] != 1 { - t.Errorf("Unexpected exists count") + if (*localStats)["exists"] != 1 && (*remoteStats)["exists"] != 1 { + t.Errorf("Unexpected exists count : \n%v \n%v", localStats, remoteStats) } // Get - should succeed and pull manifest into local - _, err = env.manifests.Get(env.manifestDigest) + _, err = env.manifests.Get(ctx, env.manifestDigest) if err != nil { t.Fatal(err) } + if (*localStats)["get"] != 1 && (*remoteStats)["get"] != 1 { t.Errorf("Unexpected get count") } @@ -216,7 +206,7 @@ func TestProxyManifests(t *testing.T) { } // Stat - should only go to local - exists, err = env.manifests.ExistsByTag("latest") + exists, err = env.manifests.Exists(ctx, env.manifestDigest) if err != nil { t.Fatal(err) } @@ -224,19 +214,21 @@ func TestProxyManifests(t *testing.T) { t.Errorf("Unexpected non-existant manifest") } - if (*localStats)["existbytag"] != 2 && (*remoteStats)["existbytag"] != 1 { + if (*localStats)["exists"] != 2 && (*remoteStats)["exists"] != 1 { t.Errorf("Unexpected exists count") - } // Get - should get from remote, to test freshness - _, err = env.manifests.Get(env.manifestDigest) + _, err = env.manifests.Get(ctx, env.manifestDigest) if err != nil { t.Fatal(err) } - if (*remoteStats)["get"] != 2 && (*remoteStats)["existsbytag"] != 1 && (*localStats)["put"] != 1 { + if (*remoteStats)["get"] != 2 && (*remoteStats)["exists"] != 1 && (*localStats)["put"] != 1 { t.Errorf("Unexpected get count") } +} + +func TestProxyTagService(t *testing.T) { } diff --git a/registry/proxy/proxyregistry.go b/registry/proxy/proxyregistry.go index 8a5f5ef6d..8e1be5f27 100644 --- a/registry/proxy/proxyregistry.go +++ b/registry/proxy/proxyregistry.go @@ -42,6 +42,7 @@ func NewRegistryPullThroughCache(ctx context.Context, registry distribution.Name s.OnManifestExpire(func(repoName string) error { return v.RemoveRepository(repoName) }) + err = s.Start() if err != nil { return nil, err @@ -78,7 +79,7 @@ func (pr *proxyingRegistry) Repository(ctx context.Context, name string) (distri if err != nil { return nil, err } - localManifests, err := localRepo.Manifests(ctx, storage.SkipLayerVerification) + localManifests, err := localRepo.Manifests(ctx, storage.SkipLayerVerification()) if err != nil { return nil, err } @@ -106,8 +107,11 @@ func (pr *proxyingRegistry) Repository(ctx context.Context, name string) (distri ctx: ctx, scheduler: pr.scheduler, }, - name: name, - signatures: localRepo.Signatures(), + name: name, + tags: proxyTagService{ + localTags: localRepo.Tags(ctx), + remoteTags: remoteRepo.Tags(ctx), + }, }, nil } @@ -115,14 +119,13 @@ func (pr *proxyingRegistry) Repository(ctx context.Context, name string) (distri // locally, or pulling it through from a remote and caching it locally if it doesn't // already exist type proxiedRepository struct { - blobStore distribution.BlobStore - manifests distribution.ManifestService - name string - signatures distribution.SignatureService + blobStore distribution.BlobStore + manifests distribution.ManifestService + name string + tags distribution.TagService } func (pr *proxiedRepository) Manifests(ctx context.Context, options ...distribution.ManifestServiceOption) (distribution.ManifestService, error) { - // options return pr.manifests, nil } @@ -134,6 +137,6 @@ func (pr *proxiedRepository) Name() string { return pr.name } -func (pr *proxiedRepository) Signatures() distribution.SignatureService { - return pr.signatures +func (pr *proxiedRepository) Tags(ctx context.Context) distribution.TagService { + return pr.tags } diff --git a/registry/proxy/proxytagservice.go b/registry/proxy/proxytagservice.go new file mode 100644 index 000000000..c52460c44 --- /dev/null +++ b/registry/proxy/proxytagservice.go @@ -0,0 +1,58 @@ +package proxy + +import ( + "github.com/docker/distribution" + "github.com/docker/distribution/context" +) + +// proxyTagService supports local and remote lookup of tags. +type proxyTagService struct { + localTags distribution.TagService + remoteTags distribution.TagService +} + +var _ distribution.TagService = proxyTagService{} + +// Get attempts to get the most recent digest for the tag by checking the remote +// tag service first and then caching it locally. If the remote is unavailable +// the local association is returned +func (pt proxyTagService) Get(ctx context.Context, tag string) (distribution.Descriptor, error) { + desc, err := pt.remoteTags.Get(ctx, tag) + if err == nil { + err := pt.localTags.Tag(ctx, tag, desc) + if err != nil { + return distribution.Descriptor{}, err + } + return desc, nil + } + + desc, err = pt.localTags.Get(ctx, tag) + if err != nil { + return distribution.Descriptor{}, err + } + return desc, nil +} + +func (pt proxyTagService) Tag(ctx context.Context, tag string, desc distribution.Descriptor) error { + return distribution.ErrUnsupported +} + +func (pt proxyTagService) Untag(ctx context.Context, tag string) error { + err := pt.localTags.Untag(ctx, tag) + if err != nil { + return err + } + return nil +} + +func (pt proxyTagService) All(ctx context.Context) ([]string, error) { + tags, err := pt.remoteTags.All(ctx) + if err == nil { + return tags, err + } + return pt.localTags.All(ctx) +} + +func (pt proxyTagService) Lookup(ctx context.Context, digest distribution.Descriptor) ([]string, error) { + return []string{}, distribution.ErrUnsupported +} diff --git a/registry/proxy/proxytagservice_test.go b/registry/proxy/proxytagservice_test.go new file mode 100644 index 000000000..8d9518c03 --- /dev/null +++ b/registry/proxy/proxytagservice_test.go @@ -0,0 +1,164 @@ +package proxy + +import ( + "sort" + "sync" + "testing" + + "github.com/docker/distribution" + "github.com/docker/distribution/context" +) + +type mockTagStore struct { + mapping map[string]distribution.Descriptor + sync.Mutex +} + +var _ distribution.TagService = &mockTagStore{} + +func (m *mockTagStore) Get(ctx context.Context, tag string) (distribution.Descriptor, error) { + m.Lock() + defer m.Unlock() + + if d, ok := m.mapping[tag]; ok { + return d, nil + } + return distribution.Descriptor{}, distribution.ErrTagUnknown{} +} + +func (m *mockTagStore) Tag(ctx context.Context, tag string, desc distribution.Descriptor) error { + m.Lock() + defer m.Unlock() + + m.mapping[tag] = desc + return nil +} + +func (m *mockTagStore) Untag(ctx context.Context, tag string) error { + m.Lock() + defer m.Unlock() + + if _, ok := m.mapping[tag]; ok { + delete(m.mapping, tag) + return nil + } + return distribution.ErrTagUnknown{} +} + +func (m *mockTagStore) All(ctx context.Context) ([]string, error) { + m.Lock() + defer m.Unlock() + + var tags []string + for tag := range m.mapping { + tags = append(tags, tag) + } + + return tags, nil +} + +func (m *mockTagStore) Lookup(ctx context.Context, digest distribution.Descriptor) ([]string, error) { + panic("not implemented") +} + +func testProxyTagService(local, remote map[string]distribution.Descriptor) *proxyTagService { + if local == nil { + local = make(map[string]distribution.Descriptor) + } + if remote == nil { + remote = make(map[string]distribution.Descriptor) + } + return &proxyTagService{ + localTags: &mockTagStore{mapping: local}, + remoteTags: &mockTagStore{mapping: remote}, + } +} + +func TestGet(t *testing.T) { + remoteDesc := distribution.Descriptor{Size: 42} + remoteTag := "remote" + proxyTags := testProxyTagService(map[string]distribution.Descriptor{remoteTag: remoteDesc}, nil) + + ctx := context.Background() + + // Get pre-loaded tag + d, err := proxyTags.Get(ctx, remoteTag) + if err != nil { + t.Fatal(err) + } + + if d != remoteDesc { + t.Fatal("unable to get put tag") + } + + local, err := proxyTags.localTags.Get(ctx, remoteTag) + if err != nil { + t.Fatal("remote tag not pulled into store") + } + + if local != remoteDesc { + t.Fatalf("unexpected descriptor pulled through") + } + + // Manually overwrite remote tag + newRemoteDesc := distribution.Descriptor{Size: 43} + err = proxyTags.remoteTags.Tag(ctx, remoteTag, newRemoteDesc) + if err != nil { + t.Fatal(err) + } + + d, err = proxyTags.Get(ctx, remoteTag) + if err != nil { + t.Fatal(err) + } + + if d != newRemoteDesc { + t.Fatal("unable to get put tag") + } + + _, err = proxyTags.localTags.Get(ctx, remoteTag) + if err != nil { + t.Fatal("remote tag not pulled into store") + } + + // untag, ensure it's removed locally, but present in remote + err = proxyTags.Untag(ctx, remoteTag) + if err != nil { + t.Fatal(err) + } + + _, err = proxyTags.localTags.Get(ctx, remoteTag) + if err == nil { + t.Fatalf("Expected error getting Untag'd tag") + } + + _, err = proxyTags.remoteTags.Get(ctx, remoteTag) + if err != nil { + t.Fatalf("remote tag should not be untagged with proxyTag.Untag") + } + + _, err = proxyTags.Get(ctx, remoteTag) + if err != nil { + t.Fatal("untagged tag should be pulled through") + } + + // Add another tag. Ensure both tags appear in enumerate + err = proxyTags.remoteTags.Tag(ctx, "funtag", distribution.Descriptor{Size: 42}) + if err != nil { + t.Fatal(err) + } + + all, err := proxyTags.All(ctx) + if err != nil { + t.Fatal(err) + } + + if len(all) != 2 { + t.Fatalf("Unexpected tag length returned from All() : %d ", len(all)) + } + + sort.Strings(all) + if all[0] != "funtag" && all[1] != "remote" { + t.Fatalf("Unexpected tags returned from All() : %v ", all) + } +} diff --git a/registry/storage/manifeststore.go b/registry/storage/manifeststore.go index 024c8e4bb..730615926 100644 --- a/registry/storage/manifeststore.go +++ b/registry/storage/manifeststore.go @@ -1,6 +1,7 @@ package storage import ( + "encoding/json" "fmt" "github.com/docker/distribution" @@ -11,20 +12,21 @@ import ( "github.com/docker/libtrust" ) +// manifestStore is a storage driver based store for storing schema1 manifests. type manifestStore struct { repository *repository - revisionStore *revisionStore - tagStore *tagStore + blobStore *linkedBlobStore ctx context.Context + signatures *signatureStore skipDependencyVerification bool } var _ distribution.ManifestService = &manifestStore{} -func (ms *manifestStore) Exists(dgst digest.Digest) (bool, error) { +func (ms *manifestStore) Exists(ctx context.Context, dgst digest.Digest) (bool, error) { context.GetLogger(ms.ctx).Debug("(*manifestStore).Exists") - _, err := ms.revisionStore.blobStore.Stat(ms.ctx, dgst) + _, err := ms.blobStore.Stat(ms.ctx, dgst) if err != nil { if err == distribution.ErrBlobUnknown { return false, nil @@ -36,76 +38,131 @@ func (ms *manifestStore) Exists(dgst digest.Digest) (bool, error) { return true, nil } -func (ms *manifestStore) Get(dgst digest.Digest) (*schema1.SignedManifest, error) { +func (ms *manifestStore) Get(ctx context.Context, dgst digest.Digest, options ...distribution.ManifestServiceOption) (distribution.Manifest, error) { context.GetLogger(ms.ctx).Debug("(*manifestStore).Get") - return ms.revisionStore.get(ms.ctx, dgst) + // Ensure that this revision is available in this repository. + _, err := ms.blobStore.Stat(ctx, dgst) + if err != nil { + if err == distribution.ErrBlobUnknown { + return nil, distribution.ErrManifestUnknownRevision{ + Name: ms.repository.Name(), + Revision: dgst, + } + } + + return nil, err + } + + // TODO(stevvooe): Need to check descriptor from above to ensure that the + // mediatype is as we expect for the manifest store. + + content, err := ms.blobStore.Get(ctx, dgst) + if err != nil { + if err == distribution.ErrBlobUnknown { + return nil, distribution.ErrManifestUnknownRevision{ + Name: ms.repository.Name(), + Revision: dgst, + } + } + + return nil, err + } + + // Fetch the signatures for the manifest + signatures, err := ms.signatures.Get(dgst) + if err != nil { + return nil, err + } + + jsig, err := libtrust.NewJSONSignature(content, signatures...) + if err != nil { + return nil, err + } + + // Extract the pretty JWS + raw, err := jsig.PrettySignature("signatures") + if err != nil { + return nil, err + } + + var sm schema1.SignedManifest + if err := json.Unmarshal(raw, &sm); err != nil { + return nil, err + } + + return &sm, nil } -// SkipLayerVerification allows a manifest to be Put before it's +// SkipLayerVerification allows a manifest to be Put before its // layers are on the filesystem -func SkipLayerVerification(ms distribution.ManifestService) error { - if ms, ok := ms.(*manifestStore); ok { +func SkipLayerVerification() distribution.ManifestServiceOption { + return skipLayerOption{} +} + +type skipLayerOption struct{} + +func (o skipLayerOption) Apply(m distribution.ManifestService) error { + if ms, ok := m.(*manifestStore); ok { ms.skipDependencyVerification = true return nil } return fmt.Errorf("skip layer verification only valid for manifestStore") } -func (ms *manifestStore) Put(manifest *schema1.SignedManifest) error { +func (ms *manifestStore) Put(ctx context.Context, manifest distribution.Manifest, options ...distribution.ManifestServiceOption) (digest.Digest, error) { context.GetLogger(ms.ctx).Debug("(*manifestStore).Put") - if err := ms.verifyManifest(ms.ctx, manifest); err != nil { - return err + sm, ok := manifest.(*schema1.SignedManifest) + if !ok { + return "", fmt.Errorf("non-v1 manifest put to signed manifestStore: %T", manifest) } - // Store the revision of the manifest - revision, err := ms.revisionStore.put(ms.ctx, manifest) + if err := ms.verifyManifest(ms.ctx, *sm); err != nil { + return "", err + } + + mt := schema1.MediaTypeManifest + payload := sm.Canonical + + revision, err := ms.blobStore.Put(ctx, mt, payload) if err != nil { - return err + context.GetLogger(ctx).Errorf("error putting payload into blobstore: %v", err) + return "", err } - // Now, tag the manifest - return ms.tagStore.tag(manifest.Tag, revision.Digest) + // Link the revision into the repository. + if err := ms.blobStore.linkBlob(ctx, revision); err != nil { + return "", err + } + + // Grab each json signature and store them. + signatures, err := sm.Signatures() + if err != nil { + return "", err + } + + if err := ms.signatures.Put(revision.Digest, signatures...); err != nil { + return "", err + } + + return revision.Digest, nil } // Delete removes the revision of the specified manfiest. -func (ms *manifestStore) Delete(dgst digest.Digest) error { +func (ms *manifestStore) Delete(ctx context.Context, dgst digest.Digest) error { context.GetLogger(ms.ctx).Debug("(*manifestStore).Delete") - return ms.revisionStore.delete(ms.ctx, dgst) + return ms.blobStore.Delete(ctx, dgst) } -func (ms *manifestStore) Tags() ([]string, error) { - context.GetLogger(ms.ctx).Debug("(*manifestStore).Tags") - return ms.tagStore.tags() -} - -func (ms *manifestStore) ExistsByTag(tag string) (bool, error) { - context.GetLogger(ms.ctx).Debug("(*manifestStore).ExistsByTag") - return ms.tagStore.exists(tag) -} - -func (ms *manifestStore) GetByTag(tag string, options ...distribution.ManifestServiceOption) (*schema1.SignedManifest, error) { - for _, option := range options { - err := option(ms) - if err != nil { - return nil, err - } - } - - context.GetLogger(ms.ctx).Debug("(*manifestStore).GetByTag") - dgst, err := ms.tagStore.resolve(tag) - if err != nil { - return nil, err - } - - return ms.revisionStore.get(ms.ctx, dgst) +func (ms *manifestStore) Enumerate(ctx context.Context, manifests []distribution.Manifest, last distribution.Manifest) (n int, err error) { + return 0, distribution.ErrUnsupported } // verifyManifest ensures that the manifest content is valid from the // perspective of the registry. It ensures that the signature is valid for the // enclosed payload. As a policy, the registry only tries to store valid -// content, leaving trust policies of that content up to consumers. -func (ms *manifestStore) verifyManifest(ctx context.Context, mnfst *schema1.SignedManifest) error { +// content, leaving trust policies of that content up to consumems. +func (ms *manifestStore) verifyManifest(ctx context.Context, mnfst schema1.SignedManifest) error { var errs distribution.ErrManifestVerification if len(mnfst.Name) > reference.NameTotalLengthMax { @@ -129,7 +186,7 @@ func (ms *manifestStore) verifyManifest(ctx context.Context, mnfst *schema1.Sign len(mnfst.History), len(mnfst.FSLayers))) } - if _, err := schema1.Verify(mnfst); err != nil { + if _, err := schema1.Verify(&mnfst); err != nil { switch err { case libtrust.ErrMissingSignatureKey, libtrust.ErrInvalidJSONContent, libtrust.ErrMissingSignatureKey: errs = append(errs, distribution.ErrManifestUnverified{}) @@ -143,15 +200,15 @@ func (ms *manifestStore) verifyManifest(ctx context.Context, mnfst *schema1.Sign } if !ms.skipDependencyVerification { - for _, fsLayer := range mnfst.FSLayers { - _, err := ms.repository.Blobs(ctx).Stat(ctx, fsLayer.BlobSum) + for _, fsLayer := range mnfst.References() { + _, err := ms.repository.Blobs(ctx).Stat(ctx, fsLayer.Digest) if err != nil { if err != distribution.ErrBlobUnknown { errs = append(errs, err) } - // On error here, we always append unknown blob errors. - errs = append(errs, distribution.ErrManifestBlobUnknown{Digest: fsLayer.BlobSum}) + // On error here, we always append unknown blob erroms. + errs = append(errs, distribution.ErrManifestBlobUnknown{Digest: fsLayer.Digest}) } } } diff --git a/registry/storage/manifeststore_test.go b/registry/storage/manifeststore_test.go index de31b364a..a41feb045 100644 --- a/registry/storage/manifeststore_test.go +++ b/registry/storage/manifeststore_test.go @@ -30,7 +30,8 @@ type manifestStoreTestEnv struct { func newManifestStoreTestEnv(t *testing.T, name, tag string) *manifestStoreTestEnv { ctx := context.Background() driver := inmemory.New() - registry, err := NewRegistry(ctx, driver, BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()), EnableDelete, EnableRedirect) + registry, err := NewRegistry(ctx, driver, BlobDescriptorCacheProvider( + memory.NewInMemoryBlobDescriptorCacheProvider()), EnableDelete, EnableRedirect) if err != nil { t.Fatalf("error creating registry: %v", err) } @@ -58,24 +59,6 @@ func TestManifestStorage(t *testing.T) { t.Fatal(err) } - exists, err := ms.ExistsByTag(env.tag) - if err != nil { - t.Fatalf("unexpected error checking manifest existence: %v", err) - } - - if exists { - t.Fatalf("manifest should not exist") - } - - if _, err := ms.GetByTag(env.tag); true { - switch err.(type) { - case distribution.ErrManifestUnknown: - break - default: - t.Fatalf("expected manifest unknown error: %#v", err) - } - } - m := schema1.Manifest{ Versioned: manifest.Versioned{ SchemaVersion: 1, @@ -114,7 +97,7 @@ func TestManifestStorage(t *testing.T) { t.Fatalf("error signing manifest: %v", err) } - err = ms.Put(sm) + _, err = ms.Put(ctx, sm) if err == nil { t.Fatalf("expected errors putting manifest with full verification") } @@ -150,30 +133,40 @@ func TestManifestStorage(t *testing.T) { } } - if err = ms.Put(sm); err != nil { + var manifestDigest digest.Digest + if manifestDigest, err = ms.Put(ctx, sm); err != nil { t.Fatalf("unexpected error putting manifest: %v", err) } - exists, err = ms.ExistsByTag(env.tag) + exists, err := ms.Exists(ctx, manifestDigest) if err != nil { - t.Fatalf("unexpected error checking manifest existence: %v", err) + t.Fatalf("unexpected error checking manifest existence: %#v", err) } if !exists { t.Fatalf("manifest should exist") } - fetchedManifest, err := ms.GetByTag(env.tag) - + fromStore, err := ms.Get(ctx, manifestDigest) if err != nil { t.Fatalf("unexpected error fetching manifest: %v", err) } + fetchedManifest, ok := fromStore.(*schema1.SignedManifest) + if !ok { + t.Fatalf("unexpected manifest type from signedstore") + } + if !reflect.DeepEqual(fetchedManifest, sm) { t.Fatalf("fetched manifest not equal: %#v != %#v", fetchedManifest, sm) } - fetchedJWS, err := libtrust.ParsePrettySignature(fetchedManifest.Raw, "signatures") + _, pl, err := fetchedManifest.Payload() + if err != nil { + t.Fatalf("error getting payload %#v", err) + } + + fetchedJWS, err := libtrust.ParsePrettySignature(pl, "signatures") if err != nil { t.Fatalf("unexpected error parsing jws: %v", err) } @@ -185,8 +178,9 @@ func TestManifestStorage(t *testing.T) { // Now that we have a payload, take a moment to check that the manifest is // return by the payload digest. + dgst := digest.FromBytes(payload) - exists, err = ms.Exists(dgst) + exists, err = ms.Exists(ctx, dgst) if err != nil { t.Fatalf("error checking manifest existence by digest: %v", err) } @@ -195,7 +189,7 @@ func TestManifestStorage(t *testing.T) { t.Fatalf("manifest %s should exist", dgst) } - fetchedByDigest, err := ms.Get(dgst) + fetchedByDigest, err := ms.Get(ctx, dgst) if err != nil { t.Fatalf("unexpected error fetching manifest by digest: %v", err) } @@ -213,20 +207,6 @@ func TestManifestStorage(t *testing.T) { t.Fatalf("unexpected number of signatures: %d != %d", len(sigs), 1) } - // Grabs the tags and check that this tagged manifest is present - tags, err := ms.Tags() - if err != nil { - t.Fatalf("unexpected error fetching tags: %v", err) - } - - if len(tags) != 1 { - t.Fatalf("unexpected tags returned: %v", tags) - } - - if tags[0] != env.tag { - t.Fatalf("unexpected tag found in tags: %v != %v", tags, []string{env.tag}) - } - // Now, push the same manifest with a different key pk2, err := libtrust.GenerateECP256PrivateKey() if err != nil { @@ -237,8 +217,12 @@ func TestManifestStorage(t *testing.T) { if err != nil { t.Fatalf("unexpected error signing manifest: %v", err) } + _, pl, err = sm2.Payload() + if err != nil { + t.Fatalf("error getting payload %#v", err) + } - jws2, err := libtrust.ParsePrettySignature(sm2.Raw, "signatures") + jws2, err := libtrust.ParsePrettySignature(pl, "signatures") if err != nil { t.Fatalf("error parsing signature: %v", err) } @@ -252,15 +236,20 @@ func TestManifestStorage(t *testing.T) { t.Fatalf("unexpected number of signatures: %d != %d", len(sigs2), 1) } - if err = ms.Put(sm2); err != nil { + if manifestDigest, err = ms.Put(ctx, sm2); err != nil { t.Fatalf("unexpected error putting manifest: %v", err) } - fetched, err := ms.GetByTag(env.tag) + fromStore, err = ms.Get(ctx, manifestDigest) if err != nil { t.Fatalf("unexpected error fetching manifest: %v", err) } + fetched, ok := fromStore.(*schema1.SignedManifest) + if !ok { + t.Fatalf("unexpected type from signed manifeststore : %T", fetched) + } + if _, err := schema1.Verify(fetched); err != nil { t.Fatalf("unexpected error verifying manifest: %v", err) } @@ -276,7 +265,12 @@ func TestManifestStorage(t *testing.T) { t.Fatalf("unexpected error getting expected signatures: %v", err) } - receivedJWS, err := libtrust.ParsePrettySignature(fetched.Raw, "signatures") + _, pl, err = fetched.Payload() + if err != nil { + t.Fatalf("error getting payload %#v", err) + } + + receivedJWS, err := libtrust.ParsePrettySignature(pl, "signatures") if err != nil { t.Fatalf("unexpected error parsing jws: %v", err) } @@ -302,12 +296,12 @@ func TestManifestStorage(t *testing.T) { } // Test deleting manifests - err = ms.Delete(dgst) + err = ms.Delete(ctx, dgst) if err != nil { t.Fatalf("unexpected an error deleting manifest by digest: %v", err) } - exists, err = ms.Exists(dgst) + exists, err = ms.Exists(ctx, dgst) if err != nil { t.Fatalf("Error querying manifest existence") } @@ -315,7 +309,7 @@ func TestManifestStorage(t *testing.T) { t.Errorf("Deleted manifest should not exist") } - deletedManifest, err := ms.Get(dgst) + deletedManifest, err := ms.Get(ctx, dgst) if err == nil { t.Errorf("Unexpected success getting deleted manifest") } @@ -331,12 +325,12 @@ func TestManifestStorage(t *testing.T) { } // Re-upload should restore manifest to a good state - err = ms.Put(sm) + _, err = ms.Put(ctx, sm) if err != nil { t.Errorf("Error re-uploading deleted manifest") } - exists, err = ms.Exists(dgst) + exists, err = ms.Exists(ctx, dgst) if err != nil { t.Fatalf("Error querying manifest existence") } @@ -344,7 +338,7 @@ func TestManifestStorage(t *testing.T) { t.Errorf("Restored manifest should exist") } - deletedManifest, err = ms.Get(dgst) + deletedManifest, err = ms.Get(ctx, dgst) if err != nil { t.Errorf("Unexpected error getting manifest") } @@ -364,7 +358,7 @@ func TestManifestStorage(t *testing.T) { if err != nil { t.Fatal(err) } - err = ms.Delete(dgst) + err = ms.Delete(ctx, dgst) if err == nil { t.Errorf("Unexpected success deleting while disabled") } diff --git a/registry/storage/registry.go b/registry/storage/registry.go index 5ef06d536..c58b91d8a 100644 --- a/registry/storage/registry.go +++ b/registry/storage/registry.go @@ -145,6 +145,15 @@ func (repo *repository) Name() string { return repo.name } +func (repo *repository) Tags(ctx context.Context) distribution.TagService { + tags := &tagStore{ + repository: repo, + blobStore: repo.registry.blobStore, + } + + return tags +} + // Manifests returns an instance of ManifestService. Instantiation is cheap and // may be context sensitive in the future. The instance should be used similar // to a request local. @@ -159,36 +168,31 @@ func (repo *repository) Manifests(ctx context.Context, options ...distribution.M ms := &manifestStore{ ctx: ctx, repository: repo, - revisionStore: &revisionStore{ - ctx: ctx, - repository: repo, - blobStore: &linkedBlobStore{ - ctx: ctx, - blobStore: repo.blobStore, - repository: repo, - deleteEnabled: repo.registry.deleteEnabled, - blobAccessController: &linkedBlobStatter{ - blobStore: repo.blobStore, - repository: repo, - linkPathFns: manifestLinkPathFns, - }, - - // TODO(stevvooe): linkPath limits this blob store to only - // manifests. This instance cannot be used for blob checks. - linkPathFns: manifestLinkPathFns, - resumableDigestEnabled: repo.resumableDigestEnabled, + blobStore: &linkedBlobStore{ + ctx: ctx, + blobStore: repo.blobStore, + repository: repo, + deleteEnabled: repo.registry.deleteEnabled, + blobAccessController: &linkedBlobStatter{ + blobStore: repo.blobStore, + repository: repo, + linkPathFns: manifestLinkPathFns, }, + + // TODO(stevvooe): linkPath limits this blob store to only + // manifests. This instance cannot be used for blob checks. + linkPathFns: manifestLinkPathFns, }, - tagStore: &tagStore{ + signatures: &signatureStore{ ctx: ctx, repository: repo, - blobStore: repo.registry.blobStore, + blobStore: repo.blobStore, }, } // Apply options for _, option := range options { - err := option(ms) + err := option.Apply(ms) if err != nil { return nil, err } @@ -225,11 +229,3 @@ func (repo *repository) Blobs(ctx context.Context) distribution.BlobStore { resumableDigestEnabled: repo.resumableDigestEnabled, } } - -func (repo *repository) Signatures() distribution.SignatureService { - return &signatureStore{ - repository: repo, - blobStore: repo.blobStore, - ctx: repo.ctx, - } -} diff --git a/registry/storage/revisionstore.go b/registry/storage/revisionstore.go deleted file mode 100644 index ed2d5dd3b..000000000 --- a/registry/storage/revisionstore.go +++ /dev/null @@ -1,111 +0,0 @@ -package storage - -import ( - "encoding/json" - - "github.com/docker/distribution" - "github.com/docker/distribution/context" - "github.com/docker/distribution/digest" - "github.com/docker/distribution/manifest/schema1" - "github.com/docker/libtrust" -) - -// revisionStore supports storing and managing manifest revisions. -type revisionStore struct { - repository *repository - blobStore *linkedBlobStore - ctx context.Context -} - -// get retrieves the manifest, keyed by revision digest. -func (rs *revisionStore) get(ctx context.Context, revision digest.Digest) (*schema1.SignedManifest, error) { - // Ensure that this revision is available in this repository. - _, err := rs.blobStore.Stat(ctx, revision) - if err != nil { - if err == distribution.ErrBlobUnknown { - return nil, distribution.ErrManifestUnknownRevision{ - Name: rs.repository.Name(), - Revision: revision, - } - } - - return nil, err - } - - // TODO(stevvooe): Need to check descriptor from above to ensure that the - // mediatype is as we expect for the manifest store. - - content, err := rs.blobStore.Get(ctx, revision) - if err != nil { - if err == distribution.ErrBlobUnknown { - return nil, distribution.ErrManifestUnknownRevision{ - Name: rs.repository.Name(), - Revision: revision, - } - } - - return nil, err - } - - // Fetch the signatures for the manifest - signatures, err := rs.repository.Signatures().Get(revision) - if err != nil { - return nil, err - } - - jsig, err := libtrust.NewJSONSignature(content, signatures...) - if err != nil { - return nil, err - } - - // Extract the pretty JWS - raw, err := jsig.PrettySignature("signatures") - if err != nil { - return nil, err - } - - var sm schema1.SignedManifest - if err := json.Unmarshal(raw, &sm); err != nil { - return nil, err - } - - return &sm, nil -} - -// put stores the manifest in the repository, if not already present. Any -// updated signatures will be stored, as well. -func (rs *revisionStore) put(ctx context.Context, sm *schema1.SignedManifest) (distribution.Descriptor, error) { - // Resolve the payload in the manifest. - payload, err := sm.Payload() - if err != nil { - return distribution.Descriptor{}, err - } - - // Digest and store the manifest payload in the blob store. - revision, err := rs.blobStore.Put(ctx, schema1.ManifestMediaType, payload) - if err != nil { - context.GetLogger(ctx).Errorf("error putting payload into blobstore: %v", err) - return distribution.Descriptor{}, err - } - - // Link the revision into the repository. - if err := rs.blobStore.linkBlob(ctx, revision); err != nil { - return distribution.Descriptor{}, err - } - - // Grab each json signature and store them. - signatures, err := sm.Signatures() - if err != nil { - return distribution.Descriptor{}, err - } - - if err := rs.repository.Signatures().Put(revision.Digest, signatures...); err != nil { - return distribution.Descriptor{}, err - } - - return revision, nil -} - -func (rs *revisionStore) delete(ctx context.Context, revision digest.Digest) error { - return rs.blobStore.Delete(ctx, revision) -} diff --git a/registry/storage/signaturestore.go b/registry/storage/signaturestore.go index f5888f64a..ede4e0e2a 100644 --- a/registry/storage/signaturestore.go +++ b/registry/storage/signaturestore.go @@ -4,7 +4,6 @@ import ( "path" "sync" - "github.com/docker/distribution" "github.com/docker/distribution/context" "github.com/docker/distribution/digest" ) @@ -15,16 +14,6 @@ type signatureStore struct { ctx context.Context } -func newSignatureStore(ctx context.Context, repo *repository, blobStore *blobStore) *signatureStore { - return &signatureStore{ - ctx: ctx, - repository: repo, - blobStore: blobStore, - } -} - -var _ distribution.SignatureService = &signatureStore{} - func (s *signatureStore) Get(dgst digest.Digest) ([][]byte, error) { signaturesPath, err := pathFor(manifestSignaturesPathSpec{ name: s.repository.Name(), diff --git a/registry/storage/tagstore.go b/registry/storage/tagstore.go index aec952860..167c7fa08 100644 --- a/registry/storage/tagstore.go +++ b/registry/storage/tagstore.go @@ -9,37 +9,41 @@ import ( storagedriver "github.com/docker/distribution/registry/storage/driver" ) +var _ distribution.TagService = &tagStore{} + // tagStore provides methods to manage manifest tags in a backend storage driver. +// This implementation uses the same on-disk layout as the (now deleted) tag +// store. This provides backward compatibility with current registry deployments +// which only makes use of the Digest field of the returned distribution.Descriptor +// but does not enable full roundtripping of Descriptor objects type tagStore struct { repository *repository blobStore *blobStore - ctx context.Context } -// tags lists the manifest tags for the specified repository. -func (ts *tagStore) tags() ([]string, error) { - p, err := pathFor(manifestTagPathSpec{ +// All returns all tags +func (ts *tagStore) All(ctx context.Context) ([]string, error) { + var tags []string + + pathSpec, err := pathFor(manifestTagPathSpec{ name: ts.repository.Name(), }) - if err != nil { - return nil, err + return tags, err } - var tags []string - entries, err := ts.blobStore.driver.List(ts.ctx, p) + entries, err := ts.blobStore.driver.List(ctx, pathSpec) if err != nil { switch err := err.(type) { case storagedriver.PathNotFoundError: - return nil, distribution.ErrRepositoryUnknown{Name: ts.repository.Name()} + return tags, distribution.ErrRepositoryUnknown{Name: ts.repository.Name()} default: - return nil, err + return tags, err } } for _, entry := range entries { _, filename := path.Split(entry) - tags = append(tags, filename) } @@ -47,7 +51,7 @@ func (ts *tagStore) tags() ([]string, error) { } // exists returns true if the specified manifest tag exists in the repository. -func (ts *tagStore) exists(tag string) (bool, error) { +func (ts *tagStore) exists(ctx context.Context, tag string) (bool, error) { tagPath, err := pathFor(manifestTagCurrentPathSpec{ name: ts.repository.Name(), tag: tag, @@ -57,7 +61,7 @@ func (ts *tagStore) exists(tag string) (bool, error) { return false, err } - exists, err := exists(ts.ctx, ts.blobStore.driver, tagPath) + exists, err := exists(ctx, ts.blobStore.driver, tagPath) if err != nil { return false, err } @@ -65,9 +69,9 @@ func (ts *tagStore) exists(tag string) (bool, error) { return exists, nil } -// tag tags the digest with the given tag, updating the the store to point at +// Tag tags the digest with the given tag, updating the the store to point at // the current tag. The digest must point to a manifest. -func (ts *tagStore) tag(tag string, revision digest.Digest) error { +func (ts *tagStore) Tag(ctx context.Context, tag string, desc distribution.Descriptor) error { currentPath, err := pathFor(manifestTagCurrentPathSpec{ name: ts.repository.Name(), tag: tag, @@ -77,43 +81,44 @@ func (ts *tagStore) tag(tag string, revision digest.Digest) error { return err } - nbs := ts.linkedBlobStore(ts.ctx, tag) + lbs := ts.linkedBlobStore(ctx, tag) + // Link into the index - if err := nbs.linkBlob(ts.ctx, distribution.Descriptor{Digest: revision}); err != nil { + if err := lbs.linkBlob(ctx, desc); err != nil { return err } // Overwrite the current link - return ts.blobStore.link(ts.ctx, currentPath, revision) + return ts.blobStore.link(ctx, currentPath, desc.Digest) } // resolve the current revision for name and tag. -func (ts *tagStore) resolve(tag string) (digest.Digest, error) { +func (ts *tagStore) Get(ctx context.Context, tag string) (distribution.Descriptor, error) { currentPath, err := pathFor(manifestTagCurrentPathSpec{ name: ts.repository.Name(), tag: tag, }) if err != nil { - return "", err + return distribution.Descriptor{}, err } - revision, err := ts.blobStore.readlink(ts.ctx, currentPath) + revision, err := ts.blobStore.readlink(ctx, currentPath) if err != nil { switch err.(type) { case storagedriver.PathNotFoundError: - return "", distribution.ErrManifestUnknown{Name: ts.repository.Name(), Tag: tag} + return distribution.Descriptor{}, distribution.ErrTagUnknown{Tag: tag} } - return "", err + return distribution.Descriptor{}, err } - return revision, nil + return distribution.Descriptor{Digest: revision}, nil } // delete removes the tag from repository, including the history of all // revisions that have the specified tag. -func (ts *tagStore) delete(tag string) error { +func (ts *tagStore) Untag(ctx context.Context, tag string) error { tagPath, err := pathFor(manifestTagPathSpec{ name: ts.repository.Name(), tag: tag, @@ -123,7 +128,7 @@ func (ts *tagStore) delete(tag string) error { return err } - return ts.blobStore.driver.Delete(ts.ctx, tagPath) + return ts.blobStore.driver.Delete(ctx, tagPath) } // linkedBlobStore returns the linkedBlobStore for the named tag, allowing one @@ -145,3 +150,10 @@ func (ts *tagStore) linkedBlobStore(ctx context.Context, tag string) *linkedBlob }}, } } + +// Lookup recovers a list of tags which refer to this digest. When a manifest is deleted by +// digest, tag entries which point to it need to be recovered to avoid dangling tags. +func (ts *tagStore) Lookup(ctx context.Context, digest distribution.Descriptor) ([]string, error) { + // An efficient implementation of this will require changes to the S3 driver. + return make([]string, 0), nil +} diff --git a/registry/storage/tagstore_test.go b/registry/storage/tagstore_test.go new file mode 100644 index 000000000..79660199e --- /dev/null +++ b/registry/storage/tagstore_test.go @@ -0,0 +1,150 @@ +package storage + +import ( + "testing" + + "github.com/docker/distribution" + "github.com/docker/distribution/context" + "github.com/docker/distribution/registry/storage/driver/inmemory" +) + +type tagsTestEnv struct { + ts distribution.TagService + ctx context.Context +} + +func testTagStore(t *testing.T) *tagsTestEnv { + ctx := context.Background() + d := inmemory.New() + reg, err := NewRegistry(ctx, d) + if err != nil { + t.Fatal(err) + } + + repo, err := reg.Repository(ctx, "a/b") + if err != nil { + t.Fatal(err) + } + + return &tagsTestEnv{ + ctx: ctx, + ts: repo.Tags(ctx), + } +} + +func TestTagStoreTag(t *testing.T) { + env := testTagStore(t) + tags := env.ts + ctx := env.ctx + + d := distribution.Descriptor{} + err := tags.Tag(ctx, "latest", d) + if err == nil { + t.Errorf("unexpected error putting malformed descriptor : %s", err) + } + + d.Digest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + err = tags.Tag(ctx, "latest", d) + if err != nil { + t.Error(err) + } + + d1, err := tags.Get(ctx, "latest") + if err != nil { + t.Error(err) + } + + if d1.Digest != d.Digest { + t.Error("put and get digest differ") + } + + // Overwrite existing + d.Digest = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + err = tags.Tag(ctx, "latest", d) + if err != nil { + t.Error(err) + } + + d1, err = tags.Get(ctx, "latest") + if err != nil { + t.Error(err) + } + + if d1.Digest != d.Digest { + t.Error("put and get digest differ") + } +} + +func TestTagStoreUnTag(t *testing.T) { + env := testTagStore(t) + tags := env.ts + ctx := env.ctx + desc := distribution.Descriptor{Digest: "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"} + + err := tags.Untag(ctx, "latest") + if err == nil { + t.Errorf("Expected error untagging non-existant tag") + } + + err = tags.Tag(ctx, "latest", desc) + if err != nil { + t.Error(err) + } + + err = tags.Untag(ctx, "latest") + if err != nil { + t.Error(err) + } + + _, err = tags.Get(ctx, "latest") + if err == nil { + t.Error("Expected error getting untagged tag") + } +} + +func TestTagAll(t *testing.T) { + env := testTagStore(t) + tagStore := env.ts + ctx := env.ctx + + alpha := "abcdefghijklmnopqrstuvwxyz" + for i := 0; i < len(alpha); i++ { + tag := alpha[i] + desc := distribution.Descriptor{Digest: "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"} + err := tagStore.Tag(ctx, string(tag), desc) + if err != nil { + t.Error(err) + } + } + + all, err := tagStore.All(ctx) + if err != nil { + t.Error(err) + } + if len(all) != len(alpha) { + t.Errorf("Unexpected count returned from enumerate") + } + + for i, c := range all { + if c != string(alpha[i]) { + t.Errorf("unexpected tag in enumerate %s", c) + } + } + + removed := "a" + err = tagStore.Untag(ctx, removed) + if err != nil { + t.Error(err) + } + + all, err = tagStore.All(ctx) + if err != nil { + t.Error(err) + } + for _, tag := range all { + if tag == removed { + t.Errorf("unexpected tag in enumerate %s", removed) + } + } + +} diff --git a/tags.go b/tags.go new file mode 100644 index 000000000..503056596 --- /dev/null +++ b/tags.go @@ -0,0 +1,27 @@ +package distribution + +import ( + "github.com/docker/distribution/context" +) + +// TagService provides access to information about tagged objects. +type TagService interface { + // Get retrieves the descriptor identified by the tag. Some + // implementations may differentiate between "trusted" tags and + // "untrusted" tags. If a tag is "untrusted", the mapping will be returned + // as an ErrTagUntrusted error, with the target descriptor. + Get(ctx context.Context, tag string) (Descriptor, error) + + // Tag associates the tag with the provided descriptor, updating the + // current association, if needed. + Tag(ctx context.Context, tag string, desc Descriptor) error + + // Untag removes the given tag association + Untag(ctx context.Context, tag string) error + + // All returns the set of tags managed by this tag service + All(ctx context.Context) ([]string, error) + + // Lookup returns the set of tags referencing the given digest. + Lookup(ctx context.Context, digest Descriptor) ([]string, error) +}