r.loginov/distribution
1
0
Fork 0
forked from TrueCloudLab/distribution
Code Pull requests Activity

Merge pull request #1268 from RichardScothern/manifest-refactor-impl

Browse source 
Implementation of the Manifest Service API refactor.
This commit is contained in:
Richard Scothern 2015-12-17 17:32:55 -08:00
commit 67d3675d55
36 changed files with 1681 additions and 823 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
docs/spec/api.md
View file

@ -402,6 +402,28 @@ for details):
The client should verify the returned manifest signature for authenticity
before fetching layers.
##### Existing Manifests
The image manifest can be checked for existence with the following url:
```
HEAD /v2/<name>/manifests/<reference>
```
The `name` and `reference` parameter identify the image and are required. The
reference may include a tag or digest.
A `404 Not Found` response will be returned if the image is unknown to the
registry. If the image exists and the response is successful the response will
be as follows:
```
200 OK
Content-Length: <length of manifest>
Docker-Content-Digest: <digest>
```
#### Pulling a Layer
Layers are stored in the blob portion of the registry, keyed by digest.

22
docs/spec/api.md.tmpl
View file

@ -402,6 +402,28 @@ for details):
The client should verify the returned manifest signature for authenticity
before fetching layers.
##### Existing Manifests
The image manifest can be checked for existence with the following url:
```
HEAD /v2/<name>/manifests/<reference>
```
The `name` and `reference` parameter identify the image and are required. The
reference may include a tag or digest.
A `404 Not Found` response will be returned if the image is unknown to the
registry. If the image exists and the response is successful the response will
be as follows:
```
200 OK
Content-Length: <length of manifest>
Docker-Content-Digest: <digest>
```
#### Pulling a Layer
Layers are stored in the blob portion of the registry, keyed by digest.

9
errors.go
View file

@ -16,6 +16,15 @@ var ErrManifestNotModified = errors.New("manifest not modified")
// performed
var ErrUnsupported = errors.New("operation unsupported")
// ErrTagUnknown is returned if the given tag is not known by the tag service
type ErrTagUnknown struct {
Tag string
}
func (err ErrTagUnknown) Error() string {
return fmt.Sprintf("unknown tag=%s", err.Tag)
}
// ErrRepositoryUnknown is returned if the named repository is not known by
// the registry.
type ErrRepositoryUnknown struct {

91
manifest/schema1/builder.go Normal file
View file

@ -0,0 +1,91 @@
package schema1
import (
"fmt"
"errors"
"github.com/docker/distribution"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/manifest"
"github.com/docker/libtrust"
)
// ManifestBuilder is a type for constructing manifests
type manifestBuilder struct {
Manifest
pk libtrust.PrivateKey
}
// NewManifestBuilder is used to build new manifests for the current schema
// version.
func NewManifestBuilder(pk libtrust.PrivateKey, name, tag, architecture string) distribution.ManifestBuilder {
return &manifestBuilder{
Manifest: Manifest{
Versioned: manifest.Versioned{
SchemaVersion: 1,
},
Name: name,
Tag: tag,
Architecture: architecture,
},
pk: pk,
}
}
// Build produces a final manifest from the given references
func (mb *manifestBuilder) Build() (distribution.Manifest, error) {
m := mb.Manifest
if len(m.FSLayers) == 0 {
return nil, errors.New("cannot build manifest with zero layers or history")
}
m.FSLayers = make([]FSLayer, len(mb.Manifest.FSLayers))
m.History = make([]History, len(mb.Manifest.History))
copy(m.FSLayers, mb.Manifest.FSLayers)
copy(m.History, mb.Manifest.History)
return Sign(&m, mb.pk)
}
// AppendReference adds a reference to the current manifestBuilder
func (mb *manifestBuilder) AppendReference(d distribution.Describable) error {
r, ok := d.(Reference)
if !ok {
return fmt.Errorf("Unable to add non-reference type to v1 builder")
}
// Entries need to be prepended
mb.Manifest.FSLayers = append([]FSLayer{{BlobSum: r.Digest}}, mb.Manifest.FSLayers...)
mb.Manifest.History = append([]History{r.History}, mb.Manifest.History...)
return nil
}
// References returns the current references added to this builder
func (mb *manifestBuilder) References() []distribution.Descriptor {
refs := make([]distribution.Descriptor, len(mb.Manifest.FSLayers))
for i := range mb.Manifest.FSLayers {
layerDigest := mb.Manifest.FSLayers[i].BlobSum
history := mb.Manifest.History[i]
ref := Reference{layerDigest, 0, history}
refs[i] = ref.Descriptor()
}
return refs
}
// Reference describes a manifest v2, schema version 1 dependency.
// An FSLayer associated with a history entry.
type Reference struct {
Digest digest.Digest
Size int64 // if we know it, set it for the descriptor.
History History
}
// Descriptor describes a reference
func (r Reference) Descriptor() distribution.Descriptor {
return distribution.Descriptor{
MediaType: MediaTypeManifestLayer,
Digest: r.Digest,
Size: r.Size,
}
}

97
manifest/schema1/builder_test.go Normal file
View file

@ -0,0 +1,97 @@
package schema1
import (
"testing"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/manifest"
"github.com/docker/libtrust"
)
func makeSignedManifest(t *testing.T, pk libtrust.PrivateKey, refs []Reference) *SignedManifest {
u := &Manifest{
Versioned: manifest.Versioned{
SchemaVersion: 1,
},
Name: "foo/bar",
Tag: "latest",
Architecture: "amd64",
}
for i := len(refs) - 1; i >= 0; i-- {
u.FSLayers = append(u.FSLayers, FSLayer{
BlobSum: refs[i].Digest,
})
u.History = append(u.History, History{
V1Compatibility: refs[i].History.V1Compatibility,
})
}
signedManifest, err := Sign(u, pk)
if err != nil {
t.Fatalf("unexpected error signing manifest: %v", err)
}
return signedManifest
}
func TestBuilder(t *testing.T) {
pk, err := libtrust.GenerateECP256PrivateKey()
if err != nil {
t.Fatalf("unexpected error generating private key: %v", err)
}
r1 := Reference{
Digest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
Size: 1,
History: History{V1Compatibility: "{\"a\" : 1 }"},
}
r2 := Reference{
Digest: "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
Size: 2,
History: History{V1Compatibility: "{\"\a\" : 2 }"},
}
handCrafted := makeSignedManifest(t, pk, []Reference{r1, r2})
b := NewManifestBuilder(pk, handCrafted.Manifest.Name, handCrafted.Manifest.Tag, handCrafted.Manifest.Architecture)
_, err = b.Build()
if err == nil {
t.Fatal("Expected error building zero length manifest")
}
err = b.AppendReference(r1)
if err != nil {
t.Fatal(err)
}
err = b.AppendReference(r2)
if err != nil {
t.Fatal(err)
}
refs := b.References()
if len(refs) != 2 {
t.Fatalf("Unexpected reference count : %d != %d", 2, len(refs))
}
// Ensure ordering
if refs[0].Digest != r2.Digest {
t.Fatalf("Unexpected reference : %v", refs[0])
}
m, err := b.Build()
if err != nil {
t.Fatal(err)
}
built, ok := m.(*SignedManifest)
if !ok {
t.Fatalf("unexpected type from Build() : %T", built)
}
d1 := digest.FromBytes(built.Canonical)
d2 := digest.FromBytes(handCrafted.Canonical)
if d1 != d2 {
t.Errorf("mismatching canonical JSON")
}
}

148
manifest/schema1/manifest.go
View file

@ -2,20 +2,22 @@ package schema1
import (
"encoding/json"
"fmt"
"github.com/docker/distribution"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/manifest"
"github.com/docker/libtrust"
)
// TODO(stevvooe): When we rev the manifest format, the contents of this
// package should be moved to manifest/v1.
const (
// ManifestMediaType specifies the mediaType for the current version. Note
// that for schema version 1, the the media is optionally
// "application/json".
ManifestMediaType = "application/vnd.docker.distribution.manifest.v1+json"
// MediaTypeManifest specifies the mediaType for the current version. Note
// that for schema version 1, the the media is optionally "application/json".
MediaTypeManifest = "application/vnd.docker.distribution.manifest.v1+json"
// MediaTypeSignedManifest specifies the mediatype for current SignedManifest version
MediaTypeSignedManifest = "application/vnd.docker.distribution.manifest.v1+prettyjws"
// MediaTypeManifestLayer specifies the media type for manifest layers
MediaTypeManifestLayer = "application/vnd.docker.container.image.rootfs.diff+x-gtar"
)
var (
@ -26,6 +28,47 @@ var (
}
)
func init() {
schema1Func := func(b []byte) (distribution.Manifest, distribution.Descriptor, error) {
sm := new(SignedManifest)
err := sm.UnmarshalJSON(b)
if err != nil {
return nil, distribution.Descriptor{}, err
}
desc := distribution.Descriptor{
Digest: digest.FromBytes(sm.Canonical),
Size: int64(len(sm.Canonical)),
MediaType: MediaTypeManifest,
}
return sm, desc, err
}
err := distribution.RegisterManifestSchema(MediaTypeManifest, schema1Func)
if err != nil {
panic(fmt.Sprintf("Unable to register manifest: %s", err))
}
err = distribution.RegisterManifestSchema("", schema1Func)
if err != nil {
panic(fmt.Sprintf("Unable to register manifest: %s", err))
}
err = distribution.RegisterManifestSchema("application/json; charset=utf-8", schema1Func)
if err != nil {
panic(fmt.Sprintf("Unable to register manifest: %s", err))
}
}
// FSLayer is a container struct for BlobSums defined in an image manifest
type FSLayer struct {
// BlobSum is the tarsum of the referenced filesystem image layer
BlobSum digest.Digest `json:"blobSum"`
}
// History stores unstructured v1 compatibility information
type History struct {
// V1Compatibility is the raw v1 compatibility information
V1Compatibility string `json:"v1Compatibility"`
}
// Manifest provides the base accessible fields for working with V2 image
// format in the registry.
type Manifest struct {
@ -49,59 +92,64 @@ type Manifest struct {
}
// SignedManifest provides an envelope for a signed image manifest, including
// the format sensitive raw bytes. It contains fields to
// the format sensitive raw bytes.
type SignedManifest struct {
Manifest
// Raw is the byte representation of the ImageManifest, used for signature
// verification. The value of Raw must be used directly during
// serialization, or the signature check will fail. The manifest byte
// Canonical is the canonical byte representation of the ImageManifest,
// without any attached signatures. The manifest byte
// representation cannot change or it will have to be re-signed.
Raw []byte `json:"-"`
Canonical []byte `json:"-"`
// all contains the byte representation of the Manifest including signatures
// and is retuend by Payload()
all []byte
}
// UnmarshalJSON populates a new ImageManifest struct from JSON data.
// UnmarshalJSON populates a new SignedManifest struct from JSON data.
func (sm *SignedManifest) UnmarshalJSON(b []byte) error {
sm.Raw = make([]byte, len(b), len(b))
copy(sm.Raw, b)
sm.all = make([]byte, len(b), len(b))
// store manifest and signatures in all
copy(sm.all, b)
p, err := sm.Payload()
jsig, err := libtrust.ParsePrettySignature(b, "signatures")
if err != nil {
return err
}
// Resolve the payload in the manifest.
bytes, err := jsig.Payload()
if err != nil {
return err
}
// sm.Canonical stores the canonical manifest JSON
sm.Canonical = make([]byte, len(bytes), len(bytes))
copy(sm.Canonical, bytes)
// Unmarshal canonical JSON into Manifest object
var manifest Manifest
if err := json.Unmarshal(p, &manifest); err != nil {
if err := json.Unmarshal(sm.Canonical, &manifest); err != nil {
return err
}
sm.Manifest = manifest
return nil
}
// Payload returns the raw, signed content of the signed manifest. The
// contents can be used to calculate the content identifier.
func (sm *SignedManifest) Payload() ([]byte, error) {
jsig, err := libtrust.ParsePrettySignature(sm.Raw, "signatures")
if err != nil {
return nil, err
// References returnes the descriptors of this manifests references
func (sm SignedManifest) References() []distribution.Descriptor {
dependencies := make([]distribution.Descriptor, len(sm.FSLayers))
for i, fsLayer := range sm.FSLayers {
dependencies[i] = distribution.Descriptor{
MediaType: "application/vnd.docker.container.image.rootfs.diff+x-gtar",
Digest: fsLayer.BlobSum,
}
}
// Resolve the payload in the manifest.
return jsig.Payload()
}
return dependencies
// Signatures returns the signatures as provided by
// (*libtrust.JSONSignature).Signatures. The byte slices are opaque jws
// signatures.
func (sm *SignedManifest) Signatures() ([][]byte, error) {
jsig, err := libtrust.ParsePrettySignature(sm.Raw, "signatures")
if err != nil {
return nil, err
}
// Resolve the payload in the manifest.
return jsig.Signatures()
}
// MarshalJSON returns the contents of raw. If Raw is nil, marshals the inner
@ -109,22 +157,28 @@ func (sm *SignedManifest) Signatures() ([][]byte, error) {
// use Raw directly, since the the content produced by json.Marshal will be
// compacted and will fail signature checks.
func (sm *SignedManifest) MarshalJSON() ([]byte, error) {
if len(sm.Raw) > 0 {
return sm.Raw, nil
if len(sm.all) > 0 {
return sm.all, nil
}
// If the raw data is not available, just dump the inner content.
return json.Marshal(&sm.Manifest)
}
// FSLayer is a container struct for BlobSums defined in an image manifest
type FSLayer struct {
// BlobSum is the digest of the referenced filesystem image layer
BlobSum digest.Digest `json:"blobSum"`
// Payload returns the signed content of the signed manifest.
func (sm SignedManifest) Payload() (string, []byte, error) {
return MediaTypeManifest, sm.all, nil
}
// History stores unstructured v1 compatibility information
type History struct {
// V1Compatibility is the raw v1 compatibility information
V1Compatibility string `json:"v1Compatibility"`
// Signatures returns the signatures as provided by
// (*libtrust.JSONSignature).Signatures. The byte slices are opaque jws
// signatures.
func (sm *SignedManifest) Signatures() ([][]byte, error) {
jsig, err := libtrust.ParsePrettySignature(sm.all, "signatures")
if err != nil {
return nil, err
}
// Resolve the payload in the manifest.
return jsig.Signatures()
}

8
manifest/schema1/manifest_test.go
View file

@ -19,15 +19,15 @@ type testEnv struct {
func TestManifestMarshaling(t *testing.T) {
env := genEnv(t)
// Check that the Raw field is the same as json.MarshalIndent with these
// Check that the all field is the same as json.MarshalIndent with these
// parameters.
p, err := json.MarshalIndent(env.signed, "", " ")
if err != nil {
t.Fatalf("error marshaling manifest: %v", err)
}
if !bytes.Equal(p, env.signed.Raw) {
t.Fatalf("manifest bytes not equal: %q != %q", string(env.signed.Raw), string(p))
if !bytes.Equal(p, env.signed.all) {
t.Fatalf("manifest bytes not equal: %q != %q", string(env.signed.all), string(p))
}
}
@ -35,7 +35,7 @@ func TestManifestUnmarshaling(t *testing.T) {
env := genEnv(t)
var signed SignedManifest
if err := json.Unmarshal(env.signed.Raw, &signed); err != nil {
if err := json.Unmarshal(env.signed.all, &signed); err != nil {
t.Fatalf("error unmarshaling signed manifest: %v", err)
}

10
manifest/schema1/sign.go
View file

@ -31,8 +31,9 @@ func Sign(m *Manifest, pk libtrust.PrivateKey) (*SignedManifest, error) {
}
return &SignedManifest{
Manifest: *m,
Raw: pretty,
Manifest: *m,
all: pretty,
Canonical: p,
}, nil
}
@ -60,7 +61,8 @@ func SignWithChain(m *Manifest, key libtrust.PrivateKey, chain []*x509.Certifica
}
return &SignedManifest{
Manifest: *m,
Raw: pretty,
Manifest: *m,
all: pretty,
Canonical: p,
}, nil
}

4
manifest/schema1/verify.go
View file

@ -10,7 +10,7 @@ import (
// Verify verifies the signature of the signed manifest returning the public
// keys used during signing.
func Verify(sm *SignedManifest) ([]libtrust.PublicKey, error) {
js, err := libtrust.ParsePrettySignature(sm.Raw, "signatures")
js, err := libtrust.ParsePrettySignature(sm.all, "signatures")
if err != nil {
logrus.WithField("err", err).Debugf("(*SignedManifest).Verify")
return nil, err
@ -23,7 +23,7 @@ func Verify(sm *SignedManifest) ([]libtrust.PublicKey, error) {
// certificate pool returning the list of verified chains. Signatures without
// an x509 chain are not checked.
func VerifyChains(sm *SignedManifest, ca *x509.CertPool) ([][]*x509.Certificate, error) {
js, err := libtrust.ParsePrettySignature(sm.Raw, "signatures")
js, err := libtrust.ParsePrettySignature(sm.all, "signatures")
if err != nil {
return nil, err
}

100
manifests.go Normal file
View file

@ -0,0 +1,100 @@
package distribution
import (
"fmt"
"github.com/docker/distribution/context"
"github.com/docker/distribution/digest"
)
// Manifest represents a registry object specifying a set of
// references and an optional target
type Manifest interface {
// References returns a list of objects which make up this manifest.
// The references are strictly ordered from base to head. A reference
// is anything which can be represented by a distribution.Descriptor
References() []Descriptor
// Payload provides the serialized format of the manifest, in addition to
// the mediatype.
Payload() (mediatype string, payload []byte, err error)
}
// ManifestBuilder creates a manifest allowing one to include dependencies.
// Instances can be obtained from a version-specific manifest package. Manifest
// specific data is passed into the function which creates the builder.
type ManifestBuilder interface {
// Build creates the manifest from his builder.
Build() (Manifest, error)
// References returns a list of objects which have been added to this
// builder. The dependencies are returned in the order they were added,
// which should be from base to head.
References() []Descriptor
// AppendReference includes the given object in the manifest after any
// existing dependencies. If the add fails, such as when adding an
// unsupported dependency, an error may be returned.
AppendReference(dependency Describable) error
}
// ManifestService describes operations on image manifests.
type ManifestService interface {
// Exists returns true if the manifest exists.
Exists(ctx context.Context, dgst digest.Digest) (bool, error)
// Get retrieves the manifest specified by the given digest
Get(ctx context.Context, dgst digest.Digest, options ...ManifestServiceOption) (Manifest, error)
// Put creates or updates the given manifest returning the manifest digest
Put(ctx context.Context, manifest Manifest, options ...ManifestServiceOption) (digest.Digest, error)
// Delete removes the manifest specified by the given digest. Deleting
// a manifest that doesn't exist will return ErrManifestNotFound
Delete(ctx context.Context, dgst digest.Digest) error
// Enumerate fills 'manifests' with the manifests in this service up
// to the size of 'manifests' and returns 'n' for the number of entries
// which were filled. 'last' contains an offset in the manifest set
// and can be used to resume iteration.
//Enumerate(ctx context.Context, manifests []Manifest, last Manifest) (n int, err error)
}
// Describable is an interface for descriptors
type Describable interface {
Descriptor() Descriptor
}
// ManifestMediaTypes returns the supported media types for manifests.
func ManifestMediaTypes() (mediaTypes []string) {
for t := range mappings {
mediaTypes = append(mediaTypes, t)
}
return
}
// UnmarshalFunc implements manifest unmarshalling a given MediaType
type UnmarshalFunc func([]byte) (Manifest, Descriptor, error)
var mappings = make(map[string]UnmarshalFunc, 0)
// UnmarshalManifest looks up manifest unmarshall functions based on
// MediaType
func UnmarshalManifest(mediatype string, p []byte) (Manifest, Descriptor, error) {
unmarshalFunc, ok := mappings[mediatype]
if !ok {
return nil, Descriptor{}, fmt.Errorf("unsupported manifest mediatype: %s", mediatype)
}
return unmarshalFunc(p)
}
// RegisterManifestSchema registers an UnmarshalFunc for a given schema type. This
// should be called from specific
func RegisterManifestSchema(mediatype string, u UnmarshalFunc) error {
if _, ok := mappings[mediatype]; ok {
return fmt.Errorf("manifest mediatype registration would overwrite existing: %s", mediatype)
}
mappings[mediatype] = u
return nil
}

17
notifications/bridge.go
View file

@ -7,7 +7,6 @@ import (
"github.com/docker/distribution"
"github.com/docker/distribution/context"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/uuid"
)
@ -53,15 +52,15 @@ func NewRequestRecord(id string, r *http.Request) RequestRecord {
}
}
func (b *bridge) ManifestPushed(repo string, sm *schema1.SignedManifest) error {
func (b *bridge) ManifestPushed(repo string, sm distribution.Manifest) error {
return b.createManifestEventAndWrite(EventActionPush, repo, sm)
}
func (b *bridge) ManifestPulled(repo string, sm *schema1.SignedManifest) error {
func (b *bridge) ManifestPulled(repo string, sm distribution.Manifest) error {
return b.createManifestEventAndWrite(EventActionPull, repo, sm)
}
func (b *bridge) ManifestDeleted(repo string, sm *schema1.SignedManifest) error {
func (b *bridge) ManifestDeleted(repo string, sm distribution.Manifest) error {
return b.createManifestEventAndWrite(EventActionDelete, repo, sm)
}
@ -77,7 +76,7 @@ func (b *bridge) BlobDeleted(repo string, desc distribution.Descriptor) error {
return b.createBlobEventAndWrite(EventActionDelete, repo, desc)
}
func (b *bridge) createManifestEventAndWrite(action string, repo string, sm *schema1.SignedManifest) error {
func (b *bridge) createManifestEventAndWrite(action string, repo string, sm distribution.Manifest) error {
manifestEvent, err := b.createManifestEvent(action, repo, sm)
if err != nil {
return err
@ -86,21 +85,21 @@ func (b *bridge) createManifestEventAndWrite(action string, repo string, sm *sch
return b.sink.Write(*manifestEvent)
}
func (b *bridge) createManifestEvent(action string, repo string, sm *schema1.SignedManifest) (*Event, error) {
func (b *bridge) createManifestEvent(action string, repo string, sm distribution.Manifest) (*Event, error) {
event := b.createEvent(action)
event.Target.MediaType = schema1.ManifestMediaType
event.Target.Repository = repo
p, err := sm.Payload()
mt, p, err := sm.Payload()
if err != nil {
return nil, err
}
event.Target.MediaType = mt
event.Target.Length = int64(len(p))
event.Target.Size = int64(len(p))
event.Target.Digest = digest.FromBytes(p)
event.Target.URL, err = b.ub.BuildManifestURL(sm.Name, event.Target.Digest.String())
event.Target.URL, err = b.ub.BuildManifestURL(repo, event.Target.Digest.String())
if err != nil {
return nil, err
}

4
notifications/bridge_test.go
View file

@ -85,7 +85,7 @@ func createTestEnv(t *testing.T, fn testSinkFn) Listener {
t.Fatalf("error signing manifest: %v", err)
}
payload, err = sm.Payload()
_, payload, err = sm.Payload()
if err != nil {
t.Fatalf("error getting manifest payload: %v", err)
}
@ -109,7 +109,7 @@ func checkCommonManifest(t *testing.T, action string, events ...Event) {
}
if event.Target.URL != u {
t.Fatalf("incorrect url passed: %q != %q", event.Target.URL, u)
t.Fatalf("incorrect url passed: \n%q != \n%q", event.Target.URL, u)
}
}

2
notifications/event_test.go
View file

@ -120,7 +120,7 @@ func TestEventEnvelopeJSONFormat(t *testing.T) {
manifestPush.Target.Digest = "sha256:0123456789abcdef0"
manifestPush.Target.Length = 1
manifestPush.Target.Size = 1
manifestPush.Target.MediaType = schema1.ManifestMediaType
manifestPush.Target.MediaType = schema1.MediaTypeManifest
manifestPush.Target.Repository = "library/test"
manifestPush.Target.URL = "http://example.com/v2/library/test/manifests/latest"

4
notifications/http_test.go
View file

@ -75,12 +75,12 @@ func TestHTTPSink(t *testing.T) {
{
statusCode: http.StatusOK,
events: []Event{
createTestEvent("push", "library/test", schema1.ManifestMediaType)},
createTestEvent("push", "library/test", schema1.MediaTypeManifest)},
},
{
statusCode: http.StatusOK,
events: []Event{
createTestEvent("push", "library/test", schema1.ManifestMediaType),
createTestEvent("push", "library/test", schema1.MediaTypeManifest),
createTestEvent("push", "library/test", layerMediaType),
createTestEvent("push", "library/test", layerMediaType),
},

28
notifications/listener.go
View file

@ -7,18 +7,17 @@ import (
"github.com/docker/distribution"
"github.com/docker/distribution/context"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/manifest/schema1"
)
// ManifestListener describes a set of methods for listening to events related to manifests.
type ManifestListener interface {
ManifestPushed(repo string, sm *schema1.SignedManifest) error
ManifestPulled(repo string, sm *schema1.SignedManifest) error
ManifestPushed(repo string, sm distribution.Manifest) error
ManifestPulled(repo string, sm distribution.Manifest) error
// TODO(stevvooe): Please note that delete support is still a little shaky
// and we'll need to propagate these in the future.
ManifestDeleted(repo string, sm *schema1.SignedManifest) error
ManifestDeleted(repo string, sm distribution.Manifest) error
}
// BlobListener describes a listener that can respond to layer related events.
@ -74,8 +73,8 @@ type manifestServiceListener struct {
parent *repositoryListener
}
func (msl *manifestServiceListener) Get(dgst digest.Digest) (*schema1.SignedManifest, error) {
sm, err := msl.ManifestService.Get(dgst)
func (msl *manifestServiceListener) Get(ctx context.Context, dgst digest.Digest, options ...distribution.ManifestServiceOption) (distribution.Manifest, error) {
sm, err := msl.ManifestService.Get(ctx, dgst)
if err == nil {
if err := msl.parent.listener.ManifestPulled(msl.parent.Repository.Name(), sm); err != nil {
logrus.Errorf("error dispatching manifest pull to listener: %v", err)
@ -85,8 +84,8 @@ func (msl *manifestServiceListener) Get(dgst digest.Digest) (*schema1.SignedMani
return sm, err
}
func (msl *manifestServiceListener) Put(sm *schema1.SignedManifest) error {
err := msl.ManifestService.Put(sm)
func (msl *manifestServiceListener) Put(ctx context.Context, sm distribution.Manifest, options ...distribution.ManifestServiceOption) (digest.Digest, error) {
dgst, err := msl.ManifestService.Put(ctx, sm, options...)
if err == nil {
if err := msl.parent.listener.ManifestPushed(msl.parent.Repository.Name(), sm); err != nil {
@ -94,18 +93,7 @@ func (msl *manifestServiceListener) Put(sm *schema1.SignedManifest) error {
}
}
return err
}
func (msl *manifestServiceListener) GetByTag(tag string, options ...distribution.ManifestServiceOption) (*schema1.SignedManifest, error) {
sm, err := msl.ManifestService.GetByTag(tag, options...)
if err == nil {
if err := msl.parent.listener.ManifestPulled(msl.parent.Repository.Name(), sm); err != nil {
logrus.Errorf("error dispatching manifest pull to listener: %v", err)
}
}
return sm, err
return dgst, err
}
type blobServiceListener struct {

35
notifications/listener_test.go
View file

@ -38,7 +38,7 @@ func TestListener(t *testing.T) {
expectedOps := map[string]int{
"manifest:push": 1,
"manifest:pull": 2,
"manifest:pull": 1,
// "manifest:delete": 0, // deletes not supported for now
"layer:push": 2,
"layer:pull": 2,
@ -55,18 +55,18 @@ type testListener struct {
ops map[string]int
}
func (tl *testListener) ManifestPushed(repo string, sm *schema1.SignedManifest) error {
func (tl *testListener) ManifestPushed(repo string, m distribution.Manifest) error {
tl.ops["manifest:push"]++
return nil
}
func (tl *testListener) ManifestPulled(repo string, sm *schema1.SignedManifest) error {
func (tl *testListener) ManifestPulled(repo string, m distribution.Manifest) error {
tl.ops["manifest:pull"]++
return nil
}
func (tl *testListener) ManifestDeleted(repo string, sm *schema1.SignedManifest) error {
func (tl *testListener) ManifestDeleted(repo string, m distribution.Manifest) error {
tl.ops["manifest:delete"]++
return nil
}
@ -93,8 +93,11 @@ func checkExerciseRepository(t *testing.T, repository distribution.Repository) {
// takes the registry through a common set of operations. This could be
// used to make cross-cutting updates by changing internals that affect
// update counts. Basically, it would make writing tests a lot easier.
ctx := context.Background()
tag := "thetag"
// todo: change this to use Builder
m := schema1.Manifest{
Versioned: manifest.Versioned{
SchemaVersion: 1,
@ -158,31 +161,19 @@ func checkExerciseRepository(t *testing.T, repository distribution.Repository) {
t.Fatal(err.Error())
}
if err = manifests.Put(sm); err != nil {
var digestPut digest.Digest
if digestPut, err = manifests.Put(ctx, sm); err != nil {
t.Fatalf("unexpected error putting the manifest: %v", err)
}
p, err := sm.Payload()
if err != nil {
t.Fatalf("unexpected error getting manifest payload: %v", err)
dgst := digest.FromBytes(sm.Canonical)
if dgst != digestPut {
t.Fatalf("mismatching digest from payload and put")
}
dgst := digest.FromBytes(p)
fetchedByManifest, err := manifests.Get(dgst)
_, err = manifests.Get(ctx, dgst)
if err != nil {
t.Fatalf("unexpected error fetching manifest: %v", err)
}
if fetchedByManifest.Tag != sm.Tag {
t.Fatalf("retrieved unexpected manifest: %v", err)
}
fetched, err := manifests.GetByTag(tag)
if err != nil {
t.Fatalf("unexpected error fetching manifest: %v", err)
}
if fetched.Tag != fetchedByManifest.Tag {
t.Fatalf("retrieved unexpected manifest: %v", err)
}
}

59
registry.go
View file

@ -2,8 +2,6 @@ package distribution
import (
"github.com/docker/distribution/context"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/manifest/schema1"
)
// Scope defines the set of items that match a namespace.
@ -44,7 +42,9 @@ type Namespace interface {
}
// ManifestServiceOption is a function argument for Manifest Service methods
type ManifestServiceOption func(ManifestService) error
type ManifestServiceOption interface {
Apply(ManifestService) error
}
// Repository is a named collection of manifests and layers.
type Repository interface {
@ -62,59 +62,10 @@ type Repository interface {
// be a BlobService for use with clients. This will allow such
// implementations to avoid implementing ServeBlob.
// Signatures returns a reference to this repository's signatures service.
Signatures() SignatureService
// Tags returns a reference to this repositories tag service
Tags(ctx context.Context) TagService
}
// TODO(stevvooe): Must add close methods to all these. May want to change the
// way instances are created to better reflect internal dependency
// relationships.
// ManifestService provides operations on image manifests.
type ManifestService interface {
// Exists returns true if the manifest exists.
Exists(dgst digest.Digest) (bool, error)
// Get retrieves the identified by the digest, if it exists.
Get(dgst digest.Digest) (*schema1.SignedManifest, error)
// Delete removes the manifest, if it exists.
Delete(dgst digest.Digest) error
// Put creates or updates the manifest.
Put(manifest *schema1.SignedManifest) error
// TODO(stevvooe): The methods after this message should be moved to a
// discrete TagService, per active proposals.
// Tags lists the tags under the named repository.
Tags() ([]string, error)
// ExistsByTag returns true if the manifest exists.
ExistsByTag(tag string) (bool, error)
// GetByTag retrieves the named manifest, if it exists.
GetByTag(tag string, options ...ManifestServiceOption) (*schema1.SignedManifest, error)
// TODO(stevvooe): There are several changes that need to be done to this
// interface:
//
// 1. Allow explicit tagging with Tag(digest digest.Digest, tag string)
// 2. Support reading tags with a re-entrant reader to avoid large
// allocations in the registry.
// 3. Long-term: Provide All() method that lets one scroll through all of
// the manifest entries.
// 4. Long-term: break out concept of signing from manifests. This is
// really a part of the distribution sprint.
// 5. Long-term: Manifest should be an interface. This code shouldn't
// really be concerned with the storage format.
}
// SignatureService provides operations on signatures.
type SignatureService interface {
// Get retrieves all of the signature blobs for the specified digest.
Get(dgst digest.Digest) ([][]byte, error)
// Put stores the signature for the provided digest.
Put(dgst digest.Digest, signatures ...[]byte) error
}

2
registry/api/v2/descriptors.go
View file

@ -495,7 +495,7 @@ var routeDescriptors = []RouteDescriptor{
Methods: []MethodDescriptor{
{
Method: "GET",
Description: "Fetch the manifest identified by `name` and `reference` where `reference` can be a tag or digest.",
Description: "Fetch the manifest identified by `name` and `reference` where `reference` can be a tag or digest. A `HEAD` request can also be issued to this endpoint to obtain resource information without receiving all data.",
Requests: []RequestDescriptor{
{
Headers: []ParameterDescriptor{

324
registry/client/repository.go
View file

@ -3,6 +3,7 @@ package client
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
@ -14,7 +15,6 @@ import (
"github.com/docker/distribution"
"github.com/docker/distribution/context"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/reference"
"github.com/docker/distribution/registry/api/v2"
"github.com/docker/distribution/registry/client/transport"
@ -156,26 +156,139 @@ func (r *repository) Manifests(ctx context.Context, options ...distribution.Mani
}, nil
}
func (r *repository) Signatures() distribution.SignatureService {
ms, _ := r.Manifests(r.context)
return &signatures{
manifests: ms,
func (r *repository) Tags(ctx context.Context) distribution.TagService {
return &tags{
client: r.client,
ub: r.ub,
context: r.context,
name: r.Name(),
}
}
type signatures struct {
manifests distribution.ManifestService
// tags implements remote tagging operations.
type tags struct {
client *http.Client
ub *v2.URLBuilder
context context.Context
name string
}
func (s *signatures) Get(dgst digest.Digest) ([][]byte, error) {
m, err := s.manifests.Get(dgst)
// All returns all tags
func (t *tags) All(ctx context.Context) ([]string, error) {
var tags []string
u, err := t.ub.BuildTagsURL(t.name)
if err != nil {
return nil, err
return tags, err
}
return m.Signatures()
resp, err := t.client.Get(u)
if err != nil {
return tags, err
}
defer resp.Body.Close()
if SuccessStatus(resp.StatusCode) {
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return tags, err
}
tagsResponse := struct {
Tags []string `json:"tags"`
}{}
if err := json.Unmarshal(b, &tagsResponse); err != nil {
return tags, err
}
tags = tagsResponse.Tags
return tags, nil
}
return tags, handleErrorResponse(resp)
}
func (s *signatures) Put(dgst digest.Digest, signatures ...[]byte) error {
func descriptorFromResponse(response *http.Response) (distribution.Descriptor, error) {
desc := distribution.Descriptor{}
headers := response.Header
ctHeader := headers.Get("Content-Type")
if ctHeader == "" {
return distribution.Descriptor{}, errors.New("missing or empty Content-Type header")
}
desc.MediaType = ctHeader
digestHeader := headers.Get("Docker-Content-Digest")
if digestHeader == "" {
bytes, err := ioutil.ReadAll(response.Body)
if err != nil {
return distribution.Descriptor{}, err
}
_, desc, err := distribution.UnmarshalManifest(ctHeader, bytes)
if err != nil {
return distribution.Descriptor{}, err
}
return desc, nil
}
dgst, err := digest.ParseDigest(digestHeader)
if err != nil {
return distribution.Descriptor{}, err
}
desc.Digest = dgst
lengthHeader := headers.Get("Content-Length")
if lengthHeader == "" {
return distribution.Descriptor{}, errors.New("missing or empty Content-Length header")
}
length, err := strconv.ParseInt(lengthHeader, 10, 64)
if err != nil {
return distribution.Descriptor{}, err
}
desc.Size = length
return desc, nil
}
// Get issues a HEAD request for a Manifest against its named endpoint in order
// to construct a descriptor for the tag. If the registry doesn't support HEADing
// a manifest, fallback to GET.
func (t *tags) Get(ctx context.Context, tag string) (distribution.Descriptor, error) {
u, err := t.ub.BuildManifestURL(t.name, tag)
if err != nil {
return distribution.Descriptor{}, err
}
var attempts int
resp, err := t.client.Head(u)
check:
if err != nil {
return distribution.Descriptor{}, err
}
switch {
case resp.StatusCode >= 200 && resp.StatusCode < 400:
return descriptorFromResponse(resp)
case resp.StatusCode == http.StatusMethodNotAllowed:
resp, err = t.client.Get(u)
attempts++
if attempts > 1 {
return distribution.Descriptor{}, err
}
goto check
default:
return distribution.Descriptor{}, handleErrorResponse(resp)
}
}
func (t *tags) Lookup(ctx context.Context, digest distribution.Descriptor) ([]string, error) {
panic("not implemented")
}
func (t *tags) Tag(ctx context.Context, tag string, desc distribution.Descriptor) error {
panic("not implemented")
}
func (t *tags) Untag(ctx context.Context, tag string) error {
panic("not implemented")
}
@ -186,44 +299,8 @@ type manifests struct {
etags map[string]string
}
func (ms *manifests) Tags() ([]string, error) {
u, err := ms.ub.BuildTagsURL(ms.name)
if err != nil {
return nil, err
}
resp, err := ms.client.Get(u)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if SuccessStatus(resp.StatusCode) {
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
tagsResponse := struct {
Tags []string `json:"tags"`
}{}
if err := json.Unmarshal(b, &tagsResponse); err != nil {
return nil, err
}
return tagsResponse.Tags, nil
}
return nil, handleErrorResponse(resp)
}
func (ms *manifests) Exists(dgst digest.Digest) (bool, error) {
// Call by Tag endpoint since the API uses the same
// URL endpoint for tags and digests.
return ms.ExistsByTag(dgst.String())
}
func (ms *manifests) ExistsByTag(tag string) (bool, error) {
u, err := ms.ub.BuildManifestURL(ms.name, tag)
func (ms *manifests) Exists(ctx context.Context, dgst digest.Digest) (bool, error) {
u, err := ms.ub.BuildManifestURL(ms.name, dgst.String())
if err != nil {
return false, err
}
@ -241,46 +318,63 @@ func (ms *manifests) ExistsByTag(tag string) (bool, error) {
return false, handleErrorResponse(resp)
}
func (ms *manifests) Get(dgst digest.Digest) (*schema1.SignedManifest, error) {
// Call by Tag endpoint since the API uses the same
// URL endpoint for tags and digests.
return ms.GetByTag(dgst.String())
}
// AddEtagToTag allows a client to supply an eTag to GetByTag which will be
// AddEtagToTag allows a client to supply an eTag to Get which will be
// used for a conditional HTTP request. If the eTag matches, a nil manifest
// and nil error will be returned. etag is automatically quoted when added to
// this map.
// and ErrManifestNotModified error will be returned. etag is automatically
// quoted when added to this map.
func AddEtagToTag(tag, etag string) distribution.ManifestServiceOption {
return func(ms distribution.ManifestService) error {
if ms, ok := ms.(*manifests); ok {
ms.etags[tag] = fmt.Sprintf(`"%s"`, etag)
return nil
}
return fmt.Errorf("etag options is a client-only option")
}
return etagOption{tag, etag}
}
func (ms *manifests) GetByTag(tag string, options ...distribution.ManifestServiceOption) (*schema1.SignedManifest, error) {
type etagOption struct{ tag, etag string }
func (o etagOption) Apply(ms distribution.ManifestService) error {
if ms, ok := ms.(*manifests); ok {
ms.etags[o.tag] = fmt.Sprintf(`"%s"`, o.etag)
return nil
}
return fmt.Errorf("etag options is a client-only option")
}
func (ms *manifests) Get(ctx context.Context, dgst digest.Digest, options ...distribution.ManifestServiceOption) (distribution.Manifest, error) {
var tag string
for _, option := range options {
err := option(ms)
if err != nil {
return nil, err
if opt, ok := option.(withTagOption); ok {
tag = opt.tag
} else {
err := option.Apply(ms)
if err != nil {
return nil, err
}
}
}
u, err := ms.ub.BuildManifestURL(ms.name, tag)
var ref string
if tag != "" {
ref = tag
} else {
ref = dgst.String()
}
u, err := ms.ub.BuildManifestURL(ms.name, ref)
if err != nil {
return nil, err
}
req, err := http.NewRequest("GET", u, nil)
if err != nil {
return nil, err
}
if _, ok := ms.etags[tag]; ok {
req.Header.Set("If-None-Match", ms.etags[tag])
for _, t := range distribution.ManifestMediaTypes() {
req.Header.Add("Accept", t)
}
if _, ok := ms.etags[ref]; ok {
req.Header.Set("If-None-Match", ms.etags[ref])
}
resp, err := ms.client.Do(req)
if err != nil {
return nil, err
@ -289,44 +383,89 @@ func (ms *manifests) GetByTag(tag string, options ...distribution.ManifestServic
if resp.StatusCode == http.StatusNotModified {
return nil, distribution.ErrManifestNotModified
} else if SuccessStatus(resp.StatusCode) {
var sm schema1.SignedManifest
decoder := json.NewDecoder(resp.Body)
mt := resp.Header.Get("Content-Type")
body, err := ioutil.ReadAll(resp.Body)
if err := decoder.Decode(&sm); err != nil {
if err != nil {
return nil, err
}
return &sm, nil
m, _, err := distribution.UnmarshalManifest(mt, body)
if err != nil {
return nil, err
}
return m, nil
}
return nil, handleErrorResponse(resp)
}
func (ms *manifests) Put(m *schema1.SignedManifest) error {
manifestURL, err := ms.ub.BuildManifestURL(ms.name, m.Tag)
if err != nil {
return err
// WithTag allows a tag to be passed into Put which enables the client
// to build a correct URL.
func WithTag(tag string) distribution.ManifestServiceOption {
return withTagOption{tag}
}
type withTagOption struct{ tag string }
func (o withTagOption) Apply(m distribution.ManifestService) error {
if _, ok := m.(*manifests); ok {
return nil
}
return fmt.Errorf("withTagOption is a client-only option")
}
// Put puts a manifest. A tag can be specified using an options parameter which uses some shared state to hold the
// tag name in order to build the correct upload URL. This state is written and read under a lock.
func (ms *manifests) Put(ctx context.Context, m distribution.Manifest, options ...distribution.ManifestServiceOption) (digest.Digest, error) {
var tag string
for _, option := range options {
if opt, ok := option.(withTagOption); ok {
tag = opt.tag
} else {
err := option.Apply(ms)
if err != nil {
return "", err
}
}
}
// todo(richardscothern): do something with options here when they become applicable
putRequest, err := http.NewRequest("PUT", manifestURL, bytes.NewReader(m.Raw))
manifestURL, err := ms.ub.BuildManifestURL(ms.name, tag)
if err != nil {
return err
return "", err
}
mediaType, p, err := m.Payload()
if err != nil {
return "", err
}
putRequest, err := http.NewRequest("PUT", manifestURL, bytes.NewReader(p))
if err != nil {
return "", err
}
putRequest.Header.Set("Content-Type", mediaType)
resp, err := ms.client.Do(putRequest)
if err != nil {
return err
return "", err
}
defer resp.Body.Close()
if SuccessStatus(resp.StatusCode) {
// TODO(dmcgowan): make use of digest header
return nil
dgstHeader := resp.Header.Get("Docker-Content-Digest")
dgst, err := digest.ParseDigest(dgstHeader)
if err != nil {
return "", err
}
return dgst, nil
}
return handleErrorResponse(resp)
return "", handleErrorResponse(resp)
}
func (ms *manifests) Delete(dgst digest.Digest) error {
func (ms *manifests) Delete(ctx context.Context, dgst digest.Digest) error {
u, err := ms.ub.BuildManifestURL(ms.name, dgst.String())
if err != nil {
return err
@ -348,6 +487,11 @@ func (ms *manifests) Delete(dgst digest.Digest) error {
return handleErrorResponse(resp)
}
// todo(richardscothern): Restore interface and implementation with merge of #1050
/*func (ms *manifests) Enumerate(ctx context.Context, manifests []distribution.Manifest, last distribution.Manifest) (n int, err error) {
panic("not supported")
}*/
type blobs struct {
name string
ub *v2.URLBuilder

125
registry/client/repository_test.go
View file

@ -42,7 +42,6 @@ func newRandomBlob(size int) (digest.Digest, []byte) {
}
func addTestFetch(repo string, dgst digest.Digest, content []byte, m *testutil.RequestResponseMap) {
*m = append(*m, testutil.RequestResponseMapping{
Request: testutil.Request{
Method: "GET",
@ -499,12 +498,7 @@ func newRandomSchemaV1Manifest(name, tag string, blobCount int) (*schema1.Signed
panic(err)
}
p, err := sm.Payload()
if err != nil {
panic(err)
}
return sm, digest.FromBytes(p), p
return sm, digest.FromBytes(sm.Canonical), sm.Canonical
}
func addTestManifestWithEtag(repo, reference string, content []byte, m *testutil.RequestResponseMap, dgst string) {
@ -525,6 +519,7 @@ func addTestManifestWithEtag(repo, reference string, content []byte, m *testutil
Headers: http.Header(map[string][]string{
"Content-Length": {"0"},
"Last-Modified": {time.Now().Add(-1 * time.Second).Format(time.ANSIC)},
"Content-Type": {schema1.MediaTypeManifest},
}),
}
} else {
@ -534,6 +529,7 @@ func addTestManifestWithEtag(repo, reference string, content []byte, m *testutil
Headers: http.Header(map[string][]string{
"Content-Length": {fmt.Sprint(len(content))},
"Last-Modified": {time.Now().Add(-1 * time.Second).Format(time.ANSIC)},
"Content-Type": {schema1.MediaTypeManifest},
}),
}
@ -553,6 +549,7 @@ func addTestManifest(repo, reference string, content []byte, m *testutil.Request
Headers: http.Header(map[string][]string{
"Content-Length": {fmt.Sprint(len(content))},
"Last-Modified": {time.Now().Add(-1 * time.Second).Format(time.ANSIC)},
"Content-Type": {schema1.MediaTypeManifest},
}),
},
})
@ -566,6 +563,7 @@ func addTestManifest(repo, reference string, content []byte, m *testutil.Request
Headers: http.Header(map[string][]string{
"Content-Length": {fmt.Sprint(len(content))},
"Last-Modified": {time.Now().Add(-1 * time.Second).Format(time.ANSIC)},
"Content-Type": {schema1.MediaTypeManifest},
}),
},
})
@ -598,12 +596,17 @@ func checkEqualManifest(m1, m2 *schema1.SignedManifest) error {
return nil
}
func TestManifestFetch(t *testing.T) {
func TestV1ManifestFetch(t *testing.T) {
ctx := context.Background()
repo := "test.example.com/repo"
m1, dgst, _ := newRandomSchemaV1Manifest(repo, "latest", 6)
var m testutil.RequestResponseMap
addTestManifest(repo, dgst.String(), m1.Raw, &m)
_, pl, err := m1.Payload()
if err != nil {
t.Fatal(err)
}
addTestManifest(repo, dgst.String(), pl, &m)
addTestManifest(repo, "latest", pl, &m)
e, c := testServer(m)
defer c()
@ -617,7 +620,7 @@ func TestManifestFetch(t *testing.T) {
t.Fatal(err)
}
ok, err := ms.Exists(dgst)
ok, err := ms.Exists(ctx, dgst)
if err != nil {
t.Fatal(err)
}
@ -625,11 +628,29 @@ func TestManifestFetch(t *testing.T) {
t.Fatal("Manifest does not exist")
}
manifest, err := ms.Get(dgst)
manifest, err := ms.Get(ctx, dgst)
if err != nil {
t.Fatal(err)
}
if err := checkEqualManifest(manifest, m1); err != nil {
v1manifest, ok := manifest.(*schema1.SignedManifest)
if !ok {
t.Fatalf("Unexpected manifest type from Get: %T", manifest)
}
if err := checkEqualManifest(v1manifest, m1); err != nil {
t.Fatal(err)
}
manifest, err = ms.Get(ctx, dgst, WithTag("latest"))
if err != nil {
t.Fatal(err)
}
v1manifest, ok = manifest.(*schema1.SignedManifest)
if !ok {
t.Fatalf("Unexpected manifest type from Get: %T", manifest)
}
if err = checkEqualManifest(v1manifest, m1); err != nil {
t.Fatal(err)
}
}
@ -643,17 +664,22 @@ func TestManifestFetchWithEtag(t *testing.T) {
e, c := testServer(m)
defer c()
r, err := NewRepository(context.Background(), repo, e, nil)
ctx := context.Background()
r, err := NewRepository(ctx, repo, e, nil)
if err != nil {
t.Fatal(err)
}
ctx := context.Background()
ms, err := r.Manifests(ctx)
if err != nil {
t.Fatal(err)
}
_, err = ms.GetByTag("latest", AddEtagToTag("latest", d1.String()))
clientManifestService, ok := ms.(*manifests)
if !ok {
panic("wrong type for client manifest service")
}
_, err = clientManifestService.Get(ctx, d1, WithTag("latest"), AddEtagToTag("latest", d1.String()))
if err != distribution.ErrManifestNotModified {
t.Fatal(err)
}
@ -690,10 +716,10 @@ func TestManifestDelete(t *testing.T) {
t.Fatal(err)
}
if err := ms.Delete(dgst1); err != nil {
if err := ms.Delete(ctx, dgst1); err != nil {
t.Fatal(err)
}
if err := ms.Delete(dgst2); err == nil {
if err := ms.Delete(ctx, dgst2); err == nil {
t.Fatal("Expected error deleting unknown manifest")
}
// TODO(dmcgowan): Check for specific unknown error
@ -702,12 +728,17 @@ func TestManifestDelete(t *testing.T) {
func TestManifestPut(t *testing.T) {
repo := "test.example.com/repo/delete"
m1, dgst, _ := newRandomSchemaV1Manifest(repo, "other", 6)
_, payload, err := m1.Payload()
if err != nil {
t.Fatal(err)
}
var m testutil.RequestResponseMap
m = append(m, testutil.RequestResponseMapping{
Request: testutil.Request{
Method: "PUT",
Route: "/v2/" + repo + "/manifests/other",
Body: m1.Raw,
Body: payload,
},
Response: testutil.Response{
StatusCode: http.StatusAccepted,
@ -731,7 +762,7 @@ func TestManifestPut(t *testing.T) {
t.Fatal(err)
}
if err := ms.Put(m1); err != nil {
if _, err := ms.Put(ctx, m1, WithTag(m1.Tag)); err != nil {
t.Fatal(err)
}
@ -751,21 +782,22 @@ func TestManifestTags(t *testing.T) {
}
`))
var m testutil.RequestResponseMap
m = append(m, testutil.RequestResponseMapping{
Request: testutil.Request{
Method: "GET",
Route: "/v2/" + repo + "/tags/list",
},
Response: testutil.Response{
StatusCode: http.StatusOK,
Body: tagsList,
Headers: http.Header(map[string][]string{
"Content-Length": {fmt.Sprint(len(tagsList))},
"Last-Modified": {time.Now().Add(-1 * time.Second).Format(time.ANSIC)},
}),
},
})
for i := 0; i < 3; i++ {
m = append(m, testutil.RequestResponseMapping{
Request: testutil.Request{
Method: "GET",
Route: "/v2/" + repo + "/tags/list",
},
Response: testutil.Response{
StatusCode: http.StatusOK,
Body: tagsList,
Headers: http.Header(map[string][]string{
"Content-Length": {fmt.Sprint(len(tagsList))},
"Last-Modified": {time.Now().Add(-1 * time.Second).Format(time.ANSIC)},
}),
},
})
}
e, c := testServer(m)
defer c()
@ -773,22 +805,29 @@ func TestManifestTags(t *testing.T) {
if err != nil {
t.Fatal(err)
}
ctx := context.Background()
ms, err := r.Manifests(ctx)
tagService := r.Tags(ctx)
tags, err := tagService.All(ctx)
if err != nil {
t.Fatal(err)
}
tags, err := ms.Tags()
if err != nil {
t.Fatal(err)
}
if len(tags) != 3 {
t.Fatalf("Wrong number of tags returned: %d, expected 3", len(tags))
}
// TODO(dmcgowan): Check array
expected := map[string]struct{}{
"tag1": {},
"tag2": {},
"funtag": {},
}
for _, t := range tags {
delete(expected, t)
}
if len(expected) != 0 {
t.Fatalf("unexpected tags returned: %v", expected)
}
// TODO(dmcgowan): Check for error cases
}
@ -821,7 +860,7 @@ func TestManifestUnauthorized(t *testing.T) {
t.Fatal(err)
}
_, err = ms.Get(dgst)
_, err = ms.Get(ctx, dgst)
if err == nil {
t.Fatal("Expected error fetching manifest")
}

149
registry/handlers/api_test.go
View file

@ -871,19 +871,15 @@ func testManifestAPI(t *testing.T, env *testEnv, args manifestArgs) (*testEnv, m
t.Fatalf("unexpected error signing manifest: %v", err)
}
payload, err := signedManifest.Payload()
checkErr(t, err, "getting manifest payload")
dgst := digest.FromBytes(payload)
dgst := digest.FromBytes(signedManifest.Canonical)
args.signedManifest = signedManifest
args.dgst = dgst
manifestDigestURL, err := env.builder.BuildManifestURL(imageName, dgst.String())
checkErr(t, err, "building manifest url")
resp = putManifest(t, "putting signed manifest", manifestURL, signedManifest)
checkResponse(t, "putting signed manifest", resp, http.StatusCreated)
resp = putManifest(t, "putting signed manifest no error", manifestURL, signedManifest)
checkResponse(t, "putting signed manifest no error", resp, http.StatusCreated)
checkHeaders(t, resp, http.Header{
"Location": []string{manifestDigestURL},
"Docker-Content-Digest": []string{dgst.String()},
@ -914,11 +910,12 @@ func testManifestAPI(t *testing.T, env *testEnv, args manifestArgs) (*testEnv, m
var fetchedManifest schema1.SignedManifest
dec := json.NewDecoder(resp.Body)
if err := dec.Decode(&fetchedManifest); err != nil {
t.Fatalf("error decoding fetched manifest: %v", err)
}
if !bytes.Equal(fetchedManifest.Raw, signedManifest.Raw) {
if !bytes.Equal(fetchedManifest.Canonical, signedManifest.Canonical) {
t.Fatalf("manifests do not match")
}
@ -940,10 +937,55 @@ func testManifestAPI(t *testing.T, env *testEnv, args manifestArgs) (*testEnv, m
t.Fatalf("error decoding fetched manifest: %v", err)
}
if !bytes.Equal(fetchedManifestByDigest.Raw, signedManifest.Raw) {
if !bytes.Equal(fetchedManifestByDigest.Canonical, signedManifest.Canonical) {
t.Fatalf("manifests do not match")
}
// check signature was roundtripped
signatures, err := fetchedManifestByDigest.Signatures()
if err != nil {
t.Fatal(err)
}
if len(signatures) != 1 {
t.Fatalf("expected 1 signature from manifest, got: %d", len(signatures))
}
// Re-sign, push and pull the same digest
sm2, err := schema1.Sign(&fetchedManifestByDigest.Manifest, env.pk)
if err != nil {
t.Fatal(err)
}
resp = putManifest(t, "re-putting signed manifest", manifestDigestURL, sm2)
checkResponse(t, "re-putting signed manifest", resp, http.StatusCreated)
resp, err = http.Get(manifestDigestURL)
checkErr(t, err, "re-fetching manifest by digest")
defer resp.Body.Close()
checkResponse(t, "re-fetching uploaded manifest", resp, http.StatusOK)
checkHeaders(t, resp, http.Header{
"Docker-Content-Digest": []string{dgst.String()},
"ETag": []string{fmt.Sprintf(`"%s"`, dgst)},
})
dec = json.NewDecoder(resp.Body)
if err := dec.Decode(&fetchedManifestByDigest); err != nil {
t.Fatalf("error decoding fetched manifest: %v", err)
}
// check two signatures were roundtripped
signatures, err = fetchedManifestByDigest.Signatures()
if err != nil {
t.Fatal(err)
}
if len(signatures) != 2 {
t.Fatalf("expected 2 signature from manifest, got: %d", len(signatures))
}
// Get by name with etag, gives 304
etag := resp.Header.Get("Etag")
req, err := http.NewRequest("GET", manifestURL, nil)
@ -956,7 +998,7 @@ func testManifestAPI(t *testing.T, env *testEnv, args manifestArgs) (*testEnv, m
t.Fatalf("Error constructing request: %s", err)
}
checkResponse(t, "fetching layer with etag", resp, http.StatusNotModified)
checkResponse(t, "fetching manifest by name with etag", resp, http.StatusNotModified)
// Get by digest with etag, gives 304
req, err = http.NewRequest("GET", manifestDigestURL, nil)
@ -969,7 +1011,7 @@ func testManifestAPI(t *testing.T, env *testEnv, args manifestArgs) (*testEnv, m
t.Fatalf("Error constructing request: %s", err)
}
checkResponse(t, "fetching layer with etag", resp, http.StatusNotModified)
checkResponse(t, "fetching manifest by dgst with etag", resp, http.StatusNotModified)
// Ensure that the tag is listed.
resp, err = http.Get(tagsURL)
@ -1143,8 +1185,13 @@ func newTestEnvWithConfig(t *testing.T, config *configuration.Configuration) *te
func putManifest(t *testing.T, msg, url string, v interface{}) *http.Response {
var body []byte
if sm, ok := v.(*schema1.SignedManifest); ok {
body = sm.Raw
_, pl, err := sm.Payload()
if err != nil {
t.Fatalf("error getting payload: %v", err)
}
body = pl
} else {
var err error
body, err = json.MarshalIndent(v, "", " ")
@ -1435,7 +1482,7 @@ func checkErr(t *testing.T, err error, msg string) {
}
}
func createRepository(env *testEnv, t *testing.T, imageName string, tag string) {
func createRepository(env *testEnv, t *testing.T, imageName string, tag string) digest.Digest {
unsignedManifest := &schema1.Manifest{
Versioned: manifest.Versioned{
SchemaVersion: 1,
@ -1459,7 +1506,6 @@ func createRepository(env *testEnv, t *testing.T, imageName string, tag string)
for i := range unsignedManifest.FSLayers {
rs, dgstStr, err := testutil.CreateRandomTarFile()
if err != nil {
t.Fatalf("error creating random layer %d: %v", i, err)
}
@ -1477,20 +1523,22 @@ func createRepository(env *testEnv, t *testing.T, imageName string, tag string)
t.Fatalf("unexpected error signing manifest: %v", err)
}
payload, err := signedManifest.Payload()
checkErr(t, err, "getting manifest payload")
dgst := digest.FromBytes(signedManifest.Canonical)
dgst := digest.FromBytes(payload)
manifestDigestURL, err := env.builder.BuildManifestURL(imageName, dgst.String())
// Create this repository by tag to ensure the tag mapping is made in the registry
manifestDigestURL, err := env.builder.BuildManifestURL(imageName, tag)
checkErr(t, err, "building manifest url")
location, err := env.builder.BuildManifestURL(imageName, dgst.String())
checkErr(t, err, "building location URL")
resp := putManifest(t, "putting signed manifest", manifestDigestURL, signedManifest)
checkResponse(t, "putting signed manifest", resp, http.StatusCreated)
checkHeaders(t, resp, http.Header{
"Location": []string{manifestDigestURL},
"Location": []string{location},
"Docker-Content-Digest": []string{dgst.String()},
})
return dgst
}
// Test mutation operations on a registry configured as a cache. Ensure that they return
@ -1577,3 +1625,64 @@ func TestCheckContextNotifier(t *testing.T) {
t.Fatalf("wrong status code - expected 200, got %d", resp.StatusCode)
}
}
func TestProxyManifestGetByTag(t *testing.T) {
truthConfig := configuration.Configuration{
Storage: configuration.Storage{
"inmemory": configuration.Parameters{},
},
}
truthConfig.HTTP.Headers = headerConfig
imageName := "foo/bar"
tag := "latest"
truthEnv := newTestEnvWithConfig(t, &truthConfig)
// create a repository in the truth registry
dgst := createRepository(truthEnv, t, imageName, tag)
proxyConfig := configuration.Configuration{
Storage: configuration.Storage{
"inmemory": configuration.Parameters{},
},
Proxy: configuration.Proxy{
RemoteURL: truthEnv.server.URL,
},
}
proxyConfig.HTTP.Headers = headerConfig
proxyEnv := newTestEnvWithConfig(t, &proxyConfig)
manifestDigestURL, err := proxyEnv.builder.BuildManifestURL(imageName, dgst.String())
checkErr(t, err, "building manifest url")
resp, err := http.Get(manifestDigestURL)
checkErr(t, err, "fetching manifest from proxy by digest")
defer resp.Body.Close()
manifestTagURL, err := proxyEnv.builder.BuildManifestURL(imageName, tag)
checkErr(t, err, "building manifest url")
resp, err = http.Get(manifestTagURL)
checkErr(t, err, "fetching manifest from proxy by tag")
defer resp.Body.Close()
checkResponse(t, "fetching manifest from proxy by tag", resp, http.StatusOK)
checkHeaders(t, resp, http.Header{
"Docker-Content-Digest": []string{dgst.String()},
})
// Create another manifest in the remote with the same image/tag pair
newDigest := createRepository(truthEnv, t, imageName, tag)
if dgst == newDigest {
t.Fatalf("non-random test data")
}
// fetch it with the same proxy URL as before. Ensure the updated content is at the same tag
resp, err = http.Get(manifestTagURL)
checkErr(t, err, "fetching manifest from proxy by tag")
defer resp.Body.Close()
checkResponse(t, "fetching manifest from proxy by tag", resp, http.StatusOK)
checkHeaders(t, resp, http.Header{
"Docker-Content-Digest": []string{newDigest.String()},
})
}

114
registry/handlers/images.go
View file

@ -2,19 +2,15 @@ package handlers
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
"strings"
"github.com/docker/distribution"
ctxu "github.com/docker/distribution/context"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/registry/api/errcode"
"github.com/docker/distribution/registry/api/v2"
"github.com/gorilla/handlers"
"golang.org/x/net/context"
)
// imageManifestDispatcher takes the request context and builds the
@ -33,7 +29,8 @@ func imageManifestDispatcher(ctx *Context, r *http.Request) http.Handler {
}
mhandler := handlers.MethodHandler{
"GET": http.HandlerFunc(imageManifestHandler.GetImageManifest),
"GET": http.HandlerFunc(imageManifestHandler.GetImageManifest),
"HEAD": http.HandlerFunc(imageManifestHandler.GetImageManifest),
}
if !ctx.readOnly {
@ -54,6 +51,8 @@ type imageManifestHandler struct {
}
// GetImageManifest fetches the image manifest from the storage backend, if it exists.
// todo(richardscothern): this assumes v2 schema 1 manifests for now but in the future
// get the version from the Accept HTTP header
func (imh *imageManifestHandler) GetImageManifest(w http.ResponseWriter, r *http.Request) {
ctxu.GetLogger(imh).Debug("GetImageManifest")
manifests, err := imh.Repository.Manifests(imh)
@ -62,42 +61,38 @@ func (imh *imageManifestHandler) GetImageManifest(w http.ResponseWriter, r *http
return
}
var sm *schema1.SignedManifest
var manifest distribution.Manifest
if imh.Tag != "" {
sm, err = manifests.GetByTag(imh.Tag)
} else {
if etagMatch(r, imh.Digest.String()) {
w.WriteHeader(http.StatusNotModified)
tags := imh.Repository.Tags(imh)
desc, err := tags.Get(imh, imh.Tag)
if err != nil {
imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithDetail(err))
return
}
sm, err = manifests.Get(imh.Digest)
imh.Digest = desc.Digest
}
if etagMatch(r, imh.Digest.String()) {
w.WriteHeader(http.StatusNotModified)
return
}
manifest, err = manifests.Get(imh, imh.Digest)
if err != nil {
imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithDetail(err))
return
}
// Get the digest, if we don't already have it.
if imh.Digest == "" {
dgst, err := digestManifest(imh, sm)
if err != nil {
imh.Errors = append(imh.Errors, v2.ErrorCodeDigestInvalid.WithDetail(err))
return
}
if etagMatch(r, dgst.String()) {
w.WriteHeader(http.StatusNotModified)
return
}
imh.Digest = dgst
ct, p, err := manifest.Payload()
if err != nil {
return
}
w.Header().Set("Content-Type", "application/json; charset=utf-8")
w.Header().Set("Content-Length", fmt.Sprint(len(sm.Raw)))
w.Header().Set("Content-Type", ct)
w.Header().Set("Content-Length", fmt.Sprint(len(p)))
w.Header().Set("Docker-Content-Digest", imh.Digest.String())
w.Header().Set("Etag", fmt.Sprintf(`"%s"`, imh.Digest))
w.Write(sm.Raw)
w.Write(p)
}
func etagMatch(r *http.Request, etag string) bool {
@ -109,7 +104,7 @@ func etagMatch(r *http.Request, etag string) bool {
return false
}
// PutImageManifest validates and stores and image in the registry.
// PutImageManifest validates and stores an image in the registry.
func (imh *imageManifestHandler) PutImageManifest(w http.ResponseWriter, r *http.Request) {
ctxu.GetLogger(imh).Debug("PutImageManifest")
manifests, err := imh.Repository.Manifests(imh)
@ -124,39 +119,28 @@ func (imh *imageManifestHandler) PutImageManifest(w http.ResponseWriter, r *http
return
}
var manifest schema1.SignedManifest
if err := json.Unmarshal(jsonBuf.Bytes(), &manifest); err != nil {
mediaType := r.Header.Get("Content-Type")
manifest, desc, err := distribution.UnmarshalManifest(mediaType, jsonBuf.Bytes())
if err != nil {
imh.Errors = append(imh.Errors, v2.ErrorCodeManifestInvalid.WithDetail(err))
return
}
dgst, err := digestManifest(imh, &manifest)
if err != nil {
imh.Errors = append(imh.Errors, v2.ErrorCodeDigestInvalid.WithDetail(err))
return
}
// Validate manifest tag or digest matches payload
if imh.Tag != "" {
if manifest.Tag != imh.Tag {
ctxu.GetLogger(imh).Errorf("invalid tag on manifest payload: %q != %q", manifest.Tag, imh.Tag)
imh.Errors = append(imh.Errors, v2.ErrorCodeTagInvalid)
return
}
imh.Digest = dgst
} else if imh.Digest != "" {
if dgst != imh.Digest {
ctxu.GetLogger(imh).Errorf("payload digest does match: %q != %q", dgst, imh.Digest)
if imh.Digest != "" {
if desc.Digest != imh.Digest {
ctxu.GetLogger(imh).Errorf("payload digest does match: %q != %q", desc.Digest, imh.Digest)
imh.Errors = append(imh.Errors, v2.ErrorCodeDigestInvalid)
return
}
} else if imh.Tag != "" {
imh.Digest = desc.Digest
} else {
imh.Errors = append(imh.Errors, v2.ErrorCodeTagInvalid.WithDetail("no tag or digest specified"))
return
}
if err := manifests.Put(&manifest); err != nil {
_, err = manifests.Put(imh, manifest)
if err != nil {
// TODO(stevvooe): These error handling switches really need to be
// handled by an app global mapper.
if err == distribution.ErrUnsupported {
@ -188,6 +172,17 @@ func (imh *imageManifestHandler) PutImageManifest(w http.ResponseWriter, r *http
return
}
// Tag this manifest
if imh.Tag != "" {
tags := imh.Repository.Tags(imh)
err = tags.Tag(imh, imh.Tag, desc)
if err != nil {
imh.Errors = append(imh.Errors, errcode.ErrorCodeUnknown.WithDetail(err))
return
}
}
// Construct a canonical url for the uploaded manifest.
location, err := imh.urlBuilder.BuildManifestURL(imh.Repository.Name(), imh.Digest.String())
if err != nil {
@ -212,7 +207,7 @@ func (imh *imageManifestHandler) DeleteImageManifest(w http.ResponseWriter, r *h
return
}
err = manifests.Delete(imh.Digest)
err = manifests.Delete(imh, imh.Digest)
if err != nil {
switch err {
case digest.ErrDigestUnsupported:
@ -233,22 +228,3 @@ func (imh *imageManifestHandler) DeleteImageManifest(w http.ResponseWriter, r *h
w.WriteHeader(http.StatusAccepted)
}
// digestManifest takes a digest of the given manifest. This belongs somewhere
// better but we'll wait for a refactoring cycle to find that real somewhere.
func digestManifest(ctx context.Context, sm *schema1.SignedManifest) (digest.Digest, error) {
p, err := sm.Payload()
if err != nil {
if !strings.Contains(err.Error(), "missing signature key") {
ctxu.GetLogger(ctx).Errorf("error getting manifest payload: %v", err)
return "", err
}
// NOTE(stevvooe): There are no signatures but we still have a
// payload. The request will fail later but this is not the
// responsibility of this part of the code.
p = sm.Raw
}
return digest.FromBytes(p), nil
}

8
registry/handlers/tags.go
View file

@ -34,13 +34,9 @@ type tagsAPIResponse struct {
// GetTags returns a json list of tags for a specific image name.
func (th *tagsHandler) GetTags(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
manifests, err := th.Repository.Manifests(th)
if err != nil {
th.Errors = append(th.Errors, err)
return
}
tags, err := manifests.Tags()
tagService := th.Repository.Tags(th)
tags, err := tagService.All(th)
if err != nil {
switch err := err.(type) {
case distribution.ErrRepositoryUnknown:

137
registry/proxy/proxymanifeststore.go
View file

@ -6,8 +6,6 @@ import (
"github.com/docker/distribution"
"github.com/docker/distribution/context"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/registry/client"
"github.com/docker/distribution/registry/proxy/scheduler"
)
@ -24,8 +22,8 @@ type proxyManifestStore struct {
var _ distribution.ManifestService = &proxyManifestStore{}
func (pms proxyManifestStore) Exists(dgst digest.Digest) (bool, error) {
exists, err := pms.localManifests.Exists(dgst)
func (pms proxyManifestStore) Exists(ctx context.Context, dgst digest.Digest) (bool, error) {
exists, err := pms.localManifests.Exists(ctx, dgst)
if err != nil {
return false, err
}
@ -33,117 +31,56 @@ func (pms proxyManifestStore) Exists(dgst digest.Digest) (bool, error) {
return true, nil
}
return pms.remoteManifests.Exists(dgst)
return pms.remoteManifests.Exists(ctx, dgst)
}
func (pms proxyManifestStore) Get(dgst digest.Digest) (*schema1.SignedManifest, error) {
sm, err := pms.localManifests.Get(dgst)
if err == nil {
proxyMetrics.ManifestPush(uint64(len(sm.Raw)))
return sm, err
func (pms proxyManifestStore) Get(ctx context.Context, dgst digest.Digest, options ...distribution.ManifestServiceOption) (distribution.Manifest, error) {
// At this point `dgst` was either specified explicitly, or returned by the
// tagstore with the most recent association.
var fromRemote bool
manifest, err := pms.localManifests.Get(ctx, dgst, options...)
if err != nil {
manifest, err = pms.remoteManifests.Get(ctx, dgst, options...)
if err != nil {
return nil, err
}
fromRemote = true
}
sm, err = pms.remoteManifests.Get(dgst)
_, payload, err := manifest.Payload()
if err != nil {
return nil, err
}
proxyMetrics.ManifestPull(uint64(len(sm.Raw)))
err = pms.localManifests.Put(sm)
if err != nil {
return nil, err
proxyMetrics.ManifestPush(uint64(len(payload)))
if fromRemote {
proxyMetrics.ManifestPull(uint64(len(payload)))
_, err = pms.localManifests.Put(ctx, manifest)
if err != nil {
return nil, err
}
// Schedule the repo for removal
pms.scheduler.AddManifest(pms.repositoryName, repositoryTTL)
// Ensure the manifest blob is cleaned up
pms.scheduler.AddBlob(dgst.String(), repositoryTTL)
}
// Schedule the repo for removal
pms.scheduler.AddManifest(pms.repositoryName, repositoryTTL)
// Ensure the manifest blob is cleaned up
pms.scheduler.AddBlob(dgst.String(), repositoryTTL)
proxyMetrics.ManifestPush(uint64(len(sm.Raw)))
return sm, err
return manifest, err
}
func (pms proxyManifestStore) Tags() ([]string, error) {
return pms.localManifests.Tags()
func (pms proxyManifestStore) Put(ctx context.Context, manifest distribution.Manifest, options ...distribution.ManifestServiceOption) (digest.Digest, error) {
var d digest.Digest
return d, distribution.ErrUnsupported
}
func (pms proxyManifestStore) ExistsByTag(tag string) (bool, error) {
exists, err := pms.localManifests.ExistsByTag(tag)
if err != nil {
return false, err
}
if exists {
return true, nil
}
return pms.remoteManifests.ExistsByTag(tag)
}
func (pms proxyManifestStore) GetByTag(tag string, options ...distribution.ManifestServiceOption) (*schema1.SignedManifest, error) {
var localDigest digest.Digest
localManifest, err := pms.localManifests.GetByTag(tag, options...)
switch err.(type) {
case distribution.ErrManifestUnknown, distribution.ErrManifestUnknownRevision:
goto fromremote
case nil:
break
default:
return nil, err
}
localDigest, err = manifestDigest(localManifest)
if err != nil {
return nil, err
}
fromremote:
var sm *schema1.SignedManifest
sm, err = pms.remoteManifests.GetByTag(tag, client.AddEtagToTag(tag, localDigest.String()))
if err != nil && err != distribution.ErrManifestNotModified {
return nil, err
}
if err == distribution.ErrManifestNotModified {
context.GetLogger(pms.ctx).Debugf("Local manifest for %q is latest, dgst=%s", tag, localDigest.String())
return localManifest, nil
}
context.GetLogger(pms.ctx).Debugf("Updated manifest for %q, dgst=%s", tag, localDigest.String())
err = pms.localManifests.Put(sm)
if err != nil {
return nil, err
}
dgst, err := manifestDigest(sm)
if err != nil {
return nil, err
}
pms.scheduler.AddBlob(dgst.String(), repositoryTTL)
pms.scheduler.AddManifest(pms.repositoryName, repositoryTTL)
proxyMetrics.ManifestPull(uint64(len(sm.Raw)))
proxyMetrics.ManifestPush(uint64(len(sm.Raw)))
return sm, err
}
func manifestDigest(sm *schema1.SignedManifest) (digest.Digest, error) {
payload, err := sm.Payload()
if err != nil {
return "", err
}
return digest.FromBytes(payload), nil
}
func (pms proxyManifestStore) Put(manifest *schema1.SignedManifest) error {
func (pms proxyManifestStore) Delete(ctx context.Context, dgst digest.Digest) error {
return distribution.ErrUnsupported
}
func (pms proxyManifestStore) Delete(dgst digest.Digest) error {
return distribution.ErrUnsupported
/*func (pms proxyManifestStore) Enumerate(ctx context.Context, manifests []distribution.Manifest, last distribution.Manifest) (n int, err error) {
return 0, distribution.ErrUnsupported
}
*/

64
registry/proxy/proxymanifeststore_test.go
View file

@ -37,40 +37,31 @@ func (te manifestStoreTestEnv) RemoteStats() *map[string]int {
return &rs
}
func (sm statsManifest) Delete(dgst digest.Digest) error {
func (sm statsManifest) Delete(ctx context.Context, dgst digest.Digest) error {
sm.stats["delete"]++
return sm.manifests.Delete(dgst)
return sm.manifests.Delete(ctx, dgst)
}
func (sm statsManifest) Exists(dgst digest.Digest) (bool, error) {
func (sm statsManifest) Exists(ctx context.Context, dgst digest.Digest) (bool, error) {
sm.stats["exists"]++
return sm.manifests.Exists(dgst)
return sm.manifests.Exists(ctx, dgst)
}
func (sm statsManifest) ExistsByTag(tag string) (bool, error) {
sm.stats["existbytag"]++
return sm.manifests.ExistsByTag(tag)
}
func (sm statsManifest) Get(dgst digest.Digest) (*schema1.SignedManifest, error) {
func (sm statsManifest) Get(ctx context.Context, dgst digest.Digest, options ...distribution.ManifestServiceOption) (distribution.Manifest, error) {
sm.stats["get"]++
return sm.manifests.Get(dgst)
return sm.manifests.Get(ctx, dgst)
}
func (sm statsManifest) GetByTag(tag string, options ...distribution.ManifestServiceOption) (*schema1.SignedManifest, error) {
sm.stats["getbytag"]++
return sm.manifests.GetByTag(tag, options...)
}
func (sm statsManifest) Put(manifest *schema1.SignedManifest) error {
func (sm statsManifest) Put(ctx context.Context, manifest distribution.Manifest, options ...distribution.ManifestServiceOption) (digest.Digest, error) {
sm.stats["put"]++
return sm.manifests.Put(manifest)
return sm.manifests.Put(ctx, manifest)
}
func (sm statsManifest) Tags() ([]string, error) {
sm.stats["tags"]++
return sm.manifests.Tags()
/*func (sm statsManifest) Enumerate(ctx context.Context, manifests []distribution.Manifest, last distribution.Manifest) (n int, err error) {
sm.stats["enumerate"]++
return sm.manifests.Enumerate(ctx, manifests, last)
}
*/
func newManifestStoreTestEnv(t *testing.T, name, tag string) *manifestStoreTestEnv {
ctx := context.Background()
@ -169,15 +160,12 @@ func populateRepo(t *testing.T, ctx context.Context, repository distribution.Rep
if err != nil {
t.Fatalf(err.Error())
}
ms.Put(sm)
dgst, err := ms.Put(ctx, sm)
if err != nil {
t.Fatalf("unexpected errors putting manifest: %v", err)
}
pl, err := sm.Payload()
if err != nil {
t.Fatal(err)
}
return digest.FromBytes(pl), nil
return dgst, nil
}
// TestProxyManifests contains basic acceptance tests
@ -189,8 +177,9 @@ func TestProxyManifests(t *testing.T) {
localStats := env.LocalStats()
remoteStats := env.RemoteStats()
ctx := context.Background()
// Stat - must check local and remote
exists, err := env.manifests.ExistsByTag("latest")
exists, err := env.manifests.Exists(ctx, env.manifestDigest)
if err != nil {
t.Fatalf("Error checking existance")
}
@ -198,15 +187,16 @@ func TestProxyManifests(t *testing.T) {
t.Errorf("Unexpected non-existant manifest")
}
if (*localStats)["existbytag"] != 1 && (*remoteStats)["existbytag"] != 1 {
t.Errorf("Unexpected exists count")
if (*localStats)["exists"] != 1 && (*remoteStats)["exists"] != 1 {
t.Errorf("Unexpected exists count : \n%v \n%v", localStats, remoteStats)
}
// Get - should succeed and pull manifest into local
_, err = env.manifests.Get(env.manifestDigest)
_, err = env.manifests.Get(ctx, env.manifestDigest)
if err != nil {
t.Fatal(err)
}
if (*localStats)["get"] != 1 && (*remoteStats)["get"] != 1 {
t.Errorf("Unexpected get count")
}
@ -216,7 +206,7 @@ func TestProxyManifests(t *testing.T) {
}
// Stat - should only go to local
exists, err = env.manifests.ExistsByTag("latest")
exists, err = env.manifests.Exists(ctx, env.manifestDigest)
if err != nil {
t.Fatal(err)
}
@ -224,19 +214,21 @@ func TestProxyManifests(t *testing.T) {
t.Errorf("Unexpected non-existant manifest")
}
if (*localStats)["existbytag"] != 2 && (*remoteStats)["existbytag"] != 1 {
if (*localStats)["exists"] != 2 && (*remoteStats)["exists"] != 1 {
t.Errorf("Unexpected exists count")
}
// Get - should get from remote, to test freshness
_, err = env.manifests.Get(env.manifestDigest)
_, err = env.manifests.Get(ctx, env.manifestDigest)
if err != nil {
t.Fatal(err)
}
if (*remoteStats)["get"] != 2 && (*remoteStats)["existsbytag"] != 1 && (*localStats)["put"] != 1 {
if (*remoteStats)["get"] != 2 && (*remoteStats)["exists"] != 1 && (*localStats)["put"] != 1 {
t.Errorf("Unexpected get count")
}
}
func TestProxyTagService(t *testing.T) {
}

23
registry/proxy/proxyregistry.go
View file

@ -42,6 +42,7 @@ func NewRegistryPullThroughCache(ctx context.Context, registry distribution.Name
s.OnManifestExpire(func(repoName string) error {
return v.RemoveRepository(repoName)
})
err = s.Start()
if err != nil {
return nil, err
@ -78,7 +79,7 @@ func (pr *proxyingRegistry) Repository(ctx context.Context, name string) (distri
if err != nil {
return nil, err
}
localManifests, err := localRepo.Manifests(ctx, storage.SkipLayerVerification)
localManifests, err := localRepo.Manifests(ctx, storage.SkipLayerVerification())
if err != nil {
return nil, err
}
@ -106,8 +107,11 @@ func (pr *proxyingRegistry) Repository(ctx context.Context, name string) (distri
ctx: ctx,
scheduler: pr.scheduler,
},
name: name,
signatures: localRepo.Signatures(),
name: name,
tags: proxyTagService{
localTags: localRepo.Tags(ctx),
remoteTags: remoteRepo.Tags(ctx),
},
}, nil
}
@ -115,14 +119,13 @@ func (pr *proxyingRegistry) Repository(ctx context.Context, name string) (distri
// locally, or pulling it through from a remote and caching it locally if it doesn't
// already exist
type proxiedRepository struct {
blobStore distribution.BlobStore
manifests distribution.ManifestService
name string
signatures distribution.SignatureService
blobStore distribution.BlobStore
manifests distribution.ManifestService
name string
tags distribution.TagService
}
func (pr *proxiedRepository) Manifests(ctx context.Context, options ...distribution.ManifestServiceOption) (distribution.ManifestService, error) {
// options
return pr.manifests, nil
}
@ -134,6 +137,6 @@ func (pr *proxiedRepository) Name() string {
return pr.name
}
func (pr *proxiedRepository) Signatures() distribution.SignatureService {
return pr.signatures
func (pr *proxiedRepository) Tags(ctx context.Context) distribution.TagService {
return pr.tags
}

58
registry/proxy/proxytagservice.go Normal file
View file

@ -0,0 +1,58 @@
package proxy
import (
"github.com/docker/distribution"
"github.com/docker/distribution/context"
)
// proxyTagService supports local and remote lookup of tags.
type proxyTagService struct {
localTags distribution.TagService
remoteTags distribution.TagService
}
var _ distribution.TagService = proxyTagService{}
// Get attempts to get the most recent digest for the tag by checking the remote
// tag service first and then caching it locally. If the remote is unavailable
// the local association is returned
func (pt proxyTagService) Get(ctx context.Context, tag string) (distribution.Descriptor, error) {
desc, err := pt.remoteTags.Get(ctx, tag)
if err == nil {
err := pt.localTags.Tag(ctx, tag, desc)
if err != nil {
return distribution.Descriptor{}, err
}
return desc, nil
}
desc, err = pt.localTags.Get(ctx, tag)
if err != nil {
return distribution.Descriptor{}, err
}
return desc, nil
}
func (pt proxyTagService) Tag(ctx context.Context, tag string, desc distribution.Descriptor) error {
return distribution.ErrUnsupported
}
func (pt proxyTagService) Untag(ctx context.Context, tag string) error {
err := pt.localTags.Untag(ctx, tag)
if err != nil {
return err
}
return nil
}
func (pt proxyTagService) All(ctx context.Context) ([]string, error) {
tags, err := pt.remoteTags.All(ctx)
if err == nil {
return tags, err
}
return pt.localTags.All(ctx)
}
func (pt proxyTagService) Lookup(ctx context.Context, digest distribution.Descriptor) ([]string, error) {
return []string{}, distribution.ErrUnsupported
}

164
registry/proxy/proxytagservice_test.go Normal file
View file

@ -0,0 +1,164 @@
package proxy
import (
"sort"
"sync"
"testing"
"github.com/docker/distribution"
"github.com/docker/distribution/context"
)
type mockTagStore struct {
mapping map[string]distribution.Descriptor
sync.Mutex
}
var _ distribution.TagService = &mockTagStore{}
func (m *mockTagStore) Get(ctx context.Context, tag string) (distribution.Descriptor, error) {
m.Lock()
defer m.Unlock()
if d, ok := m.mapping[tag]; ok {
return d, nil
}
return distribution.Descriptor{}, distribution.ErrTagUnknown{}
}
func (m *mockTagStore) Tag(ctx context.Context, tag string, desc distribution.Descriptor) error {
m.Lock()
defer m.Unlock()
m.mapping[tag] = desc
return nil
}
func (m *mockTagStore) Untag(ctx context.Context, tag string) error {
m.Lock()
defer m.Unlock()
if _, ok := m.mapping[tag]; ok {
delete(m.mapping, tag)
return nil
}
return distribution.ErrTagUnknown{}
}
func (m *mockTagStore) All(ctx context.Context) ([]string, error) {
m.Lock()
defer m.Unlock()
var tags []string
for tag := range m.mapping {
tags = append(tags, tag)
}
return tags, nil
}
func (m *mockTagStore) Lookup(ctx context.Context, digest distribution.Descriptor) ([]string, error) {
panic("not implemented")
}
func testProxyTagService(local, remote map[string]distribution.Descriptor) *proxyTagService {
if local == nil {
local = make(map[string]distribution.Descriptor)
}
if remote == nil {
remote = make(map[string]distribution.Descriptor)
}
return &proxyTagService{
localTags: &mockTagStore{mapping: local},
remoteTags: &mockTagStore{mapping: remote},
}
}
func TestGet(t *testing.T) {
remoteDesc := distribution.Descriptor{Size: 42}
remoteTag := "remote"
proxyTags := testProxyTagService(map[string]distribution.Descriptor{remoteTag: remoteDesc}, nil)
ctx := context.Background()
// Get pre-loaded tag
d, err := proxyTags.Get(ctx, remoteTag)
if err != nil {
t.Fatal(err)
}
if d != remoteDesc {
t.Fatal("unable to get put tag")
}
local, err := proxyTags.localTags.Get(ctx, remoteTag)
if err != nil {
t.Fatal("remote tag not pulled into store")
}
if local != remoteDesc {
t.Fatalf("unexpected descriptor pulled through")
}
// Manually overwrite remote tag
newRemoteDesc := distribution.Descriptor{Size: 43}
err = proxyTags.remoteTags.Tag(ctx, remoteTag, newRemoteDesc)
if err != nil {
t.Fatal(err)
}
d, err = proxyTags.Get(ctx, remoteTag)
if err != nil {
t.Fatal(err)
}
if d != newRemoteDesc {
t.Fatal("unable to get put tag")
}
_, err = proxyTags.localTags.Get(ctx, remoteTag)
if err != nil {
t.Fatal("remote tag not pulled into store")
}
// untag, ensure it's removed locally, but present in remote
err = proxyTags.Untag(ctx, remoteTag)
if err != nil {
t.Fatal(err)
}
_, err = proxyTags.localTags.Get(ctx, remoteTag)
if err == nil {
t.Fatalf("Expected error getting Untag'd tag")
}
_, err = proxyTags.remoteTags.Get(ctx, remoteTag)
if err != nil {
t.Fatalf("remote tag should not be untagged with proxyTag.Untag")
}
_, err = proxyTags.Get(ctx, remoteTag)
if err != nil {
t.Fatal("untagged tag should be pulled through")
}
// Add another tag. Ensure both tags appear in enumerate
err = proxyTags.remoteTags.Tag(ctx, "funtag", distribution.Descriptor{Size: 42})
if err != nil {
t.Fatal(err)
}
all, err := proxyTags.All(ctx)
if err != nil {
t.Fatal(err)
}
if len(all) != 2 {
t.Fatalf("Unexpected tag length returned from All() : %d ", len(all))
}
sort.Strings(all)
if all[0] != "funtag" && all[1] != "remote" {
t.Fatalf("Unexpected tags returned from All() : %v ", all)
}
}

159
registry/storage/manifeststore.go
View file

@ -1,6 +1,7 @@
package storage
import (
"encoding/json"
"fmt"
"github.com/docker/distribution"
@ -11,20 +12,21 @@ import (
"github.com/docker/libtrust"
)
// manifestStore is a storage driver based store for storing schema1 manifests.
type manifestStore struct {
repository *repository
revisionStore *revisionStore
tagStore *tagStore
blobStore *linkedBlobStore
ctx context.Context
signatures *signatureStore
skipDependencyVerification bool
}
var _ distribution.ManifestService = &manifestStore{}
func (ms *manifestStore) Exists(dgst digest.Digest) (bool, error) {
func (ms *manifestStore) Exists(ctx context.Context, dgst digest.Digest) (bool, error) {
context.GetLogger(ms.ctx).Debug("(*manifestStore).Exists")
_, err := ms.revisionStore.blobStore.Stat(ms.ctx, dgst)
_, err := ms.blobStore.Stat(ms.ctx, dgst)
if err != nil {
if err == distribution.ErrBlobUnknown {
return false, nil
@ -36,76 +38,131 @@ func (ms *manifestStore) Exists(dgst digest.Digest) (bool, error) {
return true, nil
}
func (ms *manifestStore) Get(dgst digest.Digest) (*schema1.SignedManifest, error) {
func (ms *manifestStore) Get(ctx context.Context, dgst digest.Digest, options ...distribution.ManifestServiceOption) (distribution.Manifest, error) {
context.GetLogger(ms.ctx).Debug("(*manifestStore).Get")
return ms.revisionStore.get(ms.ctx, dgst)
// Ensure that this revision is available in this repository.
_, err := ms.blobStore.Stat(ctx, dgst)
if err != nil {
if err == distribution.ErrBlobUnknown {
return nil, distribution.ErrManifestUnknownRevision{
Name: ms.repository.Name(),
Revision: dgst,
}
}
return nil, err
}
// TODO(stevvooe): Need to check descriptor from above to ensure that the
// mediatype is as we expect for the manifest store.
content, err := ms.blobStore.Get(ctx, dgst)
if err != nil {
if err == distribution.ErrBlobUnknown {
return nil, distribution.ErrManifestUnknownRevision{
Name: ms.repository.Name(),
Revision: dgst,
}
}
return nil, err
}
// Fetch the signatures for the manifest
signatures, err := ms.signatures.Get(dgst)
if err != nil {
return nil, err
}
jsig, err := libtrust.NewJSONSignature(content, signatures...)
if err != nil {
return nil, err
}
// Extract the pretty JWS
raw, err := jsig.PrettySignature("signatures")
if err != nil {
return nil, err
}
var sm schema1.SignedManifest
if err := json.Unmarshal(raw, &sm); err != nil {
return nil, err
}
return &sm, nil
}
// SkipLayerVerification allows a manifest to be Put before it's
// SkipLayerVerification allows a manifest to be Put before its
// layers are on the filesystem
func SkipLayerVerification(ms distribution.ManifestService) error {
if ms, ok := ms.(*manifestStore); ok {
func SkipLayerVerification() distribution.ManifestServiceOption {
return skipLayerOption{}
}
type skipLayerOption struct{}
func (o skipLayerOption) Apply(m distribution.ManifestService) error {
if ms, ok := m.(*manifestStore); ok {
ms.skipDependencyVerification = true
return nil
}
return fmt.Errorf("skip layer verification only valid for manifestStore")
}
func (ms *manifestStore) Put(manifest *schema1.SignedManifest) error {
func (ms *manifestStore) Put(ctx context.Context, manifest distribution.Manifest, options ...distribution.ManifestServiceOption) (digest.Digest, error) {
context.GetLogger(ms.ctx).Debug("(*manifestStore).Put")
if err := ms.verifyManifest(ms.ctx, manifest); err != nil {
return err
sm, ok := manifest.(*schema1.SignedManifest)
if !ok {
return "", fmt.Errorf("non-v1 manifest put to signed manifestStore: %T", manifest)
}
// Store the revision of the manifest
revision, err := ms.revisionStore.put(ms.ctx, manifest)
if err := ms.verifyManifest(ms.ctx, *sm); err != nil {
return "", err
}
mt := schema1.MediaTypeManifest
payload := sm.Canonical
revision, err := ms.blobStore.Put(ctx, mt, payload)
if err != nil {
return err
context.GetLogger(ctx).Errorf("error putting payload into blobstore: %v", err)
return "", err
}
// Now, tag the manifest
return ms.tagStore.tag(manifest.Tag, revision.Digest)
// Link the revision into the repository.
if err := ms.blobStore.linkBlob(ctx, revision); err != nil {
return "", err
}
// Grab each json signature and store them.
signatures, err := sm.Signatures()
if err != nil {
return "", err
}
if err := ms.signatures.Put(revision.Digest, signatures...); err != nil {
return "", err
}
return revision.Digest, nil
}
// Delete removes the revision of the specified manfiest.
func (ms *manifestStore) Delete(dgst digest.Digest) error {
func (ms *manifestStore) Delete(ctx context.Context, dgst digest.Digest) error {
context.GetLogger(ms.ctx).Debug("(*manifestStore).Delete")
return ms.revisionStore.delete(ms.ctx, dgst)
return ms.blobStore.Delete(ctx, dgst)
}
func (ms *manifestStore) Tags() ([]string, error) {
context.GetLogger(ms.ctx).Debug("(*manifestStore).Tags")
return ms.tagStore.tags()
}
func (ms *manifestStore) ExistsByTag(tag string) (bool, error) {
context.GetLogger(ms.ctx).Debug("(*manifestStore).ExistsByTag")
return ms.tagStore.exists(tag)
}
func (ms *manifestStore) GetByTag(tag string, options ...distribution.ManifestServiceOption) (*schema1.SignedManifest, error) {
for _, option := range options {
err := option(ms)
if err != nil {
return nil, err
}
}
context.GetLogger(ms.ctx).Debug("(*manifestStore).GetByTag")
dgst, err := ms.tagStore.resolve(tag)
if err != nil {
return nil, err
}
return ms.revisionStore.get(ms.ctx, dgst)
func (ms *manifestStore) Enumerate(ctx context.Context, manifests []distribution.Manifest, last distribution.Manifest) (n int, err error) {
return 0, distribution.ErrUnsupported
}
// verifyManifest ensures that the manifest content is valid from the
// perspective of the registry. It ensures that the signature is valid for the
// enclosed payload. As a policy, the registry only tries to store valid
// content, leaving trust policies of that content up to consumers.
func (ms *manifestStore) verifyManifest(ctx context.Context, mnfst *schema1.SignedManifest) error {
// content, leaving trust policies of that content up to consumems.
func (ms *manifestStore) verifyManifest(ctx context.Context, mnfst schema1.SignedManifest) error {
var errs distribution.ErrManifestVerification
if len(mnfst.Name) > reference.NameTotalLengthMax {
@ -129,7 +186,7 @@ func (ms *manifestStore) verifyManifest(ctx context.Context, mnfst *schema1.Sign
len(mnfst.History), len(mnfst.FSLayers)))
}
if _, err := schema1.Verify(mnfst); err != nil {
if _, err := schema1.Verify(&mnfst); err != nil {
switch err {
case libtrust.ErrMissingSignatureKey, libtrust.ErrInvalidJSONContent, libtrust.ErrMissingSignatureKey:
errs = append(errs, distribution.ErrManifestUnverified{})
@ -143,15 +200,15 @@ func (ms *manifestStore) verifyManifest(ctx context.Context, mnfst *schema1.Sign
}
if !ms.skipDependencyVerification {
for _, fsLayer := range mnfst.FSLayers {
_, err := ms.repository.Blobs(ctx).Stat(ctx, fsLayer.BlobSum)
for _, fsLayer := range mnfst.References() {
_, err := ms.repository.Blobs(ctx).Stat(ctx, fsLayer.Digest)
if err != nil {
if err != distribution.ErrBlobUnknown {
errs = append(errs, err)
}
// On error here, we always append unknown blob errors.
errs = append(errs, distribution.ErrManifestBlobUnknown{Digest: fsLayer.BlobSum})
// On error here, we always append unknown blob erroms.
errs = append(errs, distribution.ErrManifestBlobUnknown{Digest: fsLayer.Digest})
}
}
}

100
registry/storage/manifeststore_test.go
View file

@ -30,7 +30,8 @@ type manifestStoreTestEnv struct {
func newManifestStoreTestEnv(t *testing.T, name, tag string) *manifestStoreTestEnv {
ctx := context.Background()
driver := inmemory.New()
registry, err := NewRegistry(ctx, driver, BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()), EnableDelete, EnableRedirect)
registry, err := NewRegistry(ctx, driver, BlobDescriptorCacheProvider(
memory.NewInMemoryBlobDescriptorCacheProvider()), EnableDelete, EnableRedirect)
if err != nil {
t.Fatalf("error creating registry: %v", err)
}
@ -58,24 +59,6 @@ func TestManifestStorage(t *testing.T) {
t.Fatal(err)
}
exists, err := ms.ExistsByTag(env.tag)
if err != nil {
t.Fatalf("unexpected error checking manifest existence: %v", err)
}
if exists {
t.Fatalf("manifest should not exist")
}
if _, err := ms.GetByTag(env.tag); true {
switch err.(type) {
case distribution.ErrManifestUnknown:
break
default:
t.Fatalf("expected manifest unknown error: %#v", err)
}
}
m := schema1.Manifest{
Versioned: manifest.Versioned{
SchemaVersion: 1,
@ -114,7 +97,7 @@ func TestManifestStorage(t *testing.T) {
t.Fatalf("error signing manifest: %v", err)
}
err = ms.Put(sm)
_, err = ms.Put(ctx, sm)
if err == nil {
t.Fatalf("expected errors putting manifest with full verification")
}
@ -150,30 +133,40 @@ func TestManifestStorage(t *testing.T) {
}
}
if err = ms.Put(sm); err != nil {
var manifestDigest digest.Digest
if manifestDigest, err = ms.Put(ctx, sm); err != nil {
t.Fatalf("unexpected error putting manifest: %v", err)
}
exists, err = ms.ExistsByTag(env.tag)
exists, err := ms.Exists(ctx, manifestDigest)
if err != nil {
t.Fatalf("unexpected error checking manifest existence: %v", err)
t.Fatalf("unexpected error checking manifest existence: %#v", err)
}
if !exists {
t.Fatalf("manifest should exist")
}
fetchedManifest, err := ms.GetByTag(env.tag)
fromStore, err := ms.Get(ctx, manifestDigest)
if err != nil {
t.Fatalf("unexpected error fetching manifest: %v", err)
}
fetchedManifest, ok := fromStore.(*schema1.SignedManifest)
if !ok {
t.Fatalf("unexpected manifest type from signedstore")
}
if !reflect.DeepEqual(fetchedManifest, sm) {
t.Fatalf("fetched manifest not equal: %#v != %#v", fetchedManifest, sm)
}
fetchedJWS, err := libtrust.ParsePrettySignature(fetchedManifest.Raw, "signatures")
_, pl, err := fetchedManifest.Payload()
if err != nil {
t.Fatalf("error getting payload %#v", err)
}
fetchedJWS, err := libtrust.ParsePrettySignature(pl, "signatures")
if err != nil {
t.Fatalf("unexpected error parsing jws: %v", err)
}
@ -185,8 +178,9 @@ func TestManifestStorage(t *testing.T) {
// Now that we have a payload, take a moment to check that the manifest is
// return by the payload digest.
dgst := digest.FromBytes(payload)
exists, err = ms.Exists(dgst)
exists, err = ms.Exists(ctx, dgst)
if err != nil {
t.Fatalf("error checking manifest existence by digest: %v", err)
}
@ -195,7 +189,7 @@ func TestManifestStorage(t *testing.T) {
t.Fatalf("manifest %s should exist", dgst)
}
fetchedByDigest, err := ms.Get(dgst)
fetchedByDigest, err := ms.Get(ctx, dgst)
if err != nil {
t.Fatalf("unexpected error fetching manifest by digest: %v", err)
}
@ -213,20 +207,6 @@ func TestManifestStorage(t *testing.T) {
t.Fatalf("unexpected number of signatures: %d != %d", len(sigs), 1)
}
// Grabs the tags and check that this tagged manifest is present
tags, err := ms.Tags()
if err != nil {
t.Fatalf("unexpected error fetching tags: %v", err)
}
if len(tags) != 1 {
t.Fatalf("unexpected tags returned: %v", tags)
}
if tags[0] != env.tag {
t.Fatalf("unexpected tag found in tags: %v != %v", tags, []string{env.tag})
}
// Now, push the same manifest with a different key
pk2, err := libtrust.GenerateECP256PrivateKey()
if err != nil {
@ -237,8 +217,12 @@ func TestManifestStorage(t *testing.T) {
if err != nil {
t.Fatalf("unexpected error signing manifest: %v", err)
}
_, pl, err = sm2.Payload()
if err != nil {
t.Fatalf("error getting payload %#v", err)
}
jws2, err := libtrust.ParsePrettySignature(sm2.Raw, "signatures")
jws2, err := libtrust.ParsePrettySignature(pl, "signatures")
if err != nil {
t.Fatalf("error parsing signature: %v", err)
}
@ -252,15 +236,20 @@ func TestManifestStorage(t *testing.T) {
t.Fatalf("unexpected number of signatures: %d != %d", len(sigs2), 1)
}
if err = ms.Put(sm2); err != nil {
if manifestDigest, err = ms.Put(ctx, sm2); err != nil {
t.Fatalf("unexpected error putting manifest: %v", err)
}
fetched, err := ms.GetByTag(env.tag)
fromStore, err = ms.Get(ctx, manifestDigest)
if err != nil {
t.Fatalf("unexpected error fetching manifest: %v", err)
}
fetched, ok := fromStore.(*schema1.SignedManifest)
if !ok {
t.Fatalf("unexpected type from signed manifeststore : %T", fetched)
}
if _, err := schema1.Verify(fetched); err != nil {
t.Fatalf("unexpected error verifying manifest: %v", err)
}
@ -276,7 +265,12 @@ func TestManifestStorage(t *testing.T) {
t.Fatalf("unexpected error getting expected signatures: %v", err)
}
receivedJWS, err := libtrust.ParsePrettySignature(fetched.Raw, "signatures")
_, pl, err = fetched.Payload()
if err != nil {
t.Fatalf("error getting payload %#v", err)
}
receivedJWS, err := libtrust.ParsePrettySignature(pl, "signatures")
if err != nil {
t.Fatalf("unexpected error parsing jws: %v", err)
}
@ -302,12 +296,12 @@ func TestManifestStorage(t *testing.T) {
}
// Test deleting manifests
err = ms.Delete(dgst)
err = ms.Delete(ctx, dgst)
if err != nil {
t.Fatalf("unexpected an error deleting manifest by digest: %v", err)
}
exists, err = ms.Exists(dgst)
exists, err = ms.Exists(ctx, dgst)
if err != nil {
t.Fatalf("Error querying manifest existence")
}
@ -315,7 +309,7 @@ func TestManifestStorage(t *testing.T) {
t.Errorf("Deleted manifest should not exist")
}
deletedManifest, err := ms.Get(dgst)
deletedManifest, err := ms.Get(ctx, dgst)
if err == nil {
t.Errorf("Unexpected success getting deleted manifest")
}
@ -331,12 +325,12 @@ func TestManifestStorage(t *testing.T) {
}
// Re-upload should restore manifest to a good state
err = ms.Put(sm)
_, err = ms.Put(ctx, sm)
if err != nil {
t.Errorf("Error re-uploading deleted manifest")
}
exists, err = ms.Exists(dgst)
exists, err = ms.Exists(ctx, dgst)
if err != nil {
t.Fatalf("Error querying manifest existence")
}
@ -344,7 +338,7 @@ func TestManifestStorage(t *testing.T) {
t.Errorf("Restored manifest should exist")
}
deletedManifest, err = ms.Get(dgst)
deletedManifest, err = ms.Get(ctx, dgst)
if err != nil {
t.Errorf("Unexpected error getting manifest")
}
@ -364,7 +358,7 @@ func TestManifestStorage(t *testing.T) {
if err != nil {
t.Fatal(err)
}
err = ms.Delete(dgst)
err = ms.Delete(ctx, dgst)
if err == nil {
t.Errorf("Unexpected success deleting while disabled")
}

54
registry/storage/registry.go
View file

@ -145,6 +145,15 @@ func (repo *repository) Name() string {
return repo.name
}
func (repo *repository) Tags(ctx context.Context) distribution.TagService {
tags := &tagStore{
repository: repo,
blobStore: repo.registry.blobStore,
}
return tags
}
// Manifests returns an instance of ManifestService. Instantiation is cheap and
// may be context sensitive in the future. The instance should be used similar
// to a request local.
@ -159,36 +168,31 @@ func (repo *repository) Manifests(ctx context.Context, options ...distribution.M
ms := &manifestStore{
ctx: ctx,
repository: repo,
revisionStore: &revisionStore{
ctx: ctx,
repository: repo,
blobStore: &linkedBlobStore{
ctx: ctx,
blobStore: repo.blobStore,
repository: repo,
deleteEnabled: repo.registry.deleteEnabled,
blobAccessController: &linkedBlobStatter{
blobStore: repo.blobStore,
repository: repo,
linkPathFns: manifestLinkPathFns,
},
// TODO(stevvooe): linkPath limits this blob store to only
// manifests. This instance cannot be used for blob checks.
linkPathFns: manifestLinkPathFns,
resumableDigestEnabled: repo.resumableDigestEnabled,
blobStore: &linkedBlobStore{
ctx: ctx,
blobStore: repo.blobStore,
repository: repo,
deleteEnabled: repo.registry.deleteEnabled,
blobAccessController: &linkedBlobStatter{
blobStore: repo.blobStore,
repository: repo,
linkPathFns: manifestLinkPathFns,
},
// TODO(stevvooe): linkPath limits this blob store to only
// manifests. This instance cannot be used for blob checks.
linkPathFns: manifestLinkPathFns,
},
tagStore: &tagStore{
signatures: &signatureStore{
ctx: ctx,
repository: repo,
blobStore: repo.registry.blobStore,
blobStore: repo.blobStore,
},
}
// Apply options
for _, option := range options {
err := option(ms)
err := option.Apply(ms)
if err != nil {
return nil, err
}
@ -225,11 +229,3 @@ func (repo *repository) Blobs(ctx context.Context) distribution.BlobStore {
resumableDigestEnabled: repo.resumableDigestEnabled,
}
}
func (repo *repository) Signatures() distribution.SignatureService {
return &signatureStore{
repository: repo,
blobStore: repo.blobStore,
ctx: repo.ctx,
}
}

111
registry/storage/revisionstore.go
View file

@ -1,111 +0,0 @@
package storage
import (
"encoding/json"
"github.com/docker/distribution"
"github.com/docker/distribution/context"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/libtrust"
)
// revisionStore supports storing and managing manifest revisions.
type revisionStore struct {
repository *repository
blobStore *linkedBlobStore
ctx context.Context
}
// get retrieves the manifest, keyed by revision digest.
func (rs *revisionStore) get(ctx context.Context, revision digest.Digest) (*schema1.SignedManifest, error) {
// Ensure that this revision is available in this repository.
_, err := rs.blobStore.Stat(ctx, revision)
if err != nil {
if err == distribution.ErrBlobUnknown {
return nil, distribution.ErrManifestUnknownRevision{
Name: rs.repository.Name(),
Revision: revision,
}
}
return nil, err
}
// TODO(stevvooe): Need to check descriptor from above to ensure that the
// mediatype is as we expect for the manifest store.
content, err := rs.blobStore.Get(ctx, revision)
if err != nil {
if err == distribution.ErrBlobUnknown {
return nil, distribution.ErrManifestUnknownRevision{
Name: rs.repository.Name(),
Revision: revision,
}
}
return nil, err
}
// Fetch the signatures for the manifest
signatures, err := rs.repository.Signatures().Get(revision)
if err != nil {
return nil, err
}
jsig, err := libtrust.NewJSONSignature(content, signatures...)
if err != nil {
return nil, err
}
// Extract the pretty JWS
raw, err := jsig.PrettySignature("signatures")
if err != nil {
return nil, err
}
var sm schema1.SignedManifest
if err := json.Unmarshal(raw, &sm); err != nil {
return nil, err
}
return &sm, nil
}
// put stores the manifest in the repository, if not already present. Any
// updated signatures will be stored, as well.
func (rs *revisionStore) put(ctx context.Context, sm *schema1.SignedManifest) (distribution.Descriptor, error) {
// Resolve the payload in the manifest.
payload, err := sm.Payload()
if err != nil {
return distribution.Descriptor{}, err
}
// Digest and store the manifest payload in the blob store.
revision, err := rs.blobStore.Put(ctx, schema1.ManifestMediaType, payload)
if err != nil {
context.GetLogger(ctx).Errorf("error putting payload into blobstore: %v", err)
return distribution.Descriptor{}, err
}
// Link the revision into the repository.
if err := rs.blobStore.linkBlob(ctx, revision); err != nil {
return distribution.Descriptor{}, err
}
// Grab each json signature and store them.
signatures, err := sm.Signatures()
if err != nil {
return distribution.Descriptor{}, err
}
if err := rs.repository.Signatures().Put(revision.Digest, signatures...); err != nil {
return distribution.Descriptor{}, err
}
return revision, nil
}
func (rs *revisionStore) delete(ctx context.Context, revision digest.Digest) error {
return rs.blobStore.Delete(ctx, revision)
}

11
registry/storage/signaturestore.go
View file

@ -4,7 +4,6 @@ import (
"path"
"sync"
"github.com/docker/distribution"
"github.com/docker/distribution/context"
"github.com/docker/distribution/digest"
)
@ -15,16 +14,6 @@ type signatureStore struct {
ctx context.Context
}
func newSignatureStore(ctx context.Context, repo *repository, blobStore *blobStore) *signatureStore {
return &signatureStore{
ctx: ctx,
repository: repo,
blobStore: blobStore,
}
}
var _ distribution.SignatureService = &signatureStore{}
func (s *signatureStore) Get(dgst digest.Digest) ([][]byte, error) {
signaturesPath, err := pathFor(manifestSignaturesPathSpec{
name: s.repository.Name(),

64
registry/storage/tagstore.go
View file

@ -9,37 +9,41 @@ import (
storagedriver "github.com/docker/distribution/registry/storage/driver"
)
var _ distribution.TagService = &tagStore{}
// tagStore provides methods to manage manifest tags in a backend storage driver.
// This implementation uses the same on-disk layout as the (now deleted) tag
// store. This provides backward compatibility with current registry deployments
// which only makes use of the Digest field of the returned distribution.Descriptor
// but does not enable full roundtripping of Descriptor objects
type tagStore struct {
repository *repository
blobStore *blobStore
ctx context.Context
}
// tags lists the manifest tags for the specified repository.
func (ts *tagStore) tags() ([]string, error) {
p, err := pathFor(manifestTagPathSpec{
// All returns all tags
func (ts *tagStore) All(ctx context.Context) ([]string, error) {
var tags []string
pathSpec, err := pathFor(manifestTagPathSpec{
name: ts.repository.Name(),
})
if err != nil {
return nil, err
return tags, err
}
var tags []string
entries, err := ts.blobStore.driver.List(ts.ctx, p)
entries, err := ts.blobStore.driver.List(ctx, pathSpec)
if err != nil {
switch err := err.(type) {
case storagedriver.PathNotFoundError:
return nil, distribution.ErrRepositoryUnknown{Name: ts.repository.Name()}
return tags, distribution.ErrRepositoryUnknown{Name: ts.repository.Name()}
default:
return nil, err
return tags, err
}
}
for _, entry := range entries {
_, filename := path.Split(entry)
tags = append(tags, filename)
}
@ -47,7 +51,7 @@ func (ts *tagStore) tags() ([]string, error) {
}
// exists returns true if the specified manifest tag exists in the repository.
func (ts *tagStore) exists(tag string) (bool, error) {
func (ts *tagStore) exists(ctx context.Context, tag string) (bool, error) {
tagPath, err := pathFor(manifestTagCurrentPathSpec{
name: ts.repository.Name(),
tag: tag,
@ -57,7 +61,7 @@ func (ts *tagStore) exists(tag string) (bool, error) {
return false, err
}
exists, err := exists(ts.ctx, ts.blobStore.driver, tagPath)
exists, err := exists(ctx, ts.blobStore.driver, tagPath)
if err != nil {
return false, err
}
@ -65,9 +69,9 @@ func (ts *tagStore) exists(tag string) (bool, error) {
return exists, nil
}
// tag tags the digest with the given tag, updating the the store to point at
// Tag tags the digest with the given tag, updating the the store to point at
// the current tag. The digest must point to a manifest.
func (ts *tagStore) tag(tag string, revision digest.Digest) error {
func (ts *tagStore) Tag(ctx context.Context, tag string, desc distribution.Descriptor) error {
currentPath, err := pathFor(manifestTagCurrentPathSpec{
name: ts.repository.Name(),
tag: tag,
@ -77,43 +81,44 @@ func (ts *tagStore) tag(tag string, revision digest.Digest) error {
return err
}
nbs := ts.linkedBlobStore(ts.ctx, tag)
lbs := ts.linkedBlobStore(ctx, tag)
// Link into the index
if err := nbs.linkBlob(ts.ctx, distribution.Descriptor{Digest: revision}); err != nil {
if err := lbs.linkBlob(ctx, desc); err != nil {
return err
}
// Overwrite the current link
return ts.blobStore.link(ts.ctx, currentPath, revision)
return ts.blobStore.link(ctx, currentPath, desc.Digest)
}
// resolve the current revision for name and tag.
func (ts *tagStore) resolve(tag string) (digest.Digest, error) {
func (ts *tagStore) Get(ctx context.Context, tag string) (distribution.Descriptor, error) {
currentPath, err := pathFor(manifestTagCurrentPathSpec{
name: ts.repository.Name(),
tag: tag,
})
if err != nil {
return "", err
return distribution.Descriptor{}, err
}
revision, err := ts.blobStore.readlink(ts.ctx, currentPath)
revision, err := ts.blobStore.readlink(ctx, currentPath)
if err != nil {
switch err.(type) {
case storagedriver.PathNotFoundError:
return "", distribution.ErrManifestUnknown{Name: ts.repository.Name(), Tag: tag}
return distribution.Descriptor{}, distribution.ErrTagUnknown{Tag: tag}
}
return "", err
return distribution.Descriptor{}, err
}
return revision, nil
return distribution.Descriptor{Digest: revision}, nil
}
// delete removes the tag from repository, including the history of all
// revisions that have the specified tag.
func (ts *tagStore) delete(tag string) error {
func (ts *tagStore) Untag(ctx context.Context, tag string) error {
tagPath, err := pathFor(manifestTagPathSpec{
name: ts.repository.Name(),
tag: tag,
@ -123,7 +128,7 @@ func (ts *tagStore) delete(tag string) error {
return err
}
return ts.blobStore.driver.Delete(ts.ctx, tagPath)
return ts.blobStore.driver.Delete(ctx, tagPath)
}
// linkedBlobStore returns the linkedBlobStore for the named tag, allowing one
@ -145,3 +150,10 @@ func (ts *tagStore) linkedBlobStore(ctx context.Context, tag string) *linkedBlob
}},
}
}
// Lookup recovers a list of tags which refer to this digest. When a manifest is deleted by
// digest, tag entries which point to it need to be recovered to avoid dangling tags.
func (ts *tagStore) Lookup(ctx context.Context, digest distribution.Descriptor) ([]string, error) {
// An efficient implementation of this will require changes to the S3 driver.
return make([]string, 0), nil
}

150
registry/storage/tagstore_test.go Normal file
View file

@ -0,0 +1,150 @@
package storage
import (
"testing"
"github.com/docker/distribution"
"github.com/docker/distribution/context"
"github.com/docker/distribution/registry/storage/driver/inmemory"
)
type tagsTestEnv struct {
ts distribution.TagService
ctx context.Context
}
func testTagStore(t *testing.T) *tagsTestEnv {
ctx := context.Background()
d := inmemory.New()
reg, err := NewRegistry(ctx, d)
if err != nil {
t.Fatal(err)
}
repo, err := reg.Repository(ctx, "a/b")
if err != nil {
t.Fatal(err)
}
return &tagsTestEnv{
ctx: ctx,
ts: repo.Tags(ctx),
}
}
func TestTagStoreTag(t *testing.T) {
env := testTagStore(t)
tags := env.ts
ctx := env.ctx
d := distribution.Descriptor{}
err := tags.Tag(ctx, "latest", d)
if err == nil {
t.Errorf("unexpected error putting malformed descriptor : %s", err)
}
d.Digest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
err = tags.Tag(ctx, "latest", d)
if err != nil {
t.Error(err)
}
d1, err := tags.Get(ctx, "latest")
if err != nil {
t.Error(err)
}
if d1.Digest != d.Digest {
t.Error("put and get digest differ")
}
// Overwrite existing
d.Digest = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
err = tags.Tag(ctx, "latest", d)
if err != nil {
t.Error(err)
}
d1, err = tags.Get(ctx, "latest")
if err != nil {
t.Error(err)
}
if d1.Digest != d.Digest {
t.Error("put and get digest differ")
}
}
func TestTagStoreUnTag(t *testing.T) {
env := testTagStore(t)
tags := env.ts
ctx := env.ctx
desc := distribution.Descriptor{Digest: "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}
err := tags.Untag(ctx, "latest")
if err == nil {
t.Errorf("Expected error untagging non-existant tag")
}
err = tags.Tag(ctx, "latest", desc)
if err != nil {
t.Error(err)
}
err = tags.Untag(ctx, "latest")
if err != nil {
t.Error(err)
}
_, err = tags.Get(ctx, "latest")
if err == nil {
t.Error("Expected error getting untagged tag")
}
}
func TestTagAll(t *testing.T) {
env := testTagStore(t)
tagStore := env.ts
ctx := env.ctx
alpha := "abcdefghijklmnopqrstuvwxyz"
for i := 0; i < len(alpha); i++ {
tag := alpha[i]
desc := distribution.Descriptor{Digest: "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"}
err := tagStore.Tag(ctx, string(tag), desc)
if err != nil {
t.Error(err)
}
}
all, err := tagStore.All(ctx)
if err != nil {
t.Error(err)
}
if len(all) != len(alpha) {
t.Errorf("Unexpected count returned from enumerate")
}
for i, c := range all {
if c != string(alpha[i]) {
t.Errorf("unexpected tag in enumerate %s", c)
}
}
removed := "a"
err = tagStore.Untag(ctx, removed)
if err != nil {
t.Error(err)
}
all, err = tagStore.All(ctx)
if err != nil {
t.Error(err)
}
for _, tag := range all {
if tag == removed {
t.Errorf("unexpected tag in enumerate %s", removed)
}
}
}

27
tags.go Normal file
View file

@ -0,0 +1,27 @@
package distribution
import (
"github.com/docker/distribution/context"
)
// TagService provides access to information about tagged objects.
type TagService interface {
// Get retrieves the descriptor identified by the tag. Some
// implementations may differentiate between "trusted" tags and
// "untrusted" tags. If a tag is "untrusted", the mapping will be returned
// as an ErrTagUntrusted error, with the target descriptor.
Get(ctx context.Context, tag string) (Descriptor, error)
// Tag associates the tag with the provided descriptor, updating the
// current association, if needed.
Tag(ctx context.Context, tag string, desc Descriptor) error
// Untag removes the given tag association
Untag(ctx context.Context, tag string) error
// All returns the set of tags managed by this tag service
All(ctx context.Context) ([]string, error)
// Lookup returns the set of tags referencing the given digest.
Lookup(ctx context.Context, digest Descriptor) ([]string, error)
}