From 4bbabc6e36006c165106ab4d7ffb342ec9263883 Mon Sep 17 00:00:00 2001 From: Stephen J Day Date: Fri, 21 Nov 2014 17:04:35 -0800 Subject: [PATCH 1/3] Implement path spec for manifest storage --- storage/paths.go | 28 +++++++++++++++++++++------- storage/paths_test.go | 7 +++++++ 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/storage/paths.go b/storage/paths.go index 18aef17ec..87c0b2fda 100644 --- a/storage/paths.go +++ b/storage/paths.go @@ -24,7 +24,7 @@ const storagePathVersion = "v2" // /v2 // -> repositories/ // ->/ -// -> images/ +// -> manifests/ // // -> layers/ // -> tarsum/ @@ -48,6 +48,7 @@ const storagePathVersion = "v2" // // We cover the path formats implemented by this path mapper below. // +// manifestPathSpec: /v2/repositories//manifests/ // layerLinkPathSpec: /v2/repositories//layers/tarsum/// // layerIndexLinkPathSpec: /v2/layerindex/tarsum/// // blobPathSpec: /v2/blob/sha256// @@ -84,7 +85,13 @@ func (pm *pathMapper) path(spec pathSpec) (string, error) { // to an intermediate path object, than can be consumed and mapped by the // other version. + rootPrefix := []string{pm.root, pm.version} + repoPrefix := append(rootPrefix, "repositories") + switch v := spec.(type) { + case manifestPathSpec: + // TODO(sday): May need to store manifest by architecture. + return path.Join(append(repoPrefix, v.name, "manifests", v.tag)...), nil case layerLinkPathSpec: if !strings.HasPrefix(v.digest.Algorithm(), "tarsum") { // Only tarsum is supported, for now @@ -101,9 +108,8 @@ func (pm *pathMapper) path(spec pathSpec) (string, error) { return "", err } - p := path.Join(append([]string{pm.root, pm.version, "repositories", v.name, "layers"}, tarSumInfoPathComponents(tsi)...)...) - - return p, nil + return path.Join(append(append(repoPrefix, v.name, "layers"), + tarSumInfoPathComponents(tsi)...)...), nil case layerIndexLinkPathSpec: if !strings.HasPrefix(v.digest.Algorithm(), "tarsum") { // Only tarsum is supported, for now @@ -120,9 +126,8 @@ func (pm *pathMapper) path(spec pathSpec) (string, error) { return "", err } - p := path.Join(append([]string{pm.root, pm.version, "layerindex"}, tarSumInfoPathComponents(tsi)...)...) - - return p, nil + return path.Join(append(append(rootPrefix, "layerindex"), + tarSumInfoPathComponents(tsi)...)...), nil case blobPathSpec: p := path.Join([]string{pm.root, pm.version, "blob", v.alg, v.digest[:2], v.digest}...) return p, nil @@ -139,6 +144,15 @@ type pathSpec interface { pathSpec() } +// manifestPathSpec describes the path elements used to build a manifest path. +// The contents should be a signed manifest json file. +type manifestPathSpec struct { + name string + tag string +} + +func (manifestPathSpec) pathSpec() {} + // layerLink specifies a path for a layer link, which is a file with a blob // id. The layer link will contain a content addressable blob id reference // into the blob store. The format of the contents is as follows: diff --git a/storage/paths_test.go b/storage/paths_test.go index 5dc4c07c5..d2ff542f5 100644 --- a/storage/paths_test.go +++ b/storage/paths_test.go @@ -16,6 +16,13 @@ func TestPathMapper(t *testing.T) { expected string err error }{ + { + spec: manifestPathSpec{ + name: "foo/bar", + tag: "thetag", + }, + expected: "/pathmapper-test/repositories/foo/bar/manifests/thetag", + }, { spec: layerLinkPathSpec{ name: "foo/bar", From eaadb82e1e45547726d53c8c36a34efdc6015024 Mon Sep 17 00:00:00 2001 From: Stephen J Day Date: Fri, 21 Nov 2014 19:29:08 -0800 Subject: [PATCH 2/3] Move Manifest type into storage package This changeset move the Manifest type into the storage package to make the type accessible to client and registry without import cycles. The structure of the manifest was also changed to accuratle reflect the stages of the signing process. A straw man Manifest.Sign method has been added to start testing this concept out but will probably be accompanied by the more import SignedManifest.Verify method as the security model develops. This is probably the start of a concerted effort to consolidate types across the client and server portions of the code base but we may want to see how such a handy type, like the Manifest and SignedManifest, would work in docker core. --- client/client.go | 11 ++-- client/client_test.go | 58 +++++++++++--------- client/objectstore.go | 12 ++-- client/pull.go | 4 +- client/push.go | 6 +- errors.go | 2 +- images.go | 67 +--------------------- layer.go | 2 +- storage/manifest.go | 125 ++++++++++++++++++++++++++++++++++++++++++ 9 files changed, 179 insertions(+), 108 deletions(-) create mode 100644 storage/manifest.go diff --git a/client/client.go b/client/client.go index 944050e07..e51476cdc 100644 --- a/client/client.go +++ b/client/client.go @@ -12,17 +12,18 @@ import ( "github.com/docker/docker-registry" "github.com/docker/docker-registry/digest" + "github.com/docker/docker-registry/storage" ) // Client implements the client interface to the registry http api type Client interface { // GetImageManifest returns an image manifest for the image at the given // name, tag pair. - GetImageManifest(name, tag string) (*registry.ImageManifest, error) + GetImageManifest(name, tag string) (*storage.SignedManifest, error) // PutImageManifest uploads an image manifest for the image at the given // name, tag pair. - PutImageManifest(name, tag string, imageManifest *registry.ImageManifest) error + PutImageManifest(name, tag string, imageManifest *storage.SignedManifest) error // DeleteImage removes the image at the given name, tag pair. DeleteImage(name, tag string) error @@ -81,7 +82,7 @@ type clientImpl struct { // TODO(bbland): use consistent route generation between server and client -func (r *clientImpl) GetImageManifest(name, tag string) (*registry.ImageManifest, error) { +func (r *clientImpl) GetImageManifest(name, tag string) (*storage.SignedManifest, error) { response, err := http.Get(r.imageManifestURL(name, tag)) if err != nil { return nil, err @@ -108,7 +109,7 @@ func (r *clientImpl) GetImageManifest(name, tag string) (*registry.ImageManifest decoder := json.NewDecoder(response.Body) - manifest := new(registry.ImageManifest) + manifest := new(storage.SignedManifest) err = decoder.Decode(manifest) if err != nil { return nil, err @@ -116,7 +117,7 @@ func (r *clientImpl) GetImageManifest(name, tag string) (*registry.ImageManifest return manifest, nil } -func (r *clientImpl) PutImageManifest(name, tag string, manifest *registry.ImageManifest) error { +func (r *clientImpl) PutImageManifest(name, tag string, manifest *storage.SignedManifest) error { manifestBytes, err := json.Marshal(manifest) if err != nil { return err diff --git a/client/client_test.go b/client/client_test.go index a77e7665e..dc75789dd 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -9,9 +9,9 @@ import ( "sync" "testing" - "github.com/docker/docker-registry" "github.com/docker/docker-registry/common/testutil" "github.com/docker/docker-registry/digest" + "github.com/docker/docker-registry/storage" ) type testBlob struct { @@ -33,8 +33,8 @@ func TestPush(t *testing.T) { }, } uploadLocations := make([]string, len(testBlobs)) - blobs := make([]registry.FSLayer, len(testBlobs)) - history := make([]registry.ManifestHistory, len(testBlobs)) + blobs := make([]storage.FSLayer, len(testBlobs)) + history := make([]storage.ManifestHistory, len(testBlobs)) for i, blob := range testBlobs { // TODO(bbland): this is returning the same location for all uploads, @@ -42,17 +42,21 @@ func TestPush(t *testing.T) { // It's sort of okay because we're using unique digests, but this needs // to change at some point. uploadLocations[i] = fmt.Sprintf("/v2/%s/blob/test-uuid", name) - blobs[i] = registry.FSLayer{BlobSum: blob.digest} - history[i] = registry.ManifestHistory{V1Compatibility: blob.digest.String()} + blobs[i] = storage.FSLayer{BlobSum: blob.digest} + history[i] = storage.ManifestHistory{V1Compatibility: blob.digest.String()} } - manifest := ®istry.ImageManifest{ - Name: name, - Tag: tag, - Architecture: "x86", - FSLayers: blobs, - History: history, - SchemaVersion: 1, + manifest := &storage.SignedManifest{ + Manifest: storage.Manifest{ + Name: name, + Tag: tag, + Architecture: "x86", + FSLayers: blobs, + History: history, + Versioned: storage.Versioned{ + SchemaVersion: 1, + }, + }, } manifestBytes, err := json.Marshal(manifest) @@ -102,7 +106,7 @@ func TestPush(t *testing.T) { client := New(server.URL) objectStore := &memoryObjectStore{ mutex: new(sync.Mutex), - manifestStorage: make(map[string]*registry.ImageManifest), + manifestStorage: make(map[string]*storage.SignedManifest), layerStorage: make(map[digest.Digest]Layer), } @@ -142,21 +146,25 @@ func TestPull(t *testing.T) { contents: []byte("some other contents"), }, } - blobs := make([]registry.FSLayer, len(testBlobs)) - history := make([]registry.ManifestHistory, len(testBlobs)) + blobs := make([]storage.FSLayer, len(testBlobs)) + history := make([]storage.ManifestHistory, len(testBlobs)) for i, blob := range testBlobs { - blobs[i] = registry.FSLayer{BlobSum: blob.digest} - history[i] = registry.ManifestHistory{V1Compatibility: blob.digest.String()} + blobs[i] = storage.FSLayer{BlobSum: blob.digest} + history[i] = storage.ManifestHistory{V1Compatibility: blob.digest.String()} } - manifest := ®istry.ImageManifest{ - Name: name, - Tag: tag, - Architecture: "x86", - FSLayers: blobs, - History: history, - SchemaVersion: 1, + manifest := &storage.SignedManifest{ + Manifest: storage.Manifest{ + Name: name, + Tag: tag, + Architecture: "x86", + FSLayers: blobs, + History: history, + Versioned: storage.Versioned{ + SchemaVersion: 1, + }, + }, } manifestBytes, err := json.Marshal(manifest) @@ -190,7 +198,7 @@ func TestPull(t *testing.T) { client := New(server.URL) objectStore := &memoryObjectStore{ mutex: new(sync.Mutex), - manifestStorage: make(map[string]*registry.ImageManifest), + manifestStorage: make(map[string]*storage.SignedManifest), layerStorage: make(map[digest.Digest]Layer), } diff --git a/client/objectstore.go b/client/objectstore.go index bee73ff07..177f9acaf 100644 --- a/client/objectstore.go +++ b/client/objectstore.go @@ -8,8 +8,8 @@ import ( "io/ioutil" "sync" - "github.com/docker/docker-registry" "github.com/docker/docker-registry/digest" + "github.com/docker/docker-registry/storage" ) var ( @@ -28,11 +28,11 @@ var ( type ObjectStore interface { // Manifest retrieves the image manifest stored at the given repository name // and tag - Manifest(name, tag string) (*registry.ImageManifest, error) + Manifest(name, tag string) (*storage.SignedManifest, error) // WriteManifest stores an image manifest at the given repository name and // tag - WriteManifest(name, tag string, manifest *registry.ImageManifest) error + WriteManifest(name, tag string, manifest *storage.SignedManifest) error // Layer returns a handle to a layer for reading and writing Layer(dgst digest.Digest) (Layer, error) @@ -56,11 +56,11 @@ type Layer interface { // memoryObjectStore is an in-memory implementation of the ObjectStore interface type memoryObjectStore struct { mutex *sync.Mutex - manifestStorage map[string]*registry.ImageManifest + manifestStorage map[string]*storage.SignedManifest layerStorage map[digest.Digest]Layer } -func (objStore *memoryObjectStore) Manifest(name, tag string) (*registry.ImageManifest, error) { +func (objStore *memoryObjectStore) Manifest(name, tag string) (*storage.SignedManifest, error) { objStore.mutex.Lock() defer objStore.mutex.Unlock() @@ -71,7 +71,7 @@ func (objStore *memoryObjectStore) Manifest(name, tag string) (*registry.ImageMa return manifest, nil } -func (objStore *memoryObjectStore) WriteManifest(name, tag string, manifest *registry.ImageManifest) error { +func (objStore *memoryObjectStore) WriteManifest(name, tag string, manifest *storage.SignedManifest) error { objStore.mutex.Lock() defer objStore.mutex.Unlock() diff --git a/client/pull.go b/client/pull.go index bce067568..435e40b97 100644 --- a/client/pull.go +++ b/client/pull.go @@ -4,7 +4,7 @@ import ( "fmt" "io" - "github.com/docker/docker-registry" + "github.com/docker/docker-registry/storage" log "github.com/Sirupsen/logrus" ) @@ -77,7 +77,7 @@ func Pull(c Client, objectStore ObjectStore, name, tag string) error { return nil } -func pullLayer(c Client, objectStore ObjectStore, name string, fsLayer registry.FSLayer) error { +func pullLayer(c Client, objectStore ObjectStore, name string, fsLayer storage.FSLayer) error { log.WithField("layer", fsLayer).Info("Pulling layer") layer, err := objectStore.Layer(fsLayer.BlobSum) diff --git a/client/push.go b/client/push.go index 087260586..c0ff10d1d 100644 --- a/client/push.go +++ b/client/push.go @@ -5,7 +5,7 @@ import ( "io" "io/ioutil" - "github.com/docker/docker-registry" + "github.com/docker/docker-registry/storage" log "github.com/Sirupsen/logrus" ) @@ -15,7 +15,7 @@ import ( // push window has been successfully pushed. const simultaneousLayerPushWindow = 4 -type pushFunction func(fsLayer registry.FSLayer) error +type pushFunction func(fsLayer storage.FSLayer) error // Push implements a client push workflow for the image defined by the given // name and tag pair, using the given ObjectStore for local manifest and layer @@ -74,7 +74,7 @@ func Push(c Client, objectStore ObjectStore, name, tag string) error { return nil } -func pushLayer(c Client, objectStore ObjectStore, name string, fsLayer registry.FSLayer) error { +func pushLayer(c Client, objectStore ObjectStore, name string, fsLayer storage.FSLayer) error { log.WithField("layer", fsLayer).Info("Pushing layer") layer, err := objectStore.Layer(fsLayer.BlobSum) diff --git a/errors.go b/errors.go index e2f16ba05..b61704302 100644 --- a/errors.go +++ b/errors.go @@ -212,7 +212,7 @@ type DetailUnknownLayer struct { // Unknown should contain the contents of a layer descriptor, which is a // single FSLayer currently. - Unknown FSLayer `json:"unknown"` + Unknown storage.FSLayer `json:"unknown"` } // RepositoryNotFoundError is returned when making an operation against a diff --git a/images.go b/images.go index 534069b26..317651e22 100644 --- a/images.go +++ b/images.go @@ -2,76 +2,13 @@ package registry import ( "encoding/json" + "fmt" "net/http" - "github.com/docker/docker-registry/digest" + "github.com/docker/docker-registry/storage" "github.com/gorilla/handlers" ) -// ImageManifest defines the structure of an image manifest -type ImageManifest struct { - // Name is the name of the image's repository - Name string `json:"name"` - - // Tag is the tag of the image specified by this manifest - Tag string `json:"tag"` - - // Architecture is the host architecture on which this image is intended to - // run - Architecture string `json:"architecture"` - - // FSLayers is a list of filesystem layer blobSums contained in this image - FSLayers []FSLayer `json:"fsLayers"` - - // History is a list of unstructured historical data for v1 compatibility - History []ManifestHistory `json:"history"` - - // SchemaVersion is the image manifest schema that this image follows - SchemaVersion int `json:"schemaVersion"` - - // Raw is the byte representation of the ImageManifest, used for signature - // verification - Raw []byte `json:"-"` -} - -// imageManifest is used to avoid recursion in unmarshaling -type imageManifest ImageManifest - -// UnmarshalJSON populates a new ImageManifest struct from JSON data. -func (m *ImageManifest) UnmarshalJSON(b []byte) error { - var manifest imageManifest - err := json.Unmarshal(b, &manifest) - if err != nil { - return err - } - - *m = ImageManifest(manifest) - m.Raw = b - return nil -} - -// FSLayer is a container struct for BlobSums defined in an image manifest -type FSLayer struct { - // BlobSum is the tarsum of the referenced filesystem image layer - BlobSum digest.Digest `json:"blobSum"` -} - -// ManifestHistory stores unstructured v1 compatibility information -type ManifestHistory struct { - // V1Compatibility is the raw v1 compatibility information - V1Compatibility string `json:"v1Compatibility"` -} - -// Checksum is a container struct for an image checksum -type Checksum struct { - // HashAlgorithm is the algorithm used to compute the checksum - // Supported values: md5, sha1, sha256, sha512 - HashAlgorithm string - - // Sum is the actual checksum value for the given HashAlgorithm - Sum string -} - // imageManifestDispatcher takes the request context and builds the // appropriate handler for handling image manifest requests. func imageManifestDispatcher(ctx *Context, r *http.Request) http.Handler { diff --git a/layer.go b/layer.go index 38fdfe39b..5e1c6f45d 100644 --- a/layer.go +++ b/layer.go @@ -52,7 +52,7 @@ func (lh *layerHandler) GetLayer(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNotFound) lh.Errors.Push(ErrorCodeUnknownLayer, map[string]interface{}{ - "unknown": FSLayer{BlobSum: lh.Digest}, + "unknown": storage.FSLayer{BlobSum: lh.Digest}, }) return default: diff --git a/storage/manifest.go b/storage/manifest.go new file mode 100644 index 000000000..9921fbea6 --- /dev/null +++ b/storage/manifest.go @@ -0,0 +1,125 @@ +package storage + +import ( + "encoding/json" + "fmt" + + "github.com/docker/libtrust" + + "github.com/docker/docker-registry/digest" +) + +var ( + // ErrManifestUnknown is returned if the manifest is not known by the + // registry. + ErrManifestUnknown = fmt.Errorf("unknown manifest") + + // ErrManifestUnverified is returned when the registry is unable to verify + // the manifest. + ErrManifestUnverified = fmt.Errorf("unverified manifest") +) + +// Versioned provides a struct with just the manifest schemaVersion. Incoming +// content with unknown schema version can be decoded against this struct to +// check the version. +type Versioned struct { + // SchemaVersion is the image manifest schema that this image follows + SchemaVersion int `json:"schemaVersion"` +} + +// Manifest provides the base accessible fields for working with V2 image +// format in the registry. +type Manifest struct { + Versioned + + // Name is the name of the image's repository + Name string `json:"name"` + + // Tag is the tag of the image specified by this manifest + Tag string `json:"tag"` + + // Architecture is the host architecture on which this image is intended to + // run + Architecture string `json:"architecture"` + + // FSLayers is a list of filesystem layer blobSums contained in this image + FSLayers []FSLayer `json:"fsLayers"` + + // History is a list of unstructured historical data for v1 compatibility + History []ManifestHistory `json:"history"` +} + +// Sign signs the manifest with the provided private key, returning a +// SignedManifest. This typically won't be used within the registry, except +// for testing. +func (m *Manifest) Sign(pk libtrust.PrivateKey) (*SignedManifest, error) { + p, err := json.Marshal(m) + if err != nil { + return nil, err + } + + js, err := libtrust.NewJSONSignature(p) + if err != nil { + return nil, err + } + + if err := js.Sign(pk); err != nil { + return nil, err + } + + pretty, err := js.PrettySignature("signatures") + if err != nil { + return nil, err + } + + return &SignedManifest{ + Manifest: *m, + Raw: pretty, + }, nil +} + +// SignedManifest provides an envelope for +type SignedManifest struct { + Manifest + + // Raw is the byte representation of the ImageManifest, used for signature + // verification. The manifest byte representation cannot change or it will + // have to be re-signed. + Raw []byte `json:"-"` +} + +// UnmarshalJSON populates a new ImageManifest struct from JSON data. +func (m *SignedManifest) UnmarshalJSON(b []byte) error { + var manifest Manifest + if err := json.Unmarshal(b, &manifest); err != nil { + return err + } + + m.Manifest = manifest + m.Raw = b + + return nil +} + +// MarshalJSON returns the contents of raw. If Raw is nil, marshals the inner +// contents. +func (m *SignedManifest) MarshalJSON() ([]byte, error) { + if len(m.Raw) > 0 { + return m.Raw, nil + } + + // If the raw data is not available, just dump the inner content. + return json.Marshal(&m.Manifest) +} + +// FSLayer is a container struct for BlobSums defined in an image manifest +type FSLayer struct { + // BlobSum is the tarsum of the referenced filesystem image layer + BlobSum digest.Digest `json:"blobSum"` +} + +// ManifestHistory stores unstructured v1 compatibility information +type ManifestHistory struct { + // V1Compatibility is the raw v1 compatibility information + V1Compatibility string `json:"v1Compatibility"` +} From 4decfaa82e6e3b9754b436910670d75b4eca77e7 Mon Sep 17 00:00:00 2001 From: Stephen J Day Date: Fri, 21 Nov 2014 19:39:52 -0800 Subject: [PATCH 3/3] Initial implementation of image manifest storage This change implements the first pass at image manifest storage on top of the storagedriver. Very similar to LayerService, its much simpler due to less complexity of pushing and pulling images. Various components are still missing, such as detailed error reporting on missing layers during verification, but the base functionality is present. --- errors.go | 1 + images.go | 3 - storage/layer.go | 17 ----- storage/manifest_test.go | 139 +++++++++++++++++++++++++++++++++++++++ storage/manifeststore.go | 134 +++++++++++++++++++++++++++++++++++++ storage/services.go | 40 +++++++++++ 6 files changed, 314 insertions(+), 20 deletions(-) create mode 100644 storage/manifest_test.go create mode 100644 storage/manifeststore.go diff --git a/errors.go b/errors.go index b61704302..113097ddf 100644 --- a/errors.go +++ b/errors.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/docker/docker-registry/digest" + "github.com/docker/docker-registry/storage" ) // ErrorCode represents the error type. The errors are serialized via strings diff --git a/images.go b/images.go index 317651e22..f16a3560f 100644 --- a/images.go +++ b/images.go @@ -1,11 +1,8 @@ package registry import ( - "encoding/json" - "fmt" "net/http" - "github.com/docker/docker-registry/storage" "github.com/gorilla/handlers" ) diff --git a/storage/layer.go b/storage/layer.go index d2ddfb070..dc6b34228 100644 --- a/storage/layer.go +++ b/storage/layer.go @@ -8,23 +8,6 @@ import ( "github.com/docker/docker-registry/digest" ) -// LayerService provides operations on layer files in a backend storage. -type LayerService interface { - // Exists returns true if the layer exists. - Exists(name string, digest digest.Digest) (bool, error) - - // Fetch the layer identifed by TarSum. - Fetch(name string, digest digest.Digest) (Layer, error) - - // Upload begins a layer upload to repository identified by name, - // returning a handle. - Upload(name string) (LayerUpload, error) - - // Resume continues an in progress layer upload, returning the current - // state of the upload. - Resume(uuid string) (LayerUpload, error) -} - // Layer provides a readable and seekable layer object. Typically, // implementations are *not* goroutine safe. type Layer interface { diff --git a/storage/manifest_test.go b/storage/manifest_test.go new file mode 100644 index 000000000..c96c1dec8 --- /dev/null +++ b/storage/manifest_test.go @@ -0,0 +1,139 @@ +package storage + +import ( + "reflect" + "testing" + + "github.com/docker/libtrust" + + "github.com/docker/docker-registry/digest" + "github.com/docker/docker-registry/storagedriver/inmemory" +) + +func TestManifestStorage(t *testing.T) { + driver := inmemory.New() + ms := &manifestStore{ + driver: driver, + pathMapper: &pathMapper{ + root: "/storage/testing", + version: storagePathVersion, + }, + layerService: newMockedLayerService(), + } + + name := "foo/bar" + tag := "thetag" + + exists, err := ms.Exists(name, tag) + if err != nil { + t.Fatalf("unexpected error checking manifest existence: %v", err) + } + + if exists { + t.Fatalf("manifest should not exist") + } + + if _, err := ms.Get(name, tag); err != ErrManifestUnknown { + t.Fatalf("expected manifest unknown error: %v != %v", err, ErrManifestUnknown) + } + + manifest := Manifest{ + Versioned: Versioned{ + SchemaVersion: 1, + }, + Name: name, + Tag: tag, + FSLayers: []FSLayer{ + { + BlobSum: "asdf", + }, + { + BlobSum: "qwer", + }, + }, + } + + pk, err := libtrust.GenerateECP256PrivateKey() + if err != nil { + t.Fatalf("unexpected error generating private key: %v", err) + } + + sm, err := manifest.Sign(pk) + if err != nil { + t.Fatalf("error signing manifest: %v", err) + } + + err = ms.Put(name, tag, sm) + if err == nil { + t.Fatalf("expected errors putting manifest") + } + + // TODO(stevvooe): We expect errors describing all of the missing layers. + + ms.layerService.(*mockedExistenceLayerService).add(name, "asdf") + ms.layerService.(*mockedExistenceLayerService).add(name, "qwer") + + if err = ms.Put(name, tag, sm); err != nil { + t.Fatalf("unexpected error putting manifest: %v", err) + } + + exists, err = ms.Exists(name, tag) + if err != nil { + t.Fatalf("unexpected error checking manifest existence: %v", err) + } + + if !exists { + t.Fatalf("manifest should exist") + } + + fetchedManifest, err := ms.Get(name, tag) + if err != nil { + t.Fatalf("unexpected error fetching manifest: %v", err) + } + + if !reflect.DeepEqual(fetchedManifest, sm) { + t.Fatalf("fetched manifest not equal: %#v != %#v", fetchedManifest, sm) + } +} + +type layerKey struct { + name string + digest digest.Digest +} + +type mockedExistenceLayerService struct { + exists map[layerKey]struct{} +} + +func newMockedLayerService() *mockedExistenceLayerService { + return &mockedExistenceLayerService{ + exists: make(map[layerKey]struct{}), + } +} + +var _ LayerService = &mockedExistenceLayerService{} + +func (mels *mockedExistenceLayerService) add(name string, digest digest.Digest) { + mels.exists[layerKey{name: name, digest: digest}] = struct{}{} +} + +func (mels *mockedExistenceLayerService) remove(name string, digest digest.Digest) { + delete(mels.exists, layerKey{name: name, digest: digest}) +} + +func (mels *mockedExistenceLayerService) Exists(name string, digest digest.Digest) (bool, error) { + _, ok := mels.exists[layerKey{name: name, digest: digest}] + return ok, nil +} + +func (mockedExistenceLayerService) Fetch(name string, digest digest.Digest) (Layer, error) { + panic("not implemented") +} + +func (mockedExistenceLayerService) Upload(name string) (LayerUpload, error) { + panic("not implemented") +} + +func (mockedExistenceLayerService) Resume(uuid string) (LayerUpload, error) { + panic("not implemented") +} diff --git a/storage/manifeststore.go b/storage/manifeststore.go new file mode 100644 index 000000000..1b76c8c03 --- /dev/null +++ b/storage/manifeststore.go @@ -0,0 +1,134 @@ +package storage + +import ( + "encoding/json" + "fmt" + + "github.com/docker/libtrust" + + "github.com/docker/docker-registry/storagedriver" +) + +type manifestStore struct { + driver storagedriver.StorageDriver + pathMapper *pathMapper + layerService LayerService +} + +var _ ManifestService = &manifestStore{} + +func (ms *manifestStore) Exists(name, tag string) (bool, error) { + p, err := ms.path(name, tag) + if err != nil { + return false, err + } + + size, err := ms.driver.CurrentSize(p) + if err != nil { + return false, err + } + + if size == 0 { + return false, nil + } + + return true, nil +} + +func (ms *manifestStore) Get(name, tag string) (*SignedManifest, error) { + p, err := ms.path(name, tag) + if err != nil { + return nil, err + } + + content, err := ms.driver.GetContent(p) + if err != nil { + switch err := err.(type) { + case storagedriver.PathNotFoundError, *storagedriver.PathNotFoundError: + return nil, ErrManifestUnknown + default: + return nil, err + } + } + + var manifest SignedManifest + + if err := json.Unmarshal(content, &manifest); err != nil { + // TODO(stevvooe): Corrupted manifest error? + return nil, err + } + + // TODO(stevvooe): Verify the manifest here? + + return &manifest, nil +} + +func (ms *manifestStore) Put(name, tag string, manifest *SignedManifest) error { + p, err := ms.path(name, tag) + if err != nil { + return err + } + + if err := ms.verifyManifest(name, tag, manifest); err != nil { + return err + } + + // TODO(stevvooe): Should we get manifest first? + + return ms.driver.PutContent(p, manifest.Raw) +} + +func (ms *manifestStore) Delete(name, tag string) error { + panic("not implemented") +} + +func (ms *manifestStore) path(name, tag string) (string, error) { + return ms.pathMapper.path(manifestPathSpec{ + name: name, + tag: tag, + }) +} + +func (ms *manifestStore) verifyManifest(name, tag string, manifest *SignedManifest) error { + if manifest.Name != name { + return fmt.Errorf("name does not match manifest name") + } + + if manifest.Tag != tag { + return fmt.Errorf("tag does not match manifest tag") + } + + var errs []error + + for _, fsLayer := range manifest.FSLayers { + exists, err := ms.layerService.Exists(name, fsLayer.BlobSum) + if err != nil { + // TODO(stevvooe): Need to store information about missing blob. + errs = append(errs, err) + } + + if !exists { + errs = append(errs, fmt.Errorf("missing layer %v", fsLayer.BlobSum)) + } + } + + if len(errs) != 0 { + // TODO(stevvooe): These need to be recoverable by a caller. + return fmt.Errorf("missing layers: %v", errs) + } + + js, err := libtrust.ParsePrettySignature(manifest.Raw, "signatures") + if err != nil { + return err + } + + _, err = js.Verify() // These pubkeys need to be checked. + if err != nil { + return err + } + + // TODO(sday): Pubkey checks need to go here. This where things get fancy. + // Perhaps, an injected service would reduce coupling here. + + return nil +} diff --git a/storage/services.go b/storage/services.go index afb26d943..1f6d5e51a 100644 --- a/storage/services.go +++ b/storage/services.go @@ -1,6 +1,7 @@ package storage import ( + "github.com/docker/docker-registry/digest" "github.com/docker/docker-registry/storagedriver" ) @@ -41,3 +42,42 @@ func NewServices(driver storagedriver.StorageDriver) *Services { func (ss *Services) Layers() LayerService { return &layerStore{driver: ss.driver, pathMapper: ss.pathMapper, uploadStore: ss.layerUploadStore} } + +// Manifests returns an instance of ManifestService. Instantiation is cheap and +// may be context sensitive in the future. The instance should be used similar +// to a request local. +func (ss *Services) Manifests() ManifestService { + return &manifestStore{driver: ss.driver, pathMapper: ss.pathMapper, layerService: ss.Layers()} +} + +// ManifestService provides operations on image manifests. +type ManifestService interface { + // Exists returns true if the layer exists. + Exists(name, tag string) (bool, error) + + // Get retrieves the named manifest, if it exists. + Get(name, tag string) (*SignedManifest, error) + + // Put creates or updates the named manifest. + Put(name, tag string, manifest *SignedManifest) error + + // Delete removes the named manifest, if it exists. + Delete(name, tag string) error +} + +// LayerService provides operations on layer files in a backend storage. +type LayerService interface { + // Exists returns true if the layer exists. + Exists(name string, digest digest.Digest) (bool, error) + + // Fetch the layer identifed by TarSum. + Fetch(name string, digest digest.Digest) (Layer, error) + + // Upload begins a layer upload to repository identified by name, + // returning a handle. + Upload(name string) (LayerUpload, error) + + // Resume continues an in progress layer upload, returning the current + // state of the upload. + Resume(uuid string) (LayerUpload, error) +}