From 4decfaa82e6e3b9754b436910670d75b4eca77e7 Mon Sep 17 00:00:00 2001 From: Stephen J Day Date: Fri, 21 Nov 2014 19:39:52 -0800 Subject: [PATCH] Initial implementation of image manifest storage This change implements the first pass at image manifest storage on top of the storagedriver. Very similar to LayerService, its much simpler due to less complexity of pushing and pulling images. Various components are still missing, such as detailed error reporting on missing layers during verification, but the base functionality is present. --- errors.go | 1 + images.go | 3 - storage/layer.go | 17 ----- storage/manifest_test.go | 139 +++++++++++++++++++++++++++++++++++++++ storage/manifeststore.go | 134 +++++++++++++++++++++++++++++++++++++ storage/services.go | 40 +++++++++++ 6 files changed, 314 insertions(+), 20 deletions(-) create mode 100644 storage/manifest_test.go create mode 100644 storage/manifeststore.go diff --git a/errors.go b/errors.go index b61704302..113097ddf 100644 --- a/errors.go +++ b/errors.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/docker/docker-registry/digest" + "github.com/docker/docker-registry/storage" ) // ErrorCode represents the error type. The errors are serialized via strings diff --git a/images.go b/images.go index 317651e22..f16a3560f 100644 --- a/images.go +++ b/images.go @@ -1,11 +1,8 @@ package registry import ( - "encoding/json" - "fmt" "net/http" - "github.com/docker/docker-registry/storage" "github.com/gorilla/handlers" ) diff --git a/storage/layer.go b/storage/layer.go index d2ddfb070..dc6b34228 100644 --- a/storage/layer.go +++ b/storage/layer.go @@ -8,23 +8,6 @@ import ( "github.com/docker/docker-registry/digest" ) -// LayerService provides operations on layer files in a backend storage. -type LayerService interface { - // Exists returns true if the layer exists. - Exists(name string, digest digest.Digest) (bool, error) - - // Fetch the layer identifed by TarSum. - Fetch(name string, digest digest.Digest) (Layer, error) - - // Upload begins a layer upload to repository identified by name, - // returning a handle. - Upload(name string) (LayerUpload, error) - - // Resume continues an in progress layer upload, returning the current - // state of the upload. - Resume(uuid string) (LayerUpload, error) -} - // Layer provides a readable and seekable layer object. Typically, // implementations are *not* goroutine safe. type Layer interface { diff --git a/storage/manifest_test.go b/storage/manifest_test.go new file mode 100644 index 000000000..c96c1dec8 --- /dev/null +++ b/storage/manifest_test.go @@ -0,0 +1,139 @@ +package storage + +import ( + "reflect" + "testing" + + "github.com/docker/libtrust" + + "github.com/docker/docker-registry/digest" + "github.com/docker/docker-registry/storagedriver/inmemory" +) + +func TestManifestStorage(t *testing.T) { + driver := inmemory.New() + ms := &manifestStore{ + driver: driver, + pathMapper: &pathMapper{ + root: "/storage/testing", + version: storagePathVersion, + }, + layerService: newMockedLayerService(), + } + + name := "foo/bar" + tag := "thetag" + + exists, err := ms.Exists(name, tag) + if err != nil { + t.Fatalf("unexpected error checking manifest existence: %v", err) + } + + if exists { + t.Fatalf("manifest should not exist") + } + + if _, err := ms.Get(name, tag); err != ErrManifestUnknown { + t.Fatalf("expected manifest unknown error: %v != %v", err, ErrManifestUnknown) + } + + manifest := Manifest{ + Versioned: Versioned{ + SchemaVersion: 1, + }, + Name: name, + Tag: tag, + FSLayers: []FSLayer{ + { + BlobSum: "asdf", + }, + { + BlobSum: "qwer", + }, + }, + } + + pk, err := libtrust.GenerateECP256PrivateKey() + if err != nil { + t.Fatalf("unexpected error generating private key: %v", err) + } + + sm, err := manifest.Sign(pk) + if err != nil { + t.Fatalf("error signing manifest: %v", err) + } + + err = ms.Put(name, tag, sm) + if err == nil { + t.Fatalf("expected errors putting manifest") + } + + // TODO(stevvooe): We expect errors describing all of the missing layers. + + ms.layerService.(*mockedExistenceLayerService).add(name, "asdf") + ms.layerService.(*mockedExistenceLayerService).add(name, "qwer") + + if err = ms.Put(name, tag, sm); err != nil { + t.Fatalf("unexpected error putting manifest: %v", err) + } + + exists, err = ms.Exists(name, tag) + if err != nil { + t.Fatalf("unexpected error checking manifest existence: %v", err) + } + + if !exists { + t.Fatalf("manifest should exist") + } + + fetchedManifest, err := ms.Get(name, tag) + if err != nil { + t.Fatalf("unexpected error fetching manifest: %v", err) + } + + if !reflect.DeepEqual(fetchedManifest, sm) { + t.Fatalf("fetched manifest not equal: %#v != %#v", fetchedManifest, sm) + } +} + +type layerKey struct { + name string + digest digest.Digest +} + +type mockedExistenceLayerService struct { + exists map[layerKey]struct{} +} + +func newMockedLayerService() *mockedExistenceLayerService { + return &mockedExistenceLayerService{ + exists: make(map[layerKey]struct{}), + } +} + +var _ LayerService = &mockedExistenceLayerService{} + +func (mels *mockedExistenceLayerService) add(name string, digest digest.Digest) { + mels.exists[layerKey{name: name, digest: digest}] = struct{}{} +} + +func (mels *mockedExistenceLayerService) remove(name string, digest digest.Digest) { + delete(mels.exists, layerKey{name: name, digest: digest}) +} + +func (mels *mockedExistenceLayerService) Exists(name string, digest digest.Digest) (bool, error) { + _, ok := mels.exists[layerKey{name: name, digest: digest}] + return ok, nil +} + +func (mockedExistenceLayerService) Fetch(name string, digest digest.Digest) (Layer, error) { + panic("not implemented") +} + +func (mockedExistenceLayerService) Upload(name string) (LayerUpload, error) { + panic("not implemented") +} + +func (mockedExistenceLayerService) Resume(uuid string) (LayerUpload, error) { + panic("not implemented") +} diff --git a/storage/manifeststore.go b/storage/manifeststore.go new file mode 100644 index 000000000..1b76c8c03 --- /dev/null +++ b/storage/manifeststore.go @@ -0,0 +1,134 @@ +package storage + +import ( + "encoding/json" + "fmt" + + "github.com/docker/libtrust" + + "github.com/docker/docker-registry/storagedriver" +) + +type manifestStore struct { + driver storagedriver.StorageDriver + pathMapper *pathMapper + layerService LayerService +} + +var _ ManifestService = &manifestStore{} + +func (ms *manifestStore) Exists(name, tag string) (bool, error) { + p, err := ms.path(name, tag) + if err != nil { + return false, err + } + + size, err := ms.driver.CurrentSize(p) + if err != nil { + return false, err + } + + if size == 0 { + return false, nil + } + + return true, nil +} + +func (ms *manifestStore) Get(name, tag string) (*SignedManifest, error) { + p, err := ms.path(name, tag) + if err != nil { + return nil, err + } + + content, err := ms.driver.GetContent(p) + if err != nil { + switch err := err.(type) { + case storagedriver.PathNotFoundError, *storagedriver.PathNotFoundError: + return nil, ErrManifestUnknown + default: + return nil, err + } + } + + var manifest SignedManifest + + if err := json.Unmarshal(content, &manifest); err != nil { + // TODO(stevvooe): Corrupted manifest error? + return nil, err + } + + // TODO(stevvooe): Verify the manifest here? + + return &manifest, nil +} + +func (ms *manifestStore) Put(name, tag string, manifest *SignedManifest) error { + p, err := ms.path(name, tag) + if err != nil { + return err + } + + if err := ms.verifyManifest(name, tag, manifest); err != nil { + return err + } + + // TODO(stevvooe): Should we get manifest first? + + return ms.driver.PutContent(p, manifest.Raw) +} + +func (ms *manifestStore) Delete(name, tag string) error { + panic("not implemented") +} + +func (ms *manifestStore) path(name, tag string) (string, error) { + return ms.pathMapper.path(manifestPathSpec{ + name: name, + tag: tag, + }) +} + +func (ms *manifestStore) verifyManifest(name, tag string, manifest *SignedManifest) error { + if manifest.Name != name { + return fmt.Errorf("name does not match manifest name") + } + + if manifest.Tag != tag { + return fmt.Errorf("tag does not match manifest tag") + } + + var errs []error + + for _, fsLayer := range manifest.FSLayers { + exists, err := ms.layerService.Exists(name, fsLayer.BlobSum) + if err != nil { + // TODO(stevvooe): Need to store information about missing blob. + errs = append(errs, err) + } + + if !exists { + errs = append(errs, fmt.Errorf("missing layer %v", fsLayer.BlobSum)) + } + } + + if len(errs) != 0 { + // TODO(stevvooe): These need to be recoverable by a caller. + return fmt.Errorf("missing layers: %v", errs) + } + + js, err := libtrust.ParsePrettySignature(manifest.Raw, "signatures") + if err != nil { + return err + } + + _, err = js.Verify() // These pubkeys need to be checked. + if err != nil { + return err + } + + // TODO(sday): Pubkey checks need to go here. This where things get fancy. + // Perhaps, an injected service would reduce coupling here. + + return nil +} diff --git a/storage/services.go b/storage/services.go index afb26d943..1f6d5e51a 100644 --- a/storage/services.go +++ b/storage/services.go @@ -1,6 +1,7 @@ package storage import ( + "github.com/docker/docker-registry/digest" "github.com/docker/docker-registry/storagedriver" ) @@ -41,3 +42,42 @@ func NewServices(driver storagedriver.StorageDriver) *Services { func (ss *Services) Layers() LayerService { return &layerStore{driver: ss.driver, pathMapper: ss.pathMapper, uploadStore: ss.layerUploadStore} } + +// Manifests returns an instance of ManifestService. Instantiation is cheap and +// may be context sensitive in the future. The instance should be used similar +// to a request local. +func (ss *Services) Manifests() ManifestService { + return &manifestStore{driver: ss.driver, pathMapper: ss.pathMapper, layerService: ss.Layers()} +} + +// ManifestService provides operations on image manifests. +type ManifestService interface { + // Exists returns true if the layer exists. + Exists(name, tag string) (bool, error) + + // Get retrieves the named manifest, if it exists. + Get(name, tag string) (*SignedManifest, error) + + // Put creates or updates the named manifest. + Put(name, tag string, manifest *SignedManifest) error + + // Delete removes the named manifest, if it exists. + Delete(name, tag string) error +} + +// LayerService provides operations on layer files in a backend storage. +type LayerService interface { + // Exists returns true if the layer exists. + Exists(name string, digest digest.Digest) (bool, error) + + // Fetch the layer identifed by TarSum. + Fetch(name string, digest digest.Digest) (Layer, error) + + // Upload begins a layer upload to repository identified by name, + // returning a handle. + Upload(name string) (LayerUpload, error) + + // Resume continues an in progress layer upload, returning the current + // state of the upload. + Resume(uuid string) (LayerUpload, error) +}