Initial implementation of image manifest storage

This change implements the first pass at image manifest storage on top of the
storagedriver. Very similar to LayerService, its much simpler due to less
complexity of pushing and pulling images.

Various components are still missing, such as detailed error reporting on
missing layers during verification, but the base functionality is present.

errors.go

@@ -5,6 +5,7 @@ import (
 	"strings"
 
 	"github.com/docker/docker-registry/digest"
+	"github.com/docker/docker-registry/storage"
 )
 
 // ErrorCode represents the error type. The errors are serialized via strings

images.go

@@ -1,11 +1,8 @@
 package registry
 
 import (
-	"encoding/json"
-	"fmt"
 	"net/http"
 
-	"github.com/docker/docker-registry/storage"
 	"github.com/gorilla/handlers"
 )
 

storage/layer.go

@@ -8,23 +8,6 @@ import (
 	"github.com/docker/docker-registry/digest"
 )
 
-// LayerService provides operations on layer files in a backend storage.
-type LayerService interface {
-	// Exists returns true if the layer exists.
-	Exists(name string, digest digest.Digest) (bool, error)
-
-	// Fetch the layer identifed by TarSum.
-	Fetch(name string, digest digest.Digest) (Layer, error)
-
-	// Upload begins a layer upload to repository identified by name,
-	// returning a handle.
-	Upload(name string) (LayerUpload, error)
-
-	// Resume continues an in progress layer upload, returning the current
-	// state of the upload.
-	Resume(uuid string) (LayerUpload, error)
-}
-
 // Layer provides a readable and seekable layer object. Typically,
 // implementations are *not* goroutine safe.
 type Layer interface {

storage/manifest_test.go

@@ -0,0 +1,139 @@
+package storage
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/docker/libtrust"
+
+	"github.com/docker/docker-registry/digest"
+	"github.com/docker/docker-registry/storagedriver/inmemory"
+)
+
+func TestManifestStorage(t *testing.T) {
+	driver := inmemory.New()
+	ms := &manifestStore{
+		driver: driver,
+		pathMapper: &pathMapper{
+			root:    "/storage/testing",
+			version: storagePathVersion,
+		},
+		layerService: newMockedLayerService(),
+	}
+
+	name := "foo/bar"
+	tag := "thetag"
+
+	exists, err := ms.Exists(name, tag)
+	if err != nil {
+		t.Fatalf("unexpected error checking manifest existence: %v", err)
+	}
+
+	if exists {
+		t.Fatalf("manifest should not exist")
+	}
+
+	if _, err := ms.Get(name, tag); err != ErrManifestUnknown {
+		t.Fatalf("expected manifest unknown error: %v != %v", err, ErrManifestUnknown)
+	}
+
+	manifest := Manifest{
+		Versioned: Versioned{
+			SchemaVersion: 1,
+		},
+		Name: name,
+		Tag:  tag,
+		FSLayers: []FSLayer{
+			{
+				BlobSum: "asdf",
+			},
+			{
+				BlobSum: "qwer",
+			},
+		},
+	}
+
+	pk, err := libtrust.GenerateECP256PrivateKey()
+	if err != nil {
+		t.Fatalf("unexpected error generating private key: %v", err)
+	}
+
+	sm, err := manifest.Sign(pk)
+	if err != nil {
+		t.Fatalf("error signing manifest: %v", err)
+	}
+
+	err = ms.Put(name, tag, sm)
+	if err == nil {
+		t.Fatalf("expected errors putting manifest")
+	}
+
+	// TODO(stevvooe): We expect errors describing all of the missing layers.
+
+	ms.layerService.(*mockedExistenceLayerService).add(name, "asdf")
+	ms.layerService.(*mockedExistenceLayerService).add(name, "qwer")
+
+	if err = ms.Put(name, tag, sm); err != nil {
+		t.Fatalf("unexpected error putting manifest: %v", err)
+	}
+
+	exists, err = ms.Exists(name, tag)
+	if err != nil {
+		t.Fatalf("unexpected error checking manifest existence: %v", err)
+	}
+
+	if !exists {
+		t.Fatalf("manifest should exist")
+	}
+
+	fetchedManifest, err := ms.Get(name, tag)
+	if err != nil {
+		t.Fatalf("unexpected error fetching manifest: %v", err)
+	}
+
+	if !reflect.DeepEqual(fetchedManifest, sm) {
+		t.Fatalf("fetched manifest not equal: %#v != %#v", fetchedManifest, sm)
+	}
+}
+
+type layerKey struct {
+	name   string
+	digest digest.Digest
+}
+
+type mockedExistenceLayerService struct {
+	exists map[layerKey]struct{}
+}
+
+func newMockedLayerService() *mockedExistenceLayerService {
+	return &mockedExistenceLayerService{
+		exists: make(map[layerKey]struct{}),
+	}
+}
+
+var _ LayerService = &mockedExistenceLayerService{}
+
+func (mels *mockedExistenceLayerService) add(name string, digest digest.Digest) {
+	mels.exists[layerKey{name: name, digest: digest}] = struct{}{}
+}
+
+func (mels *mockedExistenceLayerService) remove(name string, digest digest.Digest) {
+	delete(mels.exists, layerKey{name: name, digest: digest})
+}
+
+func (mels *mockedExistenceLayerService) Exists(name string, digest digest.Digest) (bool, error) {
+	_, ok := mels.exists[layerKey{name: name, digest: digest}]
+	return ok, nil
+}
+
+func (mockedExistenceLayerService) Fetch(name string, digest digest.Digest) (Layer, error) {
+	panic("not implemented")
+}
+
+func (mockedExistenceLayerService) Upload(name string) (LayerUpload, error) {
+	panic("not implemented")
+}
+
+func (mockedExistenceLayerService) Resume(uuid string) (LayerUpload, error) {
+	panic("not implemented")
+}

storage/manifeststore.go

@@ -0,0 +1,134 @@
+package storage
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/docker/libtrust"
+
+	"github.com/docker/docker-registry/storagedriver"
+)
+
+type manifestStore struct {
+	driver       storagedriver.StorageDriver
+	pathMapper   *pathMapper
+	layerService LayerService
+}
+
+var _ ManifestService = &manifestStore{}
+
+func (ms *manifestStore) Exists(name, tag string) (bool, error) {
+	p, err := ms.path(name, tag)
+	if err != nil {
+		return false, err
+	}
+
+	size, err := ms.driver.CurrentSize(p)
+	if err != nil {
+		return false, err
+	}
+
+	if size == 0 {
+		return false, nil
+	}
+
+	return true, nil
+}
+
+func (ms *manifestStore) Get(name, tag string) (*SignedManifest, error) {
+	p, err := ms.path(name, tag)
+	if err != nil {
+		return nil, err
+	}
+
+	content, err := ms.driver.GetContent(p)
+	if err != nil {
+		switch err := err.(type) {
+		case storagedriver.PathNotFoundError, *storagedriver.PathNotFoundError:
+			return nil, ErrManifestUnknown
+		default:
+			return nil, err
+		}
+	}
+
+	var manifest SignedManifest
+
+	if err := json.Unmarshal(content, &manifest); err != nil {
+		// TODO(stevvooe): Corrupted manifest error?
+		return nil, err
+	}
+
+	// TODO(stevvooe): Verify the manifest here?
+
+	return &manifest, nil
+}
+
+func (ms *manifestStore) Put(name, tag string, manifest *SignedManifest) error {
+	p, err := ms.path(name, tag)
+	if err != nil {
+		return err
+	}
+
+	if err := ms.verifyManifest(name, tag, manifest); err != nil {
+		return err
+	}
+
+	// TODO(stevvooe): Should we get manifest first?
+
+	return ms.driver.PutContent(p, manifest.Raw)
+}
+
+func (ms *manifestStore) Delete(name, tag string) error {
+	panic("not implemented")
+}
+
+func (ms *manifestStore) path(name, tag string) (string, error) {
+	return ms.pathMapper.path(manifestPathSpec{
+		name: name,
+		tag:  tag,
+	})
+}
+
+func (ms *manifestStore) verifyManifest(name, tag string, manifest *SignedManifest) error {
+	if manifest.Name != name {
+		return fmt.Errorf("name does not match manifest name")
+	}
+
+	if manifest.Tag != tag {
+		return fmt.Errorf("tag does not match manifest tag")
+	}
+
+	var errs []error
+
+	for _, fsLayer := range manifest.FSLayers {
+		exists, err := ms.layerService.Exists(name, fsLayer.BlobSum)
+		if err != nil {
+			// TODO(stevvooe): Need to store information about missing blob.
+			errs = append(errs, err)
+		}
+
+		if !exists {
+			errs = append(errs, fmt.Errorf("missing layer %v", fsLayer.BlobSum))
+		}
+	}
+
+	if len(errs) != 0 {
+		// TODO(stevvooe): These need to be recoverable by a caller.
+		return fmt.Errorf("missing layers: %v", errs)
+	}
+
+	js, err := libtrust.ParsePrettySignature(manifest.Raw, "signatures")
+	if err != nil {
+		return err
+	}
+
+	_, err = js.Verify() // These pubkeys need to be checked.
+	if err != nil {
+		return err
+	}
+
+	// TODO(sday): Pubkey checks need to go here. This where things get fancy.
+	// Perhaps, an injected service would reduce coupling here.
+
+	return nil
+}

storage/services.go

@@ -1,6 +1,7 @@
 package storage
 
 import (
+	"github.com/docker/docker-registry/digest"
 	"github.com/docker/docker-registry/storagedriver"
 )
 
@@ -41,3 +42,42 @@ func NewServices(driver storagedriver.StorageDriver) *Services {
 func (ss *Services) Layers() LayerService {
 	return &layerStore{driver: ss.driver, pathMapper: ss.pathMapper, uploadStore: ss.layerUploadStore}
 }
+
+// Manifests returns an instance of ManifestService. Instantiation is cheap and
+// may be context sensitive in the future. The instance should be used similar
+// to a request local.
+func (ss *Services) Manifests() ManifestService {
+	return &manifestStore{driver: ss.driver, pathMapper: ss.pathMapper, layerService: ss.Layers()}
+}
+
+// ManifestService provides operations on image manifests.
+type ManifestService interface {
+	// Exists returns true if the layer exists.
+	Exists(name, tag string) (bool, error)
+
+	// Get retrieves the named manifest, if it exists.
+	Get(name, tag string) (*SignedManifest, error)
+
+	// Put creates or updates the named manifest.
+	Put(name, tag string, manifest *SignedManifest) error
+
+	// Delete removes the named manifest, if it exists.
+	Delete(name, tag string) error
+}
+
+// LayerService provides operations on layer files in a backend storage.
+type LayerService interface {
+	// Exists returns true if the layer exists.
+	Exists(name string, digest digest.Digest) (bool, error)
+
+	// Fetch the layer identifed by TarSum.
+	Fetch(name string, digest digest.Digest) (Layer, error)
+
+	// Upload begins a layer upload to repository identified by name,
+	// returning a handle.
+	Upload(name string) (LayerUpload, error)
+
+	// Resume continues an in progress layer upload, returning the current
+	// state of the upload.
+	Resume(uuid string) (LayerUpload, error)
+}