Initial implementation of image manifest storage

This change implements the first pass at image manifest storage on top of the
storagedriver. Very similar to LayerService, its much simpler due to less
complexity of pushing and pulling images.

Various components are still missing, such as detailed error reporting on
missing layers during verification, but the base functionality is present.
This commit is contained in:
Stephen J Day 2014-11-21 19:39:52 -08:00
parent eaadb82e1e
commit 4decfaa82e
6 changed files with 314 additions and 20 deletions

View file

@ -5,6 +5,7 @@ import (
"strings" "strings"
"github.com/docker/docker-registry/digest" "github.com/docker/docker-registry/digest"
"github.com/docker/docker-registry/storage"
) )
// ErrorCode represents the error type. The errors are serialized via strings // ErrorCode represents the error type. The errors are serialized via strings

View file

@ -1,11 +1,8 @@
package registry package registry
import ( import (
"encoding/json"
"fmt"
"net/http" "net/http"
"github.com/docker/docker-registry/storage"
"github.com/gorilla/handlers" "github.com/gorilla/handlers"
) )

View file

@ -8,23 +8,6 @@ import (
"github.com/docker/docker-registry/digest" "github.com/docker/docker-registry/digest"
) )
// LayerService provides operations on layer files in a backend storage.
type LayerService interface {
// Exists returns true if the layer exists.
Exists(name string, digest digest.Digest) (bool, error)
// Fetch the layer identifed by TarSum.
Fetch(name string, digest digest.Digest) (Layer, error)
// Upload begins a layer upload to repository identified by name,
// returning a handle.
Upload(name string) (LayerUpload, error)
// Resume continues an in progress layer upload, returning the current
// state of the upload.
Resume(uuid string) (LayerUpload, error)
}
// Layer provides a readable and seekable layer object. Typically, // Layer provides a readable and seekable layer object. Typically,
// implementations are *not* goroutine safe. // implementations are *not* goroutine safe.
type Layer interface { type Layer interface {

139
storage/manifest_test.go Normal file
View file

@ -0,0 +1,139 @@
package storage
import (
"reflect"
"testing"
"github.com/docker/libtrust"
"github.com/docker/docker-registry/digest"
"github.com/docker/docker-registry/storagedriver/inmemory"
)
func TestManifestStorage(t *testing.T) {
driver := inmemory.New()
ms := &manifestStore{
driver: driver,
pathMapper: &pathMapper{
root: "/storage/testing",
version: storagePathVersion,
},
layerService: newMockedLayerService(),
}
name := "foo/bar"
tag := "thetag"
exists, err := ms.Exists(name, tag)
if err != nil {
t.Fatalf("unexpected error checking manifest existence: %v", err)
}
if exists {
t.Fatalf("manifest should not exist")
}
if _, err := ms.Get(name, tag); err != ErrManifestUnknown {
t.Fatalf("expected manifest unknown error: %v != %v", err, ErrManifestUnknown)
}
manifest := Manifest{
Versioned: Versioned{
SchemaVersion: 1,
},
Name: name,
Tag: tag,
FSLayers: []FSLayer{
{
BlobSum: "asdf",
},
{
BlobSum: "qwer",
},
},
}
pk, err := libtrust.GenerateECP256PrivateKey()
if err != nil {
t.Fatalf("unexpected error generating private key: %v", err)
}
sm, err := manifest.Sign(pk)
if err != nil {
t.Fatalf("error signing manifest: %v", err)
}
err = ms.Put(name, tag, sm)
if err == nil {
t.Fatalf("expected errors putting manifest")
}
// TODO(stevvooe): We expect errors describing all of the missing layers.
ms.layerService.(*mockedExistenceLayerService).add(name, "asdf")
ms.layerService.(*mockedExistenceLayerService).add(name, "qwer")
if err = ms.Put(name, tag, sm); err != nil {
t.Fatalf("unexpected error putting manifest: %v", err)
}
exists, err = ms.Exists(name, tag)
if err != nil {
t.Fatalf("unexpected error checking manifest existence: %v", err)
}
if !exists {
t.Fatalf("manifest should exist")
}
fetchedManifest, err := ms.Get(name, tag)
if err != nil {
t.Fatalf("unexpected error fetching manifest: %v", err)
}
if !reflect.DeepEqual(fetchedManifest, sm) {
t.Fatalf("fetched manifest not equal: %#v != %#v", fetchedManifest, sm)
}
}
type layerKey struct {
name string
digest digest.Digest
}
type mockedExistenceLayerService struct {
exists map[layerKey]struct{}
}
func newMockedLayerService() *mockedExistenceLayerService {
return &mockedExistenceLayerService{
exists: make(map[layerKey]struct{}),
}
}
var _ LayerService = &mockedExistenceLayerService{}
func (mels *mockedExistenceLayerService) add(name string, digest digest.Digest) {
mels.exists[layerKey{name: name, digest: digest}] = struct{}{}
}
func (mels *mockedExistenceLayerService) remove(name string, digest digest.Digest) {
delete(mels.exists, layerKey{name: name, digest: digest})
}
func (mels *mockedExistenceLayerService) Exists(name string, digest digest.Digest) (bool, error) {
_, ok := mels.exists[layerKey{name: name, digest: digest}]
return ok, nil
}
func (mockedExistenceLayerService) Fetch(name string, digest digest.Digest) (Layer, error) {
panic("not implemented")
}
func (mockedExistenceLayerService) Upload(name string) (LayerUpload, error) {
panic("not implemented")
}
func (mockedExistenceLayerService) Resume(uuid string) (LayerUpload, error) {
panic("not implemented")
}

134
storage/manifeststore.go Normal file
View file

@ -0,0 +1,134 @@
package storage
import (
"encoding/json"
"fmt"
"github.com/docker/libtrust"
"github.com/docker/docker-registry/storagedriver"
)
type manifestStore struct {
driver storagedriver.StorageDriver
pathMapper *pathMapper
layerService LayerService
}
var _ ManifestService = &manifestStore{}
func (ms *manifestStore) Exists(name, tag string) (bool, error) {
p, err := ms.path(name, tag)
if err != nil {
return false, err
}
size, err := ms.driver.CurrentSize(p)
if err != nil {
return false, err
}
if size == 0 {
return false, nil
}
return true, nil
}
func (ms *manifestStore) Get(name, tag string) (*SignedManifest, error) {
p, err := ms.path(name, tag)
if err != nil {
return nil, err
}
content, err := ms.driver.GetContent(p)
if err != nil {
switch err := err.(type) {
case storagedriver.PathNotFoundError, *storagedriver.PathNotFoundError:
return nil, ErrManifestUnknown
default:
return nil, err
}
}
var manifest SignedManifest
if err := json.Unmarshal(content, &manifest); err != nil {
// TODO(stevvooe): Corrupted manifest error?
return nil, err
}
// TODO(stevvooe): Verify the manifest here?
return &manifest, nil
}
func (ms *manifestStore) Put(name, tag string, manifest *SignedManifest) error {
p, err := ms.path(name, tag)
if err != nil {
return err
}
if err := ms.verifyManifest(name, tag, manifest); err != nil {
return err
}
// TODO(stevvooe): Should we get manifest first?
return ms.driver.PutContent(p, manifest.Raw)
}
func (ms *manifestStore) Delete(name, tag string) error {
panic("not implemented")
}
func (ms *manifestStore) path(name, tag string) (string, error) {
return ms.pathMapper.path(manifestPathSpec{
name: name,
tag: tag,
})
}
func (ms *manifestStore) verifyManifest(name, tag string, manifest *SignedManifest) error {
if manifest.Name != name {
return fmt.Errorf("name does not match manifest name")
}
if manifest.Tag != tag {
return fmt.Errorf("tag does not match manifest tag")
}
var errs []error
for _, fsLayer := range manifest.FSLayers {
exists, err := ms.layerService.Exists(name, fsLayer.BlobSum)
if err != nil {
// TODO(stevvooe): Need to store information about missing blob.
errs = append(errs, err)
}
if !exists {
errs = append(errs, fmt.Errorf("missing layer %v", fsLayer.BlobSum))
}
}
if len(errs) != 0 {
// TODO(stevvooe): These need to be recoverable by a caller.
return fmt.Errorf("missing layers: %v", errs)
}
js, err := libtrust.ParsePrettySignature(manifest.Raw, "signatures")
if err != nil {
return err
}
_, err = js.Verify() // These pubkeys need to be checked.
if err != nil {
return err
}
// TODO(sday): Pubkey checks need to go here. This where things get fancy.
// Perhaps, an injected service would reduce coupling here.
return nil
}

View file

@ -1,6 +1,7 @@
package storage package storage
import ( import (
"github.com/docker/docker-registry/digest"
"github.com/docker/docker-registry/storagedriver" "github.com/docker/docker-registry/storagedriver"
) )
@ -41,3 +42,42 @@ func NewServices(driver storagedriver.StorageDriver) *Services {
func (ss *Services) Layers() LayerService { func (ss *Services) Layers() LayerService {
return &layerStore{driver: ss.driver, pathMapper: ss.pathMapper, uploadStore: ss.layerUploadStore} return &layerStore{driver: ss.driver, pathMapper: ss.pathMapper, uploadStore: ss.layerUploadStore}
} }
// Manifests returns an instance of ManifestService. Instantiation is cheap and
// may be context sensitive in the future. The instance should be used similar
// to a request local.
func (ss *Services) Manifests() ManifestService {
return &manifestStore{driver: ss.driver, pathMapper: ss.pathMapper, layerService: ss.Layers()}
}
// ManifestService provides operations on image manifests.
type ManifestService interface {
// Exists returns true if the layer exists.
Exists(name, tag string) (bool, error)
// Get retrieves the named manifest, if it exists.
Get(name, tag string) (*SignedManifest, error)
// Put creates or updates the named manifest.
Put(name, tag string, manifest *SignedManifest) error
// Delete removes the named manifest, if it exists.
Delete(name, tag string) error
}
// LayerService provides operations on layer files in a backend storage.
type LayerService interface {
// Exists returns true if the layer exists.
Exists(name string, digest digest.Digest) (bool, error)
// Fetch the layer identifed by TarSum.
Fetch(name string, digest digest.Digest) (Layer, error)
// Upload begins a layer upload to repository identified by name,
// returning a handle.
Upload(name string) (LayerUpload, error)
// Resume continues an in progress layer upload, returning the current
// state of the upload.
Resume(uuid string) (LayerUpload, error)
}