TrueCloudLab
/
distribution
13
0
Fork 3
Code Issues 2 Pull Requests Packages Projects Releases 1 Wiki Activity

Merge pull request #1 from owtaylor/oci-media-types 

Handle OCI manifests and image indexes without a media type

Signed-off-by: Mike Brown <brownwm@us.ibm.com>
pull/2076/head
Mike Brown 2018-06-19 09:31:43 -07:00 committed by Mike Brown
parent f6224f78ba 132abc6de5
commit 321d636e76
9 changed files with 411 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
manifest/manifestlist/manifestlist.go
View File

@ -38,6 +38,13 @@ func init() {
return nil, distribution.Descriptor{}, err
}
if m.MediaType != MediaTypeManifestList {
err = fmt.Errorf("mediaType in manifest list should be '%s' not '%s'",
MediaTypeManifestList, m.MediaType)
return nil, distribution.Descriptor{}, err
}
dgst := digest.FromBytes(b)
return m, distribution.Descriptor{Digest: dgst, Size: int64(len(b)), MediaType: MediaTypeManifestList}, err
}
@ -53,6 +60,13 @@ func init() {
return nil, distribution.Descriptor{}, err
}
if m.MediaType != "" && m.MediaType != v1.MediaTypeImageIndex {
err = fmt.Errorf("if present, mediaType in image index should be '%s' not '%s'",
v1.MediaTypeImageIndex, m.MediaType)
return nil, distribution.Descriptor{}, err
}
dgst := digest.FromBytes(b)
return m, distribution.Descriptor{Digest: dgst, Size: int64(len(b)), MediaType: v1.MediaTypeImageIndex}, err
}
@ -130,15 +144,23 @@ type DeserializedManifestList struct {
// DeserializedManifestList which contains the resulting manifest list
// and its JSON representation.
func FromDescriptors(descriptors []ManifestDescriptor) (*DeserializedManifestList, error) {
var m ManifestList
var mediaType string
if len(descriptors) > 0 && descriptors[0].Descriptor.MediaType == v1.MediaTypeImageManifest {
m = ManifestList{
Versioned: OCISchemaVersion,
}
mediaType = v1.MediaTypeImageIndex
} else {
m = ManifestList{
Versioned: SchemaVersion,
}
mediaType = MediaTypeManifestList
}
return FromDescriptorsWithMediaType(descriptors, mediaType)
}
// For testing purposes, it's useful to be able to specify the media type explicitly
func FromDescriptorsWithMediaType(descriptors []ManifestDescriptor, mediaType string) (*DeserializedManifestList, error) {
m := ManifestList{
Versioned: manifest.Versioned{
SchemaVersion: 2,
MediaType: mediaType,
},
}
m.Manifests = make([]ManifestDescriptor, len(descriptors), len(descriptors))
@ -183,5 +205,12 @@ func (m *DeserializedManifestList) MarshalJSON() ([]byte, error) {
// Payload returns the raw content of the manifest list. The contents can be
// used to calculate the content identifier.
func (m DeserializedManifestList) Payload() (string, []byte, error) {
return m.MediaType, m.canonical, nil
var mediaType string
if m.MediaType == "" {
mediaType = v1.MediaTypeImageIndex
} else {
mediaType = m.MediaType
}
return mediaType, m.canonical, nil
}

70
manifest/manifestlist/manifestlist_test.go
View File

@ -38,7 +38,7 @@ var expectedManifestListSerialization = []byte(`{
]
}`)
func TestManifestList(t *testing.T) {
func makeTestManifestList(t *testing.T, mediaType string) ([]ManifestDescriptor, *DeserializedManifestList) {
manifestDescriptors := []ManifestDescriptor{
{
Descriptor: distribution.Descriptor{
@ -65,11 +65,16 @@ func TestManifestList(t *testing.T) {
},
}
deserialized, err := FromDescriptors(manifestDescriptors)
deserialized, err := FromDescriptorsWithMediaType(manifestDescriptors, mediaType)
if err != nil {
t.Fatalf("error creating DeserializedManifestList: %v", err)
}
return manifestDescriptors, deserialized
}
func TestManifestList(t *testing.T) {
manifestDescriptors, deserialized := makeTestManifestList(t, MediaTypeManifestList)
mediaType, canonical, _ := deserialized.Payload()
if mediaType != MediaTypeManifestList {
@ -160,7 +165,7 @@ var expectedOCIImageIndexSerialization = []byte(`{
]
}`)
func TestOCIImageIndex(t *testing.T) {
func makeTestOCIImageIndex(t *testing.T, mediaType string) ([]ManifestDescriptor, *DeserializedManifestList) {
manifestDescriptors := []ManifestDescriptor{
{
Descriptor: distribution.Descriptor{
@ -196,11 +201,17 @@ func TestOCIImageIndex(t *testing.T) {
},
}
deserialized, err := FromDescriptors(manifestDescriptors)
deserialized, err := FromDescriptorsWithMediaType(manifestDescriptors, mediaType)
if err != nil {
t.Fatalf("error creating DeserializedManifestList: %v", err)
}
return manifestDescriptors, deserialized
}
func TestOCIImageIndex(t *testing.T) {
manifestDescriptors, deserialized := makeTestOCIImageIndex(t, v1.MediaTypeImageIndex)
mediaType, canonical, _ := deserialized.Payload()
if mediaType != v1.MediaTypeImageIndex {
@ -241,3 +252,54 @@ func TestOCIImageIndex(t *testing.T) {
}
}
}
func mediaTypeTest(t *testing.T, contentType string, mediaType string, shouldError bool) {
var m *DeserializedManifestList
if contentType == MediaTypeManifestList {
_, m = makeTestManifestList(t, mediaType)
} else {
_, m = makeTestOCIImageIndex(t, mediaType)
}
_, canonical, err := m.Payload()
if err != nil {
t.Fatalf("error getting payload, %v", err)
}
unmarshalled, descriptor, err := distribution.UnmarshalManifest(
contentType,
canonical)
if shouldError {
if err == nil {
t.Fatalf("bad content type should have produced error")
}
} else {
if err != nil {
t.Fatalf("error unmarshaling manifest, %v", err)
}
asManifest := unmarshalled.(*DeserializedManifestList)
if asManifest.MediaType != mediaType {
t.Fatalf("Bad media type '%v' as unmarshalled", asManifest.MediaType)
}
if descriptor.MediaType != contentType {
t.Fatalf("Bad media type '%v' for descriptor", descriptor.MediaType)
}
unmarshalledMediaType, _, _ := unmarshalled.Payload()
if unmarshalledMediaType != contentType {
t.Fatalf("Bad media type '%v' for payload", unmarshalledMediaType)
}
}
}
func TestMediaTypes(t *testing.T) {
mediaTypeTest(t, MediaTypeManifestList, "", true)
mediaTypeTest(t, MediaTypeManifestList, MediaTypeManifestList, false)
mediaTypeTest(t, MediaTypeManifestList, MediaTypeManifestList+"XXX", true)
mediaTypeTest(t, v1.MediaTypeImageIndex, "", false)
mediaTypeTest(t, v1.MediaTypeImageIndex, v1.MediaTypeImageIndex, false)
mediaTypeTest(t, v1.MediaTypeImageIndex, v1.MediaTypeImageIndex+"XXX", true)
}

30
manifest/ocischema/builder.go
View File

@ -4,12 +4,13 @@ import (
"context"
"github.com/docker/distribution"
"github.com/docker/distribution/manifest"
"github.com/opencontainers/go-digest"
"github.com/opencontainers/image-spec/specs-go/v1"
)
// builder is a type for constructing manifests.
type builder struct {
type Builder struct {
// bs is a BlobService used to publish the configuration blob.
bs distribution.BlobService
@ -22,26 +23,43 @@ type builder struct {
// Annotations contains arbitrary metadata relating to the targeted content.
annotations map[string]string
// For testing purposes
mediaType string
}
// NewManifestBuilder is used to build new manifests for the current schema
// version. It takes a BlobService so it can publish the configuration blob
// as part of the Build process, and annotations.
func NewManifestBuilder(bs distribution.BlobService, configJSON []byte, annotations map[string]string) distribution.ManifestBuilder {
mb := &builder{
mb := &Builder{
bs: bs,
configJSON: make([]byte, len(configJSON)),
annotations: annotations,
mediaType: v1.MediaTypeImageManifest,
}
copy(mb.configJSON, configJSON)
return mb
}
// For testing purposes, we want to be able to create an OCI image with
// either an MediaType either empty, or with the OCI image value
func (mb *Builder) SetMediaType(mediaType string) {
if mediaType != "" && mediaType != v1.MediaTypeImageManifest {
panic("Invalid media type for OCI image manifest")
}
mb.mediaType = mediaType
}
// Build produces a final manifest from the given references.
func (mb *builder) Build(ctx context.Context) (distribution.Manifest, error) {
func (mb *Builder) Build(ctx context.Context) (distribution.Manifest, error) {
m := Manifest{
Versioned: SchemaVersion,
Versioned: manifest.Versioned{
SchemaVersion: 2,
MediaType: mb.mediaType,
},
Layers: make([]distribution.Descriptor, len(mb.layers)),
Annotations: mb.annotations,
}
@ -76,12 +94,12 @@ func (mb *builder) Build(ctx context.Context) (distribution.Manifest, error) {
}
// AppendReference adds a reference to the current ManifestBuilder.
func (mb *builder) AppendReference(d distribution.Describable) error {
func (mb *Builder) AppendReference(d distribution.Describable) error {
mb.layers = append(mb.layers, d.Descriptor())
return nil
}
// References returns the current references added to this builder.
func (mb *builder) References() []distribution.Descriptor {
func (mb *Builder) References() []distribution.Descriptor {
return mb.layers
}

7
manifest/ocischema/manifest.go
View File

@ -97,6 +97,11 @@ func (m *DeserializedManifest) UnmarshalJSON(b []byte) error {
return err
}
if manifest.MediaType != "" && manifest.MediaType != v1.MediaTypeImageManifest {
return fmt.Errorf("if present, mediaType in manifest should be '%s' not '%s'",
v1.MediaTypeImageManifest, manifest.MediaType)
}
m.Manifest = manifest
return nil
@ -115,5 +120,5 @@ func (m *DeserializedManifest) MarshalJSON() ([]byte, error) {
// Payload returns the raw content of the manifest. The contents can be used to
// calculate the content identifier.
func (m DeserializedManifest) Payload() (string, []byte, error) {
return m.MediaType, m.canonical, nil
return v1.MediaTypeImageManifest, m.canonical, nil
}

57
manifest/ocischema/manifest_test.go
View File

@ -7,6 +7,7 @@ import (
"testing"
"github.com/docker/distribution"
"github.com/docker/distribution/manifest"
"github.com/opencontainers/image-spec/specs-go/v1"
)
@ -36,9 +37,12 @@ var expectedManifestSerialization = []byte(`{
}
}`)
func TestManifest(t *testing.T) {
manifest := Manifest{
Versioned: SchemaVersion,
func makeTestManifest(mediaType string) Manifest {
return Manifest{
Versioned: manifest.Versioned{
SchemaVersion: 2,
MediaType: mediaType,
},
Config: distribution.Descriptor{
Digest: "sha256:1a9ec845ee94c202b2d5da74a24f0ed2058318bfa9879fa541efaecba272e86b",
Size: 985,
@ -55,6 +59,10 @@ func TestManifest(t *testing.T) {
},
Annotations: map[string]string{"hot": "potato"},
}
}
func TestManifest(t *testing.T) {
manifest := makeTestManifest(v1.MediaTypeImageManifest)
deserialized, err := FromStruct(manifest)
if err != nil {
@ -131,3 +139,46 @@ func TestManifest(t *testing.T) {
t.Fatalf("unexpected annotation in reference: %s", references[1].Annotations["lettuce"])
}
}
func mediaTypeTest(t *testing.T, mediaType string, shouldError bool) {
manifest := makeTestManifest(mediaType)
deserialized, err := FromStruct(manifest)
if err != nil {
t.Fatalf("error creating DeserializedManifest: %v", err)
}
unmarshalled, descriptor, err := distribution.UnmarshalManifest(
v1.MediaTypeImageManifest,
deserialized.canonical)
if shouldError {
if err == nil {
t.Fatalf("bad content type should have produced error")
}
} else {
if err != nil {
t.Fatalf("error unmarshaling manifest, %v", err)
}
asManifest := unmarshalled.(*DeserializedManifest)
if asManifest.MediaType != mediaType {
t.Fatalf("Bad media type '%v' as unmarshalled", asManifest.MediaType)
}
if descriptor.MediaType != v1.MediaTypeImageManifest {
t.Fatalf("Bad media type '%v' for descriptor", descriptor.MediaType)
}
unmarshalledMediaType, _, _ := unmarshalled.Payload()
if unmarshalledMediaType != v1.MediaTypeImageManifest {
t.Fatalf("Bad media type '%v' for payload", unmarshalledMediaType)
}
}
}
func TestMediaTypes(t *testing.T) {
mediaTypeTest(t, "", false)
mediaTypeTest(t, v1.MediaTypeImageManifest, false)
mediaTypeTest(t, v1.MediaTypeImageManifest+"XXX", true)
}

6
manifest/schema2/manifest.go
View File

@ -116,6 +116,12 @@ func (m *DeserializedManifest) UnmarshalJSON(b []byte) error {
return err
}
if manifest.MediaType != MediaTypeManifest {
return fmt.Errorf("mediaType in manifest should be '%s' not '%s'",
MediaTypeManifest, manifest.MediaType)
}
m.Manifest = manifest
return nil

57
manifest/schema2/manifest_test.go
View File

@ -7,6 +7,7 @@ import (
"testing"
"github.com/docker/distribution"
"github.com/docker/distribution/manifest"
)
var expectedManifestSerialization = []byte(`{
@ -26,9 +27,12 @@ var expectedManifestSerialization = []byte(`{
]
}`)
func TestManifest(t *testing.T) {
manifest := Manifest{
Versioned: SchemaVersion,
func makeTestManifest(mediaType string) Manifest {
return Manifest{
Versioned: manifest.Versioned{
SchemaVersion: 2,
MediaType: mediaType,
},
Config: distribution.Descriptor{
Digest: "sha256:1a9ec845ee94c202b2d5da74a24f0ed2058318bfa9879fa541efaecba272e86b",
Size: 985,
@ -42,6 +46,10 @@ func TestManifest(t *testing.T) {
},
},
}
}
func TestManifest(t *testing.T) {
manifest := makeTestManifest(MediaTypeManifest)
deserialized, err := FromStruct(manifest)
if err != nil {
@ -109,3 +117,46 @@ func TestManifest(t *testing.T) {
t.Fatalf("unexpected size in reference: %d", references[0].Size)
}
}
func mediaTypeTest(t *testing.T, mediaType string, shouldError bool) {
manifest := makeTestManifest(mediaType)
deserialized, err := FromStruct(manifest)
if err != nil {
t.Fatalf("error creating DeserializedManifest: %v", err)
}
unmarshalled, descriptor, err := distribution.UnmarshalManifest(
MediaTypeManifest,
deserialized.canonical)
if shouldError {
if err == nil {
t.Fatalf("bad content type should have produced error")
}
} else {
if err != nil {
t.Fatalf("error unmarshaling manifest, %v", err)
}
asManifest := unmarshalled.(*DeserializedManifest)
if asManifest.MediaType != mediaType {
t.Fatalf("Bad media type '%v' as unmarshalled", asManifest.MediaType)
}
if descriptor.MediaType != MediaTypeManifest {
t.Fatalf("Bad media type '%v' for descriptor", descriptor.MediaType)
}
unmarshalledMediaType, _, _ := unmarshalled.Payload()
if unmarshalledMediaType != MediaTypeManifest {
t.Fatalf("Bad media type '%v' for payload", unmarshalledMediaType)
}
}
}
func TestMediaTypes(t *testing.T) {
mediaTypeTest(t, "", true)
mediaTypeTest(t, MediaTypeManifest, false)
mediaTypeTest(t, MediaTypeManifest+"XXX", true)
}

12
registry/storage/manifeststore.go
View File

@ -106,6 +106,18 @@ func (ms *manifestStore) Get(ctx context.Context, dgst digest.Digest, options ..
return ms.ocischemaHandler.Unmarshal(ctx, dgst, content)
case manifestlist.MediaTypeManifestList, v1.MediaTypeImageIndex:
return ms.manifestListHandler.Unmarshal(ctx, dgst, content)
case "":
// OCI image or image index - no media type in the content
// First see if it looks like an image index
res, err := ms.manifestListHandler.Unmarshal(ctx, dgst, content)
resIndex := res.(*manifestlist.DeserializedManifestList)
if err == nil && resIndex.Manifests != nil {
return resIndex, nil
}
// Otherwise, assume it must be an image manifest
return ms.ocischemaHandler.Unmarshal(ctx, dgst, content)
default:
return nil, distribution.ErrManifestVerification{fmt.Errorf("unrecognized manifest content type %s", versioned.MediaType)}
}

153
registry/storage/manifeststore_test.go
View File

@ -9,6 +9,8 @@ import (
"github.com/docker/distribution"
"github.com/docker/distribution/manifest"
"github.com/docker/distribution/manifest/manifestlist"
"github.com/docker/distribution/manifest/ocischema"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/reference"
"github.com/docker/distribution/registry/storage/cache/memory"
@ -17,6 +19,7 @@ import (
"github.com/docker/distribution/testutil"
"github.com/docker/libtrust"
"github.com/opencontainers/go-digest"
"github.com/opencontainers/image-spec/specs-go/v1"
)
type manifestStoreTestEnv struct {
@ -356,6 +359,155 @@ func testManifestStorage(t *testing.T, options ...RegistryOption) {
}
}
func TestOCIManifestStorage(t *testing.T) {
testOCIManifestStorage(t, "includeMediaTypes=true", true)
testOCIManifestStorage(t, "includeMediaTypes=false", false)
}
func testOCIManifestStorage(t *testing.T, testname string, includeMediaTypes bool) {
var imageMediaType string
var indexMediaType string
if includeMediaTypes {
imageMediaType = v1.MediaTypeImageManifest
indexMediaType = v1.MediaTypeImageIndex
} else {
imageMediaType = ""
indexMediaType = ""
}
repoName, _ := reference.WithName("foo/bar")
env := newManifestStoreTestEnv(t, repoName, "thetag",
BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()),
EnableDelete, EnableRedirect)
ctx := context.Background()
ms, err := env.repository.Manifests(ctx)
if err != nil {
t.Fatal(err)
}
// Build a manifest and store it and its layers in the registry
blobStore := env.repository.Blobs(ctx)
builder := ocischema.NewManifestBuilder(blobStore, []byte{}, map[string]string{})
builder.(*ocischema.Builder).SetMediaType(imageMediaType)
// Add some layers
for i := 0; i < 2; i++ {
rs, ds, err := testutil.CreateRandomTarFile()
if err != nil {
t.Fatalf("%s: unexpected error generating test layer file", testname)
}
dgst := digest.Digest(ds)
wr, err := env.repository.Blobs(env.ctx).Create(env.ctx)
if err != nil {
t.Fatalf("%s: unexpected error creating test upload: %v", testname, err)
}
if _, err := io.Copy(wr, rs); err != nil {
t.Fatalf("%s: unexpected error copying to upload: %v", testname, err)
}
if _, err := wr.Commit(env.ctx, distribution.Descriptor{Digest: dgst}); err != nil {
t.Fatalf("%s: unexpected error finishing upload: %v", testname, err)
}
builder.AppendReference(distribution.Descriptor{Digest: dgst})
}
manifest, err := builder.Build(ctx)
if err != nil {
t.Fatalf("%s: unexpected error generating manifest: %v", testname, err)
}
var manifestDigest digest.Digest
if manifestDigest, err = ms.Put(ctx, manifest); err != nil {
t.Fatalf("%s: unexpected error putting manifest: %v", testname, err)
}
// Also create an image index that contains the manifest
descriptor, err := env.registry.BlobStatter().Stat(ctx, manifestDigest)
if err != nil {
t.Fatalf("%s: unexpected error getting manifest descriptor", testname)
}
descriptor.MediaType = v1.MediaTypeImageManifest
platformSpec := manifestlist.PlatformSpec{
Architecture: "atari2600",
OS: "CP/M",
}
manifestDescriptors := []manifestlist.ManifestDescriptor{
manifestlist.ManifestDescriptor{
Descriptor: descriptor,
Platform: platformSpec,
},
}
imageIndex, err := manifestlist.FromDescriptorsWithMediaType(manifestDescriptors, indexMediaType)
if err != nil {
t.Fatalf("%s: unexpected error creating image index: %v", testname, err)
}
var indexDigest digest.Digest
if indexDigest, err = ms.Put(ctx, imageIndex); err != nil {
t.Fatalf("%s: unexpected error putting image index: %v", testname, err)
}
// Now check that we can retrieve the manifest
fromStore, err := ms.Get(ctx, manifestDigest)
if err != nil {
t.Fatalf("%s: unexpected error fetching manifest: %v", testname, err)
}
fetchedManifest, ok := fromStore.(*ocischema.DeserializedManifest)
if !ok {
t.Fatalf("%s: unexpected type for fetched manifest", testname)
}
if fetchedManifest.MediaType != imageMediaType {
t.Fatalf("%s: unexpected MediaType for result, %s", testname, fetchedManifest.MediaType)
}
payloadMediaType, _, err := fromStore.Payload()
if err != nil {
t.Fatalf("%s: error getting payload %v", testname, err)
}
if payloadMediaType != v1.MediaTypeImageManifest {
t.Fatalf("%s: unexpected MediaType for manifest payload, %s", testname, payloadMediaType)
}
// and the image index
fromStore, err = ms.Get(ctx, indexDigest)
if err != nil {
t.Fatalf("%s: unexpected error fetching image index: %v", testname, err)
}
fetchedIndex, ok := fromStore.(*manifestlist.DeserializedManifestList)
if !ok {
t.Fatalf("%s: unexpected type for fetched manifest", testname)
}
if fetchedIndex.MediaType != indexMediaType {
t.Fatalf("%s: unexpected MediaType for result, %s", testname, fetchedManifest.MediaType)
}
payloadMediaType, _, err = fromStore.Payload()
if err != nil {
t.Fatalf("%s: error getting payload %v", testname, err)
}
if payloadMediaType != v1.MediaTypeImageIndex {
t.Fatalf("%s: unexpected MediaType for index payload, %s", testname, payloadMediaType)
}
}
// TestLinkPathFuncs ensures that the link path functions behavior are locked
// down and implemented as expected.
func TestLinkPathFuncs(t *testing.T) {
@ -387,5 +539,4 @@ func TestLinkPathFuncs(t *testing.T) {
t.Fatalf("incorrect path returned: %q != %q", p, testcase.expected)
}
}
}