diff --git a/manifest/manifestlist/manifestlist.go b/manifest/manifestlist/manifestlist.go index 7be297140..a08f1e5a0 100644 --- a/manifest/manifestlist/manifestlist.go +++ b/manifest/manifestlist/manifestlist.go @@ -38,6 +38,13 @@ func init() { return nil, distribution.Descriptor{}, err } + if m.MediaType != MediaTypeManifestList { + err = fmt.Errorf("mediaType in manifest list should be '%s' not '%s'", + MediaTypeManifestList, m.MediaType) + + return nil, distribution.Descriptor{}, err + } + dgst := digest.FromBytes(b) return m, distribution.Descriptor{Digest: dgst, Size: int64(len(b)), MediaType: MediaTypeManifestList}, err } @@ -53,6 +60,13 @@ func init() { return nil, distribution.Descriptor{}, err } + if m.MediaType != "" && m.MediaType != v1.MediaTypeImageIndex { + err = fmt.Errorf("if present, mediaType in image index should be '%s' not '%s'", + v1.MediaTypeImageIndex, m.MediaType) + + return nil, distribution.Descriptor{}, err + } + dgst := digest.FromBytes(b) return m, distribution.Descriptor{Digest: dgst, Size: int64(len(b)), MediaType: v1.MediaTypeImageIndex}, err } @@ -130,15 +144,23 @@ type DeserializedManifestList struct { // DeserializedManifestList which contains the resulting manifest list // and its JSON representation. func FromDescriptors(descriptors []ManifestDescriptor) (*DeserializedManifestList, error) { - var m ManifestList + var mediaType string if len(descriptors) > 0 && descriptors[0].Descriptor.MediaType == v1.MediaTypeImageManifest { - m = ManifestList{ - Versioned: OCISchemaVersion, - } + mediaType = v1.MediaTypeImageIndex } else { - m = ManifestList{ - Versioned: SchemaVersion, - } + mediaType = MediaTypeManifestList + } + + return FromDescriptorsWithMediaType(descriptors, mediaType) +} + +// For testing purposes, it's useful to be able to specify the media type explicitly +func FromDescriptorsWithMediaType(descriptors []ManifestDescriptor, mediaType string) (*DeserializedManifestList, error) { + m := ManifestList{ + Versioned: manifest.Versioned{ + SchemaVersion: 2, + MediaType: mediaType, + }, } m.Manifests = make([]ManifestDescriptor, len(descriptors), len(descriptors)) @@ -183,5 +205,12 @@ func (m *DeserializedManifestList) MarshalJSON() ([]byte, error) { // Payload returns the raw content of the manifest list. The contents can be // used to calculate the content identifier. func (m DeserializedManifestList) Payload() (string, []byte, error) { - return m.MediaType, m.canonical, nil + var mediaType string + if m.MediaType == "" { + mediaType = v1.MediaTypeImageIndex + } else { + mediaType = m.MediaType + } + + return mediaType, m.canonical, nil } diff --git a/manifest/manifestlist/manifestlist_test.go b/manifest/manifestlist/manifestlist_test.go index 720b911b7..e72292f07 100644 --- a/manifest/manifestlist/manifestlist_test.go +++ b/manifest/manifestlist/manifestlist_test.go @@ -38,7 +38,7 @@ var expectedManifestListSerialization = []byte(`{ ] }`) -func TestManifestList(t *testing.T) { +func makeTestManifestList(t *testing.T, mediaType string) ([]ManifestDescriptor, *DeserializedManifestList) { manifestDescriptors := []ManifestDescriptor{ { Descriptor: distribution.Descriptor{ @@ -65,11 +65,16 @@ func TestManifestList(t *testing.T) { }, } - deserialized, err := FromDescriptors(manifestDescriptors) + deserialized, err := FromDescriptorsWithMediaType(manifestDescriptors, mediaType) if err != nil { t.Fatalf("error creating DeserializedManifestList: %v", err) } + return manifestDescriptors, deserialized +} + +func TestManifestList(t *testing.T) { + manifestDescriptors, deserialized := makeTestManifestList(t, MediaTypeManifestList) mediaType, canonical, _ := deserialized.Payload() if mediaType != MediaTypeManifestList { @@ -160,7 +165,7 @@ var expectedOCIImageIndexSerialization = []byte(`{ ] }`) -func TestOCIImageIndex(t *testing.T) { +func makeTestOCIImageIndex(t *testing.T, mediaType string) ([]ManifestDescriptor, *DeserializedManifestList) { manifestDescriptors := []ManifestDescriptor{ { Descriptor: distribution.Descriptor{ @@ -196,11 +201,17 @@ func TestOCIImageIndex(t *testing.T) { }, } - deserialized, err := FromDescriptors(manifestDescriptors) + deserialized, err := FromDescriptorsWithMediaType(manifestDescriptors, mediaType) if err != nil { t.Fatalf("error creating DeserializedManifestList: %v", err) } + return manifestDescriptors, deserialized +} + +func TestOCIImageIndex(t *testing.T) { + manifestDescriptors, deserialized := makeTestOCIImageIndex(t, v1.MediaTypeImageIndex) + mediaType, canonical, _ := deserialized.Payload() if mediaType != v1.MediaTypeImageIndex { @@ -241,3 +252,54 @@ func TestOCIImageIndex(t *testing.T) { } } } + +func mediaTypeTest(t *testing.T, contentType string, mediaType string, shouldError bool) { + var m *DeserializedManifestList + if contentType == MediaTypeManifestList { + _, m = makeTestManifestList(t, mediaType) + } else { + _, m = makeTestOCIImageIndex(t, mediaType) + } + + _, canonical, err := m.Payload() + if err != nil { + t.Fatalf("error getting payload, %v", err) + } + + unmarshalled, descriptor, err := distribution.UnmarshalManifest( + contentType, + canonical) + + if shouldError { + if err == nil { + t.Fatalf("bad content type should have produced error") + } + } else { + if err != nil { + t.Fatalf("error unmarshaling manifest, %v", err) + } + + asManifest := unmarshalled.(*DeserializedManifestList) + if asManifest.MediaType != mediaType { + t.Fatalf("Bad media type '%v' as unmarshalled", asManifest.MediaType) + } + + if descriptor.MediaType != contentType { + t.Fatalf("Bad media type '%v' for descriptor", descriptor.MediaType) + } + + unmarshalledMediaType, _, _ := unmarshalled.Payload() + if unmarshalledMediaType != contentType { + t.Fatalf("Bad media type '%v' for payload", unmarshalledMediaType) + } + } +} + +func TestMediaTypes(t *testing.T) { + mediaTypeTest(t, MediaTypeManifestList, "", true) + mediaTypeTest(t, MediaTypeManifestList, MediaTypeManifestList, false) + mediaTypeTest(t, MediaTypeManifestList, MediaTypeManifestList+"XXX", true) + mediaTypeTest(t, v1.MediaTypeImageIndex, "", false) + mediaTypeTest(t, v1.MediaTypeImageIndex, v1.MediaTypeImageIndex, false) + mediaTypeTest(t, v1.MediaTypeImageIndex, v1.MediaTypeImageIndex+"XXX", true) +} diff --git a/manifest/ocischema/builder.go b/manifest/ocischema/builder.go index 9bfef6147..22a53c0de 100644 --- a/manifest/ocischema/builder.go +++ b/manifest/ocischema/builder.go @@ -4,12 +4,13 @@ import ( "context" "github.com/docker/distribution" + "github.com/docker/distribution/manifest" "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/specs-go/v1" ) // builder is a type for constructing manifests. -type builder struct { +type Builder struct { // bs is a BlobService used to publish the configuration blob. bs distribution.BlobService @@ -22,26 +23,43 @@ type builder struct { // Annotations contains arbitrary metadata relating to the targeted content. annotations map[string]string + + // For testing purposes + mediaType string } // NewManifestBuilder is used to build new manifests for the current schema // version. It takes a BlobService so it can publish the configuration blob // as part of the Build process, and annotations. func NewManifestBuilder(bs distribution.BlobService, configJSON []byte, annotations map[string]string) distribution.ManifestBuilder { - mb := &builder{ + mb := &Builder{ bs: bs, configJSON: make([]byte, len(configJSON)), annotations: annotations, + mediaType: v1.MediaTypeImageManifest, } copy(mb.configJSON, configJSON) return mb } +// For testing purposes, we want to be able to create an OCI image with +// either an MediaType either empty, or with the OCI image value +func (mb *Builder) SetMediaType(mediaType string) { + if mediaType != "" && mediaType != v1.MediaTypeImageManifest { + panic("Invalid media type for OCI image manifest") + } + + mb.mediaType = mediaType +} + // Build produces a final manifest from the given references. -func (mb *builder) Build(ctx context.Context) (distribution.Manifest, error) { +func (mb *Builder) Build(ctx context.Context) (distribution.Manifest, error) { m := Manifest{ - Versioned: SchemaVersion, + Versioned: manifest.Versioned{ + SchemaVersion: 2, + MediaType: mb.mediaType, + }, Layers: make([]distribution.Descriptor, len(mb.layers)), Annotations: mb.annotations, } @@ -76,12 +94,12 @@ func (mb *builder) Build(ctx context.Context) (distribution.Manifest, error) { } // AppendReference adds a reference to the current ManifestBuilder. -func (mb *builder) AppendReference(d distribution.Describable) error { +func (mb *Builder) AppendReference(d distribution.Describable) error { mb.layers = append(mb.layers, d.Descriptor()) return nil } // References returns the current references added to this builder. -func (mb *builder) References() []distribution.Descriptor { +func (mb *Builder) References() []distribution.Descriptor { return mb.layers } diff --git a/manifest/ocischema/manifest.go b/manifest/ocischema/manifest.go index afab12bd2..6de17f9ac 100644 --- a/manifest/ocischema/manifest.go +++ b/manifest/ocischema/manifest.go @@ -97,6 +97,11 @@ func (m *DeserializedManifest) UnmarshalJSON(b []byte) error { return err } + if manifest.MediaType != "" && manifest.MediaType != v1.MediaTypeImageManifest { + return fmt.Errorf("if present, mediaType in manifest should be '%s' not '%s'", + v1.MediaTypeImageManifest, manifest.MediaType) + } + m.Manifest = manifest return nil @@ -115,5 +120,5 @@ func (m *DeserializedManifest) MarshalJSON() ([]byte, error) { // Payload returns the raw content of the manifest. The contents can be used to // calculate the content identifier. func (m DeserializedManifest) Payload() (string, []byte, error) { - return m.MediaType, m.canonical, nil + return v1.MediaTypeImageManifest, m.canonical, nil } diff --git a/manifest/ocischema/manifest_test.go b/manifest/ocischema/manifest_test.go index 749cb859f..c39d1c9fc 100644 --- a/manifest/ocischema/manifest_test.go +++ b/manifest/ocischema/manifest_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/docker/distribution" + "github.com/docker/distribution/manifest" "github.com/opencontainers/image-spec/specs-go/v1" ) @@ -36,9 +37,12 @@ var expectedManifestSerialization = []byte(`{ } }`) -func TestManifest(t *testing.T) { - manifest := Manifest{ - Versioned: SchemaVersion, +func makeTestManifest(mediaType string) Manifest { + return Manifest{ + Versioned: manifest.Versioned{ + SchemaVersion: 2, + MediaType: mediaType, + }, Config: distribution.Descriptor{ Digest: "sha256:1a9ec845ee94c202b2d5da74a24f0ed2058318bfa9879fa541efaecba272e86b", Size: 985, @@ -55,6 +59,10 @@ func TestManifest(t *testing.T) { }, Annotations: map[string]string{"hot": "potato"}, } +} + +func TestManifest(t *testing.T) { + manifest := makeTestManifest(v1.MediaTypeImageManifest) deserialized, err := FromStruct(manifest) if err != nil { @@ -131,3 +139,46 @@ func TestManifest(t *testing.T) { t.Fatalf("unexpected annotation in reference: %s", references[1].Annotations["lettuce"]) } } + +func mediaTypeTest(t *testing.T, mediaType string, shouldError bool) { + manifest := makeTestManifest(mediaType) + + deserialized, err := FromStruct(manifest) + if err != nil { + t.Fatalf("error creating DeserializedManifest: %v", err) + } + + unmarshalled, descriptor, err := distribution.UnmarshalManifest( + v1.MediaTypeImageManifest, + deserialized.canonical) + + if shouldError { + if err == nil { + t.Fatalf("bad content type should have produced error") + } + } else { + if err != nil { + t.Fatalf("error unmarshaling manifest, %v", err) + } + + asManifest := unmarshalled.(*DeserializedManifest) + if asManifest.MediaType != mediaType { + t.Fatalf("Bad media type '%v' as unmarshalled", asManifest.MediaType) + } + + if descriptor.MediaType != v1.MediaTypeImageManifest { + t.Fatalf("Bad media type '%v' for descriptor", descriptor.MediaType) + } + + unmarshalledMediaType, _, _ := unmarshalled.Payload() + if unmarshalledMediaType != v1.MediaTypeImageManifest { + t.Fatalf("Bad media type '%v' for payload", unmarshalledMediaType) + } + } +} + +func TestMediaTypes(t *testing.T) { + mediaTypeTest(t, "", false) + mediaTypeTest(t, v1.MediaTypeImageManifest, false) + mediaTypeTest(t, v1.MediaTypeImageManifest+"XXX", true) +} diff --git a/manifest/schema2/manifest.go b/manifest/schema2/manifest.go index 26a6a334d..8adbbd0fc 100644 --- a/manifest/schema2/manifest.go +++ b/manifest/schema2/manifest.go @@ -116,6 +116,12 @@ func (m *DeserializedManifest) UnmarshalJSON(b []byte) error { return err } + if manifest.MediaType != MediaTypeManifest { + return fmt.Errorf("mediaType in manifest should be '%s' not '%s'", + MediaTypeManifest, manifest.MediaType) + + } + m.Manifest = manifest return nil diff --git a/manifest/schema2/manifest_test.go b/manifest/schema2/manifest_test.go index 86226606f..912dda9a5 100644 --- a/manifest/schema2/manifest_test.go +++ b/manifest/schema2/manifest_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/docker/distribution" + "github.com/docker/distribution/manifest" ) var expectedManifestSerialization = []byte(`{ @@ -26,9 +27,12 @@ var expectedManifestSerialization = []byte(`{ ] }`) -func TestManifest(t *testing.T) { - manifest := Manifest{ - Versioned: SchemaVersion, +func makeTestManifest(mediaType string) Manifest { + return Manifest{ + Versioned: manifest.Versioned{ + SchemaVersion: 2, + MediaType: mediaType, + }, Config: distribution.Descriptor{ Digest: "sha256:1a9ec845ee94c202b2d5da74a24f0ed2058318bfa9879fa541efaecba272e86b", Size: 985, @@ -42,6 +46,10 @@ func TestManifest(t *testing.T) { }, }, } +} + +func TestManifest(t *testing.T) { + manifest := makeTestManifest(MediaTypeManifest) deserialized, err := FromStruct(manifest) if err != nil { @@ -109,3 +117,46 @@ func TestManifest(t *testing.T) { t.Fatalf("unexpected size in reference: %d", references[0].Size) } } + +func mediaTypeTest(t *testing.T, mediaType string, shouldError bool) { + manifest := makeTestManifest(mediaType) + + deserialized, err := FromStruct(manifest) + if err != nil { + t.Fatalf("error creating DeserializedManifest: %v", err) + } + + unmarshalled, descriptor, err := distribution.UnmarshalManifest( + MediaTypeManifest, + deserialized.canonical) + + if shouldError { + if err == nil { + t.Fatalf("bad content type should have produced error") + } + } else { + if err != nil { + t.Fatalf("error unmarshaling manifest, %v", err) + } + + asManifest := unmarshalled.(*DeserializedManifest) + if asManifest.MediaType != mediaType { + t.Fatalf("Bad media type '%v' as unmarshalled", asManifest.MediaType) + } + + if descriptor.MediaType != MediaTypeManifest { + t.Fatalf("Bad media type '%v' for descriptor", descriptor.MediaType) + } + + unmarshalledMediaType, _, _ := unmarshalled.Payload() + if unmarshalledMediaType != MediaTypeManifest { + t.Fatalf("Bad media type '%v' for payload", unmarshalledMediaType) + } + } +} + +func TestMediaTypes(t *testing.T) { + mediaTypeTest(t, "", true) + mediaTypeTest(t, MediaTypeManifest, false) + mediaTypeTest(t, MediaTypeManifest+"XXX", true) +} diff --git a/registry/storage/manifeststore.go b/registry/storage/manifeststore.go index 6543f6fd6..73bff5733 100644 --- a/registry/storage/manifeststore.go +++ b/registry/storage/manifeststore.go @@ -106,6 +106,18 @@ func (ms *manifestStore) Get(ctx context.Context, dgst digest.Digest, options .. return ms.ocischemaHandler.Unmarshal(ctx, dgst, content) case manifestlist.MediaTypeManifestList, v1.MediaTypeImageIndex: return ms.manifestListHandler.Unmarshal(ctx, dgst, content) + case "": + // OCI image or image index - no media type in the content + + // First see if it looks like an image index + res, err := ms.manifestListHandler.Unmarshal(ctx, dgst, content) + resIndex := res.(*manifestlist.DeserializedManifestList) + if err == nil && resIndex.Manifests != nil { + return resIndex, nil + } + + // Otherwise, assume it must be an image manifest + return ms.ocischemaHandler.Unmarshal(ctx, dgst, content) default: return nil, distribution.ErrManifestVerification{fmt.Errorf("unrecognized manifest content type %s", versioned.MediaType)} } diff --git a/registry/storage/manifeststore_test.go b/registry/storage/manifeststore_test.go index 2fad7611d..61a4beeee 100644 --- a/registry/storage/manifeststore_test.go +++ b/registry/storage/manifeststore_test.go @@ -9,6 +9,8 @@ import ( "github.com/docker/distribution" "github.com/docker/distribution/manifest" + "github.com/docker/distribution/manifest/manifestlist" + "github.com/docker/distribution/manifest/ocischema" "github.com/docker/distribution/manifest/schema1" "github.com/docker/distribution/reference" "github.com/docker/distribution/registry/storage/cache/memory" @@ -17,6 +19,7 @@ import ( "github.com/docker/distribution/testutil" "github.com/docker/libtrust" "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/specs-go/v1" ) type manifestStoreTestEnv struct { @@ -356,6 +359,155 @@ func testManifestStorage(t *testing.T, options ...RegistryOption) { } } +func TestOCIManifestStorage(t *testing.T) { + testOCIManifestStorage(t, "includeMediaTypes=true", true) + testOCIManifestStorage(t, "includeMediaTypes=false", false) +} + +func testOCIManifestStorage(t *testing.T, testname string, includeMediaTypes bool) { + var imageMediaType string + var indexMediaType string + if includeMediaTypes { + imageMediaType = v1.MediaTypeImageManifest + indexMediaType = v1.MediaTypeImageIndex + } else { + imageMediaType = "" + indexMediaType = "" + } + + repoName, _ := reference.WithName("foo/bar") + env := newManifestStoreTestEnv(t, repoName, "thetag", + BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()), + EnableDelete, EnableRedirect) + + ctx := context.Background() + ms, err := env.repository.Manifests(ctx) + if err != nil { + t.Fatal(err) + } + + // Build a manifest and store it and its layers in the registry + + blobStore := env.repository.Blobs(ctx) + builder := ocischema.NewManifestBuilder(blobStore, []byte{}, map[string]string{}) + builder.(*ocischema.Builder).SetMediaType(imageMediaType) + + // Add some layers + for i := 0; i < 2; i++ { + rs, ds, err := testutil.CreateRandomTarFile() + if err != nil { + t.Fatalf("%s: unexpected error generating test layer file", testname) + } + dgst := digest.Digest(ds) + + wr, err := env.repository.Blobs(env.ctx).Create(env.ctx) + if err != nil { + t.Fatalf("%s: unexpected error creating test upload: %v", testname, err) + } + + if _, err := io.Copy(wr, rs); err != nil { + t.Fatalf("%s: unexpected error copying to upload: %v", testname, err) + } + + if _, err := wr.Commit(env.ctx, distribution.Descriptor{Digest: dgst}); err != nil { + t.Fatalf("%s: unexpected error finishing upload: %v", testname, err) + } + + builder.AppendReference(distribution.Descriptor{Digest: dgst}) + } + + manifest, err := builder.Build(ctx) + if err != nil { + t.Fatalf("%s: unexpected error generating manifest: %v", testname, err) + } + + var manifestDigest digest.Digest + if manifestDigest, err = ms.Put(ctx, manifest); err != nil { + t.Fatalf("%s: unexpected error putting manifest: %v", testname, err) + } + + // Also create an image index that contains the manifest + + descriptor, err := env.registry.BlobStatter().Stat(ctx, manifestDigest) + if err != nil { + t.Fatalf("%s: unexpected error getting manifest descriptor", testname) + } + descriptor.MediaType = v1.MediaTypeImageManifest + + platformSpec := manifestlist.PlatformSpec{ + Architecture: "atari2600", + OS: "CP/M", + } + + manifestDescriptors := []manifestlist.ManifestDescriptor{ + manifestlist.ManifestDescriptor{ + Descriptor: descriptor, + Platform: platformSpec, + }, + } + + imageIndex, err := manifestlist.FromDescriptorsWithMediaType(manifestDescriptors, indexMediaType) + if err != nil { + t.Fatalf("%s: unexpected error creating image index: %v", testname, err) + } + + var indexDigest digest.Digest + if indexDigest, err = ms.Put(ctx, imageIndex); err != nil { + t.Fatalf("%s: unexpected error putting image index: %v", testname, err) + } + + // Now check that we can retrieve the manifest + + fromStore, err := ms.Get(ctx, manifestDigest) + if err != nil { + t.Fatalf("%s: unexpected error fetching manifest: %v", testname, err) + } + + fetchedManifest, ok := fromStore.(*ocischema.DeserializedManifest) + if !ok { + t.Fatalf("%s: unexpected type for fetched manifest", testname) + } + + if fetchedManifest.MediaType != imageMediaType { + t.Fatalf("%s: unexpected MediaType for result, %s", testname, fetchedManifest.MediaType) + } + + payloadMediaType, _, err := fromStore.Payload() + if err != nil { + t.Fatalf("%s: error getting payload %v", testname, err) + } + + if payloadMediaType != v1.MediaTypeImageManifest { + t.Fatalf("%s: unexpected MediaType for manifest payload, %s", testname, payloadMediaType) + } + + // and the image index + + fromStore, err = ms.Get(ctx, indexDigest) + if err != nil { + t.Fatalf("%s: unexpected error fetching image index: %v", testname, err) + } + + fetchedIndex, ok := fromStore.(*manifestlist.DeserializedManifestList) + if !ok { + t.Fatalf("%s: unexpected type for fetched manifest", testname) + } + + if fetchedIndex.MediaType != indexMediaType { + t.Fatalf("%s: unexpected MediaType for result, %s", testname, fetchedManifest.MediaType) + } + + payloadMediaType, _, err = fromStore.Payload() + if err != nil { + t.Fatalf("%s: error getting payload %v", testname, err) + } + + if payloadMediaType != v1.MediaTypeImageIndex { + t.Fatalf("%s: unexpected MediaType for index payload, %s", testname, payloadMediaType) + } + +} + // TestLinkPathFuncs ensures that the link path functions behavior are locked // down and implemented as expected. func TestLinkPathFuncs(t *testing.T) { @@ -387,5 +539,4 @@ func TestLinkPathFuncs(t *testing.T) { t.Fatalf("incorrect path returned: %q != %q", p, testcase.expected) } } - }