r.loginov/distribution
1
0
Fork 0
forked from TrueCloudLab/distribution
Code Pull requests Activity

adds support for oci manifests and manifestlists

Browse source 
Signed-off-by: Mike Brown <brownwm@us.ibm.com>
This commit is contained in:
Mike Brown 2016-11-17 12:28:05 -06:00
parent 749f6afb45
commit 9986e8ca7c
10 changed files with 957 additions and 62 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
manifest/manifestlist/manifestlist.go
View file

@ -7,11 +7,17 @@ import (
"github.com/docker/distribution"
"github.com/docker/distribution/manifest"
"github.com/docker/distribution/manifest/ocischema"
"github.com/opencontainers/go-digest"
)
// MediaTypeManifestList specifies the mediaType for manifest lists.
const MediaTypeManifestList = "application/vnd.docker.distribution.manifest.list.v2+json"
const (
// MediaTypeManifestList specifies the mediaType for manifest lists.
MediaTypeManifestList = "application/vnd.docker.distribution.manifest.list.v2+json"
// MediaTypeOCIManifestList specifies the mediaType for OCI compliant manifest
// lists.
MediaTypeOCIManifestList = "application/vnd.oci.image.manifest.list.v1+json"
)
// SchemaVersion provides a pre-initialized version structure for this
// packages version of the manifest.
@ -20,6 +26,13 @@ var SchemaVersion = manifest.Versioned{
MediaType: MediaTypeManifestList,
}
// OCISchemaVersion provides a pre-initialized version structure for this
// packages OCIschema version of the manifest.
var OCISchemaVersion = manifest.Versioned{
SchemaVersion: 2,
MediaType: MediaTypeOCIManifestList,
}
func init() {
manifestListFunc := func(b []byte) (distribution.Manifest, distribution.Descriptor, error) {
m := new(DeserializedManifestList)
@ -105,8 +118,15 @@ type DeserializedManifestList struct {
// DeserializedManifestList which contains the resulting manifest list
// and its JSON representation.
func FromDescriptors(descriptors []ManifestDescriptor) (*DeserializedManifestList, error) {
m := ManifestList{
Versioned: SchemaVersion,
var m ManifestList
if len(descriptors) > 0 && descriptors[0].Descriptor.MediaType == ocischema.MediaTypeManifest {
m = ManifestList{
Versioned: OCISchemaVersion,
}
} else {
m = ManifestList{
Versioned: SchemaVersion,
}
}
m.Manifests = make([]ManifestDescriptor, len(descriptors), len(descriptors))

103
manifest/manifestlist/manifestlist_test.go
View file

@ -69,7 +69,7 @@ func TestManifestList(t *testing.T) {
t.Fatalf("error creating DeserializedManifestList: %v", err)
}
mediaType, canonical, err := deserialized.Payload()
mediaType, canonical, _ := deserialized.Payload()
if mediaType != MediaTypeManifestList {
t.Fatalf("unexpected media type: %s", mediaType)
@ -109,3 +109,104 @@ func TestManifestList(t *testing.T) {
}
}
}
var expectedOCIManifestListSerialization = []byte(`{
"schemaVersion": 2,
"mediaType": "application/vnd.oci.image.manifest.list.v1+json",
"manifests": [
{
"mediaType": "application/vnd.oci.image.manifest.v1+json",
"size": 985,
"digest": "sha256:1a9ec845ee94c202b2d5da74a24f0ed2058318bfa9879fa541efaecba272e86b",
"platform": {
"architecture": "amd64",
"os": "linux",
"features": [
"sse4"
]
}
},
{
"mediaType": "application/vnd.oci.image.manifest.v1+json",
"size": 2392,
"digest": "sha256:6346340964309634683409684360934680934608934608934608934068934608",
"platform": {
"architecture": "sun4m",
"os": "sunos"
}
}
]
}`)
func TestOCIManifestList(t *testing.T) {
manifestDescriptors := []ManifestDescriptor{
{
Descriptor: distribution.Descriptor{
Digest: "sha256:1a9ec845ee94c202b2d5da74a24f0ed2058318bfa9879fa541efaecba272e86b",
Size: 985,
MediaType: "application/vnd.oci.image.manifest.v1+json",
},
Platform: PlatformSpec{
Architecture: "amd64",
OS: "linux",
Features: []string{"sse4"},
},
},
{
Descriptor: distribution.Descriptor{
Digest: "sha256:6346340964309634683409684360934680934608934608934608934068934608",
Size: 2392,
MediaType: "application/vnd.oci.image.manifest.v1+json",
},
Platform: PlatformSpec{
Architecture: "sun4m",
OS: "sunos",
},
},
}
deserialized, err := FromDescriptors(manifestDescriptors)
if err != nil {
t.Fatalf("error creating DeserializedManifestList: %v", err)
}
mediaType, canonical, _ := deserialized.Payload()
if mediaType != MediaTypeOCIManifestList {
t.Fatalf("unexpected media type: %s", mediaType)
}
// Check that the canonical field is the same as json.MarshalIndent
// with these parameters.
p, err := json.MarshalIndent(&deserialized.ManifestList, "", " ")
if err != nil {
t.Fatalf("error marshaling manifest list: %v", err)
}
if !bytes.Equal(p, canonical) {
t.Fatalf("manifest bytes not equal: %q != %q", string(canonical), string(p))
}
// Check that the canonical field has the expected value.
if !bytes.Equal(expectedOCIManifestListSerialization, canonical) {
t.Fatalf("manifest bytes not equal: %q != %q", string(canonical), string(expectedOCIManifestListSerialization))
}
var unmarshalled DeserializedManifestList
if err := json.Unmarshal(deserialized.canonical, &unmarshalled); err != nil {
t.Fatalf("error unmarshaling manifest: %v", err)
}
if !reflect.DeepEqual(&unmarshalled, deserialized) {
t.Fatalf("manifests are different after unmarshaling: %v != %v", unmarshalled, *deserialized)
}
references := deserialized.References()
if len(references) != 2 {
t.Fatalf("unexpected number of references: %d", len(references))
}
for i := range references {
if !reflect.DeepEqual(references[i], manifestDescriptors[i].Descriptor) {
t.Fatalf("unexpected value %d returned by References: %v", i, references[i])
}
}
}

80
manifest/ocischema/builder.go Normal file
View file

@ -0,0 +1,80 @@
package ocischema
import (
"github.com/docker/distribution"
"github.com/docker/distribution/context"
"github.com/opencontainers/go-digest"
)
// builder is a type for constructing manifests.
type builder struct {
// bs is a BlobService used to publish the configuration blob.
bs distribution.BlobService
// configJSON references
configJSON []byte
// layers is a list of layer descriptors that gets built by successive
// calls to AppendReference.
layers []distribution.Descriptor
}
// NewManifestBuilder is used to build new manifests for the current schema
// version. It takes a BlobService so it can publish the configuration blob
// as part of the Build process.
func NewManifestBuilder(bs distribution.BlobService, configJSON []byte) distribution.ManifestBuilder {
mb := &builder{
bs: bs,
configJSON: make([]byte, len(configJSON)),
}
copy(mb.configJSON, configJSON)
return mb
}
// Build produces a final manifest from the given references.
func (mb *builder) Build(ctx context.Context) (distribution.Manifest, error) {
m := Manifest{
Versioned: SchemaVersion,
Layers: make([]distribution.Descriptor, len(mb.layers)),
}
copy(m.Layers, mb.layers)
configDigest := digest.FromBytes(mb.configJSON)
var err error
m.Config, err = mb.bs.Stat(ctx, configDigest)
switch err {
case nil:
// Override MediaType, since Put always replaces the specified media
// type with application/octet-stream in the descriptor it returns.
m.Config.MediaType = MediaTypeConfig
return FromStruct(m)
case distribution.ErrBlobUnknown:
// nop
default:
return nil, err
}
// Add config to the blob store
m.Config, err = mb.bs.Put(ctx, MediaTypeConfig, mb.configJSON)
// Override MediaType, since Put always replaces the specified media
// type with application/octet-stream in the descriptor it returns.
m.Config.MediaType = MediaTypeConfig
if err != nil {
return nil, err
}
return FromStruct(m)
}
// AppendReference adds a reference to the current ManifestBuilder.
func (mb *builder) AppendReference(d distribution.Describable) error {
mb.layers = append(mb.layers, d.Descriptor())
return nil
}
// References returns the current references added to this builder.
func (mb *builder) References() []distribution.Descriptor {
return mb.layers
}

210
manifest/ocischema/builder_test.go Normal file
View file

@ -0,0 +1,210 @@
package ocischema
import (
"reflect"
"testing"
"github.com/docker/distribution"
"github.com/docker/distribution/context"
"github.com/opencontainers/go-digest"
)
type mockBlobService struct {
descriptors map[digest.Digest]distribution.Descriptor
}
func (bs *mockBlobService) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) {
if descriptor, ok := bs.descriptors[dgst]; ok {
return descriptor, nil
}
return distribution.Descriptor{}, distribution.ErrBlobUnknown
}
func (bs *mockBlobService) Get(ctx context.Context, dgst digest.Digest) ([]byte, error) {
panic("not implemented")
}
func (bs *mockBlobService) Open(ctx context.Context, dgst digest.Digest) (distribution.ReadSeekCloser, error) {
panic("not implemented")
}
func (bs *mockBlobService) Put(ctx context.Context, mediaType string, p []byte) (distribution.Descriptor, error) {
d := distribution.Descriptor{
Digest: digest.FromBytes(p),
Size: int64(len(p)),
MediaType: "application/octet-stream",
}
bs.descriptors[d.Digest] = d
return d, nil
}
func (bs *mockBlobService) Create(ctx context.Context, options ...distribution.BlobCreateOption) (distribution.BlobWriter, error) {
panic("not implemented")
}
func (bs *mockBlobService) Resume(ctx context.Context, id string) (distribution.BlobWriter, error) {
panic("not implemented")
}
func TestBuilder(t *testing.T) {
imgJSON := []byte(`{
"architecture": "amd64",
"config": {
"AttachStderr": false,
"AttachStdin": false,
"AttachStdout": false,
"Cmd": [
"/bin/sh",
"-c",
"echo hi"
],
"Domainname": "",
"Entrypoint": null,
"Env": [
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
"derived=true",
"asdf=true"
],
"Hostname": "23304fc829f9",
"Image": "sha256:4ab15c48b859c2920dd5224f92aabcd39a52794c5b3cf088fb3bbb438756c246",
"Labels": {},
"OnBuild": [],
"OpenStdin": false,
"StdinOnce": false,
"Tty": false,
"User": "",
"Volumes": null,
"WorkingDir": ""
},
"container": "e91032eb0403a61bfe085ff5a5a48e3659e5a6deae9f4d678daa2ae399d5a001",
"container_config": {
"AttachStderr": false,
"AttachStdin": false,
"AttachStdout": false,
"Cmd": [
"/bin/sh",
"-c",
"#(nop) CMD [\"/bin/sh\" \"-c\" \"echo hi\"]"
],
"Domainname": "",
"Entrypoint": null,
"Env": [
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
"derived=true",
"asdf=true"
],
"Hostname": "23304fc829f9",
"Image": "sha256:4ab15c48b859c2920dd5224f92aabcd39a52794c5b3cf088fb3bbb438756c246",
"Labels": {},
"OnBuild": [],
"OpenStdin": false,
"StdinOnce": false,
"Tty": false,
"User": "",
"Volumes": null,
"WorkingDir": ""
},
"created": "2015-11-04T23:06:32.365666163Z",
"docker_version": "1.9.0-dev",
"history": [
{
"created": "2015-10-31T22:22:54.690851953Z",
"created_by": "/bin/sh -c #(nop) ADD file:a3bc1e842b69636f9df5256c49c5374fb4eef1e281fe3f282c65fb853ee171c5 in /"
},
{
"created": "2015-10-31T22:22:55.613815829Z",
"created_by": "/bin/sh -c #(nop) CMD [\"sh\"]"
},
{
"created": "2015-11-04T23:06:30.934316144Z",
"created_by": "/bin/sh -c #(nop) ENV derived=true",
"empty_layer": true
},
{
"created": "2015-11-04T23:06:31.192097572Z",
"created_by": "/bin/sh -c #(nop) ENV asdf=true",
"empty_layer": true
},
{
"created": "2015-11-04T23:06:32.083868454Z",
"created_by": "/bin/sh -c dd if=/dev/zero of=/file bs=1024 count=1024"
},
{
"created": "2015-11-04T23:06:32.365666163Z",
"created_by": "/bin/sh -c #(nop) CMD [\"/bin/sh\" \"-c\" \"echo hi\"]",
"empty_layer": true
}
],
"os": "linux",
"rootfs": {
"diff_ids": [
"sha256:c6f988f4874bb0add23a778f753c65efe992244e148a1d2ec2a8b664fb66bbd1",
"sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef",
"sha256:13f53e08df5a220ab6d13c58b2bf83a59cbdc2e04d0a3f041ddf4b0ba4112d49"
],
"type": "layers"
}
}`)
configDigest := digest.FromBytes(imgJSON)
descriptors := []distribution.Descriptor{
{
Digest: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"),
Size: 5312,
MediaType: MediaTypeLayer,
},
{
Digest: digest.Digest("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa"),
Size: 235231,
MediaType: MediaTypeLayer,
},
{
Digest: digest.Digest("sha256:b4ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"),
Size: 639152,
MediaType: MediaTypeLayer,
},
}
bs := &mockBlobService{descriptors: make(map[digest.Digest]distribution.Descriptor)}
builder := NewManifestBuilder(bs, imgJSON)
for _, d := range descriptors {
if err := builder.AppendReference(d); err != nil {
t.Fatalf("AppendReference returned error: %v", err)
}
}
built, err := builder.Build(context.Background())
if err != nil {
t.Fatalf("Build returned error: %v", err)
}
// Check that the config was put in the blob store
_, err = bs.Stat(context.Background(), configDigest)
if err != nil {
t.Fatal("config was not put in the blob store")
}
manifest := built.(*DeserializedManifest).Manifest
if manifest.Versioned.SchemaVersion != 2 {
t.Fatal("SchemaVersion != 2")
}
target := manifest.Target()
if target.Digest != configDigest {
t.Fatalf("unexpected digest in target: %s", target.Digest.String())
}
if target.MediaType != MediaTypeConfig {
t.Fatalf("unexpected media type in target: %s", target.MediaType)
}
if target.Size != 3153 {
t.Fatalf("unexpected size in target: %d", target.Size)
}
references := manifest.References()
expected := append([]distribution.Descriptor{manifest.Target()}, descriptors...)
if !reflect.DeepEqual(references, expected) {
t.Fatal("References() does not match the descriptors added")
}
}

133
manifest/ocischema/manifest.go Normal file
View file

@ -0,0 +1,133 @@
package ocischema
import (
"encoding/json"
"errors"
"fmt"
"github.com/docker/distribution"
"github.com/docker/distribution/manifest"
"github.com/opencontainers/go-digest"
)
const (
// MediaTypeManifest specifies the mediaType for the current version.
MediaTypeManifest = "application/vnd.oci.image.manifest.v1+json"
// MediaTypeConfig specifies the mediaType for the image configuration.
MediaTypeConfig = "application/vnd.oci.image.config.v1+json"
// MediaTypePluginConfig specifies the mediaType for plugin configuration.
MediaTypePluginConfig = "application/vnd.docker.plugin.v1+json"
// MediaTypeLayer is the mediaType used for layers referenced by the manifest.
MediaTypeLayer = "application/vnd.oci.image.layer.v1.tar+gzip"
// MediaTypeForeignLayer is the mediaType used for layers that must be
// downloaded from foreign URLs.
MediaTypeForeignLayer = "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip"
)
var (
// SchemaVersion provides a pre-initialized version structure for this
// packages version of the manifest.
SchemaVersion = manifest.Versioned{
SchemaVersion: 2, // Mike: todo this could confusing cause oci version 1 is closer to docker 2 than 1
MediaType: MediaTypeManifest,
}
)
func init() {
ocischemaFunc := func(b []byte) (distribution.Manifest, distribution.Descriptor, error) {
m := new(DeserializedManifest)
err := m.UnmarshalJSON(b)
if err != nil {
return nil, distribution.Descriptor{}, err
}
dgst := digest.FromBytes(b)
return m, distribution.Descriptor{Digest: dgst, Size: int64(len(b)), MediaType: MediaTypeManifest}, err
}
err := distribution.RegisterManifestSchema(MediaTypeManifest, ocischemaFunc)
if err != nil {
panic(fmt.Sprintf("Unable to register manifest: %s", err))
}
}
// Manifest defines a schema2 manifest.
type Manifest struct {
manifest.Versioned
// Config references the image configuration as a blob.
Config distribution.Descriptor `json:"config"`
// Layers lists descriptors for the layers referenced by the
// configuration.
Layers []distribution.Descriptor `json:"layers"`
}
// References returnes the descriptors of this manifests references.
func (m Manifest) References() []distribution.Descriptor {
references := make([]distribution.Descriptor, 0, 1+len(m.Layers))
references = append(references, m.Config)
references = append(references, m.Layers...)
return references
}
// Target returns the target of this signed manifest.
func (m Manifest) Target() distribution.Descriptor {
return m.Config
}
// DeserializedManifest wraps Manifest with a copy of the original JSON.
// It satisfies the distribution.Manifest interface.
type DeserializedManifest struct {
Manifest
// canonical is the canonical byte representation of the Manifest.
canonical []byte
}
// FromStruct takes a Manifest structure, marshals it to JSON, and returns a
// DeserializedManifest which contains the manifest and its JSON representation.
func FromStruct(m Manifest) (*DeserializedManifest, error) {
var deserialized DeserializedManifest
deserialized.Manifest = m
var err error
deserialized.canonical, err = json.MarshalIndent(&m, "", " ")
return &deserialized, err
}
// UnmarshalJSON populates a new Manifest struct from JSON data.
func (m *DeserializedManifest) UnmarshalJSON(b []byte) error {
m.canonical = make([]byte, len(b), len(b))
// store manifest in canonical
copy(m.canonical, b)
// Unmarshal canonical JSON into Manifest object
var manifest Manifest
if err := json.Unmarshal(m.canonical, &manifest); err != nil {
return err
}
m.Manifest = manifest
return nil
}
// MarshalJSON returns the contents of canonical. If canonical is empty,
// marshals the inner contents.
func (m *DeserializedManifest) MarshalJSON() ([]byte, error) {
if len(m.canonical) > 0 {
return m.canonical, nil
}
return nil, errors.New("JSON representation not initialized in DeserializedManifest")
}
// Payload returns the raw content of the manifest. The contents can be used to
// calculate the content identifier.
func (m DeserializedManifest) Payload() (string, []byte, error) {
return m.MediaType, m.canonical, nil
}

111
manifest/ocischema/manifest_test.go Normal file
View file

@ -0,0 +1,111 @@
package ocischema
import (
"bytes"
"encoding/json"
"reflect"
"testing"
"github.com/docker/distribution"
)
var expectedManifestSerialization = []byte(`{
"schemaVersion": 2,
"mediaType": "application/vnd.oci.image.manifest.v1+json",
"config": {
"mediaType": "application/vnd.oci.image.config.v1+json",
"size": 985,
"digest": "sha256:1a9ec845ee94c202b2d5da74a24f0ed2058318bfa9879fa541efaecba272e86b"
},
"layers": [
{
"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip",
"size": 153263,
"digest": "sha256:62d8908bee94c202b2d35224a221aaa2058318bfa9879fa541efaecba272331b"
}
]
}`)
func TestManifest(t *testing.T) {
manifest := Manifest{
Versioned: SchemaVersion,
Config: distribution.Descriptor{
Digest: "sha256:1a9ec845ee94c202b2d5da74a24f0ed2058318bfa9879fa541efaecba272e86b",
Size: 985,
MediaType: MediaTypeConfig,
},
Layers: []distribution.Descriptor{
{
Digest: "sha256:62d8908bee94c202b2d35224a221aaa2058318bfa9879fa541efaecba272331b",
Size: 153263,
MediaType: MediaTypeLayer,
},
},
}
deserialized, err := FromStruct(manifest)
if err != nil {
t.Fatalf("error creating DeserializedManifest: %v", err)
}
mediaType, canonical, err := deserialized.Payload()
if mediaType != MediaTypeManifest {
t.Fatalf("unexpected media type: %s", mediaType)
}
// Check that the canonical field is the same as json.MarshalIndent
// with these parameters.
p, err := json.MarshalIndent(&manifest, "", " ")
if err != nil {
t.Fatalf("error marshaling manifest: %v", err)
}
if !bytes.Equal(p, canonical) {
t.Fatalf("manifest bytes not equal: %q != %q", string(canonical), string(p))
}
// Check that canonical field matches expected value.
if !bytes.Equal(expectedManifestSerialization, canonical) {
t.Fatalf("manifest bytes not equal: %q != %q", string(canonical), string(expectedManifestSerialization))
}
var unmarshalled DeserializedManifest
if err := json.Unmarshal(deserialized.canonical, &unmarshalled); err != nil {
t.Fatalf("error unmarshaling manifest: %v", err)
}
if !reflect.DeepEqual(&unmarshalled, deserialized) {
t.Fatalf("manifests are different after unmarshaling: %v != %v", unmarshalled, *deserialized)
}
target := deserialized.Target()
if target.Digest != "sha256:1a9ec845ee94c202b2d5da74a24f0ed2058318bfa9879fa541efaecba272e86b" {
t.Fatalf("unexpected digest in target: %s", target.Digest.String())
}
if target.MediaType != MediaTypeConfig {
t.Fatalf("unexpected media type in target: %s", target.MediaType)
}
if target.Size != 985 {
t.Fatalf("unexpected size in target: %d", target.Size)
}
references := deserialized.References()
if len(references) != 2 {
t.Fatalf("unexpected number of references: %d", len(references))
}
if !reflect.DeepEqual(references[0], target) {
t.Fatalf("first reference should be target: %v != %v", references[0], target)
}
// Test the second reference
if references[1].Digest != "sha256:62d8908bee94c202b2d35224a221aaa2058318bfa9879fa541efaecba272331b" {
t.Fatalf("unexpected digest in reference: %s", references[0].Digest.String())
}
if references[1].MediaType != MediaTypeLayer {
t.Fatalf("unexpected media type in reference: %s", references[0].MediaType)
}
if references[1].Size != 153263 {
t.Fatalf("unexpected size in reference: %d", references[0].Size)
}
}

30
registry/handlers/api_test.go
View file

@ -478,7 +478,7 @@ func testBlobAPI(t *testing.T, env *testEnv, args blobArgs) *testEnv {
// -----------------------------------------
// Do layer push with an empty body and different digest
uploadURLBase, uploadUUID = startPushLayer(t, env, imageName)
uploadURLBase, _ = startPushLayer(t, env, imageName)
resp, err = doPushLayer(t, env.builder, imageName, layerDigest, uploadURLBase, bytes.NewReader([]byte{}))
if err != nil {
t.Fatalf("unexpected error doing bad layer push: %v", err)
@ -494,7 +494,7 @@ func testBlobAPI(t *testing.T, env *testEnv, args blobArgs) *testEnv {
t.Fatalf("unexpected error digesting empty buffer: %v", err)
}
uploadURLBase, uploadUUID = startPushLayer(t, env, imageName)
uploadURLBase, _ = startPushLayer(t, env, imageName)
pushLayer(t, env.builder, imageName, zeroDigest, uploadURLBase, bytes.NewReader([]byte{}))
// -----------------------------------------
@ -507,7 +507,7 @@ func testBlobAPI(t *testing.T, env *testEnv, args blobArgs) *testEnv {
t.Fatalf("unexpected error digesting empty tar: %v", err)
}
uploadURLBase, uploadUUID = startPushLayer(t, env, imageName)
uploadURLBase, _ = startPushLayer(t, env, imageName)
pushLayer(t, env.builder, imageName, emptyDigest, uploadURLBase, bytes.NewReader(emptyTar))
// ------------------------------------------
@ -515,7 +515,7 @@ func testBlobAPI(t *testing.T, env *testEnv, args blobArgs) *testEnv {
layerLength, _ := layerFile.Seek(0, os.SEEK_END)
layerFile.Seek(0, os.SEEK_SET)
uploadURLBase, uploadUUID = startPushLayer(t, env, imageName)
uploadURLBase, _ = startPushLayer(t, env, imageName)
pushLayer(t, env.builder, imageName, layerDigest, uploadURLBase, layerFile)
// ------------------------------------------
@ -529,7 +529,7 @@ func testBlobAPI(t *testing.T, env *testEnv, args blobArgs) *testEnv {
canonicalDigest := canonicalDigester.Digest()
layerFile.Seek(0, 0)
uploadURLBase, uploadUUID = startPushLayer(t, env, imageName)
uploadURLBase, _ = startPushLayer(t, env, imageName)
uploadURLBase, dgst := pushChunk(t, env.builder, imageName, uploadURLBase, layerFile, layerLength)
finishUpload(t, env.builder, imageName, uploadURLBase, dgst)
@ -612,7 +612,7 @@ func testBlobAPI(t *testing.T, env *testEnv, args blobArgs) *testEnv {
t.Fatalf("Error constructing request: %s", err)
}
req.Header.Set("If-None-Match", "")
resp, err = http.DefaultClient.Do(req)
resp, _ = http.DefaultClient.Do(req)
checkResponse(t, "fetching layer with invalid etag", resp, http.StatusOK)
// Missing tests:
@ -1874,7 +1874,7 @@ func testManifestDelete(t *testing.T, env *testEnv, args manifestArgs) {
manifest := args.manifest
ref, _ := reference.WithDigest(imageName, dgst)
manifestDigestURL, err := env.builder.BuildManifestURL(ref)
manifestDigestURL, _ := env.builder.BuildManifestURL(ref)
// ---------------
// Delete by digest
resp, err := httpDelete(manifestDigestURL)
@ -1935,7 +1935,7 @@ func testManifestDelete(t *testing.T, env *testEnv, args manifestArgs) {
// Upload manifest by tag
tag := "atag"
tagRef, _ := reference.WithTag(imageName, tag)
manifestTagURL, err := env.builder.BuildManifestURL(tagRef)
manifestTagURL, _ := env.builder.BuildManifestURL(tagRef)
resp = putManifest(t, "putting manifest by tag", manifestTagURL, args.mediaType, manifest)
checkResponse(t, "putting manifest by tag", resp, http.StatusCreated)
checkHeaders(t, resp, http.Header{
@ -2502,7 +2502,7 @@ func TestRegistryAsCacheMutationAPIs(t *testing.T) {
checkResponse(t, "putting signed manifest to cache", resp, errcode.ErrorCodeUnsupported.Descriptor().HTTPStatusCode)
// Manifest Delete
resp, err = httpDelete(manifestURL)
resp, _ = httpDelete(manifestURL)
checkResponse(t, "deleting signed manifest from cache", resp, errcode.ErrorCodeUnsupported.Descriptor().HTTPStatusCode)
// Blob upload initialization
@ -2521,8 +2521,8 @@ func TestRegistryAsCacheMutationAPIs(t *testing.T) {
// Blob Delete
ref, _ := reference.WithDigest(imageName, digestSha256EmptyTar)
blobURL, err := env.builder.BuildBlobURL(ref)
resp, err = httpDelete(blobURL)
blobURL, _ := env.builder.BuildBlobURL(ref)
resp, _ = httpDelete(blobURL)
checkResponse(t, "deleting blob from cache", resp, errcode.ErrorCodeUnsupported.Descriptor().HTTPStatusCode)
}
@ -2601,9 +2601,9 @@ func TestProxyManifestGetByTag(t *testing.T) {
checkErr(t, err, "building manifest url")
resp, err = http.Get(manifestTagURL)
checkErr(t, err, "fetching manifest from proxy by tag")
checkErr(t, err, "fetching manifest from proxy by tag (error check 1)")
defer resp.Body.Close()
checkResponse(t, "fetching manifest from proxy by tag", resp, http.StatusOK)
checkResponse(t, "fetching manifest from proxy by tag (response check 1)", resp, http.StatusOK)
checkHeaders(t, resp, http.Header{
"Docker-Content-Digest": []string{dgst.String()},
})
@ -2616,9 +2616,9 @@ func TestProxyManifestGetByTag(t *testing.T) {
// fetch it with the same proxy URL as before. Ensure the updated content is at the same tag
resp, err = http.Get(manifestTagURL)
checkErr(t, err, "fetching manifest from proxy by tag")
checkErr(t, err, "fetching manifest from proxy by tag (error check 2)")
defer resp.Body.Close()
checkResponse(t, "fetching manifest from proxy by tag", resp, http.StatusOK)
checkResponse(t, "fetching manifest from proxy by tag (response check 2)", resp, http.StatusOK)
checkHeaders(t, resp, http.Header{
"Docker-Content-Digest": []string{newDigest.String()},
})

316
registry/handlers/manifests.go
View file

@ -9,6 +9,7 @@ import (
"github.com/docker/distribution"
dcontext "github.com/docker/distribution/context"
"github.com/docker/distribution/manifest/manifestlist"
"github.com/docker/distribution/manifest/ocischema"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/manifest/schema2"
"github.com/docker/distribution/reference"
@ -72,43 +73,10 @@ func (imh *manifestHandler) GetManifest(w http.ResponseWriter, r *http.Request)
imh.Errors = append(imh.Errors, err)
return
}
var manifest distribution.Manifest
if imh.Tag != "" {
tags := imh.Repository.Tags(imh)
desc, err := tags.Get(imh, imh.Tag)
if err != nil {
if _, ok := err.(distribution.ErrTagUnknown); ok {
imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithDetail(err))
} else {
imh.Errors = append(imh.Errors, errcode.ErrorCodeUnknown.WithDetail(err))
}
return
}
imh.Digest = desc.Digest
}
if etagMatch(r, imh.Digest.String()) {
w.WriteHeader(http.StatusNotModified)
return
}
var options []distribution.ManifestServiceOption
if imh.Tag != "" {
options = append(options, distribution.WithTag(imh.Tag))
}
manifest, err = manifests.Get(imh, imh.Digest, options...)
if err != nil {
if _, ok := err.(distribution.ErrManifestUnknownRevision); ok {
imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithDetail(err))
} else {
imh.Errors = append(imh.Errors, errcode.ErrorCodeUnknown.WithDetail(err))
}
return
}
supportsSchema2 := false
supportsManifestList := false
supportsOCISchema := false
supportsOCIManifestList := false
// this parsing of Accept headers is not quite as full-featured as godoc.org's parser, but we don't care about "q=" values
// https://github.com/golang/gddo/blob/e91d4165076d7474d20abda83f92d15c7ebc3e81/httputil/header/header.go#L165-L202
for _, acceptHeader := range r.Header["Accept"] {
@ -132,16 +100,259 @@ func (imh *manifestHandler) GetManifest(w http.ResponseWriter, r *http.Request)
if mediaType == manifestlist.MediaTypeManifestList {
supportsManifestList = true
}
if mediaType == ocischema.MediaTypeManifest {
supportsOCISchema = true
}
if mediaType == manifestlist.MediaTypeOCIManifestList {
supportsOCIManifestList = true
}
}
}
supportsOCI := supportsOCISchema || supportsOCIManifestList
var manifest distribution.Manifest
if imh.Tag != "" {
tags := imh.Repository.Tags(imh)
var desc distribution.Descriptor
if !supportsOCI {
desc, err = tags.Get(imh, imh.Tag)
} else {
desc, err = tags.Get(imh, imh.annotatedTag(false))
}
if err != nil {
if _, ok := err.(distribution.ErrTagUnknown); ok {
imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithDetail(err))
} else {
imh.Errors = append(imh.Errors, errcode.ErrorCodeUnknown.WithDetail(err))
}
return
}
imh.Digest = desc.Digest
}
if etagMatch(r, imh.Digest.String()) {
w.WriteHeader(http.StatusNotModified)
return
}
var options []distribution.ManifestServiceOption
if imh.Tag != "" {
options = append(options, distribution.WithTag(imh.annotatedTag(supportsOCI)))
}
manifest, err = manifests.Get(imh, imh.Digest, options...)
if err != nil {
if _, ok := err.(distribution.ErrManifestUnknownRevision); ok {
imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithDetail(err))
} else {
imh.Errors = append(imh.Errors, errcode.ErrorCodeUnknown.WithDetail(err))
}
return
}
schema2Manifest, isSchema2 := manifest.(*schema2.DeserializedManifest)
manifestList, isManifestList := manifest.(*manifestlist.DeserializedManifestList)
isAnOCIManifest := isSchema2 && (schema2Manifest.MediaType == ocischema.MediaTypeManifest)
isAnOCIManifestList := isManifestList && (manifestList.MediaType == manifestlist.MediaTypeOCIManifestList)
if (isSchema2 && !isAnOCIManifest) && (supportsOCISchema && !supportsSchema2) {
fmt.Printf("\n\nmanifest is schema2, but accept header only supports OCISchema \n\n")
w.WriteHeader(http.StatusNotFound)
return
}
if (isManifestList && !isAnOCIManifestList) && (supportsOCIManifestList && !supportsManifestList) {
fmt.Printf("\n\nmanifestlist is not OCI, but accept header only supports an OCI manifestlist\n\n")
w.WriteHeader(http.StatusNotFound)
return
}
if isAnOCIManifest && (!supportsOCISchema && supportsSchema2) {
fmt.Printf("\n\nmanifest is OCI, but accept header only supports schema2\n\n")
w.WriteHeader(http.StatusNotFound)
return
}
if isAnOCIManifestList && (!supportsOCIManifestList && supportsManifestList) {
fmt.Printf("\n\nmanifestlist is OCI, but accept header only supports non-OCI manifestlists\n\n")
w.WriteHeader(http.StatusNotFound)
return
}
// Only rewrite schema2 manifests when they are being fetched by tag.
// If they are being fetched by digest, we can't return something not
// matching the digest.
if imh.Tag != "" && isSchema2 && !(supportsSchema2 || supportsOCISchema) {
// Rewrite manifest in schema1 format
ctxu.GetLogger(imh).Infof("rewriting manifest %s in schema1 format to support old client", imh.Digest.String())
manifest, err = imh.convertSchema2Manifest(schema2Manifest)
if err != nil {
return
}
} else if imh.Tag != "" && isManifestList && !(supportsManifestList || supportsOCIManifestList) {
// Rewrite manifest in schema1 format
ctxu.GetLogger(imh).Infof("rewriting manifest list %s in schema1 format to support old client", imh.Digest.String())
// Find the image manifest corresponding to the default
// platform
var manifestDigest digest.Digest
for _, manifestDescriptor := range manifestList.Manifests {
if manifestDescriptor.Platform.Architecture == defaultArch && manifestDescriptor.Platform.OS == defaultOS {
manifestDigest = manifestDescriptor.Digest
break
}
}
if manifestDigest == "" {
imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown)
return
}
manifest, err = manifests.Get(imh, manifestDigest)
if err != nil {
if _, ok := err.(distribution.ErrManifestUnknownRevision); ok {
imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithDetail(err))
} else {
imh.Errors = append(imh.Errors, errcode.ErrorCodeUnknown.WithDetail(err))
}
return
}
// If necessary, convert the image manifest
if schema2Manifest, isSchema2 := manifest.(*schema2.DeserializedManifest); isSchema2 && !(supportsSchema2 || supportsOCISchema) {
manifest, err = imh.convertSchema2Manifest(schema2Manifest)
if err != nil {
return
}
} else {
imh.Digest = manifestDigest
}
}
ct, p, err := manifest.Payload()
if err != nil {
return
}
w.Header().Set("Content-Type", ct)
w.Header().Set("Content-Length", fmt.Sprint(len(p)))
w.Header().Set("Docker-Content-Digest", imh.Digest.String())
w.Header().Set("Etag", fmt.Sprintf(`"%s"`, imh.Digest))
w.Write(p)
}
// GetImageManifest fetches the image manifest from the storage backend, if it exists.
func (imh *manifestHandler) GetImageManifest(w http.ResponseWriter, r *http.Request) {
fmt.Printf("\n\nGetting a manifest!\n\n\n")
supportsSchema2 := false
supportsManifestList := false
supportsOCISchema := false
supportsOCIManifestList := false
// this parsing of Accept headers is not quite as full-featured as godoc.org's parser, but we don't care about "q=" values
// https://github.com/golang/gddo/blob/e91d4165076d7474d20abda83f92d15c7ebc3e81/httputil/header/header.go#L165-L202
for _, acceptHeader := range r.Header["Accept"] {
// r.Header[...] is a slice in case the request contains the same header more than once
// if the header isn't set, we'll get the zero value, which "range" will handle gracefully
// we need to split each header value on "," to get the full list of "Accept" values (per RFC 2616)
// https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.1
for _, mediaType := range strings.Split(acceptHeader, ",") {
// remove "; q=..." if present
if i := strings.Index(mediaType, ";"); i >= 0 {
mediaType = mediaType[:i]
}
// it's common (but not required) for Accept values to be space separated ("a/b, c/d, e/f")
mediaType = strings.TrimSpace(mediaType)
if mediaType == schema2.MediaTypeManifest {
supportsSchema2 = true
}
if mediaType == manifestlist.MediaTypeManifestList {
supportsManifestList = true
}
if mediaType == ocischema.MediaTypeManifest {
supportsOCISchema = true
}
if mediaType == manifestlist.MediaTypeOCIManifestList {
supportsOCIManifestList = true
}
}
}
supportsOCI := supportsOCISchema || supportsOCIManifestList
ctxu.GetLogger(imh).Debug("GetImageManifest")
manifests, err := imh.Repository.Manifests(imh)
if err != nil {
imh.Errors = append(imh.Errors, err)
return
}
var manifest distribution.Manifest
if imh.Tag != "" {
tags := imh.Repository.Tags(imh)
var desc distribution.Descriptor
if !supportsOCI {
desc, err = tags.Get(imh, imh.Tag)
if err != nil {
imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithDetail(err))
return
}
} else {
desc, err = tags.Get(imh, imh.annotatedTag(supportsOCI))
if err != nil {
desc, err = tags.Get(imh, imh.annotatedTag(false))
if err != nil {
imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithDetail(err))
return
}
}
}
imh.Digest = desc.Digest
}
if etagMatch(r, imh.Digest.String()) {
w.WriteHeader(http.StatusNotModified)
return
}
var options []distribution.ManifestServiceOption
if imh.Tag != "" {
options = append(options, distribution.WithTag(imh.annotatedTag(supportsOCI)))
}
manifest, err = manifests.Get(imh, imh.Digest, options...)
if err != nil {
imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithDetail(err))
return
}
schema2Manifest, isSchema2 := manifest.(*schema2.DeserializedManifest)
manifestList, isManifestList := manifest.(*manifestlist.DeserializedManifestList)
isAnOCIManifest := isSchema2 && (schema2Manifest.MediaType == ocischema.MediaTypeManifest)
isAnOCIManifestList := isManifestList && (manifestList.MediaType == manifestlist.MediaTypeOCIManifestList)
badCombinations := [][]bool{
{isSchema2 && !isAnOCIManifest, supportsOCISchema && !supportsSchema2},
{isManifestList && !isAnOCIManifestList, supportsOCIManifestList && !supportsManifestList},
{isAnOCIManifest, !supportsOCISchema && supportsSchema2},
{isAnOCIManifestList, !supportsOCIManifestList && supportsManifestList},
}
for i, combo := range badCombinations {
if combo[0] && combo[1] {
fmt.Printf("\n\nbad combo! %d\n\n\n", i)
w.WriteHeader(http.StatusNotFound)
return
}
}
if isAnOCIManifest {
fmt.Print("\n\nreturning OCI manifest\n\n")
} else if isSchema2 {
fmt.Print("\n\nreturning schema 2 manifest\n\n")
} else {
fmt.Print("\n\nreturning schema 1 manifest\n\n")
}
// Only rewrite schema2 manifests when they are being fetched by tag.
// If they are being fetched by digest, we can't return something not
// matching the digest.
if imh.Tag != "" && isSchema2 && !supportsSchema2 {
if imh.Tag != "" && isSchema2 && !(supportsSchema2 || supportsOCISchema) {
// Rewrite manifest in schema1 format
dcontext.GetLogger(imh).Infof("rewriting manifest %s in schema1 format to support old client", imh.Digest.String())
@ -149,7 +360,7 @@ func (imh *manifestHandler) GetManifest(w http.ResponseWriter, r *http.Request)
if err != nil {
return
}
} else if imh.Tag != "" && isManifestList && !supportsManifestList {
} else if imh.Tag != "" && isManifestList && !(supportsManifestList || supportsOCIManifestList) {
// Rewrite manifest in schema1 format
dcontext.GetLogger(imh).Infof("rewriting manifest list %s in schema1 format to support old client", imh.Digest.String())
@ -199,6 +410,8 @@ func (imh *manifestHandler) GetManifest(w http.ResponseWriter, r *http.Request)
w.Header().Set("Docker-Content-Digest", imh.Digest.String())
w.Header().Set("Etag", fmt.Sprintf(`"%s"`, imh.Digest))
w.Write(p)
fmt.Printf("\n\nSucceeded in getting the manifest!\n\n\n")
}
func (imh *manifestHandler) convertSchema2Manifest(schema2Manifest *schema2.DeserializedManifest) (distribution.Manifest, error) {
@ -286,9 +499,17 @@ func (imh *manifestHandler) PutManifest(w http.ResponseWriter, r *http.Request)
return
}
isAnOCIManifest := mediaType == ocischema.MediaTypeManifest || mediaType == manifestlist.MediaTypeOCIManifestList
if isAnOCIManifest {
fmt.Printf("\n\nPutting an OCI Manifest!\n\n\n")
} else {
fmt.Printf("\n\nPutting a Docker Manifest!\n\n\n")
}
var options []distribution.ManifestServiceOption
if imh.Tag != "" {
options = append(options, distribution.WithTag(imh.Tag))
options = append(options, distribution.WithTag(imh.annotatedTag(isAnOCIManifest)))
}
if err := imh.applyResourcePolicy(manifest); err != nil {
@ -301,10 +522,12 @@ func (imh *manifestHandler) PutManifest(w http.ResponseWriter, r *http.Request)
// TODO(stevvooe): These error handling switches really need to be
// handled by an app global mapper.
if err == distribution.ErrUnsupported {
fmt.Printf("\n\nXXX 1\n\n\n")
imh.Errors = append(imh.Errors, errcode.ErrorCodeUnsupported)
return
}
if err == distribution.ErrAccessDenied {
fmt.Printf("\n\nXXX 2\n\n\n")
imh.Errors = append(imh.Errors, errcode.ErrorCodeDenied)
return
}
@ -331,15 +554,16 @@ func (imh *manifestHandler) PutManifest(w http.ResponseWriter, r *http.Request)
default:
imh.Errors = append(imh.Errors, errcode.ErrorCodeUnknown.WithDetail(err))
}
fmt.Printf("\n\nXXX 3\n\n\n")
return
}
// Tag this manifest
if imh.Tag != "" {
tags := imh.Repository.Tags(imh)
err = tags.Tag(imh, imh.Tag, desc)
err = tags.Tag(imh, imh.annotatedTag(isAnOCIManifest), desc)
if err != nil {
fmt.Printf("\n\nXXX 4: %T: %v\n\n\n", err, err)
imh.Errors = append(imh.Errors, errcode.ErrorCodeUnknown.WithDetail(err))
return
}
@ -349,6 +573,7 @@ func (imh *manifestHandler) PutManifest(w http.ResponseWriter, r *http.Request)
// Construct a canonical url for the uploaded manifest.
ref, err := reference.WithDigest(imh.Repository.Named(), imh.Digest)
if err != nil {
fmt.Printf("\n\nXXX 5\n\n\n")
imh.Errors = append(imh.Errors, errcode.ErrorCodeUnknown.WithDetail(err))
return
}
@ -364,6 +589,8 @@ func (imh *manifestHandler) PutManifest(w http.ResponseWriter, r *http.Request)
w.Header().Set("Location", location)
w.Header().Set("Docker-Content-Digest", imh.Digest.String())
w.WriteHeader(http.StatusCreated)
fmt.Printf("\n\nSucceeded in putting manifest!\n\n\n")
}
// applyResourcePolicy checks whether the resource class matches what has
@ -478,3 +705,12 @@ func (imh *manifestHandler) DeleteManifest(w http.ResponseWriter, r *http.Reques
w.WriteHeader(http.StatusAccepted)
}
// annotatedTag will annotate OCI tags by prepending a string, and leave docker
// tags unmodified.
func (imh *manifestHandler) annotatedTag(oci bool) string {
if oci {
return "oci." + imh.Tag
}
return imh.Tag
}

2
registry/storage/driver/storagedriver.go
View file

@ -116,7 +116,7 @@ type FileWriter interface {
// number of path components separated by slashes, where each component is
// restricted to alphanumeric characters or a period, underscore, or
// hyphen.
var PathRegexp = regexp.MustCompile(`^(/[A-Za-z0-9._-]+)+$`)
var PathRegexp = regexp.MustCompile(`^(/[A-Za-z0-9._:-]+)+$`)
// ErrUnsupportedMethod may be returned in the case where a StorageDriver implementation does not support an optional method.
type ErrUnsupportedMethod struct {

6
registry/storage/manifeststore.go
View file

@ -9,6 +9,7 @@ import (
dcontext "github.com/docker/distribution/context"
"github.com/docker/distribution/manifest"
"github.com/docker/distribution/manifest/manifestlist"
"github.com/docker/distribution/manifest/ocischema"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/manifest/schema2"
"github.com/opencontainers/go-digest"
@ -48,6 +49,7 @@ type manifestStore struct {
schema1Handler ManifestHandler
schema2Handler ManifestHandler
ocischemaHandler ManifestHandler
manifestListHandler ManifestHandler
}
@ -99,7 +101,9 @@ func (ms *manifestStore) Get(ctx context.Context, dgst digest.Digest, options ..
switch versioned.MediaType {
case schema2.MediaTypeManifest:
return ms.schema2Handler.Unmarshal(ctx, dgst, content)
case manifestlist.MediaTypeManifestList:
case ocischema.MediaTypeManifest:
return ms.ocischemaHandler.Unmarshal(ctx, dgst, content)
case manifestlist.MediaTypeManifestList, manifestlist.MediaTypeOCIManifestList:
return ms.manifestListHandler.Unmarshal(ctx, dgst, content)
default:
return nil, distribution.ErrManifestVerification{fmt.Errorf("unrecognized manifest content type %s", versioned.MediaType)}