Add schema2 manifest support

Add schema2 manifest implementation.

Add a schema2 builder that creates a schema2 manifest from descriptors
and a configuration. It will add the configuration to the blob store if
necessary.

Rename the original schema1 manifest builder to ReferenceBuilder, and
create a ConfigBuilder variant that can build a schema1 manifest from an
image configuration and set of descriptors. This will be used to
translate schema2 manifests to the schema1 format for backward
compatibliity, by adding the descriptors from the existing schema2
manifest to the schema1 builder. It will also be used by engine-side
push code to create schema1 manifests from the new-style image
configration, when necessary to push a schema1 manifest.

Signed-off-by: Aaron Lehmann <aaron.lehmann@docker.com>

blobs.go

@@ -61,6 +61,15 @@ type Descriptor struct {
 	// depend on the simplicity of this type.
 }
 
+// Descriptor returns the descriptor, to make it satisfy the Describable
+// interface. Note that implementations of Describable are generally objects
+// which can be described, not simply descriptors; this exception is in place
+// to make it more convenient to pass actual descriptors to functions that
+// expect Describable objects.
+func (d Descriptor) Descriptor() Descriptor {
+	return d
+}
+
 // BlobStatter makes blob descriptors available by digest. The service may
 // provide a descriptor of a different digest if the provided digest is not
 // canonical.

manifest/schema1/config_builder.go

@@ -0,0 +1,278 @@
+package schema1
+
+import (
+	"crypto/sha512"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/context"
+	"github.com/docker/libtrust"
+
+	"github.com/docker/distribution/digest"
+	"github.com/docker/distribution/manifest"
+)
+
+type diffID digest.Digest
+
+// gzippedEmptyTar is a gzip-compressed version of an empty tar file
+// (1024 NULL bytes)
+var gzippedEmptyTar = []byte{
+	31, 139, 8, 0, 0, 9, 110, 136, 0, 255, 98, 24, 5, 163, 96, 20, 140, 88,
+	0, 8, 0, 0, 255, 255, 46, 175, 181, 239, 0, 4, 0, 0,
+}
+
+// digestSHA256GzippedEmptyTar is the canonical sha256 digest of
+// gzippedEmptyTar
+const digestSHA256GzippedEmptyTar = digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")
+
+// configManifestBuilder is a type for constructing manifests from an image
+// configuration and generic descriptors.
+type configManifestBuilder struct {
+	// bs is a BlobService used to create empty layer tars in the
+	// blob store if necessary.
+	bs distribution.BlobService
+	// pk is the libtrust private key used to sign the final manifest.
+	pk libtrust.PrivateKey
+	// configJSON is configuration supplied when the ManifestBuilder was
+	// created.
+	configJSON []byte
+	// name is the name provided to NewConfigManifestBuilder
+	name string
+	// tag is the tag provided to NewConfigManifestBuilder
+	tag string
+	// descriptors is the set of descriptors referencing the layers.
+	descriptors []distribution.Descriptor
+	// emptyTarDigest is set to a valid digest if an empty tar has been
+	// put in the blob store; otherwise it is empty.
+	emptyTarDigest digest.Digest
+}
+
+// NewConfigManifestBuilder is used to build new manifests for the current
+// schema version from an image configuration and a set of descriptors.
+// It takes a BlobService so that it can add an empty tar to the blob store
+// if the resulting manifest needs empty layers.
+func NewConfigManifestBuilder(bs distribution.BlobService, pk libtrust.PrivateKey, name, tag string, configJSON []byte) distribution.ManifestBuilder {
+	return &configManifestBuilder{
+		bs:         bs,
+		pk:         pk,
+		configJSON: configJSON,
+		name:       name,
+		tag:        tag,
+	}
+}
+
+// Build produces a final manifest from the given references
+func (mb *configManifestBuilder) Build(ctx context.Context) (m distribution.Manifest, err error) {
+	type imageRootFS struct {
+		Type      string   `json:"type"`
+		DiffIDs   []diffID `json:"diff_ids,omitempty"`
+		BaseLayer string   `json:"base_layer,omitempty"`
+	}
+
+	type imageHistory struct {
+		Created    time.Time `json:"created"`
+		Author     string    `json:"author,omitempty"`
+		CreatedBy  string    `json:"created_by,omitempty"`
+		Comment    string    `json:"comment,omitempty"`
+		EmptyLayer bool      `json:"empty_layer,omitempty"`
+	}
+
+	type imageConfig struct {
+		RootFS       *imageRootFS   `json:"rootfs,omitempty"`
+		History      []imageHistory `json:"history,omitempty"`
+		Architecture string         `json:"architecture,omitempty"`
+	}
+
+	var img imageConfig
+
+	if err := json.Unmarshal(mb.configJSON, &img); err != nil {
+		return nil, err
+	}
+
+	if len(img.History) == 0 {
+		return nil, errors.New("empty history when trying to create schema1 manifest")
+	}
+
+	if len(img.RootFS.DiffIDs) != len(mb.descriptors) {
+		return nil, errors.New("number of descriptors and number of layers in rootfs must match")
+	}
+
+	// Generate IDs for each layer
+	// For non-top-level layers, create fake V1Compatibility strings that
+	// fit the format and don't collide with anything else, but don't
+	// result in runnable images on their own.
+	type v1Compatibility struct {
+		ID              string    `json:"id"`
+		Parent          string    `json:"parent,omitempty"`
+		Comment         string    `json:"comment,omitempty"`
+		Created         time.Time `json:"created"`
+		ContainerConfig struct {
+			Cmd []string
+		} `json:"container_config,omitempty"`
+		ThrowAway bool `json:"throwaway,omitempty"`
+	}
+
+	fsLayerList := make([]FSLayer, len(img.History))
+	history := make([]History, len(img.History))
+
+	parent := ""
+	layerCounter := 0
+	for i, h := range img.History[:len(img.History)-1] {
+		var blobsum digest.Digest
+		if h.EmptyLayer {
+			if blobsum, err = mb.emptyTar(ctx); err != nil {
+				return nil, err
+			}
+		} else {
+			if len(img.RootFS.DiffIDs) <= layerCounter {
+				return nil, errors.New("too many non-empty layers in History section")
+			}
+			blobsum = mb.descriptors[layerCounter].Digest
+			layerCounter++
+		}
+
+		v1ID := digest.FromBytes([]byte(blobsum.Hex() + " " + parent)).Hex()
+
+		if i == 0 && img.RootFS.BaseLayer != "" {
+			// windows-only baselayer setup
+			baseID := sha512.Sum384([]byte(img.RootFS.BaseLayer))
+			parent = fmt.Sprintf("%x", baseID[:32])
+		}
+
+		v1Compatibility := v1Compatibility{
+			ID:      v1ID,
+			Parent:  parent,
+			Comment: h.Comment,
+			Created: h.Created,
+		}
+		v1Compatibility.ContainerConfig.Cmd = []string{img.History[i].CreatedBy}
+		if h.EmptyLayer {
+			v1Compatibility.ThrowAway = true
+		}
+		jsonBytes, err := json.Marshal(&v1Compatibility)
+		if err != nil {
+			return nil, err
+		}
+
+		reversedIndex := len(img.History) - i - 1
+		history[reversedIndex].V1Compatibility = string(jsonBytes)
+		fsLayerList[reversedIndex] = FSLayer{BlobSum: blobsum}
+
+		parent = v1ID
+	}
+
+	latestHistory := img.History[len(img.History)-1]
+
+	var blobsum digest.Digest
+	if latestHistory.EmptyLayer {
+		if blobsum, err = mb.emptyTar(ctx); err != nil {
+			return nil, err
+		}
+	} else {
+		if len(img.RootFS.DiffIDs) <= layerCounter {
+			return nil, errors.New("too many non-empty layers in History section")
+		}
+		blobsum = mb.descriptors[layerCounter].Digest
+	}
+
+	fsLayerList[0] = FSLayer{BlobSum: blobsum}
+	dgst := digest.FromBytes([]byte(blobsum.Hex() + " " + parent + " " + string(mb.configJSON)))
+
+	// Top-level v1compatibility string should be a modified version of the
+	// image config.
+	transformedConfig, err := MakeV1ConfigFromConfig(mb.configJSON, dgst.Hex(), parent, latestHistory.EmptyLayer)
+	if err != nil {
+		return nil, err
+	}
+
+	history[0].V1Compatibility = string(transformedConfig)
+
+	mfst := Manifest{
+		Versioned: manifest.Versioned{
+			SchemaVersion: 1,
+		},
+		Name:         mb.name,
+		Tag:          mb.tag,
+		Architecture: img.Architecture,
+		FSLayers:     fsLayerList,
+		History:      history,
+	}
+
+	return Sign(&mfst, mb.pk)
+}
+
+// emptyTar pushes a compressed empty tar to the blob store if one doesn't
+// already exist, and returns its blobsum.
+func (mb *configManifestBuilder) emptyTar(ctx context.Context) (digest.Digest, error) {
+	if mb.emptyTarDigest != "" {
+		// Already put an empty tar
+		return mb.emptyTarDigest, nil
+	}
+
+	descriptor, err := mb.bs.Stat(ctx, digestSHA256GzippedEmptyTar)
+	switch err {
+	case nil:
+		mb.emptyTarDigest = descriptor.Digest
+		return descriptor.Digest, nil
+	case distribution.ErrBlobUnknown:
+		// nop
+	default:
+		return "", err
+	}
+
+	// Add gzipped empty tar to the blob store
+	descriptor, err = mb.bs.Put(ctx, "", gzippedEmptyTar)
+	if err != nil {
+		return "", err
+	}
+
+	mb.emptyTarDigest = descriptor.Digest
+
+	return descriptor.Digest, nil
+}
+
+// AppendReference adds a reference to the current ManifestBuilder
+func (mb *configManifestBuilder) AppendReference(d distribution.Describable) error {
+	// todo: verification here?
+	mb.descriptors = append(mb.descriptors, d.Descriptor())
+	return nil
+}
+
+// References returns the current references added to this builder
+func (mb *configManifestBuilder) References() []distribution.Descriptor {
+	return mb.descriptors
+}
+
+// MakeV1ConfigFromConfig creates an legacy V1 image config from image config JSON
+func MakeV1ConfigFromConfig(configJSON []byte, v1ID, parentV1ID string, throwaway bool) ([]byte, error) {
+	// Top-level v1compatibility string should be a modified version of the
+	// image config.
+	var configAsMap map[string]*json.RawMessage
+	if err := json.Unmarshal(configJSON, &configAsMap); err != nil {
+		return nil, err
+	}
+
+	// Delete fields that didn't exist in old manifest
+	delete(configAsMap, "rootfs")
+	delete(configAsMap, "history")
+	configAsMap["id"] = rawJSON(v1ID)
+	if parentV1ID != "" {
+		configAsMap["parent"] = rawJSON(parentV1ID)
+	}
+	if throwaway {
+		configAsMap["throwaway"] = rawJSON(true)
+	}
+
+	return json.Marshal(configAsMap)
+}
+
+func rawJSON(value interface{}) *json.RawMessage {
+	jsonval, err := json.Marshal(value)
+	if err != nil {
+		return nil
+	}
+	return (*json.RawMessage)(&jsonval)
+}

manifest/schema1/config_builder_test.go

@@ -0,0 +1,264 @@
+package schema1
+
+import (
+	"bytes"
+	"compress/gzip"
+	"io"
+	"reflect"
+	"testing"
+
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/digest"
+	"github.com/docker/libtrust"
+)
+
+type mockBlobService struct {
+	descriptors map[digest.Digest]distribution.Descriptor
+}
+
+func (bs *mockBlobService) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) {
+	if descriptor, ok := bs.descriptors[dgst]; ok {
+		return descriptor, nil
+	}
+	return distribution.Descriptor{}, distribution.ErrBlobUnknown
+}
+
+func (bs *mockBlobService) Get(ctx context.Context, dgst digest.Digest) ([]byte, error) {
+	panic("not implemented")
+}
+
+func (bs *mockBlobService) Open(ctx context.Context, dgst digest.Digest) (distribution.ReadSeekCloser, error) {
+	panic("not implemented")
+}
+
+func (bs *mockBlobService) Put(ctx context.Context, mediaType string, p []byte) (distribution.Descriptor, error) {
+	d := distribution.Descriptor{
+		Digest:    digest.FromBytes(p),
+		Size:      int64(len(p)),
+		MediaType: mediaType,
+	}
+	bs.descriptors[d.Digest] = d
+	return d, nil
+}
+
+func (bs *mockBlobService) Create(ctx context.Context) (distribution.BlobWriter, error) {
+	panic("not implemented")
+}
+
+func (bs *mockBlobService) Resume(ctx context.Context, id string) (distribution.BlobWriter, error) {
+	panic("not implemented")
+}
+
+func TestEmptyTar(t *testing.T) {
+	// Confirm that gzippedEmptyTar expands to 1024 NULL bytes.
+	var decompressed [2048]byte
+	gzipReader, err := gzip.NewReader(bytes.NewReader(gzippedEmptyTar))
+	if err != nil {
+		t.Fatalf("NewReader returned error: %v", err)
+	}
+	n, err := gzipReader.Read(decompressed[:])
+	if n != 1024 {
+		t.Fatalf("read returned %d bytes; expected 1024", n)
+	}
+	n, err = gzipReader.Read(decompressed[1024:])
+	if n != 0 {
+		t.Fatalf("read returned %d bytes; expected 0", n)
+	}
+	if err != io.EOF {
+		t.Fatal("read did not return io.EOF")
+	}
+	gzipReader.Close()
+	for _, b := range decompressed[:1024] {
+		if b != 0 {
+			t.Fatal("nonzero byte in decompressed tar")
+		}
+	}
+
+	// Confirm that digestSHA256EmptyTar is the digest of gzippedEmptyTar.
+	dgst := digest.FromBytes(gzippedEmptyTar)
+	if dgst != digestSHA256GzippedEmptyTar {
+		t.Fatalf("digest mismatch for empty tar: expected %s got %s", digestSHA256GzippedEmptyTar, dgst)
+	}
+}
+
+func TestConfigBuilder(t *testing.T) {
+	imgJSON := `{
+    "architecture": "amd64",
+    "config": {
+        "AttachStderr": false,
+        "AttachStdin": false,
+        "AttachStdout": false,
+        "Cmd": [
+            "/bin/sh",
+            "-c",
+            "echo hi"
+        ],
+        "Domainname": "",
+        "Entrypoint": null,
+        "Env": [
+            "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+            "derived=true",
+            "asdf=true"
+        ],
+        "Hostname": "23304fc829f9",
+        "Image": "sha256:4ab15c48b859c2920dd5224f92aabcd39a52794c5b3cf088fb3bbb438756c246",
+        "Labels": {},
+        "OnBuild": [],
+        "OpenStdin": false,
+        "StdinOnce": false,
+        "Tty": false,
+        "User": "",
+        "Volumes": null,
+        "WorkingDir": ""
+    },
+    "container": "e91032eb0403a61bfe085ff5a5a48e3659e5a6deae9f4d678daa2ae399d5a001",
+    "container_config": {
+        "AttachStderr": false,
+        "AttachStdin": false,
+        "AttachStdout": false,
+        "Cmd": [
+            "/bin/sh",
+            "-c",
+            "#(nop) CMD [\"/bin/sh\" \"-c\" \"echo hi\"]"
+        ],
+        "Domainname": "",
+        "Entrypoint": null,
+        "Env": [
+            "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+            "derived=true",
+            "asdf=true"
+        ],
+        "Hostname": "23304fc829f9",
+        "Image": "sha256:4ab15c48b859c2920dd5224f92aabcd39a52794c5b3cf088fb3bbb438756c246",
+        "Labels": {},
+        "OnBuild": [],
+        "OpenStdin": false,
+        "StdinOnce": false,
+        "Tty": false,
+        "User": "",
+        "Volumes": null,
+        "WorkingDir": ""
+    },
+    "created": "2015-11-04T23:06:32.365666163Z",
+    "docker_version": "1.9.0-dev",
+    "history": [
+        {
+            "created": "2015-10-31T22:22:54.690851953Z",
+            "created_by": "/bin/sh -c #(nop) ADD file:a3bc1e842b69636f9df5256c49c5374fb4eef1e281fe3f282c65fb853ee171c5 in /"
+        },
+        {
+            "created": "2015-10-31T22:22:55.613815829Z",
+            "created_by": "/bin/sh -c #(nop) CMD [\"sh\"]"
+        },
+        {
+            "created": "2015-11-04T23:06:30.934316144Z",
+            "created_by": "/bin/sh -c #(nop) ENV derived=true",
+            "empty_layer": true
+        },
+        {
+            "created": "2015-11-04T23:06:31.192097572Z",
+            "created_by": "/bin/sh -c #(nop) ENV asdf=true",
+            "empty_layer": true
+        },
+        {
+            "created": "2015-11-04T23:06:32.083868454Z",
+            "created_by": "/bin/sh -c dd if=/dev/zero of=/file bs=1024 count=1024"
+        },
+        {
+            "created": "2015-11-04T23:06:32.365666163Z",
+            "created_by": "/bin/sh -c #(nop) CMD [\"/bin/sh\" \"-c\" \"echo hi\"]",
+            "empty_layer": true
+        }
+    ],
+    "os": "linux",
+    "rootfs": {
+        "diff_ids": [
+            "sha256:c6f988f4874bb0add23a778f753c65efe992244e148a1d2ec2a8b664fb66bbd1",
+            "sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef",
+            "sha256:13f53e08df5a220ab6d13c58b2bf83a59cbdc2e04d0a3f041ddf4b0ba4112d49"
+        ],
+        "type": "layers"
+    }
+}`
+
+	descriptors := []distribution.Descriptor{
+		{Digest: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")},
+		{Digest: digest.Digest("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa")},
+		{Digest: digest.Digest("sha256:b4ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")},
+	}
+
+	pk, err := libtrust.GenerateECP256PrivateKey()
+	if err != nil {
+		t.Fatalf("could not generate key for testing: %v", err)
+	}
+
+	bs := &mockBlobService{descriptors: make(map[digest.Digest]distribution.Descriptor)}
+	builder := NewConfigManifestBuilder(bs, pk, "testrepo", "testtag", []byte(imgJSON))
+
+	for _, d := range descriptors {
+		if err := builder.AppendReference(d); err != nil {
+			t.Fatalf("AppendReference returned error: %v", err)
+		}
+	}
+
+	signed, err := builder.Build(context.Background())
+	if err != nil {
+		t.Fatalf("Build returned error: %v", err)
+	}
+
+	// Check that the gzipped empty layer tar was put in the blob store
+	_, err = bs.Stat(context.Background(), digestSHA256GzippedEmptyTar)
+	if err != nil {
+		t.Fatal("gzipped empty tar was not put in the blob store")
+	}
+
+	manifest := signed.(*SignedManifest).Manifest
+
+	if manifest.Versioned.SchemaVersion != 1 {
+		t.Fatal("SchemaVersion != 1")
+	}
+	if manifest.Name != "testrepo" {
+		t.Fatal("incorrect name in manifest")
+	}
+	if manifest.Tag != "testtag" {
+		t.Fatal("incorrect tag in manifest")
+	}
+	if manifest.Architecture != "amd64" {
+		t.Fatal("incorrect arch in manifest")
+	}
+
+	expectedFSLayers := []FSLayer{
+		{BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")},
+		{BlobSum: digest.Digest("sha256:b4ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")},
+		{BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")},
+		{BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")},
+		{BlobSum: digest.Digest("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa")},
+		{BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")},
+	}
+
+	if len(manifest.FSLayers) != len(expectedFSLayers) {
+		t.Fatalf("wrong number of FSLayers: %d", len(manifest.FSLayers))
+	}
+	if !reflect.DeepEqual(manifest.FSLayers, expectedFSLayers) {
+		t.Fatal("wrong FSLayers list")
+	}
+
+	expectedV1Compatibility := []string{
+		`{"architecture":"amd64","config":{"AttachStderr":false,"AttachStdin":false,"AttachStdout":false,"Cmd":["/bin/sh","-c","echo hi"],"Domainname":"","Entrypoint":null,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","derived=true","asdf=true"],"Hostname":"23304fc829f9","Image":"sha256:4ab15c48b859c2920dd5224f92aabcd39a52794c5b3cf088fb3bbb438756c246","Labels":{},"OnBuild":[],"OpenStdin":false,"StdinOnce":false,"Tty":false,"User":"","Volumes":null,"WorkingDir":""},"container":"e91032eb0403a61bfe085ff5a5a48e3659e5a6deae9f4d678daa2ae399d5a001","container_config":{"AttachStderr":false,"AttachStdin":false,"AttachStdout":false,"Cmd":["/bin/sh","-c","#(nop) CMD [\"/bin/sh\" \"-c\" \"echo hi\"]"],"Domainname":"","Entrypoint":null,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","derived=true","asdf=true"],"Hostname":"23304fc829f9","Image":"sha256:4ab15c48b859c2920dd5224f92aabcd39a52794c5b3cf088fb3bbb438756c246","Labels":{},"OnBuild":[],"OpenStdin":false,"StdinOnce":false,"Tty":false,"User":"","Volumes":null,"WorkingDir":""},"created":"2015-11-04T23:06:32.365666163Z","docker_version":"1.9.0-dev","id":"0850bfdeb7b060b1004a09099846c2f023a3f2ecbf33f56b4774384b00ce0323","os":"linux","parent":"74cf9c92699240efdba1903c2748ef57105d5bedc588084c4e88f3bb1c3ef0b0","throwaway":true}`,
+		`{"id":"74cf9c92699240efdba1903c2748ef57105d5bedc588084c4e88f3bb1c3ef0b0","parent":"178be37afc7c49e951abd75525dbe0871b62ad49402f037164ee6314f754599d","created":"2015-11-04T23:06:32.083868454Z","container_config":{"Cmd":["/bin/sh -c dd if=/dev/zero of=/file bs=1024 count=1024"]}}`,
+		`{"id":"178be37afc7c49e951abd75525dbe0871b62ad49402f037164ee6314f754599d","parent":"b449305a55a283538c4574856a8b701f2a3d5ec08ef8aec47f385f20339a4866","created":"2015-11-04T23:06:31.192097572Z","container_config":{"Cmd":["/bin/sh -c #(nop) ENV asdf=true"]},"throwaway":true}`,
+		`{"id":"b449305a55a283538c4574856a8b701f2a3d5ec08ef8aec47f385f20339a4866","parent":"9e3447ca24cb96d86ebd5960cb34d1299b07e0a0e03801d90b9969a2c187dd6e","created":"2015-11-04T23:06:30.934316144Z","container_config":{"Cmd":["/bin/sh -c #(nop) ENV derived=true"]},"throwaway":true}`,
+		`{"id":"9e3447ca24cb96d86ebd5960cb34d1299b07e0a0e03801d90b9969a2c187dd6e","parent":"3690474eb5b4b26fdfbd89c6e159e8cc376ca76ef48032a30fa6aafd56337880","created":"2015-10-31T22:22:55.613815829Z","container_config":{"Cmd":["/bin/sh -c #(nop) CMD [\"sh\"]"]}}`,
+		`{"id":"3690474eb5b4b26fdfbd89c6e159e8cc376ca76ef48032a30fa6aafd56337880","created":"2015-10-31T22:22:54.690851953Z","container_config":{"Cmd":["/bin/sh -c #(nop) ADD file:a3bc1e842b69636f9df5256c49c5374fb4eef1e281fe3f282c65fb853ee171c5 in /"]}}`,
+	}
+
+	if len(manifest.History) != len(expectedV1Compatibility) {
+		t.Fatalf("wrong number of history entries: %d", len(manifest.History))
+	}
+	for i := range expectedV1Compatibility {
+		if manifest.History[i].V1Compatibility != expectedV1Compatibility[i] {
+			t.Errorf("wrong V1Compatibility %d. expected:\n%s\ngot:\n%s", i, expectedV1Compatibility[i], manifest.History[i].V1Compatibility)
+		}
+	}
+}

manifest/schema1/reference_builder.go

@@ -5,21 +5,23 @@ import (
 
 	"errors"
 	"github.com/docker/distribution"
+	"github.com/docker/distribution/context"
 	"github.com/docker/distribution/digest"
 	"github.com/docker/distribution/manifest"
 	"github.com/docker/libtrust"
 )
 
-// ManifestBuilder is a type for constructing manifests
+// referenceManifestBuilder is a type for constructing manifests from schema1
-type manifestBuilder struct {
+// dependencies.
+type referenceManifestBuilder struct {
 	Manifest
 	pk libtrust.PrivateKey
 }
 
-// NewManifestBuilder is used to build new manifests for the current schema
+// NewReferenceManifestBuilder is used to build new manifests for the current
-// version.
+// schema version using schema1 dependencies.
-func NewManifestBuilder(pk libtrust.PrivateKey, name, tag, architecture string) distribution.ManifestBuilder {
+func NewReferenceManifestBuilder(pk libtrust.PrivateKey, name, tag, architecture string) distribution.ManifestBuilder {
-	return &manifestBuilder{
+	return &referenceManifestBuilder{
 		Manifest: Manifest{
 			Versioned: manifest.Versioned{
 				SchemaVersion: 1,
@@ -32,8 +34,7 @@ func NewManifestBuilder(pk libtrust.PrivateKey, name, tag, architecture string)
 	}
 }
 
-// Build produces a final manifest from the given references
+func (mb *referenceManifestBuilder) Build(ctx context.Context) (distribution.Manifest, error) {
-func (mb *manifestBuilder) Build() (distribution.Manifest, error) {
 	m := mb.Manifest
 	if len(m.FSLayers) == 0 {
 		return nil, errors.New("cannot build manifest with zero layers or history")
@@ -47,8 +48,8 @@ func (mb *manifestBuilder) Build() (distribution.Manifest, error) {
 	return Sign(&m, mb.pk)
 }
 
-// AppendReference adds a reference to the current manifestBuilder
+// AppendReference adds a reference to the current ManifestBuilder
-func (mb *manifestBuilder) AppendReference(d distribution.Describable) error {
+func (mb *referenceManifestBuilder) AppendReference(d distribution.Describable) error {
 	r, ok := d.(Reference)
 	if !ok {
 		return fmt.Errorf("Unable to add non-reference type to v1 builder")
@@ -62,7 +63,7 @@ func (mb *manifestBuilder) AppendReference(d distribution.Describable) error {
 }
 
 // References returns the current references added to this builder
-func (mb *manifestBuilder) References() []distribution.Descriptor {
+func (mb *referenceManifestBuilder) References() []distribution.Descriptor {
 	refs := make([]distribution.Descriptor, len(mb.Manifest.FSLayers))
 	for i := range mb.Manifest.FSLayers {
 		layerDigest := mb.Manifest.FSLayers[i].BlobSum

manifest/schema1/reference_builder_test.go

@@ -3,6 +3,7 @@ package schema1
 import (
 	"testing"
 
+	"github.com/docker/distribution/context"
 	"github.com/docker/distribution/digest"
 	"github.com/docker/distribution/manifest"
 	"github.com/docker/libtrust"
@@ -34,7 +35,7 @@ func makeSignedManifest(t *testing.T, pk libtrust.PrivateKey, refs []Reference)
 	return signedManifest
 }
 
-func TestBuilder(t *testing.T) {
+func TestReferenceBuilder(t *testing.T) {
 	pk, err := libtrust.GenerateECP256PrivateKey()
 	if err != nil {
 		t.Fatalf("unexpected error generating private key: %v", err)
@@ -53,8 +54,8 @@ func TestBuilder(t *testing.T) {
 
 	handCrafted := makeSignedManifest(t, pk, []Reference{r1, r2})
 
-	b := NewManifestBuilder(pk, handCrafted.Manifest.Name, handCrafted.Manifest.Tag, handCrafted.Manifest.Architecture)
+	b := NewReferenceManifestBuilder(pk, handCrafted.Manifest.Name, handCrafted.Manifest.Tag, handCrafted.Manifest.Architecture)
-	_, err = b.Build()
+	_, err = b.Build(context.Background())
 	if err == nil {
 		t.Fatal("Expected error building zero length manifest")
 	}
@@ -79,7 +80,7 @@ func TestBuilder(t *testing.T) {
 		t.Fatalf("Unexpected reference : %v", refs[0])
 	}
 
-	m, err := b.Build()
+	m, err := b.Build(context.Background())
 	if err != nil {
 		t.Fatal(err)
 	}

manifest/schema2/builder.go

@@ -0,0 +1,78 @@
+package schema2
+
+import (
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/digest"
+	"github.com/docker/distribution/manifest"
+)
+
+// builder is a type for constructing manifests.
+type builder struct {
+	// bs is a BlobService used to publish the configuration blob.
+	bs distribution.BlobService
+
+	// configJSON references
+	configJSON []byte
+
+	// layers is a list of layer descriptors that gets built by successive
+	// calls to AppendReference.
+	layers []distribution.Descriptor
+}
+
+// NewManifestBuilder is used to build new manifests for the current schema
+// version. It takes a BlobService so it can publish the configuration blob
+// as part of the Build process.
+func NewManifestBuilder(bs distribution.BlobService, configJSON []byte) distribution.ManifestBuilder {
+	mb := &builder{
+		bs:         bs,
+		configJSON: make([]byte, len(configJSON)),
+	}
+	copy(mb.configJSON, configJSON)
+
+	return mb
+}
+
+// Build produces a final manifest from the given references.
+func (mb *builder) Build(ctx context.Context) (distribution.Manifest, error) {
+	m := Manifest{
+		Versioned: manifest.Versioned{
+			SchemaVersion: 2,
+		},
+		MediaType: MediaTypeManifest,
+		Layers:    make([]distribution.Descriptor, len(mb.layers)),
+	}
+	copy(m.Layers, mb.layers)
+
+	configDigest := digest.FromBytes(mb.configJSON)
+
+	var err error
+	m.Config, err = mb.bs.Stat(ctx, configDigest)
+	switch err {
+	case nil:
+		return FromStruct(m)
+	case distribution.ErrBlobUnknown:
+		// nop
+	default:
+		return nil, err
+	}
+
+	// Add config to the blob store
+	m.Config, err = mb.bs.Put(ctx, MediaTypeConfig, mb.configJSON)
+	if err != nil {
+		return nil, err
+	}
+
+	return FromStruct(m)
+}
+
+// AppendReference adds a reference to the current ManifestBuilder.
+func (mb *builder) AppendReference(d distribution.Describable) error {
+	mb.layers = append(mb.layers, d.Descriptor())
+	return nil
+}
+
+// References returns the current references added to this builder.
+func (mb *builder) References() []distribution.Descriptor {
+	return mb.layers
+}

manifest/schema2/builder_test.go

@@ -0,0 +1,210 @@
+package schema2
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/digest"
+)
+
+type mockBlobService struct {
+	descriptors map[digest.Digest]distribution.Descriptor
+}
+
+func (bs *mockBlobService) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) {
+	if descriptor, ok := bs.descriptors[dgst]; ok {
+		return descriptor, nil
+	}
+	return distribution.Descriptor{}, distribution.ErrBlobUnknown
+}
+
+func (bs *mockBlobService) Get(ctx context.Context, dgst digest.Digest) ([]byte, error) {
+	panic("not implemented")
+}
+
+func (bs *mockBlobService) Open(ctx context.Context, dgst digest.Digest) (distribution.ReadSeekCloser, error) {
+	panic("not implemented")
+}
+
+func (bs *mockBlobService) Put(ctx context.Context, mediaType string, p []byte) (distribution.Descriptor, error) {
+	d := distribution.Descriptor{
+		Digest:    digest.FromBytes(p),
+		Size:      int64(len(p)),
+		MediaType: mediaType,
+	}
+	bs.descriptors[d.Digest] = d
+	return d, nil
+}
+
+func (bs *mockBlobService) Create(ctx context.Context) (distribution.BlobWriter, error) {
+	panic("not implemented")
+}
+
+func (bs *mockBlobService) Resume(ctx context.Context, id string) (distribution.BlobWriter, error) {
+	panic("not implemented")
+}
+
+func TestBuilder(t *testing.T) {
+	imgJSON := []byte(`{
+    "architecture": "amd64",
+    "config": {
+        "AttachStderr": false,
+        "AttachStdin": false,
+        "AttachStdout": false,
+        "Cmd": [
+            "/bin/sh",
+            "-c",
+            "echo hi"
+        ],
+        "Domainname": "",
+        "Entrypoint": null,
+        "Env": [
+            "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+            "derived=true",
+            "asdf=true"
+        ],
+        "Hostname": "23304fc829f9",
+        "Image": "sha256:4ab15c48b859c2920dd5224f92aabcd39a52794c5b3cf088fb3bbb438756c246",
+        "Labels": {},
+        "OnBuild": [],
+        "OpenStdin": false,
+        "StdinOnce": false,
+        "Tty": false,
+        "User": "",
+        "Volumes": null,
+        "WorkingDir": ""
+    },
+    "container": "e91032eb0403a61bfe085ff5a5a48e3659e5a6deae9f4d678daa2ae399d5a001",
+    "container_config": {
+        "AttachStderr": false,
+        "AttachStdin": false,
+        "AttachStdout": false,
+        "Cmd": [
+            "/bin/sh",
+            "-c",
+            "#(nop) CMD [\"/bin/sh\" \"-c\" \"echo hi\"]"
+        ],
+        "Domainname": "",
+        "Entrypoint": null,
+        "Env": [
+            "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+            "derived=true",
+            "asdf=true"
+        ],
+        "Hostname": "23304fc829f9",
+        "Image": "sha256:4ab15c48b859c2920dd5224f92aabcd39a52794c5b3cf088fb3bbb438756c246",
+        "Labels": {},
+        "OnBuild": [],
+        "OpenStdin": false,
+        "StdinOnce": false,
+        "Tty": false,
+        "User": "",
+        "Volumes": null,
+        "WorkingDir": ""
+    },
+    "created": "2015-11-04T23:06:32.365666163Z",
+    "docker_version": "1.9.0-dev",
+    "history": [
+        {
+            "created": "2015-10-31T22:22:54.690851953Z",
+            "created_by": "/bin/sh -c #(nop) ADD file:a3bc1e842b69636f9df5256c49c5374fb4eef1e281fe3f282c65fb853ee171c5 in /"
+        },
+        {
+            "created": "2015-10-31T22:22:55.613815829Z",
+            "created_by": "/bin/sh -c #(nop) CMD [\"sh\"]"
+        },
+        {
+            "created": "2015-11-04T23:06:30.934316144Z",
+            "created_by": "/bin/sh -c #(nop) ENV derived=true",
+            "empty_layer": true
+        },
+        {
+            "created": "2015-11-04T23:06:31.192097572Z",
+            "created_by": "/bin/sh -c #(nop) ENV asdf=true",
+            "empty_layer": true
+        },
+        {
+            "created": "2015-11-04T23:06:32.083868454Z",
+            "created_by": "/bin/sh -c dd if=/dev/zero of=/file bs=1024 count=1024"
+        },
+        {
+            "created": "2015-11-04T23:06:32.365666163Z",
+            "created_by": "/bin/sh -c #(nop) CMD [\"/bin/sh\" \"-c\" \"echo hi\"]",
+            "empty_layer": true
+        }
+    ],
+    "os": "linux",
+    "rootfs": {
+        "diff_ids": [
+            "sha256:c6f988f4874bb0add23a778f753c65efe992244e148a1d2ec2a8b664fb66bbd1",
+            "sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef",
+            "sha256:13f53e08df5a220ab6d13c58b2bf83a59cbdc2e04d0a3f041ddf4b0ba4112d49"
+        ],
+        "type": "layers"
+    }
+}`)
+	configDigest := digest.FromBytes(imgJSON)
+
+	descriptors := []distribution.Descriptor{
+		{
+			Digest:    digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"),
+			Size:      5312,
+			MediaType: MediaTypeLayer,
+		},
+		{
+			Digest:    digest.Digest("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa"),
+			Size:      235231,
+			MediaType: MediaTypeLayer,
+		},
+		{
+			Digest:    digest.Digest("sha256:b4ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"),
+			Size:      639152,
+			MediaType: MediaTypeLayer,
+		},
+	}
+
+	bs := &mockBlobService{descriptors: make(map[digest.Digest]distribution.Descriptor)}
+	builder := NewManifestBuilder(bs, imgJSON)
+
+	for _, d := range descriptors {
+		if err := builder.AppendReference(d); err != nil {
+			t.Fatalf("AppendReference returned error: %v", err)
+		}
+	}
+
+	built, err := builder.Build(context.Background())
+	if err != nil {
+		t.Fatalf("Build returned error: %v", err)
+	}
+
+	// Check that the config was put in the blob store
+	_, err = bs.Stat(context.Background(), configDigest)
+	if err != nil {
+		t.Fatal("config was not put in the blob store")
+	}
+
+	manifest := built.(*DeserializedManifest).Manifest
+
+	if manifest.Versioned.SchemaVersion != 2 {
+		t.Fatal("SchemaVersion != 2")
+	}
+
+	target := manifest.Target()
+	if target.Digest != configDigest {
+		t.Fatalf("unexpected digest in target: %s", target.Digest.String())
+	}
+	if target.MediaType != MediaTypeConfig {
+		t.Fatalf("unexpected media type in target: %s", target.MediaType)
+	}
+	if target.Size != 3153 {
+		t.Fatalf("unexpected size in target: %d", target.Size)
+	}
+
+	references := manifest.References()
+
+	if !reflect.DeepEqual(references, descriptors) {
+		t.Fatal("References() does not match the descriptors added")
+	}
+}

manifest/schema2/manifest.go

@@ -0,0 +1,125 @@
+package schema2
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/digest"
+	"github.com/docker/distribution/manifest"
+)
+
+const (
+	// MediaTypeManifest specifies the mediaType for the current version.
+	MediaTypeManifest = "application/vnd.docker.distribution.manifest.v2+json"
+
+	// MediaTypeConfig specifies the mediaType for the image configuration.
+	MediaTypeConfig = "application/vnd.docker.container.image.v1+json"
+
+	// MediaTypeLayer is the mediaType used for layers referenced by the
+	// manifest.
+	MediaTypeLayer = "application/vnd.docker.image.rootfs.diff.tar.gzip"
+)
+
+var (
+	// SchemaVersion provides a pre-initialized version structure for this
+	// packages version of the manifest.
+	SchemaVersion = manifest.Versioned{
+		SchemaVersion: 2,
+	}
+)
+
+func init() {
+	schema2Func := func(b []byte) (distribution.Manifest, distribution.Descriptor, error) {
+		m := new(DeserializedManifest)
+		err := m.UnmarshalJSON(b)
+		if err != nil {
+			return nil, distribution.Descriptor{}, err
+		}
+
+		dgst := digest.FromBytes(b)
+		return m, distribution.Descriptor{Digest: dgst, Size: int64(len(b)), MediaType: MediaTypeManifest}, err
+	}
+	err := distribution.RegisterManifestSchema(MediaTypeManifest, schema2Func)
+	if err != nil {
+		panic(fmt.Sprintf("Unable to register manifest: %s", err))
+	}
+}
+
+// Manifest defines a schema2 manifest.
+type Manifest struct {
+	manifest.Versioned
+	MediaType string `json:"mediaType"`
+
+	// Config references the image configuration as a blob.
+	Config distribution.Descriptor `json:"config"`
+
+	// Layers lists descriptors for the layers referenced by the
+	// configuration.
+	Layers []distribution.Descriptor `json:"layers"`
+}
+
+// References returnes the descriptors of this manifests references.
+func (m Manifest) References() []distribution.Descriptor {
+	return m.Layers
+
+}
+
+// Target returns the target of this signed manifest.
+func (m Manifest) Target() distribution.Descriptor {
+	return m.Config
+}
+
+// DeserializedManifest wraps Manifest with a copy of the original JSON.
+// It satisfies the distribution.Manifest interface.
+type DeserializedManifest struct {
+	Manifest
+
+	// canonical is the canonical byte representation of the Manifest.
+	canonical []byte
+}
+
+// FromStruct takes a Manifest structure, marshals it to JSON, and returns a
+// DeserializedManifest which contains the manifest and its JSON representation.
+func FromStruct(m Manifest) (*DeserializedManifest, error) {
+	var deserialized DeserializedManifest
+	deserialized.Manifest = m
+
+	var err error
+	deserialized.canonical, err = json.MarshalIndent(&m, "", "   ")
+	return &deserialized, err
+}
+
+// UnmarshalJSON populates a new Manifest struct from JSON data.
+func (m *DeserializedManifest) UnmarshalJSON(b []byte) error {
+	m.canonical = make([]byte, len(b), len(b))
+	// store manifest in canonical
+	copy(m.canonical, b)
+
+	// Unmarshal canonical JSON into Manifest object
+	var manifest Manifest
+	if err := json.Unmarshal(m.canonical, &manifest); err != nil {
+		return err
+	}
+
+	m.Manifest = manifest
+
+	return nil
+}
+
+// MarshalJSON returns the contents of canonical. If canonical is empty,
+// marshals the inner contents.
+func (m *DeserializedManifest) MarshalJSON() ([]byte, error) {
+	if len(m.canonical) > 0 {
+		return m.canonical, nil
+	}
+
+	return nil, errors.New("JSON representation not initialized in DeserializedManifest")
+}
+
+// Payload returns the raw content of the manifest. The contents can be used to
+// calculate the content identifier.
+func (m DeserializedManifest) Payload() (string, []byte, error) {
+	return m.MediaType, m.canonical, nil
+}

manifest/schema2/manifest_test.go

@@ -0,0 +1,106 @@
+package schema2
+
+import (
+	"bytes"
+	"encoding/json"
+	"reflect"
+	"testing"
+
+	"github.com/docker/distribution"
+)
+
+var expectedManifestSerialization = []byte(`{
+   "schemaVersion": 2,
+   "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
+   "config": {
+      "mediaType": "application/vnd.docker.container.image.v1+json",
+      "size": 985,
+      "digest": "sha256:1a9ec845ee94c202b2d5da74a24f0ed2058318bfa9879fa541efaecba272e86b"
+   },
+   "layers": [
+      {
+         "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+         "size": 153263,
+         "digest": "sha256:62d8908bee94c202b2d35224a221aaa2058318bfa9879fa541efaecba272331b"
+      }
+   ]
+}`)
+
+func TestManifest(t *testing.T) {
+	manifest := Manifest{
+		Versioned: SchemaVersion,
+		MediaType: MediaTypeManifest,
+		Config: distribution.Descriptor{
+			Digest:    "sha256:1a9ec845ee94c202b2d5da74a24f0ed2058318bfa9879fa541efaecba272e86b",
+			Size:      985,
+			MediaType: MediaTypeConfig,
+		},
+		Layers: []distribution.Descriptor{
+			{
+				Digest:    "sha256:62d8908bee94c202b2d35224a221aaa2058318bfa9879fa541efaecba272331b",
+				Size:      153263,
+				MediaType: MediaTypeLayer,
+			},
+		},
+	}
+
+	deserialized, err := FromStruct(manifest)
+	if err != nil {
+		t.Fatalf("error creating DeserializedManifest: %v", err)
+	}
+
+	mediaType, canonical, err := deserialized.Payload()
+
+	if mediaType != MediaTypeManifest {
+		t.Fatalf("unexpected media type: %s", mediaType)
+	}
+
+	// Check that the canonical field is the same as json.MarshalIndent
+	// with these parameters.
+	p, err := json.MarshalIndent(&manifest, "", "   ")
+	if err != nil {
+		t.Fatalf("error marshaling manifest: %v", err)
+	}
+	if !bytes.Equal(p, canonical) {
+		t.Fatalf("manifest bytes not equal: %q != %q", string(canonical), string(p))
+	}
+
+	// Check that canonical field matches expected value.
+	if !bytes.Equal(expectedManifestSerialization, canonical) {
+		t.Fatalf("manifest bytes not equal: %q != %q", string(canonical), string(expectedManifestSerialization))
+	}
+
+	var unmarshalled DeserializedManifest
+	if err := json.Unmarshal(deserialized.canonical, &unmarshalled); err != nil {
+		t.Fatalf("error unmarshaling manifest: %v", err)
+	}
+
+	if !reflect.DeepEqual(&unmarshalled, deserialized) {
+		t.Fatalf("manifests are different after unmarshaling: %v != %v", unmarshalled, *deserialized)
+	}
+
+	target := deserialized.Target()
+	if target.Digest != "sha256:1a9ec845ee94c202b2d5da74a24f0ed2058318bfa9879fa541efaecba272e86b" {
+		t.Fatalf("unexpected digest in target: %s", target.Digest.String())
+	}
+	if target.MediaType != MediaTypeConfig {
+		t.Fatalf("unexpected media type in target: %s", target.MediaType)
+	}
+	if target.Size != 985 {
+		t.Fatalf("unexpected size in target: %d", target.Size)
+	}
+
+	references := deserialized.References()
+	if len(references) != 1 {
+		t.Fatalf("unexpected number of references: %d", len(references))
+	}
+	if references[0].Digest != "sha256:62d8908bee94c202b2d35224a221aaa2058318bfa9879fa541efaecba272331b" {
+		t.Fatalf("unexpected digest in reference: %s", references[0].Digest.String())
+	}
+	if references[0].MediaType != MediaTypeLayer {
+		t.Fatalf("unexpected media type in reference: %s", references[0].MediaType)
+	}
+	if references[0].Size != 153263 {
+		t.Fatalf("unexpected size in reference: %d", references[0].Size)
+	}
+}

manifests.go

@@ -25,7 +25,7 @@ type Manifest interface {
 // specific data is passed into the function which creates the builder.
 type ManifestBuilder interface {
 	// Build creates the manifest from his builder.
-	Build() (Manifest, error)
+	Build(ctx context.Context) (Manifest, error)
 
 	// References returns a list of objects which have been added to this
 	// builder. The dependencies are returned in the order they were added,