Merge pull request #1725 from jstarks/foreign_layer

Add support for layers from foreign sources
This commit is contained in:
Aaron Lehmann 2016-05-20 12:55:55 -07:00
commit 5bbf654999
8 changed files with 180 additions and 17 deletions

View file

@ -69,6 +69,9 @@ type Descriptor struct {
// against against this digest. // against against this digest.
Digest digest.Digest `json:"digest,omitempty"` Digest digest.Digest `json:"digest,omitempty"`
// URLs contains the source URLs of this content.
URLs []string `json:"urls,omitempty"`
// NOTE: Before adding a field here, please ensure that all // NOTE: Before adding a field here, please ensure that all
// other options have been exhausted. Much of the type relationships // other options have been exhausted. Much of the type relationships
// depend on the simplicity of this type. // depend on the simplicity of this type.

View file

@ -216,6 +216,14 @@ image. It's the direct replacement for the schema-1 manifest.
The digest of the content, as defined by the The digest of the content, as defined by the
[Registry V2 HTTP API Specificiation](https://docs.docker.com/registry/spec/api/#digest-parameter). [Registry V2 HTTP API Specificiation](https://docs.docker.com/registry/spec/api/#digest-parameter).
- **`urls`** *array*
For an ordinary layer, this is empty, and the layer contents can be
retrieved directly from the registry. For a layer with *`mediatype`* of
`application/vnd.docker.image.rootfs.foreign.diff.tar.gzip`, this
contains a non-empty list of URLs from which this object can be
downloaded.
## Example Image Manifest ## Example Image Manifest
*Example showing an image manifest:* *Example showing an image manifest:*

View file

@ -20,6 +20,10 @@ const (
// MediaTypeLayer is the mediaType used for layers referenced by the // MediaTypeLayer is the mediaType used for layers referenced by the
// manifest. // manifest.
MediaTypeLayer = "application/vnd.docker.image.rootfs.diff.tar.gzip" MediaTypeLayer = "application/vnd.docker.image.rootfs.diff.tar.gzip"
// MediaTypeForeignLayer is the mediaType used for layers that must be
// downloaded from foreign URLs.
MediaTypeForeignLayer = "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip"
) )
var ( var (
@ -63,7 +67,6 @@ type Manifest struct {
// References returnes the descriptors of this manifests references. // References returnes the descriptors of this manifests references.
func (m Manifest) References() []distribution.Descriptor { func (m Manifest) References() []distribution.Descriptor {
return m.Layers return m.Layers
} }
// Target returns the target of this signed manifest. // Target returns the target of this signed manifest.

View file

@ -1,6 +1,7 @@
package proxy package proxy
import ( import (
"reflect"
"sort" "sort"
"sync" "sync"
"testing" "testing"
@ -92,7 +93,7 @@ func TestGet(t *testing.T) {
t.Fatalf("Expected 1 auth challenge call, got %#v", proxyTags.authChallenger) t.Fatalf("Expected 1 auth challenge call, got %#v", proxyTags.authChallenger)
} }
if d != remoteDesc { if !reflect.DeepEqual(d, remoteDesc) {
t.Fatal("unable to get put tag") t.Fatal("unable to get put tag")
} }
@ -101,7 +102,7 @@ func TestGet(t *testing.T) {
t.Fatal("remote tag not pulled into store") t.Fatal("remote tag not pulled into store")
} }
if local != remoteDesc { if !reflect.DeepEqual(local, remoteDesc) {
t.Fatalf("unexpected descriptor pulled through") t.Fatalf("unexpected descriptor pulled through")
} }
@ -121,7 +122,7 @@ func TestGet(t *testing.T) {
t.Fatalf("Expected 2 auth challenge calls, got %#v", proxyTags.authChallenger) t.Fatalf("Expected 2 auth challenge calls, got %#v", proxyTags.authChallenger)
} }
if d != newRemoteDesc { if !reflect.DeepEqual(d, newRemoteDesc) {
t.Fatal("unable to get put tag") t.Fatal("unable to get put tag")
} }

View file

@ -7,6 +7,8 @@ import (
"io" "io"
"io/ioutil" "io/ioutil"
"os" "os"
"path"
"reflect"
"testing" "testing"
"github.com/docker/distribution" "github.com/docker/distribution"
@ -16,7 +18,6 @@ import (
"github.com/docker/distribution/registry/storage/cache/memory" "github.com/docker/distribution/registry/storage/cache/memory"
"github.com/docker/distribution/registry/storage/driver/inmemory" "github.com/docker/distribution/registry/storage/driver/inmemory"
"github.com/docker/distribution/testutil" "github.com/docker/distribution/testutil"
"path"
) )
// TestWriteSeek tests that the current file size can be // TestWriteSeek tests that the current file size can be
@ -156,7 +157,7 @@ func TestSimpleBlobUpload(t *testing.T) {
t.Fatalf("unexpected error checking for existence: %v, %#v", err, bs) t.Fatalf("unexpected error checking for existence: %v, %#v", err, bs)
} }
if statDesc != desc { if !reflect.DeepEqual(statDesc, desc) {
t.Fatalf("descriptors not equal: %v != %v", statDesc, desc) t.Fatalf("descriptors not equal: %v != %v", statDesc, desc)
} }
@ -410,7 +411,7 @@ func TestBlobMount(t *testing.T) {
t.Fatalf("unexpected error checking for existence: %v, %#v", err, sbs) t.Fatalf("unexpected error checking for existence: %v, %#v", err, sbs)
} }
if statDesc != desc { if !reflect.DeepEqual(statDesc, desc) {
t.Fatalf("descriptors not equal: %v != %v", statDesc, desc) t.Fatalf("descriptors not equal: %v != %v", statDesc, desc)
} }
@ -436,7 +437,7 @@ func TestBlobMount(t *testing.T) {
t.Fatalf("unexpected error mounting layer: %v", err) t.Fatalf("unexpected error mounting layer: %v", err)
} }
if ebm.Descriptor != desc { if !reflect.DeepEqual(ebm.Descriptor, desc) {
t.Fatalf("descriptors not equal: %v != %v", ebm.Descriptor, desc) t.Fatalf("descriptors not equal: %v != %v", ebm.Descriptor, desc)
} }
@ -446,7 +447,7 @@ func TestBlobMount(t *testing.T) {
t.Fatalf("unexpected error checking for existence: %v, %#v", err, bs) t.Fatalf("unexpected error checking for existence: %v, %#v", err, bs)
} }
if statDesc != desc { if !reflect.DeepEqual(statDesc, desc) {
t.Fatalf("descriptors not equal: %v != %v", statDesc, desc) t.Fatalf("descriptors not equal: %v != %v", statDesc, desc)
} }

View file

@ -1,6 +1,7 @@
package cachecheck package cachecheck
import ( import (
"reflect"
"testing" "testing"
"github.com/docker/distribution" "github.com/docker/distribution"
@ -79,7 +80,7 @@ func checkBlobDescriptorCacheSetAndRead(t *testing.T, ctx context.Context, provi
t.Fatalf("unexpected error statting fake2:abc: %v", err) t.Fatalf("unexpected error statting fake2:abc: %v", err)
} }
if expected != desc { if !reflect.DeepEqual(expected, desc) {
t.Fatalf("unexpected descriptor: %#v != %#v", expected, desc) t.Fatalf("unexpected descriptor: %#v != %#v", expected, desc)
} }
@ -89,7 +90,7 @@ func checkBlobDescriptorCacheSetAndRead(t *testing.T, ctx context.Context, provi
t.Fatalf("descriptor not returned for canonical key: %v", err) t.Fatalf("descriptor not returned for canonical key: %v", err)
} }
if expected != desc { if !reflect.DeepEqual(expected, desc) {
t.Fatalf("unexpected descriptor: %#v != %#v", expected, desc) t.Fatalf("unexpected descriptor: %#v != %#v", expected, desc)
} }
@ -99,7 +100,7 @@ func checkBlobDescriptorCacheSetAndRead(t *testing.T, ctx context.Context, provi
t.Fatalf("expected blob unknown in global cache: %v, %v", err, desc) t.Fatalf("expected blob unknown in global cache: %v, %v", err, desc)
} }
if desc != expected { if !reflect.DeepEqual(desc, expected) {
t.Fatalf("unexpected descriptor: %#v != %#v", expected, desc) t.Fatalf("unexpected descriptor: %#v != %#v", expected, desc)
} }
@ -109,7 +110,7 @@ func checkBlobDescriptorCacheSetAndRead(t *testing.T, ctx context.Context, provi
t.Fatalf("unexpected error checking glboal descriptor: %v", err) t.Fatalf("unexpected error checking glboal descriptor: %v", err)
} }
if desc != expected { if !reflect.DeepEqual(desc, expected) {
t.Fatalf("unexpected descriptor: %#v != %#v", expected, desc) t.Fatalf("unexpected descriptor: %#v != %#v", expected, desc)
} }
@ -126,7 +127,7 @@ func checkBlobDescriptorCacheSetAndRead(t *testing.T, ctx context.Context, provi
t.Fatalf("unexpected error getting descriptor: %v", err) t.Fatalf("unexpected error getting descriptor: %v", err)
} }
if desc != expected { if !reflect.DeepEqual(desc, expected) {
t.Fatalf("unexpected descriptor: %#v != %#v", desc, expected) t.Fatalf("unexpected descriptor: %#v != %#v", desc, expected)
} }
@ -137,7 +138,7 @@ func checkBlobDescriptorCacheSetAndRead(t *testing.T, ctx context.Context, provi
expected.MediaType = "application/octet-stream" // expect original mediatype in global expected.MediaType = "application/octet-stream" // expect original mediatype in global
if desc != expected { if !reflect.DeepEqual(desc, expected) {
t.Fatalf("unexpected descriptor: %#v != %#v", desc, expected) t.Fatalf("unexpected descriptor: %#v != %#v", desc, expected)
} }
} }
@ -163,7 +164,7 @@ func checkBlobDescriptorCacheClear(t *testing.T, ctx context.Context, provider c
t.Fatalf("unexpected error statting fake2:abc: %v", err) t.Fatalf("unexpected error statting fake2:abc: %v", err)
} }
if expected != desc { if !reflect.DeepEqual(expected, desc) {
t.Fatalf("unexpected descriptor: %#v != %#v", expected, desc) t.Fatalf("unexpected descriptor: %#v != %#v", expected, desc)
} }

View file

@ -1,15 +1,24 @@
package storage package storage
import ( import (
"errors"
"fmt" "fmt"
"net/url"
"encoding/json" "encoding/json"
"github.com/docker/distribution" "github.com/docker/distribution"
"github.com/docker/distribution/context" "github.com/docker/distribution/context"
"github.com/docker/distribution/digest" "github.com/docker/distribution/digest"
"github.com/docker/distribution/manifest/schema2" "github.com/docker/distribution/manifest/schema2"
) )
var (
errUnexpectedURL = errors.New("unexpected URL on layer")
errMissingURL = errors.New("missing URL on layer")
errInvalidURL = errors.New("invalid URL on layer")
)
//schema2ManifestHandler is a ManifestHandler that covers schema2 manifests. //schema2ManifestHandler is a ManifestHandler that covers schema2 manifests.
type schema2ManifestHandler struct { type schema2ManifestHandler struct {
repository *repository repository *repository
@ -80,7 +89,27 @@ func (ms *schema2ManifestHandler) verifyManifest(ctx context.Context, mnfst sche
} }
for _, fsLayer := range mnfst.References() { for _, fsLayer := range mnfst.References() {
_, err := ms.repository.Blobs(ctx).Stat(ctx, fsLayer.Digest) var err error
if fsLayer.MediaType != schema2.MediaTypeForeignLayer {
if len(fsLayer.URLs) == 0 {
_, err = ms.repository.Blobs(ctx).Stat(ctx, fsLayer.Digest)
} else {
err = errUnexpectedURL
}
} else {
// Clients download this layer from an external URL, so do not check for
// its presense.
if len(fsLayer.URLs) == 0 {
err = errMissingURL
}
for _, u := range fsLayer.URLs {
var pu *url.URL
pu, err = url.Parse(u)
if err != nil || (pu.Scheme != "http" && pu.Scheme != "https") || pu.Fragment != "" {
err = errInvalidURL
}
}
}
if err != nil { if err != nil {
if err != distribution.ErrBlobUnknown { if err != distribution.ErrBlobUnknown {
errs = append(errs, err) errs = append(errs, err)

View file

@ -0,0 +1,117 @@
package storage
import (
"testing"
"github.com/docker/distribution"
"github.com/docker/distribution/context"
"github.com/docker/distribution/manifest"
"github.com/docker/distribution/manifest/schema2"
"github.com/docker/distribution/registry/storage/driver/inmemory"
)
func TestVerifyManifestForeignLayer(t *testing.T) {
ctx := context.Background()
inmemoryDriver := inmemory.New()
registry := createRegistry(t, inmemoryDriver)
repo := makeRepository(t, registry, "test")
manifestService := makeManifestService(t, repo)
config, err := repo.Blobs(ctx).Put(ctx, schema2.MediaTypeConfig, nil)
if err != nil {
t.Fatal(err)
}
layer, err := repo.Blobs(ctx).Put(ctx, schema2.MediaTypeLayer, nil)
if err != nil {
t.Fatal(err)
}
foreignLayer := distribution.Descriptor{
Digest: "sha256:463435349086340864309863409683460843608348608934092322395278926a",
Size: 6323,
MediaType: schema2.MediaTypeForeignLayer,
}
template := schema2.Manifest{
Versioned: manifest.Versioned{
SchemaVersion: 2,
MediaType: schema2.MediaTypeManifest,
},
Config: config,
}
type testcase struct {
BaseLayer distribution.Descriptor
URLs []string
Err error
}
cases := []testcase{
{
foreignLayer,
nil,
errMissingURL,
},
{
layer,
[]string{"http://foo/bar"},
errUnexpectedURL,
},
{
foreignLayer,
[]string{"file:///local/file"},
errInvalidURL,
},
{
foreignLayer,
[]string{"http://foo/bar#baz"},
errInvalidURL,
},
{
foreignLayer,
[]string{""},
errInvalidURL,
},
{
foreignLayer,
[]string{"https://foo/bar", ""},
errInvalidURL,
},
{
foreignLayer,
[]string{"http://foo/bar"},
nil,
},
{
foreignLayer,
[]string{"https://foo/bar"},
nil,
},
}
for _, c := range cases {
m := template
l := c.BaseLayer
l.URLs = c.URLs
m.Layers = []distribution.Descriptor{l}
dm, err := schema2.FromStruct(m)
if err != nil {
t.Error(err)
continue
}
_, err = manifestService.Put(ctx, dm)
if verr, ok := err.(distribution.ErrManifestVerification); ok {
// Extract the first error
if len(verr) == 2 {
if _, ok = verr[1].(distribution.ErrManifestBlobUnknown); ok {
err = verr[0]
}
}
}
if err != c.Err {
t.Errorf("%#v: expected %v, got %v", l, c.Err, err)
}
}
}