From dd66aabebafd0cf20f26d92a71e1a991d9309a39 Mon Sep 17 00:00:00 2001 From: John Starks Date: Sat, 14 May 2016 14:49:08 -0700 Subject: [PATCH] Add support for layers from foreign sources This will be used to support downloading Windows base layers from Microsoft URLs. Signed-off-by: John Starks --- docs/proxy/proxytagservice_test.go | 7 +- docs/storage/blob_test.go | 11 +- docs/storage/cache/cachecheck/suite.go | 15 +-- docs/storage/schema2manifesthandler.go | 31 +++++- docs/storage/schema2manifesthandler_test.go | 117 ++++++++++++++++++++ 5 files changed, 165 insertions(+), 16 deletions(-) create mode 100644 docs/storage/schema2manifesthandler_test.go diff --git a/docs/proxy/proxytagservice_test.go b/docs/proxy/proxytagservice_test.go index a446645cb..ce0fe78ba 100644 --- a/docs/proxy/proxytagservice_test.go +++ b/docs/proxy/proxytagservice_test.go @@ -1,6 +1,7 @@ package proxy import ( + "reflect" "sort" "sync" "testing" @@ -92,7 +93,7 @@ func TestGet(t *testing.T) { t.Fatalf("Expected 1 auth challenge call, got %#v", proxyTags.authChallenger) } - if d != remoteDesc { + if !reflect.DeepEqual(d, remoteDesc) { t.Fatal("unable to get put tag") } @@ -101,7 +102,7 @@ func TestGet(t *testing.T) { t.Fatal("remote tag not pulled into store") } - if local != remoteDesc { + if !reflect.DeepEqual(local, remoteDesc) { t.Fatalf("unexpected descriptor pulled through") } @@ -121,7 +122,7 @@ func TestGet(t *testing.T) { t.Fatalf("Expected 2 auth challenge calls, got %#v", proxyTags.authChallenger) } - if d != newRemoteDesc { + if !reflect.DeepEqual(d, newRemoteDesc) { t.Fatal("unable to get put tag") } diff --git a/docs/storage/blob_test.go b/docs/storage/blob_test.go index 7e1a7cd44..f7ae70f12 100644 --- a/docs/storage/blob_test.go +++ b/docs/storage/blob_test.go @@ -7,6 +7,8 @@ import ( "io" "io/ioutil" "os" + "path" + "reflect" "testing" "github.com/docker/distribution" @@ -16,7 +18,6 @@ import ( "github.com/docker/distribution/registry/storage/cache/memory" "github.com/docker/distribution/registry/storage/driver/inmemory" "github.com/docker/distribution/testutil" - "path" ) // TestWriteSeek tests that the current file size can be @@ -156,7 +157,7 @@ func TestSimpleBlobUpload(t *testing.T) { t.Fatalf("unexpected error checking for existence: %v, %#v", err, bs) } - if statDesc != desc { + if !reflect.DeepEqual(statDesc, desc) { t.Fatalf("descriptors not equal: %v != %v", statDesc, desc) } @@ -410,7 +411,7 @@ func TestBlobMount(t *testing.T) { t.Fatalf("unexpected error checking for existence: %v, %#v", err, sbs) } - if statDesc != desc { + if !reflect.DeepEqual(statDesc, desc) { t.Fatalf("descriptors not equal: %v != %v", statDesc, desc) } @@ -436,7 +437,7 @@ func TestBlobMount(t *testing.T) { t.Fatalf("unexpected error mounting layer: %v", err) } - if ebm.Descriptor != desc { + if !reflect.DeepEqual(ebm.Descriptor, desc) { t.Fatalf("descriptors not equal: %v != %v", ebm.Descriptor, desc) } @@ -446,7 +447,7 @@ func TestBlobMount(t *testing.T) { t.Fatalf("unexpected error checking for existence: %v, %#v", err, bs) } - if statDesc != desc { + if !reflect.DeepEqual(statDesc, desc) { t.Fatalf("descriptors not equal: %v != %v", statDesc, desc) } diff --git a/docs/storage/cache/cachecheck/suite.go b/docs/storage/cache/cachecheck/suite.go index 13e9c1322..cba5addd3 100644 --- a/docs/storage/cache/cachecheck/suite.go +++ b/docs/storage/cache/cachecheck/suite.go @@ -1,6 +1,7 @@ package cachecheck import ( + "reflect" "testing" "github.com/docker/distribution" @@ -79,7 +80,7 @@ func checkBlobDescriptorCacheSetAndRead(t *testing.T, ctx context.Context, provi t.Fatalf("unexpected error statting fake2:abc: %v", err) } - if expected != desc { + if !reflect.DeepEqual(expected, desc) { t.Fatalf("unexpected descriptor: %#v != %#v", expected, desc) } @@ -89,7 +90,7 @@ func checkBlobDescriptorCacheSetAndRead(t *testing.T, ctx context.Context, provi t.Fatalf("descriptor not returned for canonical key: %v", err) } - if expected != desc { + if !reflect.DeepEqual(expected, desc) { t.Fatalf("unexpected descriptor: %#v != %#v", expected, desc) } @@ -99,7 +100,7 @@ func checkBlobDescriptorCacheSetAndRead(t *testing.T, ctx context.Context, provi t.Fatalf("expected blob unknown in global cache: %v, %v", err, desc) } - if desc != expected { + if !reflect.DeepEqual(desc, expected) { t.Fatalf("unexpected descriptor: %#v != %#v", expected, desc) } @@ -109,7 +110,7 @@ func checkBlobDescriptorCacheSetAndRead(t *testing.T, ctx context.Context, provi t.Fatalf("unexpected error checking glboal descriptor: %v", err) } - if desc != expected { + if !reflect.DeepEqual(desc, expected) { t.Fatalf("unexpected descriptor: %#v != %#v", expected, desc) } @@ -126,7 +127,7 @@ func checkBlobDescriptorCacheSetAndRead(t *testing.T, ctx context.Context, provi t.Fatalf("unexpected error getting descriptor: %v", err) } - if desc != expected { + if !reflect.DeepEqual(desc, expected) { t.Fatalf("unexpected descriptor: %#v != %#v", desc, expected) } @@ -137,7 +138,7 @@ func checkBlobDescriptorCacheSetAndRead(t *testing.T, ctx context.Context, provi expected.MediaType = "application/octet-stream" // expect original mediatype in global - if desc != expected { + if !reflect.DeepEqual(desc, expected) { t.Fatalf("unexpected descriptor: %#v != %#v", desc, expected) } } @@ -163,7 +164,7 @@ func checkBlobDescriptorCacheClear(t *testing.T, ctx context.Context, provider c t.Fatalf("unexpected error statting fake2:abc: %v", err) } - if expected != desc { + if !reflect.DeepEqual(expected, desc) { t.Fatalf("unexpected descriptor: %#v != %#v", expected, desc) } diff --git a/docs/storage/schema2manifesthandler.go b/docs/storage/schema2manifesthandler.go index 115786e26..6456efa4e 100644 --- a/docs/storage/schema2manifesthandler.go +++ b/docs/storage/schema2manifesthandler.go @@ -1,15 +1,24 @@ package storage import ( + "errors" "fmt" + "net/url" "encoding/json" + "github.com/docker/distribution" "github.com/docker/distribution/context" "github.com/docker/distribution/digest" "github.com/docker/distribution/manifest/schema2" ) +var ( + errUnexpectedURL = errors.New("unexpected URL on layer") + errMissingURL = errors.New("missing URL on layer") + errInvalidURL = errors.New("invalid URL on layer") +) + //schema2ManifestHandler is a ManifestHandler that covers schema2 manifests. type schema2ManifestHandler struct { repository *repository @@ -80,7 +89,27 @@ func (ms *schema2ManifestHandler) verifyManifest(ctx context.Context, mnfst sche } for _, fsLayer := range mnfst.References() { - _, err := ms.repository.Blobs(ctx).Stat(ctx, fsLayer.Digest) + var err error + if fsLayer.MediaType != schema2.MediaTypeForeignLayer { + if len(fsLayer.URLs) == 0 { + _, err = ms.repository.Blobs(ctx).Stat(ctx, fsLayer.Digest) + } else { + err = errUnexpectedURL + } + } else { + // Clients download this layer from an external URL, so do not check for + // its presense. + if len(fsLayer.URLs) == 0 { + err = errMissingURL + } + for _, u := range fsLayer.URLs { + var pu *url.URL + pu, err = url.Parse(u) + if err != nil || (pu.Scheme != "http" && pu.Scheme != "https") || pu.Fragment != "" { + err = errInvalidURL + } + } + } if err != nil { if err != distribution.ErrBlobUnknown { errs = append(errs, err) diff --git a/docs/storage/schema2manifesthandler_test.go b/docs/storage/schema2manifesthandler_test.go new file mode 100644 index 000000000..c2f61edf4 --- /dev/null +++ b/docs/storage/schema2manifesthandler_test.go @@ -0,0 +1,117 @@ +package storage + +import ( + "testing" + + "github.com/docker/distribution" + "github.com/docker/distribution/context" + "github.com/docker/distribution/manifest" + "github.com/docker/distribution/manifest/schema2" + "github.com/docker/distribution/registry/storage/driver/inmemory" +) + +func TestVerifyManifestForeignLayer(t *testing.T) { + ctx := context.Background() + inmemoryDriver := inmemory.New() + registry := createRegistry(t, inmemoryDriver) + repo := makeRepository(t, registry, "test") + manifestService := makeManifestService(t, repo) + + config, err := repo.Blobs(ctx).Put(ctx, schema2.MediaTypeConfig, nil) + if err != nil { + t.Fatal(err) + } + + layer, err := repo.Blobs(ctx).Put(ctx, schema2.MediaTypeLayer, nil) + if err != nil { + t.Fatal(err) + } + + foreignLayer := distribution.Descriptor{ + Digest: "sha256:463435349086340864309863409683460843608348608934092322395278926a", + Size: 6323, + MediaType: schema2.MediaTypeForeignLayer, + } + + template := schema2.Manifest{ + Versioned: manifest.Versioned{ + SchemaVersion: 2, + MediaType: schema2.MediaTypeManifest, + }, + Config: config, + } + + type testcase struct { + BaseLayer distribution.Descriptor + URLs []string + Err error + } + + cases := []testcase{ + { + foreignLayer, + nil, + errMissingURL, + }, + { + layer, + []string{"http://foo/bar"}, + errUnexpectedURL, + }, + { + foreignLayer, + []string{"file:///local/file"}, + errInvalidURL, + }, + { + foreignLayer, + []string{"http://foo/bar#baz"}, + errInvalidURL, + }, + { + foreignLayer, + []string{""}, + errInvalidURL, + }, + { + foreignLayer, + []string{"https://foo/bar", ""}, + errInvalidURL, + }, + { + foreignLayer, + []string{"http://foo/bar"}, + nil, + }, + { + foreignLayer, + []string{"https://foo/bar"}, + nil, + }, + } + + for _, c := range cases { + m := template + l := c.BaseLayer + l.URLs = c.URLs + m.Layers = []distribution.Descriptor{l} + dm, err := schema2.FromStruct(m) + if err != nil { + t.Error(err) + continue + } + + _, err = manifestService.Put(ctx, dm) + if verr, ok := err.(distribution.ErrManifestVerification); ok { + // Extract the first error + if len(verr) == 2 { + if _, ok = verr[1].(distribution.ErrManifestBlobUnknown); ok { + err = verr[0] + } + } + } + if err != c.Err { + t.Errorf("%#v: expected %v, got %v", l, c.Err, err) + } + } +}