Enable proxying registries to downgrade fetched manifests to Schema 1.

Ensure Accept headers are sent with TagService.Get (which hits manifest
endpoints).  Add support for remote Get and Put for the proxied blobstore.

Signed-off-by: Richard Scothern <richard.scothern@gmail.com>

.mailmap

@@ -2,6 +2,7 @@ Stephen J Day <stephen.day@docker.com>  Stephen Day <stevvooe@users.noreply.gith
 Stephen J Day <stephen.day@docker.com>  Stephen Day <stevvooe@gmail.com>
 Olivier Gambier <olivier@docker.com>    Olivier Gambier <dmp42@users.noreply.github.com>
 Brian Bland <brian.bland@docker.com>    Brian Bland <r4nd0m1n4t0r@gmail.com>
+Brian Bland <brian.bland@docker.com> Brian Bland <brian.t.bland@gmail.com>
 Josh Hawn <josh.hawn@docker.com>        Josh Hawn <jlhawn@berkeley.edu>
 Richard Scothern <richard.scothern@docker.com> Richard <richard.scothern@gmail.com>
 Richard Scothern <richard.scothern@docker.com> Richard Scothern <richard.scothern@gmail.com>
@@ -11,4 +12,4 @@ Jessie Frazelle <jessie@docker.com>  <jfrazelle@users.noreply.github.com>
 Sharif Nassar <sharif@mrwacky.com> Sharif Nassar <mrwacky42@users.noreply.github.com>
 Sven Dowideit <SvenDowideit@home.org.au> Sven Dowideit <SvenDowideit@users.noreply.github.com>
 Vincent Giersch <vincent.giersch@ovh.net> Vincent Giersch <vincent@giersch.fr>
-davidli <wenquan.li@hp.com> davidli <wenquan.li@hpe.com>
+davidli <wenquan.li@hp.com> davidli <wenquan.li@hpe.com>

AUTHORS

@@ -21,6 +21,7 @@ Ben Firshman <ben@firshman.co.uk>
 bin liu <liubin0329@gmail.com>
 Brian Bland <brian.bland@docker.com>
 burnettk <burnettk@gmail.com>
+Carson A <ca@carsonoid.net>
 Chris Dillon <squarism@gmail.com>
 Daisuke Fujita <dtanshi45@gmail.com>
 Darren Shepherd <darren@rancher.com>
@@ -33,11 +34,13 @@ davidli <wenquan.li@hp.com>
 Dejan Golja <dejan@golja.org>
 Derek McGowan <derek@mcgstyle.net>
 Diogo Mónica <diogo.monica@gmail.com>
+DJ Enriquez <dj.enriquez@infospace.com>
 Donald Huang <don.hcd@gmail.com>
 Doug Davis <dug@us.ibm.com>
 farmerworking <farmerworking@gmail.com>
 Florentin Raud <florentin.raud@gmail.com>
 Frederick F. Kautz IV <fkautz@alumni.cmu.edu>
+gabriell nascimento <gabriell@bluesoft.com.br>
 harche <p.harshal@gmail.com>
 Henri Gomez <henri.gomez@gmail.com>
 Hu Keping <hukeping@huawei.com>
@@ -55,7 +58,9 @@ Josh Hawn <josh.hawn@docker.com>
 Julien Fernandez <julien.fernandez@gmail.com>
 Kelsey Hightower <kelsey.hightower@gmail.com>
 Kenneth Lim <kennethlimcp@gmail.com>
+Kenny Leung <kleung@google.com>
 Li Yi <denverdino@gmail.com>
+Liu Hua <sdu.liu@huawei.com>
 Louis Kottmann <louis.kottmann@gmail.com>
 Luke Carpenter <x@rubynerd.net>
 Mary Anthony <mary@docker.com>
@@ -76,7 +81,9 @@ Olivier Jacques <olivier.jacques@hp.com>
 Patrick Devine <patrick.devine@docker.com>
 Philip Misiowiec <philip@atlashealth.com>
 Richard Scothern <richard.scothern@docker.com>
+Rodolfo Carvalho <rhcarvalho@gmail.com>
 Rusty Conover <rusty@luckydinosaur.com>
+Sean Boran <Boran@users.noreply.github.com>
 Sebastiaan van Stijn <github@gone.nl>
 Sharif Nassar <sharif@mrwacky.com>
 Shawn Falkner-Horine <dreadpirateshawn@gmail.com>
@@ -93,11 +100,13 @@ Thomas Sjögren <konstruktoid@users.noreply.github.com>
 Tianon Gravi <admwiggin@gmail.com>
 Tibor Vass <teabee89@gmail.com>
 Tonis Tiigi <tonistiigi@gmail.com>
+Trevor Pounds <trevor.pounds@gmail.com>
 Troels Thomsen <troels@thomsen.io>
 Vincent Batts <vbatts@redhat.com>
 Vincent Demeester <vincent@sbr.pm>
 Vincent Giersch <vincent.giersch@ovh.net>
 W. Trevor King <wking@tremily.us>
+weiyuan.yl <weiyuan.yl@alibaba-inc.com>
 xg.song <xg.song@venusource.com>
 xiekeyang <xiekeyang@huawei.com>
 Yann ROBERT <yann.robert@anantaplex.fr>

MAINTAINERS

@@ -32,6 +32,11 @@
 	Email = "aaron.lehmann@docker.com"
 	GitHub = "aaronlehmann"
 
+	[people.brianbland]
+	Name = "Brian Bland"
+	Email = "brian.bland@docker.com"
+	GitHub = "BrianBland"
+
 	[people.dmcgowan]
 	Name = "Derek McGowan"
 	Email = "derek@mcgstyle.net"

configuration/configuration.go

@@ -145,6 +145,21 @@ type Configuration struct {
 	Health Health `yaml:"health,omitempty"`
 
 	Proxy Proxy `yaml:"proxy,omitempty"`
+
+	// Compatibility is used for configurations of working with older or deprecated features.
+	Compatibility struct {
+		// Schema1 configures how schema1 manifests will be handled
+		Schema1 struct {
+			// TrustKey is the signing key to use for adding the signature to
+			// schema1 manifests.
+			TrustKey string `yaml:"signingkeyfile,omitempty"`
+
+			// DisableSignatureStore will cause all signatures attached to schema1 manifests
+			// to be ignored. Signatures will be generated on all schema1 manifest requests
+			// rather than only requests which converted schema2 to schema1.
+			DisableSignatureStore bool `yaml:"disablesignaturestore,omitempty"`
+		} `yaml:"schema1,omitempty"`
+	} `yaml:"compatibility,omitempty"`
 }
 
 // LogHook is composed of hook Level and Type.

manifest/schema1/manifest.go

@@ -102,7 +102,7 @@ type SignedManifest struct {
 	Canonical []byte `json:"-"`
 
 	// all contains the byte representation of the Manifest including signatures
-	// and is retuend by Payload()
+	// and is returned by Payload()
 	all []byte
 }
 

reference/reference.go

@@ -6,7 +6,7 @@
 // 	reference                       := repository [ ":" tag ] [ "@" digest ]
 //	name                            := [hostname '/'] component ['/' component]*
 //	hostname                        := hostcomponent ['.' hostcomponent]* [':' port-number]
-//	hostcomponent                   := /([a-z0-9]|[a-z0-9][a-z0-9-]*[a-z0-9])/
+//	hostcomponent                   := /([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9])/
 //	port-number                     := /[0-9]+/
 //	component                       := alpha-numeric [separator alpha-numeric]*
 // 	alpha-numeric                   := /[a-z0-9]+/

reference/regexp.go

@@ -22,7 +22,7 @@ var (
 	// hostnameComponentRegexp restricts the registry hostname component of a
 	// repository name to start with a component as defined by hostnameRegexp
 	// and followed by an optional port.
-	hostnameComponentRegexp = match(`(?:[a-z0-9]|[a-z0-9][a-z0-9-]*[a-z0-9])`)
+	hostnameComponentRegexp = match(`(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9])`)
 
 	// hostnameRegexp defines the structure of potential hostname components
 	// that may be part of image names. This is purposely a subset of what is

reference/regexp_test.go

@@ -111,6 +111,10 @@ func TestHostRegexp(t *testing.T) {
 			input: "xn--n3h.com", // ☃.com in punycode
 			match: true,
 		},
+		{
+			input: "Asdf.com", // uppercase character
+			match: true,
+		},
 	}
 	r := regexp.MustCompile(`^` + hostnameRegexp.String() + `$`)
 	for i := range hostcases {
@@ -399,6 +403,14 @@ func TestFullNameRegexp(t *testing.T) {
 			match: true,
 			subs:  []string{"registry.io", "foo/project--id.module--name.ver---sion--name"},
 		},
+		{
+			input: "Asdf.com/foo/bar", // uppercase character in hostname
+			match: true,
+		},
+		{
+			input: "Foo/FarB", // uppercase characters in remote name
+			match: false,
+		},
 	}
 	for i := range testcases {
 		checkRegexp(t, anchoredNameRegexp, testcases[i])

registry/client/repository.go

@@ -257,9 +257,18 @@ func (t *tags) Get(ctx context.Context, tag string) (distribution.Descriptor, er
 	if err != nil {
 		return distribution.Descriptor{}, err
 	}
-	var attempts int
-	resp, err := t.client.Head(u)
 
+	req, err := http.NewRequest("HEAD", u, nil)
+	if err != nil {
+		return distribution.Descriptor{}, err
+	}
+
+	for _, t := range distribution.ManifestMediaTypes() {
+		req.Header.Add("Accept", t)
+	}
+
+	var attempts int
+	resp, err := t.client.Do(req)
 check:
 	if err != nil {
 		return distribution.Descriptor{}, err
@@ -269,7 +278,16 @@ check:
 	case resp.StatusCode >= 200 && resp.StatusCode < 400:
 		return descriptorFromResponse(resp)
 	case resp.StatusCode == http.StatusMethodNotAllowed:
-		resp, err = t.client.Get(u)
+		req, err = http.NewRequest("GET", u, nil)
+		if err != nil {
+			return distribution.Descriptor{}, err
+		}
+
+		for _, t := range distribution.ManifestMediaTypes() {
+			req.Header.Add("Accept", t)
+		}
+
+		resp, err = t.client.Do(req)
 		attempts++
 		if attempts > 1 {
 			return distribution.Descriptor{}, err

registry/handlers/api_test.go

@@ -1378,19 +1378,28 @@ func testManifestAPISchema2(t *testing.T, env *testEnv, imageName reference.Name
 	}
 	defer resp.Body.Close()
 
-	checkResponse(t, "fetching uploaded manifest as schema1", resp, http.StatusOK)
-	checkHeaders(t, resp, http.Header{
-		"Docker-Content-Digest": []string{dgst.String()},
-		"ETag":                  []string{fmt.Sprintf(`"%s"`, dgst)},
-	})
-
-	var fetchedSchema1Manifest schema1.SignedManifest
-	dec = json.NewDecoder(resp.Body)
-
-	if err := dec.Decode(&fetchedSchema1Manifest); err != nil {
-		t.Fatalf("error decoding fetched schema1 manifest: %v", err)
+	manifestBytes, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatalf("error reading response body: %v", err)
 	}
 
+	checkResponse(t, "fetching uploaded manifest as schema1", resp, http.StatusOK)
+
+	m, desc, err := distribution.UnmarshalManifest(schema1.MediaTypeManifest, manifestBytes)
+	if err != nil {
+		t.Fatalf("unexpected error unmarshalling manifest: %v", err)
+	}
+
+	fetchedSchema1Manifest, ok := m.(*schema1.SignedManifest)
+	if !ok {
+		t.Fatalf("expecting schema1 manifest")
+	}
+
+	checkHeaders(t, resp, http.Header{
+		"Docker-Content-Digest": []string{desc.Digest.String()},
+		"ETag":                  []string{fmt.Sprintf(`"%s"`, desc.Digest)},
+	})
+
 	if fetchedSchema1Manifest.Manifest.SchemaVersion != 1 {
 		t.Fatal("wrong schema version")
 	}
@@ -1603,19 +1612,28 @@ func testManifestAPIManifestList(t *testing.T, env *testEnv, args manifestArgs)
 	}
 	defer resp.Body.Close()
 
-	checkResponse(t, "fetching uploaded manifest list as schema1", resp, http.StatusOK)
-	checkHeaders(t, resp, http.Header{
-		"Docker-Content-Digest": []string{dgst.String()},
-		"ETag":                  []string{fmt.Sprintf(`"%s"`, dgst)},
-	})
-
-	var fetchedSchema1Manifest schema1.SignedManifest
-	dec = json.NewDecoder(resp.Body)
-
-	if err := dec.Decode(&fetchedSchema1Manifest); err != nil {
-		t.Fatalf("error decoding fetched schema1 manifest: %v", err)
+	manifestBytes, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatalf("error reading response body: %v", err)
 	}
 
+	checkResponse(t, "fetching uploaded manifest list as schema1", resp, http.StatusOK)
+
+	m, desc, err := distribution.UnmarshalManifest(schema1.MediaTypeManifest, manifestBytes)
+	if err != nil {
+		t.Fatalf("unexpected error unmarshalling manifest: %v", err)
+	}
+
+	fetchedSchema1Manifest, ok := m.(*schema1.SignedManifest)
+	if !ok {
+		t.Fatalf("expecting schema1 manifest")
+	}
+
+	checkHeaders(t, resp, http.Header{
+		"Docker-Content-Digest": []string{desc.Digest.String()},
+		"ETag":                  []string{fmt.Sprintf(`"%s"`, desc.Digest)},
+	})
+
 	if fetchedSchema1Manifest.Manifest.SchemaVersion != 1 {
 		t.Fatal("wrong schema version")
 	}

registry/handlers/app.go

@@ -146,11 +146,18 @@ func NewApp(ctx context.Context, configuration *configuration.Configuration) *Ap
 	app.configureRedis(configuration)
 	app.configureLogHook(configuration)
 
-	// Generate an ephemeral key to be used for signing converted manifests
-	// for clients that don't support schema2.
-	app.trustKey, err = libtrust.GenerateECP256PrivateKey()
-	if err != nil {
-		panic(err)
+	if configuration.Compatibility.Schema1.TrustKey != "" {
+		app.trustKey, err = libtrust.LoadKeyFile(configuration.Compatibility.Schema1.TrustKey)
+		if err != nil {
+			panic(fmt.Sprintf(`could not load schema1 "signingkey" parameter: %v`, err))
+		}
+	} else {
+		// Generate an ephemeral key to be used for signing converted manifests
+		// for clients that don't support schema2.
+		app.trustKey, err = libtrust.GenerateECP256PrivateKey()
+		if err != nil {
+			panic(err)
+		}
 	}
 
 	if configuration.HTTP.Host != "" {
@@ -167,6 +174,11 @@ func NewApp(ctx context.Context, configuration *configuration.Configuration) *Ap
 		options = append(options, storage.DisableDigestResumption)
 	}
 
+	if configuration.Compatibility.Schema1.DisableSignatureStore {
+		options = append(options, storage.DisableSchema1Signatures)
+		options = append(options, storage.Schema1SigningKey(app.trustKey))
+	}
+
 	// configure deletion
 	if d, ok := configuration.Storage["delete"]; ok {
 		e, ok := d["enabled"]

registry/handlers/images.go

@@ -196,6 +196,7 @@ func (imh *imageManifestHandler) convertSchema2Manifest(schema2Manifest *schema2
 		imh.Errors = append(imh.Errors, v2.ErrorCodeManifestInvalid.WithDetail(err))
 		return nil, err
 	}
+	imh.Digest = digest.FromBytes(manifest.(*schema1.SignedManifest).Canonical)
 
 	return manifest, nil
 }

registry/proxy/proxyauth.go

@@ -8,6 +8,7 @@ import (
 )
 
 const tokenURL = "https://auth.docker.io/token"
+const challengeHeader = "Docker-Distribution-Api-Version"
 
 type userpass struct {
 	username string
@@ -24,12 +25,8 @@ func (c credentials) Basic(u *url.URL) (string, string) {
 	return up.username, up.password
 }
 
-// ConfigureAuth authorizes with the upstream registry
-func ConfigureAuth(remoteURL, username, password string, cm auth.ChallengeManager) (auth.CredentialStore, error) {
-	if err := ping(cm, remoteURL+"/v2/", "Docker-Distribution-Api-Version"); err != nil {
-		return nil, err
-	}
-
+// configureAuth stores credentials for challenge responses
+func configureAuth(username, password string) (auth.CredentialStore, error) {
 	creds := map[string]userpass{
 		tokenURL: {
 			username: username,

registry/proxy/proxyblobstore.go

@@ -22,6 +22,7 @@ type proxyBlobStore struct {
 	remoteStore    distribution.BlobService
 	scheduler      *scheduler.TTLExpirationScheduler
 	repositoryName reference.Named
+	authChallenger authChallenger
 }
 
 var _ distribution.BlobStore = &proxyBlobStore{}
@@ -121,6 +122,10 @@ func (pbs *proxyBlobStore) ServeBlob(ctx context.Context, w http.ResponseWriter,
 		return nil
 	}
 
+	if err := pbs.authChallenger.tryEstablishChallenges(ctx); err != nil {
+		return err
+	}
+
 	mu.Lock()
 	_, ok := inflight[dgst]
 	if ok {
@@ -162,9 +167,35 @@ func (pbs *proxyBlobStore) Stat(ctx context.Context, dgst digest.Digest) (distri
 		return distribution.Descriptor{}, err
 	}
 
+	if err := pbs.authChallenger.tryEstablishChallenges(ctx); err != nil {
+		return distribution.Descriptor{}, err
+	}
+
 	return pbs.remoteStore.Stat(ctx, dgst)
 }
 
+func (pbs *proxyBlobStore) Get(ctx context.Context, dgst digest.Digest) ([]byte, error) {
+	blob, err := pbs.localStore.Get(ctx, dgst)
+	if err == nil {
+		return blob, nil
+	}
+
+	if err := pbs.authChallenger.tryEstablishChallenges(ctx); err != nil {
+		return []byte{}, err
+	}
+
+	blob, err = pbs.remoteStore.Get(ctx, dgst)
+	if err != nil {
+		return []byte{}, err
+	}
+
+	_, err = pbs.localStore.Put(ctx, "", blob)
+	if err != nil {
+		return []byte{}, err
+	}
+	return blob, nil
+}
+
 // Unsupported functions
 func (pbs *proxyBlobStore) Put(ctx context.Context, mediaType string, p []byte) (distribution.Descriptor, error) {
 	return distribution.Descriptor{}, distribution.ErrUnsupported
@@ -186,10 +217,6 @@ func (pbs *proxyBlobStore) Open(ctx context.Context, dgst digest.Digest) (distri
 	return nil, distribution.ErrUnsupported
 }
 
-func (pbs *proxyBlobStore) Get(ctx context.Context, dgst digest.Digest) ([]byte, error) {
-	return nil, distribution.ErrUnsupported
-}
-
 func (pbs *proxyBlobStore) Delete(ctx context.Context, dgst digest.Digest) error {
 	return distribution.ErrUnsupported
 }

registry/proxy/proxyblobstore_test.go

@@ -168,6 +168,7 @@ func makeTestEnv(t *testing.T, name string) *testEnv {
 		remoteStore:    truthBlobs,
 		localStore:     localBlobs,
 		scheduler:      s,
+		authChallenger: &mockChallenger{},
 	}
 
 	te := &testEnv{
@@ -217,6 +218,40 @@ func populate(t *testing.T, te *testEnv, blobCount, size, numUnique int) {
 	te.inRemote = inRemote
 	te.numUnique = numUnique
 }
+func TestProxyStoreGet(t *testing.T) {
+	te := makeTestEnv(t, "foo/bar")
+
+	localStats := te.LocalStats()
+	remoteStats := te.RemoteStats()
+
+	populate(t, te, 1, 10, 1)
+	_, err := te.store.Get(te.ctx, te.inRemote[0].Digest)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if (*localStats)["get"] != 1 && (*localStats)["put"] != 1 {
+		t.Errorf("Unexpected local counts")
+	}
+
+	if (*remoteStats)["get"] != 1 {
+		t.Errorf("Unexpected remote get count")
+	}
+
+	_, err = te.store.Get(te.ctx, te.inRemote[0].Digest)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if (*localStats)["get"] != 2 && (*localStats)["put"] != 1 {
+		t.Errorf("Unexpected local counts")
+	}
+
+	if (*remoteStats)["get"] != 1 {
+		t.Errorf("Unexpected remote get count")
+	}
+
+}
 
 func TestProxyStoreStat(t *testing.T) {
 	te := makeTestEnv(t, "foo/bar")
@@ -242,6 +277,11 @@ func TestProxyStoreStat(t *testing.T) {
 	if (*remoteStats)["stat"] != remoteBlobCount {
 		t.Errorf("Unexpected remote stat count")
 	}
+
+	if te.store.authChallenger.(*mockChallenger).count != len(te.inRemote) {
+		t.Fatalf("Unexpected auth challenge count, got %#v", te.store.authChallenger)
+	}
+
 }
 
 func TestProxyStoreServeHighConcurrency(t *testing.T) {

registry/proxy/proxymanifeststore.go

@@ -19,6 +19,7 @@ type proxyManifestStore struct {
 	remoteManifests distribution.ManifestService
 	repositoryName  reference.Named
 	scheduler       *scheduler.TTLExpirationScheduler
+	authChallenger  authChallenger
 }
 
 var _ distribution.ManifestService = &proxyManifestStore{}
@@ -31,7 +32,9 @@ func (pms proxyManifestStore) Exists(ctx context.Context, dgst digest.Digest) (b
 	if exists {
 		return true, nil
 	}
-
+	if err := pms.authChallenger.tryEstablishChallenges(ctx); err != nil {
+		return false, err
+	}
 	return pms.remoteManifests.Exists(ctx, dgst)
 }
 
@@ -41,6 +44,10 @@ func (pms proxyManifestStore) Get(ctx context.Context, dgst digest.Digest, optio
 	var fromRemote bool
 	manifest, err := pms.localManifests.Get(ctx, dgst, options...)
 	if err != nil {
+		if err := pms.authChallenger.tryEstablishChallenges(ctx); err != nil {
+			return nil, err
+		}
+
 		manifest, err = pms.remoteManifests.Get(ctx, dgst, options...)
 		if err != nil {
 			return nil, err

registry/proxy/proxymanifeststore_test.go

@@ -2,6 +2,7 @@ package proxy
 
 import (
 	"io"
+	"sync"
 	"testing"
 
 	"github.com/docker/distribution"
@@ -10,6 +11,7 @@ import (
 	"github.com/docker/distribution/manifest"
 	"github.com/docker/distribution/manifest/schema1"
 	"github.com/docker/distribution/reference"
+	"github.com/docker/distribution/registry/client/auth"
 	"github.com/docker/distribution/registry/proxy/scheduler"
 	"github.com/docker/distribution/registry/storage"
 	"github.com/docker/distribution/registry/storage/cache/memory"
@@ -64,6 +66,28 @@ func (sm statsManifest) Put(ctx context.Context, manifest distribution.Manifest,
 }
 */
 
+type mockChallenger struct {
+	sync.Mutex
+	count int
+}
+
+// Called for remote operations only
+func (m *mockChallenger) tryEstablishChallenges(context.Context) error {
+	m.Lock()
+	defer m.Unlock()
+
+	m.count++
+	return nil
+}
+
+func (m *mockChallenger) credentialStore() auth.CredentialStore {
+	return nil
+}
+
+func (m *mockChallenger) challengeManager() auth.ChallengeManager {
+	return nil
+}
+
 func newManifestStoreTestEnv(t *testing.T, name, tag string) *manifestStoreTestEnv {
 	nameRef, err := reference.ParseNamed(name)
 	if err != nil {
@@ -120,6 +144,7 @@ func newManifestStoreTestEnv(t *testing.T, name, tag string) *manifestStoreTestE
 			remoteManifests: truthManifests,
 			scheduler:       s,
 			repositoryName:  nameRef,
+			authChallenger:  &mockChallenger{},
 		},
 	}
 }
@@ -198,6 +223,10 @@ func TestProxyManifests(t *testing.T) {
 		t.Errorf("Unexpected exists count : \n%v \n%v", localStats, remoteStats)
 	}
 
+	if env.manifests.authChallenger.(*mockChallenger).count != 1 {
+		t.Fatalf("Expected 1 auth challenge, got %#v", env.manifests.authChallenger)
+	}
+
 	// Get - should succeed and pull manifest into local
 	_, err = env.manifests.Get(ctx, env.manifestDigest)
 	if err != nil {
@@ -212,6 +241,10 @@ func TestProxyManifests(t *testing.T) {
 		t.Errorf("Expected local put")
 	}
 
+	if env.manifests.authChallenger.(*mockChallenger).count != 2 {
+		t.Fatalf("Expected 2 auth challenges, got %#v", env.manifests.authChallenger)
+	}
+
 	// Stat - should only go to local
 	exists, err = env.manifests.Exists(ctx, env.manifestDigest)
 	if err != nil {
@@ -225,17 +258,18 @@ func TestProxyManifests(t *testing.T) {
 		t.Errorf("Unexpected exists count")
 	}
 
-	// Get - should get from remote, to test freshness
+	if env.manifests.authChallenger.(*mockChallenger).count != 2 {
+		t.Fatalf("Expected 2 auth challenges, got %#v", env.manifests.authChallenger)
+	}
+
+	// Get proxied - won't require another authchallenge
 	_, err = env.manifests.Get(ctx, env.manifestDigest)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	if (*remoteStats)["get"] != 2 && (*remoteStats)["exists"] != 1 && (*localStats)["put"] != 1 {
-		t.Errorf("Unexpected get count")
+	if env.manifests.authChallenger.(*mockChallenger).count != 2 {
+		t.Fatalf("Expected 2 auth challenges, got %#v", env.manifests.authChallenger)
 	}
-}
-
-func TestProxyTagService(t *testing.T) {
 
 }

registry/proxy/proxyregistry.go

@@ -1,10 +1,11 @@
 package proxy
 
 import (
+	"fmt"
 	"net/http"
 	"net/url"
+	"sync"
 
-	"fmt"
 	"github.com/docker/distribution"
 	"github.com/docker/distribution/configuration"
 	"github.com/docker/distribution/context"
@@ -19,13 +20,10 @@ import (
 
 // proxyingRegistry fetches content from a remote registry and caches it locally
 type proxyingRegistry struct {
-	embedded distribution.Namespace // provides local registry functionality
-
-	scheduler *scheduler.TTLExpirationScheduler
-
-	remoteURL        string
-	credentialStore  auth.CredentialStore
-	challengeManager auth.ChallengeManager
+	embedded       distribution.Namespace // provides local registry functionality
+	scheduler      *scheduler.TTLExpirationScheduler
+	remoteURL      string
+	authChallenger authChallenger
 }
 
 // NewRegistryPullThroughCache creates a registry acting as a pull through cache
@@ -93,18 +91,20 @@ func NewRegistryPullThroughCache(ctx context.Context, registry distribution.Name
 		return nil, err
 	}
 
-	challengeManager := auth.NewSimpleChallengeManager()
-	cs, err := ConfigureAuth(config.RemoteURL, config.Username, config.Password, challengeManager)
+	cs, err := configureAuth(config.Username, config.Password)
 	if err != nil {
 		return nil, err
 	}
 
 	return &proxyingRegistry{
-		embedded:         registry,
-		scheduler:        s,
-		challengeManager: challengeManager,
-		credentialStore:  cs,
-		remoteURL:        config.RemoteURL,
+		embedded:  registry,
+		scheduler: s,
+		remoteURL: config.RemoteURL,
+		authChallenger: &remoteAuthChallenger{
+			remoteURL: config.RemoteURL,
+			cm:        auth.NewSimpleChallengeManager(),
+			cs:        cs,
+		},
 	}, nil
 }
 
@@ -117,8 +117,10 @@ func (pr *proxyingRegistry) Repositories(ctx context.Context, repos []string, la
 }
 
 func (pr *proxyingRegistry) Repository(ctx context.Context, name reference.Named) (distribution.Repository, error) {
+	c := pr.authChallenger
+
 	tr := transport.NewTransport(http.DefaultTransport,
-		auth.NewAuthorizer(pr.challengeManager, auth.NewTokenHandler(http.DefaultTransport, pr.credentialStore, name.Name(), "pull")))
+		auth.NewAuthorizer(c.challengeManager(), auth.NewTokenHandler(http.DefaultTransport, c.credentialStore(), name.Name(), "pull")))
 
 	localRepo, err := pr.embedded.Repository(ctx, name)
 	if err != nil {
@@ -145,6 +147,7 @@ func (pr *proxyingRegistry) Repository(ctx context.Context, name reference.Named
 			remoteStore:    remoteRepo.Blobs(ctx),
 			scheduler:      pr.scheduler,
 			repositoryName: name,
+			authChallenger: pr.authChallenger,
 		},
 		manifests: &proxyManifestStore{
 			repositoryName:  name,
@@ -152,15 +155,63 @@ func (pr *proxyingRegistry) Repository(ctx context.Context, name reference.Named
 			remoteManifests: remoteManifests,
 			ctx:             ctx,
 			scheduler:       pr.scheduler,
+			authChallenger:  pr.authChallenger,
 		},
 		name: name,
 		tags: &proxyTagService{
-			localTags:  localRepo.Tags(ctx),
-			remoteTags: remoteRepo.Tags(ctx),
+			localTags:      localRepo.Tags(ctx),
+			remoteTags:     remoteRepo.Tags(ctx),
+			authChallenger: pr.authChallenger,
 		},
 	}, nil
 }
 
+// authChallenger encapsulates a request to the upstream to establish credential challenges
+type authChallenger interface {
+	tryEstablishChallenges(context.Context) error
+	challengeManager() auth.ChallengeManager
+	credentialStore() auth.CredentialStore
+}
+
+type remoteAuthChallenger struct {
+	remoteURL string
+	sync.Mutex
+	cm auth.ChallengeManager
+	cs auth.CredentialStore
+}
+
+func (r *remoteAuthChallenger) credentialStore() auth.CredentialStore {
+	return r.cs
+}
+
+func (r *remoteAuthChallenger) challengeManager() auth.ChallengeManager {
+	return r.cm
+}
+
+// tryEstablishChallenges will attempt to get a challenge type for the upstream if none currently exist
+func (r *remoteAuthChallenger) tryEstablishChallenges(ctx context.Context) error {
+	r.Lock()
+	defer r.Unlock()
+
+	remoteURL := r.remoteURL + "/v2/"
+	challenges, err := r.cm.GetChallenges(remoteURL)
+	if err != nil {
+		return err
+	}
+
+	if len(challenges) > 0 {
+		return nil
+	}
+
+	// establish challenge type with upstream
+	if err := ping(r.cm, remoteURL, challengeHeader); err != nil {
+		return err
+	}
+
+	context.GetLogger(ctx).Infof("Challenge established with upstream : %s %s", remoteURL, r.cm)
+	return nil
+}
+
 // proxiedRepository uses proxying blob and manifest services to serve content
 // locally, or pulling it through from a remote and caching it locally if it doesn't
 // already exist

registry/proxy/proxytagservice.go

@@ -7,8 +7,9 @@ import (
 
 // proxyTagService supports local and remote lookup of tags.
 type proxyTagService struct {
-	localTags  distribution.TagService
-	remoteTags distribution.TagService
+	localTags      distribution.TagService
+	remoteTags     distribution.TagService
+	authChallenger authChallenger
 }
 
 var _ distribution.TagService = proxyTagService{}
@@ -17,16 +18,19 @@ var _ distribution.TagService = proxyTagService{}
 // tag service first and then caching it locally.  If the remote is unavailable
 // the local association is returned
 func (pt proxyTagService) Get(ctx context.Context, tag string) (distribution.Descriptor, error) {
-	desc, err := pt.remoteTags.Get(ctx, tag)
+	err := pt.authChallenger.tryEstablishChallenges(ctx)
 	if err == nil {
-		err := pt.localTags.Tag(ctx, tag, desc)
-		if err != nil {
-			return distribution.Descriptor{}, err
+		desc, err := pt.remoteTags.Get(ctx, tag)
+		if err == nil {
+			err := pt.localTags.Tag(ctx, tag, desc)
+			if err != nil {
+				return distribution.Descriptor{}, err
+			}
+			return desc, nil
 		}
-		return desc, nil
 	}
 
-	desc, err = pt.localTags.Get(ctx, tag)
+	desc, err := pt.localTags.Get(ctx, tag)
 	if err != nil {
 		return distribution.Descriptor{}, err
 	}
@@ -46,9 +50,12 @@ func (pt proxyTagService) Untag(ctx context.Context, tag string) error {
 }
 
 func (pt proxyTagService) All(ctx context.Context) ([]string, error) {
-	tags, err := pt.remoteTags.All(ctx)
+	err := pt.authChallenger.tryEstablishChallenges(ctx)
 	if err == nil {
-		return tags, err
+		tags, err := pt.remoteTags.All(ctx)
+		if err == nil {
+			return tags, err
+		}
 	}
 	return pt.localTags.All(ctx)
 }

registry/proxy/proxytagservice_test.go

@@ -69,8 +69,9 @@ func testProxyTagService(local, remote map[string]distribution.Descriptor) *prox
 		remote = make(map[string]distribution.Descriptor)
 	}
 	return &proxyTagService{
-		localTags:  &mockTagStore{mapping: local},
-		remoteTags: &mockTagStore{mapping: remote},
+		localTags:      &mockTagStore{mapping: local},
+		remoteTags:     &mockTagStore{mapping: remote},
+		authChallenger: &mockChallenger{},
 	}
 }
 
@@ -87,6 +88,10 @@ func TestGet(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	if proxyTags.authChallenger.(*mockChallenger).count != 1 {
+		t.Fatalf("Expected 1 auth challenge call, got %#v", proxyTags.authChallenger)
+	}
+
 	if d != remoteDesc {
 		t.Fatal("unable to get put tag")
 	}
@@ -112,6 +117,10 @@ func TestGet(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	if proxyTags.authChallenger.(*mockChallenger).count != 2 {
+		t.Fatalf("Expected 2 auth challenge calls, got %#v", proxyTags.authChallenger)
+	}
+
 	if d != newRemoteDesc {
 		t.Fatal("unable to get put tag")
 	}
@@ -142,7 +151,11 @@ func TestGet(t *testing.T) {
 		t.Fatal("untagged tag should be pulled through")
 	}
 
-	// Add another tag.  Ensure both tags appear in enumerate
+	if proxyTags.authChallenger.(*mockChallenger).count != 3 {
+		t.Fatalf("Expected 3 auth challenge calls, got %#v", proxyTags.authChallenger)
+	}
+
+	// Add another tag.  Ensure both tags appear in 'All'
 	err = proxyTags.remoteTags.Tag(ctx, "funtag", distribution.Descriptor{Size: 42})
 	if err != nil {
 		t.Fatal(err)
@@ -161,4 +174,8 @@ func TestGet(t *testing.T) {
 	if all[0] != "funtag" && all[1] != "remote" {
 		t.Fatalf("Unexpected tags returned from All() : %v ", all)
 	}
+
+	if proxyTags.authChallenger.(*mockChallenger).count != 4 {
+		t.Fatalf("Expected 4 auth challenge calls, got %#v", proxyTags.authChallenger)
+	}
 }

registry/storage/manifeststore_test.go

@@ -28,11 +28,10 @@ type manifestStoreTestEnv struct {
 	tag        string
 }
 
-func newManifestStoreTestEnv(t *testing.T, name reference.Named, tag string) *manifestStoreTestEnv {
+func newManifestStoreTestEnv(t *testing.T, name reference.Named, tag string, options ...RegistryOption) *manifestStoreTestEnv {
 	ctx := context.Background()
 	driver := inmemory.New()
-	registry, err := NewRegistry(ctx, driver, BlobDescriptorCacheProvider(
-		memory.NewInMemoryBlobDescriptorCacheProvider()), EnableDelete, EnableRedirect)
+	registry, err := NewRegistry(ctx, driver, options...)
 	if err != nil {
 		t.Fatalf("error creating registry: %v", err)
 	}
@@ -53,13 +52,26 @@ func newManifestStoreTestEnv(t *testing.T, name reference.Named, tag string) *ma
 }
 
 func TestManifestStorage(t *testing.T) {
+	testManifestStorage(t, BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()), EnableDelete, EnableRedirect)
+}
+
+func TestManifestStorageDisabledSignatures(t *testing.T) {
+	k, err := libtrust.GenerateECP256PrivateKey()
+	if err != nil {
+		t.Fatal(err)
+	}
+	testManifestStorage(t, BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()), EnableDelete, EnableRedirect, DisableSchema1Signatures, Schema1SigningKey(k))
+}
+
+func testManifestStorage(t *testing.T, options ...RegistryOption) {
 	repoName, _ := reference.ParseNamed("foo/bar")
-	env := newManifestStoreTestEnv(t, repoName, "thetag")
+	env := newManifestStoreTestEnv(t, repoName, "thetag", options...)
 	ctx := context.Background()
 	ms, err := env.repository.Manifests(ctx)
 	if err != nil {
 		t.Fatal(err)
 	}
+	equalSignatures := env.registry.(*registry).schema1SignaturesEnabled
 
 	m := schema1.Manifest{
 		Versioned: manifest.Versioned{
@@ -159,8 +171,14 @@ func TestManifestStorage(t *testing.T) {
 		t.Fatalf("unexpected manifest type from signedstore")
 	}
 
-	if !reflect.DeepEqual(fetchedManifest, sm) {
-		t.Fatalf("fetched manifest not equal: %#v != %#v", fetchedManifest, sm)
+	if !bytes.Equal(fetchedManifest.Canonical, sm.Canonical) {
+		t.Fatalf("fetched payload does not match original payload: %q != %q", fetchedManifest.Canonical, sm.Canonical)
+	}
+
+	if equalSignatures {
+		if !reflect.DeepEqual(fetchedManifest, sm) {
+			t.Fatalf("fetched manifest not equal: %#v != %#v", fetchedManifest.Manifest, sm.Manifest)
+		}
 	}
 
 	_, pl, err := fetchedManifest.Payload()
@@ -196,8 +214,19 @@ func TestManifestStorage(t *testing.T) {
 		t.Fatalf("unexpected error fetching manifest by digest: %v", err)
 	}
 
-	if !reflect.DeepEqual(fetchedByDigest, fetchedManifest) {
-		t.Fatalf("fetched manifest not equal: %#v != %#v", fetchedByDigest, fetchedManifest)
+	byDigestManifest, ok := fetchedByDigest.(*schema1.SignedManifest)
+	if !ok {
+		t.Fatalf("unexpected manifest type from signedstore")
+	}
+
+	if !bytes.Equal(byDigestManifest.Canonical, fetchedManifest.Canonical) {
+		t.Fatalf("fetched manifest not equal: %q != %q", byDigestManifest.Canonical, fetchedManifest.Canonical)
+	}
+
+	if equalSignatures {
+		if !reflect.DeepEqual(fetchedByDigest, fetchedManifest) {
+			t.Fatalf("fetched manifest not equal: %#v != %#v", fetchedByDigest, fetchedManifest)
+		}
 	}
 
 	sigs, err := fetchedJWS.Signatures()
@@ -286,14 +315,16 @@ func TestManifestStorage(t *testing.T) {
 		t.Fatalf("payloads are not equal")
 	}
 
-	receivedSigs, err := receivedJWS.Signatures()
-	if err != nil {
-		t.Fatalf("error getting signatures: %v", err)
-	}
+	if equalSignatures {
+		receivedSigs, err := receivedJWS.Signatures()
+		if err != nil {
+			t.Fatalf("error getting signatures: %v", err)
+		}
 
-	for i, sig := range receivedSigs {
-		if !bytes.Equal(sig, expectedSigs[i]) {
-			t.Fatalf("mismatched signatures from remote: %v != %v", string(sig), string(expectedSigs[i]))
+		for i, sig := range receivedSigs {
+			if !bytes.Equal(sig, expectedSigs[i]) {
+				t.Fatalf("mismatched signatures from remote: %v != %v", string(sig), string(expectedSigs[i]))
+			}
 		}
 	}
 

registry/storage/registry.go

@@ -6,6 +6,7 @@ import (
 	"github.com/docker/distribution/reference"
 	"github.com/docker/distribution/registry/storage/cache"
 	storagedriver "github.com/docker/distribution/registry/storage/driver"
+	"github.com/docker/libtrust"
 )
 
 // registry is the top-level implementation of Registry for use in the storage
@@ -17,6 +18,8 @@ type registry struct {
 	blobDescriptorCacheProvider cache.BlobDescriptorCacheProvider
 	deleteEnabled               bool
 	resumableDigestEnabled      bool
+	schema1SignaturesEnabled    bool
+	schema1SigningKey           libtrust.PrivateKey
 }
 
 // RegistryOption is the type used for functional options for NewRegistry.
@@ -43,6 +46,24 @@ func DisableDigestResumption(registry *registry) error {
 	return nil
 }
 
+// DisableSchema1Signatures is a functional option for NewRegistry. It disables
+// signature storage and ensures all schema1 manifests will only be returned
+// with a signature from a provided signing key.
+func DisableSchema1Signatures(registry *registry) error {
+	registry.schema1SignaturesEnabled = false
+	return nil
+}
+
+// Schema1SigningKey returns a functional option for NewRegistry. It sets the
+// signing key for adding a signature to all schema1 manifests. This should be
+// used in conjunction with disabling signature store.
+func Schema1SigningKey(key libtrust.PrivateKey) RegistryOption {
+	return func(registry *registry) error {
+		registry.schema1SigningKey = key
+		return nil
+	}
+}
+
 // BlobDescriptorCacheProvider returns a functional option for
 // NewRegistry. It creates a cached blob statter for use by the
 // registry.
@@ -85,8 +106,9 @@ func NewRegistry(ctx context.Context, driver storagedriver.StorageDriver, option
 			statter: statter,
 			pathFn:  bs.path,
 		},
-		statter:                statter,
-		resumableDigestEnabled: true,
+		statter:                  statter,
+		resumableDigestEnabled:   true,
+		schema1SignaturesEnabled: true,
 	}
 
 	for _, option := range options {

registry/storage/signedmanifesthandler.go

@@ -25,10 +25,17 @@ var _ ManifestHandler = &signedManifestHandler{}
 
 func (ms *signedManifestHandler) Unmarshal(ctx context.Context, dgst digest.Digest, content []byte) (distribution.Manifest, error) {
 	context.GetLogger(ms.ctx).Debug("(*signedManifestHandler).Unmarshal")
-	// Fetch the signatures for the manifest
-	signatures, err := ms.signatures.Get(dgst)
-	if err != nil {
-		return nil, err
+
+	var (
+		signatures [][]byte
+		err        error
+	)
+	if ms.repository.schema1SignaturesEnabled {
+		// Fetch the signatures for the manifest
+		signatures, err = ms.signatures.Get(dgst)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	jsig, err := libtrust.NewJSONSignature(content, signatures...)
@@ -36,6 +43,14 @@ func (ms *signedManifestHandler) Unmarshal(ctx context.Context, dgst digest.Dige
 		return nil, err
 	}
 
+	if ms.repository.schema1SigningKey != nil {
+		if err := jsig.Sign(ms.repository.schema1SigningKey); err != nil {
+			return nil, err
+		}
+	} else if !ms.repository.schema1SignaturesEnabled {
+		return nil, fmt.Errorf("missing signing key with signature store disabled")
+	}
+
 	// Extract the pretty JWS
 	raw, err := jsig.PrettySignature("signatures")
 	if err != nil {
@@ -75,14 +90,16 @@ func (ms *signedManifestHandler) Put(ctx context.Context, manifest distribution.
 		return "", err
 	}
 
-	// Grab each json signature and store them.
-	signatures, err := sm.Signatures()
-	if err != nil {
-		return "", err
-	}
+	if ms.repository.schema1SignaturesEnabled {
+		// Grab each json signature and store them.
+		signatures, err := sm.Signatures()
+		if err != nil {
+			return "", err
+		}
 
-	if err := ms.signatures.Put(revision.Digest, signatures...); err != nil {
-		return "", err
+		if err := ms.signatures.Put(revision.Digest, signatures...); err != nil {
+			return "", err
+		}
 	}
 
 	return revision.Digest, nil

version/version.go

@@ -8,4 +8,4 @@ var Package = "github.com/docker/distribution"
 // the latest release tag by hand, always suffixed by "+unknown". During
 // build, it will be replaced by the actual version. The value here will be
 // used if the registry is run after a go get based install.
-var Version = "v2.1.0+unknown"
+var Version = "v2.3.0+unknown"