TrueCloudLab/distribution
13
0
Fork 3
Code Issues 2 Pull requests Projects Releases 2 Packages Wiki Activity Actions

Merge pull request #1268 from RichardScothern/manifest-refactor-impl

Browse source 
Implementation of the Manifest Service API refactor.
This commit is contained in:
Richard Scothern 2015-12-17 17:32:55 -08:00
commit 13b56c9d20
18 changed files with 1161 additions and 656 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
docs/api/v2/descriptors.go
View file

@ -495,7 +495,7 @@ var routeDescriptors = []RouteDescriptor{
Methods: []MethodDescriptor{
{
Method: "GET",
Description: "Fetch the manifest identified by `name` and `reference` where `reference` can be a tag or digest.",
Description: "Fetch the manifest identified by `name` and `reference` where `reference` can be a tag or digest. A `HEAD` request can also be issued to this endpoint to obtain resource information without receiving all data.",
Requests: []RequestDescriptor{
{
Headers: []ParameterDescriptor{

324
docs/client/repository.go
View file

@ -3,6 +3,7 @@ package client
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
@ -14,7 +15,6 @@ import (
"github.com/docker/distribution"
"github.com/docker/distribution/context"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/reference"
"github.com/docker/distribution/registry/api/v2"
"github.com/docker/distribution/registry/client/transport"
@ -156,26 +156,139 @@ func (r *repository) Manifests(ctx context.Context, options ...distribution.Mani
}, nil
}
func (r *repository) Signatures() distribution.SignatureService {
ms, _ := r.Manifests(r.context)
return &signatures{
manifests: ms,
func (r *repository) Tags(ctx context.Context) distribution.TagService {
return &tags{
client: r.client,
ub: r.ub,
context: r.context,
name: r.Name(),
}
}
type signatures struct {
manifests distribution.ManifestService
// tags implements remote tagging operations.
type tags struct {
client *http.Client
ub *v2.URLBuilder
context context.Context
name string
}
func (s *signatures) Get(dgst digest.Digest) ([][]byte, error) {
m, err := s.manifests.Get(dgst)
// All returns all tags
func (t *tags) All(ctx context.Context) ([]string, error) {
var tags []string
u, err := t.ub.BuildTagsURL(t.name)
if err != nil {
return nil, err
return tags, err
}
return m.Signatures()
resp, err := t.client.Get(u)
if err != nil {
return tags, err
}
defer resp.Body.Close()
if SuccessStatus(resp.StatusCode) {
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return tags, err
}
tagsResponse := struct {
Tags []string `json:"tags"`
}{}
if err := json.Unmarshal(b, &tagsResponse); err != nil {
return tags, err
}
tags = tagsResponse.Tags
return tags, nil
}
return tags, handleErrorResponse(resp)
}
func (s *signatures) Put(dgst digest.Digest, signatures ...[]byte) error {
func descriptorFromResponse(response *http.Response) (distribution.Descriptor, error) {
desc := distribution.Descriptor{}
headers := response.Header
ctHeader := headers.Get("Content-Type")
if ctHeader == "" {
return distribution.Descriptor{}, errors.New("missing or empty Content-Type header")
}
desc.MediaType = ctHeader
digestHeader := headers.Get("Docker-Content-Digest")
if digestHeader == "" {
bytes, err := ioutil.ReadAll(response.Body)
if err != nil {
return distribution.Descriptor{}, err
}
_, desc, err := distribution.UnmarshalManifest(ctHeader, bytes)
if err != nil {
return distribution.Descriptor{}, err
}
return desc, nil
}
dgst, err := digest.ParseDigest(digestHeader)
if err != nil {
return distribution.Descriptor{}, err
}
desc.Digest = dgst
lengthHeader := headers.Get("Content-Length")
if lengthHeader == "" {
return distribution.Descriptor{}, errors.New("missing or empty Content-Length header")
}
length, err := strconv.ParseInt(lengthHeader, 10, 64)
if err != nil {
return distribution.Descriptor{}, err
}
desc.Size = length
return desc, nil
}
// Get issues a HEAD request for a Manifest against its named endpoint in order
// to construct a descriptor for the tag. If the registry doesn't support HEADing
// a manifest, fallback to GET.
func (t *tags) Get(ctx context.Context, tag string) (distribution.Descriptor, error) {
u, err := t.ub.BuildManifestURL(t.name, tag)
if err != nil {
return distribution.Descriptor{}, err
}
var attempts int
resp, err := t.client.Head(u)
check:
if err != nil {
return distribution.Descriptor{}, err
}
switch {
case resp.StatusCode >= 200 && resp.StatusCode < 400:
return descriptorFromResponse(resp)
case resp.StatusCode == http.StatusMethodNotAllowed:
resp, err = t.client.Get(u)
attempts++
if attempts > 1 {
return distribution.Descriptor{}, err
}
goto check
default:
return distribution.Descriptor{}, handleErrorResponse(resp)
}
}
func (t *tags) Lookup(ctx context.Context, digest distribution.Descriptor) ([]string, error) {
panic("not implemented")
}
func (t *tags) Tag(ctx context.Context, tag string, desc distribution.Descriptor) error {
panic("not implemented")
}
func (t *tags) Untag(ctx context.Context, tag string) error {
panic("not implemented")
}
@ -186,44 +299,8 @@ type manifests struct {
etags map[string]string
}
func (ms *manifests) Tags() ([]string, error) {
u, err := ms.ub.BuildTagsURL(ms.name)
if err != nil {
return nil, err
}
resp, err := ms.client.Get(u)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if SuccessStatus(resp.StatusCode) {
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
tagsResponse := struct {
Tags []string `json:"tags"`
}{}
if err := json.Unmarshal(b, &tagsResponse); err != nil {
return nil, err
}
return tagsResponse.Tags, nil
}
return nil, handleErrorResponse(resp)
}
func (ms *manifests) Exists(dgst digest.Digest) (bool, error) {
// Call by Tag endpoint since the API uses the same
// URL endpoint for tags and digests.
return ms.ExistsByTag(dgst.String())
}
func (ms *manifests) ExistsByTag(tag string) (bool, error) {
u, err := ms.ub.BuildManifestURL(ms.name, tag)
func (ms *manifests) Exists(ctx context.Context, dgst digest.Digest) (bool, error) {
u, err := ms.ub.BuildManifestURL(ms.name, dgst.String())
if err != nil {
return false, err
}
@ -241,46 +318,63 @@ func (ms *manifests) ExistsByTag(tag string) (bool, error) {
return false, handleErrorResponse(resp)
}
func (ms *manifests) Get(dgst digest.Digest) (*schema1.SignedManifest, error) {
// Call by Tag endpoint since the API uses the same
// URL endpoint for tags and digests.
return ms.GetByTag(dgst.String())
}
// AddEtagToTag allows a client to supply an eTag to GetByTag which will be
// AddEtagToTag allows a client to supply an eTag to Get which will be
// used for a conditional HTTP request. If the eTag matches, a nil manifest
// and nil error will be returned. etag is automatically quoted when added to
// this map.
// and ErrManifestNotModified error will be returned. etag is automatically
// quoted when added to this map.
func AddEtagToTag(tag, etag string) distribution.ManifestServiceOption {
return func(ms distribution.ManifestService) error {
if ms, ok := ms.(*manifests); ok {
ms.etags[tag] = fmt.Sprintf(`"%s"`, etag)
return nil
}
return fmt.Errorf("etag options is a client-only option")
}
return etagOption{tag, etag}
}
func (ms *manifests) GetByTag(tag string, options ...distribution.ManifestServiceOption) (*schema1.SignedManifest, error) {
type etagOption struct{ tag, etag string }
func (o etagOption) Apply(ms distribution.ManifestService) error {
if ms, ok := ms.(*manifests); ok {
ms.etags[o.tag] = fmt.Sprintf(`"%s"`, o.etag)
return nil
}
return fmt.Errorf("etag options is a client-only option")
}
func (ms *manifests) Get(ctx context.Context, dgst digest.Digest, options ...distribution.ManifestServiceOption) (distribution.Manifest, error) {
var tag string
for _, option := range options {
err := option(ms)
if err != nil {
return nil, err
if opt, ok := option.(withTagOption); ok {
tag = opt.tag
} else {
err := option.Apply(ms)
if err != nil {
return nil, err
}
}
}
u, err := ms.ub.BuildManifestURL(ms.name, tag)
var ref string
if tag != "" {
ref = tag
} else {
ref = dgst.String()
}
u, err := ms.ub.BuildManifestURL(ms.name, ref)
if err != nil {
return nil, err
}
req, err := http.NewRequest("GET", u, nil)
if err != nil {
return nil, err
}
if _, ok := ms.etags[tag]; ok {
req.Header.Set("If-None-Match", ms.etags[tag])
for _, t := range distribution.ManifestMediaTypes() {
req.Header.Add("Accept", t)
}
if _, ok := ms.etags[ref]; ok {
req.Header.Set("If-None-Match", ms.etags[ref])
}
resp, err := ms.client.Do(req)
if err != nil {
return nil, err
@ -289,44 +383,89 @@ func (ms *manifests) GetByTag(tag string, options ...distribution.ManifestServic
if resp.StatusCode == http.StatusNotModified {
return nil, distribution.ErrManifestNotModified
} else if SuccessStatus(resp.StatusCode) {
var sm schema1.SignedManifest
decoder := json.NewDecoder(resp.Body)
mt := resp.Header.Get("Content-Type")
body, err := ioutil.ReadAll(resp.Body)
if err := decoder.Decode(&sm); err != nil {
if err != nil {
return nil, err
}
return &sm, nil
m, _, err := distribution.UnmarshalManifest(mt, body)
if err != nil {
return nil, err
}
return m, nil
}
return nil, handleErrorResponse(resp)
}
func (ms *manifests) Put(m *schema1.SignedManifest) error {
manifestURL, err := ms.ub.BuildManifestURL(ms.name, m.Tag)
if err != nil {
return err
// WithTag allows a tag to be passed into Put which enables the client
// to build a correct URL.
func WithTag(tag string) distribution.ManifestServiceOption {
return withTagOption{tag}
}
type withTagOption struct{ tag string }
func (o withTagOption) Apply(m distribution.ManifestService) error {
if _, ok := m.(*manifests); ok {
return nil
}
return fmt.Errorf("withTagOption is a client-only option")
}
// Put puts a manifest. A tag can be specified using an options parameter which uses some shared state to hold the
// tag name in order to build the correct upload URL. This state is written and read under a lock.
func (ms *manifests) Put(ctx context.Context, m distribution.Manifest, options ...distribution.ManifestServiceOption) (digest.Digest, error) {
var tag string
for _, option := range options {
if opt, ok := option.(withTagOption); ok {
tag = opt.tag
} else {
err := option.Apply(ms)
if err != nil {
return "", err
}
}
}
// todo(richardscothern): do something with options here when they become applicable
putRequest, err := http.NewRequest("PUT", manifestURL, bytes.NewReader(m.Raw))
manifestURL, err := ms.ub.BuildManifestURL(ms.name, tag)
if err != nil {
return err
return "", err
}
mediaType, p, err := m.Payload()
if err != nil {
return "", err
}
putRequest, err := http.NewRequest("PUT", manifestURL, bytes.NewReader(p))
if err != nil {
return "", err
}
putRequest.Header.Set("Content-Type", mediaType)
resp, err := ms.client.Do(putRequest)
if err != nil {
return err
return "", err
}
defer resp.Body.Close()
if SuccessStatus(resp.StatusCode) {
// TODO(dmcgowan): make use of digest header
return nil
dgstHeader := resp.Header.Get("Docker-Content-Digest")
dgst, err := digest.ParseDigest(dgstHeader)
if err != nil {
return "", err
}
return dgst, nil
}
return handleErrorResponse(resp)
return "", handleErrorResponse(resp)
}
func (ms *manifests) Delete(dgst digest.Digest) error {
func (ms *manifests) Delete(ctx context.Context, dgst digest.Digest) error {
u, err := ms.ub.BuildManifestURL(ms.name, dgst.String())
if err != nil {
return err
@ -348,6 +487,11 @@ func (ms *manifests) Delete(dgst digest.Digest) error {
return handleErrorResponse(resp)
}
// todo(richardscothern): Restore interface and implementation with merge of #1050
/*func (ms *manifests) Enumerate(ctx context.Context, manifests []distribution.Manifest, last distribution.Manifest) (n int, err error) {
panic("not supported")
}*/
type blobs struct {
name string
ub *v2.URLBuilder

125
docs/client/repository_test.go
View file

@ -42,7 +42,6 @@ func newRandomBlob(size int) (digest.Digest, []byte) {
}
func addTestFetch(repo string, dgst digest.Digest, content []byte, m *testutil.RequestResponseMap) {
*m = append(*m, testutil.RequestResponseMapping{
Request: testutil.Request{
Method: "GET",
@ -499,12 +498,7 @@ func newRandomSchemaV1Manifest(name, tag string, blobCount int) (*schema1.Signed
panic(err)
}
p, err := sm.Payload()
if err != nil {
panic(err)
}
return sm, digest.FromBytes(p), p
return sm, digest.FromBytes(sm.Canonical), sm.Canonical
}
func addTestManifestWithEtag(repo, reference string, content []byte, m *testutil.RequestResponseMap, dgst string) {
@ -525,6 +519,7 @@ func addTestManifestWithEtag(repo, reference string, content []byte, m *testutil
Headers: http.Header(map[string][]string{
"Content-Length": {"0"},
"Last-Modified": {time.Now().Add(-1 * time.Second).Format(time.ANSIC)},
"Content-Type": {schema1.MediaTypeManifest},
}),
}
} else {
@ -534,6 +529,7 @@ func addTestManifestWithEtag(repo, reference string, content []byte, m *testutil
Headers: http.Header(map[string][]string{
"Content-Length": {fmt.Sprint(len(content))},
"Last-Modified": {time.Now().Add(-1 * time.Second).Format(time.ANSIC)},
"Content-Type": {schema1.MediaTypeManifest},
}),
}
@ -553,6 +549,7 @@ func addTestManifest(repo, reference string, content []byte, m *testutil.Request
Headers: http.Header(map[string][]string{
"Content-Length": {fmt.Sprint(len(content))},
"Last-Modified": {time.Now().Add(-1 * time.Second).Format(time.ANSIC)},
"Content-Type": {schema1.MediaTypeManifest},
}),
},
})
@ -566,6 +563,7 @@ func addTestManifest(repo, reference string, content []byte, m *testutil.Request
Headers: http.Header(map[string][]string{
"Content-Length": {fmt.Sprint(len(content))},
"Last-Modified": {time.Now().Add(-1 * time.Second).Format(time.ANSIC)},
"Content-Type": {schema1.MediaTypeManifest},
}),
},
})
@ -598,12 +596,17 @@ func checkEqualManifest(m1, m2 *schema1.SignedManifest) error {
return nil
}
func TestManifestFetch(t *testing.T) {
func TestV1ManifestFetch(t *testing.T) {
ctx := context.Background()
repo := "test.example.com/repo"
m1, dgst, _ := newRandomSchemaV1Manifest(repo, "latest", 6)
var m testutil.RequestResponseMap
addTestManifest(repo, dgst.String(), m1.Raw, &m)
_, pl, err := m1.Payload()
if err != nil {
t.Fatal(err)
}
addTestManifest(repo, dgst.String(), pl, &m)
addTestManifest(repo, "latest", pl, &m)
e, c := testServer(m)
defer c()
@ -617,7 +620,7 @@ func TestManifestFetch(t *testing.T) {
t.Fatal(err)
}
ok, err := ms.Exists(dgst)
ok, err := ms.Exists(ctx, dgst)
if err != nil {
t.Fatal(err)
}
@ -625,11 +628,29 @@ func TestManifestFetch(t *testing.T) {
t.Fatal("Manifest does not exist")
}
manifest, err := ms.Get(dgst)
manifest, err := ms.Get(ctx, dgst)
if err != nil {
t.Fatal(err)
}
if err := checkEqualManifest(manifest, m1); err != nil {
v1manifest, ok := manifest.(*schema1.SignedManifest)
if !ok {
t.Fatalf("Unexpected manifest type from Get: %T", manifest)
}
if err := checkEqualManifest(v1manifest, m1); err != nil {
t.Fatal(err)
}
manifest, err = ms.Get(ctx, dgst, WithTag("latest"))
if err != nil {
t.Fatal(err)
}
v1manifest, ok = manifest.(*schema1.SignedManifest)
if !ok {
t.Fatalf("Unexpected manifest type from Get: %T", manifest)
}
if err = checkEqualManifest(v1manifest, m1); err != nil {
t.Fatal(err)
}
}
@ -643,17 +664,22 @@ func TestManifestFetchWithEtag(t *testing.T) {
e, c := testServer(m)
defer c()
r, err := NewRepository(context.Background(), repo, e, nil)
ctx := context.Background()
r, err := NewRepository(ctx, repo, e, nil)
if err != nil {
t.Fatal(err)
}
ctx := context.Background()
ms, err := r.Manifests(ctx)
if err != nil {
t.Fatal(err)
}
_, err = ms.GetByTag("latest", AddEtagToTag("latest", d1.String()))
clientManifestService, ok := ms.(*manifests)
if !ok {
panic("wrong type for client manifest service")
}
_, err = clientManifestService.Get(ctx, d1, WithTag("latest"), AddEtagToTag("latest", d1.String()))
if err != distribution.ErrManifestNotModified {
t.Fatal(err)
}
@ -690,10 +716,10 @@ func TestManifestDelete(t *testing.T) {
t.Fatal(err)
}
if err := ms.Delete(dgst1); err != nil {
if err := ms.Delete(ctx, dgst1); err != nil {
t.Fatal(err)
}
if err := ms.Delete(dgst2); err == nil {
if err := ms.Delete(ctx, dgst2); err == nil {
t.Fatal("Expected error deleting unknown manifest")
}
// TODO(dmcgowan): Check for specific unknown error
@ -702,12 +728,17 @@ func TestManifestDelete(t *testing.T) {
func TestManifestPut(t *testing.T) {
repo := "test.example.com/repo/delete"
m1, dgst, _ := newRandomSchemaV1Manifest(repo, "other", 6)
_, payload, err := m1.Payload()
if err != nil {
t.Fatal(err)
}
var m testutil.RequestResponseMap
m = append(m, testutil.RequestResponseMapping{
Request: testutil.Request{
Method: "PUT",
Route: "/v2/" + repo + "/manifests/other",
Body: m1.Raw,
Body: payload,
},
Response: testutil.Response{
StatusCode: http.StatusAccepted,
@ -731,7 +762,7 @@ func TestManifestPut(t *testing.T) {
t.Fatal(err)
}
if err := ms.Put(m1); err != nil {
if _, err := ms.Put(ctx, m1, WithTag(m1.Tag)); err != nil {
t.Fatal(err)
}
@ -751,21 +782,22 @@ func TestManifestTags(t *testing.T) {
}
`))
var m testutil.RequestResponseMap
m = append(m, testutil.RequestResponseMapping{
Request: testutil.Request{
Method: "GET",
Route: "/v2/" + repo + "/tags/list",
},
Response: testutil.Response{
StatusCode: http.StatusOK,
Body: tagsList,
Headers: http.Header(map[string][]string{
"Content-Length": {fmt.Sprint(len(tagsList))},
"Last-Modified": {time.Now().Add(-1 * time.Second).Format(time.ANSIC)},
}),
},
})
for i := 0; i < 3; i++ {
m = append(m, testutil.RequestResponseMapping{
Request: testutil.Request{
Method: "GET",
Route: "/v2/" + repo + "/tags/list",
},
Response: testutil.Response{
StatusCode: http.StatusOK,
Body: tagsList,
Headers: http.Header(map[string][]string{
"Content-Length": {fmt.Sprint(len(tagsList))},
"Last-Modified": {time.Now().Add(-1 * time.Second).Format(time.ANSIC)},
}),
},
})
}
e, c := testServer(m)
defer c()
@ -773,22 +805,29 @@ func TestManifestTags(t *testing.T) {
if err != nil {
t.Fatal(err)
}
ctx := context.Background()
ms, err := r.Manifests(ctx)
tagService := r.Tags(ctx)
tags, err := tagService.All(ctx)
if err != nil {
t.Fatal(err)
}
tags, err := ms.Tags()
if err != nil {
t.Fatal(err)
}
if len(tags) != 3 {
t.Fatalf("Wrong number of tags returned: %d, expected 3", len(tags))
}
// TODO(dmcgowan): Check array
expected := map[string]struct{}{
"tag1": {},
"tag2": {},
"funtag": {},
}
for _, t := range tags {
delete(expected, t)
}
if len(expected) != 0 {
t.Fatalf("unexpected tags returned: %v", expected)
}
// TODO(dmcgowan): Check for error cases
}
@ -821,7 +860,7 @@ func TestManifestUnauthorized(t *testing.T) {
t.Fatal(err)
}
_, err = ms.Get(dgst)
_, err = ms.Get(ctx, dgst)
if err == nil {
t.Fatal("Expected error fetching manifest")
}

149
docs/handlers/api_test.go
View file

@ -871,19 +871,15 @@ func testManifestAPI(t *testing.T, env *testEnv, args manifestArgs) (*testEnv, m
t.Fatalf("unexpected error signing manifest: %v", err)
}
payload, err := signedManifest.Payload()
checkErr(t, err, "getting manifest payload")
dgst := digest.FromBytes(payload)
dgst := digest.FromBytes(signedManifest.Canonical)
args.signedManifest = signedManifest
args.dgst = dgst
manifestDigestURL, err := env.builder.BuildManifestURL(imageName, dgst.String())
checkErr(t, err, "building manifest url")
resp = putManifest(t, "putting signed manifest", manifestURL, signedManifest)
checkResponse(t, "putting signed manifest", resp, http.StatusCreated)
resp = putManifest(t, "putting signed manifest no error", manifestURL, signedManifest)
checkResponse(t, "putting signed manifest no error", resp, http.StatusCreated)
checkHeaders(t, resp, http.Header{
"Location": []string{manifestDigestURL},
"Docker-Content-Digest": []string{dgst.String()},
@ -914,11 +910,12 @@ func testManifestAPI(t *testing.T, env *testEnv, args manifestArgs) (*testEnv, m
var fetchedManifest schema1.SignedManifest
dec := json.NewDecoder(resp.Body)
if err := dec.Decode(&fetchedManifest); err != nil {
t.Fatalf("error decoding fetched manifest: %v", err)
}
if !bytes.Equal(fetchedManifest.Raw, signedManifest.Raw) {
if !bytes.Equal(fetchedManifest.Canonical, signedManifest.Canonical) {
t.Fatalf("manifests do not match")
}
@ -940,10 +937,55 @@ func testManifestAPI(t *testing.T, env *testEnv, args manifestArgs) (*testEnv, m
t.Fatalf("error decoding fetched manifest: %v", err)
}
if !bytes.Equal(fetchedManifestByDigest.Raw, signedManifest.Raw) {
if !bytes.Equal(fetchedManifestByDigest.Canonical, signedManifest.Canonical) {
t.Fatalf("manifests do not match")
}
// check signature was roundtripped
signatures, err := fetchedManifestByDigest.Signatures()
if err != nil {
t.Fatal(err)
}
if len(signatures) != 1 {
t.Fatalf("expected 1 signature from manifest, got: %d", len(signatures))
}
// Re-sign, push and pull the same digest
sm2, err := schema1.Sign(&fetchedManifestByDigest.Manifest, env.pk)
if err != nil {
t.Fatal(err)
}
resp = putManifest(t, "re-putting signed manifest", manifestDigestURL, sm2)
checkResponse(t, "re-putting signed manifest", resp, http.StatusCreated)
resp, err = http.Get(manifestDigestURL)
checkErr(t, err, "re-fetching manifest by digest")
defer resp.Body.Close()
checkResponse(t, "re-fetching uploaded manifest", resp, http.StatusOK)
checkHeaders(t, resp, http.Header{
"Docker-Content-Digest": []string{dgst.String()},
"ETag": []string{fmt.Sprintf(`"%s"`, dgst)},
})
dec = json.NewDecoder(resp.Body)
if err := dec.Decode(&fetchedManifestByDigest); err != nil {
t.Fatalf("error decoding fetched manifest: %v", err)
}
// check two signatures were roundtripped
signatures, err = fetchedManifestByDigest.Signatures()
if err != nil {
t.Fatal(err)
}
if len(signatures) != 2 {
t.Fatalf("expected 2 signature from manifest, got: %d", len(signatures))
}
// Get by name with etag, gives 304
etag := resp.Header.Get("Etag")
req, err := http.NewRequest("GET", manifestURL, nil)
@ -956,7 +998,7 @@ func testManifestAPI(t *testing.T, env *testEnv, args manifestArgs) (*testEnv, m
t.Fatalf("Error constructing request: %s", err)
}
checkResponse(t, "fetching layer with etag", resp, http.StatusNotModified)
checkResponse(t, "fetching manifest by name with etag", resp, http.StatusNotModified)
// Get by digest with etag, gives 304
req, err = http.NewRequest("GET", manifestDigestURL, nil)
@ -969,7 +1011,7 @@ func testManifestAPI(t *testing.T, env *testEnv, args manifestArgs) (*testEnv, m
t.Fatalf("Error constructing request: %s", err)
}
checkResponse(t, "fetching layer with etag", resp, http.StatusNotModified)
checkResponse(t, "fetching manifest by dgst with etag", resp, http.StatusNotModified)
// Ensure that the tag is listed.
resp, err = http.Get(tagsURL)
@ -1143,8 +1185,13 @@ func newTestEnvWithConfig(t *testing.T, config *configuration.Configuration) *te
func putManifest(t *testing.T, msg, url string, v interface{}) *http.Response {
var body []byte
if sm, ok := v.(*schema1.SignedManifest); ok {
body = sm.Raw
_, pl, err := sm.Payload()
if err != nil {
t.Fatalf("error getting payload: %v", err)
}
body = pl
} else {
var err error
body, err = json.MarshalIndent(v, "", " ")
@ -1435,7 +1482,7 @@ func checkErr(t *testing.T, err error, msg string) {
}
}
func createRepository(env *testEnv, t *testing.T, imageName string, tag string) {
func createRepository(env *testEnv, t *testing.T, imageName string, tag string) digest.Digest {
unsignedManifest := &schema1.Manifest{
Versioned: manifest.Versioned{
SchemaVersion: 1,
@ -1459,7 +1506,6 @@ func createRepository(env *testEnv, t *testing.T, imageName string, tag string)
for i := range unsignedManifest.FSLayers {
rs, dgstStr, err := testutil.CreateRandomTarFile()
if err != nil {
t.Fatalf("error creating random layer %d: %v", i, err)
}
@ -1477,20 +1523,22 @@ func createRepository(env *testEnv, t *testing.T, imageName string, tag string)
t.Fatalf("unexpected error signing manifest: %v", err)
}
payload, err := signedManifest.Payload()
checkErr(t, err, "getting manifest payload")
dgst := digest.FromBytes(signedManifest.Canonical)
dgst := digest.FromBytes(payload)
manifestDigestURL, err := env.builder.BuildManifestURL(imageName, dgst.String())
// Create this repository by tag to ensure the tag mapping is made in the registry
manifestDigestURL, err := env.builder.BuildManifestURL(imageName, tag)
checkErr(t, err, "building manifest url")
location, err := env.builder.BuildManifestURL(imageName, dgst.String())
checkErr(t, err, "building location URL")
resp := putManifest(t, "putting signed manifest", manifestDigestURL, signedManifest)
checkResponse(t, "putting signed manifest", resp, http.StatusCreated)
checkHeaders(t, resp, http.Header{
"Location": []string{manifestDigestURL},
"Location": []string{location},
"Docker-Content-Digest": []string{dgst.String()},
})
return dgst
}
// Test mutation operations on a registry configured as a cache. Ensure that they return
@ -1577,3 +1625,64 @@ func TestCheckContextNotifier(t *testing.T) {
t.Fatalf("wrong status code - expected 200, got %d", resp.StatusCode)
}
}
func TestProxyManifestGetByTag(t *testing.T) {
truthConfig := configuration.Configuration{
Storage: configuration.Storage{
"inmemory": configuration.Parameters{},
},
}
truthConfig.HTTP.Headers = headerConfig
imageName := "foo/bar"
tag := "latest"
truthEnv := newTestEnvWithConfig(t, &truthConfig)
// create a repository in the truth registry
dgst := createRepository(truthEnv, t, imageName, tag)
proxyConfig := configuration.Configuration{
Storage: configuration.Storage{
"inmemory": configuration.Parameters{},
},
Proxy: configuration.Proxy{
RemoteURL: truthEnv.server.URL,
},
}
proxyConfig.HTTP.Headers = headerConfig
proxyEnv := newTestEnvWithConfig(t, &proxyConfig)
manifestDigestURL, err := proxyEnv.builder.BuildManifestURL(imageName, dgst.String())
checkErr(t, err, "building manifest url")
resp, err := http.Get(manifestDigestURL)
checkErr(t, err, "fetching manifest from proxy by digest")
defer resp.Body.Close()
manifestTagURL, err := proxyEnv.builder.BuildManifestURL(imageName, tag)
checkErr(t, err, "building manifest url")
resp, err = http.Get(manifestTagURL)
checkErr(t, err, "fetching manifest from proxy by tag")
defer resp.Body.Close()
checkResponse(t, "fetching manifest from proxy by tag", resp, http.StatusOK)
checkHeaders(t, resp, http.Header{
"Docker-Content-Digest": []string{dgst.String()},
})
// Create another manifest in the remote with the same image/tag pair
newDigest := createRepository(truthEnv, t, imageName, tag)
if dgst == newDigest {
t.Fatalf("non-random test data")
}
// fetch it with the same proxy URL as before. Ensure the updated content is at the same tag
resp, err = http.Get(manifestTagURL)
checkErr(t, err, "fetching manifest from proxy by tag")
defer resp.Body.Close()
checkResponse(t, "fetching manifest from proxy by tag", resp, http.StatusOK)
checkHeaders(t, resp, http.Header{
"Docker-Content-Digest": []string{newDigest.String()},
})
}

114
docs/handlers/images.go
View file

@ -2,19 +2,15 @@ package handlers
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
"strings"
"github.com/docker/distribution"
ctxu "github.com/docker/distribution/context"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/registry/api/errcode"
"github.com/docker/distribution/registry/api/v2"
"github.com/gorilla/handlers"
"golang.org/x/net/context"
)
// imageManifestDispatcher takes the request context and builds the
@ -33,7 +29,8 @@ func imageManifestDispatcher(ctx *Context, r *http.Request) http.Handler {
}
mhandler := handlers.MethodHandler{
"GET": http.HandlerFunc(imageManifestHandler.GetImageManifest),
"GET": http.HandlerFunc(imageManifestHandler.GetImageManifest),
"HEAD": http.HandlerFunc(imageManifestHandler.GetImageManifest),
}
if !ctx.readOnly {
@ -54,6 +51,8 @@ type imageManifestHandler struct {
}
// GetImageManifest fetches the image manifest from the storage backend, if it exists.
// todo(richardscothern): this assumes v2 schema 1 manifests for now but in the future
// get the version from the Accept HTTP header
func (imh *imageManifestHandler) GetImageManifest(w http.ResponseWriter, r *http.Request) {
ctxu.GetLogger(imh).Debug("GetImageManifest")
manifests, err := imh.Repository.Manifests(imh)
@ -62,42 +61,38 @@ func (imh *imageManifestHandler) GetImageManifest(w http.ResponseWriter, r *http
return
}
var sm *schema1.SignedManifest
var manifest distribution.Manifest
if imh.Tag != "" {
sm, err = manifests.GetByTag(imh.Tag)
} else {
if etagMatch(r, imh.Digest.String()) {
w.WriteHeader(http.StatusNotModified)
tags := imh.Repository.Tags(imh)
desc, err := tags.Get(imh, imh.Tag)
if err != nil {
imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithDetail(err))
return
}
sm, err = manifests.Get(imh.Digest)
imh.Digest = desc.Digest
}
if etagMatch(r, imh.Digest.String()) {
w.WriteHeader(http.StatusNotModified)
return
}
manifest, err = manifests.Get(imh, imh.Digest)
if err != nil {
imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithDetail(err))
return
}
// Get the digest, if we don't already have it.
if imh.Digest == "" {
dgst, err := digestManifest(imh, sm)
if err != nil {
imh.Errors = append(imh.Errors, v2.ErrorCodeDigestInvalid.WithDetail(err))
return
}
if etagMatch(r, dgst.String()) {
w.WriteHeader(http.StatusNotModified)
return
}
imh.Digest = dgst
ct, p, err := manifest.Payload()
if err != nil {
return
}
w.Header().Set("Content-Type", "application/json; charset=utf-8")
w.Header().Set("Content-Length", fmt.Sprint(len(sm.Raw)))
w.Header().Set("Content-Type", ct)
w.Header().Set("Content-Length", fmt.Sprint(len(p)))
w.Header().Set("Docker-Content-Digest", imh.Digest.String())
w.Header().Set("Etag", fmt.Sprintf(`"%s"`, imh.Digest))
w.Write(sm.Raw)
w.Write(p)
}
func etagMatch(r *http.Request, etag string) bool {
@ -109,7 +104,7 @@ func etagMatch(r *http.Request, etag string) bool {
return false
}
// PutImageManifest validates and stores and image in the registry.
// PutImageManifest validates and stores an image in the registry.
func (imh *imageManifestHandler) PutImageManifest(w http.ResponseWriter, r *http.Request) {
ctxu.GetLogger(imh).Debug("PutImageManifest")
manifests, err := imh.Repository.Manifests(imh)
@ -124,39 +119,28 @@ func (imh *imageManifestHandler) PutImageManifest(w http.ResponseWriter, r *http
return
}
var manifest schema1.SignedManifest
if err := json.Unmarshal(jsonBuf.Bytes(), &manifest); err != nil {
mediaType := r.Header.Get("Content-Type")
manifest, desc, err := distribution.UnmarshalManifest(mediaType, jsonBuf.Bytes())
if err != nil {
imh.Errors = append(imh.Errors, v2.ErrorCodeManifestInvalid.WithDetail(err))
return
}
dgst, err := digestManifest(imh, &manifest)
if err != nil {
imh.Errors = append(imh.Errors, v2.ErrorCodeDigestInvalid.WithDetail(err))
return
}
// Validate manifest tag or digest matches payload
if imh.Tag != "" {
if manifest.Tag != imh.Tag {
ctxu.GetLogger(imh).Errorf("invalid tag on manifest payload: %q != %q", manifest.Tag, imh.Tag)
imh.Errors = append(imh.Errors, v2.ErrorCodeTagInvalid)
return
}
imh.Digest = dgst
} else if imh.Digest != "" {
if dgst != imh.Digest {
ctxu.GetLogger(imh).Errorf("payload digest does match: %q != %q", dgst, imh.Digest)
if imh.Digest != "" {
if desc.Digest != imh.Digest {
ctxu.GetLogger(imh).Errorf("payload digest does match: %q != %q", desc.Digest, imh.Digest)
imh.Errors = append(imh.Errors, v2.ErrorCodeDigestInvalid)
return
}
} else if imh.Tag != "" {
imh.Digest = desc.Digest
} else {
imh.Errors = append(imh.Errors, v2.ErrorCodeTagInvalid.WithDetail("no tag or digest specified"))
return
}
if err := manifests.Put(&manifest); err != nil {
_, err = manifests.Put(imh, manifest)
if err != nil {
// TODO(stevvooe): These error handling switches really need to be
// handled by an app global mapper.
if err == distribution.ErrUnsupported {
@ -188,6 +172,17 @@ func (imh *imageManifestHandler) PutImageManifest(w http.ResponseWriter, r *http
return
}
// Tag this manifest
if imh.Tag != "" {
tags := imh.Repository.Tags(imh)
err = tags.Tag(imh, imh.Tag, desc)
if err != nil {
imh.Errors = append(imh.Errors, errcode.ErrorCodeUnknown.WithDetail(err))
return
}
}
// Construct a canonical url for the uploaded manifest.
location, err := imh.urlBuilder.BuildManifestURL(imh.Repository.Name(), imh.Digest.String())
if err != nil {
@ -212,7 +207,7 @@ func (imh *imageManifestHandler) DeleteImageManifest(w http.ResponseWriter, r *h
return
}
err = manifests.Delete(imh.Digest)
err = manifests.Delete(imh, imh.Digest)
if err != nil {
switch err {
case digest.ErrDigestUnsupported:
@ -233,22 +228,3 @@ func (imh *imageManifestHandler) DeleteImageManifest(w http.ResponseWriter, r *h
w.WriteHeader(http.StatusAccepted)
}
// digestManifest takes a digest of the given manifest. This belongs somewhere
// better but we'll wait for a refactoring cycle to find that real somewhere.
func digestManifest(ctx context.Context, sm *schema1.SignedManifest) (digest.Digest, error) {
p, err := sm.Payload()
if err != nil {
if !strings.Contains(err.Error(), "missing signature key") {
ctxu.GetLogger(ctx).Errorf("error getting manifest payload: %v", err)
return "", err
}
// NOTE(stevvooe): There are no signatures but we still have a
// payload. The request will fail later but this is not the
// responsibility of this part of the code.
p = sm.Raw
}
return digest.FromBytes(p), nil
}

8
docs/handlers/tags.go
View file

@ -34,13 +34,9 @@ type tagsAPIResponse struct {
// GetTags returns a json list of tags for a specific image name.
func (th *tagsHandler) GetTags(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
manifests, err := th.Repository.Manifests(th)
if err != nil {
th.Errors = append(th.Errors, err)
return
}
tags, err := manifests.Tags()
tagService := th.Repository.Tags(th)
tags, err := tagService.All(th)
if err != nil {
switch err := err.(type) {
case distribution.ErrRepositoryUnknown:

137
docs/proxy/proxymanifeststore.go
View file

@ -6,8 +6,6 @@ import (
"github.com/docker/distribution"
"github.com/docker/distribution/context"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/registry/client"
"github.com/docker/distribution/registry/proxy/scheduler"
)
@ -24,8 +22,8 @@ type proxyManifestStore struct {
var _ distribution.ManifestService = &proxyManifestStore{}
func (pms proxyManifestStore) Exists(dgst digest.Digest) (bool, error) {
exists, err := pms.localManifests.Exists(dgst)
func (pms proxyManifestStore) Exists(ctx context.Context, dgst digest.Digest) (bool, error) {
exists, err := pms.localManifests.Exists(ctx, dgst)
if err != nil {
return false, err
}
@ -33,117 +31,56 @@ func (pms proxyManifestStore) Exists(dgst digest.Digest) (bool, error) {
return true, nil
}
return pms.remoteManifests.Exists(dgst)
return pms.remoteManifests.Exists(ctx, dgst)
}
func (pms proxyManifestStore) Get(dgst digest.Digest) (*schema1.SignedManifest, error) {
sm, err := pms.localManifests.Get(dgst)
if err == nil {
proxyMetrics.ManifestPush(uint64(len(sm.Raw)))
return sm, err
func (pms proxyManifestStore) Get(ctx context.Context, dgst digest.Digest, options ...distribution.ManifestServiceOption) (distribution.Manifest, error) {
// At this point `dgst` was either specified explicitly, or returned by the
// tagstore with the most recent association.
var fromRemote bool
manifest, err := pms.localManifests.Get(ctx, dgst, options...)
if err != nil {
manifest, err = pms.remoteManifests.Get(ctx, dgst, options...)
if err != nil {
return nil, err
}
fromRemote = true
}
sm, err = pms.remoteManifests.Get(dgst)
_, payload, err := manifest.Payload()
if err != nil {
return nil, err
}
proxyMetrics.ManifestPull(uint64(len(sm.Raw)))
err = pms.localManifests.Put(sm)
if err != nil {
return nil, err
proxyMetrics.ManifestPush(uint64(len(payload)))
if fromRemote {
proxyMetrics.ManifestPull(uint64(len(payload)))
_, err = pms.localManifests.Put(ctx, manifest)
if err != nil {
return nil, err
}
// Schedule the repo for removal
pms.scheduler.AddManifest(pms.repositoryName, repositoryTTL)
// Ensure the manifest blob is cleaned up
pms.scheduler.AddBlob(dgst.String(), repositoryTTL)
}
// Schedule the repo for removal
pms.scheduler.AddManifest(pms.repositoryName, repositoryTTL)
// Ensure the manifest blob is cleaned up
pms.scheduler.AddBlob(dgst.String(), repositoryTTL)
proxyMetrics.ManifestPush(uint64(len(sm.Raw)))
return sm, err
return manifest, err
}
func (pms proxyManifestStore) Tags() ([]string, error) {
return pms.localManifests.Tags()
func (pms proxyManifestStore) Put(ctx context.Context, manifest distribution.Manifest, options ...distribution.ManifestServiceOption) (digest.Digest, error) {
var d digest.Digest
return d, distribution.ErrUnsupported
}
func (pms proxyManifestStore) ExistsByTag(tag string) (bool, error) {
exists, err := pms.localManifests.ExistsByTag(tag)
if err != nil {
return false, err
}
if exists {
return true, nil
}
return pms.remoteManifests.ExistsByTag(tag)
}
func (pms proxyManifestStore) GetByTag(tag string, options ...distribution.ManifestServiceOption) (*schema1.SignedManifest, error) {
var localDigest digest.Digest
localManifest, err := pms.localManifests.GetByTag(tag, options...)
switch err.(type) {
case distribution.ErrManifestUnknown, distribution.ErrManifestUnknownRevision:
goto fromremote
case nil:
break
default:
return nil, err
}
localDigest, err = manifestDigest(localManifest)
if err != nil {
return nil, err
}
fromremote:
var sm *schema1.SignedManifest
sm, err = pms.remoteManifests.GetByTag(tag, client.AddEtagToTag(tag, localDigest.String()))
if err != nil && err != distribution.ErrManifestNotModified {
return nil, err
}
if err == distribution.ErrManifestNotModified {
context.GetLogger(pms.ctx).Debugf("Local manifest for %q is latest, dgst=%s", tag, localDigest.String())
return localManifest, nil
}
context.GetLogger(pms.ctx).Debugf("Updated manifest for %q, dgst=%s", tag, localDigest.String())
err = pms.localManifests.Put(sm)
if err != nil {
return nil, err
}
dgst, err := manifestDigest(sm)
if err != nil {
return nil, err
}
pms.scheduler.AddBlob(dgst.String(), repositoryTTL)
pms.scheduler.AddManifest(pms.repositoryName, repositoryTTL)
proxyMetrics.ManifestPull(uint64(len(sm.Raw)))
proxyMetrics.ManifestPush(uint64(len(sm.Raw)))
return sm, err
}
func manifestDigest(sm *schema1.SignedManifest) (digest.Digest, error) {
payload, err := sm.Payload()
if err != nil {
return "", err
}
return digest.FromBytes(payload), nil
}
func (pms proxyManifestStore) Put(manifest *schema1.SignedManifest) error {
func (pms proxyManifestStore) Delete(ctx context.Context, dgst digest.Digest) error {
return distribution.ErrUnsupported
}
func (pms proxyManifestStore) Delete(dgst digest.Digest) error {
return distribution.ErrUnsupported
/*func (pms proxyManifestStore) Enumerate(ctx context.Context, manifests []distribution.Manifest, last distribution.Manifest) (n int, err error) {
return 0, distribution.ErrUnsupported
}
*/

64
docs/proxy/proxymanifeststore_test.go
View file

@ -37,40 +37,31 @@ func (te manifestStoreTestEnv) RemoteStats() *map[string]int {
return &rs
}
func (sm statsManifest) Delete(dgst digest.Digest) error {
func (sm statsManifest) Delete(ctx context.Context, dgst digest.Digest) error {
sm.stats["delete"]++
return sm.manifests.Delete(dgst)
return sm.manifests.Delete(ctx, dgst)
}
func (sm statsManifest) Exists(dgst digest.Digest) (bool, error) {
func (sm statsManifest) Exists(ctx context.Context, dgst digest.Digest) (bool, error) {
sm.stats["exists"]++
return sm.manifests.Exists(dgst)
return sm.manifests.Exists(ctx, dgst)
}
func (sm statsManifest) ExistsByTag(tag string) (bool, error) {
sm.stats["existbytag"]++
return sm.manifests.ExistsByTag(tag)
}
func (sm statsManifest) Get(dgst digest.Digest) (*schema1.SignedManifest, error) {
func (sm statsManifest) Get(ctx context.Context, dgst digest.Digest, options ...distribution.ManifestServiceOption) (distribution.Manifest, error) {
sm.stats["get"]++
return sm.manifests.Get(dgst)
return sm.manifests.Get(ctx, dgst)
}
func (sm statsManifest) GetByTag(tag string, options ...distribution.ManifestServiceOption) (*schema1.SignedManifest, error) {
sm.stats["getbytag"]++
return sm.manifests.GetByTag(tag, options...)
}
func (sm statsManifest) Put(manifest *schema1.SignedManifest) error {
func (sm statsManifest) Put(ctx context.Context, manifest distribution.Manifest, options ...distribution.ManifestServiceOption) (digest.Digest, error) {
sm.stats["put"]++
return sm.manifests.Put(manifest)
return sm.manifests.Put(ctx, manifest)
}
func (sm statsManifest) Tags() ([]string, error) {
sm.stats["tags"]++
return sm.manifests.Tags()
/*func (sm statsManifest) Enumerate(ctx context.Context, manifests []distribution.Manifest, last distribution.Manifest) (n int, err error) {
sm.stats["enumerate"]++
return sm.manifests.Enumerate(ctx, manifests, last)
}
*/
func newManifestStoreTestEnv(t *testing.T, name, tag string) *manifestStoreTestEnv {
ctx := context.Background()
@ -169,15 +160,12 @@ func populateRepo(t *testing.T, ctx context.Context, repository distribution.Rep
if err != nil {
t.Fatalf(err.Error())
}
ms.Put(sm)
dgst, err := ms.Put(ctx, sm)
if err != nil {
t.Fatalf("unexpected errors putting manifest: %v", err)
}
pl, err := sm.Payload()
if err != nil {
t.Fatal(err)
}
return digest.FromBytes(pl), nil
return dgst, nil
}
// TestProxyManifests contains basic acceptance tests
@ -189,8 +177,9 @@ func TestProxyManifests(t *testing.T) {
localStats := env.LocalStats()
remoteStats := env.RemoteStats()
ctx := context.Background()
// Stat - must check local and remote
exists, err := env.manifests.ExistsByTag("latest")
exists, err := env.manifests.Exists(ctx, env.manifestDigest)
if err != nil {
t.Fatalf("Error checking existance")
}
@ -198,15 +187,16 @@ func TestProxyManifests(t *testing.T) {
t.Errorf("Unexpected non-existant manifest")
}
if (*localStats)["existbytag"] != 1 && (*remoteStats)["existbytag"] != 1 {
t.Errorf("Unexpected exists count")
if (*localStats)["exists"] != 1 && (*remoteStats)["exists"] != 1 {
t.Errorf("Unexpected exists count : \n%v \n%v", localStats, remoteStats)
}
// Get - should succeed and pull manifest into local
_, err = env.manifests.Get(env.manifestDigest)
_, err = env.manifests.Get(ctx, env.manifestDigest)
if err != nil {
t.Fatal(err)
}
if (*localStats)["get"] != 1 && (*remoteStats)["get"] != 1 {
t.Errorf("Unexpected get count")
}
@ -216,7 +206,7 @@ func TestProxyManifests(t *testing.T) {
}
// Stat - should only go to local
exists, err = env.manifests.ExistsByTag("latest")
exists, err = env.manifests.Exists(ctx, env.manifestDigest)
if err != nil {
t.Fatal(err)
}
@ -224,19 +214,21 @@ func TestProxyManifests(t *testing.T) {
t.Errorf("Unexpected non-existant manifest")
}
if (*localStats)["existbytag"] != 2 && (*remoteStats)["existbytag"] != 1 {
if (*localStats)["exists"] != 2 && (*remoteStats)["exists"] != 1 {
t.Errorf("Unexpected exists count")
}
// Get - should get from remote, to test freshness
_, err = env.manifests.Get(env.manifestDigest)
_, err = env.manifests.Get(ctx, env.manifestDigest)
if err != nil {
t.Fatal(err)
}
if (*remoteStats)["get"] != 2 && (*remoteStats)["existsbytag"] != 1 && (*localStats)["put"] != 1 {
if (*remoteStats)["get"] != 2 && (*remoteStats)["exists"] != 1 && (*localStats)["put"] != 1 {
t.Errorf("Unexpected get count")
}
}
func TestProxyTagService(t *testing.T) {
}

23
docs/proxy/proxyregistry.go
View file

@ -42,6 +42,7 @@ func NewRegistryPullThroughCache(ctx context.Context, registry distribution.Name
s.OnManifestExpire(func(repoName string) error {
return v.RemoveRepository(repoName)
})
err = s.Start()
if err != nil {
return nil, err
@ -78,7 +79,7 @@ func (pr *proxyingRegistry) Repository(ctx context.Context, name string) (distri
if err != nil {
return nil, err
}
localManifests, err := localRepo.Manifests(ctx, storage.SkipLayerVerification)
localManifests, err := localRepo.Manifests(ctx, storage.SkipLayerVerification())
if err != nil {
return nil, err
}
@ -106,8 +107,11 @@ func (pr *proxyingRegistry) Repository(ctx context.Context, name string) (distri
ctx: ctx,
scheduler: pr.scheduler,
},
name: name,
signatures: localRepo.Signatures(),
name: name,
tags: proxyTagService{
localTags: localRepo.Tags(ctx),
remoteTags: remoteRepo.Tags(ctx),
},
}, nil
}
@ -115,14 +119,13 @@ func (pr *proxyingRegistry) Repository(ctx context.Context, name string) (distri
// locally, or pulling it through from a remote and caching it locally if it doesn't
// already exist
type proxiedRepository struct {
blobStore distribution.BlobStore
manifests distribution.ManifestService
name string
signatures distribution.SignatureService
blobStore distribution.BlobStore
manifests distribution.ManifestService
name string
tags distribution.TagService
}
func (pr *proxiedRepository) Manifests(ctx context.Context, options ...distribution.ManifestServiceOption) (distribution.ManifestService, error) {
// options
return pr.manifests, nil
}
@ -134,6 +137,6 @@ func (pr *proxiedRepository) Name() string {
return pr.name
}
func (pr *proxiedRepository) Signatures() distribution.SignatureService {
return pr.signatures
func (pr *proxiedRepository) Tags(ctx context.Context) distribution.TagService {
return pr.tags
}

58
docs/proxy/proxytagservice.go Normal file
View file

@ -0,0 +1,58 @@
package proxy
import (
"github.com/docker/distribution"
"github.com/docker/distribution/context"
)
// proxyTagService supports local and remote lookup of tags.
type proxyTagService struct {
localTags distribution.TagService
remoteTags distribution.TagService
}
var _ distribution.TagService = proxyTagService{}
// Get attempts to get the most recent digest for the tag by checking the remote
// tag service first and then caching it locally. If the remote is unavailable
// the local association is returned
func (pt proxyTagService) Get(ctx context.Context, tag string) (distribution.Descriptor, error) {
desc, err := pt.remoteTags.Get(ctx, tag)
if err == nil {
err := pt.localTags.Tag(ctx, tag, desc)
if err != nil {
return distribution.Descriptor{}, err
}
return desc, nil
}
desc, err = pt.localTags.Get(ctx, tag)
if err != nil {
return distribution.Descriptor{}, err
}
return desc, nil
}
func (pt proxyTagService) Tag(ctx context.Context, tag string, desc distribution.Descriptor) error {
return distribution.ErrUnsupported
}
func (pt proxyTagService) Untag(ctx context.Context, tag string) error {
err := pt.localTags.Untag(ctx, tag)
if err != nil {
return err
}
return nil
}
func (pt proxyTagService) All(ctx context.Context) ([]string, error) {
tags, err := pt.remoteTags.All(ctx)
if err == nil {
return tags, err
}
return pt.localTags.All(ctx)
}
func (pt proxyTagService) Lookup(ctx context.Context, digest distribution.Descriptor) ([]string, error) {
return []string{}, distribution.ErrUnsupported
}

164
docs/proxy/proxytagservice_test.go Normal file
View file

@ -0,0 +1,164 @@
package proxy
import (
"sort"
"sync"
"testing"
"github.com/docker/distribution"
"github.com/docker/distribution/context"
)
type mockTagStore struct {
mapping map[string]distribution.Descriptor
sync.Mutex
}
var _ distribution.TagService = &mockTagStore{}
func (m *mockTagStore) Get(ctx context.Context, tag string) (distribution.Descriptor, error) {
m.Lock()
defer m.Unlock()
if d, ok := m.mapping[tag]; ok {
return d, nil
}
return distribution.Descriptor{}, distribution.ErrTagUnknown{}
}
func (m *mockTagStore) Tag(ctx context.Context, tag string, desc distribution.Descriptor) error {
m.Lock()
defer m.Unlock()
m.mapping[tag] = desc
return nil
}
func (m *mockTagStore) Untag(ctx context.Context, tag string) error {
m.Lock()
defer m.Unlock()
if _, ok := m.mapping[tag]; ok {
delete(m.mapping, tag)
return nil
}
return distribution.ErrTagUnknown{}
}
func (m *mockTagStore) All(ctx context.Context) ([]string, error) {
m.Lock()
defer m.Unlock()
var tags []string
for tag := range m.mapping {
tags = append(tags, tag)
}
return tags, nil
}
func (m *mockTagStore) Lookup(ctx context.Context, digest distribution.Descriptor) ([]string, error) {
panic("not implemented")
}
func testProxyTagService(local, remote map[string]distribution.Descriptor) *proxyTagService {
if local == nil {
local = make(map[string]distribution.Descriptor)
}
if remote == nil {
remote = make(map[string]distribution.Descriptor)
}
return &proxyTagService{
localTags: &mockTagStore{mapping: local},
remoteTags: &mockTagStore{mapping: remote},
}
}
func TestGet(t *testing.T) {
remoteDesc := distribution.Descriptor{Size: 42}
remoteTag := "remote"
proxyTags := testProxyTagService(map[string]distribution.Descriptor{remoteTag: remoteDesc}, nil)
ctx := context.Background()
// Get pre-loaded tag
d, err := proxyTags.Get(ctx, remoteTag)
if err != nil {
t.Fatal(err)
}
if d != remoteDesc {
t.Fatal("unable to get put tag")
}
local, err := proxyTags.localTags.Get(ctx, remoteTag)
if err != nil {
t.Fatal("remote tag not pulled into store")
}
if local != remoteDesc {
t.Fatalf("unexpected descriptor pulled through")
}
// Manually overwrite remote tag
newRemoteDesc := distribution.Descriptor{Size: 43}
err = proxyTags.remoteTags.Tag(ctx, remoteTag, newRemoteDesc)
if err != nil {
t.Fatal(err)
}
d, err = proxyTags.Get(ctx, remoteTag)
if err != nil {
t.Fatal(err)
}
if d != newRemoteDesc {
t.Fatal("unable to get put tag")
}
_, err = proxyTags.localTags.Get(ctx, remoteTag)
if err != nil {
t.Fatal("remote tag not pulled into store")
}
// untag, ensure it's removed locally, but present in remote
err = proxyTags.Untag(ctx, remoteTag)
if err != nil {
t.Fatal(err)
}
_, err = proxyTags.localTags.Get(ctx, remoteTag)
if err == nil {
t.Fatalf("Expected error getting Untag'd tag")
}
_, err = proxyTags.remoteTags.Get(ctx, remoteTag)
if err != nil {
t.Fatalf("remote tag should not be untagged with proxyTag.Untag")
}
_, err = proxyTags.Get(ctx, remoteTag)
if err != nil {
t.Fatal("untagged tag should be pulled through")
}
// Add another tag. Ensure both tags appear in enumerate
err = proxyTags.remoteTags.Tag(ctx, "funtag", distribution.Descriptor{Size: 42})
if err != nil {
t.Fatal(err)
}
all, err := proxyTags.All(ctx)
if err != nil {
t.Fatal(err)
}
if len(all) != 2 {
t.Fatalf("Unexpected tag length returned from All() : %d ", len(all))
}
sort.Strings(all)
if all[0] != "funtag" && all[1] != "remote" {
t.Fatalf("Unexpected tags returned from All() : %v ", all)
}
}

159
docs/storage/manifeststore.go
View file

@ -1,6 +1,7 @@
package storage
import (
"encoding/json"
"fmt"
"github.com/docker/distribution"
@ -11,20 +12,21 @@ import (
"github.com/docker/libtrust"
)
// manifestStore is a storage driver based store for storing schema1 manifests.
type manifestStore struct {
repository *repository
revisionStore *revisionStore
tagStore *tagStore
blobStore *linkedBlobStore
ctx context.Context
signatures *signatureStore
skipDependencyVerification bool
}
var _ distribution.ManifestService = &manifestStore{}
func (ms *manifestStore) Exists(dgst digest.Digest) (bool, error) {
func (ms *manifestStore) Exists(ctx context.Context, dgst digest.Digest) (bool, error) {
context.GetLogger(ms.ctx).Debug("(*manifestStore).Exists")
_, err := ms.revisionStore.blobStore.Stat(ms.ctx, dgst)
_, err := ms.blobStore.Stat(ms.ctx, dgst)
if err != nil {
if err == distribution.ErrBlobUnknown {
return false, nil
@ -36,76 +38,131 @@ func (ms *manifestStore) Exists(dgst digest.Digest) (bool, error) {
return true, nil
}
func (ms *manifestStore) Get(dgst digest.Digest) (*schema1.SignedManifest, error) {
func (ms *manifestStore) Get(ctx context.Context, dgst digest.Digest, options ...distribution.ManifestServiceOption) (distribution.Manifest, error) {
context.GetLogger(ms.ctx).Debug("(*manifestStore).Get")
return ms.revisionStore.get(ms.ctx, dgst)
// Ensure that this revision is available in this repository.
_, err := ms.blobStore.Stat(ctx, dgst)
if err != nil {
if err == distribution.ErrBlobUnknown {
return nil, distribution.ErrManifestUnknownRevision{
Name: ms.repository.Name(),
Revision: dgst,
}
}
return nil, err
}
// TODO(stevvooe): Need to check descriptor from above to ensure that the
// mediatype is as we expect for the manifest store.
content, err := ms.blobStore.Get(ctx, dgst)
if err != nil {
if err == distribution.ErrBlobUnknown {
return nil, distribution.ErrManifestUnknownRevision{
Name: ms.repository.Name(),
Revision: dgst,
}
}
return nil, err
}
// Fetch the signatures for the manifest
signatures, err := ms.signatures.Get(dgst)
if err != nil {
return nil, err
}
jsig, err := libtrust.NewJSONSignature(content, signatures...)
if err != nil {
return nil, err
}
// Extract the pretty JWS
raw, err := jsig.PrettySignature("signatures")
if err != nil {
return nil, err
}
var sm schema1.SignedManifest
if err := json.Unmarshal(raw, &sm); err != nil {
return nil, err
}
return &sm, nil
}
// SkipLayerVerification allows a manifest to be Put before it's
// SkipLayerVerification allows a manifest to be Put before its
// layers are on the filesystem
func SkipLayerVerification(ms distribution.ManifestService) error {
if ms, ok := ms.(*manifestStore); ok {
func SkipLayerVerification() distribution.ManifestServiceOption {
return skipLayerOption{}
}
type skipLayerOption struct{}
func (o skipLayerOption) Apply(m distribution.ManifestService) error {
if ms, ok := m.(*manifestStore); ok {
ms.skipDependencyVerification = true
return nil
}
return fmt.Errorf("skip layer verification only valid for manifestStore")
}
func (ms *manifestStore) Put(manifest *schema1.SignedManifest) error {
func (ms *manifestStore) Put(ctx context.Context, manifest distribution.Manifest, options ...distribution.ManifestServiceOption) (digest.Digest, error) {
context.GetLogger(ms.ctx).Debug("(*manifestStore).Put")
if err := ms.verifyManifest(ms.ctx, manifest); err != nil {
return err
sm, ok := manifest.(*schema1.SignedManifest)
if !ok {
return "", fmt.Errorf("non-v1 manifest put to signed manifestStore: %T", manifest)
}
// Store the revision of the manifest
revision, err := ms.revisionStore.put(ms.ctx, manifest)
if err := ms.verifyManifest(ms.ctx, *sm); err != nil {
return "", err
}
mt := schema1.MediaTypeManifest
payload := sm.Canonical
revision, err := ms.blobStore.Put(ctx, mt, payload)
if err != nil {
return err
context.GetLogger(ctx).Errorf("error putting payload into blobstore: %v", err)
return "", err
}
// Now, tag the manifest
return ms.tagStore.tag(manifest.Tag, revision.Digest)
// Link the revision into the repository.
if err := ms.blobStore.linkBlob(ctx, revision); err != nil {
return "", err
}
// Grab each json signature and store them.
signatures, err := sm.Signatures()
if err != nil {
return "", err
}
if err := ms.signatures.Put(revision.Digest, signatures...); err != nil {
return "", err
}
return revision.Digest, nil
}
// Delete removes the revision of the specified manfiest.
func (ms *manifestStore) Delete(dgst digest.Digest) error {
func (ms *manifestStore) Delete(ctx context.Context, dgst digest.Digest) error {
context.GetLogger(ms.ctx).Debug("(*manifestStore).Delete")
return ms.revisionStore.delete(ms.ctx, dgst)
return ms.blobStore.Delete(ctx, dgst)
}
func (ms *manifestStore) Tags() ([]string, error) {
context.GetLogger(ms.ctx).Debug("(*manifestStore).Tags")
return ms.tagStore.tags()
}
func (ms *manifestStore) ExistsByTag(tag string) (bool, error) {
context.GetLogger(ms.ctx).Debug("(*manifestStore).ExistsByTag")
return ms.tagStore.exists(tag)
}
func (ms *manifestStore) GetByTag(tag string, options ...distribution.ManifestServiceOption) (*schema1.SignedManifest, error) {
for _, option := range options {
err := option(ms)
if err != nil {
return nil, err
}
}
context.GetLogger(ms.ctx).Debug("(*manifestStore).GetByTag")
dgst, err := ms.tagStore.resolve(tag)
if err != nil {
return nil, err
}
return ms.revisionStore.get(ms.ctx, dgst)
func (ms *manifestStore) Enumerate(ctx context.Context, manifests []distribution.Manifest, last distribution.Manifest) (n int, err error) {
return 0, distribution.ErrUnsupported
}
// verifyManifest ensures that the manifest content is valid from the
// perspective of the registry. It ensures that the signature is valid for the
// enclosed payload. As a policy, the registry only tries to store valid
// content, leaving trust policies of that content up to consumers.
func (ms *manifestStore) verifyManifest(ctx context.Context, mnfst *schema1.SignedManifest) error {
// content, leaving trust policies of that content up to consumems.
func (ms *manifestStore) verifyManifest(ctx context.Context, mnfst schema1.SignedManifest) error {
var errs distribution.ErrManifestVerification
if len(mnfst.Name) > reference.NameTotalLengthMax {
@ -129,7 +186,7 @@ func (ms *manifestStore) verifyManifest(ctx context.Context, mnfst *schema1.Sign
len(mnfst.History), len(mnfst.FSLayers)))
}
if _, err := schema1.Verify(mnfst); err != nil {
if _, err := schema1.Verify(&mnfst); err != nil {
switch err {
case libtrust.ErrMissingSignatureKey, libtrust.ErrInvalidJSONContent, libtrust.ErrMissingSignatureKey:
errs = append(errs, distribution.ErrManifestUnverified{})
@ -143,15 +200,15 @@ func (ms *manifestStore) verifyManifest(ctx context.Context, mnfst *schema1.Sign
}
if !ms.skipDependencyVerification {
for _, fsLayer := range mnfst.FSLayers {
_, err := ms.repository.Blobs(ctx).Stat(ctx, fsLayer.BlobSum)
for _, fsLayer := range mnfst.References() {
_, err := ms.repository.Blobs(ctx).Stat(ctx, fsLayer.Digest)
if err != nil {
if err != distribution.ErrBlobUnknown {
errs = append(errs, err)
}
// On error here, we always append unknown blob errors.
errs = append(errs, distribution.ErrManifestBlobUnknown{Digest: fsLayer.BlobSum})
// On error here, we always append unknown blob erroms.
errs = append(errs, distribution.ErrManifestBlobUnknown{Digest: fsLayer.Digest})
}
}
}

100
docs/storage/manifeststore_test.go
View file

@ -30,7 +30,8 @@ type manifestStoreTestEnv struct {
func newManifestStoreTestEnv(t *testing.T, name, tag string) *manifestStoreTestEnv {
ctx := context.Background()
driver := inmemory.New()
registry, err := NewRegistry(ctx, driver, BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()), EnableDelete, EnableRedirect)
registry, err := NewRegistry(ctx, driver, BlobDescriptorCacheProvider(
memory.NewInMemoryBlobDescriptorCacheProvider()), EnableDelete, EnableRedirect)
if err != nil {
t.Fatalf("error creating registry: %v", err)
}
@ -58,24 +59,6 @@ func TestManifestStorage(t *testing.T) {
t.Fatal(err)
}
exists, err := ms.ExistsByTag(env.tag)
if err != nil {
t.Fatalf("unexpected error checking manifest existence: %v", err)
}
if exists {
t.Fatalf("manifest should not exist")
}
if _, err := ms.GetByTag(env.tag); true {
switch err.(type) {
case distribution.ErrManifestUnknown:
break
default:
t.Fatalf("expected manifest unknown error: %#v", err)
}
}
m := schema1.Manifest{
Versioned: manifest.Versioned{
SchemaVersion: 1,
@ -114,7 +97,7 @@ func TestManifestStorage(t *testing.T) {
t.Fatalf("error signing manifest: %v", err)
}
err = ms.Put(sm)
_, err = ms.Put(ctx, sm)
if err == nil {
t.Fatalf("expected errors putting manifest with full verification")
}
@ -150,30 +133,40 @@ func TestManifestStorage(t *testing.T) {
}
}
if err = ms.Put(sm); err != nil {
var manifestDigest digest.Digest
if manifestDigest, err = ms.Put(ctx, sm); err != nil {
t.Fatalf("unexpected error putting manifest: %v", err)
}
exists, err = ms.ExistsByTag(env.tag)
exists, err := ms.Exists(ctx, manifestDigest)
if err != nil {
t.Fatalf("unexpected error checking manifest existence: %v", err)
t.Fatalf("unexpected error checking manifest existence: %#v", err)
}
if !exists {
t.Fatalf("manifest should exist")
}
fetchedManifest, err := ms.GetByTag(env.tag)
fromStore, err := ms.Get(ctx, manifestDigest)
if err != nil {
t.Fatalf("unexpected error fetching manifest: %v", err)
}
fetchedManifest, ok := fromStore.(*schema1.SignedManifest)
if !ok {
t.Fatalf("unexpected manifest type from signedstore")
}
if !reflect.DeepEqual(fetchedManifest, sm) {
t.Fatalf("fetched manifest not equal: %#v != %#v", fetchedManifest, sm)
}
fetchedJWS, err := libtrust.ParsePrettySignature(fetchedManifest.Raw, "signatures")
_, pl, err := fetchedManifest.Payload()
if err != nil {
t.Fatalf("error getting payload %#v", err)
}
fetchedJWS, err := libtrust.ParsePrettySignature(pl, "signatures")
if err != nil {
t.Fatalf("unexpected error parsing jws: %v", err)
}
@ -185,8 +178,9 @@ func TestManifestStorage(t *testing.T) {
// Now that we have a payload, take a moment to check that the manifest is
// return by the payload digest.
dgst := digest.FromBytes(payload)
exists, err = ms.Exists(dgst)
exists, err = ms.Exists(ctx, dgst)
if err != nil {
t.Fatalf("error checking manifest existence by digest: %v", err)
}
@ -195,7 +189,7 @@ func TestManifestStorage(t *testing.T) {
t.Fatalf("manifest %s should exist", dgst)
}
fetchedByDigest, err := ms.Get(dgst)
fetchedByDigest, err := ms.Get(ctx, dgst)
if err != nil {
t.Fatalf("unexpected error fetching manifest by digest: %v", err)
}
@ -213,20 +207,6 @@ func TestManifestStorage(t *testing.T) {
t.Fatalf("unexpected number of signatures: %d != %d", len(sigs), 1)
}
// Grabs the tags and check that this tagged manifest is present
tags, err := ms.Tags()
if err != nil {
t.Fatalf("unexpected error fetching tags: %v", err)
}
if len(tags) != 1 {
t.Fatalf("unexpected tags returned: %v", tags)
}
if tags[0] != env.tag {
t.Fatalf("unexpected tag found in tags: %v != %v", tags, []string{env.tag})
}
// Now, push the same manifest with a different key
pk2, err := libtrust.GenerateECP256PrivateKey()
if err != nil {
@ -237,8 +217,12 @@ func TestManifestStorage(t *testing.T) {
if err != nil {
t.Fatalf("unexpected error signing manifest: %v", err)
}
_, pl, err = sm2.Payload()
if err != nil {
t.Fatalf("error getting payload %#v", err)
}
jws2, err := libtrust.ParsePrettySignature(sm2.Raw, "signatures")
jws2, err := libtrust.ParsePrettySignature(pl, "signatures")
if err != nil {
t.Fatalf("error parsing signature: %v", err)
}
@ -252,15 +236,20 @@ func TestManifestStorage(t *testing.T) {
t.Fatalf("unexpected number of signatures: %d != %d", len(sigs2), 1)
}
if err = ms.Put(sm2); err != nil {
if manifestDigest, err = ms.Put(ctx, sm2); err != nil {
t.Fatalf("unexpected error putting manifest: %v", err)
}
fetched, err := ms.GetByTag(env.tag)
fromStore, err = ms.Get(ctx, manifestDigest)
if err != nil {
t.Fatalf("unexpected error fetching manifest: %v", err)
}
fetched, ok := fromStore.(*schema1.SignedManifest)
if !ok {
t.Fatalf("unexpected type from signed manifeststore : %T", fetched)
}
if _, err := schema1.Verify(fetched); err != nil {
t.Fatalf("unexpected error verifying manifest: %v", err)
}
@ -276,7 +265,12 @@ func TestManifestStorage(t *testing.T) {
t.Fatalf("unexpected error getting expected signatures: %v", err)
}
receivedJWS, err := libtrust.ParsePrettySignature(fetched.Raw, "signatures")
_, pl, err = fetched.Payload()
if err != nil {
t.Fatalf("error getting payload %#v", err)
}
receivedJWS, err := libtrust.ParsePrettySignature(pl, "signatures")
if err != nil {
t.Fatalf("unexpected error parsing jws: %v", err)
}
@ -302,12 +296,12 @@ func TestManifestStorage(t *testing.T) {
}
// Test deleting manifests
err = ms.Delete(dgst)
err = ms.Delete(ctx, dgst)
if err != nil {
t.Fatalf("unexpected an error deleting manifest by digest: %v", err)
}
exists, err = ms.Exists(dgst)
exists, err = ms.Exists(ctx, dgst)
if err != nil {
t.Fatalf("Error querying manifest existence")
}
@ -315,7 +309,7 @@ func TestManifestStorage(t *testing.T) {
t.Errorf("Deleted manifest should not exist")
}
deletedManifest, err := ms.Get(dgst)
deletedManifest, err := ms.Get(ctx, dgst)
if err == nil {
t.Errorf("Unexpected success getting deleted manifest")
}
@ -331,12 +325,12 @@ func TestManifestStorage(t *testing.T) {
}
// Re-upload should restore manifest to a good state
err = ms.Put(sm)
_, err = ms.Put(ctx, sm)
if err != nil {
t.Errorf("Error re-uploading deleted manifest")
}
exists, err = ms.Exists(dgst)
exists, err = ms.Exists(ctx, dgst)
if err != nil {
t.Fatalf("Error querying manifest existence")
}
@ -344,7 +338,7 @@ func TestManifestStorage(t *testing.T) {
t.Errorf("Restored manifest should exist")
}
deletedManifest, err = ms.Get(dgst)
deletedManifest, err = ms.Get(ctx, dgst)
if err != nil {
t.Errorf("Unexpected error getting manifest")
}
@ -364,7 +358,7 @@ func TestManifestStorage(t *testing.T) {
if err != nil {
t.Fatal(err)
}
err = ms.Delete(dgst)
err = ms.Delete(ctx, dgst)
if err == nil {
t.Errorf("Unexpected success deleting while disabled")
}

54
docs/storage/registry.go
View file

@ -145,6 +145,15 @@ func (repo *repository) Name() string {
return repo.name
}
func (repo *repository) Tags(ctx context.Context) distribution.TagService {
tags := &tagStore{
repository: repo,
blobStore: repo.registry.blobStore,
}
return tags
}
// Manifests returns an instance of ManifestService. Instantiation is cheap and
// may be context sensitive in the future. The instance should be used similar
// to a request local.
@ -159,36 +168,31 @@ func (repo *repository) Manifests(ctx context.Context, options ...distribution.M
ms := &manifestStore{
ctx: ctx,
repository: repo,
revisionStore: &revisionStore{
ctx: ctx,
repository: repo,
blobStore: &linkedBlobStore{
ctx: ctx,
blobStore: repo.blobStore,
repository: repo,
deleteEnabled: repo.registry.deleteEnabled,
blobAccessController: &linkedBlobStatter{
blobStore: repo.blobStore,
repository: repo,
linkPathFns: manifestLinkPathFns,
},
// TODO(stevvooe): linkPath limits this blob store to only
// manifests. This instance cannot be used for blob checks.
linkPathFns: manifestLinkPathFns,
resumableDigestEnabled: repo.resumableDigestEnabled,
blobStore: &linkedBlobStore{
ctx: ctx,
blobStore: repo.blobStore,
repository: repo,
deleteEnabled: repo.registry.deleteEnabled,
blobAccessController: &linkedBlobStatter{
blobStore: repo.blobStore,
repository: repo,
linkPathFns: manifestLinkPathFns,
},
// TODO(stevvooe): linkPath limits this blob store to only
// manifests. This instance cannot be used for blob checks.
linkPathFns: manifestLinkPathFns,
},
tagStore: &tagStore{
signatures: &signatureStore{
ctx: ctx,
repository: repo,
blobStore: repo.registry.blobStore,
blobStore: repo.blobStore,
},
}
// Apply options
for _, option := range options {
err := option(ms)
err := option.Apply(ms)
if err != nil {
return nil, err
}
@ -225,11 +229,3 @@ func (repo *repository) Blobs(ctx context.Context) distribution.BlobStore {
resumableDigestEnabled: repo.resumableDigestEnabled,
}
}
func (repo *repository) Signatures() distribution.SignatureService {
return &signatureStore{
repository: repo,
blobStore: repo.blobStore,
ctx: repo.ctx,
}
}

111
docs/storage/revisionstore.go
View file

@ -1,111 +0,0 @@
package storage
import (
"encoding/json"
"github.com/docker/distribution"
"github.com/docker/distribution/context"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/libtrust"
)
// revisionStore supports storing and managing manifest revisions.
type revisionStore struct {
repository *repository
blobStore *linkedBlobStore
ctx context.Context
}
// get retrieves the manifest, keyed by revision digest.
func (rs *revisionStore) get(ctx context.Context, revision digest.Digest) (*schema1.SignedManifest, error) {
// Ensure that this revision is available in this repository.
_, err := rs.blobStore.Stat(ctx, revision)
if err != nil {
if err == distribution.ErrBlobUnknown {
return nil, distribution.ErrManifestUnknownRevision{
Name: rs.repository.Name(),
Revision: revision,
}
}
return nil, err
}
// TODO(stevvooe): Need to check descriptor from above to ensure that the
// mediatype is as we expect for the manifest store.
content, err := rs.blobStore.Get(ctx, revision)
if err != nil {
if err == distribution.ErrBlobUnknown {
return nil, distribution.ErrManifestUnknownRevision{
Name: rs.repository.Name(),
Revision: revision,
}
}
return nil, err
}
// Fetch the signatures for the manifest
signatures, err := rs.repository.Signatures().Get(revision)
if err != nil {
return nil, err
}
jsig, err := libtrust.NewJSONSignature(content, signatures...)
if err != nil {
return nil, err
}
// Extract the pretty JWS
raw, err := jsig.PrettySignature("signatures")
if err != nil {
return nil, err
}
var sm schema1.SignedManifest
if err := json.Unmarshal(raw, &sm); err != nil {
return nil, err
}
return &sm, nil
}
// put stores the manifest in the repository, if not already present. Any
// updated signatures will be stored, as well.
func (rs *revisionStore) put(ctx context.Context, sm *schema1.SignedManifest) (distribution.Descriptor, error) {
// Resolve the payload in the manifest.
payload, err := sm.Payload()
if err != nil {
return distribution.Descriptor{}, err
}
// Digest and store the manifest payload in the blob store.
revision, err := rs.blobStore.Put(ctx, schema1.ManifestMediaType, payload)
if err != nil {
context.GetLogger(ctx).Errorf("error putting payload into blobstore: %v", err)
return distribution.Descriptor{}, err
}
// Link the revision into the repository.
if err := rs.blobStore.linkBlob(ctx, revision); err != nil {
return distribution.Descriptor{}, err
}
// Grab each json signature and store them.
signatures, err := sm.Signatures()
if err != nil {
return distribution.Descriptor{}, err
}
if err := rs.repository.Signatures().Put(revision.Digest, signatures...); err != nil {
return distribution.Descriptor{}, err
}
return revision, nil
}
func (rs *revisionStore) delete(ctx context.Context, revision digest.Digest) error {
return rs.blobStore.Delete(ctx, revision)
}

11
docs/storage/signaturestore.go
View file

@ -4,7 +4,6 @@ import (
"path"
"sync"
"github.com/docker/distribution"
"github.com/docker/distribution/context"
"github.com/docker/distribution/digest"
)
@ -15,16 +14,6 @@ type signatureStore struct {
ctx context.Context
}
func newSignatureStore(ctx context.Context, repo *repository, blobStore *blobStore) *signatureStore {
return &signatureStore{
ctx: ctx,
repository: repo,
blobStore: blobStore,
}
}
var _ distribution.SignatureService = &signatureStore{}
func (s *signatureStore) Get(dgst digest.Digest) ([][]byte, error) {
signaturesPath, err := pathFor(manifestSignaturesPathSpec{
name: s.repository.Name(),

64
docs/storage/tagstore.go
View file

@ -9,37 +9,41 @@ import (
storagedriver "github.com/docker/distribution/registry/storage/driver"
)
var _ distribution.TagService = &tagStore{}
// tagStore provides methods to manage manifest tags in a backend storage driver.
// This implementation uses the same on-disk layout as the (now deleted) tag
// store. This provides backward compatibility with current registry deployments
// which only makes use of the Digest field of the returned distribution.Descriptor
// but does not enable full roundtripping of Descriptor objects
type tagStore struct {
repository *repository
blobStore *blobStore
ctx context.Context
}
// tags lists the manifest tags for the specified repository.
func (ts *tagStore) tags() ([]string, error) {
p, err := pathFor(manifestTagPathSpec{
// All returns all tags
func (ts *tagStore) All(ctx context.Context) ([]string, error) {
var tags []string
pathSpec, err := pathFor(manifestTagPathSpec{
name: ts.repository.Name(),
})
if err != nil {
return nil, err
return tags, err
}
var tags []string
entries, err := ts.blobStore.driver.List(ts.ctx, p)
entries, err := ts.blobStore.driver.List(ctx, pathSpec)
if err != nil {
switch err := err.(type) {
case storagedriver.PathNotFoundError:
return nil, distribution.ErrRepositoryUnknown{Name: ts.repository.Name()}
return tags, distribution.ErrRepositoryUnknown{Name: ts.repository.Name()}
default:
return nil, err
return tags, err
}
}
for _, entry := range entries {
_, filename := path.Split(entry)
tags = append(tags, filename)
}
@ -47,7 +51,7 @@ func (ts *tagStore) tags() ([]string, error) {
}
// exists returns true if the specified manifest tag exists in the repository.
func (ts *tagStore) exists(tag string) (bool, error) {
func (ts *tagStore) exists(ctx context.Context, tag string) (bool, error) {
tagPath, err := pathFor(manifestTagCurrentPathSpec{
name: ts.repository.Name(),
tag: tag,
@ -57,7 +61,7 @@ func (ts *tagStore) exists(tag string) (bool, error) {
return false, err
}
exists, err := exists(ts.ctx, ts.blobStore.driver, tagPath)
exists, err := exists(ctx, ts.blobStore.driver, tagPath)
if err != nil {
return false, err
}
@ -65,9 +69,9 @@ func (ts *tagStore) exists(tag string) (bool, error) {
return exists, nil
}
// tag tags the digest with the given tag, updating the the store to point at
// Tag tags the digest with the given tag, updating the the store to point at
// the current tag. The digest must point to a manifest.
func (ts *tagStore) tag(tag string, revision digest.Digest) error {
func (ts *tagStore) Tag(ctx context.Context, tag string, desc distribution.Descriptor) error {
currentPath, err := pathFor(manifestTagCurrentPathSpec{
name: ts.repository.Name(),
tag: tag,
@ -77,43 +81,44 @@ func (ts *tagStore) tag(tag string, revision digest.Digest) error {
return err
}
nbs := ts.linkedBlobStore(ts.ctx, tag)
lbs := ts.linkedBlobStore(ctx, tag)
// Link into the index
if err := nbs.linkBlob(ts.ctx, distribution.Descriptor{Digest: revision}); err != nil {
if err := lbs.linkBlob(ctx, desc); err != nil {
return err
}
// Overwrite the current link
return ts.blobStore.link(ts.ctx, currentPath, revision)
return ts.blobStore.link(ctx, currentPath, desc.Digest)
}
// resolve the current revision for name and tag.
func (ts *tagStore) resolve(tag string) (digest.Digest, error) {
func (ts *tagStore) Get(ctx context.Context, tag string) (distribution.Descriptor, error) {
currentPath, err := pathFor(manifestTagCurrentPathSpec{
name: ts.repository.Name(),
tag: tag,
})
if err != nil {
return "", err
return distribution.Descriptor{}, err
}
revision, err := ts.blobStore.readlink(ts.ctx, currentPath)
revision, err := ts.blobStore.readlink(ctx, currentPath)
if err != nil {
switch err.(type) {
case storagedriver.PathNotFoundError:
return "", distribution.ErrManifestUnknown{Name: ts.repository.Name(), Tag: tag}
return distribution.Descriptor{}, distribution.ErrTagUnknown{Tag: tag}
}
return "", err
return distribution.Descriptor{}, err
}
return revision, nil
return distribution.Descriptor{Digest: revision}, nil
}
// delete removes the tag from repository, including the history of all
// revisions that have the specified tag.
func (ts *tagStore) delete(tag string) error {
func (ts *tagStore) Untag(ctx context.Context, tag string) error {
tagPath, err := pathFor(manifestTagPathSpec{
name: ts.repository.Name(),
tag: tag,
@ -123,7 +128,7 @@ func (ts *tagStore) delete(tag string) error {
return err
}
return ts.blobStore.driver.Delete(ts.ctx, tagPath)
return ts.blobStore.driver.Delete(ctx, tagPath)
}
// linkedBlobStore returns the linkedBlobStore for the named tag, allowing one
@ -145,3 +150,10 @@ func (ts *tagStore) linkedBlobStore(ctx context.Context, tag string) *linkedBlob
}},
}
}
// Lookup recovers a list of tags which refer to this digest. When a manifest is deleted by
// digest, tag entries which point to it need to be recovered to avoid dangling tags.
func (ts *tagStore) Lookup(ctx context.Context, digest distribution.Descriptor) ([]string, error) {
// An efficient implementation of this will require changes to the S3 driver.
return make([]string, 0), nil
}

150
docs/storage/tagstore_test.go Normal file
View file

@ -0,0 +1,150 @@
package storage
import (
"testing"
"github.com/docker/distribution"
"github.com/docker/distribution/context"
"github.com/docker/distribution/registry/storage/driver/inmemory"
)
type tagsTestEnv struct {
ts distribution.TagService
ctx context.Context
}
func testTagStore(t *testing.T) *tagsTestEnv {
ctx := context.Background()
d := inmemory.New()
reg, err := NewRegistry(ctx, d)
if err != nil {
t.Fatal(err)
}
repo, err := reg.Repository(ctx, "a/b")
if err != nil {
t.Fatal(err)
}
return &tagsTestEnv{
ctx: ctx,
ts: repo.Tags(ctx),
}
}
func TestTagStoreTag(t *testing.T) {
env := testTagStore(t)
tags := env.ts
ctx := env.ctx
d := distribution.Descriptor{}
err := tags.Tag(ctx, "latest", d)
if err == nil {
t.Errorf("unexpected error putting malformed descriptor : %s", err)
}
d.Digest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
err = tags.Tag(ctx, "latest", d)
if err != nil {
t.Error(err)
}
d1, err := tags.Get(ctx, "latest")
if err != nil {
t.Error(err)
}
if d1.Digest != d.Digest {
t.Error("put and get digest differ")
}
// Overwrite existing
d.Digest = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
err = tags.Tag(ctx, "latest", d)
if err != nil {
t.Error(err)
}
d1, err = tags.Get(ctx, "latest")
if err != nil {
t.Error(err)
}
if d1.Digest != d.Digest {
t.Error("put and get digest differ")
}
}
func TestTagStoreUnTag(t *testing.T) {
env := testTagStore(t)
tags := env.ts
ctx := env.ctx
desc := distribution.Descriptor{Digest: "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}
err := tags.Untag(ctx, "latest")
if err == nil {
t.Errorf("Expected error untagging non-existant tag")
}
err = tags.Tag(ctx, "latest", desc)
if err != nil {
t.Error(err)
}
err = tags.Untag(ctx, "latest")
if err != nil {
t.Error(err)
}
_, err = tags.Get(ctx, "latest")
if err == nil {
t.Error("Expected error getting untagged tag")
}
}
func TestTagAll(t *testing.T) {
env := testTagStore(t)
tagStore := env.ts
ctx := env.ctx
alpha := "abcdefghijklmnopqrstuvwxyz"
for i := 0; i < len(alpha); i++ {
tag := alpha[i]
desc := distribution.Descriptor{Digest: "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"}
err := tagStore.Tag(ctx, string(tag), desc)
if err != nil {
t.Error(err)
}
}
all, err := tagStore.All(ctx)
if err != nil {
t.Error(err)
}
if len(all) != len(alpha) {
t.Errorf("Unexpected count returned from enumerate")
}
for i, c := range all {
if c != string(alpha[i]) {
t.Errorf("unexpected tag in enumerate %s", c)
}
}
removed := "a"
err = tagStore.Untag(ctx, removed)
if err != nil {
t.Error(err)
}
all, err = tagStore.All(ctx)
if err != nil {
t.Error(err)
}
for _, tag := range all {
if tag == removed {
t.Errorf("unexpected tag in enumerate %s", removed)
}
}
}