From 1d68d81b424ae295bdbca431f8f5419b06c1cd32 Mon Sep 17 00:00:00 2001 From: Stephen J Day Date: Mon, 1 Jun 2015 19:10:51 -0700 Subject: [PATCH 1/5] Catalog V2 API specification proposal This contains a proposal for a catalog API, provided access to the internal contents of a registry instance. The API endpoint is prefixed with an underscore, which is illegal in images names, to prevent collisions with repositories names. To avoid issues with large result sets, a paginated version of the API is proposed. We make an addition to the tags API to support pagination to ensure the specification is conistent. Signed-off-by: Stephen J Day --- docs/api/v2/descriptors.go | 135 +++++++++++++++++++++++++++++++++++++ docs/api/v2/routes.go | 1 + 2 files changed, 136 insertions(+) diff --git a/docs/api/v2/descriptors.go b/docs/api/v2/descriptors.go index f2551ffeb..4eec6492b 100644 --- a/docs/api/v2/descriptors.go +++ b/docs/api/v2/descriptors.go @@ -87,6 +87,23 @@ var ( Format: "", } + paginationParameters = []ParameterDescriptor{ + { + Name: "n", + Type: "integer", + Description: "Limit the number of entries in each response. It not present, all entries will be returned.", + Format: "", + Required: false, + }, + { + Name: "last", + Type: "string", + Description: "Result set will include values lexically after last.", + Format: "", + Required: false, + }, + } + unauthorizedResponse = ResponseDescriptor{ Description: "The client does not have access to the repository.", StatusCode: http.StatusUnauthorized, @@ -269,6 +286,9 @@ type ResponseDescriptor struct { // Headers covers any headers that may be returned from the response. Headers []ParameterDescriptor + // Fields describes any fields that may be present in the response. + Fields []ParameterDescriptor + // ErrorCodes enumerates the error codes that may be returned along with // the response. ErrorCodes []errcode.ErrorCode @@ -427,6 +447,44 @@ var routeDescriptors = []RouteDescriptor{ }, }, }, + { + Description: "Return a portion of the tags for the specified repository.", + PathParameters: []ParameterDescriptor{nameParameterDescriptor}, + QueryParameters: paginationParameters, + Successes: []ResponseDescriptor{ + { + StatusCode: http.StatusOK, + Description: "A list of tags for the named repository.", + Headers: []ParameterDescriptor{ + { + Name: "Content-Length", + Type: "integer", + Description: "Length of the JSON response body.", + Format: "", + }, + }, + Fields: []ParameterDescriptor{ + { + Name: "next", + Type: "url", + Description: "Provides the URL to get the next set of results, if available.", + Format: "", + }, + }, + Body: BodyDescriptor{ + ContentType: "application/json; charset=utf-8", + Format: `{ + "name": , + "tags": [ + , + ... + ], + "next": "?last=&n=" +}`, + }, + }, + }, + }, }, }, }, @@ -1320,6 +1378,83 @@ var routeDescriptors = []RouteDescriptor{ }, }, }, + { + Name: RouteNameCatalog, + Path: "/v2/_catalog", + Entity: "Catalog", + Description: "List a set of available repositories in the local registry cluster. Does not provide any indication of what may be available upstream. Applications can only determine if a repository is available but not if it is not available.", + Methods: []MethodDescriptor{ + { + Method: "GET", + Description: "Retrieve a sorted, json list of repositories available in the registry.", + Requests: []RequestDescriptor{ + { + Name: "Catalog Fetch Complete", + Description: "Request an unabridged list of repositories available.", + Successes: []ResponseDescriptor{ + { + Description: "Returns the unabridged list of repositories as a json response.", + StatusCode: http.StatusOK, + Headers: []ParameterDescriptor{ + { + Name: "Content-Length", + Type: "integer", + Description: "Length of the JSON response body.", + Format: "", + }, + }, + Body: BodyDescriptor{ + ContentType: "application/json; charset=utf-8", + Format: `{ + "repositories": [ + , + ... + ] +}`, + }, + }, + }, + }, + { + Name: "Catalog Fetch Paginated", + Description: "Return the specified portion of repositories.", + QueryParameters: paginationParameters, + Successes: []ResponseDescriptor{ + { + StatusCode: http.StatusOK, + Body: BodyDescriptor{ + ContentType: "application/json; charset=utf-8", + Format: `{ + "repositories": [ + , + ... + ] + "next": "?last=&n=" +}`, + }, + Headers: []ParameterDescriptor{ + { + Name: "Content-Length", + Type: "integer", + Description: "Length of the JSON response body.", + Format: "", + }, + }, + Fields: []ParameterDescriptor{ + { + Name: "next", + Type: "url", + Description: "Provides the URL to get the next set of results, if available.", + Format: "", + }, + }, + }, + }, + }, + }, + }, + }, + }, } var routeDescriptorsMap map[string]RouteDescriptor diff --git a/docs/api/v2/routes.go b/docs/api/v2/routes.go index 69f9d9012..d18860f56 100644 --- a/docs/api/v2/routes.go +++ b/docs/api/v2/routes.go @@ -11,6 +11,7 @@ const ( RouteNameBlob = "blob" RouteNameBlobUpload = "blob-upload" RouteNameBlobUploadChunk = "blob-upload-chunk" + RouteNameCatalog = "catalog" ) var allEndpoints = []string{ From 0790a298ed04744b6d65d21f21c17a70cd67c02b Mon Sep 17 00:00:00 2001 From: Stephen J Day Date: Tue, 2 Jun 2015 20:16:59 -0700 Subject: [PATCH 2/5] Paginate catalog and tag results with Link header Move the specification to use a Link header, rather than a "next" entry in the json results. This prevents requiring clients from parsing the request body to issue the next request. It also ensures that the returned response body does not change in between requests. The ordering of the specification has been slightly tweaked, as well. Listing image tags has been moved after the catalog specification. Tag pagination now heavily references catalog pagination. Signed-off-by: Stephen J Day --- docs/api/v2/descriptors.go | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/docs/api/v2/descriptors.go b/docs/api/v2/descriptors.go index 4eec6492b..ee895b722 100644 --- a/docs/api/v2/descriptors.go +++ b/docs/api/v2/descriptors.go @@ -87,6 +87,13 @@ var ( Format: "", } + linkHeader = ParameterDescriptor{ + Name: "Link", + Type: "link", + Description: "RFC5988 compliant rel='next' with URL to next result set, if available", + Format: `<?n=&last=>; rel="next"`, + } + paginationParameters = []ParameterDescriptor{ { Name: "n", @@ -462,14 +469,7 @@ var routeDescriptors = []RouteDescriptor{ Description: "Length of the JSON response body.", Format: "", }, - }, - Fields: []ParameterDescriptor{ - { - Name: "next", - Type: "url", - Description: "Provides the URL to get the next set of results, if available.", - Format: "", - }, + linkHeader, }, Body: BodyDescriptor{ ContentType: "application/json; charset=utf-8", @@ -479,7 +479,6 @@ var routeDescriptors = []RouteDescriptor{ , ... ], - "next": "?last=&n=" }`, }, }, @@ -1439,14 +1438,7 @@ var routeDescriptors = []RouteDescriptor{ Description: "Length of the JSON response body.", Format: "", }, - }, - Fields: []ParameterDescriptor{ - { - Name: "next", - Type: "url", - Description: "Provides the URL to get the next set of results, if available.", - Format: "", - }, + linkHeader, }, }, }, From f3207e76c878e4859018185c4fec9162d327e1e8 Mon Sep 17 00:00:00 2001 From: Patrick Devine Date: Mon, 13 Jul 2015 13:08:13 -0700 Subject: [PATCH 3/5] Catalog for V2 API Implementation This change adds a basic catalog endpoint to the API, which returns a list, or partial list, of all of the repositories contained in the registry. Calls to this endpoint are somewhat expensive, as every call requires walking a large part of the registry. Instead, to maintain a list of repositories, you would first call the catalog endpoint to get an initial list, and then use the events API to maintain any future repositories. Signed-off-by: Patrick Devine --- docs/api/v2/routes.go | 1 + docs/api/v2/urls.go | 12 +++ docs/client/repository.go | 68 +++++++++++++++++ docs/client/repository_test.go | 41 ++++++++++ docs/handlers/api_test.go | 136 +++++++++++++++++++++++++++++++++ docs/handlers/app.go | 28 ++++++- docs/handlers/catalog.go | 82 ++++++++++++++++++++ docs/handlers/context.go | 3 + docs/storage/catalog.go | 62 +++++++++++++++ docs/storage/catalog_test.go | 127 ++++++++++++++++++++++++++++++ docs/storage/registry.go | 9 +++ 11 files changed, 568 insertions(+), 1 deletion(-) create mode 100644 docs/handlers/catalog.go create mode 100644 docs/storage/catalog.go create mode 100644 docs/storage/catalog_test.go diff --git a/docs/api/v2/routes.go b/docs/api/v2/routes.go index d18860f56..5b80d5be7 100644 --- a/docs/api/v2/routes.go +++ b/docs/api/v2/routes.go @@ -16,6 +16,7 @@ const ( var allEndpoints = []string{ RouteNameManifest, + RouteNameCatalog, RouteNameTags, RouteNameBlob, RouteNameBlobUpload, diff --git a/docs/api/v2/urls.go b/docs/api/v2/urls.go index 60aad5659..429743940 100644 --- a/docs/api/v2/urls.go +++ b/docs/api/v2/urls.go @@ -100,6 +100,18 @@ func (ub *URLBuilder) BuildBaseURL() (string, error) { return baseURL.String(), nil } +// BuildCatalogURL constructs a url get a catalog of repositories +func (ub *URLBuilder) BuildCatalogURL(values ...url.Values) (string, error) { + route := ub.cloneRoute(RouteNameCatalog) + + catalogURL, err := route.URL() + if err != nil { + return "", err + } + + return appendValuesURL(catalogURL, values...).String(), nil +} + // BuildTagsURL constructs a url to list the tags in the named repository. func (ub *URLBuilder) BuildTagsURL(name string) (string, error) { route := ub.cloneRoute(RouteNameTags) diff --git a/docs/client/repository.go b/docs/client/repository.go index fc90cb6e4..6d2fd6e79 100644 --- a/docs/client/repository.go +++ b/docs/client/repository.go @@ -444,3 +444,71 @@ func (bs *blobStatter) Stat(ctx context.Context, dgst digest.Digest) (distributi return distribution.Descriptor{}, handleErrorResponse(resp) } } + +// NewCatalog can be used to get a list of repositories +func NewCatalog(ctx context.Context, baseURL string, transport http.RoundTripper) (distribution.CatalogService, error) { + ub, err := v2.NewURLBuilderFromString(baseURL) + if err != nil { + return nil, err + } + + client := &http.Client{ + Transport: transport, + Timeout: 1 * time.Minute, + } + + return &catalog{ + client: client, + ub: ub, + context: ctx, + }, nil +} + +type catalog struct { + client *http.Client + ub *v2.URLBuilder + context context.Context +} + +func (c *catalog) Get(maxEntries int, last string) ([]string, bool, error) { + var repos []string + + values := url.Values{} + + if maxEntries > 0 { + values.Add("n", strconv.Itoa(maxEntries)) + } + + if last != "" { + values.Add("last", last) + } + + u, err := c.ub.BuildCatalogURL(values) + if err != nil { + return nil, false, err + } + + resp, err := c.client.Get(u) + if err != nil { + return nil, false, err + } + defer resp.Body.Close() + + switch resp.StatusCode { + case http.StatusOK: + var ctlg struct { + Repositories []string `json:"repositories"` + } + decoder := json.NewDecoder(resp.Body) + + if err := decoder.Decode(&ctlg); err != nil { + return nil, false, err + } + + repos = ctlg.Repositories + default: + return nil, false, handleErrorResponse(resp) + } + + return repos, false, nil +} diff --git a/docs/client/repository_test.go b/docs/client/repository_test.go index 3a91be980..e9735cd48 100644 --- a/docs/client/repository_test.go +++ b/docs/client/repository_test.go @@ -8,6 +8,7 @@ import ( "log" "net/http" "net/http/httptest" + "strconv" "strings" "testing" "time" @@ -77,6 +78,23 @@ func addTestFetch(repo string, dgst digest.Digest, content []byte, m *testutil.R }) } +func addTestCatalog(content []byte, m *testutil.RequestResponseMap) { + *m = append(*m, testutil.RequestResponseMapping{ + Request: testutil.Request{ + Method: "GET", + Route: "/v2/_catalog", + }, + Response: testutil.Response{ + StatusCode: http.StatusOK, + Body: content, + Headers: http.Header(map[string][]string{ + "Content-Length": {strconv.Itoa(len(content))}, + "Content-Type": {"application/json; charset=utf-8"}, + }), + }, + }) +} + func TestBlobFetch(t *testing.T) { d1, b1 := newRandomBlob(1024) var m testutil.RequestResponseMap @@ -732,3 +750,26 @@ func TestManifestUnauthorized(t *testing.T) { t.Fatalf("Unexpected message value: %q, expected %q", v2Err.Message, expected) } } + +func TestCatalog(t *testing.T) { + var m testutil.RequestResponseMap + addTestCatalog([]byte("{\"repositories\":[\"foo\", \"bar\", \"baz\"]}"), &m) + + e, c := testServer(m) + defer c() + + ctx := context.Background() + ctlg, err := NewCatalog(ctx, e, nil) + if err != nil { + t.Fatal(err) + } + + repos, _, err := ctlg.Get(0, "") + if err != nil { + t.Fatal(err) + } + + if len(repos) != 3 { + t.Fatalf("Got wrong number of repos") + } +} diff --git a/docs/handlers/api_test.go b/docs/handlers/api_test.go index 8d6319417..d768a116d 100644 --- a/docs/handlers/api_test.go +++ b/docs/handlers/api_test.go @@ -60,6 +60,85 @@ func TestCheckAPI(t *testing.T) { } } +func TestCatalogAPI(t *testing.T) { + env := newTestEnv(t) + + values := url.Values{"last": []string{""}, "n": []string{"100"}} + + catalogURL, err := env.builder.BuildCatalogURL(values) + if err != nil { + t.Fatalf("unexpected error building catalog url: %v", err) + } + + // ----------------------------------- + // try to get an empty catalog + resp, err := http.Get(catalogURL) + if err != nil { + t.Fatalf("unexpected error issuing request: %v", err) + } + defer resp.Body.Close() + + checkResponse(t, "issuing catalog api check", resp, http.StatusOK) + + var ctlg struct { + Repositories []string `json:"repositories"` + } + + dec := json.NewDecoder(resp.Body) + if err := dec.Decode(&ctlg); err != nil { + t.Fatalf("error decoding fetched manifest: %v", err) + } + + // we haven't pushed anything to the registry yet + if ctlg.Repositories != nil { + t.Fatalf("repositories has unexpected values") + } + + if resp.Header.Get("Link") != "" { + t.Fatalf("repositories has more data when none expected") + } + + // ----------------------------------- + // push something to the registry and try again + imageName := "foo/bar" + createRepository(env, t, imageName, "sometag") + + resp, err = http.Get(catalogURL) + if err != nil { + t.Fatalf("unexpected error issuing request: %v", err) + } + defer resp.Body.Close() + + checkResponse(t, "issuing catalog api check", resp, http.StatusOK) + + dec = json.NewDecoder(resp.Body) + if err = dec.Decode(&ctlg); err != nil { + t.Fatalf("error decoding fetched manifest: %v", err) + } + + if len(ctlg.Repositories) != 1 { + t.Fatalf("repositories has unexpected values") + } + + if !contains(ctlg.Repositories, imageName) { + t.Fatalf("didn't find our repository '%s' in the catalog", imageName) + } + + if resp.Header.Get("Link") != "" { + t.Fatalf("repositories has more data when none expected") + } + +} + +func contains(elems []string, e string) bool { + for _, elem := range elems { + if elem == e { + return true + } + } + return false +} + func TestURLPrefix(t *testing.T) { config := configuration.Configuration{ Storage: configuration.Storage{ @@ -869,3 +948,60 @@ func checkErr(t *testing.T, err error, msg string) { t.Fatalf("unexpected error %s: %v", msg, err) } } + +func createRepository(env *testEnv, t *testing.T, imageName string, tag string) { + unsignedManifest := &manifest.Manifest{ + Versioned: manifest.Versioned{ + SchemaVersion: 1, + }, + Name: imageName, + Tag: tag, + FSLayers: []manifest.FSLayer{ + { + BlobSum: "asdf", + }, + { + BlobSum: "qwer", + }, + }, + } + + // Push 2 random layers + expectedLayers := make(map[digest.Digest]io.ReadSeeker) + + for i := range unsignedManifest.FSLayers { + rs, dgstStr, err := testutil.CreateRandomTarFile() + + if err != nil { + t.Fatalf("error creating random layer %d: %v", i, err) + } + dgst := digest.Digest(dgstStr) + + expectedLayers[dgst] = rs + unsignedManifest.FSLayers[i].BlobSum = dgst + + uploadURLBase, _ := startPushLayer(t, env.builder, imageName) + pushLayer(t, env.builder, imageName, dgst, uploadURLBase, rs) + } + + signedManifest, err := manifest.Sign(unsignedManifest, env.pk) + if err != nil { + t.Fatalf("unexpected error signing manifest: %v", err) + } + + payload, err := signedManifest.Payload() + checkErr(t, err, "getting manifest payload") + + dgst, err := digest.FromBytes(payload) + checkErr(t, err, "digesting manifest") + + manifestDigestURL, err := env.builder.BuildManifestURL(imageName, dgst.String()) + checkErr(t, err, "building manifest url") + + resp := putManifest(t, "putting signed manifest", manifestDigestURL, signedManifest) + checkResponse(t, "putting signed manifest", resp, http.StatusAccepted) + checkHeaders(t, resp, http.Header{ + "Location": []string{manifestDigestURL}, + "Docker-Content-Digest": []string{dgst.String()}, + }) +} diff --git a/docs/handlers/app.go b/docs/handlers/app.go index c895222bd..45f97966f 100644 --- a/docs/handlers/app.go +++ b/docs/handlers/app.go @@ -69,6 +69,7 @@ func NewApp(ctx context.Context, configuration configuration.Configuration) *App return http.HandlerFunc(apiBase) }) app.register(v2.RouteNameManifest, imageManifestDispatcher) + app.register(v2.RouteNameCatalog, catalogDispatcher) app.register(v2.RouteNameTags, tagsDispatcher) app.register(v2.RouteNameBlob, blobDispatcher) app.register(v2.RouteNameBlobUpload, blobUploadDispatcher) @@ -366,6 +367,9 @@ func (app *App) dispatcher(dispatch dispatchFunc) http.Handler { // Add username to request logging context.Context = ctxu.WithLogger(context.Context, ctxu.GetLogger(context.Context, "auth.user.name")) + catalog := app.registry.Catalog(context) + context.Catalog = catalog + if app.nameRequired(r) { repository, err := app.registry.Repository(context, getName(context)) @@ -493,6 +497,7 @@ func (app *App) authorized(w http.ResponseWriter, r *http.Request, context *Cont } return fmt.Errorf("forbidden: no repository name") } + accessRecords = appendCatalogAccessRecord(accessRecords, r) } ctx, err := app.accessController.Authorized(context.Context, accessRecords...) @@ -538,7 +543,8 @@ func (app *App) eventBridge(ctx *Context, r *http.Request) notifications.Listene // nameRequired returns true if the route requires a name. func (app *App) nameRequired(r *http.Request) bool { route := mux.CurrentRoute(r) - return route == nil || route.GetName() != v2.RouteNameBase + routeName := route.GetName() + return route == nil || (routeName != v2.RouteNameBase && routeName != v2.RouteNameCatalog) } // apiBase implements a simple yes-man for doing overall checks against the @@ -588,6 +594,26 @@ func appendAccessRecords(records []auth.Access, method string, repo string) []au return records } +// Add the access record for the catalog if it's our current route +func appendCatalogAccessRecord(accessRecords []auth.Access, r *http.Request) []auth.Access { + route := mux.CurrentRoute(r) + routeName := route.GetName() + + if routeName == v2.RouteNameCatalog { + resource := auth.Resource{ + Type: "registry", + Name: "catalog", + } + + accessRecords = append(accessRecords, + auth.Access{ + Resource: resource, + Action: "*", + }) + } + return accessRecords +} + // applyRegistryMiddleware wraps a registry instance with the configured middlewares func applyRegistryMiddleware(registry distribution.Namespace, middlewares []configuration.Middleware) (distribution.Namespace, error) { for _, mw := range middlewares { diff --git a/docs/handlers/catalog.go b/docs/handlers/catalog.go new file mode 100644 index 000000000..fd2af76e7 --- /dev/null +++ b/docs/handlers/catalog.go @@ -0,0 +1,82 @@ +package handlers + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + "strconv" + + "github.com/docker/distribution/registry/api/errcode" + "github.com/gorilla/handlers" +) + +const maximumReturnedEntries = 100 + +func catalogDispatcher(ctx *Context, r *http.Request) http.Handler { + catalogHandler := &catalogHandler{ + Context: ctx, + } + + return handlers.MethodHandler{ + "GET": http.HandlerFunc(catalogHandler.GetCatalog), + } +} + +type catalogHandler struct { + *Context +} + +type catalogAPIResponse struct { + Repositories []string `json:"repositories"` +} + +func (ch *catalogHandler) GetCatalog(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + lastEntry := q.Get("last") + maxEntries, err := strconv.Atoi(q.Get("n")) + if err != nil || maxEntries < 0 { + maxEntries = maximumReturnedEntries + } + + repos, moreEntries, err := ch.Catalog.Get(maxEntries, lastEntry) + if err != nil { + ch.Errors = append(ch.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) + return + } + + w.Header().Set("Content-Type", "application/json; charset=utf-8") + + // Add a link header if there are more entries to retrieve + if moreEntries { + urlStr, err := createLinkEntry(r.URL.String(), maxEntries, repos) + if err != nil { + ch.Errors = append(ch.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) + return + } + w.Header().Set("Link", urlStr) + } + + enc := json.NewEncoder(w) + if err := enc.Encode(catalogAPIResponse{ + Repositories: repos, + }); err != nil { + ch.Errors = append(ch.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) + return + } +} + +// Use the original URL from the request to create a new URL for +// the link header +func createLinkEntry(origURL string, maxEntries int, repos []string) (string, error) { + calledURL, err := url.Parse(origURL) + if err != nil { + return "", err + } + + calledURL.RawQuery = fmt.Sprintf("n=%d&last=%s", maxEntries, repos[len(repos)-1]) + calledURL.Fragment = "" + urlStr := fmt.Sprintf("<%s>; rel=\"next\"", calledURL.String()) + + return urlStr, nil +} diff --git a/docs/handlers/context.go b/docs/handlers/context.go index 85a171237..6625551d4 100644 --- a/docs/handlers/context.go +++ b/docs/handlers/context.go @@ -32,6 +32,9 @@ type Context struct { urlBuilder *v2.URLBuilder + // Catalog allows getting a complete list of the contents of the registry. + Catalog distribution.CatalogService + // TODO(stevvooe): The goal is too completely factor this context and // dispatching out of the web application. Ideally, we should lean on // context.Context for injection of these resources. diff --git a/docs/storage/catalog.go b/docs/storage/catalog.go new file mode 100644 index 000000000..ce184dba4 --- /dev/null +++ b/docs/storage/catalog.go @@ -0,0 +1,62 @@ +package storage + +import ( + "path" + "sort" + "strings" + + log "github.com/Sirupsen/logrus" + "github.com/docker/distribution" + "github.com/docker/distribution/context" + storageDriver "github.com/docker/distribution/registry/storage/driver" +) + +type catalogSvc struct { + ctx context.Context + driver storageDriver.StorageDriver +} + +var _ distribution.CatalogService = &catalogSvc{} + +// Get returns a list, or partial list, of repositories in the registry. +// Because it's a quite expensive operation, it should only be used when building up +// an initial set of repositories. +func (c *catalogSvc) Get(maxEntries int, lastEntry string) ([]string, bool, error) { + log.Infof("Retrieving up to %d entries of the catalog starting with '%s'", maxEntries, lastEntry) + var repos []string + + root, err := defaultPathMapper.path(repositoriesRootPathSpec{}) + if err != nil { + return repos, false, err + } + + Walk(c.ctx, c.driver, root, func(fileInfo storageDriver.FileInfo) error { + filePath := fileInfo.Path() + + // lop the base path off + repoPath := filePath[len(root)+1:] + + _, file := path.Split(repoPath) + if file == "_layers" { + repoPath = strings.TrimSuffix(repoPath, "/_layers") + if repoPath > lastEntry { + repos = append(repos, repoPath) + } + return ErrSkipDir + } else if strings.HasPrefix(file, "_") { + return ErrSkipDir + } + + return nil + }) + + sort.Strings(repos) + + moreEntries := false + if len(repos) > maxEntries { + moreEntries = true + repos = repos[0:maxEntries] + } + + return repos, moreEntries, nil +} diff --git a/docs/storage/catalog_test.go b/docs/storage/catalog_test.go new file mode 100644 index 000000000..8d9f38545 --- /dev/null +++ b/docs/storage/catalog_test.go @@ -0,0 +1,127 @@ +package storage + +import ( + "testing" + + "github.com/docker/distribution" + "github.com/docker/distribution/context" + "github.com/docker/distribution/registry/storage/cache/memory" + "github.com/docker/distribution/registry/storage/driver" + "github.com/docker/distribution/registry/storage/driver/inmemory" +) + +type setupEnv struct { + ctx context.Context + driver driver.StorageDriver + expected []string + registry distribution.Namespace + catalog distribution.CatalogService +} + +func setupFS(t *testing.T) *setupEnv { + d := inmemory.New() + c := []byte("") + ctx := context.Background() + registry := NewRegistryWithDriver(ctx, d, memory.NewInMemoryBlobDescriptorCacheProvider()) + rootpath, _ := defaultPathMapper.path(repositoriesRootPathSpec{}) + + repos := []string{ + "/foo/a/_layers/1", + "/foo/b/_layers/2", + "/bar/c/_layers/3", + "/bar/d/_layers/4", + "/foo/d/in/_layers/5", + "/an/invalid/repo", + "/bar/d/_layers/ignored/dir/6", + } + + for _, repo := range repos { + if err := d.PutContent(ctx, rootpath+repo, c); err != nil { + t.Fatalf("Unable to put to inmemory fs") + } + } + + catalog := registry.Catalog(ctx) + + expected := []string{ + "bar/c", + "bar/d", + "foo/a", + "foo/b", + "foo/d/in", + } + + return &setupEnv{ + ctx: ctx, + driver: d, + expected: expected, + registry: registry, + catalog: catalog, + } +} + +func TestCatalog(t *testing.T) { + env := setupFS(t) + + repos, more, _ := env.catalog.Get(100, "") + + if !testEq(repos, env.expected) { + t.Errorf("Expected catalog repos err") + } + + if more { + t.Errorf("Catalog has more values which we aren't expecting") + } +} + +func TestCatalogInParts(t *testing.T) { + env := setupFS(t) + + chunkLen := 2 + + repos, more, _ := env.catalog.Get(chunkLen, "") + if !testEq(repos, env.expected[0:chunkLen]) { + t.Errorf("Expected catalog first chunk err") + } + + if !more { + t.Errorf("Expected more values in catalog") + } + + lastRepo := repos[len(repos)-1] + repos, more, _ = env.catalog.Get(chunkLen, lastRepo) + + if !testEq(repos, env.expected[chunkLen:chunkLen*2]) { + t.Errorf("Expected catalog second chunk err") + } + + if !more { + t.Errorf("Expected more values in catalog") + } + + lastRepo = repos[len(repos)-1] + repos, more, _ = env.catalog.Get(chunkLen, lastRepo) + + if !testEq(repos, env.expected[chunkLen*2:chunkLen*3-1]) { + t.Errorf("Expected catalog third chunk err") + } + + if more { + t.Errorf("Catalog has more values which we aren't expecting") + } + +} + +func testEq(a, b []string) bool { + if len(a) != len(b) { + return false + } + + for count := range a { + if a[count] != b[count] { + return false + } + } + + return true +} diff --git a/docs/storage/registry.go b/docs/storage/registry.go index cf0fe3e78..170355554 100644 --- a/docs/storage/registry.go +++ b/docs/storage/registry.go @@ -55,6 +55,15 @@ func (reg *registry) Scope() distribution.Scope { return distribution.GlobalScope } +// Catalog returns an instance of the catalog service which can be +// used to dump all of the repositories in a registry +func (reg *registry) Catalog(ctx context.Context) distribution.CatalogService { + return &catalogSvc{ + ctx: ctx, + driver: reg.blobStore.driver, + } +} + // Repository returns an instance of the repository tied to the registry. // Instances should not be shared between goroutines but are cheap to // allocate. In general, they should be request scoped. From bf62b7ebb72d4872f438704e27506d18873262ae Mon Sep 17 00:00:00 2001 From: Patrick Devine Date: Fri, 17 Jul 2015 11:42:47 -0700 Subject: [PATCH 4/5] Create Repositories method This change removes the Catalog Service and replaces it with a more simplistic Repositories() method for obtaining a catalog of all repositories. The Repositories method takes a pre-allocated slice and fills it up to the size of the slice and returns the amount filled. The catalog is returned lexicographically and will start being filled from the last entry passed to Repositories(). If there are no more entries to fill, io.EOF will be returned. Signed-off-by: Patrick Devine Conflicts: registry/client/repository.go registry/handlers/api_test.go --- docs/client/repository.go | 75 ++++++++++++++++-------------- docs/client/repository_test.go | 71 ++++++++++++++++++++++------ docs/handlers/api_test.go | 85 ++++++++++++++++++++++++++++++---- docs/handlers/app.go | 3 -- docs/handlers/catalog.go | 25 +++++++--- docs/handlers/context.go | 3 -- docs/storage/catalog.go | 51 ++++++++++---------- docs/storage/catalog_test.go | 61 +++++++++++------------- docs/storage/registry.go | 9 ---- 9 files changed, 246 insertions(+), 137 deletions(-) diff --git a/docs/client/repository.go b/docs/client/repository.go index 6d2fd6e79..6979cc4d2 100644 --- a/docs/client/repository.go +++ b/docs/client/repository.go @@ -445,34 +445,7 @@ func (bs *blobStatter) Stat(ctx context.Context, dgst digest.Digest) (distributi } } -// NewCatalog can be used to get a list of repositories -func NewCatalog(ctx context.Context, baseURL string, transport http.RoundTripper) (distribution.CatalogService, error) { - ub, err := v2.NewURLBuilderFromString(baseURL) - if err != nil { - return nil, err - } - - client := &http.Client{ - Transport: transport, - Timeout: 1 * time.Minute, - } - - return &catalog{ - client: client, - ub: ub, - context: ctx, - }, nil -} - -type catalog struct { - client *http.Client - ub *v2.URLBuilder - context context.Context -} - -func (c *catalog) Get(maxEntries int, last string) ([]string, bool, error) { - var repos []string - +func buildCatalogValues(maxEntries int, last string) url.Values { values := url.Values{} if maxEntries > 0 { @@ -483,14 +456,35 @@ func (c *catalog) Get(maxEntries int, last string) ([]string, bool, error) { values.Add("last", last) } - u, err := c.ub.BuildCatalogURL(values) + return values +} + +// Repositories returns a lexigraphically sorted catalog given a base URL. The 'entries' slice will be filled up to the size +// of the slice, starting at the value provided in 'last'. The number of entries will be returned along with io.EOF if there +// are no more entries +func Repositories(ctx context.Context, baseURL string, entries []string, last string, transport http.RoundTripper) (int, error) { + var numFilled int + var returnErr error + + ub, err := v2.NewURLBuilderFromString(baseURL) if err != nil { - return nil, false, err + return 0, err } - resp, err := c.client.Get(u) + client := &http.Client{ + Transport: transport, + Timeout: 1 * time.Minute, + } + + values := buildCatalogValues(len(entries), last) + u, err := ub.BuildCatalogURL(values) if err != nil { - return nil, false, err + return 0, err + } + + resp, err := client.Get(u) + if err != nil { + return 0, err } defer resp.Body.Close() @@ -502,13 +496,22 @@ func (c *catalog) Get(maxEntries int, last string) ([]string, bool, error) { decoder := json.NewDecoder(resp.Body) if err := decoder.Decode(&ctlg); err != nil { - return nil, false, err + return 0, err + } + + for cnt := range ctlg.Repositories { + entries[cnt] = ctlg.Repositories[cnt] + } + numFilled = len(ctlg.Repositories) + + link := resp.Header.Get("Link") + if link == "" { + returnErr = io.EOF } - repos = ctlg.Repositories default: - return nil, false, handleErrorResponse(resp) + return 0, handleErrorResponse(resp) } - return repos, false, nil + return numFilled, returnErr } diff --git a/docs/client/repository_test.go b/docs/client/repository_test.go index e9735cd48..b803d754b 100644 --- a/docs/client/repository_test.go +++ b/docs/client/repository_test.go @@ -5,6 +5,7 @@ import ( "crypto/rand" "encoding/json" "fmt" + "io" "log" "net/http" "net/http/httptest" @@ -78,19 +79,24 @@ func addTestFetch(repo string, dgst digest.Digest, content []byte, m *testutil.R }) } -func addTestCatalog(content []byte, m *testutil.RequestResponseMap) { +func addTestCatalog(route string, content []byte, link string, m *testutil.RequestResponseMap) { + headers := map[string][]string{ + "Content-Length": {strconv.Itoa(len(content))}, + "Content-Type": {"application/json; charset=utf-8"}, + } + if link != "" { + headers["Link"] = append(headers["Link"], link) + } + *m = append(*m, testutil.RequestResponseMapping{ Request: testutil.Request{ Method: "GET", - Route: "/v2/_catalog", + Route: route, }, Response: testutil.Response{ StatusCode: http.StatusOK, Body: content, - Headers: http.Header(map[string][]string{ - "Content-Length": {strconv.Itoa(len(content))}, - "Content-Type": {"application/json; charset=utf-8"}, - }), + Headers: http.Header(headers), }, }) } @@ -753,23 +759,58 @@ func TestManifestUnauthorized(t *testing.T) { func TestCatalog(t *testing.T) { var m testutil.RequestResponseMap - addTestCatalog([]byte("{\"repositories\":[\"foo\", \"bar\", \"baz\"]}"), &m) + addTestCatalog( + "/v2/_catalog?n=5", + []byte("{\"repositories\":[\"foo\", \"bar\", \"baz\"]}"), "", &m) e, c := testServer(m) defer c() + entries := make([]string, 5) + ctx := context.Background() - ctlg, err := NewCatalog(ctx, e, nil) - if err != nil { + numFilled, err := Repositories(ctx, e, entries, "", nil) + if err != io.EOF { t.Fatal(err) } - repos, _, err := ctlg.Get(0, "") - if err != nil { - t.Fatal(err) - } - - if len(repos) != 3 { + if numFilled != 3 { + t.Fatalf("Got wrong number of repos") + } +} + +func TestCatalogInParts(t *testing.T) { + var m testutil.RequestResponseMap + addTestCatalog( + "/v2/_catalog?n=2", + []byte("{\"repositories\":[\"bar\", \"baz\"]}"), + "", &m) + addTestCatalog( + "/v2/_catalog?last=baz&n=2", + []byte("{\"repositories\":[\"foo\"]}"), + "", &m) + + e, c := testServer(m) + defer c() + + entries := make([]string, 2) + + ctx := context.Background() + numFilled, err := Repositories(ctx, e, entries, "", nil) + if err != nil { + t.Fatal(err) + } + + if numFilled != 2 { + t.Fatalf("Got wrong number of repos") + } + + numFilled, err = Repositories(ctx, e, entries, "baz", nil) + if err != io.EOF { + t.Fatal(err) + } + + if numFilled != 1 { t.Fatalf("Got wrong number of repos") } } diff --git a/docs/handlers/api_test.go b/docs/handlers/api_test.go index d768a116d..4473eb995 100644 --- a/docs/handlers/api_test.go +++ b/docs/handlers/api_test.go @@ -13,6 +13,8 @@ import ( "os" "path" "reflect" + "regexp" + "strconv" "strings" "testing" @@ -60,10 +62,14 @@ func TestCheckAPI(t *testing.T) { } } +// TestCatalogAPI tests the /v2/_catalog endpoint func TestCatalogAPI(t *testing.T) { + chunkLen := 2 env := newTestEnv(t) - values := url.Values{"last": []string{""}, "n": []string{"100"}} + values := url.Values{ + "last": []string{""}, + "n": []string{strconv.Itoa(chunkLen)}} catalogURL, err := env.builder.BuildCatalogURL(values) if err != nil { @@ -90,7 +96,7 @@ func TestCatalogAPI(t *testing.T) { } // we haven't pushed anything to the registry yet - if ctlg.Repositories != nil { + if len(ctlg.Repositories) != 0 { t.Fatalf("repositories has unexpected values") } @@ -100,8 +106,49 @@ func TestCatalogAPI(t *testing.T) { // ----------------------------------- // push something to the registry and try again - imageName := "foo/bar" - createRepository(env, t, imageName, "sometag") + images := []string{"foo/aaaa", "foo/bbbb", "foo/cccc"} + + for _, image := range images { + createRepository(env, t, image, "sometag") + } + + resp, err = http.Get(catalogURL) + if err != nil { + t.Fatalf("unexpected error issuing request: %v", err) + } + defer resp.Body.Close() + + checkResponse(t, "issuing catalog api check", resp, http.StatusOK) + + dec = json.NewDecoder(resp.Body) + if err = dec.Decode(&ctlg); err != nil { + t.Fatalf("error decoding fetched manifest: %v", err) + } + + if len(ctlg.Repositories) != chunkLen { + t.Fatalf("repositories has unexpected values") + } + + for _, image := range images[:chunkLen] { + if !contains(ctlg.Repositories, image) { + t.Fatalf("didn't find our repository '%s' in the catalog", image) + } + } + + link := resp.Header.Get("Link") + if link == "" { + t.Fatalf("repositories has less data than expected") + } + + newValues := checkLink(t, link, chunkLen, ctlg.Repositories[len(ctlg.Repositories)-1]) + + // ----------------------------------- + // get the last chunk of data + + catalogURL, err = env.builder.BuildCatalogURL(newValues) + if err != nil { + t.Fatalf("unexpected error building catalog url: %v", err) + } resp, err = http.Get(catalogURL) if err != nil { @@ -120,14 +167,36 @@ func TestCatalogAPI(t *testing.T) { t.Fatalf("repositories has unexpected values") } - if !contains(ctlg.Repositories, imageName) { - t.Fatalf("didn't find our repository '%s' in the catalog", imageName) + lastImage := images[len(images)-1] + if !contains(ctlg.Repositories, lastImage) { + t.Fatalf("didn't find our repository '%s' in the catalog", lastImage) } - if resp.Header.Get("Link") != "" { - t.Fatalf("repositories has more data when none expected") + link = resp.Header.Get("Link") + if link != "" { + t.Fatalf("catalog has unexpected data") + } +} + +func checkLink(t *testing.T, urlStr string, numEntries int, last string) url.Values { + re := regexp.MustCompile("<(/v2/_catalog.*)>; rel=\"next\"") + matches := re.FindStringSubmatch(urlStr) + + if len(matches) != 2 { + t.Fatalf("Catalog link address response was incorrect") + } + linkURL, _ := url.Parse(matches[1]) + urlValues := linkURL.Query() + + if urlValues.Get("n") != strconv.Itoa(numEntries) { + t.Fatalf("Catalog link entry size is incorrect") } + if urlValues.Get("last") != last { + t.Fatal("Catalog link last entry is incorrect") + } + + return urlValues } func contains(elems []string, e string) bool { diff --git a/docs/handlers/app.go b/docs/handlers/app.go index 45f97966f..f61b2c1e0 100644 --- a/docs/handlers/app.go +++ b/docs/handlers/app.go @@ -367,9 +367,6 @@ func (app *App) dispatcher(dispatch dispatchFunc) http.Handler { // Add username to request logging context.Context = ctxu.WithLogger(context.Context, ctxu.GetLogger(context.Context, "auth.user.name")) - catalog := app.registry.Catalog(context) - context.Catalog = catalog - if app.nameRequired(r) { repository, err := app.registry.Repository(context, getName(context)) diff --git a/docs/handlers/catalog.go b/docs/handlers/catalog.go index fd2af76e7..6ec1fe550 100644 --- a/docs/handlers/catalog.go +++ b/docs/handlers/catalog.go @@ -3,6 +3,7 @@ package handlers import ( "encoding/json" "fmt" + "io" "net/http" "net/url" "strconv" @@ -32,6 +33,8 @@ type catalogAPIResponse struct { } func (ch *catalogHandler) GetCatalog(w http.ResponseWriter, r *http.Request) { + var moreEntries = true + q := r.URL.Query() lastEntry := q.Get("last") maxEntries, err := strconv.Atoi(q.Get("n")) @@ -39,8 +42,12 @@ func (ch *catalogHandler) GetCatalog(w http.ResponseWriter, r *http.Request) { maxEntries = maximumReturnedEntries } - repos, moreEntries, err := ch.Catalog.Get(maxEntries, lastEntry) - if err != nil { + repos := make([]string, maxEntries) + + filled, err := ch.App.registry.Repositories(ch.Context, repos, lastEntry) + if err == io.EOF { + moreEntries = false + } else if err != nil { ch.Errors = append(ch.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) return } @@ -49,7 +56,8 @@ func (ch *catalogHandler) GetCatalog(w http.ResponseWriter, r *http.Request) { // Add a link header if there are more entries to retrieve if moreEntries { - urlStr, err := createLinkEntry(r.URL.String(), maxEntries, repos) + lastEntry = repos[len(repos)-1] + urlStr, err := createLinkEntry(r.URL.String(), maxEntries, lastEntry) if err != nil { ch.Errors = append(ch.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) return @@ -59,7 +67,7 @@ func (ch *catalogHandler) GetCatalog(w http.ResponseWriter, r *http.Request) { enc := json.NewEncoder(w) if err := enc.Encode(catalogAPIResponse{ - Repositories: repos, + Repositories: repos[0:filled], }); err != nil { ch.Errors = append(ch.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) return @@ -68,13 +76,18 @@ func (ch *catalogHandler) GetCatalog(w http.ResponseWriter, r *http.Request) { // Use the original URL from the request to create a new URL for // the link header -func createLinkEntry(origURL string, maxEntries int, repos []string) (string, error) { +func createLinkEntry(origURL string, maxEntries int, lastEntry string) (string, error) { calledURL, err := url.Parse(origURL) if err != nil { return "", err } - calledURL.RawQuery = fmt.Sprintf("n=%d&last=%s", maxEntries, repos[len(repos)-1]) + v := url.Values{} + v.Add("n", strconv.Itoa(maxEntries)) + v.Add("last", lastEntry) + + calledURL.RawQuery = v.Encode() + calledURL.Fragment = "" urlStr := fmt.Sprintf("<%s>; rel=\"next\"", calledURL.String()) diff --git a/docs/handlers/context.go b/docs/handlers/context.go index 6625551d4..85a171237 100644 --- a/docs/handlers/context.go +++ b/docs/handlers/context.go @@ -32,9 +32,6 @@ type Context struct { urlBuilder *v2.URLBuilder - // Catalog allows getting a complete list of the contents of the registry. - Catalog distribution.CatalogService - // TODO(stevvooe): The goal is too completely factor this context and // dispatching out of the web application. Ideally, we should lean on // context.Context for injection of these resources. diff --git a/docs/storage/catalog.go b/docs/storage/catalog.go index ce184dba4..470894b71 100644 --- a/docs/storage/catalog.go +++ b/docs/storage/catalog.go @@ -1,36 +1,38 @@ package storage import ( + "errors" + "io" "path" "sort" "strings" - log "github.com/Sirupsen/logrus" - "github.com/docker/distribution" "github.com/docker/distribution/context" - storageDriver "github.com/docker/distribution/registry/storage/driver" + "github.com/docker/distribution/registry/storage/driver" ) -type catalogSvc struct { - ctx context.Context - driver storageDriver.StorageDriver -} - -var _ distribution.CatalogService = &catalogSvc{} - -// Get returns a list, or partial list, of repositories in the registry. +// Returns a list, or partial list, of repositories in the registry. // Because it's a quite expensive operation, it should only be used when building up // an initial set of repositories. -func (c *catalogSvc) Get(maxEntries int, lastEntry string) ([]string, bool, error) { - log.Infof("Retrieving up to %d entries of the catalog starting with '%s'", maxEntries, lastEntry) - var repos []string +func (reg *registry) Repositories(ctx context.Context, repos []string, last string) (n int, err error) { + var foundRepos []string + var errVal error + + if len(repos) == 0 { + return 0, errors.New("no space in slice") + } root, err := defaultPathMapper.path(repositoriesRootPathSpec{}) if err != nil { - return repos, false, err + return 0, err } - Walk(c.ctx, c.driver, root, func(fileInfo storageDriver.FileInfo) error { + // Walk each of the directories in our storage. Unfortunately since there's no + // guarantee that storage will return files in lexigraphical order, we have + // to store everything another slice, sort it and then copy it back to our + // passed in slice. + + Walk(ctx, reg.blobStore.driver, root, func(fileInfo driver.FileInfo) error { filePath := fileInfo.Path() // lop the base path off @@ -39,8 +41,8 @@ func (c *catalogSvc) Get(maxEntries int, lastEntry string) ([]string, bool, erro _, file := path.Split(repoPath) if file == "_layers" { repoPath = strings.TrimSuffix(repoPath, "/_layers") - if repoPath > lastEntry { - repos = append(repos, repoPath) + if repoPath > last { + foundRepos = append(foundRepos, repoPath) } return ErrSkipDir } else if strings.HasPrefix(file, "_") { @@ -50,13 +52,14 @@ func (c *catalogSvc) Get(maxEntries int, lastEntry string) ([]string, bool, erro return nil }) - sort.Strings(repos) + sort.Strings(foundRepos) + n = copy(repos, foundRepos) - moreEntries := false - if len(repos) > maxEntries { - moreEntries = true - repos = repos[0:maxEntries] + // Signal that we have no more entries by setting EOF + if len(foundRepos) <= len(repos) { + errVal = io.EOF } - return repos, moreEntries, nil + return n, errVal + } diff --git a/docs/storage/catalog_test.go b/docs/storage/catalog_test.go index 8d9f38545..a9a046a77 100644 --- a/docs/storage/catalog_test.go +++ b/docs/storage/catalog_test.go @@ -1,6 +1,7 @@ package storage import ( + "io" "testing" "github.com/docker/distribution" @@ -15,7 +16,6 @@ type setupEnv struct { driver driver.StorageDriver expected []string registry distribution.Namespace - catalog distribution.CatalogService } func setupFS(t *testing.T) *setupEnv { @@ -41,8 +41,6 @@ func setupFS(t *testing.T) *setupEnv { } } - catalog := registry.Catalog(ctx) - expected := []string{ "bar/c", "bar/d", @@ -56,20 +54,21 @@ func setupFS(t *testing.T) *setupEnv { driver: d, expected: expected, registry: registry, - catalog: catalog, } } func TestCatalog(t *testing.T) { env := setupFS(t) - repos, more, _ := env.catalog.Get(100, "") + p := make([]string, 50) - if !testEq(repos, env.expected) { + numFilled, err := env.registry.Repositories(env.ctx, p, "") + + if !testEq(p, env.expected, numFilled) { t.Errorf("Expected catalog repos err") } - if more { + if err != io.EOF { t.Errorf("Catalog has more values which we aren't expecting") } } @@ -78,50 +77,46 @@ func TestCatalogInParts(t *testing.T) { env := setupFS(t) chunkLen := 2 + p := make([]string, chunkLen) - repos, more, _ := env.catalog.Get(chunkLen, "") - if !testEq(repos, env.expected[0:chunkLen]) { + numFilled, err := env.registry.Repositories(env.ctx, p, "") + if err == io.EOF || numFilled != len(p) { + t.Errorf("Expected more values in catalog") + } + + if !testEq(p, env.expected[0:chunkLen], numFilled) { t.Errorf("Expected catalog first chunk err") } - if !more { + lastRepo := p[len(p)-1] + numFilled, err = env.registry.Repositories(env.ctx, p, lastRepo) + + if err == io.EOF || numFilled != len(p) { t.Errorf("Expected more values in catalog") } - lastRepo := repos[len(repos)-1] - repos, more, _ = env.catalog.Get(chunkLen, lastRepo) - - if !testEq(repos, env.expected[chunkLen:chunkLen*2]) { + if !testEq(p, env.expected[chunkLen:chunkLen*2], numFilled) { t.Errorf("Expected catalog second chunk err") } - if !more { - t.Errorf("Expected more values in catalog") - } + lastRepo = p[len(p)-1] + numFilled, err = env.registry.Repositories(env.ctx, p, lastRepo) - lastRepo = repos[len(repos)-1] - repos, more, _ = env.catalog.Get(chunkLen, lastRepo) - - if !testEq(repos, env.expected[chunkLen*2:chunkLen*3-1]) { - t.Errorf("Expected catalog third chunk err") - } - - if more { + if err != io.EOF { t.Errorf("Catalog has more values which we aren't expecting") } -} - -func testEq(a, b []string) bool { - if len(a) != len(b) { - return false + if !testEq(p, env.expected[chunkLen*2:chunkLen*3-1], numFilled) { + t.Errorf("Expected catalog third chunk err") } - for count := range a { - if a[count] != b[count] { +} + +func testEq(a, b []string, size int) bool { + for cnt := 0; cnt < size-1; cnt++ { + if a[cnt] != b[cnt] { return false } } - return true } diff --git a/docs/storage/registry.go b/docs/storage/registry.go index 170355554..cf0fe3e78 100644 --- a/docs/storage/registry.go +++ b/docs/storage/registry.go @@ -55,15 +55,6 @@ func (reg *registry) Scope() distribution.Scope { return distribution.GlobalScope } -// Catalog returns an instance of the catalog service which can be -// used to dump all of the repositories in a registry -func (reg *registry) Catalog(ctx context.Context) distribution.CatalogService { - return &catalogSvc{ - ctx: ctx, - driver: reg.blobStore.driver, - } -} - // Repository returns an instance of the repository tied to the registry. // Instances should not be shared between goroutines but are cheap to // allocate. In general, they should be request scoped. From a49594a0e19560969396f5fcbed657062524be8f Mon Sep 17 00:00:00 2001 From: Patrick Devine Date: Wed, 22 Jul 2015 15:18:03 -0700 Subject: [PATCH 5/5] Add Registry to client bindings for Repositories The way Repositories() was initially called was somewhat different than other parts of the client bindings because there was no way to instantiate a Namespace. This change implements a NewRegistry() function which changes it so that Repositories() can be called the way one would expect. It doesn't implement any of the other functions of Namespaces. Signed-off-by: Patrick Devine --- docs/client/repository.go | 134 +++++++++++++++++++-------------- docs/client/repository_test.go | 16 +++- 2 files changed, 90 insertions(+), 60 deletions(-) diff --git a/docs/client/repository.go b/docs/client/repository.go index 6979cc4d2..29effcce8 100644 --- a/docs/client/repository.go +++ b/docs/client/repository.go @@ -21,6 +21,83 @@ import ( "github.com/docker/distribution/registry/storage/cache/memory" ) +// Registry provides an interface for calling Repositories, which returns a catalog of repositories. +type Registry interface { + Repositories(ctx context.Context, repos []string, last string) (n int, err error) +} + +// NewRegistry creates a registry namespace which can be used to get a listing of repositories +func NewRegistry(ctx context.Context, baseURL string, transport http.RoundTripper) (Registry, error) { + ub, err := v2.NewURLBuilderFromString(baseURL) + if err != nil { + return nil, err + } + + client := &http.Client{ + Transport: transport, + Timeout: 1 * time.Minute, + } + + return ®istry{ + client: client, + ub: ub, + context: ctx, + }, nil +} + +type registry struct { + client *http.Client + ub *v2.URLBuilder + context context.Context +} + +// Repositories returns a lexigraphically sorted catalog given a base URL. The 'entries' slice will be filled up to the size +// of the slice, starting at the value provided in 'last'. The number of entries will be returned along with io.EOF if there +// are no more entries +func (r *registry) Repositories(ctx context.Context, entries []string, last string) (int, error) { + var numFilled int + var returnErr error + + values := buildCatalogValues(len(entries), last) + u, err := r.ub.BuildCatalogURL(values) + if err != nil { + return 0, err + } + + resp, err := r.client.Get(u) + if err != nil { + return 0, err + } + defer resp.Body.Close() + + switch resp.StatusCode { + case http.StatusOK: + var ctlg struct { + Repositories []string `json:"repositories"` + } + decoder := json.NewDecoder(resp.Body) + + if err := decoder.Decode(&ctlg); err != nil { + return 0, err + } + + for cnt := range ctlg.Repositories { + entries[cnt] = ctlg.Repositories[cnt] + } + numFilled = len(ctlg.Repositories) + + link := resp.Header.Get("Link") + if link == "" { + returnErr = io.EOF + } + + default: + return 0, handleErrorResponse(resp) + } + + return numFilled, returnErr +} + // NewRepository creates a new Repository for the given repository name and base URL func NewRepository(ctx context.Context, name, baseURL string, transport http.RoundTripper) (distribution.Repository, error) { if err := v2.ValidateRepositoryName(name); err != nil { @@ -458,60 +535,3 @@ func buildCatalogValues(maxEntries int, last string) url.Values { return values } - -// Repositories returns a lexigraphically sorted catalog given a base URL. The 'entries' slice will be filled up to the size -// of the slice, starting at the value provided in 'last'. The number of entries will be returned along with io.EOF if there -// are no more entries -func Repositories(ctx context.Context, baseURL string, entries []string, last string, transport http.RoundTripper) (int, error) { - var numFilled int - var returnErr error - - ub, err := v2.NewURLBuilderFromString(baseURL) - if err != nil { - return 0, err - } - - client := &http.Client{ - Transport: transport, - Timeout: 1 * time.Minute, - } - - values := buildCatalogValues(len(entries), last) - u, err := ub.BuildCatalogURL(values) - if err != nil { - return 0, err - } - - resp, err := client.Get(u) - if err != nil { - return 0, err - } - defer resp.Body.Close() - - switch resp.StatusCode { - case http.StatusOK: - var ctlg struct { - Repositories []string `json:"repositories"` - } - decoder := json.NewDecoder(resp.Body) - - if err := decoder.Decode(&ctlg); err != nil { - return 0, err - } - - for cnt := range ctlg.Repositories { - entries[cnt] = ctlg.Repositories[cnt] - } - numFilled = len(ctlg.Repositories) - - link := resp.Header.Get("Link") - if link == "" { - returnErr = io.EOF - } - - default: - return 0, handleErrorResponse(resp) - } - - return numFilled, returnErr -} diff --git a/docs/client/repository_test.go b/docs/client/repository_test.go index b803d754b..232501aa3 100644 --- a/docs/client/repository_test.go +++ b/docs/client/repository_test.go @@ -768,8 +768,13 @@ func TestCatalog(t *testing.T) { entries := make([]string, 5) + r, err := NewRegistry(context.Background(), e, nil) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() - numFilled, err := Repositories(ctx, e, entries, "", nil) + numFilled, err := r.Repositories(ctx, entries, "") if err != io.EOF { t.Fatal(err) } @@ -795,8 +800,13 @@ func TestCatalogInParts(t *testing.T) { entries := make([]string, 2) + r, err := NewRegistry(context.Background(), e, nil) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() - numFilled, err := Repositories(ctx, e, entries, "", nil) + numFilled, err := r.Repositories(ctx, entries, "") if err != nil { t.Fatal(err) } @@ -805,7 +815,7 @@ func TestCatalogInParts(t *testing.T) { t.Fatalf("Got wrong number of repos") } - numFilled, err = Repositories(ctx, e, entries, "baz", nil) + numFilled, err = r.Repositories(ctx, entries, "baz") if err != io.EOF { t.Fatal(err) }