From aebe850f733186f2b4119089d7b425911663e095 Mon Sep 17 00:00:00 2001 From: Stephen J Day Date: Mon, 1 Jun 2015 19:10:51 -0700 Subject: [PATCH 1/5] Catalog V2 API specification proposal This contains a proposal for a catalog API, provided access to the internal contents of a registry instance. The API endpoint is prefixed with an underscore, which is illegal in images names, to prevent collisions with repositories names. To avoid issues with large result sets, a paginated version of the API is proposed. We make an addition to the tags API to support pagination to ensure the specification is conistent. Signed-off-by: Stephen J Day --- docs/spec/api.md | 331 ++++++++++++++++++++++++++++++++- docs/spec/api.md.tmpl | 184 +++++++++++++++++- registry/api/v2/descriptors.go | 135 ++++++++++++++ registry/api/v2/routes.go | 1 + 4 files changed, 640 insertions(+), 11 deletions(-) diff --git a/docs/spec/api.md b/docs/spec/api.md index 483f15f03..ba4bc1ca7 100644 --- a/docs/spec/api.md +++ b/docs/spec/api.md @@ -120,6 +120,16 @@ indicating what is different. Optionally, we may start marking parts of the specification to correspond with the versions enumerated here.
+ +
2.0.4
> +
+
    +
  • Added support for listing registry contents.
    • +
  • Added pagination to tags API.
    • +
  • Added common approach to support pagination.
    • +
+
+
2.0.3
  • Allow repository name components to be one character.
    • @@ -131,7 +141,6 @@ specification to correspond with the versions enumerated here.
  • Added section covering digest format.
  • Added more clarification that manifest cannot be deleted by tag.
    • -
    2.0.1
      @@ -745,7 +754,9 @@ each unknown blob. The response format is as follows: ] } -#### Listing Image Tags + + +### Listing Image Tags It may be necessary to list all of the tags under a given repository. The tags for an image repository can be retrieved with the following request: @@ -766,8 +777,166 @@ The response will be in the following format: } For repositories with a large number of tags, this response may be quite -large, so care should be taken by the client when parsing the response to -reduce copying. +large. If such a response is expected, one should use the pagination. + +#### Pagination + +Paginated tag results can be retrieved by adding the appropriate pagination +parameters. Starting a paginated flow may begin as follows: + +``` +GET /v2//tags/list?n= +``` + +The above specifies that a tags response should be returned, from the start of +the result set, ordered lexically, limiting the number of results to `n`. The +response to such a request would look as follows: + +``` +200 OK +Content-Type: application/json + +{ + "name": , + "tags": [ + , + ... + ] + "next": ?n=&last= +} +``` + +> __TODO(stevvooe):__ Consider using a Header here, rather than a body parameter. A +header would allow one to issue the next request before parsing the response +body. + +To get the next result set, a client would issue the request as follows, using +the value of "next" from the response body: + +``` +GET /v2//tags/list?n=&last= +``` + +The above process should then be repeated until the `next` parameter is no +longer set in the response. + +The behavior of `last` is quite simple and can be demonstrated with an +example. Let's say the repository has the following tags: + +``` +a +b +c +d +``` + +If the value of `n` is 2, _a_ and _b_ will be returned on the first response. +The `next` url within the respone will have `n` set to 2 and last set to _b_: + +``` +"next": ?n=2&last=b +``` + +The client can then issue the response, receiving the values _c_ and _d_. Note +that n may change on second to last response or be omitted fully, if the +server may so choose. + +### Listing Repositories + +Images are stored in collections, known as a _repository_, which is keyed by a +`name`, as seen throughout the API specification. A registry instance may +contain several repositories. The list of available repositories, or image +names, is made available through the _catalog_. + +The catalog for a given registry can be retrived with the following request: + +``` +GET /v2/_catalog +``` + +The response will be in the following format: + +``` +200 OK +Content-Type: application/json + +{ + "repositories": [ + , + ... + ] +} +``` + +For registries with a large number of repositories, this response may be quite +large. If such a response is expected, one should use the pagination. + +#### Pagination + +Paginated repository results can be retrieved by adding the appropriate +pagination parameters, which are similar to those available in the tag API. +Starting a paginated flow may begin as follows: + +``` +GET /v2/_catalog?n= +``` + +The above specifies that a catalog response should be returned, from the start of +the result set, ordered lexically, limiting the number of results to `n`. The +response to such a request would look as follows: + +``` +200 OK +Content-Type: application/json + +{ + "repositories": [ + , + ... + ] + "next": ?n=&last= +} +``` + +> __TODO(stevvooe):__ Consider using a Header here, rather than a body parameter. A +header would allow one to issue the next request before parsing the response +body. + +To get the next result set, a client would issue the request as follows, using +the value of "next" from the response body: + +``` +GET /v2/_catalog?n=&last= +``` + +The above process should then be repeated until the `next` parameter is no +longer set in the response. + +The result set of repository names is represented abstractly as a lexically +sorted list, where the position in that list can be specified by the query +term `last`. The entries in the response start _after_ the term specified by +`last`, up to `n` entries. + +The behavior of `last` is quite simple when demonstrated with an example. +Let's say the registry has the following repositories: + +``` +a +b +c +d +``` + +If the value of `n` is 2, _a_ and _b_ will be returned on the first response. +The `next` url within the respone will have `n` set to 2 and last set to _b_: + +``` +"next": ?n=2&last=b +``` + +The client can then issue the request with above value of `next`, receiving +the values _c_ and _d_. Note that n may change on second to last response or +be omitted fully, if the server may so choose. ### Deleting an Image @@ -817,6 +986,7 @@ A list of methods and URIs are covered in the table below: | PATCH | `/v2//blobs/uploads/` | Blob Upload | Upload a chunk of data for the specified upload. | | PUT | `/v2//blobs/uploads/` | Blob Upload | Complete the upload specified by `uuid`, optionally appending the body as the final chunk. | | DELETE | `/v2//blobs/uploads/` | Blob Upload | Cancel outstanding upload processes, releasing associated resources. If this is not called, the unfinished uploads will eventually timeout. | +| GET | `/v2/_catalog` | Catalog | Retrieve a sorted, json list of repositories available in the registry. | The detail for each endpoint is covered in the following sections. @@ -886,7 +1056,6 @@ The API implements V2 protocol and is accessible. - ###### On Failure: Unauthorized ``` @@ -1056,6 +1225,57 @@ The error codes that may be included in the response body are enumerated below: +``` +GET /v2//tags/list?n=last= +``` + +Return a portion of the tags for the specified repository. + + +The following parameters should be specified on the request: + +|Name|Kind|Description| +|----|----|-----------| +|`name`|path|Name of the target repository.| +|`n`|query|Limit the number of entries in each response. It not present, all entries will be returned.| +|`last`|query|Result set will include values lexically after last.| + + + + +###### On Success: OK + +``` +200 OK +Content-Length: +Content-Type: application/json; charset=utf-8 + +{ + "name": , + "tags": [ + , + ... + ], + "next": "?last=&n=" +} +``` + +A list of tags for the named repository. +The following fields may be returned in the response body: + +|Name|Description| +|----|-----------| +|`next`|Provides the URL to get the next set of results, if available.| + +The following headers will be returned with the response: + +|Name|Description| +|----|-----------| +|`Content-Length`|Length of the JSON response body.| + + + + ### Manifest @@ -1453,7 +1673,6 @@ The following parameters should be specified on the request: - ###### On Failure: Invalid Name or Reference ``` @@ -2907,3 +3126,103 @@ The error codes that may be included in the response body are enumerated below: +### Catalog + +List a set of available repositories in the local registry cluster. Does not provide any indication of what may be available upstream. Applications can only determine if a repository is available but not if it is not available. + + + +#### GET Catalog + +Retrieve a sorted, json list of repositories available in the registry. + + +##### Catalog Fetch Complete + +``` +GET /v2/_catalog +``` + +Request an unabridged list of repositories available. + + + + + +###### On Success: OK + +``` +200 OK +Content-Length: +Content-Type: application/json; charset=utf-8 + +{ + "repositories": [ + , + ... + ] +} +``` + +Returns the unabridged list of repositories as a json response. + +The following headers will be returned with the response: + +|Name|Description| +|----|-----------| +|`Content-Length`|Length of the JSON response body.| + + + +##### Catalog Fetch Paginated + +``` +GET /v2/_catalog?n=last= +``` + +Return the specified portion of repositories. + + +The following parameters should be specified on the request: + +|Name|Kind|Description| +|----|----|-----------| +|`n`|query|Limit the number of entries in each response. It not present, all entries will be returned.| +|`last`|query|Result set will include values lexically after last.| + + + + +###### On Success: OK + +``` +200 OK +Content-Length: +Content-Type: application/json; charset=utf-8 + +{ + "repositories": [ + , + ... + ] + "next": "?last=&n=" +} +``` + + +The following fields may be returned in the response body: + +|Name|Description| +|----|-----------| +|`next`|Provides the URL to get the next set of results, if available.| + +The following headers will be returned with the response: + +|Name|Description| +|----|-----------| +|`Content-Length`|Length of the JSON response body.| + + + + + diff --git a/docs/spec/api.md.tmpl b/docs/spec/api.md.tmpl index bc6d6f926..218f30c92 100644 --- a/docs/spec/api.md.tmpl +++ b/docs/spec/api.md.tmpl @@ -120,6 +120,16 @@ indicating what is different. Optionally, we may start marking parts of the specification to correspond with the versions enumerated here.
      + +
      2.0.4
      > +
      +
        +
      • Added support for listing registry contents.
        • +
      • Added pagination to tags API.
        • +
      • Added common approach to support pagination.
        • +
      +
      +
      2.0.3
    • Allow repository name components to be one character.
      • @@ -131,7 +141,6 @@ specification to correspond with the versions enumerated here.
    • Added section covering digest format.
    • Added more clarification that manifest cannot be deleted by tag.
      • -
      2.0.1
        @@ -745,7 +754,9 @@ each unknown blob. The response format is as follows: ] } -#### Listing Image Tags + + +### Listing Image Tags It may be necessary to list all of the tags under a given repository. The tags for an image repository can be retrieved with the following request: @@ -766,8 +777,166 @@ The response will be in the following format: } For repositories with a large number of tags, this response may be quite -large, so care should be taken by the client when parsing the response to -reduce copying. +large. If such a response is expected, one should use the pagination. + +#### Pagination + +Paginated tag results can be retrieved by adding the appropriate pagination +parameters. Starting a paginated flow may begin as follows: + +``` +GET /v2//tags/list?n= +``` + +The above specifies that a tags response should be returned, from the start of +the result set, ordered lexically, limiting the number of results to `n`. The +response to such a request would look as follows: + +``` +200 OK +Content-Type: application/json + +{ + "name": , + "tags": [ + , + ... + ] + "next": ?n=&last= +} +``` + +> __TODO(stevvooe):__ Consider using a Header here, rather than a body parameter. A +header would allow one to issue the next request before parsing the response +body. + +To get the next result set, a client would issue the request as follows, using +the value of "next" from the response body: + +``` +GET /v2//tags/list?n=&last= +``` + +The above process should then be repeated until the `next` parameter is no +longer set in the response. + +The behavior of `last` is quite simple and can be demonstrated with an +example. Let's say the repository has the following tags: + +``` +a +b +c +d +``` + +If the value of `n` is 2, _a_ and _b_ will be returned on the first response. +The `next` url within the respone will have `n` set to 2 and last set to _b_: + +``` +"next": ?n=2&last=b +``` + +The client can then issue the response, receiving the values _c_ and _d_. Note +that n may change on second to last response or be omitted fully, if the +server may so choose. + +### Listing Repositories + +Images are stored in collections, known as a _repository_, which is keyed by a +`name`, as seen throughout the API specification. A registry instance may +contain several repositories. The list of available repositories, or image +names, is made available through the _catalog_. + +The catalog for a given registry can be retrived with the following request: + +``` +GET /v2/_catalog +``` + +The response will be in the following format: + +``` +200 OK +Content-Type: application/json + +{ + "repositories": [ + , + ... + ] +} +``` + +For registries with a large number of repositories, this response may be quite +large. If such a response is expected, one should use the pagination. + +#### Pagination + +Paginated repository results can be retrieved by adding the appropriate +pagination parameters, which are similar to those available in the tag API. +Starting a paginated flow may begin as follows: + +``` +GET /v2/_catalog?n= +``` + +The above specifies that a catalog response should be returned, from the start of +the result set, ordered lexically, limiting the number of results to `n`. The +response to such a request would look as follows: + +``` +200 OK +Content-Type: application/json + +{ + "repositories": [ + , + ... + ] + "next": ?n=&last= +} +``` + +> __TODO(stevvooe):__ Consider using a Header here, rather than a body parameter. A +header would allow one to issue the next request before parsing the response +body. + +To get the next result set, a client would issue the request as follows, using +the value of "next" from the response body: + +``` +GET /v2/_catalog?n=&last= +``` + +The above process should then be repeated until the `next` parameter is no +longer set in the response. + +The result set of repository names is represented abstractly as a lexically +sorted list, where the position in that list can be specified by the query +term `last`. The entries in the response start _after_ the term specified by +`last`, up to `n` entries. + +The behavior of `last` is quite simple when demonstrated with an example. +Let's say the registry has the following repositories: + +``` +a +b +c +d +``` + +If the value of `n` is 2, _a_ and _b_ will be returned on the first response. +The `next` url within the respone will have `n` set to 2 and last set to _b_: + +``` +"next": ?n=2&last=b +``` + +The client can then issue the request with above value of `next`, receiving +the values _c_ and _d_. Note that n may change on second to last response or +be omitted fully, if the server may so choose. ### Deleting an Image @@ -867,8 +1036,13 @@ Content-Type: {{.Body.ContentType}}{{end}}{{if .Body.Format}} ``` {{.Description}} +{{if .Fields}}The following fields may be returned in the response body: -{{if .Headers}}The following headers will be returned with the response: +|Name|Description| +|----|-----------| +{{range .Fields}}|`{{.Name}}`|{{.Description}}| +{{end}}{{end}}{{if .Headers}} +The following headers will be returned with the response: |Name|Description| |----|-----------| diff --git a/registry/api/v2/descriptors.go b/registry/api/v2/descriptors.go index f2551ffeb..4eec6492b 100644 --- a/registry/api/v2/descriptors.go +++ b/registry/api/v2/descriptors.go @@ -87,6 +87,23 @@ var ( Format: "", } + paginationParameters = []ParameterDescriptor{ + { + Name: "n", + Type: "integer", + Description: "Limit the number of entries in each response. It not present, all entries will be returned.", + Format: "", + Required: false, + }, + { + Name: "last", + Type: "string", + Description: "Result set will include values lexically after last.", + Format: "", + Required: false, + }, + } + unauthorizedResponse = ResponseDescriptor{ Description: "The client does not have access to the repository.", StatusCode: http.StatusUnauthorized, @@ -269,6 +286,9 @@ type ResponseDescriptor struct { // Headers covers any headers that may be returned from the response. Headers []ParameterDescriptor + // Fields describes any fields that may be present in the response. + Fields []ParameterDescriptor + // ErrorCodes enumerates the error codes that may be returned along with // the response. ErrorCodes []errcode.ErrorCode @@ -427,6 +447,44 @@ var routeDescriptors = []RouteDescriptor{ }, }, }, + { + Description: "Return a portion of the tags for the specified repository.", + PathParameters: []ParameterDescriptor{nameParameterDescriptor}, + QueryParameters: paginationParameters, + Successes: []ResponseDescriptor{ + { + StatusCode: http.StatusOK, + Description: "A list of tags for the named repository.", + Headers: []ParameterDescriptor{ + { + Name: "Content-Length", + Type: "integer", + Description: "Length of the JSON response body.", + Format: "", + }, + }, + Fields: []ParameterDescriptor{ + { + Name: "next", + Type: "url", + Description: "Provides the URL to get the next set of results, if available.", + Format: "", + }, + }, + Body: BodyDescriptor{ + ContentType: "application/json; charset=utf-8", + Format: `{ + "name": , + "tags": [ + , + ... + ], + "next": "?last=&n=" +}`, + }, + }, + }, + }, }, }, }, @@ -1320,6 +1378,83 @@ var routeDescriptors = []RouteDescriptor{ }, }, }, + { + Name: RouteNameCatalog, + Path: "/v2/_catalog", + Entity: "Catalog", + Description: "List a set of available repositories in the local registry cluster. Does not provide any indication of what may be available upstream. Applications can only determine if a repository is available but not if it is not available.", + Methods: []MethodDescriptor{ + { + Method: "GET", + Description: "Retrieve a sorted, json list of repositories available in the registry.", + Requests: []RequestDescriptor{ + { + Name: "Catalog Fetch Complete", + Description: "Request an unabridged list of repositories available.", + Successes: []ResponseDescriptor{ + { + Description: "Returns the unabridged list of repositories as a json response.", + StatusCode: http.StatusOK, + Headers: []ParameterDescriptor{ + { + Name: "Content-Length", + Type: "integer", + Description: "Length of the JSON response body.", + Format: "", + }, + }, + Body: BodyDescriptor{ + ContentType: "application/json; charset=utf-8", + Format: `{ + "repositories": [ + , + ... + ] +}`, + }, + }, + }, + }, + { + Name: "Catalog Fetch Paginated", + Description: "Return the specified portion of repositories.", + QueryParameters: paginationParameters, + Successes: []ResponseDescriptor{ + { + StatusCode: http.StatusOK, + Body: BodyDescriptor{ + ContentType: "application/json; charset=utf-8", + Format: `{ + "repositories": [ + , + ... + ] + "next": "?last=&n=" +}`, + }, + Headers: []ParameterDescriptor{ + { + Name: "Content-Length", + Type: "integer", + Description: "Length of the JSON response body.", + Format: "", + }, + }, + Fields: []ParameterDescriptor{ + { + Name: "next", + Type: "url", + Description: "Provides the URL to get the next set of results, if available.", + Format: "", + }, + }, + }, + }, + }, + }, + }, + }, + }, } var routeDescriptorsMap map[string]RouteDescriptor diff --git a/registry/api/v2/routes.go b/registry/api/v2/routes.go index 69f9d9012..d18860f56 100644 --- a/registry/api/v2/routes.go +++ b/registry/api/v2/routes.go @@ -11,6 +11,7 @@ const ( RouteNameBlob = "blob" RouteNameBlobUpload = "blob-upload" RouteNameBlobUploadChunk = "blob-upload-chunk" + RouteNameCatalog = "catalog" ) var allEndpoints = []string{ From 006214d902d5e0ac922d9bc2711428379f172fef Mon Sep 17 00:00:00 2001 From: Stephen J Day Date: Tue, 2 Jun 2015 20:16:59 -0700 Subject: [PATCH 2/5] Paginate catalog and tag results with Link header Move the specification to use a Link header, rather than a "next" entry in the json results. This prevents requiring clients from parsing the request body to issue the next request. It also ensures that the returned response body does not change in between requests. The ordering of the specification has been slightly tweaked, as well. Listing image tags has been moved after the catalog specification. Tag pagination now heavily references catalog pagination. Signed-off-by: Stephen J Day --- docs/spec/api.md | 278 ++++++++++++++++----------------- docs/spec/api.md.tmpl | 263 ++++++++++++++++--------------- registry/api/v2/descriptors.go | 26 ++- 3 files changed, 283 insertions(+), 284 deletions(-) diff --git a/docs/spec/api.md b/docs/spec/api.md index ba4bc1ca7..a3606d8db 100644 --- a/docs/spec/api.md +++ b/docs/spec/api.md @@ -754,7 +754,129 @@ each unknown blob. The response format is as follows: ] } +### Listing Repositories +Images are stored in collections, known as a _repository_, which is keyed by a +`name`, as seen throughout the API specification. A registry instance may +contain several repositories. The list of available repositories is made +available through the _catalog_. + +The catalog for a given registry can be retrived with the following request: + +``` +GET /v2/_catalog +``` + +The response will be in the following format: + +``` +200 OK +Content-Type: application/json + +{ + "repositories": [ + , + ... + ] +} +``` + +Note that the contents of the response are specific to the registry +implementation. Some registries may opt to provide a full catalog output, +limit it based on the user's access level or omit upstream results, if +providing mirroring functionality. Subsequently, the presence of a repository +in the catalog listing only means that the registry *may* provide access to +the repository at the time of the request. Conversely, a missing entry does +*not* mean that the registry does not have the repository. More succinctly, +the presence of a repository only guarantees that it is there but not that it +is _not_ there. + +For registries with a large number of repositories, this response may be quite +large. If such a response is expected, one should use pagination. + +#### Pagination + +Paginated catalog results can be retrieved by adding an `n` parameter to the +request URL, declaring that the response should be limited to `n` results. +Starting a paginated flow begins as follows: + +``` +GET /v2/_catalog?n= +``` + +The above specifies that a catalog response should be returned, from the start of +the result set, ordered lexically, limiting the number of results to `n`. The +response to such a request would look as follows: + +``` +200 OK +Content-Type: application/json +Link: <?n=&last=>; rel="next" + +{ + "repositories": [ + , + ... + ] +} +``` + +The above includes the _first_ `n` entries from the result set. To get the +_next_ `n` entries, one can create a URL where the argument `last` has the +value from `repositories[len(repositories)-1]`. If there are indeed more +results, the URL for the next block is encoded in an +[RFC5988](https://tools.ietf.org/html/rfc5988) `Link` header, as a "next" +relation. The presence of the `Link` header communicates to the client that +the entire result set has not been returned and another request must be +issued. If the header is not present, the client can assume that all results +have been recieved. + +> __NOTE:__ In the request template above, note that the brackets +> are required. For example, if the url is +> `http://example.com/v2/_catalog?n=20&last=b`, the value of the header would +> be `; rel="next"`. Please see +> [RFC5988](https://tools.ietf.org/html/rfc5988) for details. + +Compliant client implementations should always use the `Link` header +value when proceeding through results linearly. The client may construct URLs +to skip forward in the catalog. + +To get the next result set, a client would issue the request as follows, using +the URL encoded in the described `Link` header: + +``` +GET /v2/_catalog?n=&last= +``` + +The above process should then be repeated until the `Link` header is no longer +set. + +The catalog result set is represented abstractly as a lexically sorted list, +where the position in that list can be specified by the query term `last`. The +entries in the response start _after_ the term specified by `last`, up to `n` +entries. + +The behavior of `last` is quite simple when demonstrated with an example. Let +us say the registry has the following repositories: + +``` +a +b +c +d +``` + +If the value of `n` is 2, _a_ and _b_ will be returned on the first response. +The `Link` header returned on the response will have `n` set to 2 and last set +to _b_: + +``` +Link: <?n=2&last=b>; rel="next" +``` + +The client can then issue the request with above value from the `Link` header, +receiving the values _c_ and _d_. Note that n may change on second to last +response or be omitted fully, if the server may so choose. ### Listing Image Tags @@ -781,8 +903,12 @@ large. If such a response is expected, one should use the pagination. #### Pagination -Paginated tag results can be retrieved by adding the appropriate pagination -parameters. Starting a paginated flow may begin as follows: +Paginated tag results can be retrieved by adding the appropriate parameters to +the request URL described above. The behavior of tag pagination is identical +to that specified for catalog pagination. We cover a simple flow to highlight +any differences. + +Starting a paginated flow may begin as follows: ``` GET /v2//tags/list?n= @@ -795,6 +921,7 @@ response to such a request would look as follows: ``` 200 OK Content-Type: application/json +Link: <?n=&last=>; rel="next" { "name": , @@ -802,141 +929,21 @@ Content-Type: application/json , ... ] - "next": ?n=&last= } ``` -> __TODO(stevvooe):__ Consider using a Header here, rather than a body parameter. A -header would allow one to issue the next request before parsing the response -body. - To get the next result set, a client would issue the request as follows, using -the value of "next" from the response body: +the value encoded in the [RFC5988](https://tools.ietf.org/html/rfc5988) `Link` +header: ``` GET /v2//tags/list?n=&last= ``` -The above process should then be repeated until the `next` parameter is no -longer set in the response. - -The behavior of `last` is quite simple and can be demonstrated with an -example. Let's say the repository has the following tags: - -``` -a -b -c -d -``` - -If the value of `n` is 2, _a_ and _b_ will be returned on the first response. -The `next` url within the respone will have `n` set to 2 and last set to _b_: - -``` -"next": ?n=2&last=b -``` - -The client can then issue the response, receiving the values _c_ and _d_. Note -that n may change on second to last response or be omitted fully, if the -server may so choose. - -### Listing Repositories - -Images are stored in collections, known as a _repository_, which is keyed by a -`name`, as seen throughout the API specification. A registry instance may -contain several repositories. The list of available repositories, or image -names, is made available through the _catalog_. - -The catalog for a given registry can be retrived with the following request: - -``` -GET /v2/_catalog -``` - -The response will be in the following format: - -``` -200 OK -Content-Type: application/json - -{ - "repositories": [ - , - ... - ] -} -``` - -For registries with a large number of repositories, this response may be quite -large. If such a response is expected, one should use the pagination. - -#### Pagination - -Paginated repository results can be retrieved by adding the appropriate -pagination parameters, which are similar to those available in the tag API. -Starting a paginated flow may begin as follows: - -``` -GET /v2/_catalog?n= -``` - -The above specifies that a catalog response should be returned, from the start of -the result set, ordered lexically, limiting the number of results to `n`. The -response to such a request would look as follows: - -``` -200 OK -Content-Type: application/json - -{ - "repositories": [ - , - ... - ] - "next": ?n=&last= -} -``` - -> __TODO(stevvooe):__ Consider using a Header here, rather than a body parameter. A -header would allow one to issue the next request before parsing the response -body. - -To get the next result set, a client would issue the request as follows, using -the value of "next" from the response body: - -``` -GET /v2/_catalog?n=&last= -``` - -The above process should then be repeated until the `next` parameter is no -longer set in the response. - -The result set of repository names is represented abstractly as a lexically -sorted list, where the position in that list can be specified by the query -term `last`. The entries in the response start _after_ the term specified by -`last`, up to `n` entries. - -The behavior of `last` is quite simple when demonstrated with an example. -Let's say the registry has the following repositories: - -``` -a -b -c -d -``` - -If the value of `n` is 2, _a_ and _b_ will be returned on the first response. -The `next` url within the respone will have `n` set to 2 and last set to _b_: - -``` -"next": ?n=2&last=b -``` - -The client can then issue the request with above value of `next`, receiving -the values _c_ and _d_. Note that n may change on second to last response or -be omitted fully, if the server may so choose. +The above process should then be repeated until the `Link` header is no longer +set in the response. The behavior of the `last` parameter, the provided +response result, lexical ordering and encoding of the `Link` header are +identical to that of catalog pagination. ### Deleting an Image @@ -1142,6 +1149,7 @@ The following parameters should be specified on the request: ``` 200 OK Content-Length: +Link: <?n=&last=>; rel="next" Content-Type: application/json; charset=utf-8 { @@ -1256,22 +1264,17 @@ Content-Type: application/json; charset=utf-8 , ... ], - "next": "?last=&n=" } ``` A list of tags for the named repository. -The following fields may be returned in the response body: - -|Name|Description| -|----|-----------| -|`next`|Provides the URL to get the next set of results, if available.| The following headers will be returned with the response: |Name|Description| |----|-----------| |`Content-Length`|Length of the JSON response body.| +|`Link`|RFC5988 compliant rel='next' with URL to next result set, if available| @@ -3154,6 +3157,7 @@ Request an unabridged list of repositories available. ``` 200 OK Content-Length: +Link: <?n=&last=>; rel="next" Content-Type: application/json; charset=utf-8 { @@ -3210,17 +3214,13 @@ Content-Type: application/json; charset=utf-8 ``` -The following fields may be returned in the response body: - -|Name|Description| -|----|-----------| -|`next`|Provides the URL to get the next set of results, if available.| The following headers will be returned with the response: |Name|Description| |----|-----------| |`Content-Length`|Length of the JSON response body.| +|`Link`|RFC5988 compliant rel='next' with URL to next result set, if available| diff --git a/docs/spec/api.md.tmpl b/docs/spec/api.md.tmpl index 218f30c92..ceee0c727 100644 --- a/docs/spec/api.md.tmpl +++ b/docs/spec/api.md.tmpl @@ -754,7 +754,129 @@ each unknown blob. The response format is as follows: ] } +### Listing Repositories +Images are stored in collections, known as a _repository_, which is keyed by a +`name`, as seen throughout the API specification. A registry instance may +contain several repositories. The list of available repositories is made +available through the _catalog_. + +The catalog for a given registry can be retrived with the following request: + +``` +GET /v2/_catalog +``` + +The response will be in the following format: + +``` +200 OK +Content-Type: application/json + +{ + "repositories": [ + , + ... + ] +} +``` + +Note that the contents of the response are specific to the registry +implementation. Some registries may opt to provide a full catalog output, +limit it based on the user's access level or omit upstream results, if +providing mirroring functionality. Subsequently, the presence of a repository +in the catalog listing only means that the registry *may* provide access to +the repository at the time of the request. Conversely, a missing entry does +*not* mean that the registry does not have the repository. More succinctly, +the presence of a repository only guarantees that it is there but not that it +is _not_ there. + +For registries with a large number of repositories, this response may be quite +large. If such a response is expected, one should use pagination. + +#### Pagination + +Paginated catalog results can be retrieved by adding an `n` parameter to the +request URL, declaring that the response should be limited to `n` results. +Starting a paginated flow begins as follows: + +``` +GET /v2/_catalog?n= +``` + +The above specifies that a catalog response should be returned, from the start of +the result set, ordered lexically, limiting the number of results to `n`. The +response to such a request would look as follows: + +``` +200 OK +Content-Type: application/json +Link: <?n=&last=>; rel="next" + +{ + "repositories": [ + , + ... + ] +} +``` + +The above includes the _first_ `n` entries from the result set. To get the +_next_ `n` entries, one can create a URL where the argument `last` has the +value from `repositories[len(repositories)-1]`. If there are indeed more +results, the URL for the next block is encoded in an +[RFC5988](https://tools.ietf.org/html/rfc5988) `Link` header, as a "next" +relation. The presence of the `Link` header communicates to the client that +the entire result set has not been returned and another request must be +issued. If the header is not present, the client can assume that all results +have been recieved. + +> __NOTE:__ In the request template above, note that the brackets +> are required. For example, if the url is +> `http://example.com/v2/_catalog?n=20&last=b`, the value of the header would +> be `; rel="next"`. Please see +> [RFC5988](https://tools.ietf.org/html/rfc5988) for details. + +Compliant client implementations should always use the `Link` header +value when proceeding through results linearly. The client may construct URLs +to skip forward in the catalog. + +To get the next result set, a client would issue the request as follows, using +the URL encoded in the described `Link` header: + +``` +GET /v2/_catalog?n=&last= +``` + +The above process should then be repeated until the `Link` header is no longer +set. + +The catalog result set is represented abstractly as a lexically sorted list, +where the position in that list can be specified by the query term `last`. The +entries in the response start _after_ the term specified by `last`, up to `n` +entries. + +The behavior of `last` is quite simple when demonstrated with an example. Let +us say the registry has the following repositories: + +``` +a +b +c +d +``` + +If the value of `n` is 2, _a_ and _b_ will be returned on the first response. +The `Link` header returned on the response will have `n` set to 2 and last set +to _b_: + +``` +Link: <?n=2&last=b>; rel="next" +``` + +The client can then issue the request with above value from the `Link` header, +receiving the values _c_ and _d_. Note that n may change on second to last +response or be omitted fully, if the server may so choose. ### Listing Image Tags @@ -781,8 +903,12 @@ large. If such a response is expected, one should use the pagination. #### Pagination -Paginated tag results can be retrieved by adding the appropriate pagination -parameters. Starting a paginated flow may begin as follows: +Paginated tag results can be retrieved by adding the appropriate parameters to +the request URL described above. The behavior of tag pagination is identical +to that specified for catalog pagination. We cover a simple flow to highlight +any differences. + +Starting a paginated flow may begin as follows: ``` GET /v2//tags/list?n= @@ -795,6 +921,7 @@ response to such a request would look as follows: ``` 200 OK Content-Type: application/json +Link: <?n=&last=>; rel="next" { "name": , @@ -802,141 +929,21 @@ Content-Type: application/json , ... ] - "next": ?n=&last= } ``` -> __TODO(stevvooe):__ Consider using a Header here, rather than a body parameter. A -header would allow one to issue the next request before parsing the response -body. - To get the next result set, a client would issue the request as follows, using -the value of "next" from the response body: +the value encoded in the [RFC5988](https://tools.ietf.org/html/rfc5988) `Link` +header: ``` GET /v2//tags/list?n=&last= ``` -The above process should then be repeated until the `next` parameter is no -longer set in the response. - -The behavior of `last` is quite simple and can be demonstrated with an -example. Let's say the repository has the following tags: - -``` -a -b -c -d -``` - -If the value of `n` is 2, _a_ and _b_ will be returned on the first response. -The `next` url within the respone will have `n` set to 2 and last set to _b_: - -``` -"next": ?n=2&last=b -``` - -The client can then issue the response, receiving the values _c_ and _d_. Note -that n may change on second to last response or be omitted fully, if the -server may so choose. - -### Listing Repositories - -Images are stored in collections, known as a _repository_, which is keyed by a -`name`, as seen throughout the API specification. A registry instance may -contain several repositories. The list of available repositories, or image -names, is made available through the _catalog_. - -The catalog for a given registry can be retrived with the following request: - -``` -GET /v2/_catalog -``` - -The response will be in the following format: - -``` -200 OK -Content-Type: application/json - -{ - "repositories": [ - , - ... - ] -} -``` - -For registries with a large number of repositories, this response may be quite -large. If such a response is expected, one should use the pagination. - -#### Pagination - -Paginated repository results can be retrieved by adding the appropriate -pagination parameters, which are similar to those available in the tag API. -Starting a paginated flow may begin as follows: - -``` -GET /v2/_catalog?n= -``` - -The above specifies that a catalog response should be returned, from the start of -the result set, ordered lexically, limiting the number of results to `n`. The -response to such a request would look as follows: - -``` -200 OK -Content-Type: application/json - -{ - "repositories": [ - , - ... - ] - "next": ?n=&last= -} -``` - -> __TODO(stevvooe):__ Consider using a Header here, rather than a body parameter. A -header would allow one to issue the next request before parsing the response -body. - -To get the next result set, a client would issue the request as follows, using -the value of "next" from the response body: - -``` -GET /v2/_catalog?n=&last= -``` - -The above process should then be repeated until the `next` parameter is no -longer set in the response. - -The result set of repository names is represented abstractly as a lexically -sorted list, where the position in that list can be specified by the query -term `last`. The entries in the response start _after_ the term specified by -`last`, up to `n` entries. - -The behavior of `last` is quite simple when demonstrated with an example. -Let's say the registry has the following repositories: - -``` -a -b -c -d -``` - -If the value of `n` is 2, _a_ and _b_ will be returned on the first response. -The `next` url within the respone will have `n` set to 2 and last set to _b_: - -``` -"next": ?n=2&last=b -``` - -The client can then issue the request with above value of `next`, receiving -the values _c_ and _d_. Note that n may change on second to last response or -be omitted fully, if the server may so choose. +The above process should then be repeated until the `Link` header is no longer +set in the response. The behavior of the `last` parameter, the provided +response result, lexical ordering and encoding of the `Link` header are +identical to that of catalog pagination. ### Deleting an Image diff --git a/registry/api/v2/descriptors.go b/registry/api/v2/descriptors.go index 4eec6492b..ee895b722 100644 --- a/registry/api/v2/descriptors.go +++ b/registry/api/v2/descriptors.go @@ -87,6 +87,13 @@ var ( Format: "", } + linkHeader = ParameterDescriptor{ + Name: "Link", + Type: "link", + Description: "RFC5988 compliant rel='next' with URL to next result set, if available", + Format: `<?n=&last=>; rel="next"`, + } + paginationParameters = []ParameterDescriptor{ { Name: "n", @@ -462,14 +469,7 @@ var routeDescriptors = []RouteDescriptor{ Description: "Length of the JSON response body.", Format: "", }, - }, - Fields: []ParameterDescriptor{ - { - Name: "next", - Type: "url", - Description: "Provides the URL to get the next set of results, if available.", - Format: "", - }, + linkHeader, }, Body: BodyDescriptor{ ContentType: "application/json; charset=utf-8", @@ -479,7 +479,6 @@ var routeDescriptors = []RouteDescriptor{ , ... ], - "next": "?last=&n=" }`, }, }, @@ -1439,14 +1438,7 @@ var routeDescriptors = []RouteDescriptor{ Description: "Length of the JSON response body.", Format: "", }, - }, - Fields: []ParameterDescriptor{ - { - Name: "next", - Type: "url", - Description: "Provides the URL to get the next set of results, if available.", - Format: "", - }, + linkHeader, }, }, }, From 74563efe988d1d6c2b772f3397f6506fe8b972e6 Mon Sep 17 00:00:00 2001 From: Patrick Devine Date: Mon, 13 Jul 2015 13:08:13 -0700 Subject: [PATCH 3/5] Catalog for V2 API Implementation This change adds a basic catalog endpoint to the API, which returns a list, or partial list, of all of the repositories contained in the registry. Calls to this endpoint are somewhat expensive, as every call requires walking a large part of the registry. Instead, to maintain a list of repositories, you would first call the catalog endpoint to get an initial list, and then use the events API to maintain any future repositories. Signed-off-by: Patrick Devine --- registry.go | 9 ++ registry/api/v2/routes.go | 1 + registry/api/v2/urls.go | 12 +++ registry/client/repository.go | 68 +++++++++++++++ registry/client/repository_test.go | 41 +++++++++ registry/handlers/api_test.go | 136 +++++++++++++++++++++++++++++ registry/handlers/app.go | 28 +++++- registry/handlers/catalog.go | 82 +++++++++++++++++ registry/handlers/context.go | 3 + registry/storage/catalog.go | 62 +++++++++++++ registry/storage/catalog_test.go | 127 +++++++++++++++++++++++++++ registry/storage/registry.go | 9 ++ 12 files changed, 577 insertions(+), 1 deletion(-) create mode 100644 registry/handlers/catalog.go create mode 100644 registry/storage/catalog.go create mode 100644 registry/storage/catalog_test.go diff --git a/registry.go b/registry.go index 763a189b2..2a6637a3d 100644 --- a/registry.go +++ b/registry.go @@ -35,6 +35,9 @@ type Namespace interface { // registry may or may not have the repository but should always return a // reference. Repository(ctx context.Context, name string) (Repository, error) + + // Catalog returns a reference which can be used for listing repositories + Catalog(ctx context.Context) CatalogService } // ManifestServiceOption is a function argument for Manifest Service methods @@ -112,3 +115,9 @@ type SignatureService interface { // Put stores the signature for the provided digest. Put(dgst digest.Digest, signatures ...[]byte) error } + +// CatalogService provides a way of retrieving the names of each of the repositories +type CatalogService interface { + // Get retrieves repository names from the registry. + Get(n int, q string) (p []string, moreEntries bool, err error) +} diff --git a/registry/api/v2/routes.go b/registry/api/v2/routes.go index d18860f56..5b80d5be7 100644 --- a/registry/api/v2/routes.go +++ b/registry/api/v2/routes.go @@ -16,6 +16,7 @@ const ( var allEndpoints = []string{ RouteNameManifest, + RouteNameCatalog, RouteNameTags, RouteNameBlob, RouteNameBlobUpload, diff --git a/registry/api/v2/urls.go b/registry/api/v2/urls.go index 60aad5659..429743940 100644 --- a/registry/api/v2/urls.go +++ b/registry/api/v2/urls.go @@ -100,6 +100,18 @@ func (ub *URLBuilder) BuildBaseURL() (string, error) { return baseURL.String(), nil } +// BuildCatalogURL constructs a url get a catalog of repositories +func (ub *URLBuilder) BuildCatalogURL(values ...url.Values) (string, error) { + route := ub.cloneRoute(RouteNameCatalog) + + catalogURL, err := route.URL() + if err != nil { + return "", err + } + + return appendValuesURL(catalogURL, values...).String(), nil +} + // BuildTagsURL constructs a url to list the tags in the named repository. func (ub *URLBuilder) BuildTagsURL(name string) (string, error) { route := ub.cloneRoute(RouteNameTags) diff --git a/registry/client/repository.go b/registry/client/repository.go index fc90cb6e4..6d2fd6e79 100644 --- a/registry/client/repository.go +++ b/registry/client/repository.go @@ -444,3 +444,71 @@ func (bs *blobStatter) Stat(ctx context.Context, dgst digest.Digest) (distributi return distribution.Descriptor{}, handleErrorResponse(resp) } } + +// NewCatalog can be used to get a list of repositories +func NewCatalog(ctx context.Context, baseURL string, transport http.RoundTripper) (distribution.CatalogService, error) { + ub, err := v2.NewURLBuilderFromString(baseURL) + if err != nil { + return nil, err + } + + client := &http.Client{ + Transport: transport, + Timeout: 1 * time.Minute, + } + + return &catalog{ + client: client, + ub: ub, + context: ctx, + }, nil +} + +type catalog struct { + client *http.Client + ub *v2.URLBuilder + context context.Context +} + +func (c *catalog) Get(maxEntries int, last string) ([]string, bool, error) { + var repos []string + + values := url.Values{} + + if maxEntries > 0 { + values.Add("n", strconv.Itoa(maxEntries)) + } + + if last != "" { + values.Add("last", last) + } + + u, err := c.ub.BuildCatalogURL(values) + if err != nil { + return nil, false, err + } + + resp, err := c.client.Get(u) + if err != nil { + return nil, false, err + } + defer resp.Body.Close() + + switch resp.StatusCode { + case http.StatusOK: + var ctlg struct { + Repositories []string `json:"repositories"` + } + decoder := json.NewDecoder(resp.Body) + + if err := decoder.Decode(&ctlg); err != nil { + return nil, false, err + } + + repos = ctlg.Repositories + default: + return nil, false, handleErrorResponse(resp) + } + + return repos, false, nil +} diff --git a/registry/client/repository_test.go b/registry/client/repository_test.go index 3a91be980..e9735cd48 100644 --- a/registry/client/repository_test.go +++ b/registry/client/repository_test.go @@ -8,6 +8,7 @@ import ( "log" "net/http" "net/http/httptest" + "strconv" "strings" "testing" "time" @@ -77,6 +78,23 @@ func addTestFetch(repo string, dgst digest.Digest, content []byte, m *testutil.R }) } +func addTestCatalog(content []byte, m *testutil.RequestResponseMap) { + *m = append(*m, testutil.RequestResponseMapping{ + Request: testutil.Request{ + Method: "GET", + Route: "/v2/_catalog", + }, + Response: testutil.Response{ + StatusCode: http.StatusOK, + Body: content, + Headers: http.Header(map[string][]string{ + "Content-Length": {strconv.Itoa(len(content))}, + "Content-Type": {"application/json; charset=utf-8"}, + }), + }, + }) +} + func TestBlobFetch(t *testing.T) { d1, b1 := newRandomBlob(1024) var m testutil.RequestResponseMap @@ -732,3 +750,26 @@ func TestManifestUnauthorized(t *testing.T) { t.Fatalf("Unexpected message value: %q, expected %q", v2Err.Message, expected) } } + +func TestCatalog(t *testing.T) { + var m testutil.RequestResponseMap + addTestCatalog([]byte("{\"repositories\":[\"foo\", \"bar\", \"baz\"]}"), &m) + + e, c := testServer(m) + defer c() + + ctx := context.Background() + ctlg, err := NewCatalog(ctx, e, nil) + if err != nil { + t.Fatal(err) + } + + repos, _, err := ctlg.Get(0, "") + if err != nil { + t.Fatal(err) + } + + if len(repos) != 3 { + t.Fatalf("Got wrong number of repos") + } +} diff --git a/registry/handlers/api_test.go b/registry/handlers/api_test.go index 8d6319417..d768a116d 100644 --- a/registry/handlers/api_test.go +++ b/registry/handlers/api_test.go @@ -60,6 +60,85 @@ func TestCheckAPI(t *testing.T) { } } +func TestCatalogAPI(t *testing.T) { + env := newTestEnv(t) + + values := url.Values{"last": []string{""}, "n": []string{"100"}} + + catalogURL, err := env.builder.BuildCatalogURL(values) + if err != nil { + t.Fatalf("unexpected error building catalog url: %v", err) + } + + // ----------------------------------- + // try to get an empty catalog + resp, err := http.Get(catalogURL) + if err != nil { + t.Fatalf("unexpected error issuing request: %v", err) + } + defer resp.Body.Close() + + checkResponse(t, "issuing catalog api check", resp, http.StatusOK) + + var ctlg struct { + Repositories []string `json:"repositories"` + } + + dec := json.NewDecoder(resp.Body) + if err := dec.Decode(&ctlg); err != nil { + t.Fatalf("error decoding fetched manifest: %v", err) + } + + // we haven't pushed anything to the registry yet + if ctlg.Repositories != nil { + t.Fatalf("repositories has unexpected values") + } + + if resp.Header.Get("Link") != "" { + t.Fatalf("repositories has more data when none expected") + } + + // ----------------------------------- + // push something to the registry and try again + imageName := "foo/bar" + createRepository(env, t, imageName, "sometag") + + resp, err = http.Get(catalogURL) + if err != nil { + t.Fatalf("unexpected error issuing request: %v", err) + } + defer resp.Body.Close() + + checkResponse(t, "issuing catalog api check", resp, http.StatusOK) + + dec = json.NewDecoder(resp.Body) + if err = dec.Decode(&ctlg); err != nil { + t.Fatalf("error decoding fetched manifest: %v", err) + } + + if len(ctlg.Repositories) != 1 { + t.Fatalf("repositories has unexpected values") + } + + if !contains(ctlg.Repositories, imageName) { + t.Fatalf("didn't find our repository '%s' in the catalog", imageName) + } + + if resp.Header.Get("Link") != "" { + t.Fatalf("repositories has more data when none expected") + } + +} + +func contains(elems []string, e string) bool { + for _, elem := range elems { + if elem == e { + return true + } + } + return false +} + func TestURLPrefix(t *testing.T) { config := configuration.Configuration{ Storage: configuration.Storage{ @@ -869,3 +948,60 @@ func checkErr(t *testing.T, err error, msg string) { t.Fatalf("unexpected error %s: %v", msg, err) } } + +func createRepository(env *testEnv, t *testing.T, imageName string, tag string) { + unsignedManifest := &manifest.Manifest{ + Versioned: manifest.Versioned{ + SchemaVersion: 1, + }, + Name: imageName, + Tag: tag, + FSLayers: []manifest.FSLayer{ + { + BlobSum: "asdf", + }, + { + BlobSum: "qwer", + }, + }, + } + + // Push 2 random layers + expectedLayers := make(map[digest.Digest]io.ReadSeeker) + + for i := range unsignedManifest.FSLayers { + rs, dgstStr, err := testutil.CreateRandomTarFile() + + if err != nil { + t.Fatalf("error creating random layer %d: %v", i, err) + } + dgst := digest.Digest(dgstStr) + + expectedLayers[dgst] = rs + unsignedManifest.FSLayers[i].BlobSum = dgst + + uploadURLBase, _ := startPushLayer(t, env.builder, imageName) + pushLayer(t, env.builder, imageName, dgst, uploadURLBase, rs) + } + + signedManifest, err := manifest.Sign(unsignedManifest, env.pk) + if err != nil { + t.Fatalf("unexpected error signing manifest: %v", err) + } + + payload, err := signedManifest.Payload() + checkErr(t, err, "getting manifest payload") + + dgst, err := digest.FromBytes(payload) + checkErr(t, err, "digesting manifest") + + manifestDigestURL, err := env.builder.BuildManifestURL(imageName, dgst.String()) + checkErr(t, err, "building manifest url") + + resp := putManifest(t, "putting signed manifest", manifestDigestURL, signedManifest) + checkResponse(t, "putting signed manifest", resp, http.StatusAccepted) + checkHeaders(t, resp, http.Header{ + "Location": []string{manifestDigestURL}, + "Docker-Content-Digest": []string{dgst.String()}, + }) +} diff --git a/registry/handlers/app.go b/registry/handlers/app.go index c895222bd..45f97966f 100644 --- a/registry/handlers/app.go +++ b/registry/handlers/app.go @@ -69,6 +69,7 @@ func NewApp(ctx context.Context, configuration configuration.Configuration) *App return http.HandlerFunc(apiBase) }) app.register(v2.RouteNameManifest, imageManifestDispatcher) + app.register(v2.RouteNameCatalog, catalogDispatcher) app.register(v2.RouteNameTags, tagsDispatcher) app.register(v2.RouteNameBlob, blobDispatcher) app.register(v2.RouteNameBlobUpload, blobUploadDispatcher) @@ -366,6 +367,9 @@ func (app *App) dispatcher(dispatch dispatchFunc) http.Handler { // Add username to request logging context.Context = ctxu.WithLogger(context.Context, ctxu.GetLogger(context.Context, "auth.user.name")) + catalog := app.registry.Catalog(context) + context.Catalog = catalog + if app.nameRequired(r) { repository, err := app.registry.Repository(context, getName(context)) @@ -493,6 +497,7 @@ func (app *App) authorized(w http.ResponseWriter, r *http.Request, context *Cont } return fmt.Errorf("forbidden: no repository name") } + accessRecords = appendCatalogAccessRecord(accessRecords, r) } ctx, err := app.accessController.Authorized(context.Context, accessRecords...) @@ -538,7 +543,8 @@ func (app *App) eventBridge(ctx *Context, r *http.Request) notifications.Listene // nameRequired returns true if the route requires a name. func (app *App) nameRequired(r *http.Request) bool { route := mux.CurrentRoute(r) - return route == nil || route.GetName() != v2.RouteNameBase + routeName := route.GetName() + return route == nil || (routeName != v2.RouteNameBase && routeName != v2.RouteNameCatalog) } // apiBase implements a simple yes-man for doing overall checks against the @@ -588,6 +594,26 @@ func appendAccessRecords(records []auth.Access, method string, repo string) []au return records } +// Add the access record for the catalog if it's our current route +func appendCatalogAccessRecord(accessRecords []auth.Access, r *http.Request) []auth.Access { + route := mux.CurrentRoute(r) + routeName := route.GetName() + + if routeName == v2.RouteNameCatalog { + resource := auth.Resource{ + Type: "registry", + Name: "catalog", + } + + accessRecords = append(accessRecords, + auth.Access{ + Resource: resource, + Action: "*", + }) + } + return accessRecords +} + // applyRegistryMiddleware wraps a registry instance with the configured middlewares func applyRegistryMiddleware(registry distribution.Namespace, middlewares []configuration.Middleware) (distribution.Namespace, error) { for _, mw := range middlewares { diff --git a/registry/handlers/catalog.go b/registry/handlers/catalog.go new file mode 100644 index 000000000..fd2af76e7 --- /dev/null +++ b/registry/handlers/catalog.go @@ -0,0 +1,82 @@ +package handlers + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + "strconv" + + "github.com/docker/distribution/registry/api/errcode" + "github.com/gorilla/handlers" +) + +const maximumReturnedEntries = 100 + +func catalogDispatcher(ctx *Context, r *http.Request) http.Handler { + catalogHandler := &catalogHandler{ + Context: ctx, + } + + return handlers.MethodHandler{ + "GET": http.HandlerFunc(catalogHandler.GetCatalog), + } +} + +type catalogHandler struct { + *Context +} + +type catalogAPIResponse struct { + Repositories []string `json:"repositories"` +} + +func (ch *catalogHandler) GetCatalog(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + lastEntry := q.Get("last") + maxEntries, err := strconv.Atoi(q.Get("n")) + if err != nil || maxEntries < 0 { + maxEntries = maximumReturnedEntries + } + + repos, moreEntries, err := ch.Catalog.Get(maxEntries, lastEntry) + if err != nil { + ch.Errors = append(ch.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) + return + } + + w.Header().Set("Content-Type", "application/json; charset=utf-8") + + // Add a link header if there are more entries to retrieve + if moreEntries { + urlStr, err := createLinkEntry(r.URL.String(), maxEntries, repos) + if err != nil { + ch.Errors = append(ch.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) + return + } + w.Header().Set("Link", urlStr) + } + + enc := json.NewEncoder(w) + if err := enc.Encode(catalogAPIResponse{ + Repositories: repos, + }); err != nil { + ch.Errors = append(ch.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) + return + } +} + +// Use the original URL from the request to create a new URL for +// the link header +func createLinkEntry(origURL string, maxEntries int, repos []string) (string, error) { + calledURL, err := url.Parse(origURL) + if err != nil { + return "", err + } + + calledURL.RawQuery = fmt.Sprintf("n=%d&last=%s", maxEntries, repos[len(repos)-1]) + calledURL.Fragment = "" + urlStr := fmt.Sprintf("<%s>; rel=\"next\"", calledURL.String()) + + return urlStr, nil +} diff --git a/registry/handlers/context.go b/registry/handlers/context.go index 85a171237..6625551d4 100644 --- a/registry/handlers/context.go +++ b/registry/handlers/context.go @@ -32,6 +32,9 @@ type Context struct { urlBuilder *v2.URLBuilder + // Catalog allows getting a complete list of the contents of the registry. + Catalog distribution.CatalogService + // TODO(stevvooe): The goal is too completely factor this context and // dispatching out of the web application. Ideally, we should lean on // context.Context for injection of these resources. diff --git a/registry/storage/catalog.go b/registry/storage/catalog.go new file mode 100644 index 000000000..ce184dba4 --- /dev/null +++ b/registry/storage/catalog.go @@ -0,0 +1,62 @@ +package storage + +import ( + "path" + "sort" + "strings" + + log "github.com/Sirupsen/logrus" + "github.com/docker/distribution" + "github.com/docker/distribution/context" + storageDriver "github.com/docker/distribution/registry/storage/driver" +) + +type catalogSvc struct { + ctx context.Context + driver storageDriver.StorageDriver +} + +var _ distribution.CatalogService = &catalogSvc{} + +// Get returns a list, or partial list, of repositories in the registry. +// Because it's a quite expensive operation, it should only be used when building up +// an initial set of repositories. +func (c *catalogSvc) Get(maxEntries int, lastEntry string) ([]string, bool, error) { + log.Infof("Retrieving up to %d entries of the catalog starting with '%s'", maxEntries, lastEntry) + var repos []string + + root, err := defaultPathMapper.path(repositoriesRootPathSpec{}) + if err != nil { + return repos, false, err + } + + Walk(c.ctx, c.driver, root, func(fileInfo storageDriver.FileInfo) error { + filePath := fileInfo.Path() + + // lop the base path off + repoPath := filePath[len(root)+1:] + + _, file := path.Split(repoPath) + if file == "_layers" { + repoPath = strings.TrimSuffix(repoPath, "/_layers") + if repoPath > lastEntry { + repos = append(repos, repoPath) + } + return ErrSkipDir + } else if strings.HasPrefix(file, "_") { + return ErrSkipDir + } + + return nil + }) + + sort.Strings(repos) + + moreEntries := false + if len(repos) > maxEntries { + moreEntries = true + repos = repos[0:maxEntries] + } + + return repos, moreEntries, nil +} diff --git a/registry/storage/catalog_test.go b/registry/storage/catalog_test.go new file mode 100644 index 000000000..8d9f38545 --- /dev/null +++ b/registry/storage/catalog_test.go @@ -0,0 +1,127 @@ +package storage + +import ( + "testing" + + "github.com/docker/distribution" + "github.com/docker/distribution/context" + "github.com/docker/distribution/registry/storage/cache/memory" + "github.com/docker/distribution/registry/storage/driver" + "github.com/docker/distribution/registry/storage/driver/inmemory" +) + +type setupEnv struct { + ctx context.Context + driver driver.StorageDriver + expected []string + registry distribution.Namespace + catalog distribution.CatalogService +} + +func setupFS(t *testing.T) *setupEnv { + d := inmemory.New() + c := []byte("") + ctx := context.Background() + registry := NewRegistryWithDriver(ctx, d, memory.NewInMemoryBlobDescriptorCacheProvider()) + rootpath, _ := defaultPathMapper.path(repositoriesRootPathSpec{}) + + repos := []string{ + "/foo/a/_layers/1", + "/foo/b/_layers/2", + "/bar/c/_layers/3", + "/bar/d/_layers/4", + "/foo/d/in/_layers/5", + "/an/invalid/repo", + "/bar/d/_layers/ignored/dir/6", + } + + for _, repo := range repos { + if err := d.PutContent(ctx, rootpath+repo, c); err != nil { + t.Fatalf("Unable to put to inmemory fs") + } + } + + catalog := registry.Catalog(ctx) + + expected := []string{ + "bar/c", + "bar/d", + "foo/a", + "foo/b", + "foo/d/in", + } + + return &setupEnv{ + ctx: ctx, + driver: d, + expected: expected, + registry: registry, + catalog: catalog, + } +} + +func TestCatalog(t *testing.T) { + env := setupFS(t) + + repos, more, _ := env.catalog.Get(100, "") + + if !testEq(repos, env.expected) { + t.Errorf("Expected catalog repos err") + } + + if more { + t.Errorf("Catalog has more values which we aren't expecting") + } +} + +func TestCatalogInParts(t *testing.T) { + env := setupFS(t) + + chunkLen := 2 + + repos, more, _ := env.catalog.Get(chunkLen, "") + if !testEq(repos, env.expected[0:chunkLen]) { + t.Errorf("Expected catalog first chunk err") + } + + if !more { + t.Errorf("Expected more values in catalog") + } + + lastRepo := repos[len(repos)-1] + repos, more, _ = env.catalog.Get(chunkLen, lastRepo) + + if !testEq(repos, env.expected[chunkLen:chunkLen*2]) { + t.Errorf("Expected catalog second chunk err") + } + + if !more { + t.Errorf("Expected more values in catalog") + } + + lastRepo = repos[len(repos)-1] + repos, more, _ = env.catalog.Get(chunkLen, lastRepo) + + if !testEq(repos, env.expected[chunkLen*2:chunkLen*3-1]) { + t.Errorf("Expected catalog third chunk err") + } + + if more { + t.Errorf("Catalog has more values which we aren't expecting") + } + +} + +func testEq(a, b []string) bool { + if len(a) != len(b) { + return false + } + + for count := range a { + if a[count] != b[count] { + return false + } + } + + return true +} diff --git a/registry/storage/registry.go b/registry/storage/registry.go index cf0fe3e78..170355554 100644 --- a/registry/storage/registry.go +++ b/registry/storage/registry.go @@ -55,6 +55,15 @@ func (reg *registry) Scope() distribution.Scope { return distribution.GlobalScope } +// Catalog returns an instance of the catalog service which can be +// used to dump all of the repositories in a registry +func (reg *registry) Catalog(ctx context.Context) distribution.CatalogService { + return &catalogSvc{ + ctx: ctx, + driver: reg.blobStore.driver, + } +} + // Repository returns an instance of the repository tied to the registry. // Instances should not be shared between goroutines but are cheap to // allocate. In general, they should be request scoped. From b7e26bac741c76cb792f8e14c41a2163b5dae8df Mon Sep 17 00:00:00 2001 From: Patrick Devine Date: Fri, 17 Jul 2015 11:42:47 -0700 Subject: [PATCH 4/5] Create Repositories method This change removes the Catalog Service and replaces it with a more simplistic Repositories() method for obtaining a catalog of all repositories. The Repositories method takes a pre-allocated slice and fills it up to the size of the slice and returns the amount filled. The catalog is returned lexicographically and will start being filled from the last entry passed to Repositories(). If there are no more entries to fill, io.EOF will be returned. Signed-off-by: Patrick Devine Conflicts: registry/client/repository.go registry/handlers/api_test.go --- registry.go | 13 ++--- registry/client/repository.go | 75 +++++++++++++------------- registry/client/repository_test.go | 71 +++++++++++++++++++------ registry/handlers/api_test.go | 85 +++++++++++++++++++++++++++--- registry/handlers/app.go | 3 -- registry/handlers/catalog.go | 25 ++++++--- registry/handlers/context.go | 3 -- registry/storage/catalog.go | 51 +++++++++--------- registry/storage/catalog_test.go | 61 ++++++++++----------- registry/storage/registry.go | 9 ---- 10 files changed, 251 insertions(+), 145 deletions(-) diff --git a/registry.go b/registry.go index 2a6637a3d..1a3de01d0 100644 --- a/registry.go +++ b/registry.go @@ -36,8 +36,11 @@ type Namespace interface { // reference. Repository(ctx context.Context, name string) (Repository, error) - // Catalog returns a reference which can be used for listing repositories - Catalog(ctx context.Context) CatalogService + // Repositories fills 'repos' with a lexigraphically sorted catalog of repositories + // up to the size of 'repos' and returns the value 'n' for the number of entries + // which were filled. 'last' contains an offset in the catalog, and 'err' will be + // set to io.EOF if there are no more entries to obtain. + Repositories(ctx context.Context, repos []string, last string) (n int, err error) } // ManifestServiceOption is a function argument for Manifest Service methods @@ -115,9 +118,3 @@ type SignatureService interface { // Put stores the signature for the provided digest. Put(dgst digest.Digest, signatures ...[]byte) error } - -// CatalogService provides a way of retrieving the names of each of the repositories -type CatalogService interface { - // Get retrieves repository names from the registry. - Get(n int, q string) (p []string, moreEntries bool, err error) -} diff --git a/registry/client/repository.go b/registry/client/repository.go index 6d2fd6e79..6979cc4d2 100644 --- a/registry/client/repository.go +++ b/registry/client/repository.go @@ -445,34 +445,7 @@ func (bs *blobStatter) Stat(ctx context.Context, dgst digest.Digest) (distributi } } -// NewCatalog can be used to get a list of repositories -func NewCatalog(ctx context.Context, baseURL string, transport http.RoundTripper) (distribution.CatalogService, error) { - ub, err := v2.NewURLBuilderFromString(baseURL) - if err != nil { - return nil, err - } - - client := &http.Client{ - Transport: transport, - Timeout: 1 * time.Minute, - } - - return &catalog{ - client: client, - ub: ub, - context: ctx, - }, nil -} - -type catalog struct { - client *http.Client - ub *v2.URLBuilder - context context.Context -} - -func (c *catalog) Get(maxEntries int, last string) ([]string, bool, error) { - var repos []string - +func buildCatalogValues(maxEntries int, last string) url.Values { values := url.Values{} if maxEntries > 0 { @@ -483,14 +456,35 @@ func (c *catalog) Get(maxEntries int, last string) ([]string, bool, error) { values.Add("last", last) } - u, err := c.ub.BuildCatalogURL(values) + return values +} + +// Repositories returns a lexigraphically sorted catalog given a base URL. The 'entries' slice will be filled up to the size +// of the slice, starting at the value provided in 'last'. The number of entries will be returned along with io.EOF if there +// are no more entries +func Repositories(ctx context.Context, baseURL string, entries []string, last string, transport http.RoundTripper) (int, error) { + var numFilled int + var returnErr error + + ub, err := v2.NewURLBuilderFromString(baseURL) if err != nil { - return nil, false, err + return 0, err } - resp, err := c.client.Get(u) + client := &http.Client{ + Transport: transport, + Timeout: 1 * time.Minute, + } + + values := buildCatalogValues(len(entries), last) + u, err := ub.BuildCatalogURL(values) if err != nil { - return nil, false, err + return 0, err + } + + resp, err := client.Get(u) + if err != nil { + return 0, err } defer resp.Body.Close() @@ -502,13 +496,22 @@ func (c *catalog) Get(maxEntries int, last string) ([]string, bool, error) { decoder := json.NewDecoder(resp.Body) if err := decoder.Decode(&ctlg); err != nil { - return nil, false, err + return 0, err + } + + for cnt := range ctlg.Repositories { + entries[cnt] = ctlg.Repositories[cnt] + } + numFilled = len(ctlg.Repositories) + + link := resp.Header.Get("Link") + if link == "" { + returnErr = io.EOF } - repos = ctlg.Repositories default: - return nil, false, handleErrorResponse(resp) + return 0, handleErrorResponse(resp) } - return repos, false, nil + return numFilled, returnErr } diff --git a/registry/client/repository_test.go b/registry/client/repository_test.go index e9735cd48..b803d754b 100644 --- a/registry/client/repository_test.go +++ b/registry/client/repository_test.go @@ -5,6 +5,7 @@ import ( "crypto/rand" "encoding/json" "fmt" + "io" "log" "net/http" "net/http/httptest" @@ -78,19 +79,24 @@ func addTestFetch(repo string, dgst digest.Digest, content []byte, m *testutil.R }) } -func addTestCatalog(content []byte, m *testutil.RequestResponseMap) { +func addTestCatalog(route string, content []byte, link string, m *testutil.RequestResponseMap) { + headers := map[string][]string{ + "Content-Length": {strconv.Itoa(len(content))}, + "Content-Type": {"application/json; charset=utf-8"}, + } + if link != "" { + headers["Link"] = append(headers["Link"], link) + } + *m = append(*m, testutil.RequestResponseMapping{ Request: testutil.Request{ Method: "GET", - Route: "/v2/_catalog", + Route: route, }, Response: testutil.Response{ StatusCode: http.StatusOK, Body: content, - Headers: http.Header(map[string][]string{ - "Content-Length": {strconv.Itoa(len(content))}, - "Content-Type": {"application/json; charset=utf-8"}, - }), + Headers: http.Header(headers), }, }) } @@ -753,23 +759,58 @@ func TestManifestUnauthorized(t *testing.T) { func TestCatalog(t *testing.T) { var m testutil.RequestResponseMap - addTestCatalog([]byte("{\"repositories\":[\"foo\", \"bar\", \"baz\"]}"), &m) + addTestCatalog( + "/v2/_catalog?n=5", + []byte("{\"repositories\":[\"foo\", \"bar\", \"baz\"]}"), "", &m) e, c := testServer(m) defer c() + entries := make([]string, 5) + ctx := context.Background() - ctlg, err := NewCatalog(ctx, e, nil) - if err != nil { + numFilled, err := Repositories(ctx, e, entries, "", nil) + if err != io.EOF { t.Fatal(err) } - repos, _, err := ctlg.Get(0, "") - if err != nil { - t.Fatal(err) - } - - if len(repos) != 3 { + if numFilled != 3 { + t.Fatalf("Got wrong number of repos") + } +} + +func TestCatalogInParts(t *testing.T) { + var m testutil.RequestResponseMap + addTestCatalog( + "/v2/_catalog?n=2", + []byte("{\"repositories\":[\"bar\", \"baz\"]}"), + "", &m) + addTestCatalog( + "/v2/_catalog?last=baz&n=2", + []byte("{\"repositories\":[\"foo\"]}"), + "", &m) + + e, c := testServer(m) + defer c() + + entries := make([]string, 2) + + ctx := context.Background() + numFilled, err := Repositories(ctx, e, entries, "", nil) + if err != nil { + t.Fatal(err) + } + + if numFilled != 2 { + t.Fatalf("Got wrong number of repos") + } + + numFilled, err = Repositories(ctx, e, entries, "baz", nil) + if err != io.EOF { + t.Fatal(err) + } + + if numFilled != 1 { t.Fatalf("Got wrong number of repos") } } diff --git a/registry/handlers/api_test.go b/registry/handlers/api_test.go index d768a116d..4473eb995 100644 --- a/registry/handlers/api_test.go +++ b/registry/handlers/api_test.go @@ -13,6 +13,8 @@ import ( "os" "path" "reflect" + "regexp" + "strconv" "strings" "testing" @@ -60,10 +62,14 @@ func TestCheckAPI(t *testing.T) { } } +// TestCatalogAPI tests the /v2/_catalog endpoint func TestCatalogAPI(t *testing.T) { + chunkLen := 2 env := newTestEnv(t) - values := url.Values{"last": []string{""}, "n": []string{"100"}} + values := url.Values{ + "last": []string{""}, + "n": []string{strconv.Itoa(chunkLen)}} catalogURL, err := env.builder.BuildCatalogURL(values) if err != nil { @@ -90,7 +96,7 @@ func TestCatalogAPI(t *testing.T) { } // we haven't pushed anything to the registry yet - if ctlg.Repositories != nil { + if len(ctlg.Repositories) != 0 { t.Fatalf("repositories has unexpected values") } @@ -100,8 +106,49 @@ func TestCatalogAPI(t *testing.T) { // ----------------------------------- // push something to the registry and try again - imageName := "foo/bar" - createRepository(env, t, imageName, "sometag") + images := []string{"foo/aaaa", "foo/bbbb", "foo/cccc"} + + for _, image := range images { + createRepository(env, t, image, "sometag") + } + + resp, err = http.Get(catalogURL) + if err != nil { + t.Fatalf("unexpected error issuing request: %v", err) + } + defer resp.Body.Close() + + checkResponse(t, "issuing catalog api check", resp, http.StatusOK) + + dec = json.NewDecoder(resp.Body) + if err = dec.Decode(&ctlg); err != nil { + t.Fatalf("error decoding fetched manifest: %v", err) + } + + if len(ctlg.Repositories) != chunkLen { + t.Fatalf("repositories has unexpected values") + } + + for _, image := range images[:chunkLen] { + if !contains(ctlg.Repositories, image) { + t.Fatalf("didn't find our repository '%s' in the catalog", image) + } + } + + link := resp.Header.Get("Link") + if link == "" { + t.Fatalf("repositories has less data than expected") + } + + newValues := checkLink(t, link, chunkLen, ctlg.Repositories[len(ctlg.Repositories)-1]) + + // ----------------------------------- + // get the last chunk of data + + catalogURL, err = env.builder.BuildCatalogURL(newValues) + if err != nil { + t.Fatalf("unexpected error building catalog url: %v", err) + } resp, err = http.Get(catalogURL) if err != nil { @@ -120,14 +167,36 @@ func TestCatalogAPI(t *testing.T) { t.Fatalf("repositories has unexpected values") } - if !contains(ctlg.Repositories, imageName) { - t.Fatalf("didn't find our repository '%s' in the catalog", imageName) + lastImage := images[len(images)-1] + if !contains(ctlg.Repositories, lastImage) { + t.Fatalf("didn't find our repository '%s' in the catalog", lastImage) } - if resp.Header.Get("Link") != "" { - t.Fatalf("repositories has more data when none expected") + link = resp.Header.Get("Link") + if link != "" { + t.Fatalf("catalog has unexpected data") + } +} + +func checkLink(t *testing.T, urlStr string, numEntries int, last string) url.Values { + re := regexp.MustCompile("<(/v2/_catalog.*)>; rel=\"next\"") + matches := re.FindStringSubmatch(urlStr) + + if len(matches) != 2 { + t.Fatalf("Catalog link address response was incorrect") + } + linkURL, _ := url.Parse(matches[1]) + urlValues := linkURL.Query() + + if urlValues.Get("n") != strconv.Itoa(numEntries) { + t.Fatalf("Catalog link entry size is incorrect") } + if urlValues.Get("last") != last { + t.Fatal("Catalog link last entry is incorrect") + } + + return urlValues } func contains(elems []string, e string) bool { diff --git a/registry/handlers/app.go b/registry/handlers/app.go index 45f97966f..f61b2c1e0 100644 --- a/registry/handlers/app.go +++ b/registry/handlers/app.go @@ -367,9 +367,6 @@ func (app *App) dispatcher(dispatch dispatchFunc) http.Handler { // Add username to request logging context.Context = ctxu.WithLogger(context.Context, ctxu.GetLogger(context.Context, "auth.user.name")) - catalog := app.registry.Catalog(context) - context.Catalog = catalog - if app.nameRequired(r) { repository, err := app.registry.Repository(context, getName(context)) diff --git a/registry/handlers/catalog.go b/registry/handlers/catalog.go index fd2af76e7..6ec1fe550 100644 --- a/registry/handlers/catalog.go +++ b/registry/handlers/catalog.go @@ -3,6 +3,7 @@ package handlers import ( "encoding/json" "fmt" + "io" "net/http" "net/url" "strconv" @@ -32,6 +33,8 @@ type catalogAPIResponse struct { } func (ch *catalogHandler) GetCatalog(w http.ResponseWriter, r *http.Request) { + var moreEntries = true + q := r.URL.Query() lastEntry := q.Get("last") maxEntries, err := strconv.Atoi(q.Get("n")) @@ -39,8 +42,12 @@ func (ch *catalogHandler) GetCatalog(w http.ResponseWriter, r *http.Request) { maxEntries = maximumReturnedEntries } - repos, moreEntries, err := ch.Catalog.Get(maxEntries, lastEntry) - if err != nil { + repos := make([]string, maxEntries) + + filled, err := ch.App.registry.Repositories(ch.Context, repos, lastEntry) + if err == io.EOF { + moreEntries = false + } else if err != nil { ch.Errors = append(ch.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) return } @@ -49,7 +56,8 @@ func (ch *catalogHandler) GetCatalog(w http.ResponseWriter, r *http.Request) { // Add a link header if there are more entries to retrieve if moreEntries { - urlStr, err := createLinkEntry(r.URL.String(), maxEntries, repos) + lastEntry = repos[len(repos)-1] + urlStr, err := createLinkEntry(r.URL.String(), maxEntries, lastEntry) if err != nil { ch.Errors = append(ch.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) return @@ -59,7 +67,7 @@ func (ch *catalogHandler) GetCatalog(w http.ResponseWriter, r *http.Request) { enc := json.NewEncoder(w) if err := enc.Encode(catalogAPIResponse{ - Repositories: repos, + Repositories: repos[0:filled], }); err != nil { ch.Errors = append(ch.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) return @@ -68,13 +76,18 @@ func (ch *catalogHandler) GetCatalog(w http.ResponseWriter, r *http.Request) { // Use the original URL from the request to create a new URL for // the link header -func createLinkEntry(origURL string, maxEntries int, repos []string) (string, error) { +func createLinkEntry(origURL string, maxEntries int, lastEntry string) (string, error) { calledURL, err := url.Parse(origURL) if err != nil { return "", err } - calledURL.RawQuery = fmt.Sprintf("n=%d&last=%s", maxEntries, repos[len(repos)-1]) + v := url.Values{} + v.Add("n", strconv.Itoa(maxEntries)) + v.Add("last", lastEntry) + + calledURL.RawQuery = v.Encode() + calledURL.Fragment = "" urlStr := fmt.Sprintf("<%s>; rel=\"next\"", calledURL.String()) diff --git a/registry/handlers/context.go b/registry/handlers/context.go index 6625551d4..85a171237 100644 --- a/registry/handlers/context.go +++ b/registry/handlers/context.go @@ -32,9 +32,6 @@ type Context struct { urlBuilder *v2.URLBuilder - // Catalog allows getting a complete list of the contents of the registry. - Catalog distribution.CatalogService - // TODO(stevvooe): The goal is too completely factor this context and // dispatching out of the web application. Ideally, we should lean on // context.Context for injection of these resources. diff --git a/registry/storage/catalog.go b/registry/storage/catalog.go index ce184dba4..470894b71 100644 --- a/registry/storage/catalog.go +++ b/registry/storage/catalog.go @@ -1,36 +1,38 @@ package storage import ( + "errors" + "io" "path" "sort" "strings" - log "github.com/Sirupsen/logrus" - "github.com/docker/distribution" "github.com/docker/distribution/context" - storageDriver "github.com/docker/distribution/registry/storage/driver" + "github.com/docker/distribution/registry/storage/driver" ) -type catalogSvc struct { - ctx context.Context - driver storageDriver.StorageDriver -} - -var _ distribution.CatalogService = &catalogSvc{} - -// Get returns a list, or partial list, of repositories in the registry. +// Returns a list, or partial list, of repositories in the registry. // Because it's a quite expensive operation, it should only be used when building up // an initial set of repositories. -func (c *catalogSvc) Get(maxEntries int, lastEntry string) ([]string, bool, error) { - log.Infof("Retrieving up to %d entries of the catalog starting with '%s'", maxEntries, lastEntry) - var repos []string +func (reg *registry) Repositories(ctx context.Context, repos []string, last string) (n int, err error) { + var foundRepos []string + var errVal error + + if len(repos) == 0 { + return 0, errors.New("no space in slice") + } root, err := defaultPathMapper.path(repositoriesRootPathSpec{}) if err != nil { - return repos, false, err + return 0, err } - Walk(c.ctx, c.driver, root, func(fileInfo storageDriver.FileInfo) error { + // Walk each of the directories in our storage. Unfortunately since there's no + // guarantee that storage will return files in lexigraphical order, we have + // to store everything another slice, sort it and then copy it back to our + // passed in slice. + + Walk(ctx, reg.blobStore.driver, root, func(fileInfo driver.FileInfo) error { filePath := fileInfo.Path() // lop the base path off @@ -39,8 +41,8 @@ func (c *catalogSvc) Get(maxEntries int, lastEntry string) ([]string, bool, erro _, file := path.Split(repoPath) if file == "_layers" { repoPath = strings.TrimSuffix(repoPath, "/_layers") - if repoPath > lastEntry { - repos = append(repos, repoPath) + if repoPath > last { + foundRepos = append(foundRepos, repoPath) } return ErrSkipDir } else if strings.HasPrefix(file, "_") { @@ -50,13 +52,14 @@ func (c *catalogSvc) Get(maxEntries int, lastEntry string) ([]string, bool, erro return nil }) - sort.Strings(repos) + sort.Strings(foundRepos) + n = copy(repos, foundRepos) - moreEntries := false - if len(repos) > maxEntries { - moreEntries = true - repos = repos[0:maxEntries] + // Signal that we have no more entries by setting EOF + if len(foundRepos) <= len(repos) { + errVal = io.EOF } - return repos, moreEntries, nil + return n, errVal + } diff --git a/registry/storage/catalog_test.go b/registry/storage/catalog_test.go index 8d9f38545..a9a046a77 100644 --- a/registry/storage/catalog_test.go +++ b/registry/storage/catalog_test.go @@ -1,6 +1,7 @@ package storage import ( + "io" "testing" "github.com/docker/distribution" @@ -15,7 +16,6 @@ type setupEnv struct { driver driver.StorageDriver expected []string registry distribution.Namespace - catalog distribution.CatalogService } func setupFS(t *testing.T) *setupEnv { @@ -41,8 +41,6 @@ func setupFS(t *testing.T) *setupEnv { } } - catalog := registry.Catalog(ctx) - expected := []string{ "bar/c", "bar/d", @@ -56,20 +54,21 @@ func setupFS(t *testing.T) *setupEnv { driver: d, expected: expected, registry: registry, - catalog: catalog, } } func TestCatalog(t *testing.T) { env := setupFS(t) - repos, more, _ := env.catalog.Get(100, "") + p := make([]string, 50) - if !testEq(repos, env.expected) { + numFilled, err := env.registry.Repositories(env.ctx, p, "") + + if !testEq(p, env.expected, numFilled) { t.Errorf("Expected catalog repos err") } - if more { + if err != io.EOF { t.Errorf("Catalog has more values which we aren't expecting") } } @@ -78,50 +77,46 @@ func TestCatalogInParts(t *testing.T) { env := setupFS(t) chunkLen := 2 + p := make([]string, chunkLen) - repos, more, _ := env.catalog.Get(chunkLen, "") - if !testEq(repos, env.expected[0:chunkLen]) { + numFilled, err := env.registry.Repositories(env.ctx, p, "") + if err == io.EOF || numFilled != len(p) { + t.Errorf("Expected more values in catalog") + } + + if !testEq(p, env.expected[0:chunkLen], numFilled) { t.Errorf("Expected catalog first chunk err") } - if !more { + lastRepo := p[len(p)-1] + numFilled, err = env.registry.Repositories(env.ctx, p, lastRepo) + + if err == io.EOF || numFilled != len(p) { t.Errorf("Expected more values in catalog") } - lastRepo := repos[len(repos)-1] - repos, more, _ = env.catalog.Get(chunkLen, lastRepo) - - if !testEq(repos, env.expected[chunkLen:chunkLen*2]) { + if !testEq(p, env.expected[chunkLen:chunkLen*2], numFilled) { t.Errorf("Expected catalog second chunk err") } - if !more { - t.Errorf("Expected more values in catalog") - } + lastRepo = p[len(p)-1] + numFilled, err = env.registry.Repositories(env.ctx, p, lastRepo) - lastRepo = repos[len(repos)-1] - repos, more, _ = env.catalog.Get(chunkLen, lastRepo) - - if !testEq(repos, env.expected[chunkLen*2:chunkLen*3-1]) { - t.Errorf("Expected catalog third chunk err") - } - - if more { + if err != io.EOF { t.Errorf("Catalog has more values which we aren't expecting") } -} - -func testEq(a, b []string) bool { - if len(a) != len(b) { - return false + if !testEq(p, env.expected[chunkLen*2:chunkLen*3-1], numFilled) { + t.Errorf("Expected catalog third chunk err") } - for count := range a { - if a[count] != b[count] { +} + +func testEq(a, b []string, size int) bool { + for cnt := 0; cnt < size-1; cnt++ { + if a[cnt] != b[cnt] { return false } } - return true } diff --git a/registry/storage/registry.go b/registry/storage/registry.go index 170355554..cf0fe3e78 100644 --- a/registry/storage/registry.go +++ b/registry/storage/registry.go @@ -55,15 +55,6 @@ func (reg *registry) Scope() distribution.Scope { return distribution.GlobalScope } -// Catalog returns an instance of the catalog service which can be -// used to dump all of the repositories in a registry -func (reg *registry) Catalog(ctx context.Context) distribution.CatalogService { - return &catalogSvc{ - ctx: ctx, - driver: reg.blobStore.driver, - } -} - // Repository returns an instance of the repository tied to the registry. // Instances should not be shared between goroutines but are cheap to // allocate. In general, they should be request scoped. From 14749fdce40d405b020beff9e5f139734e4b9347 Mon Sep 17 00:00:00 2001 From: Patrick Devine Date: Wed, 22 Jul 2015 15:18:03 -0700 Subject: [PATCH 5/5] Add Registry to client bindings for Repositories The way Repositories() was initially called was somewhat different than other parts of the client bindings because there was no way to instantiate a Namespace. This change implements a NewRegistry() function which changes it so that Repositories() can be called the way one would expect. It doesn't implement any of the other functions of Namespaces. Signed-off-by: Patrick Devine --- registry/client/repository.go | 134 +++++++++++++++++------------ registry/client/repository_test.go | 16 +++- 2 files changed, 90 insertions(+), 60 deletions(-) diff --git a/registry/client/repository.go b/registry/client/repository.go index 6979cc4d2..29effcce8 100644 --- a/registry/client/repository.go +++ b/registry/client/repository.go @@ -21,6 +21,83 @@ import ( "github.com/docker/distribution/registry/storage/cache/memory" ) +// Registry provides an interface for calling Repositories, which returns a catalog of repositories. +type Registry interface { + Repositories(ctx context.Context, repos []string, last string) (n int, err error) +} + +// NewRegistry creates a registry namespace which can be used to get a listing of repositories +func NewRegistry(ctx context.Context, baseURL string, transport http.RoundTripper) (Registry, error) { + ub, err := v2.NewURLBuilderFromString(baseURL) + if err != nil { + return nil, err + } + + client := &http.Client{ + Transport: transport, + Timeout: 1 * time.Minute, + } + + return ®istry{ + client: client, + ub: ub, + context: ctx, + }, nil +} + +type registry struct { + client *http.Client + ub *v2.URLBuilder + context context.Context +} + +// Repositories returns a lexigraphically sorted catalog given a base URL. The 'entries' slice will be filled up to the size +// of the slice, starting at the value provided in 'last'. The number of entries will be returned along with io.EOF if there +// are no more entries +func (r *registry) Repositories(ctx context.Context, entries []string, last string) (int, error) { + var numFilled int + var returnErr error + + values := buildCatalogValues(len(entries), last) + u, err := r.ub.BuildCatalogURL(values) + if err != nil { + return 0, err + } + + resp, err := r.client.Get(u) + if err != nil { + return 0, err + } + defer resp.Body.Close() + + switch resp.StatusCode { + case http.StatusOK: + var ctlg struct { + Repositories []string `json:"repositories"` + } + decoder := json.NewDecoder(resp.Body) + + if err := decoder.Decode(&ctlg); err != nil { + return 0, err + } + + for cnt := range ctlg.Repositories { + entries[cnt] = ctlg.Repositories[cnt] + } + numFilled = len(ctlg.Repositories) + + link := resp.Header.Get("Link") + if link == "" { + returnErr = io.EOF + } + + default: + return 0, handleErrorResponse(resp) + } + + return numFilled, returnErr +} + // NewRepository creates a new Repository for the given repository name and base URL func NewRepository(ctx context.Context, name, baseURL string, transport http.RoundTripper) (distribution.Repository, error) { if err := v2.ValidateRepositoryName(name); err != nil { @@ -458,60 +535,3 @@ func buildCatalogValues(maxEntries int, last string) url.Values { return values } - -// Repositories returns a lexigraphically sorted catalog given a base URL. The 'entries' slice will be filled up to the size -// of the slice, starting at the value provided in 'last'. The number of entries will be returned along with io.EOF if there -// are no more entries -func Repositories(ctx context.Context, baseURL string, entries []string, last string, transport http.RoundTripper) (int, error) { - var numFilled int - var returnErr error - - ub, err := v2.NewURLBuilderFromString(baseURL) - if err != nil { - return 0, err - } - - client := &http.Client{ - Transport: transport, - Timeout: 1 * time.Minute, - } - - values := buildCatalogValues(len(entries), last) - u, err := ub.BuildCatalogURL(values) - if err != nil { - return 0, err - } - - resp, err := client.Get(u) - if err != nil { - return 0, err - } - defer resp.Body.Close() - - switch resp.StatusCode { - case http.StatusOK: - var ctlg struct { - Repositories []string `json:"repositories"` - } - decoder := json.NewDecoder(resp.Body) - - if err := decoder.Decode(&ctlg); err != nil { - return 0, err - } - - for cnt := range ctlg.Repositories { - entries[cnt] = ctlg.Repositories[cnt] - } - numFilled = len(ctlg.Repositories) - - link := resp.Header.Get("Link") - if link == "" { - returnErr = io.EOF - } - - default: - return 0, handleErrorResponse(resp) - } - - return numFilled, returnErr -} diff --git a/registry/client/repository_test.go b/registry/client/repository_test.go index b803d754b..232501aa3 100644 --- a/registry/client/repository_test.go +++ b/registry/client/repository_test.go @@ -768,8 +768,13 @@ func TestCatalog(t *testing.T) { entries := make([]string, 5) + r, err := NewRegistry(context.Background(), e, nil) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() - numFilled, err := Repositories(ctx, e, entries, "", nil) + numFilled, err := r.Repositories(ctx, entries, "") if err != io.EOF { t.Fatal(err) } @@ -795,8 +800,13 @@ func TestCatalogInParts(t *testing.T) { entries := make([]string, 2) + r, err := NewRegistry(context.Background(), e, nil) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() - numFilled, err := Repositories(ctx, e, entries, "", nil) + numFilled, err := r.Repositories(ctx, entries, "") if err != nil { t.Fatal(err) } @@ -805,7 +815,7 @@ func TestCatalogInParts(t *testing.T) { t.Fatalf("Got wrong number of repos") } - numFilled, err = Repositories(ctx, e, entries, "baz", nil) + numFilled, err = r.Repositories(ctx, entries, "baz") if err != io.EOF { t.Fatal(err) }