From 53bd19b98f3bf58717d39fd8adb17acab45df89e Mon Sep 17 00:00:00 2001 From: Brian Bland Date: Mon, 10 Nov 2014 19:58:10 -0800 Subject: [PATCH 1/2] Adds a low level registry http client interface and implementation --- client/client.go | 506 +++++++++++++++++++++++++++++++++++++++++++++++ errors.go | 95 ++++++++- images.go | 47 +++++ 3 files changed, 641 insertions(+), 7 deletions(-) create mode 100644 client/client.go diff --git a/client/client.go b/client/client.go new file mode 100644 index 000000000..1270e2564 --- /dev/null +++ b/client/client.go @@ -0,0 +1,506 @@ +package client + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "regexp" + "strconv" + + "github.com/docker/docker-registry" +) + +// Client implements the client interface to the registry http api +type Client interface { + // GetImageManifest returns an image manifest for the image at the given + // name, tag pair + GetImageManifest(name, tag string) (*registry.ImageManifest, error) + + // PutImageManifest uploads an image manifest for the image at the given + // name, tag pair + PutImageManifest(name, tag string, imageManifest *registry.ImageManifest) error + + // DeleteImage removes the image at the given name, tag pair + DeleteImage(name, tag string) error + + // ListImageTags returns a list of all image tags with the given repository + // name + ListImageTags(name string) ([]string, error) + + // GetImageLayer returns the image layer at the given name, tarsum pair in + // the form of an io.ReadCloser with the length of this layer + // A nonzero byteOffset can be provided to receive a partial layer beginning + // at the given offset + GetImageLayer(name, tarsum string, byteOffset int) (io.ReadCloser, int, error) + + // InitiateLayerUpload starts an image upload for the given name, tarsum + // pair and returns a unique location url to use for other layer upload + // methods + // Returns a *registry.LayerAlreadyExistsError if the layer already exists + // on the registry + InitiateLayerUpload(name, tarsum string) (string, error) + + // GetLayerUploadStatus returns the byte offset and length of the layer at + // the given upload location + GetLayerUploadStatus(location string) (int, int, error) + + // UploadLayer uploads a full image layer to the registry + UploadLayer(location string, layer io.ReadCloser, length int, checksum *registry.Checksum) error + + // UploadLayerChunk uploads a layer chunk with a given length and startByte + // to the registry + // FinishChunkedLayerUpload must be called to finalize this upload + UploadLayerChunk(location string, layerChunk io.ReadCloser, length, startByte int) error + + // FinishChunkedLayerUpload completes a chunked layer upload at a given + // location + FinishChunkedLayerUpload(location string, length int, checksum *registry.Checksum) error + + // CancelLayerUpload deletes all content at the unfinished layer upload + // location and invalidates any future calls to this layer upload + CancelLayerUpload(location string) error +} + +// New returns a new Client which operates against a registry with the +// given base endpoint +// This endpoint should not include /v2/ or any part of the url after this +func New(endpoint string) Client { + return &clientImpl{endpoint} +} + +// clientImpl is the default implementation of the Client interface +type clientImpl struct { + Endpoint string +} + +// TODO(bbland): use consistent route generation between server and client + +func (r *clientImpl) GetImageManifest(name, tag string) (*registry.ImageManifest, error) { + response, err := http.Get(r.imageManifestUrl(name, tag)) + if err != nil { + return nil, err + } + defer response.Body.Close() + + // TODO(bbland): handle other status codes, like 5xx errors + switch { + case response.StatusCode == http.StatusOK: + break + case response.StatusCode == http.StatusNotFound: + return nil, ®istry.ImageManifestNotFoundError{name, tag} + case response.StatusCode >= 400 && response.StatusCode < 500: + errors := new(registry.Errors) + decoder := json.NewDecoder(response.Body) + err = decoder.Decode(&errors) + if err != nil { + return nil, err + } + return nil, errors + default: + return nil, ®istry.UnexpectedHttpStatusError{response.Status} + } + + decoder := json.NewDecoder(response.Body) + + manifest := new(registry.ImageManifest) + err = decoder.Decode(manifest) + if err != nil { + return nil, err + } + return manifest, nil +} + +func (r *clientImpl) PutImageManifest(name, tag string, manifest *registry.ImageManifest) error { + manifestBytes, err := json.Marshal(manifest) + if err != nil { + return err + } + + putRequest, err := http.NewRequest("PUT", + r.imageManifestUrl(name, tag), bytes.NewReader(manifestBytes)) + if err != nil { + return err + } + + response, err := http.DefaultClient.Do(putRequest) + if err != nil { + return err + } + defer response.Body.Close() + + // TODO(bbland): handle other status codes, like 5xx errors + switch { + case response.StatusCode == http.StatusOK: + return nil + case response.StatusCode >= 400 && response.StatusCode < 500: + errors := new(registry.Errors) + decoder := json.NewDecoder(response.Body) + err = decoder.Decode(&errors) + if err != nil { + return err + } + return errors + default: + return ®istry.UnexpectedHttpStatusError{response.Status} + } +} + +func (r *clientImpl) DeleteImage(name, tag string) error { + deleteRequest, err := http.NewRequest("DELETE", + r.imageManifestUrl(name, tag), nil) + if err != nil { + return err + } + + response, err := http.DefaultClient.Do(deleteRequest) + if err != nil { + return err + } + defer response.Body.Close() + + // TODO(bbland): handle other status codes, like 5xx errors + switch { + case response.StatusCode == http.StatusNoContent: + break + case response.StatusCode == http.StatusNotFound: + return ®istry.ImageManifestNotFoundError{name, tag} + case response.StatusCode >= 400 && response.StatusCode < 500: + errors := new(registry.Errors) + decoder := json.NewDecoder(response.Body) + err = decoder.Decode(&errors) + if err != nil { + return err + } + return errors + default: + return ®istry.UnexpectedHttpStatusError{response.Status} + } + + return nil +} + +func (r *clientImpl) ListImageTags(name string) ([]string, error) { + response, err := http.Get(fmt.Sprintf("%s/v2/%s/tags", r.Endpoint, name)) + if err != nil { + return nil, err + } + defer response.Body.Close() + + // TODO(bbland): handle other status codes, like 5xx errors + switch { + case response.StatusCode == http.StatusOK: + break + case response.StatusCode == http.StatusNotFound: + return nil, ®istry.RepositoryNotFoundError{name} + case response.StatusCode >= 400 && response.StatusCode < 500: + errors := new(registry.Errors) + decoder := json.NewDecoder(response.Body) + err = decoder.Decode(&errors) + if err != nil { + return nil, err + } + return nil, errors + default: + return nil, ®istry.UnexpectedHttpStatusError{response.Status} + } + + tags := struct { + Tags []string `json:"tags"` + }{} + + decoder := json.NewDecoder(response.Body) + err = decoder.Decode(&tags) + if err != nil { + return nil, err + } + + return tags.Tags, nil +} + +func (r *clientImpl) GetImageLayer(name, tarsum string, byteOffset int) (io.ReadCloser, int, error) { + getRequest, err := http.NewRequest("GET", + fmt.Sprintf("%s/v2/%s/layer/%s", r.Endpoint, name, tarsum), nil) + if err != nil { + return nil, 0, err + } + + getRequest.Header.Add("Range", fmt.Sprintf("%d-", byteOffset)) + response, err := http.DefaultClient.Do(getRequest) + if err != nil { + return nil, 0, err + } + + if response.StatusCode == http.StatusNotFound { + return nil, 0, ®istry.LayerNotFoundError{name, tarsum} + } + // TODO(bbland): handle other status codes, like 5xx errors + switch { + case response.StatusCode == http.StatusOK: + lengthHeader := response.Header.Get("Content-Length") + length, err := strconv.ParseInt(lengthHeader, 10, 0) + if err != nil { + return nil, 0, err + } + return response.Body, int(length), nil + case response.StatusCode == http.StatusNotFound: + response.Body.Close() + return nil, 0, ®istry.LayerNotFoundError{name, tarsum} + case response.StatusCode >= 400 && response.StatusCode < 500: + errors := new(registry.Errors) + decoder := json.NewDecoder(response.Body) + err = decoder.Decode(&errors) + if err != nil { + return nil, 0, err + } + return nil, 0, errors + default: + response.Body.Close() + return nil, 0, ®istry.UnexpectedHttpStatusError{response.Status} + } +} + +func (r *clientImpl) InitiateLayerUpload(name, tarsum string) (string, error) { + postRequest, err := http.NewRequest("POST", + fmt.Sprintf("%s/v2/%s/layer/%s/upload", r.Endpoint, name, tarsum), nil) + if err != nil { + return "", err + } + + response, err := http.DefaultClient.Do(postRequest) + if err != nil { + return "", err + } + defer response.Body.Close() + + // TODO(bbland): handle other status codes, like 5xx errors + switch { + case response.StatusCode == http.StatusAccepted: + return response.Header.Get("Location"), nil + case response.StatusCode == http.StatusNotModified: + return "", ®istry.LayerAlreadyExistsError{name, tarsum} + case response.StatusCode >= 400 && response.StatusCode < 500: + errors := new(registry.Errors) + decoder := json.NewDecoder(response.Body) + err = decoder.Decode(&errors) + if err != nil { + return "", err + } + return "", errors + default: + return "", ®istry.UnexpectedHttpStatusError{response.Status} + } +} + +func (r *clientImpl) GetLayerUploadStatus(location string) (int, int, error) { + response, err := http.Get(fmt.Sprintf("%s%s", r.Endpoint, location)) + if err != nil { + return 0, 0, err + } + defer response.Body.Close() + + // TODO(bbland): handle other status codes, like 5xx errors + switch { + case response.StatusCode == http.StatusNoContent: + return parseRangeHeader(response.Header.Get("Range")) + case response.StatusCode == http.StatusNotFound: + return 0, 0, ®istry.LayerUploadNotFoundError{location} + case response.StatusCode >= 400 && response.StatusCode < 500: + errors := new(registry.Errors) + decoder := json.NewDecoder(response.Body) + err = decoder.Decode(&errors) + if err != nil { + return 0, 0, err + } + return 0, 0, errors + default: + return 0, 0, ®istry.UnexpectedHttpStatusError{response.Status} + } +} + +func (r *clientImpl) UploadLayer(location string, layer io.ReadCloser, length int, checksum *registry.Checksum) error { + defer layer.Close() + + putRequest, err := http.NewRequest("PUT", + fmt.Sprintf("%s%s", r.Endpoint, location), layer) + if err != nil { + return err + } + + queryValues := new(url.Values) + queryValues.Set("length", fmt.Sprint(length)) + queryValues.Set(checksum.HashAlgorithm, checksum.Sum) + putRequest.URL.RawQuery = queryValues.Encode() + + putRequest.Header.Set("Content-Type", "application/octet-stream") + putRequest.Header.Set("Content-Length", fmt.Sprint(length)) + + response, err := http.DefaultClient.Do(putRequest) + if err != nil { + return err + } + defer response.Body.Close() + + // TODO(bbland): handle other status codes, like 5xx errors + switch { + case response.StatusCode == http.StatusCreated: + return nil + case response.StatusCode == http.StatusNotFound: + return ®istry.LayerUploadNotFoundError{location} + case response.StatusCode >= 400 && response.StatusCode < 500: + errors := new(registry.Errors) + decoder := json.NewDecoder(response.Body) + err = decoder.Decode(&errors) + if err != nil { + return err + } + return errors + default: + return ®istry.UnexpectedHttpStatusError{response.Status} + } +} + +func (r *clientImpl) UploadLayerChunk(location string, layerChunk io.ReadCloser, length, startByte int) error { + defer layerChunk.Close() + + putRequest, err := http.NewRequest("PUT", + fmt.Sprintf("%s%s", r.Endpoint, location), layerChunk) + if err != nil { + return err + } + + endByte := startByte + length + + putRequest.Header.Set("Content-Type", "application/octet-stream") + putRequest.Header.Set("Content-Length", fmt.Sprint(length)) + putRequest.Header.Set("Content-Range", + fmt.Sprintf("%d-%d/%d", startByte, endByte, endByte)) + + response, err := http.DefaultClient.Do(putRequest) + if err != nil { + return err + } + defer response.Body.Close() + + // TODO(bbland): handle other status codes, like 5xx errors + switch { + case response.StatusCode == http.StatusAccepted: + return nil + case response.StatusCode == http.StatusRequestedRangeNotSatisfiable: + lastValidRange, layerSize, err := parseRangeHeader(response.Header.Get("Range")) + if err != nil { + return err + } + return ®istry.LayerUploadInvalidRangeError{location, lastValidRange, layerSize} + case response.StatusCode == http.StatusNotFound: + return ®istry.LayerUploadNotFoundError{location} + case response.StatusCode >= 400 && response.StatusCode < 500: + errors := new(registry.Errors) + decoder := json.NewDecoder(response.Body) + err = decoder.Decode(&errors) + if err != nil { + return err + } + return errors + default: + return ®istry.UnexpectedHttpStatusError{response.Status} + } +} + +func (r *clientImpl) FinishChunkedLayerUpload(location string, length int, checksum *registry.Checksum) error { + putRequest, err := http.NewRequest("PUT", + fmt.Sprintf("%s%s", r.Endpoint, location), nil) + if err != nil { + return err + } + + queryValues := new(url.Values) + queryValues.Set("length", fmt.Sprint(length)) + queryValues.Set(checksum.HashAlgorithm, checksum.Sum) + putRequest.URL.RawQuery = queryValues.Encode() + + putRequest.Header.Set("Content-Type", "application/octet-stream") + putRequest.Header.Set("Content-Length", "0") + putRequest.Header.Set("Content-Range", + fmt.Sprintf("%d-%d/%d", length, length, length)) + + response, err := http.DefaultClient.Do(putRequest) + if err != nil { + return err + } + defer response.Body.Close() + + // TODO(bbland): handle other status codes, like 5xx errors + switch { + case response.StatusCode == http.StatusCreated: + return nil + case response.StatusCode == http.StatusNotFound: + return ®istry.LayerUploadNotFoundError{location} + case response.StatusCode >= 400 && response.StatusCode < 500: + errors := new(registry.Errors) + decoder := json.NewDecoder(response.Body) + err = decoder.Decode(&errors) + if err != nil { + return err + } + return errors + default: + return ®istry.UnexpectedHttpStatusError{response.Status} + } +} + +func (r *clientImpl) CancelLayerUpload(location string) error { + deleteRequest, err := http.NewRequest("DELETE", + fmt.Sprintf("%s%s", r.Endpoint, location), nil) + if err != nil { + return err + } + + response, err := http.DefaultClient.Do(deleteRequest) + if err != nil { + return err + } + defer response.Body.Close() + + // TODO(bbland): handle other status codes, like 5xx errors + switch { + case response.StatusCode == http.StatusNoContent: + return nil + case response.StatusCode == http.StatusNotFound: + return ®istry.LayerUploadNotFoundError{location} + case response.StatusCode >= 400 && response.StatusCode < 500: + errors := new(registry.Errors) + decoder := json.NewDecoder(response.Body) + err = decoder.Decode(&errors) + if err != nil { + return err + } + return errors + default: + return ®istry.UnexpectedHttpStatusError{response.Status} + } +} + +// imageManifestUrl is a helper method for returning the full url to an image +// manifest +func (r *clientImpl) imageManifestUrl(name, tag string) string { + return fmt.Sprintf("%s/v2/%s/image/%s", r.Endpoint, name, tag) +} + +// parseRangeHeader parses out the offset and length from a returned Range +// header +func parseRangeHeader(byteRangeHeader string) (int, int, error) { + r := regexp.MustCompile("bytes=0-(\\d+)/(\\d+)") + submatches := r.FindStringSubmatch(byteRangeHeader) + offset, err := strconv.ParseInt(submatches[1], 10, 0) + if err != nil { + return 0, 0, err + } + length, err := strconv.ParseInt(submatches[2], 10, 0) + if err != nil { + return 0, 0, err + } + return int(offset), int(length), nil +} diff --git a/errors.go b/errors.go index 53dcb6bf6..29cfdd705 100644 --- a/errors.go +++ b/errors.go @@ -162,16 +162,97 @@ func (errs *Errors) Error() string { } } -// detailUnknownLayer provides detail for unknown layer errors, returned by +// DetailUnknownLayer provides detail for unknown layer errors, returned by // image manifest push for layers that are not yet transferred. This intended // to only be used on the backend to return detail for this specific error. type DetailUnknownLayer struct { // Unknown should contain the contents of a layer descriptor, which is a - // single json object with the key "blobSum" currently. - Unknown struct { - - // BlobSum contains the uniquely identifying tarsum of the layer. - BlobSum string `json:"blobSum"` - } `json:"unknown"` + // single FSLayer currently. + Unknown FSLayer `json:"unknown"` +} + +// RepositoryNotFoundError is returned when making an operation against a +// repository that does not exist in the registry +type RepositoryNotFoundError struct { + Name string +} + +func (e *RepositoryNotFoundError) Error() string { + return fmt.Sprintf("No repository found with Name: %s", e.Name) +} + +// ImageManifestNotFoundError is returned when making an operation against a +// given image manifest that does not exist in the registry +type ImageManifestNotFoundError struct { + Name string + Tag string +} + +func (e *ImageManifestNotFoundError) Error() string { + return fmt.Sprintf("No manifest found with Name: %s, Tag: %s", + e.Name, e.Tag) +} + +// LayerAlreadyExistsError is returned when attempting to create a new layer +// that already exists in the registry +type LayerAlreadyExistsError struct { + Name string + TarSum string +} + +func (e *LayerAlreadyExistsError) Error() string { + return fmt.Sprintf("Layer already found with Name: %s, TarSum: %s", + e.Name, e.TarSum) +} + +// LayerNotFoundError is returned when making an operation against a given image +// layer that does not exist in the registry +type LayerNotFoundError struct { + Name string + TarSum string +} + +func (e *LayerNotFoundError) Error() string { + return fmt.Sprintf("No layer found with Name: %s, TarSum: %s", + e.Name, e.TarSum) +} + +// LayerUploadNotFoundError is returned when making a layer upload operation +// against an invalid layer upload location url +// This may be the result of using a cancelled, completed, or stale upload +// locationn +type LayerUploadNotFoundError struct { + Location string +} + +func (e *LayerUploadNotFoundError) Error() string { + return fmt.Sprintf("No layer found upload found at Location: %s", + e.Location) +} + +// LayerUploadInvalidRangeError is returned when attempting to upload an image +// layer chunk that is out of order +// This provides the known LayerSize and LastValidRange which can be used to +// resume the upload +type LayerUploadInvalidRangeError struct { + Location string + LastValidRange int + LayerSize int +} + +func (e *LayerUploadInvalidRangeError) Error() string { + return fmt.Sprintf( + "Invalid range provided for upload at Location: %s. Last Valid Range: %d, Layer Size: %d", + e.Location, e.LastValidRange, e.LayerSize) +} + +// UnexpectedHttpStatusError is returned when an unexpected http status is +// returned when making a registry api call +type UnexpectedHttpStatusError struct { + Status string +} + +func (e *UnexpectedHttpStatusError) Error() string { + return fmt.Sprintf("Received unexpected http status: %s", e.Status) } diff --git a/images.go b/images.go index f16a3560f..927a4b60e 100644 --- a/images.go +++ b/images.go @@ -6,6 +6,53 @@ import ( "github.com/gorilla/handlers" ) +// ImageManifest defines the structure of an image manifest +type ImageManifest struct { + // Name is the name of the image's repository + Name string `json:"name"` + + // Tag is the tag of the image specified by this manifest + Tag string `json:"tag"` + + // Architecture is the host architecture on which this image is intended to + // run + Architecture string `json:"architecture"` + + // FSLayers is a list of filesystem layer blobSums contained in this image + FSLayers []FSLayer `json:"fsLayers"` + + // History is a list of unstructured historical data for v1 compatibility + History []ManifestHistory `json:"history"` + + // Signature is the JWT with which the image is signed + Signature string `json:"signature,omitempty"` + + // SchemaVersion is the image manifest schema that this image follows + SchemaVersion int `json:"schemaVersion"` +} + +// FSLayer is a container struct for BlobSums defined in an image manifest +type FSLayer struct { + // BlobSum is the tarsum of the referenced filesystem image layer + BlobSum string `json:"blobSum"` +} + +// ManifestHistory stores unstructured v1 compatibility information +type ManifestHistory struct { + // V1Compatibility is the raw v1 compatibility information + V1Compatibility string `json:"v1Compatibility"` +} + +// Checksum is a container struct for an image checksum +type Checksum struct { + // HashAlgorithm is the algorithm used to compute the checksum + // Supported values: md5, sha1, sha256, sha512 + HashAlgorithm string + + // Sum is the actual checksum value for the given HashAlgorithm + Sum string +} + // imageManifestDispatcher takes the request context and builds the // appropriate handler for handling image manifest requests. func imageManifestDispatcher(ctx *Context, r *http.Request) http.Handler { From b25e16a56ce7fb6decc81909263a93e438f2b064 Mon Sep 17 00:00:00 2001 From: Brian Bland Date: Wed, 12 Nov 2014 15:26:35 -0800 Subject: [PATCH 2/2] Adds Raw bytes field to ImageManifest This can be used for proper json signature validation --- images.go | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/images.go b/images.go index 927a4b60e..84fe217df 100644 --- a/images.go +++ b/images.go @@ -1,6 +1,7 @@ package registry import ( + "encoding/json" "net/http" "github.com/gorilla/handlers" @@ -24,11 +25,27 @@ type ImageManifest struct { // History is a list of unstructured historical data for v1 compatibility History []ManifestHistory `json:"history"` - // Signature is the JWT with which the image is signed - Signature string `json:"signature,omitempty"` - // SchemaVersion is the image manifest schema that this image follows SchemaVersion int `json:"schemaVersion"` + + // Raw is the byte representation of the ImageManifest, used for signature + // verification + Raw []byte `json:"-"` +} + +// imageManifest is used to avoid recursion in unmarshaling +type imageManifest ImageManifest + +func (m *ImageManifest) UnmarshalJSON(b []byte) error { + var manifest imageManifest + err := json.Unmarshal(b, &manifest) + if err != nil { + return err + } + + *m = ImageManifest(manifest) + m.Raw = b + return nil } // FSLayer is a container struct for BlobSums defined in an image manifest