Merge pull request #758 from BrianBland/ng-client-routes-update

Updates client to newer routes and changes "layer" to "blob"
pull/4/head
Olivier Gambier 2014-11-20 15:29:15 -08:00
commit 0111cf2bd4
8 changed files with 261 additions and 211 deletions

View File

@ -11,62 +11,65 @@ import (
"strconv" "strconv"
"github.com/docker/docker-registry" "github.com/docker/docker-registry"
"github.com/docker/docker-registry/digest"
) )
// Client implements the client interface to the registry http api // Client implements the client interface to the registry http api
type Client interface { type Client interface {
// GetImageManifest returns an image manifest for the image at the given // GetImageManifest returns an image manifest for the image at the given
// name, tag pair // name, tag pair.
GetImageManifest(name, tag string) (*registry.ImageManifest, error) GetImageManifest(name, tag string) (*registry.ImageManifest, error)
// PutImageManifest uploads an image manifest for the image at the given // PutImageManifest uploads an image manifest for the image at the given
// name, tag pair // name, tag pair.
PutImageManifest(name, tag string, imageManifest *registry.ImageManifest) error PutImageManifest(name, tag string, imageManifest *registry.ImageManifest) error
// DeleteImage removes the image at the given name, tag pair // DeleteImage removes the image at the given name, tag pair.
DeleteImage(name, tag string) error DeleteImage(name, tag string) error
// ListImageTags returns a list of all image tags with the given repository // ListImageTags returns a list of all image tags with the given repository
// name // name.
ListImageTags(name string) ([]string, error) ListImageTags(name string) ([]string, error)
// GetImageLayer returns the image layer at the given name, tarsum pair in // BlobLength returns the length of the blob stored at the given name,
// the form of an io.ReadCloser with the length of this layer // digest pair.
// A nonzero byteOffset can be provided to receive a partial layer beginning // Returns a length value of -1 on error or if the blob does not exist.
// at the given offset BlobLength(name string, dgst digest.Digest) (int, error)
GetImageLayer(name, tarsum string, byteOffset int) (io.ReadCloser, int, error)
// InitiateLayerUpload starts an image upload for the given name, tarsum // GetBlob returns the blob stored at the given name, digest pair in the
// pair and returns a unique location url to use for other layer upload // form of an io.ReadCloser with the length of this blob.
// methods // A nonzero byteOffset can be provided to receive a partial blob beginning
// Returns a *registry.LayerAlreadyExistsError if the layer already exists // at the given offset.
// on the registry GetBlob(name string, dgst digest.Digest, byteOffset int) (io.ReadCloser, int, error)
InitiateLayerUpload(name, tarsum string) (string, error)
// GetLayerUploadStatus returns the byte offset and length of the layer at // InitiateBlobUpload starts a blob upload in the given repository namespace
// the given upload location // and returns a unique location url to use for other blob upload methods.
GetLayerUploadStatus(location string) (int, int, error) InitiateBlobUpload(name string) (string, error)
// UploadLayer uploads a full image layer to the registry // GetBlobUploadStatus returns the byte offset and length of the blob at the
UploadLayer(location string, layer io.ReadCloser, length int, checksum *registry.Checksum) error // given upload location.
GetBlobUploadStatus(location string) (int, int, error)
// UploadLayerChunk uploads a layer chunk with a given length and startByte // UploadBlob uploads a full blob to the registry.
// to the registry UploadBlob(location string, blob io.ReadCloser, length int, dgst digest.Digest) error
// FinishChunkedLayerUpload must be called to finalize this upload
UploadLayerChunk(location string, layerChunk io.ReadCloser, length, startByte int) error
// FinishChunkedLayerUpload completes a chunked layer upload at a given // UploadBlobChunk uploads a blob chunk with a given length and startByte to
// location // the registry.
FinishChunkedLayerUpload(location string, length int, checksum *registry.Checksum) error // FinishChunkedBlobUpload must be called to finalize this upload.
UploadBlobChunk(location string, blobChunk io.ReadCloser, length, startByte int) error
// CancelLayerUpload deletes all content at the unfinished layer upload // FinishChunkedBlobUpload completes a chunked blob upload at a given
// location and invalidates any future calls to this layer upload // location.
CancelLayerUpload(location string) error FinishChunkedBlobUpload(location string, length int, dgst digest.Digest) error
// CancelBlobUpload deletes all content at the unfinished blob upload
// location and invalidates any future calls to this blob upload.
CancelBlobUpload(location string) error
} }
// New returns a new Client which operates against a registry with the // New returns a new Client which operates against a registry with the
// given base endpoint // given base endpoint
// This endpoint should not include /v2/ or any part of the url after this // This endpoint should not include /v2/ or any part of the url after this.
func New(endpoint string) Client { func New(endpoint string) Client {
return &clientImpl{endpoint} return &clientImpl{endpoint}
} }
@ -220,9 +223,41 @@ func (r *clientImpl) ListImageTags(name string) ([]string, error) {
return tags.Tags, nil return tags.Tags, nil
} }
func (r *clientImpl) GetImageLayer(name, tarsum string, byteOffset int) (io.ReadCloser, int, error) { func (r *clientImpl) BlobLength(name string, dgst digest.Digest) (int, error) {
response, err := http.Head(fmt.Sprintf("%s/v2/%s/blob/%s", r.Endpoint, name, dgst))
if err != nil {
return -1, err
}
defer response.Body.Close()
// TODO(bbland): handle other status codes, like 5xx errors
switch {
case response.StatusCode == http.StatusOK:
lengthHeader := response.Header.Get("Content-Length")
length, err := strconv.ParseInt(lengthHeader, 10, 0)
if err != nil {
return -1, err
}
return int(length), nil
case response.StatusCode == http.StatusNotFound:
return -1, nil
case response.StatusCode >= 400 && response.StatusCode < 500:
errors := new(registry.Errors)
decoder := json.NewDecoder(response.Body)
err = decoder.Decode(&errors)
if err != nil {
return -1, err
}
return -1, errors
default:
response.Body.Close()
return -1, &registry.UnexpectedHTTPStatusError{Status: response.Status}
}
}
func (r *clientImpl) GetBlob(name string, dgst digest.Digest, byteOffset int) (io.ReadCloser, int, error) {
getRequest, err := http.NewRequest("GET", getRequest, err := http.NewRequest("GET",
fmt.Sprintf("%s/v2/%s/layer/%s", r.Endpoint, name, tarsum), nil) fmt.Sprintf("%s/v2/%s/blob/%s", r.Endpoint, name, dgst), nil)
if err != nil { if err != nil {
return nil, 0, err return nil, 0, err
} }
@ -233,9 +268,6 @@ func (r *clientImpl) GetImageLayer(name, tarsum string, byteOffset int) (io.Read
return nil, 0, err return nil, 0, err
} }
if response.StatusCode == http.StatusNotFound {
return nil, 0, &registry.LayerNotFoundError{Name: name, TarSum: tarsum}
}
// TODO(bbland): handle other status codes, like 5xx errors // TODO(bbland): handle other status codes, like 5xx errors
switch { switch {
case response.StatusCode == http.StatusOK: case response.StatusCode == http.StatusOK:
@ -247,7 +279,7 @@ func (r *clientImpl) GetImageLayer(name, tarsum string, byteOffset int) (io.Read
return response.Body, int(length), nil return response.Body, int(length), nil
case response.StatusCode == http.StatusNotFound: case response.StatusCode == http.StatusNotFound:
response.Body.Close() response.Body.Close()
return nil, 0, &registry.LayerNotFoundError{Name: name, TarSum: tarsum} return nil, 0, &registry.BlobNotFoundError{Name: name, Digest: dgst}
case response.StatusCode >= 400 && response.StatusCode < 500: case response.StatusCode >= 400 && response.StatusCode < 500:
errors := new(registry.Errors) errors := new(registry.Errors)
decoder := json.NewDecoder(response.Body) decoder := json.NewDecoder(response.Body)
@ -262,9 +294,9 @@ func (r *clientImpl) GetImageLayer(name, tarsum string, byteOffset int) (io.Read
} }
} }
func (r *clientImpl) InitiateLayerUpload(name, tarsum string) (string, error) { func (r *clientImpl) InitiateBlobUpload(name string) (string, error) {
postRequest, err := http.NewRequest("POST", postRequest, err := http.NewRequest("POST",
fmt.Sprintf("%s/v2/%s/layer/%s/upload/", r.Endpoint, name, tarsum), nil) fmt.Sprintf("%s/v2/%s/blob/upload/", r.Endpoint, name), nil)
if err != nil { if err != nil {
return "", err return "", err
} }
@ -279,8 +311,8 @@ func (r *clientImpl) InitiateLayerUpload(name, tarsum string) (string, error) {
switch { switch {
case response.StatusCode == http.StatusAccepted: case response.StatusCode == http.StatusAccepted:
return response.Header.Get("Location"), nil return response.Header.Get("Location"), nil
case response.StatusCode == http.StatusNotModified: // case response.StatusCode == http.StatusNotFound:
return "", &registry.LayerAlreadyExistsError{Name: name, TarSum: tarsum} // return
case response.StatusCode >= 400 && response.StatusCode < 500: case response.StatusCode >= 400 && response.StatusCode < 500:
errors := new(registry.Errors) errors := new(registry.Errors)
decoder := json.NewDecoder(response.Body) decoder := json.NewDecoder(response.Body)
@ -294,7 +326,7 @@ func (r *clientImpl) InitiateLayerUpload(name, tarsum string) (string, error) {
} }
} }
func (r *clientImpl) GetLayerUploadStatus(location string) (int, int, error) { func (r *clientImpl) GetBlobUploadStatus(location string) (int, int, error) {
response, err := http.Get(fmt.Sprintf("%s%s", r.Endpoint, location)) response, err := http.Get(fmt.Sprintf("%s%s", r.Endpoint, location))
if err != nil { if err != nil {
return 0, 0, err return 0, 0, err
@ -306,7 +338,7 @@ func (r *clientImpl) GetLayerUploadStatus(location string) (int, int, error) {
case response.StatusCode == http.StatusNoContent: case response.StatusCode == http.StatusNoContent:
return parseRangeHeader(response.Header.Get("Range")) return parseRangeHeader(response.Header.Get("Range"))
case response.StatusCode == http.StatusNotFound: case response.StatusCode == http.StatusNotFound:
return 0, 0, &registry.LayerUploadNotFoundError{Location: location} return 0, 0, &registry.BlobUploadNotFoundError{Location: location}
case response.StatusCode >= 400 && response.StatusCode < 500: case response.StatusCode >= 400 && response.StatusCode < 500:
errors := new(registry.Errors) errors := new(registry.Errors)
decoder := json.NewDecoder(response.Body) decoder := json.NewDecoder(response.Body)
@ -320,18 +352,18 @@ func (r *clientImpl) GetLayerUploadStatus(location string) (int, int, error) {
} }
} }
func (r *clientImpl) UploadLayer(location string, layer io.ReadCloser, length int, checksum *registry.Checksum) error { func (r *clientImpl) UploadBlob(location string, blob io.ReadCloser, length int, dgst digest.Digest) error {
defer layer.Close() defer blob.Close()
putRequest, err := http.NewRequest("PUT", putRequest, err := http.NewRequest("PUT",
fmt.Sprintf("%s%s", r.Endpoint, location), layer) fmt.Sprintf("%s%s", r.Endpoint, location), blob)
if err != nil { if err != nil {
return err return err
} }
queryValues := url.Values{} queryValues := url.Values{}
queryValues.Set("length", fmt.Sprint(length)) queryValues.Set("length", fmt.Sprint(length))
queryValues.Set(checksum.HashAlgorithm, checksum.Sum) queryValues.Set("digest", dgst.String())
putRequest.URL.RawQuery = queryValues.Encode() putRequest.URL.RawQuery = queryValues.Encode()
putRequest.Header.Set("Content-Type", "application/octet-stream") putRequest.Header.Set("Content-Type", "application/octet-stream")
@ -348,7 +380,7 @@ func (r *clientImpl) UploadLayer(location string, layer io.ReadCloser, length in
case response.StatusCode == http.StatusCreated: case response.StatusCode == http.StatusCreated:
return nil return nil
case response.StatusCode == http.StatusNotFound: case response.StatusCode == http.StatusNotFound:
return &registry.LayerUploadNotFoundError{Location: location} return &registry.BlobUploadNotFoundError{Location: location}
case response.StatusCode >= 400 && response.StatusCode < 500: case response.StatusCode >= 400 && response.StatusCode < 500:
errors := new(registry.Errors) errors := new(registry.Errors)
decoder := json.NewDecoder(response.Body) decoder := json.NewDecoder(response.Body)
@ -362,11 +394,11 @@ func (r *clientImpl) UploadLayer(location string, layer io.ReadCloser, length in
} }
} }
func (r *clientImpl) UploadLayerChunk(location string, layerChunk io.ReadCloser, length, startByte int) error { func (r *clientImpl) UploadBlobChunk(location string, blobChunk io.ReadCloser, length, startByte int) error {
defer layerChunk.Close() defer blobChunk.Close()
putRequest, err := http.NewRequest("PUT", putRequest, err := http.NewRequest("PUT",
fmt.Sprintf("%s%s", r.Endpoint, location), layerChunk) fmt.Sprintf("%s%s", r.Endpoint, location), blobChunk)
if err != nil { if err != nil {
return err return err
} }
@ -389,17 +421,17 @@ func (r *clientImpl) UploadLayerChunk(location string, layerChunk io.ReadCloser,
case response.StatusCode == http.StatusAccepted: case response.StatusCode == http.StatusAccepted:
return nil return nil
case response.StatusCode == http.StatusRequestedRangeNotSatisfiable: case response.StatusCode == http.StatusRequestedRangeNotSatisfiable:
lastValidRange, layerSize, err := parseRangeHeader(response.Header.Get("Range")) lastValidRange, blobSize, err := parseRangeHeader(response.Header.Get("Range"))
if err != nil { if err != nil {
return err return err
} }
return &registry.LayerUploadInvalidRangeError{ return &registry.BlobUploadInvalidRangeError{
Location: location, Location: location,
LastValidRange: lastValidRange, LastValidRange: lastValidRange,
LayerSize: layerSize, BlobSize: blobSize,
} }
case response.StatusCode == http.StatusNotFound: case response.StatusCode == http.StatusNotFound:
return &registry.LayerUploadNotFoundError{Location: location} return &registry.BlobUploadNotFoundError{Location: location}
case response.StatusCode >= 400 && response.StatusCode < 500: case response.StatusCode >= 400 && response.StatusCode < 500:
errors := new(registry.Errors) errors := new(registry.Errors)
decoder := json.NewDecoder(response.Body) decoder := json.NewDecoder(response.Body)
@ -413,7 +445,7 @@ func (r *clientImpl) UploadLayerChunk(location string, layerChunk io.ReadCloser,
} }
} }
func (r *clientImpl) FinishChunkedLayerUpload(location string, length int, checksum *registry.Checksum) error { func (r *clientImpl) FinishChunkedBlobUpload(location string, length int, dgst digest.Digest) error {
putRequest, err := http.NewRequest("PUT", putRequest, err := http.NewRequest("PUT",
fmt.Sprintf("%s%s", r.Endpoint, location), nil) fmt.Sprintf("%s%s", r.Endpoint, location), nil)
if err != nil { if err != nil {
@ -422,7 +454,7 @@ func (r *clientImpl) FinishChunkedLayerUpload(location string, length int, check
queryValues := new(url.Values) queryValues := new(url.Values)
queryValues.Set("length", fmt.Sprint(length)) queryValues.Set("length", fmt.Sprint(length))
queryValues.Set(checksum.HashAlgorithm, checksum.Sum) queryValues.Set("digest", dgst.String())
putRequest.URL.RawQuery = queryValues.Encode() putRequest.URL.RawQuery = queryValues.Encode()
putRequest.Header.Set("Content-Type", "application/octet-stream") putRequest.Header.Set("Content-Type", "application/octet-stream")
@ -441,7 +473,7 @@ func (r *clientImpl) FinishChunkedLayerUpload(location string, length int, check
case response.StatusCode == http.StatusCreated: case response.StatusCode == http.StatusCreated:
return nil return nil
case response.StatusCode == http.StatusNotFound: case response.StatusCode == http.StatusNotFound:
return &registry.LayerUploadNotFoundError{Location: location} return &registry.BlobUploadNotFoundError{Location: location}
case response.StatusCode >= 400 && response.StatusCode < 500: case response.StatusCode >= 400 && response.StatusCode < 500:
errors := new(registry.Errors) errors := new(registry.Errors)
decoder := json.NewDecoder(response.Body) decoder := json.NewDecoder(response.Body)
@ -455,7 +487,7 @@ func (r *clientImpl) FinishChunkedLayerUpload(location string, length int, check
} }
} }
func (r *clientImpl) CancelLayerUpload(location string) error { func (r *clientImpl) CancelBlobUpload(location string) error {
deleteRequest, err := http.NewRequest("DELETE", deleteRequest, err := http.NewRequest("DELETE",
fmt.Sprintf("%s%s", r.Endpoint, location), nil) fmt.Sprintf("%s%s", r.Endpoint, location), nil)
if err != nil { if err != nil {
@ -473,7 +505,7 @@ func (r *clientImpl) CancelLayerUpload(location string) error {
case response.StatusCode == http.StatusNoContent: case response.StatusCode == http.StatusNoContent:
return nil return nil
case response.StatusCode == http.StatusNotFound: case response.StatusCode == http.StatusNotFound:
return &registry.LayerUploadNotFoundError{Location: location} return &registry.BlobUploadNotFoundError{Location: location}
case response.StatusCode >= 400 && response.StatusCode < 500: case response.StatusCode >= 400 && response.StatusCode < 500:
errors := new(registry.Errors) errors := new(registry.Errors)
decoder := json.NewDecoder(response.Body) decoder := json.NewDecoder(response.Body)
@ -490,7 +522,7 @@ func (r *clientImpl) CancelLayerUpload(location string) error {
// imageManifestURL is a helper method for returning the full url to an image // imageManifestURL is a helper method for returning the full url to an image
// manifest // manifest
func (r *clientImpl) imageManifestURL(name, tag string) string { func (r *clientImpl) imageManifestURL(name, tag string) string {
return fmt.Sprintf("%s/v2/%s/image/%s", r.Endpoint, name, tag) return fmt.Sprintf("%s/v2/%s/manifest/%s", r.Endpoint, name, tag)
} }
// parseRangeHeader parses out the offset and length from a returned Range // parseRangeHeader parses out the offset and length from a returned Range

View File

@ -10,88 +10,91 @@ import (
"testing" "testing"
"github.com/docker/docker-registry" "github.com/docker/docker-registry"
"github.com/docker/docker-registry/test" "github.com/docker/docker-registry/common/testutil"
"github.com/docker/docker-registry/digest"
) )
type testLayer struct { type testBlob struct {
tarSum string digest digest.Digest
contents []byte contents []byte
} }
func TestPush(t *testing.T) { func TestPush(t *testing.T) {
name := "hello/world" name := "hello/world"
tag := "sometag" tag := "sometag"
testLayers := []testLayer{ testBlobs := []testBlob{
{ {
tarSum: "12345", digest: "12345",
contents: []byte("some contents"), contents: []byte("some contents"),
}, },
{ {
tarSum: "98765", digest: "98765",
contents: []byte("some other contents"), contents: []byte("some other contents"),
}, },
} }
uploadLocations := make([]string, len(testLayers)) uploadLocations := make([]string, len(testBlobs))
layers := make([]registry.FSLayer, len(testLayers)) blobs := make([]registry.FSLayer, len(testBlobs))
history := make([]registry.ManifestHistory, len(testLayers)) history := make([]registry.ManifestHistory, len(testBlobs))
for i, layer := range testLayers { for i, blob := range testBlobs {
uploadLocations[i] = fmt.Sprintf("/v2/%s/layer/%s/upload-location-%d", name, layer.tarSum, i) // TODO(bbland): this is returning the same location for all uploads,
layers[i] = registry.FSLayer{BlobSum: layer.tarSum} // because we can't know which blob will get which location.
history[i] = registry.ManifestHistory{V1Compatibility: layer.tarSum} // It's sort of okay because we're using unique digests, but this needs
// to change at some point.
uploadLocations[i] = fmt.Sprintf("/v2/%s/blob/test-uuid", name)
blobs[i] = registry.FSLayer{BlobSum: blob.digest}
history[i] = registry.ManifestHistory{V1Compatibility: blob.digest.String()}
} }
manifest := &registry.ImageManifest{ manifest := &registry.ImageManifest{
Name: name, Name: name,
Tag: tag, Tag: tag,
Architecture: "x86", Architecture: "x86",
FSLayers: layers, FSLayers: blobs,
History: history, History: history,
SchemaVersion: 1, SchemaVersion: 1,
} }
manifestBytes, err := json.Marshal(manifest) manifestBytes, err := json.Marshal(manifest)
layerRequestResponseMappings := make([]test.RequestResponseMapping, 2*len(testLayers)) blobRequestResponseMappings := make([]testutil.RequestResponseMapping, 2*len(testBlobs))
for i, layer := range testLayers { for i, blob := range testBlobs {
layerRequestResponseMappings[2*i] = test.RequestResponseMapping{ blobRequestResponseMappings[2*i] = testutil.RequestResponseMapping{
Request: test.Request{ Request: testutil.Request{
Method: "POST", Method: "POST",
Route: "/v2/" + name + "/layer/" + layer.tarSum + "/upload/", Route: "/v2/" + name + "/blob/upload/",
}, },
Responses: []test.Response{ Response: testutil.Response{
{ StatusCode: http.StatusAccepted,
StatusCode: http.StatusAccepted, Headers: http.Header(map[string][]string{
Headers: http.Header(map[string][]string{ "Location": {uploadLocations[i]},
"Location": {uploadLocations[i]}, }),
}),
},
}, },
} }
layerRequestResponseMappings[2*i+1] = test.RequestResponseMapping{ blobRequestResponseMappings[2*i+1] = testutil.RequestResponseMapping{
Request: test.Request{ Request: testutil.Request{
Method: "PUT", Method: "PUT",
Route: uploadLocations[i], Route: uploadLocations[i],
Body: layer.contents, QueryParams: map[string][]string{
}, "length": {fmt.Sprint(len(blob.contents))},
Responses: []test.Response{ "digest": {blob.digest.String()},
{
StatusCode: http.StatusCreated,
}, },
Body: blob.contents,
},
Response: testutil.Response{
StatusCode: http.StatusCreated,
}, },
} }
} }
handler := test.NewHandler(append(layerRequestResponseMappings, test.RequestResponseMap{ handler := testutil.NewHandler(append(blobRequestResponseMappings, testutil.RequestResponseMap{
test.RequestResponseMapping{ testutil.RequestResponseMapping{
Request: test.Request{ Request: testutil.Request{
Method: "PUT", Method: "PUT",
Route: "/v2/" + name + "/image/" + tag, Route: "/v2/" + name + "/manifest/" + tag,
Body: manifestBytes, Body: manifestBytes,
}, },
Responses: []test.Response{ Response: testutil.Response{
{ StatusCode: http.StatusOK,
StatusCode: http.StatusOK,
},
}, },
}, },
}...)) }...))
@ -100,11 +103,11 @@ func TestPush(t *testing.T) {
objectStore := &memoryObjectStore{ objectStore := &memoryObjectStore{
mutex: new(sync.Mutex), mutex: new(sync.Mutex),
manifestStorage: make(map[string]*registry.ImageManifest), manifestStorage: make(map[string]*registry.ImageManifest),
layerStorage: make(map[string]Layer), layerStorage: make(map[digest.Digest]Layer),
} }
for _, layer := range testLayers { for _, blob := range testBlobs {
l, err := objectStore.Layer(layer.tarSum) l, err := objectStore.Layer(blob.digest)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -114,7 +117,7 @@ func TestPush(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
writer.Write(layer.contents) writer.Write(blob.contents)
writer.Close() writer.Close()
} }
@ -129,61 +132,57 @@ func TestPush(t *testing.T) {
func TestPull(t *testing.T) { func TestPull(t *testing.T) {
name := "hello/world" name := "hello/world"
tag := "sometag" tag := "sometag"
testLayers := []testLayer{ testBlobs := []testBlob{
{ {
tarSum: "12345", digest: "12345",
contents: []byte("some contents"), contents: []byte("some contents"),
}, },
{ {
tarSum: "98765", digest: "98765",
contents: []byte("some other contents"), contents: []byte("some other contents"),
}, },
} }
layers := make([]registry.FSLayer, len(testLayers)) blobs := make([]registry.FSLayer, len(testBlobs))
history := make([]registry.ManifestHistory, len(testLayers)) history := make([]registry.ManifestHistory, len(testBlobs))
for i, layer := range testLayers { for i, blob := range testBlobs {
layers[i] = registry.FSLayer{BlobSum: layer.tarSum} blobs[i] = registry.FSLayer{BlobSum: blob.digest}
history[i] = registry.ManifestHistory{V1Compatibility: layer.tarSum} history[i] = registry.ManifestHistory{V1Compatibility: blob.digest.String()}
} }
manifest := &registry.ImageManifest{ manifest := &registry.ImageManifest{
Name: name, Name: name,
Tag: tag, Tag: tag,
Architecture: "x86", Architecture: "x86",
FSLayers: layers, FSLayers: blobs,
History: history, History: history,
SchemaVersion: 1, SchemaVersion: 1,
} }
manifestBytes, err := json.Marshal(manifest) manifestBytes, err := json.Marshal(manifest)
layerRequestResponseMappings := make([]test.RequestResponseMapping, len(testLayers)) blobRequestResponseMappings := make([]testutil.RequestResponseMapping, len(testBlobs))
for i, layer := range testLayers { for i, blob := range testBlobs {
layerRequestResponseMappings[i] = test.RequestResponseMapping{ blobRequestResponseMappings[i] = testutil.RequestResponseMapping{
Request: test.Request{ Request: testutil.Request{
Method: "GET", Method: "GET",
Route: "/v2/" + name + "/layer/" + layer.tarSum, Route: "/v2/" + name + "/blob/" + blob.digest.String(),
}, },
Responses: []test.Response{ Response: testutil.Response{
{ StatusCode: http.StatusOK,
StatusCode: http.StatusOK, Body: blob.contents,
Body: layer.contents,
},
}, },
} }
} }
handler := test.NewHandler(append(layerRequestResponseMappings, test.RequestResponseMap{ handler := testutil.NewHandler(append(blobRequestResponseMappings, testutil.RequestResponseMap{
test.RequestResponseMapping{ testutil.RequestResponseMapping{
Request: test.Request{ Request: testutil.Request{
Method: "GET", Method: "GET",
Route: "/v2/" + name + "/image/" + tag, Route: "/v2/" + name + "/manifest/" + tag,
}, },
Responses: []test.Response{ Response: testutil.Response{
{ StatusCode: http.StatusOK,
StatusCode: http.StatusOK, Body: manifestBytes,
Body: manifestBytes,
},
}, },
}, },
}...)) }...))
@ -192,7 +191,7 @@ func TestPull(t *testing.T) {
objectStore := &memoryObjectStore{ objectStore := &memoryObjectStore{
mutex: new(sync.Mutex), mutex: new(sync.Mutex),
manifestStorage: make(map[string]*registry.ImageManifest), manifestStorage: make(map[string]*registry.ImageManifest),
layerStorage: make(map[string]Layer), layerStorage: make(map[digest.Digest]Layer),
} }
err = Pull(client, objectStore, name, tag) err = Pull(client, objectStore, name, tag)
@ -214,8 +213,8 @@ func TestPull(t *testing.T) {
t.Fatal("Incorrect manifest") t.Fatal("Incorrect manifest")
} }
for _, layer := range testLayers { for _, blob := range testBlobs {
l, err := objectStore.Layer(layer.tarSum) l, err := objectStore.Layer(blob.digest)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -226,13 +225,13 @@ func TestPull(t *testing.T) {
} }
defer reader.Close() defer reader.Close()
layerBytes, err := ioutil.ReadAll(reader) blobBytes, err := ioutil.ReadAll(reader)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if string(layerBytes) != string(layer.contents) { if string(blobBytes) != string(blob.contents) {
t.Fatal("Incorrect layer") t.Fatal("Incorrect blob")
} }
} }
} }

View File

@ -9,6 +9,7 @@ import (
"sync" "sync"
"github.com/docker/docker-registry" "github.com/docker/docker-registry"
"github.com/docker/docker-registry/digest"
) )
var ( var (
@ -34,7 +35,7 @@ type ObjectStore interface {
WriteManifest(name, tag string, manifest *registry.ImageManifest) error WriteManifest(name, tag string, manifest *registry.ImageManifest) error
// Layer returns a handle to a layer for reading and writing // Layer returns a handle to a layer for reading and writing
Layer(blobSum string) (Layer, error) Layer(dgst digest.Digest) (Layer, error)
} }
// Layer is a generic image layer interface. // Layer is a generic image layer interface.
@ -56,7 +57,7 @@ type Layer interface {
type memoryObjectStore struct { type memoryObjectStore struct {
mutex *sync.Mutex mutex *sync.Mutex
manifestStorage map[string]*registry.ImageManifest manifestStorage map[string]*registry.ImageManifest
layerStorage map[string]Layer layerStorage map[digest.Digest]Layer
} }
func (objStore *memoryObjectStore) Manifest(name, tag string) (*registry.ImageManifest, error) { func (objStore *memoryObjectStore) Manifest(name, tag string) (*registry.ImageManifest, error) {
@ -78,14 +79,14 @@ func (objStore *memoryObjectStore) WriteManifest(name, tag string, manifest *reg
return nil return nil
} }
func (objStore *memoryObjectStore) Layer(blobSum string) (Layer, error) { func (objStore *memoryObjectStore) Layer(dgst digest.Digest) (Layer, error) {
objStore.mutex.Lock() objStore.mutex.Lock()
defer objStore.mutex.Unlock() defer objStore.mutex.Unlock()
layer, ok := objStore.layerStorage[blobSum] layer, ok := objStore.layerStorage[dgst]
if !ok { if !ok {
layer = &memoryLayer{cond: sync.NewCond(new(sync.Mutex))} layer = &memoryLayer{cond: sync.NewCond(new(sync.Mutex))}
objStore.layerStorage[blobSum] = layer objStore.layerStorage[dgst] = layer
} }
return layer, nil return layer, nil

View File

@ -99,7 +99,7 @@ func pullLayer(c Client, objectStore ObjectStore, name string, fsLayer registry.
} }
defer writer.Close() defer writer.Close()
layerReader, length, err := c.GetImageLayer(name, fsLayer.BlobSum, 0) layerReader, length, err := c.GetBlob(name, fsLayer.BlobSum, 0)
if err != nil { if err != nil {
log.WithFields(log.Fields{ log.WithFields(log.Fields{
"error": err, "error": err,

View File

@ -2,7 +2,6 @@ package client
import ( import (
"bytes" "bytes"
"crypto/sha1"
"io" "io"
"io/ioutil" "io/ioutil"
@ -89,25 +88,10 @@ func pushLayer(c Client, objectStore ObjectStore, name string, fsLayer registry.
}).Warn("Unable to read local layer") }).Warn("Unable to read local layer")
return err return err
} }
defer layerReader.Close()
location, err := c.InitiateLayerUpload(name, fsLayer.BlobSum)
if _, ok := err.(*registry.LayerAlreadyExistsError); ok {
log.WithField("layer", fsLayer).Info("Layer already exists")
return nil
}
if err != nil {
log.WithFields(log.Fields{
"error": err,
"layer": fsLayer,
}).Warn("Unable to upload layer")
return err
}
layerBuffer := new(bytes.Buffer) layerBuffer := new(bytes.Buffer)
checksum := sha1.New() layerSize, err := io.Copy(layerBuffer, layerReader)
teeReader := io.TeeReader(layerReader, checksum)
_, err = io.Copy(layerBuffer, teeReader)
if err != nil { if err != nil {
log.WithFields(log.Fields{ log.WithFields(log.Fields{
"error": err, "error": err,
@ -116,9 +100,29 @@ func pushLayer(c Client, objectStore ObjectStore, name string, fsLayer registry.
return err return err
} }
err = c.UploadLayer(location, ioutil.NopCloser(layerBuffer), layerBuffer.Len(), length, err := c.BlobLength(name, fsLayer.BlobSum)
&registry.Checksum{HashAlgorithm: "sha1", Sum: string(checksum.Sum(nil))}, if err != nil {
) log.WithFields(log.Fields{
"error": err,
"layer": fsLayer,
}).Warn("Unable to check existence of remote layer")
return err
}
if length >= 0 {
log.WithField("layer", fsLayer).Info("Layer already exists")
return nil
}
location, err := c.InitiateBlobUpload(name)
if err != nil {
log.WithFields(log.Fields{
"error": err,
"layer": fsLayer,
}).Warn("Unable to upload layer")
return err
}
err = c.UploadBlob(location, ioutil.NopCloser(layerBuffer), int(layerSize), fsLayer.BlobSum)
if err != nil { if err != nil {
log.WithFields(log.Fields{ log.WithFields(log.Fields{
"error": err, "error": err,

View File

@ -1,4 +1,4 @@
package test package testutil
import ( import (
"bytes" "bytes"
@ -6,16 +6,18 @@ import (
"io" "io"
"io/ioutil" "io/ioutil"
"net/http" "net/http"
"sort"
"strings"
) )
// RequestResponseMap is a mapping from Requests to Responses // RequestResponseMap is an ordered mapping from Requests to Responses
type RequestResponseMap []RequestResponseMapping type RequestResponseMap []RequestResponseMapping
// RequestResponseMapping defines an ordered list of Responses to be sent in // RequestResponseMapping defines a Response to be sent in response to a given
// response to a given Request // Request
type RequestResponseMapping struct { type RequestResponseMapping struct {
Request Request Request Request
Responses []Response Response Response
} }
// TODO(bbland): add support for request headers // TODO(bbland): add support for request headers
@ -28,12 +30,28 @@ type Request struct {
// Route is the http route of this request // Route is the http route of this request
Route string Route string
// QueryParams are the query parameters of this request
QueryParams map[string][]string
// Body is the byte contents of the http request // Body is the byte contents of the http request
Body []byte Body []byte
} }
func (r Request) String() string { func (r Request) String() string {
return fmt.Sprintf("%s %s\n%s", r.Method, r.Route, r.Body) queryString := ""
if len(r.QueryParams) > 0 {
queryString = "?"
keys := make([]string, 0, len(r.QueryParams))
for k := range r.QueryParams {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
queryString += strings.Join(r.QueryParams[k], "&") + "&"
}
queryString = queryString[:len(queryString)-1]
}
return fmt.Sprintf("%s %s%s\n%s", r.Method, r.Route, queryString, r.Body)
} }
// Response is a simplified http.Response object // Response is a simplified http.Response object
@ -61,7 +79,12 @@ type testHandler struct {
func NewHandler(requestResponseMap RequestResponseMap) http.Handler { func NewHandler(requestResponseMap RequestResponseMap) http.Handler {
responseMap := make(map[string][]Response) responseMap := make(map[string][]Response)
for _, mapping := range requestResponseMap { for _, mapping := range requestResponseMap {
responseMap[mapping.Request.String()] = mapping.Responses responses, ok := responseMap[mapping.Request.String()]
if ok {
responseMap[mapping.Request.String()] = append(responses, mapping.Response)
} else {
responseMap[mapping.Request.String()] = []Response{mapping.Response}
}
} }
return &testHandler{responseMap: responseMap} return &testHandler{responseMap: responseMap}
} }
@ -71,9 +94,10 @@ func (app *testHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
requestBody, _ := ioutil.ReadAll(r.Body) requestBody, _ := ioutil.ReadAll(r.Body)
request := Request{ request := Request{
Method: r.Method, Method: r.Method,
Route: r.URL.Path, Route: r.URL.Path,
Body: requestBody, QueryParams: r.URL.Query(),
Body: requestBody,
} }
responses, ok := app.responseMap[request.String()] responses, ok := app.responseMap[request.String()]

View File

@ -3,6 +3,8 @@ package registry
import ( import (
"fmt" "fmt"
"strings" "strings"
"github.com/docker/docker-registry/digest"
) )
// ErrorCode represents the error type. The errors are serialized via strings // ErrorCode represents the error type. The errors are serialized via strings
@ -224,57 +226,44 @@ func (e *ImageManifestNotFoundError) Error() string {
e.Name, e.Tag) e.Name, e.Tag)
} }
// LayerAlreadyExistsError is returned when attempting to create a new layer // BlobNotFoundError is returned when making an operation against a given image
// that already exists in the registry.
type LayerAlreadyExistsError struct {
Name string
TarSum string
}
func (e *LayerAlreadyExistsError) Error() string {
return fmt.Sprintf("Layer already found with Name: %s, TarSum: %s",
e.Name, e.TarSum)
}
// LayerNotFoundError is returned when making an operation against a given image
// layer that does not exist in the registry. // layer that does not exist in the registry.
type LayerNotFoundError struct { type BlobNotFoundError struct {
Name string Name string
TarSum string Digest digest.Digest
} }
func (e *LayerNotFoundError) Error() string { func (e *BlobNotFoundError) Error() string {
return fmt.Sprintf("No layer found with Name: %s, TarSum: %s", return fmt.Sprintf("No blob found with Name: %s, Digest: %s",
e.Name, e.TarSum) e.Name, e.Digest)
} }
// LayerUploadNotFoundError is returned when making a layer upload operation // BlobUploadNotFoundError is returned when making a blob upload operation against an
// against an invalid layer upload location url // invalid blob upload location url.
// This may be the result of using a cancelled, completed, or stale upload // This may be the result of using a cancelled, completed, or stale upload
// location. // location.
type LayerUploadNotFoundError struct { type BlobUploadNotFoundError struct {
Location string Location string
} }
func (e *LayerUploadNotFoundError) Error() string { func (e *BlobUploadNotFoundError) Error() string {
return fmt.Sprintf("No layer found upload found at Location: %s", return fmt.Sprintf("No blob upload found at Location: %s", e.Location)
e.Location)
} }
// LayerUploadInvalidRangeError is returned when attempting to upload an image // BlobUploadInvalidRangeError is returned when attempting to upload an image
// layer chunk that is out of order. // blob chunk that is out of order.
// This provides the known LayerSize and LastValidRange which can be used to // This provides the known BlobSize and LastValidRange which can be used to
// resume the upload. // resume the upload.
type LayerUploadInvalidRangeError struct { type BlobUploadInvalidRangeError struct {
Location string Location string
LastValidRange int LastValidRange int
LayerSize int BlobSize int
} }
func (e *LayerUploadInvalidRangeError) Error() string { func (e *BlobUploadInvalidRangeError) Error() string {
return fmt.Sprintf( return fmt.Sprintf(
"Invalid range provided for upload at Location: %s. Last Valid Range: %d, Layer Size: %d", "Invalid range provided for upload at Location: %s. Last Valid Range: %d, Blob Size: %d",
e.Location, e.LastValidRange, e.LayerSize) e.Location, e.LastValidRange, e.BlobSize)
} }
// UnexpectedHTTPStatusError is returned when an unexpected HTTP status is // UnexpectedHTTPStatusError is returned when an unexpected HTTP status is

View File

@ -4,6 +4,7 @@ import (
"encoding/json" "encoding/json"
"net/http" "net/http"
"github.com/docker/docker-registry/digest"
"github.com/gorilla/handlers" "github.com/gorilla/handlers"
) )
@ -52,7 +53,7 @@ func (m *ImageManifest) UnmarshalJSON(b []byte) error {
// FSLayer is a container struct for BlobSums defined in an image manifest // FSLayer is a container struct for BlobSums defined in an image manifest
type FSLayer struct { type FSLayer struct {
// BlobSum is the tarsum of the referenced filesystem image layer // BlobSum is the tarsum of the referenced filesystem image layer
BlobSum string `json:"blobSum"` BlobSum digest.Digest `json:"blobSum"`
} }
// ManifestHistory stores unstructured v1 compatibility information // ManifestHistory stores unstructured v1 compatibility information