TrueCloudLab
/
distribution
13
0
Fork 3
Code Issues 2 Pull Requests Packages Projects Releases 1 Wiki Activity

Merge pull request #792 from stevvooe/next-generation 

Initial implementation of Manifest HTTP API
pull/4/head
Olivier Gambier 2014-12-01 10:10:35 -08:00
parent 9da63d0bd2 e809796f59
commit af72f6cc94
17 changed files with 749 additions and 263 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

491
api_test.go
View File

@ -1,6 +1,8 @@
package registry
import (
"bytes"
"encoding/json"
"fmt"
"io"
"net/http"
@ -10,7 +12,9 @@ import (
"os"
"testing"
"github.com/Sirupsen/logrus"
"github.com/docker/libtrust"
"github.com/docker/docker-registry/storage"
_ "github.com/docker/docker-registry/storagedriver/inmemory"
"github.com/gorilla/handlers"
@ -34,11 +38,10 @@ func TestLayerAPI(t *testing.T) {
app := NewApp(config)
server := httptest.NewServer(handlers.CombinedLoggingHandler(os.Stderr, app))
router := v2APIRouter()
builder, err := newURLBuilderFromString(server.URL)
u, err := url.Parse(server.URL)
if err != nil {
t.Fatalf("error parsing server url: %v", err)
t.Fatalf("error creating url builder: %v", err)
}
imageName := "foo/bar"
@ -52,154 +55,65 @@ func TestLayerAPI(t *testing.T) {
// -----------------------------------
// Test fetch for non-existent content
r, err := router.GetRoute(routeNameBlob).Host(u.Host).
URL("name", imageName,
"digest", tarSumStr)
layerURL, err := builder.buildLayerURL(imageName, layerDigest)
if err != nil {
t.Fatalf("error building url: %v", err)
}
resp, err := http.Get(r.String())
resp, err := http.Get(layerURL)
if err != nil {
t.Fatalf("unexpected error fetching non-existent layer: %v", err)
}
switch resp.StatusCode {
case http.StatusNotFound:
break // expected
default:
d, err := httputil.DumpResponse(resp, true)
if err != nil {
t.Fatalf("unexpected status fetching non-existent layer: %v, %v", resp.StatusCode, resp.Status)
}
t.Logf("response:\n%s", string(d))
t.Fatalf("unexpected status fetching non-existent layer: %v, %v", resp.StatusCode, resp.Status)
}
checkResponse(t, "fetching non-existent content", resp, http.StatusNotFound)
// ------------------------------------------
// Test head request for non-existent content
resp, err = http.Head(r.String())
if err != nil {
t.Fatalf("unexpected error checking head on non-existent layer: %v", err)
}
switch resp.StatusCode {
case http.StatusNotFound:
break // expected
default:
d, err := httputil.DumpResponse(resp, true)
if err != nil {
t.Fatalf("unexpected status checking head on non-existent layer: %v, %v", resp.StatusCode, resp.Status)
}
t.Logf("response:\n%s", string(d))
t.Fatalf("unexpected status checking head on non-existent layer: %v, %v", resp.StatusCode, resp.Status)
}
// ------------------------------------------
// Upload a layer
r, err = router.GetRoute(routeNameBlobUpload).Host(u.Host).
URL("name", imageName)
if err != nil {
t.Fatalf("error starting layer upload: %v", err)
}
resp, err = http.Post(r.String(), "", nil)
if err != nil {
t.Fatalf("error starting layer upload: %v", err)
}
if resp.StatusCode != http.StatusAccepted {
d, err := httputil.DumpResponse(resp, true)
if err != nil {
t.Fatalf("unexpected status starting layer upload: %v, %v", resp.StatusCode, resp.Status)
}
t.Logf("response:\n%s", string(d))
t.Fatalf("unexpected status starting layer upload: %v, %v", resp.StatusCode, resp.Status)
}
if resp.Header.Get("Location") == "" { // TODO(stevvooe): Need better check here.
t.Fatalf("unexpected Location: %q != %q", resp.Header.Get("Location"), "foo")
}
if resp.Header.Get("Content-Length") != "0" {
t.Fatalf("unexpected content-length: %q != %q", resp.Header.Get("Content-Length"), "0")
}
layerLength, _ := layerFile.Seek(0, os.SEEK_END)
layerFile.Seek(0, os.SEEK_SET)
uploadURLStr := resp.Header.Get("Location")
// TODO(sday): Cancel the layer upload here and restart.
query := url.Values{
"digest": []string{layerDigest.String()},
"length": []string{fmt.Sprint(layerLength)},
}
uploadURL, err := url.Parse(uploadURLStr)
if err != nil {
t.Fatalf("unexpected error parsing url: %v", err)
}
uploadURL.RawQuery = query.Encode()
// Just do a monolithic upload
req, err := http.NewRequest("PUT", uploadURL.String(), layerFile)
if err != nil {
t.Fatalf("unexpected error creating new request: %v", err)
}
resp, err = http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("unexpected error doing put: %v", err)
}
defer resp.Body.Close()
switch resp.StatusCode {
case http.StatusCreated:
break // expected
default:
d, err := httputil.DumpResponse(resp, true)
if err != nil {
t.Fatalf("unexpected status putting chunk: %v, %v", resp.StatusCode, resp.Status)
}
t.Logf("response:\n%s", string(d))
t.Fatalf("unexpected status putting chunk: %v, %v", resp.StatusCode, resp.Status)
}
if resp.Header.Get("Location") == "" {
t.Fatalf("unexpected Location: %q", resp.Header.Get("Location"))
}
if resp.Header.Get("Content-Length") != "0" {
t.Fatalf("unexpected content-length: %q != %q", resp.Header.Get("Content-Length"), "0")
}
layerURL := resp.Header.Get("Location")
// ------------------------
// Use a head request to see if the layer exists.
resp, err = http.Head(layerURL)
if err != nil {
t.Fatalf("unexpected error checking head on non-existent layer: %v", err)
}
switch resp.StatusCode {
case http.StatusOK:
break // expected
default:
d, err := httputil.DumpResponse(resp, true)
if err != nil {
t.Fatalf("unexpected status checking head on layer: %v, %v", resp.StatusCode, resp.Status)
}
checkResponse(t, "checking head on non-existent layer", resp, http.StatusNotFound)
t.Logf("response:\n%s", string(d))
t.Fatalf("unexpected status checking head on layer: %v, %v", resp.StatusCode, resp.Status)
// ------------------------------------------
// Upload a layer
layerUploadURL, err := builder.buildLayerUploadURL(imageName)
if err != nil {
t.Fatalf("error building upload url: %v", err)
}
logrus.Infof("fetch the layer")
resp, err = http.Post(layerUploadURL, "", nil)
if err != nil {
t.Fatalf("error starting layer upload: %v", err)
}
checkResponse(t, "starting layer upload", resp, http.StatusAccepted)
checkHeaders(t, resp, http.Header{
"Location": []string{"*"},
"Content-Length": []string{"0"},
})
layerLength, _ := layerFile.Seek(0, os.SEEK_END)
layerFile.Seek(0, os.SEEK_SET)
// TODO(sday): Cancel the layer upload here and restart.
uploadURLBase := startPushLayer(t, builder, imageName)
pushLayer(t, builder, imageName, layerDigest, uploadURLBase, layerFile)
// ------------------------
// Use a head request to see if the layer exists.
resp, err = http.Head(layerURL)
if err != nil {
t.Fatalf("unexpected error checking head on existing layer: %v", err)
}
checkResponse(t, "checking head on existing layer", resp, http.StatusOK)
checkHeaders(t, resp, http.Header{
"Content-Length": []string{fmt.Sprint(layerLength)},
})
// ----------------
// Fetch the layer!
resp, err = http.Get(layerURL)
@ -207,30 +121,299 @@ func TestLayerAPI(t *testing.T) {
t.Fatalf("unexpected error fetching layer: %v", err)
}
switch resp.StatusCode {
case http.StatusOK:
break // expected
default:
d, err := httputil.DumpResponse(resp, true)
if err != nil {
t.Fatalf("unexpected status fetching layer: %v, %v", resp.StatusCode, resp.Status)
}
t.Logf("response:\n%s", string(d))
t.Fatalf("unexpected status fetching layer: %v, %v", resp.StatusCode, resp.Status)
}
checkResponse(t, "fetching layer", resp, http.StatusOK)
checkHeaders(t, resp, http.Header{
"Content-Length": []string{fmt.Sprint(layerLength)},
})
// Verify the body
verifier := digest.NewDigestVerifier(layerDigest)
io.Copy(verifier, resp.Body)
if !verifier.Verified() {
d, err := httputil.DumpResponse(resp, true)
if err != nil {
t.Fatalf("unexpected status checking head on layer ayo!: %v, %v", resp.StatusCode, resp.Status)
}
t.Logf("response:\n%s", string(d))
t.Fatalf("response body did not pass verification")
}
}
func TestManifestAPI(t *testing.T) {
pk, err := libtrust.GenerateECP256PrivateKey()
if err != nil {
t.Fatalf("unexpected error generating private key: %v", err)
}
config := configuration.Configuration{
Storage: configuration.Storage{
"inmemory": configuration.Parameters{},
},
}
app := NewApp(config)
server := httptest.NewServer(handlers.CombinedLoggingHandler(os.Stderr, app))
builder, err := newURLBuilderFromString(server.URL)
if err != nil {
t.Fatalf("unexpected error creating url builder: %v", err)
}
imageName := "foo/bar"
tag := "thetag"
manifestURL, err := builder.buildManifestURL(imageName, tag)
if err != nil {
t.Fatalf("unexpected error getting manifest url: %v", err)
}
// -----------------------------
// Attempt to fetch the manifest
resp, err := http.Get(manifestURL)
if err != nil {
t.Fatalf("unexpected error getting manifest: %v", err)
}
defer resp.Body.Close()
checkResponse(t, "getting non-existent manifest", resp, http.StatusNotFound)
// TODO(stevvooe): Shoot. The error setup is not working out. The content-
// type headers are being set after writing the status code.
// if resp.Header.Get("Content-Type") != "application/json" {
// t.Fatalf("unexpected content type: %v != 'application/json'",
// resp.Header.Get("Content-Type"))
// }
dec := json.NewDecoder(resp.Body)
var respErrs struct {
Errors []Error
}
if err := dec.Decode(&respErrs); err != nil {
t.Fatalf("unexpected error decoding error response: %v", err)
}
if len(respErrs.Errors) == 0 {
t.Fatalf("expected errors in response")
}
if respErrs.Errors[0].Code != ErrorCodeUnknownManifest {
t.Fatalf("expected manifest unknown error: got %v", respErrs)
}
// --------------------------------
// Attempt to push unsigned manifest with missing layers
unsignedManifest := &storage.Manifest{
Name: imageName,
Tag: tag,
FSLayers: []storage.FSLayer{
{
BlobSum: "asdf",
},
{
BlobSum: "qwer",
},
},
}
resp = putManifest(t, "putting unsigned manifest", manifestURL, unsignedManifest)
defer resp.Body.Close()
checkResponse(t, "posting unsigned manifest", resp, http.StatusBadRequest)
dec = json.NewDecoder(resp.Body)
if err := dec.Decode(&respErrs); err != nil {
t.Fatalf("unexpected error decoding error response: %v", err)
}
var unverified int
var missingLayers int
var invalidDigests int
for _, err := range respErrs.Errors {
switch err.Code {
case ErrorCodeUnverifiedManifest:
unverified++
case ErrorCodeUnknownLayer:
missingLayers++
case ErrorCodeInvalidDigest:
// TODO(stevvooe): This error isn't quite descriptive enough --
// the layer with an invalid digest isn't identified.
invalidDigests++
default:
t.Fatalf("unexpected error: %v", err)
}
}
if unverified != 1 {
t.Fatalf("should have received one unverified manifest error: %v", respErrs)
}
if missingLayers != 2 {
t.Fatalf("should have received two missing layer errors: %v", respErrs)
}
if invalidDigests != 2 {
t.Fatalf("should have received two invalid digest errors: %v", respErrs)
}
// Push 2 random layers
expectedLayers := make(map[digest.Digest]io.ReadSeeker)
for i := range unsignedManifest.FSLayers {
rs, dgstStr, err := testutil.CreateRandomTarFile()
if err != nil {
t.Fatalf("error creating random layer %d: %v", i, err)
}
dgst := digest.Digest(dgstStr)
expectedLayers[dgst] = rs
unsignedManifest.FSLayers[i].BlobSum = dgst
uploadURLBase := startPushLayer(t, builder, imageName)
pushLayer(t, builder, imageName, dgst, uploadURLBase, rs)
}
// -------------------
// Push the signed manifest with all layers pushed.
signedManifest, err := unsignedManifest.Sign(pk)
if err != nil {
t.Fatalf("unexpected error signing manifest: %v", err)
}
resp = putManifest(t, "putting signed manifest", manifestURL, signedManifest)
checkResponse(t, "putting manifest", resp, http.StatusOK)
resp, err = http.Get(manifestURL)
if err != nil {
t.Fatalf("unexpected error fetching manifest: %v", err)
}
defer resp.Body.Close()
checkResponse(t, "fetching uploaded manifest", resp, http.StatusOK)
var fetchedManifest storage.SignedManifest
dec = json.NewDecoder(resp.Body)
if err := dec.Decode(&fetchedManifest); err != nil {
t.Fatalf("error decoding fetched manifest: %v", err)
}
if !bytes.Equal(fetchedManifest.Raw, signedManifest.Raw) {
t.Fatalf("manifests do not match")
}
}
func putManifest(t *testing.T, msg, url string, v interface{}) *http.Response {
body, err := json.Marshal(v)
if err != nil {
t.Fatalf("unexpected error marshaling %v: %v", v, err)
}
req, err := http.NewRequest("PUT", url, bytes.NewReader(body))
if err != nil {
t.Fatalf("error creating request for %s: %v", msg, err)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("error doing put request while %s: %v", msg, err)
}
return resp
}
func startPushLayer(t *testing.T, ub *urlBuilder, name string) string {
layerUploadURL, err := ub.buildLayerUploadURL(name)
if err != nil {
t.Fatalf("unexpected error building layer upload url: %v", err)
}
resp, err := http.Post(layerUploadURL, "", nil)
if err != nil {
t.Fatalf("unexpected error starting layer push: %v", err)
}
defer resp.Body.Close()
checkResponse(t, fmt.Sprintf("pushing starting layer push %v", name), resp, http.StatusAccepted)
checkHeaders(t, resp, http.Header{
"Location": []string{"*"},
"Content-Length": []string{"0"},
})
return resp.Header.Get("Location")
}
// pushLayer pushes the layer content returning the url on success.
func pushLayer(t *testing.T, ub *urlBuilder, name string, dgst digest.Digest, uploadURLBase string, rs io.ReadSeeker) string {
rsLength, _ := rs.Seek(0, os.SEEK_END)
rs.Seek(0, os.SEEK_SET)
uploadURL := appendValues(uploadURLBase, url.Values{
"digest": []string{dgst.String()},
"size": []string{fmt.Sprint(rsLength)},
})
// Just do a monolithic upload
req, err := http.NewRequest("PUT", uploadURL, rs)
if err != nil {
t.Fatalf("unexpected error creating new request: %v", err)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("unexpected error doing put: %v", err)
}
defer resp.Body.Close()
checkResponse(t, "putting monolithic chunk", resp, http.StatusCreated)
expectedLayerURL, err := ub.buildLayerURL(name, dgst)
if err != nil {
t.Fatalf("error building expected layer url: %v", err)
}
checkHeaders(t, resp, http.Header{
"Location": []string{expectedLayerURL},
"Content-Length": []string{"0"},
})
return resp.Header.Get("Location")
}
func checkResponse(t *testing.T, msg string, resp *http.Response, expectedStatus int) {
if resp.StatusCode != expectedStatus {
t.Logf("unexpected status %s: %v != %v", msg, resp.StatusCode, expectedStatus)
maybeDumpResponse(t, resp)
t.FailNow()
}
}
func maybeDumpResponse(t *testing.T, resp *http.Response) {
if d, err := httputil.DumpResponse(resp, true); err != nil {
t.Logf("error dumping response: %v", err)
} else {
t.Logf("response:\n%s", string(d))
}
}
// matchHeaders checks that the response has at least the headers. If not, the
// test will fail. If a passed in header value is "*", any non-zero value will
// suffice as a match.
func checkHeaders(t *testing.T, resp *http.Response, headers http.Header) {
for k, vs := range headers {
if resp.Header.Get(k) == "" {
t.Fatalf("response missing header %q", k)
}
for _, v := range vs {
if v == "*" {
// Just ensure there is some value.
if len(resp.Header[k]) > 0 {
continue
}
}
for _, hv := range resp.Header[k] {
if hv != v {
t.Fatalf("header value not matched in response: %q != %q", hv, v)
}
}
}
}
}

5
app.go
View File

@ -108,8 +108,9 @@ func (app *App) dispatcher(dispatch dispatchFunc) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
context := &Context{
App: app,
Name: vars["name"],
App: app,
Name: vars["name"],
urlBuilder: newURLBuilderFromRequest(r),
}
// Store vars for underlying handlers.

2
context.go
View File

@ -24,4 +24,6 @@ type Context struct {
// log provides a context specific logger.
log *logrus.Entry
urlBuilder *urlBuilder
}

29
errors.go
View File

@ -34,6 +34,14 @@ const (
// match the provided tag.
ErrorCodeInvalidTag
// ErrorCodeUnknownManifest returned when image manifest name and tag is
// unknown, accompanied by a 404 status.
ErrorCodeUnknownManifest
// ErrorCodeInvalidManifest returned when an image manifest is invalid,
// typically during a PUT operation.
ErrorCodeInvalidManifest
// ErrorCodeUnverifiedManifest is returned when the manifest fails signature
// validation.
ErrorCodeUnverifiedManifest
@ -56,6 +64,8 @@ var errorCodeStrings = map[ErrorCode]string{
ErrorCodeInvalidLength: "INVALID_LENGTH",
ErrorCodeInvalidName: "INVALID_NAME",
ErrorCodeInvalidTag: "INVALID_TAG",
ErrorCodeUnknownManifest: "UNKNOWN_MANIFEST",
ErrorCodeInvalidManifest: "INVALID_MANIFEST",
ErrorCodeUnverifiedManifest: "UNVERIFIED_MANIFEST",
ErrorCodeUnknownLayer: "UNKNOWN_LAYER",
ErrorCodeUnknownLayerUpload: "UNKNOWN_LAYER_UPLOAD",
@ -66,12 +76,14 @@ var errorCodesMessages = map[ErrorCode]string{
ErrorCodeUnknown: "unknown error",
ErrorCodeInvalidDigest: "provided digest did not match uploaded content",
ErrorCodeInvalidLength: "provided length did not match content length",
ErrorCodeInvalidName: "Manifest name did not match URI",
ErrorCodeInvalidTag: "Manifest tag did not match URI",
ErrorCodeUnverifiedManifest: "Manifest failed signature validation",
ErrorCodeUnknownLayer: "Referenced layer not available",
ErrorCodeInvalidName: "manifest name did not match URI",
ErrorCodeInvalidTag: "manifest tag did not match URI",
ErrorCodeUnknownManifest: "manifest not known",
ErrorCodeInvalidManifest: "manifest is invalid",
ErrorCodeUnverifiedManifest: "manifest failed signature validation",
ErrorCodeUnknownLayer: "referenced layer not available",
ErrorCodeUnknownLayerUpload: "cannot resume unknown layer upload",
ErrorCodeUntrustedSignature: "Manifest signed by untrusted source",
ErrorCodeUntrustedSignature: "manifest signed by untrusted source",
}
var stringToErrorCode map[string]ErrorCode
@ -178,7 +190,12 @@ func (errs *Errors) Push(code ErrorCode, details ...interface{}) {
// PushErr pushes an error interface onto the error stack.
func (errs *Errors) PushErr(err error) {
errs.Errors = append(errs.Errors, err)
switch err.(type) {
case Error:
errs.Errors = append(errs.Errors, err)
default:
errs.Errors = append(errs.Errors, Error{Message: err.Error()})
}
}
func (errs *Errors) Error() string {

2
errors_test.go
View File

@ -69,7 +69,7 @@ func TestErrorsManagement(t *testing.T) {
t.Fatalf("error marashaling errors: %v", err)
}
expectedJSON := "{\"errors\":[{\"code\":\"INVALID_DIGEST\",\"message\":\"provided digest did not match uploaded content\"},{\"code\":\"UNKNOWN_LAYER\",\"message\":\"Referenced layer not available\",\"detail\":{\"unknown\":{\"blobSum\":\"sometestblobsumdoesntmatter\"}}}]}"
expectedJSON := "{\"errors\":[{\"code\":\"INVALID_DIGEST\",\"message\":\"provided digest did not match uploaded content\"},{\"code\":\"UNKNOWN_LAYER\",\"message\":\"referenced layer not available\",\"detail\":{\"unknown\":{\"blobSum\":\"sometestblobsumdoesntmatter\"}}}]}"
if string(p) != expectedJSON {
t.Fatalf("unexpected json: %q != %q", string(p), expectedJSON)

9
helpers.go
View File

@ -4,8 +4,6 @@ import (
"encoding/json"
"io"
"net/http"
"github.com/gorilla/mux"
)
// serveJSON marshals v and sets the content-type header to
@ -32,10 +30,3 @@ func closeResources(handler http.Handler, closers ...io.Closer) http.Handler {
handler.ServeHTTP(w, r)
})
}
// clondedRoute returns a clone of the named route from the router.
func clonedRoute(router *mux.Router, name string) *mux.Route {
route := new(mux.Route)
*route = *router.GetRoute(name) // clone the route
return route
}

67
images.go
View File

@ -1,8 +1,13 @@
package registry
import (
"encoding/json"
"fmt"
"net/http"
"github.com/docker/docker-registry/digest"
"github.com/docker/docker-registry/storage"
"github.com/gorilla/handlers"
)
@ -32,15 +37,77 @@ type imageManifestHandler struct {
// GetImageManifest fetches the image manifest from the storage backend, if it exists.
func (imh *imageManifestHandler) GetImageManifest(w http.ResponseWriter, r *http.Request) {
manifests := imh.services.Manifests()
manifest, err := manifests.Get(imh.Name, imh.Tag)
if err != nil {
imh.Errors.Push(ErrorCodeUnknownManifest, err)
w.WriteHeader(http.StatusNotFound)
return
}
w.Header().Set("Content-Type", "application/json")
w.Header().Set("Content-Length", fmt.Sprint(len(manifest.Raw)))
w.Write(manifest.Raw)
}
// PutImageManifest validates and stores and image in the registry.
func (imh *imageManifestHandler) PutImageManifest(w http.ResponseWriter, r *http.Request) {
manifests := imh.services.Manifests()
dec := json.NewDecoder(r.Body)
var manifest storage.SignedManifest
if err := dec.Decode(&manifest); err != nil {
imh.Errors.Push(ErrorCodeInvalidManifest, err)
w.WriteHeader(http.StatusBadRequest)
return
}
if err := manifests.Put(imh.Name, imh.Tag, &manifest); err != nil {
// TODO(stevvooe): These error handling switches really need to be
// handled by an app global mapper.
switch err := err.(type) {
case storage.ErrManifestVerification:
for _, verificationError := range err {
switch verificationError := verificationError.(type) {
case storage.ErrUnknownLayer:
imh.Errors.Push(ErrorCodeUnknownLayer, verificationError.FSLayer)
case storage.ErrManifestUnverified:
imh.Errors.Push(ErrorCodeUnverifiedManifest)
default:
if verificationError == digest.ErrDigestInvalidFormat {
// TODO(stevvooe): We need to really need to move all
// errors to types. Its much more straightforward.
imh.Errors.Push(ErrorCodeInvalidDigest)
} else {
imh.Errors.PushErr(verificationError)
}
}
}
default:
imh.Errors.PushErr(err)
}
w.WriteHeader(http.StatusBadRequest)
return
}
}
// DeleteImageManifest removes the image with the given tag from the registry.
func (imh *imageManifestHandler) DeleteImageManifest(w http.ResponseWriter, r *http.Request) {
manifests := imh.services.Manifests()
if err := manifests.Delete(imh.Name, imh.Tag); err != nil {
switch err := err.(type) {
case storage.ErrUnknownManifest:
imh.Errors.Push(ErrorCodeUnknownManifest, err)
w.WriteHeader(http.StatusNotFound)
default:
imh.Errors.Push(ErrorCodeUnknown, err)
w.WriteHeader(http.StatusBadRequest)
}
return
}
w.Header().Set("Content-Length", "0")
w.WriteHeader(http.StatusAccepted)
}

26
layer.go
View File

@ -6,7 +6,6 @@ import (
"github.com/docker/docker-registry/digest"
"github.com/docker/docker-registry/storage"
"github.com/gorilla/handlers"
"github.com/gorilla/mux"
)
// layerDispatcher uses the request context to build a layerHandler.
@ -47,33 +46,16 @@ func (lh *layerHandler) GetLayer(w http.ResponseWriter, r *http.Request) {
layer, err := layers.Fetch(lh.Name, lh.Digest)
if err != nil {
switch err {
case storage.ErrLayerUnknown:
switch err := err.(type) {
case storage.ErrUnknownLayer:
w.WriteHeader(http.StatusNotFound)
lh.Errors.Push(ErrorCodeUnknownLayer,
map[string]interface{}{
"unknown": storage.FSLayer{BlobSum: lh.Digest},
})
return
lh.Errors.Push(ErrorCodeUnknownLayer, err.FSLayer)
default:
lh.Errors.Push(ErrorCodeUnknown, err)
return
}
return
}
defer layer.Close()
http.ServeContent(w, r, layer.Digest().String(), layer.CreatedAt(), layer)
}
func buildLayerURL(router *mux.Router, r *http.Request, layer storage.Layer) (string, error) {
route := clonedRoute(router, routeNameBlob)
layerURL, err := route.Schemes(r.URL.Scheme).Host(r.Host).
URL("name", layer.Name(),
"digest", layer.Digest().String())
if err != nil {
return "", err
}
return layerURL.String(), nil
}

19
layerupload.go
View File

@ -10,7 +10,6 @@ import (
"github.com/docker/docker-registry/digest"
"github.com/docker/docker-registry/storage"
"github.com/gorilla/handlers"
"github.com/gorilla/mux"
)
// layerUploadDispatcher constructs and returns the layer upload handler for
@ -151,7 +150,7 @@ func (luh *layerUploadHandler) CancelLayerUpload(w http.ResponseWriter, r *http.
// chunk responses. This sets the correct headers but the response status is
// left to the caller.
func (luh *layerUploadHandler) layerUploadResponse(w http.ResponseWriter, r *http.Request) error {
uploadURL, err := buildLayerUploadURL(luh.router, r, luh.Upload)
uploadURL, err := luh.urlBuilder.forLayerUpload(luh.Upload)
if err != nil {
logrus.Infof("error building upload url: %s", err)
return err
@ -171,7 +170,7 @@ var errNotReadyToComplete = fmt.Errorf("not ready to complete upload")
func (luh *layerUploadHandler) maybeCompleteUpload(w http.ResponseWriter, r *http.Request) error {
// If we get a digest and length, we can finish the upload.
dgstStr := r.FormValue("digest") // TODO(stevvooe): Support multiple digest parameters!
sizeStr := r.FormValue("length")
sizeStr := r.FormValue("size")
if dgstStr == "" || sizeStr == "" {
return errNotReadyToComplete
@ -200,7 +199,7 @@ func (luh *layerUploadHandler) completeUpload(w http.ResponseWriter, r *http.Req
return
}
layerURL, err := buildLayerURL(luh.router, r, layer)
layerURL, err := luh.urlBuilder.forLayer(layer)
if err != nil {
luh.Errors.Push(ErrorCodeUnknown, err)
w.WriteHeader(http.StatusInternalServerError)
@ -211,15 +210,3 @@ func (luh *layerUploadHandler) completeUpload(w http.ResponseWriter, r *http.Req
w.Header().Set("Content-Length", "0")
w.WriteHeader(http.StatusCreated)
}
func buildLayerUploadURL(router *mux.Router, r *http.Request, upload storage.LayerUpload) (string, error) {
route := clonedRoute(router, routeNameBlobUploadResume)
uploadURL, err := route.Schemes(r.URL.Scheme).Host(r.Host).
URL("name", upload.Name(), "uuid", upload.UUID())
if err != nil {
return "", err
}
return uploadURL.String(), nil
}

24
storage/layer.go
View File

@ -53,9 +53,6 @@ type LayerUpload interface {
}
var (
// ErrLayerUnknown returned when layer cannot be found.
ErrLayerUnknown = fmt.Errorf("unknown layer")
// ErrLayerExists returned when layer already exists
ErrLayerExists = fmt.Errorf("layer exists")
@ -65,9 +62,6 @@ var (
// ErrLayerUploadUnknown returned when upload is not found.
ErrLayerUploadUnknown = fmt.Errorf("layer upload unknown")
// ErrLayerInvalidDigest returned when tarsum check fails.
ErrLayerInvalidDigest = fmt.Errorf("invalid layer digest")
// ErrLayerInvalidLength returned when length check fails.
ErrLayerInvalidLength = fmt.Errorf("invalid layer length")
@ -75,3 +69,21 @@ var (
// Layer or LayerUpload.
ErrLayerClosed = fmt.Errorf("layer closed")
)
// ErrUnknownLayer returned when layer cannot be found.
type ErrUnknownLayer struct {
FSLayer FSLayer
}
func (err ErrUnknownLayer) Error() string {
return fmt.Sprintf("unknown layer %v", err.FSLayer.BlobSum)
}
// ErrLayerInvalidDigest returned when tarsum check fails.
type ErrLayerInvalidDigest struct {
FSLayer FSLayer
}
func (err ErrLayerInvalidDigest) Error() string {
return fmt.Sprintf("invalid digest for referenced layer: %v", err.FSLayer.BlobSum)
}

8
storage/layer_test.go
View File

@ -169,11 +169,13 @@ func TestSimpleLayerRead(t *testing.T) {
t.Fatalf("error expected fetching unknown layer")
}
if err != ErrLayerUnknown {
t.Fatalf("unexpected error fetching non-existent layer: %v", err)
} else {
switch err.(type) {
case ErrUnknownLayer:
err = nil
default:
t.Fatalf("unexpected error fetching non-existent layer: %v", err)
}
randomLayerDigest, err := writeTestLayer(driver, ls.pathMapper, imageName, dgst, randomLayerReader)
if err != nil {
t.Fatalf("unexpected error writing test layer: %v", err)

7
storage/layerstore.go
View File

@ -19,7 +19,8 @@ func (ls *layerStore) Exists(name string, digest digest.Digest) (bool, error) {
_, err := ls.Fetch(name, digest)
if err != nil {
if err == ErrLayerUnknown {
switch err.(type) {
case ErrUnknownLayer:
return false, nil
}
@ -34,7 +35,7 @@ func (ls *layerStore) Fetch(name string, digest digest.Digest) (Layer, error) {
if err != nil {
switch err := err.(type) {
case storagedriver.PathNotFoundError, *storagedriver.PathNotFoundError:
return nil, ErrLayerUnknown
return nil, ErrUnknownLayer{FSLayer{BlobSum: digest}}
default:
return nil, err
}
@ -44,7 +45,7 @@ func (ls *layerStore) Fetch(name string, digest digest.Digest) (Layer, error) {
if err != nil {
switch err := err.(type) {
case storagedriver.PathNotFoundError, *storagedriver.PathNotFoundError:
return nil, ErrLayerUnknown
return nil, ErrUnknownLayer{FSLayer{BlobSum: digest}}
default:
return nil, err
}

2
storage/layerupload.go
View File

@ -275,7 +275,7 @@ func (luc *layerUploadController) validateLayer(fp layerFile, size int64, dgst d
}
if !digestVerifier.Verified() {
return "", ErrLayerInvalidDigest
return "", ErrLayerInvalidDigest{FSLayer{BlobSum: dgst}}
}
return dgst, nil

110
storage/manifest.go
View File

@ -1,23 +1,48 @@
package storage
import (
"crypto/x509"
"encoding/json"
"fmt"
"strings"
"github.com/docker/libtrust"
"github.com/docker/docker-registry/digest"
)
var (
// ErrManifestUnknown is returned if the manifest is not known by the
// registry.
ErrManifestUnknown = fmt.Errorf("unknown manifest")
// ErrUnknownManifest is returned if the manifest is not known by the
// registry.
type ErrUnknownManifest struct {
Name string
Tag string
}
// ErrManifestUnverified is returned when the registry is unable to verify
// the manifest.
ErrManifestUnverified = fmt.Errorf("unverified manifest")
)
func (err ErrUnknownManifest) Error() string {
return fmt.Sprintf("unknown manifest name=%s tag=%s", err.Name, err.Tag)
}
// ErrManifestUnverified is returned when the registry is unable to verify
// the manifest.
type ErrManifestUnverified struct{}
func (ErrManifestUnverified) Error() string {
return fmt.Sprintf("unverified manifest")
}
// ErrManifestVerification provides a type to collect errors encountered
// during manifest verification. Currently, it accepts errors of all types,
// but it may be narrowed to those involving manifest verification.
type ErrManifestVerification []error
func (errs ErrManifestVerification) Error() string {
var parts []string
for _, err := range errs {
parts = append(parts, err.Error())
}
return fmt.Sprintf("errors verifying manifest: %v", strings.Join(parts, ","))
}
// Versioned provides a struct with just the manifest schemaVersion. Incoming
// content with unknown schema version can be decoded against this struct to
@ -78,7 +103,37 @@ func (m *Manifest) Sign(pk libtrust.PrivateKey) (*SignedManifest, error) {
}, nil
}
// SignedManifest provides an envelope for
// SignWithChain signs the manifest with the given private key and x509 chain.
// The public key of the first element in the chain must be the public key
// corresponding with the sign key.
func (m *Manifest) SignWithChain(key libtrust.PrivateKey, chain []*x509.Certificate) (*SignedManifest, error) {
p, err := json.Marshal(m)
if err != nil {
return nil, err
}
js, err := libtrust.NewJSONSignature(p)
if err != nil {
return nil, err
}
if err := js.SignWithChain(key, chain); err != nil {
return nil, err
}
pretty, err := js.PrettySignature("signatures")
if err != nil {
return nil, err
}
return &SignedManifest{
Manifest: *m,
Raw: pretty,
}, nil
}
// SignedManifest provides an envelope for a signed image manifest, including
// the format sensitive raw bytes. It contains fields to
type SignedManifest struct {
Manifest
@ -88,28 +143,51 @@ type SignedManifest struct {
Raw []byte `json:"-"`
}
// Verify verifies the signature of the signed manifest returning the public
// keys used during signing.
func (sm *SignedManifest) Verify() ([]libtrust.PublicKey, error) {
js, err := libtrust.ParsePrettySignature(sm.Raw, "signatures")
if err != nil {
return nil, err
}
return js.Verify()
}
// VerifyChains verifies the signature of the signed manifest against the
// certificate pool returning the list of verified chains. Signatures without
// an x509 chain are not checked.
func (sm *SignedManifest) VerifyChains(ca *x509.CertPool) ([][]*x509.Certificate, error) {
js, err := libtrust.ParsePrettySignature(sm.Raw, "signatures")
if err != nil {
return nil, err
}
return js.VerifyChains(ca)
}
// UnmarshalJSON populates a new ImageManifest struct from JSON data.
func (m *SignedManifest) UnmarshalJSON(b []byte) error {
func (sm *SignedManifest) UnmarshalJSON(b []byte) error {
var manifest Manifest
if err := json.Unmarshal(b, &manifest); err != nil {
return err
}
m.Manifest = manifest
m.Raw = b
sm.Manifest = manifest
sm.Raw = b
return nil
}
// MarshalJSON returns the contents of raw. If Raw is nil, marshals the inner
// contents.
func (m *SignedManifest) MarshalJSON() ([]byte, error) {
if len(m.Raw) > 0 {
return m.Raw, nil
func (sm *SignedManifest) MarshalJSON() ([]byte, error) {
if len(sm.Raw) > 0 {
return sm.Raw, nil
}
// If the raw data is not available, just dump the inner content.
return json.Marshal(&m.Manifest)
return json.Marshal(&sm.Manifest)
}
// FSLayer is a container struct for BlobSums defined in an image manifest

9
storage/manifest_test.go
View File

@ -33,8 +33,13 @@ func TestManifestStorage(t *testing.T) {
t.Fatalf("manifest should not exist")
}
if _, err := ms.Get(name, tag); err != ErrManifestUnknown {
t.Fatalf("expected manifest unknown error: %v != %v", err, ErrManifestUnknown)
if _, err := ms.Get(name, tag); true {
switch err.(type) {
case ErrUnknownManifest:
break
default:
t.Fatalf("expected manifest unknown error: %#v", err)
}
}
manifest := Manifest{

61
storage/manifeststore.go
View File

@ -4,9 +4,8 @@ import (
"encoding/json"
"fmt"
"github.com/docker/libtrust"
"github.com/docker/docker-registry/storagedriver"
"github.com/docker/libtrust"
)
type manifestStore struct {
@ -45,7 +44,7 @@ func (ms *manifestStore) Get(name, tag string) (*SignedManifest, error) {
if err != nil {
switch err := err.(type) {
case storagedriver.PathNotFoundError, *storagedriver.PathNotFoundError:
return nil, ErrManifestUnknown
return nil, ErrUnknownManifest{Name: name, Tag: tag}
default:
return nil, err
}
@ -73,13 +72,28 @@ func (ms *manifestStore) Put(name, tag string, manifest *SignedManifest) error {
return err
}
// TODO(stevvooe): Should we get manifest first?
// TODO(stevvooe): Should we get old manifest first? Perhaps, write, then
// move to ensure a valid manifest?
return ms.driver.PutContent(p, manifest.Raw)
}
func (ms *manifestStore) Delete(name, tag string) error {
panic("not implemented")
p, err := ms.path(name, tag)
if err != nil {
return err
}
if err := ms.driver.Delete(p); err != nil {
switch err := err.(type) {
case storagedriver.PathNotFoundError, *storagedriver.PathNotFoundError:
return ErrUnknownManifest{Name: name, Tag: tag}
default:
return err
}
}
return nil
}
func (ms *manifestStore) path(name, tag string) (string, error) {
@ -90,6 +104,12 @@ func (ms *manifestStore) path(name, tag string) (string, error) {
}
func (ms *manifestStore) verifyManifest(name, tag string, manifest *SignedManifest) error {
// TODO(stevvooe): This verification is present here, but this needs to be
// lifted out of the storage infrastructure and moved into a package
// oriented towards defining verifiers and reporting them with
// granularity.
var errs ErrManifestVerification
if manifest.Name != name {
return fmt.Errorf("name does not match manifest name")
}
@ -98,37 +118,34 @@ func (ms *manifestStore) verifyManifest(name, tag string, manifest *SignedManife
return fmt.Errorf("tag does not match manifest tag")
}
var errs []error
// TODO(stevvooe): These pubkeys need to be checked with either Verify or
// VerifyWithChains. We need to define the exact source of the CA.
// Perhaps, its a configuration value injected into manifest store.
if _, err := manifest.Verify(); err != nil {
switch err {
case libtrust.ErrMissingSignatureKey, libtrust.ErrInvalidJSONContent, libtrust.ErrMissingSignatureKey:
errs = append(errs, ErrManifestUnverified{})
default:
errs = append(errs, err)
}
}
for _, fsLayer := range manifest.FSLayers {
exists, err := ms.layerService.Exists(name, fsLayer.BlobSum)
if err != nil {
// TODO(stevvooe): Need to store information about missing blob.
errs = append(errs, err)
}
if !exists {
errs = append(errs, fmt.Errorf("missing layer %v", fsLayer.BlobSum))
errs = append(errs, ErrUnknownLayer{FSLayer: fsLayer})
}
}
if len(errs) != 0 {
// TODO(stevvooe): These need to be recoverable by a caller.
return fmt.Errorf("missing layers: %v", errs)
return errs
}
js, err := libtrust.ParsePrettySignature(manifest.Raw, "signatures")
if err != nil {
return err
}
_, err = js.Verify() // These pubkeys need to be checked.
if err != nil {
return err
}
// TODO(sday): Pubkey checks need to go here. This where things get fancy.
// Perhaps, an injected service would reduce coupling here.
return nil
}

141
urls.go 100644
View File

@ -0,0 +1,141 @@
package registry
import (
"net/http"
"net/url"
"github.com/docker/docker-registry/digest"
"github.com/docker/docker-registry/storage"
"github.com/gorilla/mux"
)
type urlBuilder struct {
url *url.URL // url root (ie http://localhost/)
router *mux.Router
}
func newURLBuilder(root *url.URL) *urlBuilder {
return &urlBuilder{
url: root,
router: v2APIRouter(),
}
}
func newURLBuilderFromRequest(r *http.Request) *urlBuilder {
u := &url.URL{
Scheme: r.URL.Scheme,
Host: r.Host,
}
return newURLBuilder(u)
}
func newURLBuilderFromString(root string) (*urlBuilder, error) {
u, err := url.Parse(root)
if err != nil {
return nil, err
}
return newURLBuilder(u), nil
}
func (ub *urlBuilder) forManifest(m *storage.Manifest) (string, error) {
return ub.buildManifestURL(m.Name, m.Tag)
}
func (ub *urlBuilder) buildManifestURL(name, tag string) (string, error) {
route := clonedRoute(ub.router, routeNameImageManifest)
manifestURL, err := route.
Schemes(ub.url.Scheme).
Host(ub.url.Host).
URL("name", name, "tag", tag)
if err != nil {
return "", err
}
return manifestURL.String(), nil
}
func (ub *urlBuilder) forLayer(l storage.Layer) (string, error) {
return ub.buildLayerURL(l.Name(), l.Digest())
}
func (ub *urlBuilder) buildLayerURL(name string, dgst digest.Digest) (string, error) {
route := clonedRoute(ub.router, routeNameBlob)
layerURL, err := route.
Schemes(ub.url.Scheme).
Host(ub.url.Host).
URL("name", name, "digest", dgst.String())
if err != nil {
return "", err
}
return layerURL.String(), nil
}
func (ub *urlBuilder) buildLayerUploadURL(name string) (string, error) {
route := clonedRoute(ub.router, routeNameBlobUpload)
uploadURL, err := route.
Schemes(ub.url.Scheme).
Host(ub.url.Host).
URL("name", name)
if err != nil {
return "", err
}
return uploadURL.String(), nil
}
func (ub *urlBuilder) forLayerUpload(layerUpload storage.LayerUpload) (string, error) {
return ub.buildLayerUploadResumeURL(layerUpload.Name(), layerUpload.UUID())
}
func (ub *urlBuilder) buildLayerUploadResumeURL(name, uuid string, values ...url.Values) (string, error) {
route := clonedRoute(ub.router, routeNameBlobUploadResume)
uploadURL, err := route.
Schemes(ub.url.Scheme).
Host(ub.url.Host).
URL("name", name, "uuid", uuid)
if err != nil {
return "", err
}
return appendValuesURL(uploadURL, values...).String(), nil
}
// appendValuesURL appends the parameters to the url.
func appendValuesURL(u *url.URL, values ...url.Values) *url.URL {
merged := u.Query()
for _, v := range values {
for k, vv := range v {
merged[k] = append(merged[k], vv...)
}
}
u.RawQuery = merged.Encode()
return u
}
// appendValues appends the parameters to the url. Panics if the string is not
// a url.
func appendValues(u string, values ...url.Values) string {
up, err := url.Parse(u)
if err != nil {
panic(err) // should never happen
}
return appendValuesURL(up, values...).String()
}
// clondedRoute returns a clone of the named route from the router.
func clonedRoute(router *mux.Router, name string) *mux.Route {
route := new(mux.Route)
*route = *router.GetRoute(name) // clone the route
return route
}