From ce614b6de8b2f760bbb6b426447f159821e0ea5d Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Fri, 17 Apr 2015 13:32:51 -0700 Subject: [PATCH 01/26] Add client implementation of distribution interface Adds functionality to create a Repository client which connects to a remote endpoint. Signed-off-by: Derek McGowan (github: dmcgowan) --- registry/client/authchallenge.go | 150 +++++++ registry/client/endpoint.go | 266 ++++++++++++ registry/client/errors.go | 37 ++ registry/client/repository.go | 657 +++++++++++++++++++++++++++++ registry/client/repository_test.go | 605 ++++++++++++++++++++++++++ registry/client/token.go | 78 ++++ 6 files changed, 1793 insertions(+) create mode 100644 registry/client/authchallenge.go create mode 100644 registry/client/endpoint.go create mode 100644 registry/client/repository.go create mode 100644 registry/client/repository_test.go create mode 100644 registry/client/token.go diff --git a/registry/client/authchallenge.go b/registry/client/authchallenge.go new file mode 100644 index 00000000..0485f42d --- /dev/null +++ b/registry/client/authchallenge.go @@ -0,0 +1,150 @@ +package client + +import ( + "net/http" + "strings" +) + +// Octet types from RFC 2616. +type octetType byte + +// AuthorizationChallenge carries information +// from a WWW-Authenticate response header. +type AuthorizationChallenge struct { + Scheme string + Parameters map[string]string +} + +var octetTypes [256]octetType + +const ( + isToken octetType = 1 << iota + isSpace +) + +func init() { + // OCTET = + // CHAR = + // CTL = + // CR = + // LF = + // SP = + // HT = + // <"> = + // CRLF = CR LF + // LWS = [CRLF] 1*( SP | HT ) + // TEXT = + // separators = "(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\" | <"> + // | "/" | "[" | "]" | "?" | "=" | "{" | "}" | SP | HT + // token = 1* + // qdtext = > + + for c := 0; c < 256; c++ { + var t octetType + isCtl := c <= 31 || c == 127 + isChar := 0 <= c && c <= 127 + isSeparator := strings.IndexRune(" \t\"(),/:;<=>?@[]\\{}", rune(c)) >= 0 + if strings.IndexRune(" \t\r\n", rune(c)) >= 0 { + t |= isSpace + } + if isChar && !isCtl && !isSeparator { + t |= isToken + } + octetTypes[c] = t + } +} + +func parseAuthHeader(header http.Header) []AuthorizationChallenge { + var challenges []AuthorizationChallenge + for _, h := range header[http.CanonicalHeaderKey("WWW-Authenticate")] { + v, p := parseValueAndParams(h) + if v != "" { + challenges = append(challenges, AuthorizationChallenge{Scheme: v, Parameters: p}) + } + } + return challenges +} + +func parseValueAndParams(header string) (value string, params map[string]string) { + params = make(map[string]string) + value, s := expectToken(header) + if value == "" { + return + } + value = strings.ToLower(value) + s = "," + skipSpace(s) + for strings.HasPrefix(s, ",") { + var pkey string + pkey, s = expectToken(skipSpace(s[1:])) + if pkey == "" { + return + } + if !strings.HasPrefix(s, "=") { + return + } + var pvalue string + pvalue, s = expectTokenOrQuoted(s[1:]) + if pvalue == "" { + return + } + pkey = strings.ToLower(pkey) + params[pkey] = pvalue + s = skipSpace(s) + } + return +} + +func skipSpace(s string) (rest string) { + i := 0 + for ; i < len(s); i++ { + if octetTypes[s[i]]&isSpace == 0 { + break + } + } + return s[i:] +} + +func expectToken(s string) (token, rest string) { + i := 0 + for ; i < len(s); i++ { + if octetTypes[s[i]]&isToken == 0 { + break + } + } + return s[:i], s[i:] +} + +func expectTokenOrQuoted(s string) (value string, rest string) { + if !strings.HasPrefix(s, "\"") { + return expectToken(s) + } + s = s[1:] + for i := 0; i < len(s); i++ { + switch s[i] { + case '"': + return s[:i], s[i+1:] + case '\\': + p := make([]byte, len(s)-1) + j := copy(p, s[:i]) + escape := true + for i = i + i; i < len(s); i++ { + b := s[i] + switch { + case escape: + escape = false + p[j] = b + j++ + case b == '\\': + escape = true + case b == '"': + return string(p[:j]), s[i+1:] + default: + p[j] = b + j++ + } + } + return "", "" + } + } + return "", "" +} diff --git a/registry/client/endpoint.go b/registry/client/endpoint.go new file mode 100644 index 00000000..83d3d991 --- /dev/null +++ b/registry/client/endpoint.go @@ -0,0 +1,266 @@ +package client + +import ( + "fmt" + "net/http" + "net/url" + "strings" + "sync" + "time" + + "github.com/Sirupsen/logrus" + "github.com/docker/distribution/registry/api/v2" +) + +// Authorizer is used to apply Authorization to an HTTP request +type Authorizer interface { + // Authorizer updates an HTTP request with the needed authorization + Authorize(req *http.Request) error +} + +// CredentialStore is an interface for getting credentials for +// a given URL +type CredentialStore interface { + // Basic returns basic auth for the given URL + Basic(*url.URL) (string, string) +} + +// RepositoryEndpoint represents a single host endpoint serving up +// the distribution API. +type RepositoryEndpoint struct { + Endpoint string + Mirror bool + + Header http.Header + Credentials CredentialStore + + ub *v2.URLBuilder +} + +type nullAuthorizer struct{} + +func (na nullAuthorizer) Authorize(req *http.Request) error { + return nil +} + +type repositoryTransport struct { + Transport http.RoundTripper + Header http.Header + Authorizer Authorizer +} + +func (rt *repositoryTransport) RoundTrip(req *http.Request) (*http.Response, error) { + reqCopy := new(http.Request) + *reqCopy = *req + + // Copy existing headers then static headers + reqCopy.Header = make(http.Header, len(req.Header)+len(rt.Header)) + for k, s := range req.Header { + reqCopy.Header[k] = append([]string(nil), s...) + } + for k, s := range rt.Header { + reqCopy.Header[k] = append(reqCopy.Header[k], s...) + } + + if rt.Authorizer != nil { + if err := rt.Authorizer.Authorize(reqCopy); err != nil { + return nil, err + } + } + + logrus.Debugf("HTTP: %s %s", req.Method, req.URL) + + if rt.Transport != nil { + return rt.Transport.RoundTrip(reqCopy) + } + return http.DefaultTransport.RoundTrip(reqCopy) +} + +type authTransport struct { + Transport http.RoundTripper + Header http.Header +} + +func (rt *authTransport) RoundTrip(req *http.Request) (*http.Response, error) { + reqCopy := new(http.Request) + *reqCopy = *req + + // Copy existing headers then static headers + reqCopy.Header = make(http.Header, len(req.Header)+len(rt.Header)) + for k, s := range req.Header { + reqCopy.Header[k] = append([]string(nil), s...) + } + for k, s := range rt.Header { + reqCopy.Header[k] = append(reqCopy.Header[k], s...) + } + + logrus.Debugf("HTTP: %s %s", req.Method, req.URL) + + if rt.Transport != nil { + return rt.Transport.RoundTrip(reqCopy) + } + return http.DefaultTransport.RoundTrip(reqCopy) +} + +// URLBuilder returns a new URL builder +func (e *RepositoryEndpoint) URLBuilder() (*v2.URLBuilder, error) { + if e.ub == nil { + var err error + e.ub, err = v2.NewURLBuilderFromString(e.Endpoint) + if err != nil { + return nil, err + } + } + + return e.ub, nil +} + +// HTTPClient returns a new HTTP client configured for this endpoint +func (e *RepositoryEndpoint) HTTPClient(name string) (*http.Client, error) { + transport := &repositoryTransport{ + Header: e.Header, + } + client := &http.Client{ + Transport: transport, + } + + challenges, err := e.ping(client) + if err != nil { + return nil, err + } + actions := []string{"pull"} + if !e.Mirror { + actions = append(actions, "push") + } + + transport.Authorizer = &endpointAuthorizer{ + client: &http.Client{Transport: &authTransport{Header: e.Header}}, + challenges: challenges, + creds: e.Credentials, + resource: "repository", + scope: name, + actions: actions, + } + + return client, nil +} + +func (e *RepositoryEndpoint) ping(client *http.Client) ([]AuthorizationChallenge, error) { + ub, err := e.URLBuilder() + if err != nil { + return nil, err + } + u, err := ub.BuildBaseURL() + if err != nil { + return nil, err + } + + req, err := http.NewRequest("GET", u, nil) + if err != nil { + return nil, err + } + req.Header = make(http.Header, len(e.Header)) + for k, s := range e.Header { + req.Header[k] = append([]string(nil), s...) + } + + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var supportsV2 bool +HeaderLoop: + for _, supportedVersions := range resp.Header[http.CanonicalHeaderKey("Docker-Distribution-API-Version")] { + for _, versionName := range strings.Fields(supportedVersions) { + if versionName == "registry/2.0" { + supportsV2 = true + break HeaderLoop + } + } + } + + if !supportsV2 { + return nil, fmt.Errorf("%s does not appear to be a v2 registry endpoint", e.Endpoint) + } + + if resp.StatusCode == http.StatusUnauthorized { + // Parse the WWW-Authenticate Header and store the challenges + // on this endpoint object. + return parseAuthHeader(resp.Header), nil + } else if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unable to get valid ping response: %d", resp.StatusCode) + } + + return nil, nil +} + +type endpointAuthorizer struct { + client *http.Client + challenges []AuthorizationChallenge + creds CredentialStore + + resource string + scope string + actions []string + + tokenLock sync.Mutex + tokenCache string + tokenExpiration time.Time +} + +func (ta *endpointAuthorizer) Authorize(req *http.Request) error { + token, err := ta.getToken() + if err != nil { + return err + } + if token != "" { + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + } else if ta.creds != nil { + username, password := ta.creds.Basic(req.URL) + if username != "" && password != "" { + req.SetBasicAuth(username, password) + } + } + return nil +} + +func (ta *endpointAuthorizer) getToken() (string, error) { + ta.tokenLock.Lock() + defer ta.tokenLock.Unlock() + now := time.Now() + if now.Before(ta.tokenExpiration) { + //log.Debugf("Using cached token for %q", ta.auth.Username) + return ta.tokenCache, nil + } + + for _, challenge := range ta.challenges { + switch strings.ToLower(challenge.Scheme) { + case "basic": + // no token necessary + case "bearer": + //log.Debugf("Getting bearer token with %s for %s", challenge.Parameters, ta.auth.Username) + params := map[string]string{} + for k, v := range challenge.Parameters { + params[k] = v + } + params["scope"] = fmt.Sprintf("%s:%s:%s", ta.resource, ta.scope, strings.Join(ta.actions, ",")) + token, err := getToken(ta.creds, params, ta.client) + if err != nil { + return "", err + } + ta.tokenCache = token + ta.tokenExpiration = now.Add(time.Minute) + + return token, nil + default: + //log.Infof("Unsupported auth scheme: %q", challenge.Scheme) + } + } + + // Do not expire cache since there are no challenges which use a token + ta.tokenExpiration = time.Now().Add(time.Hour * 24) + + return "", nil +} diff --git a/registry/client/errors.go b/registry/client/errors.go index 3e89e674..4ef2cc23 100644 --- a/registry/client/errors.go +++ b/registry/client/errors.go @@ -1,9 +1,14 @@ package client import ( + "bytes" + "encoding/json" "fmt" + "io/ioutil" + "net/http" "github.com/docker/distribution/digest" + "github.com/docker/distribution/registry/api/v2" ) // RepositoryNotFoundError is returned when making an operation against a @@ -77,3 +82,35 @@ type UnexpectedHTTPStatusError struct { func (e *UnexpectedHTTPStatusError) Error() string { return fmt.Sprintf("Received unexpected HTTP status: %s", e.Status) } + +// UnexpectedHTTPResponseError is returned when an expected HTTP status code +// is returned, but the content was unexpected and failed to be parsed. +type UnexpectedHTTPResponseError struct { + ParseErr error + Response []byte +} + +func (e *UnexpectedHTTPResponseError) Error() string { + shortenedResponse := string(e.Response) + if len(shortenedResponse) > 15 { + shortenedResponse = shortenedResponse[:12] + "..." + } + return fmt.Sprintf("Error parsing HTTP response: %s: %q", e.ParseErr.Error(), shortenedResponse) +} + +func parseHTTPErrorResponse(response *http.Response) error { + var errors v2.Errors + body, err := ioutil.ReadAll(response.Body) + if err != nil { + return err + } + decoder := json.NewDecoder(bytes.NewReader(body)) + err = decoder.Decode(&errors) + if err != nil { + return &UnexpectedHTTPResponseError{ + ParseErr: err, + Response: body, + } + } + return &errors +} diff --git a/registry/client/repository.go b/registry/client/repository.go new file mode 100644 index 00000000..a96390fa --- /dev/null +++ b/registry/client/repository.go @@ -0,0 +1,657 @@ +package client + +import ( + "bufio" + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "os" + "strconv" + "time" + + ctxu "github.com/docker/distribution/context" + + "github.com/docker/distribution/manifest" + + "github.com/docker/distribution/digest" + + "github.com/docker/distribution" + "github.com/docker/distribution/registry/api/v2" + "golang.org/x/net/context" +) + +// NewRepositoryClient creates a new Repository for the given repository name and endpoint +func NewRepositoryClient(ctx context.Context, name string, endpoint *RepositoryEndpoint) (distribution.Repository, error) { + if err := v2.ValidateRespositoryName(name); err != nil { + return nil, err + } + + ub, err := endpoint.URLBuilder() + if err != nil { + return nil, err + } + + client, err := endpoint.HTTPClient(name) + if err != nil { + return nil, err + } + + return &repository{ + client: client, + ub: ub, + name: name, + context: ctx, + mirror: endpoint.Mirror, + }, nil +} + +type repository struct { + client *http.Client + ub *v2.URLBuilder + context context.Context + name string + mirror bool +} + +func (r *repository) Name() string { + return r.name +} + +func (r *repository) Layers() distribution.LayerService { + return &layers{ + repository: r, + } +} + +func (r *repository) Manifests() distribution.ManifestService { + return &manifests{ + repository: r, + } +} + +func (r *repository) Signatures() distribution.SignatureService { + return &signatures{ + repository: r, + } +} + +type signatures struct { + *repository +} + +func (s *signatures) Get(dgst digest.Digest) ([][]byte, error) { + panic("not implemented") +} + +func (s *signatures) Put(dgst digest.Digest, signatures ...[]byte) error { + panic("not implemented") +} + +type manifests struct { + *repository +} + +func (ms *manifests) Tags() ([]string, error) { + panic("not implemented") +} + +func (ms *manifests) Exists(dgst digest.Digest) (bool, error) { + return ms.ExistsByTag(dgst.String()) +} + +func (ms *manifests) ExistsByTag(tag string) (bool, error) { + u, err := ms.ub.BuildManifestURL(ms.name, tag) + if err != nil { + return false, err + } + + resp, err := ms.client.Head(u) + if err != nil { + return false, err + } + + switch { + case resp.StatusCode == http.StatusOK: + return true, nil + case resp.StatusCode == http.StatusNotFound: + return false, nil + case resp.StatusCode >= 400 && resp.StatusCode < 500: + return false, parseHTTPErrorResponse(resp) + default: + return false, &UnexpectedHTTPStatusError{Status: resp.Status} + } +} + +func (ms *manifests) Get(dgst digest.Digest) (*manifest.SignedManifest, error) { + return ms.GetByTag(dgst.String()) +} + +func (ms *manifests) GetByTag(tag string) (*manifest.SignedManifest, error) { + u, err := ms.ub.BuildManifestURL(ms.name, tag) + if err != nil { + return nil, err + } + + resp, err := ms.client.Get(u) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + switch { + case resp.StatusCode == http.StatusOK: + var sm manifest.SignedManifest + decoder := json.NewDecoder(resp.Body) + + if err := decoder.Decode(&sm); err != nil { + return nil, err + } + + return &sm, nil + case resp.StatusCode >= 400 && resp.StatusCode < 500: + return nil, parseHTTPErrorResponse(resp) + default: + return nil, &UnexpectedHTTPStatusError{Status: resp.Status} + } +} + +func (ms *manifests) Put(m *manifest.SignedManifest) error { + manifestURL, err := ms.ub.BuildManifestURL(ms.name, m.Tag) + if err != nil { + return err + } + + putRequest, err := http.NewRequest("PUT", manifestURL, bytes.NewReader(m.Raw)) + if err != nil { + return err + } + + resp, err := ms.client.Do(putRequest) + if err != nil { + return err + } + defer resp.Body.Close() + + switch { + case resp.StatusCode == http.StatusAccepted: + // TODO(dmcgowan): Use or check digest header + return nil + case resp.StatusCode >= 400 && resp.StatusCode < 500: + return parseHTTPErrorResponse(resp) + default: + return &UnexpectedHTTPStatusError{Status: resp.Status} + } +} + +func (ms *manifests) Delete(dgst digest.Digest) error { + u, err := ms.ub.BuildManifestURL(ms.name, dgst.String()) + if err != nil { + return err + } + req, err := http.NewRequest("DELETE", u, nil) + if err != nil { + return err + } + + resp, err := ms.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + switch { + case resp.StatusCode == http.StatusOK: + return nil + case resp.StatusCode >= 400 && resp.StatusCode < 500: + return parseHTTPErrorResponse(resp) + default: + return &UnexpectedHTTPStatusError{Status: resp.Status} + } +} + +type layers struct { + *repository +} + +func sanitizeLocation(location, source string) (string, error) { + locationURL, err := url.Parse(location) + if err != nil { + return "", err + } + + if locationURL.Scheme == "" { + sourceURL, err := url.Parse(source) + if err != nil { + return "", err + } + locationURL = &url.URL{ + Scheme: sourceURL.Scheme, + Host: sourceURL.Host, + Path: location, + } + location = locationURL.String() + } + return location, nil +} + +func (ls *layers) Exists(dgst digest.Digest) (bool, error) { + _, err := ls.fetchLayer(dgst) + if err != nil { + switch err := err.(type) { + case distribution.ErrUnknownLayer: + return false, nil + default: + return false, err + } + } + + return true, nil +} + +func (ls *layers) Fetch(dgst digest.Digest) (distribution.Layer, error) { + return ls.fetchLayer(dgst) +} + +func (ls *layers) Upload() (distribution.LayerUpload, error) { + u, err := ls.ub.BuildBlobUploadURL(ls.name) + + resp, err := ls.client.Post(u, "", nil) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + switch { + case resp.StatusCode == http.StatusAccepted: + // TODO(dmcgowan): Check for invalid UUID + uuid := resp.Header.Get("Docker-Upload-UUID") + location, err := sanitizeLocation(resp.Header.Get("Location"), u) + if err != nil { + return nil, err + } + + return &httpLayerUpload{ + layers: ls, + uuid: uuid, + startedAt: time.Now(), + location: location, + }, nil + case resp.StatusCode >= 400 && resp.StatusCode < 500: + return nil, parseHTTPErrorResponse(resp) + default: + return nil, &UnexpectedHTTPStatusError{Status: resp.Status} + } +} + +func (ls *layers) Resume(uuid string) (distribution.LayerUpload, error) { + panic("not implemented") +} + +func (ls *layers) fetchLayer(dgst digest.Digest) (distribution.Layer, error) { + u, err := ls.ub.BuildBlobURL(ls.name, dgst) + if err != nil { + return nil, err + } + + resp, err := ls.client.Head(u) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + switch { + case resp.StatusCode == http.StatusOK: + lengthHeader := resp.Header.Get("Content-Length") + length, err := strconv.ParseInt(lengthHeader, 10, 64) + if err != nil { + return nil, fmt.Errorf("error parsing content-length: %v", err) + } + + var t time.Time + lastModified := resp.Header.Get("Last-Modified") + if lastModified != "" { + t, err = http.ParseTime(lastModified) + if err != nil { + return nil, fmt.Errorf("error parsing last-modified: %v", err) + } + } + + return &httpLayer{ + layers: ls, + size: length, + digest: dgst, + createdAt: t, + }, nil + case resp.StatusCode == http.StatusNotFound: + return nil, distribution.ErrUnknownLayer{ + FSLayer: manifest.FSLayer{ + BlobSum: dgst, + }, + } + case resp.StatusCode >= 400 && resp.StatusCode < 500: + return nil, parseHTTPErrorResponse(resp) + default: + return nil, &UnexpectedHTTPStatusError{Status: resp.Status} + } +} + +type httpLayer struct { + *layers + + size int64 + digest digest.Digest + createdAt time.Time + + rc io.ReadCloser // remote read closer + brd *bufio.Reader // internal buffered io + offset int64 + err error +} + +func (hl *httpLayer) CreatedAt() time.Time { + return hl.createdAt +} + +func (hl *httpLayer) Digest() digest.Digest { + return hl.digest +} + +func (hl *httpLayer) Read(p []byte) (n int, err error) { + if hl.err != nil { + return 0, hl.err + } + + rd, err := hl.reader() + if err != nil { + return 0, err + } + + n, err = rd.Read(p) + hl.offset += int64(n) + + // Simulate io.EOR error if we reach filesize. + if err == nil && hl.offset >= hl.size { + err = io.EOF + } + + return n, err +} + +func (hl *httpLayer) Seek(offset int64, whence int) (int64, error) { + if hl.err != nil { + return 0, hl.err + } + + var err error + newOffset := hl.offset + + switch whence { + case os.SEEK_CUR: + newOffset += int64(offset) + case os.SEEK_END: + newOffset = hl.size + int64(offset) + case os.SEEK_SET: + newOffset = int64(offset) + } + + if newOffset < 0 { + err = fmt.Errorf("cannot seek to negative position") + } else { + if hl.offset != newOffset { + hl.reset() + } + + // No problems, set the offset. + hl.offset = newOffset + } + + return hl.offset, err +} + +func (hl *httpLayer) Close() error { + if hl.err != nil { + return hl.err + } + + // close and release reader chain + if hl.rc != nil { + hl.rc.Close() + } + + hl.rc = nil + hl.brd = nil + + hl.err = fmt.Errorf("httpLayer: closed") + + return nil +} + +func (hl *httpLayer) reset() { + if hl.err != nil { + return + } + if hl.rc != nil { + hl.rc.Close() + hl.rc = nil + } +} + +func (hl *httpLayer) reader() (io.Reader, error) { + if hl.err != nil { + return nil, hl.err + } + + if hl.rc != nil { + return hl.brd, nil + } + + // If the offset is great than or equal to size, return a empty, noop reader. + if hl.offset >= hl.size { + return ioutil.NopCloser(bytes.NewReader([]byte{})), nil + } + + blobURL, err := hl.ub.BuildBlobURL(hl.name, hl.digest) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("GET", blobURL, nil) + if err != nil { + return nil, err + } + + if hl.offset > 0 { + // TODO(stevvooe): Get this working correctly. + + // If we are at different offset, issue a range request from there. + req.Header.Add("Range", fmt.Sprintf("1-")) + ctxu.GetLogger(hl.context).Infof("Range: %s", req.Header.Get("Range")) + } + + resp, err := hl.client.Do(req) + if err != nil { + return nil, err + } + + switch { + case resp.StatusCode == 200: + hl.rc = resp.Body + default: + defer resp.Body.Close() + return nil, fmt.Errorf("unexpected status resolving reader: %v", resp.Status) + } + + if hl.brd == nil { + hl.brd = bufio.NewReader(hl.rc) + } else { + hl.brd.Reset(hl.rc) + } + + return hl.brd, nil +} + +func (hl *httpLayer) Length() int64 { + return hl.size +} + +func (hl *httpLayer) Handler(r *http.Request) (http.Handler, error) { + panic("Not implemented") +} + +type httpLayerUpload struct { + *layers + + uuid string + startedAt time.Time + + location string // always the last value of the location header. + offset int64 + closed bool +} + +var _ distribution.LayerUpload = &httpLayerUpload{} + +func (hlu *httpLayerUpload) ReadFrom(r io.Reader) (n int64, err error) { + req, err := http.NewRequest("PATCH", hlu.location, r) + if err != nil { + return 0, err + } + defer req.Body.Close() + + resp, err := hlu.client.Do(req) + if err != nil { + return 0, err + } + + switch { + case resp.StatusCode == http.StatusAccepted: + // TODO(dmcgowan): Validate headers + hlu.uuid = resp.Header.Get("Docker-Upload-UUID") + hlu.location, err = sanitizeLocation(resp.Header.Get("Location"), hlu.location) + if err != nil { + return 0, err + } + rng := resp.Header.Get("Range") + var start, end int64 + if n, err := fmt.Sscanf(rng, "%d-%d", &start, &end); err != nil { + return 0, err + } else if n != 2 || end < start { + return 0, fmt.Errorf("bad range format: %s", rng) + } + + return (end - start + 1), nil + case resp.StatusCode == http.StatusNotFound: + return 0, &BlobUploadNotFoundError{Location: hlu.location} + case resp.StatusCode >= 400 && resp.StatusCode < 500: + return 0, parseHTTPErrorResponse(resp) + default: + return 0, &UnexpectedHTTPStatusError{Status: resp.Status} + } +} + +func (hlu *httpLayerUpload) Write(p []byte) (n int, err error) { + req, err := http.NewRequest("PATCH", hlu.location, bytes.NewReader(p)) + if err != nil { + return 0, err + } + req.Header.Set("Content-Range", fmt.Sprintf("%d-%d", hlu.offset, hlu.offset+int64(len(p)-1))) + req.Header.Set("Content-Length", fmt.Sprintf("%d", len(p))) + req.Header.Set("Content-Type", "application/octet-stream") + + resp, err := hlu.client.Do(req) + if err != nil { + return 0, err + } + + switch { + case resp.StatusCode == http.StatusAccepted: + // TODO(dmcgowan): Validate headers + hlu.uuid = resp.Header.Get("Docker-Upload-UUID") + hlu.location, err = sanitizeLocation(resp.Header.Get("Location"), hlu.location) + if err != nil { + return 0, err + } + rng := resp.Header.Get("Range") + var start, end int + if n, err := fmt.Sscanf(rng, "%d-%d", &start, &end); err != nil { + return 0, err + } else if n != 2 || end < start { + return 0, fmt.Errorf("bad range format: %s", rng) + } + + return (end - start + 1), nil + case resp.StatusCode == http.StatusNotFound: + return 0, &BlobUploadNotFoundError{Location: hlu.location} + case resp.StatusCode >= 400 && resp.StatusCode < 500: + return 0, parseHTTPErrorResponse(resp) + default: + return 0, &UnexpectedHTTPStatusError{Status: resp.Status} + } +} + +func (hlu *httpLayerUpload) Seek(offset int64, whence int) (int64, error) { + newOffset := hlu.offset + + switch whence { + case os.SEEK_CUR: + newOffset += int64(offset) + case os.SEEK_END: + return newOffset, errors.New("Cannot seek from end on incomplete upload") + case os.SEEK_SET: + newOffset = int64(offset) + } + + hlu.offset = newOffset + + return hlu.offset, nil +} + +func (hlu *httpLayerUpload) UUID() string { + return hlu.uuid +} + +func (hlu *httpLayerUpload) StartedAt() time.Time { + return hlu.startedAt +} + +func (hlu *httpLayerUpload) Finish(digest digest.Digest) (distribution.Layer, error) { + // TODO(dmcgowan): Check if already finished, if so just fetch + req, err := http.NewRequest("PUT", hlu.location, nil) + if err != nil { + return nil, err + } + + values := req.URL.Query() + values.Set("digest", digest.String()) + req.URL.RawQuery = values.Encode() + + resp, err := hlu.client.Do(req) + if err != nil { + return nil, err + } + + switch { + case resp.StatusCode == http.StatusCreated: + return hlu.Layers().Fetch(digest) + case resp.StatusCode == http.StatusNotFound: + return nil, &BlobUploadNotFoundError{Location: hlu.location} + case resp.StatusCode >= 400 && resp.StatusCode < 500: + return nil, parseHTTPErrorResponse(resp) + default: + return nil, &UnexpectedHTTPStatusError{Status: resp.Status} + } +} + +func (hlu *httpLayerUpload) Cancel() error { + panic("not implemented") +} + +func (hlu *httpLayerUpload) Close() error { + hlu.closed = true + return nil +} diff --git a/registry/client/repository_test.go b/registry/client/repository_test.go new file mode 100644 index 00000000..67138db6 --- /dev/null +++ b/registry/client/repository_test.go @@ -0,0 +1,605 @@ +package client + +import ( + "bytes" + "crypto/rand" + "encoding/json" + "fmt" + "io/ioutil" + "log" + "net/http" + "net/http/httptest" + "testing" + "time" + + "code.google.com/p/go-uuid/uuid" + + "github.com/docker/distribution/digest" + "github.com/docker/distribution/manifest" + "github.com/docker/distribution/testutil" + "golang.org/x/net/context" +) + +func testServer(rrm testutil.RequestResponseMap) (*RepositoryEndpoint, func()) { + h := testutil.NewHandler(rrm) + s := httptest.NewServer(h) + e := RepositoryEndpoint{Endpoint: s.URL, Mirror: false} + return &e, s.Close +} + +func newRandomBlob(size int) (digest.Digest, []byte) { + b := make([]byte, size) + if n, err := rand.Read(b); err != nil { + panic(err) + } else if n != size { + panic("unable to read enough bytes") + } + + dgst, err := digest.FromBytes(b) + if err != nil { + panic(err) + } + + return dgst, b +} + +func addTestFetch(repo string, dgst digest.Digest, content []byte, m *testutil.RequestResponseMap) { + *m = append(*m, testutil.RequestResponseMapping{ + Request: testutil.Request{ + Method: "GET", + Route: "/v2/" + repo + "/blobs/" + dgst.String(), + }, + Response: testutil.Response{ + StatusCode: http.StatusOK, + Body: content, + Headers: http.Header(map[string][]string{ + "Content-Length": {fmt.Sprint(len(content))}, + "Last-Modified": {time.Now().Add(-1 * time.Second).Format(time.ANSIC)}, + }), + }, + }) + *m = append(*m, testutil.RequestResponseMapping{ + Request: testutil.Request{ + Method: "HEAD", + Route: "/v2/" + repo + "/blobs/" + dgst.String(), + }, + Response: testutil.Response{ + StatusCode: http.StatusOK, + Headers: http.Header(map[string][]string{ + "Content-Length": {fmt.Sprint(len(content))}, + "Last-Modified": {time.Now().Add(-1 * time.Second).Format(time.ANSIC)}, + }), + }, + }) +} + +func addPing(m *testutil.RequestResponseMap) { + *m = append(*m, testutil.RequestResponseMapping{ + Request: testutil.Request{ + Method: "GET", + Route: "/v2/", + }, + Response: testutil.Response{ + StatusCode: http.StatusOK, + Headers: http.Header(map[string][]string{ + "Docker-Distribution-API-Version": {"registry/2.0"}, + }), + }, + }) +} + +func TestLayerFetch(t *testing.T) { + d1, b1 := newRandomBlob(1024) + var m testutil.RequestResponseMap + addTestFetch("test.example.com/repo1", d1, b1, &m) + addPing(&m) + + e, c := testServer(m) + defer c() + + r, err := NewRepositoryClient(context.Background(), "test.example.com/repo1", e) + if err != nil { + t.Fatal(err) + } + l := r.Layers() + + layer, err := l.Fetch(d1) + if err != nil { + t.Fatal(err) + } + b, err := ioutil.ReadAll(layer) + if err != nil { + t.Fatal(err) + } + if bytes.Compare(b, b1) != 0 { + t.Fatalf("Wrong bytes values fetched: [%d]byte != [%d]byte", len(b), len(b1)) + } + + // TODO(dmcgowan): Test error cases +} + +func TestLayerExists(t *testing.T) { + d1, b1 := newRandomBlob(1024) + var m testutil.RequestResponseMap + addTestFetch("test.example.com/repo1", d1, b1, &m) + addPing(&m) + + e, c := testServer(m) + defer c() + + r, err := NewRepositoryClient(context.Background(), "test.example.com/repo1", e) + if err != nil { + t.Fatal(err) + } + l := r.Layers() + + ok, err := l.Exists(d1) + if err != nil { + t.Fatal(err) + } + if !ok { + t.Fatalf("Blob does not exist: %s", d1) + } + + // TODO(dmcgowan): Test error cases +} + +func TestLayerUploadChunked(t *testing.T) { + dgst, b1 := newRandomBlob(1024) + var m testutil.RequestResponseMap + addPing(&m) + chunks := [][]byte{ + b1[0:256], + b1[256:512], + b1[512:513], + b1[513:1024], + } + repo := "test.example.com/uploadrepo" + uuids := []string{uuid.New()} + m = append(m, testutil.RequestResponseMapping{ + Request: testutil.Request{ + Method: "POST", + Route: "/v2/" + repo + "/blobs/uploads/", + }, + Response: testutil.Response{ + StatusCode: http.StatusAccepted, + Headers: http.Header(map[string][]string{ + "Content-Length": {"0"}, + "Location": {"/v2/" + repo + "/blobs/uploads/" + uuids[0]}, + "Docker-Upload-UUID": {uuids[0]}, + "Range": {"0-0"}, + }), + }, + }) + offset := 0 + for i, chunk := range chunks { + uuids = append(uuids, uuid.New()) + newOffset := offset + len(chunk) + m = append(m, testutil.RequestResponseMapping{ + Request: testutil.Request{ + Method: "PATCH", + Route: "/v2/" + repo + "/blobs/uploads/" + uuids[i], + Body: chunk, + }, + Response: testutil.Response{ + StatusCode: http.StatusAccepted, + Headers: http.Header(map[string][]string{ + "Content-Length": {"0"}, + "Location": {"/v2/" + repo + "/blobs/uploads/" + uuids[i+1]}, + "Docker-Upload-UUID": {uuids[i+1]}, + "Range": {fmt.Sprintf("%d-%d", offset, newOffset-1)}, + }), + }, + }) + offset = newOffset + } + m = append(m, testutil.RequestResponseMapping{ + Request: testutil.Request{ + Method: "PUT", + Route: "/v2/" + repo + "/blobs/uploads/" + uuids[len(uuids)-1], + QueryParams: map[string][]string{ + "digest": {dgst.String()}, + }, + }, + Response: testutil.Response{ + StatusCode: http.StatusCreated, + Headers: http.Header(map[string][]string{ + "Content-Length": {"0"}, + "Docker-Content-Digest": {dgst.String()}, + "Content-Range": {fmt.Sprintf("0-%d", offset-1)}, + }), + }, + }) + m = append(m, testutil.RequestResponseMapping{ + Request: testutil.Request{ + Method: "HEAD", + Route: "/v2/" + repo + "/blobs/" + dgst.String(), + }, + Response: testutil.Response{ + StatusCode: http.StatusOK, + Headers: http.Header(map[string][]string{ + "Content-Length": {fmt.Sprint(offset)}, + "Last-Modified": {time.Now().Add(-1 * time.Second).Format(time.ANSIC)}, + }), + }, + }) + + e, c := testServer(m) + defer c() + + r, err := NewRepositoryClient(context.Background(), repo, e) + if err != nil { + t.Fatal(err) + } + l := r.Layers() + + upload, err := l.Upload() + if err != nil { + t.Fatal(err) + } + + if upload.UUID() != uuids[0] { + log.Fatalf("Unexpected UUID %s; expected %s", upload.UUID(), uuids[0]) + } + + for _, chunk := range chunks { + n, err := upload.Write(chunk) + if err != nil { + t.Fatal(err) + } + if n != len(chunk) { + t.Fatalf("Unexpected length returned from write: %d; expected: %d", n, len(chunk)) + } + } + + layer, err := upload.Finish(dgst) + if err != nil { + t.Fatal(err) + } + + if layer.Length() != int64(len(b1)) { + t.Fatalf("Unexpected layer size: %d; expected: %d", layer.Length(), len(b1)) + } +} + +func TestLayerUploadMonolithic(t *testing.T) { + dgst, b1 := newRandomBlob(1024) + var m testutil.RequestResponseMap + addPing(&m) + repo := "test.example.com/uploadrepo" + uploadID := uuid.New() + m = append(m, testutil.RequestResponseMapping{ + Request: testutil.Request{ + Method: "POST", + Route: "/v2/" + repo + "/blobs/uploads/", + }, + Response: testutil.Response{ + StatusCode: http.StatusAccepted, + Headers: http.Header(map[string][]string{ + "Content-Length": {"0"}, + "Location": {"/v2/" + repo + "/blobs/uploads/" + uploadID}, + "Docker-Upload-UUID": {uploadID}, + "Range": {"0-0"}, + }), + }, + }) + m = append(m, testutil.RequestResponseMapping{ + Request: testutil.Request{ + Method: "PATCH", + Route: "/v2/" + repo + "/blobs/uploads/" + uploadID, + Body: b1, + }, + Response: testutil.Response{ + StatusCode: http.StatusAccepted, + Headers: http.Header(map[string][]string{ + "Location": {"/v2/" + repo + "/blobs/uploads/" + uploadID}, + "Docker-Upload-UUID": {uploadID}, + "Content-Length": {"0"}, + "Docker-Content-Digest": {dgst.String()}, + "Range": {fmt.Sprintf("0-%d", len(b1)-1)}, + }), + }, + }) + m = append(m, testutil.RequestResponseMapping{ + Request: testutil.Request{ + Method: "PUT", + Route: "/v2/" + repo + "/blobs/uploads/" + uploadID, + QueryParams: map[string][]string{ + "digest": {dgst.String()}, + }, + }, + Response: testutil.Response{ + StatusCode: http.StatusCreated, + Headers: http.Header(map[string][]string{ + "Content-Length": {"0"}, + "Docker-Content-Digest": {dgst.String()}, + "Content-Range": {fmt.Sprintf("0-%d", len(b1)-1)}, + }), + }, + }) + m = append(m, testutil.RequestResponseMapping{ + Request: testutil.Request{ + Method: "HEAD", + Route: "/v2/" + repo + "/blobs/" + dgst.String(), + }, + Response: testutil.Response{ + StatusCode: http.StatusOK, + Headers: http.Header(map[string][]string{ + "Content-Length": {fmt.Sprint(len(b1))}, + "Last-Modified": {time.Now().Add(-1 * time.Second).Format(time.ANSIC)}, + }), + }, + }) + + e, c := testServer(m) + defer c() + + r, err := NewRepositoryClient(context.Background(), repo, e) + if err != nil { + t.Fatal(err) + } + l := r.Layers() + + upload, err := l.Upload() + if err != nil { + t.Fatal(err) + } + + if upload.UUID() != uploadID { + log.Fatalf("Unexpected UUID %s; expected %s", upload.UUID(), uploadID) + } + + n, err := upload.ReadFrom(bytes.NewReader(b1)) + if err != nil { + t.Fatal(err) + } + if n != int64(len(b1)) { + t.Fatalf("Unexpected ReadFrom length: %d; expected: %d", n, len(b1)) + } + + layer, err := upload.Finish(dgst) + if err != nil { + t.Fatal(err) + } + + if layer.Length() != int64(len(b1)) { + t.Fatalf("Unexpected layer size: %d; expected: %d", layer.Length(), len(b1)) + } +} + +func TestLayerUploadResume(t *testing.T) { + // TODO(dmcgowan): implement +} + +func newRandomSchema1Manifest(name, tag string, blobCount int) (*manifest.SignedManifest, digest.Digest) { + blobs := make([]manifest.FSLayer, blobCount) + history := make([]manifest.History, blobCount) + + for i := 0; i < blobCount; i++ { + dgst, blob := newRandomBlob((i % 5) * 16) + + blobs[i] = manifest.FSLayer{BlobSum: dgst} + history[i] = manifest.History{V1Compatibility: fmt.Sprintf("{\"Hex\": \"%x\"}", blob)} + } + + m := &manifest.SignedManifest{ + Manifest: manifest.Manifest{ + Name: name, + Tag: tag, + Architecture: "x86", + FSLayers: blobs, + History: history, + Versioned: manifest.Versioned{ + SchemaVersion: 1, + }, + }, + } + manifestBytes, err := json.Marshal(m) + if err != nil { + panic(err) + } + dgst, err := digest.FromBytes(manifestBytes) + if err != nil { + panic(err) + } + + m.Raw = manifestBytes + + return m, dgst +} + +func addTestManifest(repo, reference string, content []byte, m *testutil.RequestResponseMap) { + *m = append(*m, testutil.RequestResponseMapping{ + Request: testutil.Request{ + Method: "GET", + Route: "/v2/" + repo + "/manifests/" + reference, + }, + Response: testutil.Response{ + StatusCode: http.StatusOK, + Body: content, + Headers: http.Header(map[string][]string{ + "Content-Length": {fmt.Sprint(len(content))}, + "Last-Modified": {time.Now().Add(-1 * time.Second).Format(time.ANSIC)}, + }), + }, + }) + *m = append(*m, testutil.RequestResponseMapping{ + Request: testutil.Request{ + Method: "HEAD", + Route: "/v2/" + repo + "/manifests/" + reference, + }, + Response: testutil.Response{ + StatusCode: http.StatusOK, + Headers: http.Header(map[string][]string{ + "Content-Length": {fmt.Sprint(len(content))}, + "Last-Modified": {time.Now().Add(-1 * time.Second).Format(time.ANSIC)}, + }), + }, + }) + +} + +func checkEqualManifest(m1, m2 *manifest.SignedManifest) error { + if m1.Name != m2.Name { + return fmt.Errorf("name does not match %q != %q", m1.Name, m2.Name) + } + if m1.Tag != m2.Tag { + return fmt.Errorf("tag does not match %q != %q", m1.Tag, m2.Tag) + } + if len(m1.FSLayers) != len(m2.FSLayers) { + return fmt.Errorf("fs layer length does not match %d != %d", len(m1.FSLayers), len(m2.FSLayers)) + } + for i := range m1.FSLayers { + if m1.FSLayers[i].BlobSum != m2.FSLayers[i].BlobSum { + return fmt.Errorf("blobsum does not match %q != %q", m1.FSLayers[i].BlobSum, m2.FSLayers[i].BlobSum) + } + } + if len(m1.History) != len(m2.History) { + return fmt.Errorf("history length does not match %d != %d", len(m1.History), len(m2.History)) + } + for i := range m1.History { + if m1.History[i].V1Compatibility != m2.History[i].V1Compatibility { + return fmt.Errorf("blobsum does not match %q != %q", m1.History[i].V1Compatibility, m2.History[i].V1Compatibility) + } + } + return nil +} + +func TestManifestFetch(t *testing.T) { + repo := "test.example.com/repo" + m1, dgst := newRandomSchema1Manifest(repo, "latest", 6) + var m testutil.RequestResponseMap + addPing(&m) + addTestManifest(repo, dgst.String(), m1.Raw, &m) + + e, c := testServer(m) + defer c() + + r, err := NewRepositoryClient(context.Background(), repo, e) + if err != nil { + t.Fatal(err) + } + ms := r.Manifests() + + ok, err := ms.Exists(dgst) + if err != nil { + t.Fatal(err) + } + if !ok { + t.Fatal("Manifest does not exist") + } + + manifest, err := ms.Get(dgst) + if err != nil { + t.Fatal(err) + } + if err := checkEqualManifest(manifest, m1); err != nil { + t.Fatal(err) + } +} + +func TestManifestFetchByTag(t *testing.T) { + repo := "test.example.com/repo/by/tag" + m1, _ := newRandomSchema1Manifest(repo, "latest", 6) + var m testutil.RequestResponseMap + addPing(&m) + addTestManifest(repo, "latest", m1.Raw, &m) + + e, c := testServer(m) + defer c() + + r, err := NewRepositoryClient(context.Background(), repo, e) + if err != nil { + t.Fatal(err) + } + + ms := r.Manifests() + ok, err := ms.ExistsByTag("latest") + if err != nil { + t.Fatal(err) + } + if !ok { + t.Fatal("Manifest does not exist") + } + + manifest, err := ms.GetByTag("latest") + if err != nil { + t.Fatal(err) + } + if err := checkEqualManifest(manifest, m1); err != nil { + t.Fatal(err) + } +} + +func TestManifestDelete(t *testing.T) { + repo := "test.example.com/repo/delete" + _, dgst1 := newRandomSchema1Manifest(repo, "latest", 6) + _, dgst2 := newRandomSchema1Manifest(repo, "latest", 6) + var m testutil.RequestResponseMap + addPing(&m) + m = append(m, testutil.RequestResponseMapping{ + Request: testutil.Request{ + Method: "DELETE", + Route: "/v2/" + repo + "/manifests/" + dgst1.String(), + }, + Response: testutil.Response{ + StatusCode: http.StatusOK, + Headers: http.Header(map[string][]string{ + "Content-Length": {"0"}, + }), + }, + }) + + e, c := testServer(m) + defer c() + + r, err := NewRepositoryClient(context.Background(), repo, e) + if err != nil { + t.Fatal(err) + } + + ms := r.Manifests() + if err := ms.Delete(dgst1); err != nil { + t.Fatal(err) + } + if err := ms.Delete(dgst2); err == nil { + t.Fatal("Expected error deleting unknown manifest") + } + // TODO(dmcgowan): Check for specific unknown error +} + +func TestManifestPut(t *testing.T) { + repo := "test.example.com/repo/delete" + m1, dgst := newRandomSchema1Manifest(repo, "other", 6) + var m testutil.RequestResponseMap + addPing(&m) + m = append(m, testutil.RequestResponseMapping{ + Request: testutil.Request{ + Method: "PUT", + Route: "/v2/" + repo + "/manifests/other", + Body: m1.Raw, + }, + Response: testutil.Response{ + StatusCode: http.StatusAccepted, + Headers: http.Header(map[string][]string{ + "Content-Length": {"0"}, + "Docker-Content-Digest": {dgst.String()}, + }), + }, + }) + + e, c := testServer(m) + defer c() + + r, err := NewRepositoryClient(context.Background(), repo, e) + if err != nil { + t.Fatal(err) + } + + ms := r.Manifests() + if err := ms.Put(m1); err != nil { + t.Fatal(err) + } + + // TODO(dmcgowan): Check for error cases +} diff --git a/registry/client/token.go b/registry/client/token.go new file mode 100644 index 00000000..6439e01e --- /dev/null +++ b/registry/client/token.go @@ -0,0 +1,78 @@ +package client + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "strings" +) + +type tokenResponse struct { + Token string `json:"token"` +} + +func getToken(creds CredentialStore, params map[string]string, client *http.Client) (token string, err error) { + realm, ok := params["realm"] + if !ok { + return "", errors.New("no realm specified for token auth challenge") + } + + realmURL, err := url.Parse(realm) + if err != nil { + return "", fmt.Errorf("invalid token auth challenge realm: %s", err) + } + + // TODO(dmcgowan): Handle empty scheme + + req, err := http.NewRequest("GET", realmURL.String(), nil) + if err != nil { + return "", err + } + + reqParams := req.URL.Query() + service := params["service"] + scope := params["scope"] + + if service != "" { + reqParams.Add("service", service) + } + + for _, scopeField := range strings.Fields(scope) { + reqParams.Add("scope", scopeField) + } + + if creds != nil { + username, password := creds.Basic(realmURL) + if username != "" && password != "" { + reqParams.Add("account", username) + req.SetBasicAuth(username, password) + } + } + + req.URL.RawQuery = reqParams.Encode() + + resp, err := client.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("token auth attempt for registry: %s request failed with status: %d %s", req.URL, resp.StatusCode, http.StatusText(resp.StatusCode)) + } + + decoder := json.NewDecoder(resp.Body) + + tr := new(tokenResponse) + if err = decoder.Decode(tr); err != nil { + return "", fmt.Errorf("unable to decode token response: %s", err) + } + + if tr.Token == "" { + return "", errors.New("authorization server did not include a token in the response") + } + + return tr.Token, nil +} From 174a732c94d219e5fd90f3565351eb00d90bc0df Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 6 May 2015 11:12:33 -0700 Subject: [PATCH 02/26] Remove deprecated client interface Signed-off-by: Derek McGowan (github: dmcgowan) --- registry/client/client.go | 573 --------------------------------- registry/client/client_test.go | 440 ------------------------- registry/client/objectstore.go | 239 -------------- registry/client/pull.go | 151 --------- registry/client/push.go | 137 -------- 5 files changed, 1540 deletions(-) delete mode 100644 registry/client/client.go delete mode 100644 registry/client/client_test.go delete mode 100644 registry/client/objectstore.go delete mode 100644 registry/client/pull.go delete mode 100644 registry/client/push.go diff --git a/registry/client/client.go b/registry/client/client.go deleted file mode 100644 index 36be960d..00000000 --- a/registry/client/client.go +++ /dev/null @@ -1,573 +0,0 @@ -package client - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "net/http" - "regexp" - "strconv" - - "github.com/docker/distribution/digest" - "github.com/docker/distribution/manifest" - "github.com/docker/distribution/registry/api/v2" -) - -// Client implements the client interface to the registry http api -type Client interface { - // GetImageManifest returns an image manifest for the image at the given - // name, tag pair. - GetImageManifest(name, tag string) (*manifest.SignedManifest, error) - - // PutImageManifest uploads an image manifest for the image at the given - // name, tag pair. - PutImageManifest(name, tag string, imageManifest *manifest.SignedManifest) error - - // DeleteImage removes the image at the given name, tag pair. - DeleteImage(name, tag string) error - - // ListImageTags returns a list of all image tags with the given repository - // name. - ListImageTags(name string) ([]string, error) - - // BlobLength returns the length of the blob stored at the given name, - // digest pair. - // Returns a length value of -1 on error or if the blob does not exist. - BlobLength(name string, dgst digest.Digest) (int, error) - - // GetBlob returns the blob stored at the given name, digest pair in the - // form of an io.ReadCloser with the length of this blob. - // A nonzero byteOffset can be provided to receive a partial blob beginning - // at the given offset. - GetBlob(name string, dgst digest.Digest, byteOffset int) (io.ReadCloser, int, error) - - // InitiateBlobUpload starts a blob upload in the given repository namespace - // and returns a unique location url to use for other blob upload methods. - InitiateBlobUpload(name string) (string, error) - - // GetBlobUploadStatus returns the byte offset and length of the blob at the - // given upload location. - GetBlobUploadStatus(location string) (int, int, error) - - // UploadBlob uploads a full blob to the registry. - UploadBlob(location string, blob io.ReadCloser, length int, dgst digest.Digest) error - - // UploadBlobChunk uploads a blob chunk with a given length and startByte to - // the registry. - // FinishChunkedBlobUpload must be called to finalize this upload. - UploadBlobChunk(location string, blobChunk io.ReadCloser, length, startByte int) error - - // FinishChunkedBlobUpload completes a chunked blob upload at a given - // location. - FinishChunkedBlobUpload(location string, length int, dgst digest.Digest) error - - // CancelBlobUpload deletes all content at the unfinished blob upload - // location and invalidates any future calls to this blob upload. - CancelBlobUpload(location string) error -} - -var ( - patternRangeHeader = regexp.MustCompile("bytes=0-(\\d+)/(\\d+)") -) - -// New returns a new Client which operates against a registry with the -// given base endpoint -// This endpoint should not include /v2/ or any part of the url after this. -func New(endpoint string) (Client, error) { - ub, err := v2.NewURLBuilderFromString(endpoint) - if err != nil { - return nil, err - } - - return &clientImpl{ - endpoint: endpoint, - ub: ub, - }, nil -} - -// clientImpl is the default implementation of the Client interface -type clientImpl struct { - endpoint string - ub *v2.URLBuilder -} - -// TODO(bbland): use consistent route generation between server and client - -func (r *clientImpl) GetImageManifest(name, tag string) (*manifest.SignedManifest, error) { - manifestURL, err := r.ub.BuildManifestURL(name, tag) - if err != nil { - return nil, err - } - - response, err := http.Get(manifestURL) - if err != nil { - return nil, err - } - defer response.Body.Close() - - // TODO(bbland): handle other status codes, like 5xx errors - switch { - case response.StatusCode == http.StatusOK: - break - case response.StatusCode == http.StatusNotFound: - return nil, &ImageManifestNotFoundError{Name: name, Tag: tag} - case response.StatusCode >= 400 && response.StatusCode < 500: - var errs v2.Errors - - decoder := json.NewDecoder(response.Body) - err = decoder.Decode(&errs) - if err != nil { - return nil, err - } - return nil, &errs - default: - return nil, &UnexpectedHTTPStatusError{Status: response.Status} - } - - decoder := json.NewDecoder(response.Body) - - manifest := new(manifest.SignedManifest) - err = decoder.Decode(manifest) - if err != nil { - return nil, err - } - return manifest, nil -} - -func (r *clientImpl) PutImageManifest(name, tag string, manifest *manifest.SignedManifest) error { - manifestURL, err := r.ub.BuildManifestURL(name, tag) - if err != nil { - return err - } - - putRequest, err := http.NewRequest("PUT", manifestURL, bytes.NewReader(manifest.Raw)) - if err != nil { - return err - } - - response, err := http.DefaultClient.Do(putRequest) - if err != nil { - return err - } - defer response.Body.Close() - - // TODO(bbland): handle other status codes, like 5xx errors - switch { - case response.StatusCode == http.StatusOK || response.StatusCode == http.StatusAccepted: - return nil - case response.StatusCode >= 400 && response.StatusCode < 500: - var errors v2.Errors - decoder := json.NewDecoder(response.Body) - err = decoder.Decode(&errors) - if err != nil { - return err - } - - return &errors - default: - return &UnexpectedHTTPStatusError{Status: response.Status} - } -} - -func (r *clientImpl) DeleteImage(name, tag string) error { - manifestURL, err := r.ub.BuildManifestURL(name, tag) - if err != nil { - return err - } - - deleteRequest, err := http.NewRequest("DELETE", manifestURL, nil) - if err != nil { - return err - } - - response, err := http.DefaultClient.Do(deleteRequest) - if err != nil { - return err - } - defer response.Body.Close() - - // TODO(bbland): handle other status codes, like 5xx errors - switch { - case response.StatusCode == http.StatusNoContent: - break - case response.StatusCode == http.StatusNotFound: - return &ImageManifestNotFoundError{Name: name, Tag: tag} - case response.StatusCode >= 400 && response.StatusCode < 500: - var errs v2.Errors - decoder := json.NewDecoder(response.Body) - err = decoder.Decode(&errs) - if err != nil { - return err - } - return &errs - default: - return &UnexpectedHTTPStatusError{Status: response.Status} - } - - return nil -} - -func (r *clientImpl) ListImageTags(name string) ([]string, error) { - tagsURL, err := r.ub.BuildTagsURL(name) - if err != nil { - return nil, err - } - - response, err := http.Get(tagsURL) - if err != nil { - return nil, err - } - defer response.Body.Close() - - // TODO(bbland): handle other status codes, like 5xx errors - switch { - case response.StatusCode == http.StatusOK: - break - case response.StatusCode == http.StatusNotFound: - return nil, &RepositoryNotFoundError{Name: name} - case response.StatusCode >= 400 && response.StatusCode < 500: - var errs v2.Errors - decoder := json.NewDecoder(response.Body) - err = decoder.Decode(&errs) - if err != nil { - return nil, err - } - return nil, &errs - default: - return nil, &UnexpectedHTTPStatusError{Status: response.Status} - } - - tags := struct { - Tags []string `json:"tags"` - }{} - - decoder := json.NewDecoder(response.Body) - err = decoder.Decode(&tags) - if err != nil { - return nil, err - } - - return tags.Tags, nil -} - -func (r *clientImpl) BlobLength(name string, dgst digest.Digest) (int, error) { - blobURL, err := r.ub.BuildBlobURL(name, dgst) - if err != nil { - return -1, err - } - - response, err := http.Head(blobURL) - if err != nil { - return -1, err - } - defer response.Body.Close() - - // TODO(bbland): handle other status codes, like 5xx errors - switch { - case response.StatusCode == http.StatusOK: - lengthHeader := response.Header.Get("Content-Length") - length, err := strconv.ParseInt(lengthHeader, 10, 64) - if err != nil { - return -1, err - } - return int(length), nil - case response.StatusCode == http.StatusNotFound: - return -1, nil - case response.StatusCode >= 400 && response.StatusCode < 500: - var errs v2.Errors - decoder := json.NewDecoder(response.Body) - err = decoder.Decode(&errs) - if err != nil { - return -1, err - } - return -1, &errs - default: - return -1, &UnexpectedHTTPStatusError{Status: response.Status} - } -} - -func (r *clientImpl) GetBlob(name string, dgst digest.Digest, byteOffset int) (io.ReadCloser, int, error) { - blobURL, err := r.ub.BuildBlobURL(name, dgst) - if err != nil { - return nil, 0, err - } - - getRequest, err := http.NewRequest("GET", blobURL, nil) - if err != nil { - return nil, 0, err - } - - getRequest.Header.Add("Range", fmt.Sprintf("%d-", byteOffset)) - response, err := http.DefaultClient.Do(getRequest) - if err != nil { - return nil, 0, err - } - - // TODO(bbland): handle other status codes, like 5xx errors - switch { - case response.StatusCode == http.StatusOK: - lengthHeader := response.Header.Get("Content-Length") - length, err := strconv.ParseInt(lengthHeader, 10, 0) - if err != nil { - return nil, 0, err - } - return response.Body, int(length), nil - case response.StatusCode == http.StatusNotFound: - response.Body.Close() - return nil, 0, &BlobNotFoundError{Name: name, Digest: dgst} - case response.StatusCode >= 400 && response.StatusCode < 500: - var errs v2.Errors - decoder := json.NewDecoder(response.Body) - err = decoder.Decode(&errs) - if err != nil { - return nil, 0, err - } - return nil, 0, &errs - default: - response.Body.Close() - return nil, 0, &UnexpectedHTTPStatusError{Status: response.Status} - } -} - -func (r *clientImpl) InitiateBlobUpload(name string) (string, error) { - uploadURL, err := r.ub.BuildBlobUploadURL(name) - if err != nil { - return "", err - } - - postRequest, err := http.NewRequest("POST", uploadURL, nil) - if err != nil { - return "", err - } - - response, err := http.DefaultClient.Do(postRequest) - if err != nil { - return "", err - } - defer response.Body.Close() - - // TODO(bbland): handle other status codes, like 5xx errors - switch { - case response.StatusCode == http.StatusAccepted: - return response.Header.Get("Location"), nil - // case response.StatusCode == http.StatusNotFound: - // return - case response.StatusCode >= 400 && response.StatusCode < 500: - var errs v2.Errors - decoder := json.NewDecoder(response.Body) - err = decoder.Decode(&errs) - if err != nil { - return "", err - } - return "", &errs - default: - return "", &UnexpectedHTTPStatusError{Status: response.Status} - } -} - -func (r *clientImpl) GetBlobUploadStatus(location string) (int, int, error) { - response, err := http.Get(location) - if err != nil { - return 0, 0, err - } - defer response.Body.Close() - - // TODO(bbland): handle other status codes, like 5xx errors - switch { - case response.StatusCode == http.StatusNoContent: - return parseRangeHeader(response.Header.Get("Range")) - case response.StatusCode == http.StatusNotFound: - return 0, 0, &BlobUploadNotFoundError{Location: location} - case response.StatusCode >= 400 && response.StatusCode < 500: - var errs v2.Errors - decoder := json.NewDecoder(response.Body) - err = decoder.Decode(&errs) - if err != nil { - return 0, 0, err - } - return 0, 0, &errs - default: - return 0, 0, &UnexpectedHTTPStatusError{Status: response.Status} - } -} - -func (r *clientImpl) UploadBlob(location string, blob io.ReadCloser, length int, dgst digest.Digest) error { - defer blob.Close() - - putRequest, err := http.NewRequest("PUT", location, blob) - if err != nil { - return err - } - - values := putRequest.URL.Query() - values.Set("digest", dgst.String()) - putRequest.URL.RawQuery = values.Encode() - - putRequest.Header.Set("Content-Type", "application/octet-stream") - putRequest.Header.Set("Content-Length", fmt.Sprint(length)) - - response, err := http.DefaultClient.Do(putRequest) - if err != nil { - return err - } - defer response.Body.Close() - - // TODO(bbland): handle other status codes, like 5xx errors - switch { - case response.StatusCode == http.StatusCreated: - return nil - case response.StatusCode == http.StatusNotFound: - return &BlobUploadNotFoundError{Location: location} - case response.StatusCode >= 400 && response.StatusCode < 500: - var errs v2.Errors - decoder := json.NewDecoder(response.Body) - err = decoder.Decode(&errs) - if err != nil { - return err - } - return &errs - default: - return &UnexpectedHTTPStatusError{Status: response.Status} - } -} - -func (r *clientImpl) UploadBlobChunk(location string, blobChunk io.ReadCloser, length, startByte int) error { - defer blobChunk.Close() - - putRequest, err := http.NewRequest("PUT", location, blobChunk) - if err != nil { - return err - } - - endByte := startByte + length - - putRequest.Header.Set("Content-Type", "application/octet-stream") - putRequest.Header.Set("Content-Length", fmt.Sprint(length)) - putRequest.Header.Set("Content-Range", - fmt.Sprintf("%d-%d/%d", startByte, endByte, endByte)) - - response, err := http.DefaultClient.Do(putRequest) - if err != nil { - return err - } - defer response.Body.Close() - - // TODO(bbland): handle other status codes, like 5xx errors - switch { - case response.StatusCode == http.StatusAccepted: - return nil - case response.StatusCode == http.StatusRequestedRangeNotSatisfiable: - lastValidRange, blobSize, err := parseRangeHeader(response.Header.Get("Range")) - if err != nil { - return err - } - return &BlobUploadInvalidRangeError{ - Location: location, - LastValidRange: lastValidRange, - BlobSize: blobSize, - } - case response.StatusCode == http.StatusNotFound: - return &BlobUploadNotFoundError{Location: location} - case response.StatusCode >= 400 && response.StatusCode < 500: - var errs v2.Errors - decoder := json.NewDecoder(response.Body) - err = decoder.Decode(&errs) - if err != nil { - return err - } - return &errs - default: - return &UnexpectedHTTPStatusError{Status: response.Status} - } -} - -func (r *clientImpl) FinishChunkedBlobUpload(location string, length int, dgst digest.Digest) error { - putRequest, err := http.NewRequest("PUT", location, nil) - if err != nil { - return err - } - - values := putRequest.URL.Query() - values.Set("digest", dgst.String()) - putRequest.URL.RawQuery = values.Encode() - - putRequest.Header.Set("Content-Type", "application/octet-stream") - putRequest.Header.Set("Content-Length", "0") - putRequest.Header.Set("Content-Range", - fmt.Sprintf("%d-%d/%d", length, length, length)) - - response, err := http.DefaultClient.Do(putRequest) - if err != nil { - return err - } - defer response.Body.Close() - - // TODO(bbland): handle other status codes, like 5xx errors - switch { - case response.StatusCode == http.StatusCreated: - return nil - case response.StatusCode == http.StatusNotFound: - return &BlobUploadNotFoundError{Location: location} - case response.StatusCode >= 400 && response.StatusCode < 500: - var errs v2.Errors - decoder := json.NewDecoder(response.Body) - err = decoder.Decode(&errs) - if err != nil { - return err - } - return &errs - default: - return &UnexpectedHTTPStatusError{Status: response.Status} - } -} - -func (r *clientImpl) CancelBlobUpload(location string) error { - deleteRequest, err := http.NewRequest("DELETE", location, nil) - if err != nil { - return err - } - - response, err := http.DefaultClient.Do(deleteRequest) - if err != nil { - return err - } - defer response.Body.Close() - - // TODO(bbland): handle other status codes, like 5xx errors - switch { - case response.StatusCode == http.StatusNoContent: - return nil - case response.StatusCode == http.StatusNotFound: - return &BlobUploadNotFoundError{Location: location} - case response.StatusCode >= 400 && response.StatusCode < 500: - var errs v2.Errors - decoder := json.NewDecoder(response.Body) - err = decoder.Decode(&errs) - if err != nil { - return err - } - return &errs - default: - return &UnexpectedHTTPStatusError{Status: response.Status} - } -} - -// parseRangeHeader parses out the offset and length from a returned Range -// header -func parseRangeHeader(byteRangeHeader string) (int, int, error) { - submatches := patternRangeHeader.FindStringSubmatch(byteRangeHeader) - if submatches == nil || len(submatches) < 3 { - return 0, 0, fmt.Errorf("Malformed Range header") - } - - offset, err := strconv.Atoi(submatches[1]) - if err != nil { - return 0, 0, err - } - length, err := strconv.Atoi(submatches[2]) - if err != nil { - return 0, 0, err - } - return offset, length, nil -} diff --git a/registry/client/client_test.go b/registry/client/client_test.go deleted file mode 100644 index 2c1d1cc2..00000000 --- a/registry/client/client_test.go +++ /dev/null @@ -1,440 +0,0 @@ -package client - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - "net/http/httptest" - "sync" - "testing" - - "github.com/docker/distribution/digest" - "github.com/docker/distribution/manifest" - "github.com/docker/distribution/testutil" -) - -type testBlob struct { - digest digest.Digest - contents []byte -} - -func TestRangeHeaderParser(t *testing.T) { - const ( - malformedRangeHeader = "bytes=0-A/C" - emptyRangeHeader = "" - rFirst = 100 - rSecond = 200 - ) - - var ( - wellformedRangeHeader = fmt.Sprintf("bytes=0-%d/%d", rFirst, rSecond) - ) - - if _, _, err := parseRangeHeader(malformedRangeHeader); err == nil { - t.Fatalf("malformedRangeHeader: error expected, got nil") - } - - if _, _, err := parseRangeHeader(emptyRangeHeader); err == nil { - t.Fatalf("emptyRangeHeader: error expected, got nil") - } - - first, second, err := parseRangeHeader(wellformedRangeHeader) - if err != nil { - t.Fatalf("wellformedRangeHeader: unexpected error %v", err) - } - - if first != rFirst || second != rSecond { - t.Fatalf("Range has been parsed unproperly: %d/%d", first, second) - } - -} - -func TestPush(t *testing.T) { - name := "hello/world" - tag := "sometag" - testBlobs := []testBlob{ - { - digest: "tarsum.v2+sha256:12345", - contents: []byte("some contents"), - }, - { - digest: "tarsum.v2+sha256:98765", - contents: []byte("some other contents"), - }, - } - uploadLocations := make([]string, len(testBlobs)) - blobs := make([]manifest.FSLayer, len(testBlobs)) - history := make([]manifest.History, len(testBlobs)) - - for i, blob := range testBlobs { - // TODO(bbland): this is returning the same location for all uploads, - // because we can't know which blob will get which location. - // It's sort of okay because we're using unique digests, but this needs - // to change at some point. - uploadLocations[i] = fmt.Sprintf("/v2/%s/blobs/test-uuid", name) - blobs[i] = manifest.FSLayer{BlobSum: blob.digest} - history[i] = manifest.History{V1Compatibility: blob.digest.String()} - } - - m := &manifest.SignedManifest{ - Manifest: manifest.Manifest{ - Name: name, - Tag: tag, - Architecture: "x86", - FSLayers: blobs, - History: history, - Versioned: manifest.Versioned{ - SchemaVersion: 1, - }, - }, - } - var err error - m.Raw, err = json.Marshal(m) - - blobRequestResponseMappings := make([]testutil.RequestResponseMapping, 2*len(testBlobs)) - for i, blob := range testBlobs { - blobRequestResponseMappings[2*i] = testutil.RequestResponseMapping{ - Request: testutil.Request{ - Method: "POST", - Route: "/v2/" + name + "/blobs/uploads/", - }, - Response: testutil.Response{ - StatusCode: http.StatusAccepted, - Headers: http.Header(map[string][]string{ - "Location": {uploadLocations[i]}, - }), - }, - } - blobRequestResponseMappings[2*i+1] = testutil.RequestResponseMapping{ - Request: testutil.Request{ - Method: "PUT", - Route: uploadLocations[i], - QueryParams: map[string][]string{ - "digest": {blob.digest.String()}, - }, - Body: blob.contents, - }, - Response: testutil.Response{ - StatusCode: http.StatusCreated, - }, - } - } - - handler := testutil.NewHandler(append(blobRequestResponseMappings, testutil.RequestResponseMapping{ - Request: testutil.Request{ - Method: "PUT", - Route: "/v2/" + name + "/manifests/" + tag, - Body: m.Raw, - }, - Response: testutil.Response{ - StatusCode: http.StatusOK, - }, - })) - var server *httptest.Server - - // HACK(stevvooe): Super hack to follow: the request response map approach - // above does not let us correctly format the location header to the - // server url. This handler intercepts and re-writes the location header - // to the server url. - - hack := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w = &headerInterceptingResponseWriter{ResponseWriter: w, serverURL: server.URL} - handler.ServeHTTP(w, r) - }) - - server = httptest.NewServer(hack) - client, err := New(server.URL) - if err != nil { - t.Fatalf("error creating client: %v", err) - } - objectStore := &memoryObjectStore{ - mutex: new(sync.Mutex), - manifestStorage: make(map[string]*manifest.SignedManifest), - layerStorage: make(map[digest.Digest]Layer), - } - - for _, blob := range testBlobs { - l, err := objectStore.Layer(blob.digest) - if err != nil { - t.Fatal(err) - } - - writer, err := l.Writer() - if err != nil { - t.Fatal(err) - } - - writer.SetSize(len(blob.contents)) - writer.Write(blob.contents) - writer.Close() - } - - objectStore.WriteManifest(name, tag, m) - - err = Push(client, objectStore, name, tag) - if err != nil { - t.Fatal(err) - } -} - -func TestPull(t *testing.T) { - name := "hello/world" - tag := "sometag" - testBlobs := []testBlob{ - { - digest: "tarsum.v2+sha256:12345", - contents: []byte("some contents"), - }, - { - digest: "tarsum.v2+sha256:98765", - contents: []byte("some other contents"), - }, - } - blobs := make([]manifest.FSLayer, len(testBlobs)) - history := make([]manifest.History, len(testBlobs)) - - for i, blob := range testBlobs { - blobs[i] = manifest.FSLayer{BlobSum: blob.digest} - history[i] = manifest.History{V1Compatibility: blob.digest.String()} - } - - m := &manifest.SignedManifest{ - Manifest: manifest.Manifest{ - Name: name, - Tag: tag, - Architecture: "x86", - FSLayers: blobs, - History: history, - Versioned: manifest.Versioned{ - SchemaVersion: 1, - }, - }, - } - manifestBytes, err := json.Marshal(m) - - blobRequestResponseMappings := make([]testutil.RequestResponseMapping, len(testBlobs)) - for i, blob := range testBlobs { - blobRequestResponseMappings[i] = testutil.RequestResponseMapping{ - Request: testutil.Request{ - Method: "GET", - Route: "/v2/" + name + "/blobs/" + blob.digest.String(), - }, - Response: testutil.Response{ - StatusCode: http.StatusOK, - Body: blob.contents, - }, - } - } - - handler := testutil.NewHandler(append(blobRequestResponseMappings, testutil.RequestResponseMapping{ - Request: testutil.Request{ - Method: "GET", - Route: "/v2/" + name + "/manifests/" + tag, - }, - Response: testutil.Response{ - StatusCode: http.StatusOK, - Body: manifestBytes, - }, - })) - server := httptest.NewServer(handler) - client, err := New(server.URL) - if err != nil { - t.Fatalf("error creating client: %v", err) - } - objectStore := &memoryObjectStore{ - mutex: new(sync.Mutex), - manifestStorage: make(map[string]*manifest.SignedManifest), - layerStorage: make(map[digest.Digest]Layer), - } - - err = Pull(client, objectStore, name, tag) - if err != nil { - t.Fatal(err) - } - - m, err = objectStore.Manifest(name, tag) - if err != nil { - t.Fatal(err) - } - - mBytes, err := json.Marshal(m) - if err != nil { - t.Fatal(err) - } - - if string(mBytes) != string(manifestBytes) { - t.Fatal("Incorrect manifest") - } - - for _, blob := range testBlobs { - l, err := objectStore.Layer(blob.digest) - if err != nil { - t.Fatal(err) - } - - reader, err := l.Reader() - if err != nil { - t.Fatal(err) - } - defer reader.Close() - - blobBytes, err := ioutil.ReadAll(reader) - if err != nil { - t.Fatal(err) - } - - if string(blobBytes) != string(blob.contents) { - t.Fatal("Incorrect blob") - } - } -} - -func TestPullResume(t *testing.T) { - name := "hello/world" - tag := "sometag" - testBlobs := []testBlob{ - { - digest: "tarsum.v2+sha256:12345", - contents: []byte("some contents"), - }, - { - digest: "tarsum.v2+sha256:98765", - contents: []byte("some other contents"), - }, - } - layers := make([]manifest.FSLayer, len(testBlobs)) - history := make([]manifest.History, len(testBlobs)) - - for i, layer := range testBlobs { - layers[i] = manifest.FSLayer{BlobSum: layer.digest} - history[i] = manifest.History{V1Compatibility: layer.digest.String()} - } - - m := &manifest.Manifest{ - Name: name, - Tag: tag, - Architecture: "x86", - FSLayers: layers, - History: history, - Versioned: manifest.Versioned{ - SchemaVersion: 1, - }, - } - manifestBytes, err := json.Marshal(m) - - layerRequestResponseMappings := make([]testutil.RequestResponseMapping, 2*len(testBlobs)) - for i, blob := range testBlobs { - layerRequestResponseMappings[2*i] = testutil.RequestResponseMapping{ - Request: testutil.Request{ - Method: "GET", - Route: "/v2/" + name + "/blobs/" + blob.digest.String(), - }, - Response: testutil.Response{ - StatusCode: http.StatusOK, - Body: blob.contents[:len(blob.contents)/2], - Headers: http.Header(map[string][]string{ - "Content-Length": {fmt.Sprint(len(blob.contents))}, - }), - }, - } - layerRequestResponseMappings[2*i+1] = testutil.RequestResponseMapping{ - Request: testutil.Request{ - Method: "GET", - Route: "/v2/" + name + "/blobs/" + blob.digest.String(), - }, - Response: testutil.Response{ - StatusCode: http.StatusOK, - Body: blob.contents[len(blob.contents)/2:], - }, - } - } - - for i := 0; i < 3; i++ { - layerRequestResponseMappings = append(layerRequestResponseMappings, testutil.RequestResponseMapping{ - Request: testutil.Request{ - Method: "GET", - Route: "/v2/" + name + "/manifests/" + tag, - }, - Response: testutil.Response{ - StatusCode: http.StatusOK, - Body: manifestBytes, - }, - }) - } - - handler := testutil.NewHandler(layerRequestResponseMappings) - server := httptest.NewServer(handler) - client, err := New(server.URL) - if err != nil { - t.Fatalf("error creating client: %v", err) - } - objectStore := &memoryObjectStore{ - mutex: new(sync.Mutex), - manifestStorage: make(map[string]*manifest.SignedManifest), - layerStorage: make(map[digest.Digest]Layer), - } - - for attempts := 0; attempts < 3; attempts++ { - err = Pull(client, objectStore, name, tag) - if err == nil { - break - } - } - - if err != nil { - t.Fatal(err) - } - - sm, err := objectStore.Manifest(name, tag) - if err != nil { - t.Fatal(err) - } - - mBytes, err := json.Marshal(sm) - if err != nil { - t.Fatal(err) - } - - if string(mBytes) != string(manifestBytes) { - t.Fatal("Incorrect manifest") - } - - for _, blob := range testBlobs { - l, err := objectStore.Layer(blob.digest) - if err != nil { - t.Fatal(err) - } - - reader, err := l.Reader() - if err != nil { - t.Fatal(err) - } - defer reader.Close() - - layerBytes, err := ioutil.ReadAll(reader) - if err != nil { - t.Fatal(err) - } - - if string(layerBytes) != string(blob.contents) { - t.Fatal("Incorrect blob") - } - } -} - -// headerInterceptingResponseWriter is a hacky workaround to re-write the -// location header to have the server url. -type headerInterceptingResponseWriter struct { - http.ResponseWriter - serverURL string -} - -func (hirw *headerInterceptingResponseWriter) WriteHeader(status int) { - location := hirw.Header().Get("Location") - if location != "" { - hirw.Header().Set("Location", hirw.serverURL+location) - } - - hirw.ResponseWriter.WriteHeader(status) -} diff --git a/registry/client/objectstore.go b/registry/client/objectstore.go deleted file mode 100644 index 5969c9d2..00000000 --- a/registry/client/objectstore.go +++ /dev/null @@ -1,239 +0,0 @@ -package client - -import ( - "bytes" - "fmt" - "io" - "sync" - - "github.com/docker/distribution/digest" - "github.com/docker/distribution/manifest" -) - -var ( - // ErrLayerAlreadyExists is returned when attempting to create a layer with - // a tarsum that is already in use. - ErrLayerAlreadyExists = fmt.Errorf("Layer already exists") - - // ErrLayerLocked is returned when attempting to write to a layer which is - // currently being written to. - ErrLayerLocked = fmt.Errorf("Layer locked") -) - -// ObjectStore is an interface which is designed to approximate the docker -// engine storage. This interface is subject to change to conform to the -// future requirements of the engine. -type ObjectStore interface { - // Manifest retrieves the image manifest stored at the given repository name - // and tag - Manifest(name, tag string) (*manifest.SignedManifest, error) - - // WriteManifest stores an image manifest at the given repository name and - // tag - WriteManifest(name, tag string, manifest *manifest.SignedManifest) error - - // Layer returns a handle to a layer for reading and writing - Layer(dgst digest.Digest) (Layer, error) -} - -// Layer is a generic image layer interface. -// A Layer may not be written to if it is already complete. -type Layer interface { - // Reader returns a LayerReader or an error if the layer has not been - // written to or is currently being written to. - Reader() (LayerReader, error) - - // Writer returns a LayerWriter or an error if the layer has been fully - // written to or is currently being written to. - Writer() (LayerWriter, error) - - // Wait blocks until the Layer can be read from. - Wait() error -} - -// LayerReader is a read-only handle to a Layer, which exposes the CurrentSize -// and full Size in addition to implementing the io.ReadCloser interface. -type LayerReader interface { - io.ReadCloser - - // CurrentSize returns the number of bytes written to the underlying Layer - CurrentSize() int - - // Size returns the full size of the underlying Layer - Size() int -} - -// LayerWriter is a write-only handle to a Layer, which exposes the CurrentSize -// and full Size in addition to implementing the io.WriteCloser interface. -// SetSize must be called on this LayerWriter before it can be written to. -type LayerWriter interface { - io.WriteCloser - - // CurrentSize returns the number of bytes written to the underlying Layer - CurrentSize() int - - // Size returns the full size of the underlying Layer - Size() int - - // SetSize sets the full size of the underlying Layer. - // This must be called before any calls to Write - SetSize(int) error -} - -// memoryObjectStore is an in-memory implementation of the ObjectStore interface -type memoryObjectStore struct { - mutex *sync.Mutex - manifestStorage map[string]*manifest.SignedManifest - layerStorage map[digest.Digest]Layer -} - -func (objStore *memoryObjectStore) Manifest(name, tag string) (*manifest.SignedManifest, error) { - objStore.mutex.Lock() - defer objStore.mutex.Unlock() - - manifest, ok := objStore.manifestStorage[name+":"+tag] - if !ok { - return nil, fmt.Errorf("No manifest found with Name: %q, Tag: %q", name, tag) - } - return manifest, nil -} - -func (objStore *memoryObjectStore) WriteManifest(name, tag string, manifest *manifest.SignedManifest) error { - objStore.mutex.Lock() - defer objStore.mutex.Unlock() - - objStore.manifestStorage[name+":"+tag] = manifest - return nil -} - -func (objStore *memoryObjectStore) Layer(dgst digest.Digest) (Layer, error) { - objStore.mutex.Lock() - defer objStore.mutex.Unlock() - - layer, ok := objStore.layerStorage[dgst] - if !ok { - layer = &memoryLayer{cond: sync.NewCond(new(sync.Mutex))} - objStore.layerStorage[dgst] = layer - } - - return layer, nil -} - -type memoryLayer struct { - cond *sync.Cond - contents []byte - expectedSize int - writing bool -} - -func (ml *memoryLayer) Reader() (LayerReader, error) { - ml.cond.L.Lock() - defer ml.cond.L.Unlock() - - if ml.contents == nil { - return nil, fmt.Errorf("Layer has not been written to yet") - } - if ml.writing { - return nil, ErrLayerLocked - } - - return &memoryLayerReader{ml: ml, reader: bytes.NewReader(ml.contents)}, nil -} - -func (ml *memoryLayer) Writer() (LayerWriter, error) { - ml.cond.L.Lock() - defer ml.cond.L.Unlock() - - if ml.contents != nil { - if ml.writing { - return nil, ErrLayerLocked - } - if ml.expectedSize == len(ml.contents) { - return nil, ErrLayerAlreadyExists - } - } else { - ml.contents = make([]byte, 0) - } - - ml.writing = true - return &memoryLayerWriter{ml: ml, buffer: bytes.NewBuffer(ml.contents)}, nil -} - -func (ml *memoryLayer) Wait() error { - ml.cond.L.Lock() - defer ml.cond.L.Unlock() - - if ml.contents == nil { - return fmt.Errorf("No writer to wait on") - } - - for ml.writing { - ml.cond.Wait() - } - - return nil -} - -type memoryLayerReader struct { - ml *memoryLayer - reader *bytes.Reader -} - -func (mlr *memoryLayerReader) Read(p []byte) (int, error) { - return mlr.reader.Read(p) -} - -func (mlr *memoryLayerReader) Close() error { - return nil -} - -func (mlr *memoryLayerReader) CurrentSize() int { - return len(mlr.ml.contents) -} - -func (mlr *memoryLayerReader) Size() int { - return mlr.ml.expectedSize -} - -type memoryLayerWriter struct { - ml *memoryLayer - buffer *bytes.Buffer -} - -func (mlw *memoryLayerWriter) Write(p []byte) (int, error) { - if mlw.ml.expectedSize == 0 { - return 0, fmt.Errorf("Must set size before writing to layer") - } - wrote, err := mlw.buffer.Write(p) - mlw.ml.contents = mlw.buffer.Bytes() - return wrote, err -} - -func (mlw *memoryLayerWriter) Close() error { - mlw.ml.cond.L.Lock() - defer mlw.ml.cond.L.Unlock() - - return mlw.close() -} - -func (mlw *memoryLayerWriter) close() error { - mlw.ml.writing = false - mlw.ml.cond.Broadcast() - return nil -} - -func (mlw *memoryLayerWriter) CurrentSize() int { - return len(mlw.ml.contents) -} - -func (mlw *memoryLayerWriter) Size() int { - return mlw.ml.expectedSize -} - -func (mlw *memoryLayerWriter) SetSize(size int) error { - if !mlw.ml.writing { - return fmt.Errorf("Layer is closed for writing") - } - mlw.ml.expectedSize = size - return nil -} diff --git a/registry/client/pull.go b/registry/client/pull.go deleted file mode 100644 index 385158db..00000000 --- a/registry/client/pull.go +++ /dev/null @@ -1,151 +0,0 @@ -package client - -import ( - "fmt" - "io" - - log "github.com/Sirupsen/logrus" - - "github.com/docker/distribution/manifest" -) - -// simultaneousLayerPullWindow is the size of the parallel layer pull window. -// A layer may not be pulled until the layer preceeding it by the length of the -// pull window has been successfully pulled. -const simultaneousLayerPullWindow = 4 - -// Pull implements a client pull workflow for the image defined by the given -// name and tag pair, using the given ObjectStore for local manifest and layer -// storage -func Pull(c Client, objectStore ObjectStore, name, tag string) error { - manifest, err := c.GetImageManifest(name, tag) - if err != nil { - return err - } - log.WithField("manifest", manifest).Info("Pulled manifest") - - if len(manifest.FSLayers) != len(manifest.History) { - return fmt.Errorf("Length of history not equal to number of layers") - } - if len(manifest.FSLayers) == 0 { - return fmt.Errorf("Image has no layers") - } - - errChans := make([]chan error, len(manifest.FSLayers)) - for i := range manifest.FSLayers { - errChans[i] = make(chan error) - } - - // To avoid leak of goroutines we must notify - // pullLayer goroutines about a cancelation, - // otherwise they will lock forever. - cancelCh := make(chan struct{}) - - // Iterate over each layer in the manifest, simultaneously pulling no more - // than simultaneousLayerPullWindow layers at a time. If an error is - // received from a layer pull, we abort the push. - for i := 0; i < len(manifest.FSLayers)+simultaneousLayerPullWindow; i++ { - dependentLayer := i - simultaneousLayerPullWindow - if dependentLayer >= 0 { - err := <-errChans[dependentLayer] - if err != nil { - log.WithField("error", err).Warn("Pull aborted") - close(cancelCh) - return err - } - } - - if i < len(manifest.FSLayers) { - go func(i int) { - select { - case errChans[i] <- pullLayer(c, objectStore, name, manifest.FSLayers[i]): - case <-cancelCh: // no chance to recv until cancelCh's closed - } - }(i) - } - } - - err = objectStore.WriteManifest(name, tag, manifest) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "manifest": manifest, - }).Warn("Unable to write image manifest") - return err - } - - return nil -} - -func pullLayer(c Client, objectStore ObjectStore, name string, fsLayer manifest.FSLayer) error { - log.WithField("layer", fsLayer).Info("Pulling layer") - - layer, err := objectStore.Layer(fsLayer.BlobSum) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "layer": fsLayer, - }).Warn("Unable to write local layer") - return err - } - - layerWriter, err := layer.Writer() - if err == ErrLayerAlreadyExists { - log.WithField("layer", fsLayer).Info("Layer already exists") - return nil - } - if err == ErrLayerLocked { - log.WithField("layer", fsLayer).Info("Layer download in progress, waiting") - layer.Wait() - return nil - } - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "layer": fsLayer, - }).Warn("Unable to write local layer") - return err - } - defer layerWriter.Close() - - if layerWriter.CurrentSize() > 0 { - log.WithFields(log.Fields{ - "layer": fsLayer, - "currentSize": layerWriter.CurrentSize(), - "size": layerWriter.Size(), - }).Info("Layer partially downloaded, resuming") - } - - layerReader, length, err := c.GetBlob(name, fsLayer.BlobSum, layerWriter.CurrentSize()) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "layer": fsLayer, - }).Warn("Unable to download layer") - return err - } - defer layerReader.Close() - - layerWriter.SetSize(layerWriter.CurrentSize() + length) - - _, err = io.Copy(layerWriter, layerReader) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "layer": fsLayer, - }).Warn("Unable to download layer") - return err - } - if layerWriter.CurrentSize() != layerWriter.Size() { - log.WithFields(log.Fields{ - "size": layerWriter.Size(), - "currentSize": layerWriter.CurrentSize(), - "layer": fsLayer, - }).Warn("Layer invalid size") - return fmt.Errorf( - "Wrote incorrect number of bytes for layer %v. Expected %d, Wrote %d", - fsLayer, layerWriter.Size(), layerWriter.CurrentSize(), - ) - } - return nil -} diff --git a/registry/client/push.go b/registry/client/push.go deleted file mode 100644 index c26bd174..00000000 --- a/registry/client/push.go +++ /dev/null @@ -1,137 +0,0 @@ -package client - -import ( - "fmt" - - log "github.com/Sirupsen/logrus" - "github.com/docker/distribution/manifest" -) - -// simultaneousLayerPushWindow is the size of the parallel layer push window. -// A layer may not be pushed until the layer preceeding it by the length of the -// push window has been successfully pushed. -const simultaneousLayerPushWindow = 4 - -type pushFunction func(fsLayer manifest.FSLayer) error - -// Push implements a client push workflow for the image defined by the given -// name and tag pair, using the given ObjectStore for local manifest and layer -// storage -func Push(c Client, objectStore ObjectStore, name, tag string) error { - manifest, err := objectStore.Manifest(name, tag) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "name": name, - "tag": tag, - }).Info("No image found") - return err - } - - errChans := make([]chan error, len(manifest.FSLayers)) - for i := range manifest.FSLayers { - errChans[i] = make(chan error) - } - - cancelCh := make(chan struct{}) - - // Iterate over each layer in the manifest, simultaneously pushing no more - // than simultaneousLayerPushWindow layers at a time. If an error is - // received from a layer push, we abort the push. - for i := 0; i < len(manifest.FSLayers)+simultaneousLayerPushWindow; i++ { - dependentLayer := i - simultaneousLayerPushWindow - if dependentLayer >= 0 { - err := <-errChans[dependentLayer] - if err != nil { - log.WithField("error", err).Warn("Push aborted") - close(cancelCh) - return err - } - } - - if i < len(manifest.FSLayers) { - go func(i int) { - select { - case errChans[i] <- pushLayer(c, objectStore, name, manifest.FSLayers[i]): - case <-cancelCh: // recv broadcast notification about cancelation - } - }(i) - } - } - - err = c.PutImageManifest(name, tag, manifest) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "manifest": manifest, - }).Warn("Unable to upload manifest") - return err - } - - return nil -} - -func pushLayer(c Client, objectStore ObjectStore, name string, fsLayer manifest.FSLayer) error { - log.WithField("layer", fsLayer).Info("Pushing layer") - - layer, err := objectStore.Layer(fsLayer.BlobSum) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "layer": fsLayer, - }).Warn("Unable to read local layer") - return err - } - - layerReader, err := layer.Reader() - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "layer": fsLayer, - }).Warn("Unable to read local layer") - return err - } - defer layerReader.Close() - - if layerReader.CurrentSize() != layerReader.Size() { - log.WithFields(log.Fields{ - "layer": fsLayer, - "currentSize": layerReader.CurrentSize(), - "size": layerReader.Size(), - }).Warn("Local layer incomplete") - return fmt.Errorf("Local layer incomplete") - } - - length, err := c.BlobLength(name, fsLayer.BlobSum) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "layer": fsLayer, - }).Warn("Unable to check existence of remote layer") - return err - } - if length >= 0 { - log.WithField("layer", fsLayer).Info("Layer already exists") - return nil - } - - location, err := c.InitiateBlobUpload(name) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "layer": fsLayer, - }).Warn("Unable to upload layer") - return err - } - - err = c.UploadBlob(location, layerReader, int(layerReader.CurrentSize()), fsLayer.BlobSum) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "layer": fsLayer, - }).Warn("Unable to upload layer") - return err - } - - return nil -} From b1ba2183ee74a482fe97e925cbc157669ac40d2e Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Thu, 7 May 2015 13:16:52 -0700 Subject: [PATCH 03/26] Add unit tests for auth challenge and endpoint Signed-off-by: Derek McGowan (github: dmcgowan) --- registry/client/authchallenge.go | 2 +- registry/client/authchallenge_test.go | 37 ++++ registry/client/endpoint.go | 2 + registry/client/endpoint_test.go | 259 ++++++++++++++++++++++++++ registry/client/repository.go | 4 +- registry/client/repository_test.go | 16 +- testutil/handler.go | 9 +- 7 files changed, 315 insertions(+), 14 deletions(-) create mode 100644 registry/client/authchallenge_test.go create mode 100644 registry/client/endpoint_test.go diff --git a/registry/client/authchallenge.go b/registry/client/authchallenge.go index 0485f42d..f45704b1 100644 --- a/registry/client/authchallenge.go +++ b/registry/client/authchallenge.go @@ -127,7 +127,7 @@ func expectTokenOrQuoted(s string) (value string, rest string) { p := make([]byte, len(s)-1) j := copy(p, s[:i]) escape := true - for i = i + i; i < len(s); i++ { + for i = i + 1; i < len(s); i++ { b := s[i] switch { case escape: diff --git a/registry/client/authchallenge_test.go b/registry/client/authchallenge_test.go new file mode 100644 index 00000000..bb3016ee --- /dev/null +++ b/registry/client/authchallenge_test.go @@ -0,0 +1,37 @@ +package client + +import ( + "net/http" + "testing" +) + +func TestAuthChallengeParse(t *testing.T) { + header := http.Header{} + header.Add("WWW-Authenticate", `Bearer realm="https://auth.example.com/token",service="registry.example.com",other=fun,slashed="he\"\l\lo"`) + + challenges := parseAuthHeader(header) + if len(challenges) != 1 { + t.Fatalf("Unexpected number of auth challenges: %d, expected 1", len(challenges)) + } + + if expected := "bearer"; challenges[0].Scheme != expected { + t.Fatalf("Unexpected scheme: %s, expected: %s", challenges[0].Scheme, expected) + } + + if expected := "https://auth.example.com/token"; challenges[0].Parameters["realm"] != expected { + t.Fatalf("Unexpected param: %s, expected: %s", challenges[0].Parameters["realm"], expected) + } + + if expected := "registry.example.com"; challenges[0].Parameters["service"] != expected { + t.Fatalf("Unexpected param: %s, expected: %s", challenges[0].Parameters["service"], expected) + } + + if expected := "fun"; challenges[0].Parameters["other"] != expected { + t.Fatalf("Unexpected param: %s, expected: %s", challenges[0].Parameters["other"], expected) + } + + if expected := "he\"llo"; challenges[0].Parameters["slashed"] != expected { + t.Fatalf("Unexpected param: %s, expected: %s", challenges[0].Parameters["slashed"], expected) + } + +} diff --git a/registry/client/endpoint.go b/registry/client/endpoint.go index 83d3d991..9889dc66 100644 --- a/registry/client/endpoint.go +++ b/registry/client/endpoint.go @@ -117,6 +117,8 @@ func (e *RepositoryEndpoint) URLBuilder() (*v2.URLBuilder, error) { // HTTPClient returns a new HTTP client configured for this endpoint func (e *RepositoryEndpoint) HTTPClient(name string) (*http.Client, error) { + // TODO(dmcgowan): create http.Transport + transport := &repositoryTransport{ Header: e.Header, } diff --git a/registry/client/endpoint_test.go b/registry/client/endpoint_test.go new file mode 100644 index 00000000..42bdc357 --- /dev/null +++ b/registry/client/endpoint_test.go @@ -0,0 +1,259 @@ +package client + +import ( + "encoding/base64" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/docker/distribution/testutil" +) + +type testAuthenticationWrapper struct { + headers http.Header + authCheck func(string) bool + next http.Handler +} + +func (w *testAuthenticationWrapper) ServeHTTP(rw http.ResponseWriter, r *http.Request) { + auth := r.Header.Get("Authorization") + if auth == "" || !w.authCheck(auth) { + h := rw.Header() + for k, values := range w.headers { + h[k] = values + } + rw.WriteHeader(http.StatusUnauthorized) + return + } + w.next.ServeHTTP(rw, r) +} + +func testServerWithAuth(rrm testutil.RequestResponseMap, authenticate string, authCheck func(string) bool) (*RepositoryEndpoint, func()) { + h := testutil.NewHandler(rrm) + wrapper := &testAuthenticationWrapper{ + + headers: http.Header(map[string][]string{ + "Docker-Distribution-API-Version": {"registry/2.0"}, + "WWW-Authenticate": {authenticate}, + }), + authCheck: authCheck, + next: h, + } + + s := httptest.NewServer(wrapper) + e := RepositoryEndpoint{Endpoint: s.URL, Mirror: false} + return &e, s.Close +} + +type testCredentialStore struct { + username string + password string +} + +func (tcs *testCredentialStore) Basic(*url.URL) (string, string) { + return tcs.username, tcs.password +} + +func TestEndpointAuthorizeToken(t *testing.T) { + service := "localhost.localdomain" + repo1 := "some/registry" + repo2 := "other/registry" + scope1 := fmt.Sprintf("repository:%s:pull,push", repo1) + scope2 := fmt.Sprintf("repository:%s:pull,push", repo2) + + tokenMap := testutil.RequestResponseMap([]testutil.RequestResponseMapping{ + { + Request: testutil.Request{ + Method: "GET", + Route: fmt.Sprintf("/token?scope=%s&service=%s", url.QueryEscape(scope1), service), + }, + Response: testutil.Response{ + StatusCode: http.StatusOK, + Body: []byte(`{"token":"statictoken"}`), + }, + }, + { + Request: testutil.Request{ + Method: "GET", + Route: fmt.Sprintf("/token?scope=%s&service=%s", url.QueryEscape(scope2), service), + }, + Response: testutil.Response{ + StatusCode: http.StatusOK, + Body: []byte(`{"token":"badtoken"}`), + }, + }, + }) + te, tc := testServer(tokenMap) + defer tc() + + m := testutil.RequestResponseMap([]testutil.RequestResponseMapping{ + { + Request: testutil.Request{ + Method: "GET", + Route: "/hello", + }, + Response: testutil.Response{ + StatusCode: http.StatusAccepted, + }, + }, + }) + + authenicate := fmt.Sprintf("Bearer realm=%q,service=%q", te.Endpoint+"/token", service) + validCheck := func(a string) bool { + return a == "Bearer statictoken" + } + e, c := testServerWithAuth(m, authenicate, validCheck) + defer c() + + client, err := e.HTTPClient(repo1) + if err != nil { + t.Fatalf("Error creating http client: %s", err) + } + + req, _ := http.NewRequest("GET", e.Endpoint+"/hello", nil) + resp, err := client.Do(req) + if err != nil { + t.Fatalf("Error sending get request: %s", err) + } + + if resp.StatusCode != http.StatusAccepted { + t.Fatalf("Unexpected status code: %d, expected %d", resp.StatusCode, http.StatusAccepted) + } + + badCheck := func(a string) bool { + return a == "Bearer statictoken" + } + e2, c2 := testServerWithAuth(m, authenicate, badCheck) + defer c2() + + client2, err := e2.HTTPClient(repo2) + if err != nil { + t.Fatalf("Error creating http client: %s", err) + } + + req, _ = http.NewRequest("GET", e.Endpoint+"/hello", nil) + resp, err = client2.Do(req) + if err != nil { + t.Fatalf("Error sending get request: %s", err) + } + + if resp.StatusCode != http.StatusUnauthorized { + t.Fatalf("Unexpected status code: %d, expected %d", resp.StatusCode, http.StatusUnauthorized) + } +} + +func basicAuth(username, password string) string { + auth := username + ":" + password + return base64.StdEncoding.EncodeToString([]byte(auth)) +} + +func TestEndpointAuthorizeTokenBasic(t *testing.T) { + service := "localhost.localdomain" + repo := "some/fun/registry" + scope := fmt.Sprintf("repository:%s:pull,push", repo) + username := "tokenuser" + password := "superSecretPa$$word" + + tokenMap := testutil.RequestResponseMap([]testutil.RequestResponseMapping{ + { + Request: testutil.Request{ + Method: "GET", + Route: fmt.Sprintf("/token?account=%s&scope=%s&service=%s", username, url.QueryEscape(scope), service), + }, + Response: testutil.Response{ + StatusCode: http.StatusOK, + Body: []byte(`{"token":"statictoken"}`), + }, + }, + }) + + authenicate1 := fmt.Sprintf("Basic realm=localhost") + basicCheck := func(a string) bool { + return a == fmt.Sprintf("Basic %s", basicAuth(username, password)) + } + te, tc := testServerWithAuth(tokenMap, authenicate1, basicCheck) + defer tc() + + m := testutil.RequestResponseMap([]testutil.RequestResponseMapping{ + { + Request: testutil.Request{ + Method: "GET", + Route: "/hello", + }, + Response: testutil.Response{ + StatusCode: http.StatusAccepted, + }, + }, + }) + + authenicate2 := fmt.Sprintf("Bearer realm=%q,service=%q", te.Endpoint+"/token", service) + bearerCheck := func(a string) bool { + return a == "Bearer statictoken" + } + e, c := testServerWithAuth(m, authenicate2, bearerCheck) + defer c() + + e.Credentials = &testCredentialStore{ + username: username, + password: password, + } + + client, err := e.HTTPClient(repo) + if err != nil { + t.Fatalf("Error creating http client: %s", err) + } + + req, _ := http.NewRequest("GET", e.Endpoint+"/hello", nil) + resp, err := client.Do(req) + if err != nil { + t.Fatalf("Error sending get request: %s", err) + } + + if resp.StatusCode != http.StatusAccepted { + t.Fatalf("Unexpected status code: %d, expected %d", resp.StatusCode, http.StatusAccepted) + } +} + +func TestEndpointAuthorizeBasic(t *testing.T) { + m := testutil.RequestResponseMap([]testutil.RequestResponseMapping{ + { + Request: testutil.Request{ + Method: "GET", + Route: "/hello", + }, + Response: testutil.Response{ + StatusCode: http.StatusAccepted, + }, + }, + }) + + username := "user1" + password := "funSecretPa$$word" + authenicate := fmt.Sprintf("Basic realm=localhost") + validCheck := func(a string) bool { + return a == fmt.Sprintf("Basic %s", basicAuth(username, password)) + } + e, c := testServerWithAuth(m, authenicate, validCheck) + defer c() + e.Credentials = &testCredentialStore{ + username: username, + password: password, + } + + client, err := e.HTTPClient("test/repo/basic") + if err != nil { + t.Fatalf("Error creating http client: %s", err) + } + + req, _ := http.NewRequest("GET", e.Endpoint+"/hello", nil) + resp, err := client.Do(req) + if err != nil { + t.Fatalf("Error sending get request: %s", err) + } + + if resp.StatusCode != http.StatusAccepted { + t.Fatalf("Unexpected status code: %d, expected %d", resp.StatusCode, http.StatusAccepted) + } +} diff --git a/registry/client/repository.go b/registry/client/repository.go index a96390fa..578c3fca 100644 --- a/registry/client/repository.go +++ b/registry/client/repository.go @@ -25,8 +25,8 @@ import ( "golang.org/x/net/context" ) -// NewRepositoryClient creates a new Repository for the given repository name and endpoint -func NewRepositoryClient(ctx context.Context, name string, endpoint *RepositoryEndpoint) (distribution.Repository, error) { +// NewRepository creates a new Repository for the given repository name and endpoint +func NewRepository(ctx context.Context, name string, endpoint *RepositoryEndpoint) (distribution.Repository, error) { if err := v2.ValidateRespositoryName(name); err != nil { return nil, err } diff --git a/registry/client/repository_test.go b/registry/client/repository_test.go index 67138db6..b96c52e5 100644 --- a/registry/client/repository_test.go +++ b/registry/client/repository_test.go @@ -97,7 +97,7 @@ func TestLayerFetch(t *testing.T) { e, c := testServer(m) defer c() - r, err := NewRepositoryClient(context.Background(), "test.example.com/repo1", e) + r, err := NewRepository(context.Background(), "test.example.com/repo1", e) if err != nil { t.Fatal(err) } @@ -127,7 +127,7 @@ func TestLayerExists(t *testing.T) { e, c := testServer(m) defer c() - r, err := NewRepositoryClient(context.Background(), "test.example.com/repo1", e) + r, err := NewRepository(context.Background(), "test.example.com/repo1", e) if err != nil { t.Fatal(err) } @@ -227,7 +227,7 @@ func TestLayerUploadChunked(t *testing.T) { e, c := testServer(m) defer c() - r, err := NewRepositoryClient(context.Background(), repo, e) + r, err := NewRepository(context.Background(), repo, e) if err != nil { t.Fatal(err) } @@ -334,7 +334,7 @@ func TestLayerUploadMonolithic(t *testing.T) { e, c := testServer(m) defer c() - r, err := NewRepositoryClient(context.Background(), repo, e) + r, err := NewRepository(context.Background(), repo, e) if err != nil { t.Fatal(err) } @@ -475,7 +475,7 @@ func TestManifestFetch(t *testing.T) { e, c := testServer(m) defer c() - r, err := NewRepositoryClient(context.Background(), repo, e) + r, err := NewRepository(context.Background(), repo, e) if err != nil { t.Fatal(err) } @@ -508,7 +508,7 @@ func TestManifestFetchByTag(t *testing.T) { e, c := testServer(m) defer c() - r, err := NewRepositoryClient(context.Background(), repo, e) + r, err := NewRepository(context.Background(), repo, e) if err != nil { t.Fatal(err) } @@ -553,7 +553,7 @@ func TestManifestDelete(t *testing.T) { e, c := testServer(m) defer c() - r, err := NewRepositoryClient(context.Background(), repo, e) + r, err := NewRepository(context.Background(), repo, e) if err != nil { t.Fatal(err) } @@ -591,7 +591,7 @@ func TestManifestPut(t *testing.T) { e, c := testServer(m) defer c() - r, err := NewRepositoryClient(context.Background(), repo, e) + r, err := NewRepository(context.Background(), repo, e) if err != nil { t.Fatal(err) } diff --git a/testutil/handler.go b/testutil/handler.go index fa118cd1..10850e24 100644 --- a/testutil/handler.go +++ b/testutil/handler.go @@ -6,6 +6,7 @@ import ( "io" "io/ioutil" "net/http" + "net/url" "sort" "strings" ) @@ -40,16 +41,18 @@ type Request struct { func (r Request) String() string { queryString := "" if len(r.QueryParams) > 0 { - queryString = "?" keys := make([]string, 0, len(r.QueryParams)) + queryParts := make([]string, 0, len(r.QueryParams)) for k := range r.QueryParams { keys = append(keys, k) } sort.Strings(keys) for _, k := range keys { - queryString += strings.Join(r.QueryParams[k], "&") + "&" + for _, val := range r.QueryParams[k] { + queryParts = append(queryParts, fmt.Sprintf("%s=%s", k, url.QueryEscape(val))) + } } - queryString = queryString[:len(queryString)-1] + queryString = "?" + strings.Join(queryParts, "&") } return fmt.Sprintf("%s %s%s\n%s", r.Method, r.Route, queryString, r.Body) } From 6f9fbf99a94ffd2e35860fa2273b743775485270 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Thu, 7 May 2015 16:11:04 -0700 Subject: [PATCH 04/26] Split layer and upload from repository Layer upload moved to its own file with its own unit tests Signed-off-by: Derek McGowan (github: dmcgowan) --- registry/client/errors.go | 6 +- registry/client/layer.go | 178 +++++++++++++++ registry/client/layer_upload.go | 164 ++++++++++++++ registry/client/layer_upload_test.go | 223 ++++++++++++++++++ registry/client/repository.go | 326 +-------------------------- 5 files changed, 569 insertions(+), 328 deletions(-) create mode 100644 registry/client/layer.go create mode 100644 registry/client/layer_upload.go create mode 100644 registry/client/layer_upload_test.go diff --git a/registry/client/errors.go b/registry/client/errors.go index 4ef2cc23..e02b0f73 100644 --- a/registry/client/errors.go +++ b/registry/client/errors.go @@ -1,7 +1,6 @@ package client import ( - "bytes" "encoding/json" "fmt" "io/ioutil" @@ -104,9 +103,8 @@ func parseHTTPErrorResponse(response *http.Response) error { if err != nil { return err } - decoder := json.NewDecoder(bytes.NewReader(body)) - err = decoder.Decode(&errors) - if err != nil { + + if err := json.Unmarshal(body, &errors); err != nil { return &UnexpectedHTTPResponseError{ ParseErr: err, Response: body, diff --git a/registry/client/layer.go b/registry/client/layer.go new file mode 100644 index 00000000..f61a9034 --- /dev/null +++ b/registry/client/layer.go @@ -0,0 +1,178 @@ +package client + +import ( + "bufio" + "bytes" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "time" + + "github.com/docker/distribution/context" + "github.com/docker/distribution/digest" +) + +type httpLayer struct { + *layers + + size int64 + digest digest.Digest + createdAt time.Time + + rc io.ReadCloser // remote read closer + brd *bufio.Reader // internal buffered io + offset int64 + err error +} + +func (hl *httpLayer) CreatedAt() time.Time { + return hl.createdAt +} + +func (hl *httpLayer) Digest() digest.Digest { + return hl.digest +} + +func (hl *httpLayer) Read(p []byte) (n int, err error) { + if hl.err != nil { + return 0, hl.err + } + + rd, err := hl.reader() + if err != nil { + return 0, err + } + + n, err = rd.Read(p) + hl.offset += int64(n) + + // Simulate io.EOR error if we reach filesize. + if err == nil && hl.offset >= hl.size { + err = io.EOF + } + + return n, err +} + +func (hl *httpLayer) Seek(offset int64, whence int) (int64, error) { + if hl.err != nil { + return 0, hl.err + } + + var err error + newOffset := hl.offset + + switch whence { + case os.SEEK_CUR: + newOffset += int64(offset) + case os.SEEK_END: + newOffset = hl.size + int64(offset) + case os.SEEK_SET: + newOffset = int64(offset) + } + + if newOffset < 0 { + err = fmt.Errorf("cannot seek to negative position") + } else { + if hl.offset != newOffset { + hl.reset() + } + + // No problems, set the offset. + hl.offset = newOffset + } + + return hl.offset, err +} + +func (hl *httpLayer) Close() error { + if hl.err != nil { + return hl.err + } + + // close and release reader chain + if hl.rc != nil { + hl.rc.Close() + } + + hl.rc = nil + hl.brd = nil + + hl.err = fmt.Errorf("httpLayer: closed") + + return nil +} + +func (hl *httpLayer) reset() { + if hl.err != nil { + return + } + if hl.rc != nil { + hl.rc.Close() + hl.rc = nil + } +} + +func (hl *httpLayer) reader() (io.Reader, error) { + if hl.err != nil { + return nil, hl.err + } + + if hl.rc != nil { + return hl.brd, nil + } + + // If the offset is great than or equal to size, return a empty, noop reader. + if hl.offset >= hl.size { + return ioutil.NopCloser(bytes.NewReader([]byte{})), nil + } + + blobURL, err := hl.ub.BuildBlobURL(hl.name, hl.digest) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("GET", blobURL, nil) + if err != nil { + return nil, err + } + + if hl.offset > 0 { + // TODO(stevvooe): Get this working correctly. + + // If we are at different offset, issue a range request from there. + req.Header.Add("Range", fmt.Sprintf("1-")) + context.GetLogger(hl.context).Infof("Range: %s", req.Header.Get("Range")) + } + + resp, err := hl.client.Do(req) + if err != nil { + return nil, err + } + + switch { + case resp.StatusCode == 200: + hl.rc = resp.Body + default: + defer resp.Body.Close() + return nil, fmt.Errorf("unexpected status resolving reader: %v", resp.Status) + } + + if hl.brd == nil { + hl.brd = bufio.NewReader(hl.rc) + } else { + hl.brd.Reset(hl.rc) + } + + return hl.brd, nil +} + +func (hl *httpLayer) Length() int64 { + return hl.size +} + +func (hl *httpLayer) Handler(r *http.Request) (http.Handler, error) { + panic("Not implemented") +} diff --git a/registry/client/layer_upload.go b/registry/client/layer_upload.go new file mode 100644 index 00000000..ce0794c2 --- /dev/null +++ b/registry/client/layer_upload.go @@ -0,0 +1,164 @@ +package client + +import ( + "bytes" + "errors" + "fmt" + "io" + "net/http" + "os" + "time" + + "github.com/docker/distribution" + "github.com/docker/distribution/digest" +) + +type httpLayerUpload struct { + repo distribution.Repository + client *http.Client + + uuid string + startedAt time.Time + + location string // always the last value of the location header. + offset int64 + closed bool +} + +func (hlu *httpLayerUpload) handleErrorResponse(resp *http.Response) error { + switch { + case resp.StatusCode == http.StatusNotFound: + return &BlobUploadNotFoundError{Location: hlu.location} + case resp.StatusCode >= 400 && resp.StatusCode < 500: + return parseHTTPErrorResponse(resp) + default: + return &UnexpectedHTTPStatusError{Status: resp.Status} + } +} + +func (hlu *httpLayerUpload) ReadFrom(r io.Reader) (n int64, err error) { + req, err := http.NewRequest("PATCH", hlu.location, r) + if err != nil { + return 0, err + } + defer req.Body.Close() + + resp, err := hlu.client.Do(req) + if err != nil { + return 0, err + } + + if resp.StatusCode != http.StatusAccepted { + return 0, hlu.handleErrorResponse(resp) + } + + // TODO(dmcgowan): Validate headers + hlu.uuid = resp.Header.Get("Docker-Upload-UUID") + hlu.location, err = sanitizeLocation(resp.Header.Get("Location"), hlu.location) + if err != nil { + return 0, err + } + rng := resp.Header.Get("Range") + var start, end int64 + if n, err := fmt.Sscanf(rng, "%d-%d", &start, &end); err != nil { + return 0, err + } else if n != 2 || end < start { + return 0, fmt.Errorf("bad range format: %s", rng) + } + + return (end - start + 1), nil + +} + +func (hlu *httpLayerUpload) Write(p []byte) (n int, err error) { + req, err := http.NewRequest("PATCH", hlu.location, bytes.NewReader(p)) + if err != nil { + return 0, err + } + req.Header.Set("Content-Range", fmt.Sprintf("%d-%d", hlu.offset, hlu.offset+int64(len(p)-1))) + req.Header.Set("Content-Length", fmt.Sprintf("%d", len(p))) + req.Header.Set("Content-Type", "application/octet-stream") + + resp, err := hlu.client.Do(req) + if err != nil { + return 0, err + } + + if resp.StatusCode != http.StatusAccepted { + return 0, hlu.handleErrorResponse(resp) + } + + // TODO(dmcgowan): Validate headers + hlu.uuid = resp.Header.Get("Docker-Upload-UUID") + hlu.location, err = sanitizeLocation(resp.Header.Get("Location"), hlu.location) + if err != nil { + return 0, err + } + rng := resp.Header.Get("Range") + var start, end int + if n, err := fmt.Sscanf(rng, "%d-%d", &start, &end); err != nil { + return 0, err + } else if n != 2 || end < start { + return 0, fmt.Errorf("bad range format: %s", rng) + } + + return (end - start + 1), nil + +} + +func (hlu *httpLayerUpload) Seek(offset int64, whence int) (int64, error) { + newOffset := hlu.offset + + switch whence { + case os.SEEK_CUR: + newOffset += int64(offset) + case os.SEEK_END: + return newOffset, errors.New("Cannot seek from end on incomplete upload") + case os.SEEK_SET: + newOffset = int64(offset) + } + + hlu.offset = newOffset + + return hlu.offset, nil +} + +func (hlu *httpLayerUpload) UUID() string { + return hlu.uuid +} + +func (hlu *httpLayerUpload) StartedAt() time.Time { + return hlu.startedAt +} + +func (hlu *httpLayerUpload) Finish(digest digest.Digest) (distribution.Layer, error) { + // TODO(dmcgowan): Check if already finished, if so just fetch + req, err := http.NewRequest("PUT", hlu.location, nil) + if err != nil { + return nil, err + } + + values := req.URL.Query() + values.Set("digest", digest.String()) + req.URL.RawQuery = values.Encode() + + resp, err := hlu.client.Do(req) + if err != nil { + return nil, err + } + + if resp.StatusCode != http.StatusCreated { + return nil, hlu.handleErrorResponse(resp) + } + + return hlu.repo.Layers().Fetch(digest) +} + +func (hlu *httpLayerUpload) Cancel() error { + panic("not implemented") +} + +func (hlu *httpLayerUpload) Close() error { + hlu.closed = true + return nil +} diff --git a/registry/client/layer_upload_test.go b/registry/client/layer_upload_test.go new file mode 100644 index 00000000..1aa5cf1e --- /dev/null +++ b/registry/client/layer_upload_test.go @@ -0,0 +1,223 @@ +package client + +import ( + "bytes" + "fmt" + "net/http" + "testing" + + "github.com/docker/distribution" + "github.com/docker/distribution/registry/api/v2" + "github.com/docker/distribution/testutil" +) + +// Test implements distribution.LayerUpload +var _ distribution.LayerUpload = &httpLayerUpload{} + +func TestUploadReadFrom(t *testing.T) { + _, b := newRandomBlob(64) + repo := "test/upload/readfrom" + locationPath := fmt.Sprintf("/v2/%s/uploads/testid", repo) + + m := testutil.RequestResponseMap([]testutil.RequestResponseMapping{ + { + Request: testutil.Request{ + Method: "GET", + Route: "/v2/", + }, + Response: testutil.Response{ + StatusCode: http.StatusOK, + Headers: http.Header(map[string][]string{ + "Docker-Distribution-API-Version": {"registry/2.0"}, + }), + }, + }, + // Test Valid case + { + Request: testutil.Request{ + Method: "PATCH", + Route: locationPath, + Body: b, + }, + Response: testutil.Response{ + StatusCode: http.StatusAccepted, + Headers: http.Header(map[string][]string{ + "Docker-Upload-UUID": {"46603072-7a1b-4b41-98f9-fd8a7da89f9b"}, + "Location": {locationPath}, + "Range": {"0-63"}, + }), + }, + }, + // Test invalid range + { + Request: testutil.Request{ + Method: "PATCH", + Route: locationPath, + Body: b, + }, + Response: testutil.Response{ + StatusCode: http.StatusAccepted, + Headers: http.Header(map[string][]string{ + "Docker-Upload-UUID": {"46603072-7a1b-4b41-98f9-fd8a7da89f9b"}, + "Location": {locationPath}, + "Range": {""}, + }), + }, + }, + // Test 404 + { + Request: testutil.Request{ + Method: "PATCH", + Route: locationPath, + Body: b, + }, + Response: testutil.Response{ + StatusCode: http.StatusNotFound, + }, + }, + // Test 400 valid json + { + Request: testutil.Request{ + Method: "PATCH", + Route: locationPath, + Body: b, + }, + Response: testutil.Response{ + StatusCode: http.StatusBadRequest, + Body: []byte(` + { + "errors": [ + { + "code": "BLOB_UPLOAD_INVALID", + "message": "invalid upload identifier", + "detail": "more detail" + } + ] + }`), + }, + }, + // Test 400 invalid json + { + Request: testutil.Request{ + Method: "PATCH", + Route: locationPath, + Body: b, + }, + Response: testutil.Response{ + StatusCode: http.StatusBadRequest, + Body: []byte("something bad happened"), + }, + }, + // Test 500 + { + Request: testutil.Request{ + Method: "PATCH", + Route: locationPath, + Body: b, + }, + Response: testutil.Response{ + StatusCode: http.StatusInternalServerError, + }, + }, + }) + + e, c := testServer(m) + defer c() + + client, err := e.HTTPClient(repo) + if err != nil { + t.Fatalf("Error creating client: %s", err) + } + layerUpload := &httpLayerUpload{ + client: client, + } + + // Valid case + layerUpload.location = e.Endpoint + locationPath + n, err := layerUpload.ReadFrom(bytes.NewReader(b)) + if err != nil { + t.Fatalf("Error calling ReadFrom: %s", err) + } + if n != 64 { + t.Fatalf("Wrong length returned from ReadFrom: %d, expected 64", n) + } + + // Bad range + layerUpload.location = e.Endpoint + locationPath + _, err = layerUpload.ReadFrom(bytes.NewReader(b)) + if err == nil { + t.Fatalf("Expected error when bad range received") + } + + // 404 + layerUpload.location = e.Endpoint + locationPath + _, err = layerUpload.ReadFrom(bytes.NewReader(b)) + if err == nil { + t.Fatalf("Expected error when not found") + } + if blobErr, ok := err.(*BlobUploadNotFoundError); !ok { + t.Fatalf("Wrong error type %T: %s", err, err) + } else if expected := e.Endpoint + locationPath; blobErr.Location != expected { + t.Fatalf("Unexpected location: %s, expected %s", blobErr.Location, expected) + } + + // 400 valid json + layerUpload.location = e.Endpoint + locationPath + _, err = layerUpload.ReadFrom(bytes.NewReader(b)) + if err == nil { + t.Fatalf("Expected error when not found") + } + if uploadErr, ok := err.(*v2.Errors); !ok { + t.Fatalf("Wrong error type %T: %s", err, err) + } else if len(uploadErr.Errors) != 1 { + t.Fatalf("Unexpected number of errors: %d, expected 1", len(uploadErr.Errors)) + } else { + v2Err := uploadErr.Errors[0] + if v2Err.Code != v2.ErrorCodeBlobUploadInvalid { + t.Fatalf("Unexpected error code: %s, expected %s", v2Err.Code.String(), v2.ErrorCodeBlobUploadInvalid.String()) + } + if expected := "invalid upload identifier"; v2Err.Message != expected { + t.Fatalf("Unexpected error message: %s, expected %s", v2Err.Message, expected) + } + if expected := "more detail"; v2Err.Detail.(string) != expected { + t.Fatalf("Unexpected error message: %s, expected %s", v2Err.Detail.(string), expected) + } + } + + // 400 invalid json + layerUpload.location = e.Endpoint + locationPath + _, err = layerUpload.ReadFrom(bytes.NewReader(b)) + if err == nil { + t.Fatalf("Expected error when not found") + } + if uploadErr, ok := err.(*UnexpectedHTTPResponseError); !ok { + t.Fatalf("Wrong error type %T: %s", err, err) + } else { + respStr := string(uploadErr.Response) + if expected := "something bad happened"; respStr != expected { + t.Fatalf("Unexpected response string: %s, expected: %s", respStr, expected) + } + } + + // 500 + layerUpload.location = e.Endpoint + locationPath + _, err = layerUpload.ReadFrom(bytes.NewReader(b)) + if err == nil { + t.Fatalf("Expected error when not found") + } + if uploadErr, ok := err.(*UnexpectedHTTPStatusError); !ok { + t.Fatalf("Wrong error type %T: %s", err, err) + } else if expected := "500 " + http.StatusText(http.StatusInternalServerError); uploadErr.Status != expected { + t.Fatalf("Unexpected response status: %s, expected %s", uploadErr.Status, expected) + } +} + +//repo distribution.Repository +//client *http.Client + +//uuid string +//startedAt time.Time + +//location string // always the last value of the location header. +//offset int64 +//closed bool diff --git a/registry/client/repository.go b/registry/client/repository.go index 578c3fca..22a02373 100644 --- a/registry/client/repository.go +++ b/registry/client/repository.go @@ -1,21 +1,14 @@ package client import ( - "bufio" "bytes" "encoding/json" - "errors" "fmt" - "io" - "io/ioutil" "net/http" "net/url" - "os" "strconv" "time" - ctxu "github.com/docker/distribution/context" - "github.com/docker/distribution/manifest" "github.com/docker/distribution/digest" @@ -276,7 +269,8 @@ func (ls *layers) Upload() (distribution.LayerUpload, error) { } return &httpLayerUpload{ - layers: ls, + repo: ls.repository, + client: ls.client, uuid: uuid, startedAt: time.Now(), location: location, @@ -339,319 +333,3 @@ func (ls *layers) fetchLayer(dgst digest.Digest) (distribution.Layer, error) { return nil, &UnexpectedHTTPStatusError{Status: resp.Status} } } - -type httpLayer struct { - *layers - - size int64 - digest digest.Digest - createdAt time.Time - - rc io.ReadCloser // remote read closer - brd *bufio.Reader // internal buffered io - offset int64 - err error -} - -func (hl *httpLayer) CreatedAt() time.Time { - return hl.createdAt -} - -func (hl *httpLayer) Digest() digest.Digest { - return hl.digest -} - -func (hl *httpLayer) Read(p []byte) (n int, err error) { - if hl.err != nil { - return 0, hl.err - } - - rd, err := hl.reader() - if err != nil { - return 0, err - } - - n, err = rd.Read(p) - hl.offset += int64(n) - - // Simulate io.EOR error if we reach filesize. - if err == nil && hl.offset >= hl.size { - err = io.EOF - } - - return n, err -} - -func (hl *httpLayer) Seek(offset int64, whence int) (int64, error) { - if hl.err != nil { - return 0, hl.err - } - - var err error - newOffset := hl.offset - - switch whence { - case os.SEEK_CUR: - newOffset += int64(offset) - case os.SEEK_END: - newOffset = hl.size + int64(offset) - case os.SEEK_SET: - newOffset = int64(offset) - } - - if newOffset < 0 { - err = fmt.Errorf("cannot seek to negative position") - } else { - if hl.offset != newOffset { - hl.reset() - } - - // No problems, set the offset. - hl.offset = newOffset - } - - return hl.offset, err -} - -func (hl *httpLayer) Close() error { - if hl.err != nil { - return hl.err - } - - // close and release reader chain - if hl.rc != nil { - hl.rc.Close() - } - - hl.rc = nil - hl.brd = nil - - hl.err = fmt.Errorf("httpLayer: closed") - - return nil -} - -func (hl *httpLayer) reset() { - if hl.err != nil { - return - } - if hl.rc != nil { - hl.rc.Close() - hl.rc = nil - } -} - -func (hl *httpLayer) reader() (io.Reader, error) { - if hl.err != nil { - return nil, hl.err - } - - if hl.rc != nil { - return hl.brd, nil - } - - // If the offset is great than or equal to size, return a empty, noop reader. - if hl.offset >= hl.size { - return ioutil.NopCloser(bytes.NewReader([]byte{})), nil - } - - blobURL, err := hl.ub.BuildBlobURL(hl.name, hl.digest) - if err != nil { - return nil, err - } - - req, err := http.NewRequest("GET", blobURL, nil) - if err != nil { - return nil, err - } - - if hl.offset > 0 { - // TODO(stevvooe): Get this working correctly. - - // If we are at different offset, issue a range request from there. - req.Header.Add("Range", fmt.Sprintf("1-")) - ctxu.GetLogger(hl.context).Infof("Range: %s", req.Header.Get("Range")) - } - - resp, err := hl.client.Do(req) - if err != nil { - return nil, err - } - - switch { - case resp.StatusCode == 200: - hl.rc = resp.Body - default: - defer resp.Body.Close() - return nil, fmt.Errorf("unexpected status resolving reader: %v", resp.Status) - } - - if hl.brd == nil { - hl.brd = bufio.NewReader(hl.rc) - } else { - hl.brd.Reset(hl.rc) - } - - return hl.brd, nil -} - -func (hl *httpLayer) Length() int64 { - return hl.size -} - -func (hl *httpLayer) Handler(r *http.Request) (http.Handler, error) { - panic("Not implemented") -} - -type httpLayerUpload struct { - *layers - - uuid string - startedAt time.Time - - location string // always the last value of the location header. - offset int64 - closed bool -} - -var _ distribution.LayerUpload = &httpLayerUpload{} - -func (hlu *httpLayerUpload) ReadFrom(r io.Reader) (n int64, err error) { - req, err := http.NewRequest("PATCH", hlu.location, r) - if err != nil { - return 0, err - } - defer req.Body.Close() - - resp, err := hlu.client.Do(req) - if err != nil { - return 0, err - } - - switch { - case resp.StatusCode == http.StatusAccepted: - // TODO(dmcgowan): Validate headers - hlu.uuid = resp.Header.Get("Docker-Upload-UUID") - hlu.location, err = sanitizeLocation(resp.Header.Get("Location"), hlu.location) - if err != nil { - return 0, err - } - rng := resp.Header.Get("Range") - var start, end int64 - if n, err := fmt.Sscanf(rng, "%d-%d", &start, &end); err != nil { - return 0, err - } else if n != 2 || end < start { - return 0, fmt.Errorf("bad range format: %s", rng) - } - - return (end - start + 1), nil - case resp.StatusCode == http.StatusNotFound: - return 0, &BlobUploadNotFoundError{Location: hlu.location} - case resp.StatusCode >= 400 && resp.StatusCode < 500: - return 0, parseHTTPErrorResponse(resp) - default: - return 0, &UnexpectedHTTPStatusError{Status: resp.Status} - } -} - -func (hlu *httpLayerUpload) Write(p []byte) (n int, err error) { - req, err := http.NewRequest("PATCH", hlu.location, bytes.NewReader(p)) - if err != nil { - return 0, err - } - req.Header.Set("Content-Range", fmt.Sprintf("%d-%d", hlu.offset, hlu.offset+int64(len(p)-1))) - req.Header.Set("Content-Length", fmt.Sprintf("%d", len(p))) - req.Header.Set("Content-Type", "application/octet-stream") - - resp, err := hlu.client.Do(req) - if err != nil { - return 0, err - } - - switch { - case resp.StatusCode == http.StatusAccepted: - // TODO(dmcgowan): Validate headers - hlu.uuid = resp.Header.Get("Docker-Upload-UUID") - hlu.location, err = sanitizeLocation(resp.Header.Get("Location"), hlu.location) - if err != nil { - return 0, err - } - rng := resp.Header.Get("Range") - var start, end int - if n, err := fmt.Sscanf(rng, "%d-%d", &start, &end); err != nil { - return 0, err - } else if n != 2 || end < start { - return 0, fmt.Errorf("bad range format: %s", rng) - } - - return (end - start + 1), nil - case resp.StatusCode == http.StatusNotFound: - return 0, &BlobUploadNotFoundError{Location: hlu.location} - case resp.StatusCode >= 400 && resp.StatusCode < 500: - return 0, parseHTTPErrorResponse(resp) - default: - return 0, &UnexpectedHTTPStatusError{Status: resp.Status} - } -} - -func (hlu *httpLayerUpload) Seek(offset int64, whence int) (int64, error) { - newOffset := hlu.offset - - switch whence { - case os.SEEK_CUR: - newOffset += int64(offset) - case os.SEEK_END: - return newOffset, errors.New("Cannot seek from end on incomplete upload") - case os.SEEK_SET: - newOffset = int64(offset) - } - - hlu.offset = newOffset - - return hlu.offset, nil -} - -func (hlu *httpLayerUpload) UUID() string { - return hlu.uuid -} - -func (hlu *httpLayerUpload) StartedAt() time.Time { - return hlu.startedAt -} - -func (hlu *httpLayerUpload) Finish(digest digest.Digest) (distribution.Layer, error) { - // TODO(dmcgowan): Check if already finished, if so just fetch - req, err := http.NewRequest("PUT", hlu.location, nil) - if err != nil { - return nil, err - } - - values := req.URL.Query() - values.Set("digest", digest.String()) - req.URL.RawQuery = values.Encode() - - resp, err := hlu.client.Do(req) - if err != nil { - return nil, err - } - - switch { - case resp.StatusCode == http.StatusCreated: - return hlu.Layers().Fetch(digest) - case resp.StatusCode == http.StatusNotFound: - return nil, &BlobUploadNotFoundError{Location: hlu.location} - case resp.StatusCode >= 400 && resp.StatusCode < 500: - return nil, parseHTTPErrorResponse(resp) - default: - return nil, &UnexpectedHTTPStatusError{Status: resp.Status} - } -} - -func (hlu *httpLayerUpload) Cancel() error { - panic("not implemented") -} - -func (hlu *httpLayerUpload) Close() error { - hlu.closed = true - return nil -} From c7ef45130b0448dd3c9335f85991454c420a66db Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Fri, 8 May 2015 16:29:23 -0700 Subject: [PATCH 05/26] Cleanup session and config interface Signed-off-by: Derek McGowan (github: dmcgowan) --- registry/client/authchallenge.go | 10 +- registry/client/endpoint.go | 268 ----------------- registry/client/layer_upload_test.go | 17 +- registry/client/repository.go | 8 +- registry/client/repository_test.go | 21 +- registry/client/session.go | 282 ++++++++++++++++++ .../{endpoint_test.go => session_test.go} | 63 ++-- registry/client/token.go | 78 ----- registry/client/transport.go | 120 ++++++++ 9 files changed, 475 insertions(+), 392 deletions(-) delete mode 100644 registry/client/endpoint.go create mode 100644 registry/client/session.go rename registry/client/{endpoint_test.go => session_test.go} (79%) delete mode 100644 registry/client/token.go create mode 100644 registry/client/transport.go diff --git a/registry/client/authchallenge.go b/registry/client/authchallenge.go index f45704b1..a9cce3cc 100644 --- a/registry/client/authchallenge.go +++ b/registry/client/authchallenge.go @@ -8,9 +8,9 @@ import ( // Octet types from RFC 2616. type octetType byte -// AuthorizationChallenge carries information +// authorizationChallenge carries information // from a WWW-Authenticate response header. -type AuthorizationChallenge struct { +type authorizationChallenge struct { Scheme string Parameters map[string]string } @@ -54,12 +54,12 @@ func init() { } } -func parseAuthHeader(header http.Header) []AuthorizationChallenge { - var challenges []AuthorizationChallenge +func parseAuthHeader(header http.Header) []authorizationChallenge { + var challenges []authorizationChallenge for _, h := range header[http.CanonicalHeaderKey("WWW-Authenticate")] { v, p := parseValueAndParams(h) if v != "" { - challenges = append(challenges, AuthorizationChallenge{Scheme: v, Parameters: p}) + challenges = append(challenges, authorizationChallenge{Scheme: v, Parameters: p}) } } return challenges diff --git a/registry/client/endpoint.go b/registry/client/endpoint.go deleted file mode 100644 index 9889dc66..00000000 --- a/registry/client/endpoint.go +++ /dev/null @@ -1,268 +0,0 @@ -package client - -import ( - "fmt" - "net/http" - "net/url" - "strings" - "sync" - "time" - - "github.com/Sirupsen/logrus" - "github.com/docker/distribution/registry/api/v2" -) - -// Authorizer is used to apply Authorization to an HTTP request -type Authorizer interface { - // Authorizer updates an HTTP request with the needed authorization - Authorize(req *http.Request) error -} - -// CredentialStore is an interface for getting credentials for -// a given URL -type CredentialStore interface { - // Basic returns basic auth for the given URL - Basic(*url.URL) (string, string) -} - -// RepositoryEndpoint represents a single host endpoint serving up -// the distribution API. -type RepositoryEndpoint struct { - Endpoint string - Mirror bool - - Header http.Header - Credentials CredentialStore - - ub *v2.URLBuilder -} - -type nullAuthorizer struct{} - -func (na nullAuthorizer) Authorize(req *http.Request) error { - return nil -} - -type repositoryTransport struct { - Transport http.RoundTripper - Header http.Header - Authorizer Authorizer -} - -func (rt *repositoryTransport) RoundTrip(req *http.Request) (*http.Response, error) { - reqCopy := new(http.Request) - *reqCopy = *req - - // Copy existing headers then static headers - reqCopy.Header = make(http.Header, len(req.Header)+len(rt.Header)) - for k, s := range req.Header { - reqCopy.Header[k] = append([]string(nil), s...) - } - for k, s := range rt.Header { - reqCopy.Header[k] = append(reqCopy.Header[k], s...) - } - - if rt.Authorizer != nil { - if err := rt.Authorizer.Authorize(reqCopy); err != nil { - return nil, err - } - } - - logrus.Debugf("HTTP: %s %s", req.Method, req.URL) - - if rt.Transport != nil { - return rt.Transport.RoundTrip(reqCopy) - } - return http.DefaultTransport.RoundTrip(reqCopy) -} - -type authTransport struct { - Transport http.RoundTripper - Header http.Header -} - -func (rt *authTransport) RoundTrip(req *http.Request) (*http.Response, error) { - reqCopy := new(http.Request) - *reqCopy = *req - - // Copy existing headers then static headers - reqCopy.Header = make(http.Header, len(req.Header)+len(rt.Header)) - for k, s := range req.Header { - reqCopy.Header[k] = append([]string(nil), s...) - } - for k, s := range rt.Header { - reqCopy.Header[k] = append(reqCopy.Header[k], s...) - } - - logrus.Debugf("HTTP: %s %s", req.Method, req.URL) - - if rt.Transport != nil { - return rt.Transport.RoundTrip(reqCopy) - } - return http.DefaultTransport.RoundTrip(reqCopy) -} - -// URLBuilder returns a new URL builder -func (e *RepositoryEndpoint) URLBuilder() (*v2.URLBuilder, error) { - if e.ub == nil { - var err error - e.ub, err = v2.NewURLBuilderFromString(e.Endpoint) - if err != nil { - return nil, err - } - } - - return e.ub, nil -} - -// HTTPClient returns a new HTTP client configured for this endpoint -func (e *RepositoryEndpoint) HTTPClient(name string) (*http.Client, error) { - // TODO(dmcgowan): create http.Transport - - transport := &repositoryTransport{ - Header: e.Header, - } - client := &http.Client{ - Transport: transport, - } - - challenges, err := e.ping(client) - if err != nil { - return nil, err - } - actions := []string{"pull"} - if !e.Mirror { - actions = append(actions, "push") - } - - transport.Authorizer = &endpointAuthorizer{ - client: &http.Client{Transport: &authTransport{Header: e.Header}}, - challenges: challenges, - creds: e.Credentials, - resource: "repository", - scope: name, - actions: actions, - } - - return client, nil -} - -func (e *RepositoryEndpoint) ping(client *http.Client) ([]AuthorizationChallenge, error) { - ub, err := e.URLBuilder() - if err != nil { - return nil, err - } - u, err := ub.BuildBaseURL() - if err != nil { - return nil, err - } - - req, err := http.NewRequest("GET", u, nil) - if err != nil { - return nil, err - } - req.Header = make(http.Header, len(e.Header)) - for k, s := range e.Header { - req.Header[k] = append([]string(nil), s...) - } - - resp, err := client.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - var supportsV2 bool -HeaderLoop: - for _, supportedVersions := range resp.Header[http.CanonicalHeaderKey("Docker-Distribution-API-Version")] { - for _, versionName := range strings.Fields(supportedVersions) { - if versionName == "registry/2.0" { - supportsV2 = true - break HeaderLoop - } - } - } - - if !supportsV2 { - return nil, fmt.Errorf("%s does not appear to be a v2 registry endpoint", e.Endpoint) - } - - if resp.StatusCode == http.StatusUnauthorized { - // Parse the WWW-Authenticate Header and store the challenges - // on this endpoint object. - return parseAuthHeader(resp.Header), nil - } else if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("unable to get valid ping response: %d", resp.StatusCode) - } - - return nil, nil -} - -type endpointAuthorizer struct { - client *http.Client - challenges []AuthorizationChallenge - creds CredentialStore - - resource string - scope string - actions []string - - tokenLock sync.Mutex - tokenCache string - tokenExpiration time.Time -} - -func (ta *endpointAuthorizer) Authorize(req *http.Request) error { - token, err := ta.getToken() - if err != nil { - return err - } - if token != "" { - req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) - } else if ta.creds != nil { - username, password := ta.creds.Basic(req.URL) - if username != "" && password != "" { - req.SetBasicAuth(username, password) - } - } - return nil -} - -func (ta *endpointAuthorizer) getToken() (string, error) { - ta.tokenLock.Lock() - defer ta.tokenLock.Unlock() - now := time.Now() - if now.Before(ta.tokenExpiration) { - //log.Debugf("Using cached token for %q", ta.auth.Username) - return ta.tokenCache, nil - } - - for _, challenge := range ta.challenges { - switch strings.ToLower(challenge.Scheme) { - case "basic": - // no token necessary - case "bearer": - //log.Debugf("Getting bearer token with %s for %s", challenge.Parameters, ta.auth.Username) - params := map[string]string{} - for k, v := range challenge.Parameters { - params[k] = v - } - params["scope"] = fmt.Sprintf("%s:%s:%s", ta.resource, ta.scope, strings.Join(ta.actions, ",")) - token, err := getToken(ta.creds, params, ta.client) - if err != nil { - return "", err - } - ta.tokenCache = token - ta.tokenExpiration = now.Add(time.Minute) - - return token, nil - default: - //log.Infof("Unsupported auth scheme: %q", challenge.Scheme) - } - } - - // Do not expire cache since there are no challenges which use a token - ta.tokenExpiration = time.Now().Add(time.Hour * 24) - - return "", nil -} diff --git a/registry/client/layer_upload_test.go b/registry/client/layer_upload_test.go index 1aa5cf1e..9e22cb7c 100644 --- a/registry/client/layer_upload_test.go +++ b/registry/client/layer_upload_test.go @@ -124,7 +124,8 @@ func TestUploadReadFrom(t *testing.T) { e, c := testServer(m) defer c() - client, err := e.HTTPClient(repo) + repoConfig := &RepositoryConfig{} + client, err := repoConfig.HTTPClient() if err != nil { t.Fatalf("Error creating client: %s", err) } @@ -133,7 +134,7 @@ func TestUploadReadFrom(t *testing.T) { } // Valid case - layerUpload.location = e.Endpoint + locationPath + layerUpload.location = e + locationPath n, err := layerUpload.ReadFrom(bytes.NewReader(b)) if err != nil { t.Fatalf("Error calling ReadFrom: %s", err) @@ -143,26 +144,26 @@ func TestUploadReadFrom(t *testing.T) { } // Bad range - layerUpload.location = e.Endpoint + locationPath + layerUpload.location = e + locationPath _, err = layerUpload.ReadFrom(bytes.NewReader(b)) if err == nil { t.Fatalf("Expected error when bad range received") } // 404 - layerUpload.location = e.Endpoint + locationPath + layerUpload.location = e + locationPath _, err = layerUpload.ReadFrom(bytes.NewReader(b)) if err == nil { t.Fatalf("Expected error when not found") } if blobErr, ok := err.(*BlobUploadNotFoundError); !ok { t.Fatalf("Wrong error type %T: %s", err, err) - } else if expected := e.Endpoint + locationPath; blobErr.Location != expected { + } else if expected := e + locationPath; blobErr.Location != expected { t.Fatalf("Unexpected location: %s, expected %s", blobErr.Location, expected) } // 400 valid json - layerUpload.location = e.Endpoint + locationPath + layerUpload.location = e + locationPath _, err = layerUpload.ReadFrom(bytes.NewReader(b)) if err == nil { t.Fatalf("Expected error when not found") @@ -185,7 +186,7 @@ func TestUploadReadFrom(t *testing.T) { } // 400 invalid json - layerUpload.location = e.Endpoint + locationPath + layerUpload.location = e + locationPath _, err = layerUpload.ReadFrom(bytes.NewReader(b)) if err == nil { t.Fatalf("Expected error when not found") @@ -200,7 +201,7 @@ func TestUploadReadFrom(t *testing.T) { } // 500 - layerUpload.location = e.Endpoint + locationPath + layerUpload.location = e + locationPath _, err = layerUpload.ReadFrom(bytes.NewReader(b)) if err == nil { t.Fatalf("Expected error when not found") diff --git a/registry/client/repository.go b/registry/client/repository.go index 22a02373..d5f75bda 100644 --- a/registry/client/repository.go +++ b/registry/client/repository.go @@ -19,17 +19,17 @@ import ( ) // NewRepository creates a new Repository for the given repository name and endpoint -func NewRepository(ctx context.Context, name string, endpoint *RepositoryEndpoint) (distribution.Repository, error) { +func NewRepository(ctx context.Context, name, endpoint string, repoConfig *RepositoryConfig) (distribution.Repository, error) { if err := v2.ValidateRespositoryName(name); err != nil { return nil, err } - ub, err := endpoint.URLBuilder() + ub, err := v2.NewURLBuilderFromString(endpoint) if err != nil { return nil, err } - client, err := endpoint.HTTPClient(name) + client, err := repoConfig.HTTPClient() if err != nil { return nil, err } @@ -39,7 +39,7 @@ func NewRepository(ctx context.Context, name string, endpoint *RepositoryEndpoin ub: ub, name: name, context: ctx, - mirror: endpoint.Mirror, + mirror: repoConfig.AllowMirrors, }, nil } diff --git a/registry/client/repository_test.go b/registry/client/repository_test.go index b96c52e5..1674213d 100644 --- a/registry/client/repository_test.go +++ b/registry/client/repository_test.go @@ -20,11 +20,10 @@ import ( "golang.org/x/net/context" ) -func testServer(rrm testutil.RequestResponseMap) (*RepositoryEndpoint, func()) { +func testServer(rrm testutil.RequestResponseMap) (string, func()) { h := testutil.NewHandler(rrm) s := httptest.NewServer(h) - e := RepositoryEndpoint{Endpoint: s.URL, Mirror: false} - return &e, s.Close + return s.URL, s.Close } func newRandomBlob(size int) (digest.Digest, []byte) { @@ -97,7 +96,7 @@ func TestLayerFetch(t *testing.T) { e, c := testServer(m) defer c() - r, err := NewRepository(context.Background(), "test.example.com/repo1", e) + r, err := NewRepository(context.Background(), "test.example.com/repo1", e, &RepositoryConfig{}) if err != nil { t.Fatal(err) } @@ -127,7 +126,7 @@ func TestLayerExists(t *testing.T) { e, c := testServer(m) defer c() - r, err := NewRepository(context.Background(), "test.example.com/repo1", e) + r, err := NewRepository(context.Background(), "test.example.com/repo1", e, &RepositoryConfig{}) if err != nil { t.Fatal(err) } @@ -227,7 +226,7 @@ func TestLayerUploadChunked(t *testing.T) { e, c := testServer(m) defer c() - r, err := NewRepository(context.Background(), repo, e) + r, err := NewRepository(context.Background(), repo, e, &RepositoryConfig{}) if err != nil { t.Fatal(err) } @@ -334,7 +333,7 @@ func TestLayerUploadMonolithic(t *testing.T) { e, c := testServer(m) defer c() - r, err := NewRepository(context.Background(), repo, e) + r, err := NewRepository(context.Background(), repo, e, &RepositoryConfig{}) if err != nil { t.Fatal(err) } @@ -475,7 +474,7 @@ func TestManifestFetch(t *testing.T) { e, c := testServer(m) defer c() - r, err := NewRepository(context.Background(), repo, e) + r, err := NewRepository(context.Background(), repo, e, &RepositoryConfig{}) if err != nil { t.Fatal(err) } @@ -508,7 +507,7 @@ func TestManifestFetchByTag(t *testing.T) { e, c := testServer(m) defer c() - r, err := NewRepository(context.Background(), repo, e) + r, err := NewRepository(context.Background(), repo, e, &RepositoryConfig{}) if err != nil { t.Fatal(err) } @@ -553,7 +552,7 @@ func TestManifestDelete(t *testing.T) { e, c := testServer(m) defer c() - r, err := NewRepository(context.Background(), repo, e) + r, err := NewRepository(context.Background(), repo, e, &RepositoryConfig{}) if err != nil { t.Fatal(err) } @@ -591,7 +590,7 @@ func TestManifestPut(t *testing.T) { e, c := testServer(m) defer c() - r, err := NewRepository(context.Background(), repo, e) + r, err := NewRepository(context.Background(), repo, e, &RepositoryConfig{}) if err != nil { t.Fatal(err) } diff --git a/registry/client/session.go b/registry/client/session.go new file mode 100644 index 00000000..bd8abe0f --- /dev/null +++ b/registry/client/session.go @@ -0,0 +1,282 @@ +package client + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "strings" + "sync" + "time" +) + +// Authorizer is used to apply Authorization to an HTTP request +type Authorizer interface { + // Authorizer updates an HTTP request with the needed authorization + Authorize(req *http.Request) error +} + +// CredentialStore is an interface for getting credentials for +// a given URL +type CredentialStore interface { + // Basic returns basic auth for the given URL + Basic(*url.URL) (string, string) +} + +// RepositoryConfig holds the base configuration needed to communicate +// with a registry including a method of authorization and HTTP headers. +type RepositoryConfig struct { + Header http.Header + AuthSource Authorizer + AllowMirrors bool +} + +// HTTPClient returns a new HTTP client configured for this configuration +func (rc *RepositoryConfig) HTTPClient() (*http.Client, error) { + // TODO(dmcgowan): create base http.Transport with proper TLS configuration + + transport := &Transport{ + ExtraHeader: rc.Header, + AuthSource: rc.AuthSource, + } + + client := &http.Client{ + Transport: transport, + } + + return client, nil +} + +// TokenScope represents the scope at which a token will be requested. +// This represents a specific action on a registry resource. +type TokenScope struct { + Resource string + Scope string + Actions []string +} + +func (ts TokenScope) String() string { + return fmt.Sprintf("%s:%s:%s", ts.Resource, ts.Scope, strings.Join(ts.Actions, ",")) +} + +// NewTokenAuthorizer returns an authorizer which is capable of getting a token +// from a token server. The expected authorization method will be discovered +// by the authorizer, getting the token server endpoint from the URL being +// requested. Basic authentication may either be done to the token source or +// directly with the requested endpoint depending on the endpoint's +// WWW-Authenticate header. +func NewTokenAuthorizer(creds CredentialStore, header http.Header, scope TokenScope) Authorizer { + return &tokenAuthorizer{ + header: header, + creds: creds, + scope: scope, + challenges: map[string][]authorizationChallenge{}, + } +} + +type tokenAuthorizer struct { + header http.Header + challenges map[string][]authorizationChallenge + creds CredentialStore + scope TokenScope + + tokenLock sync.Mutex + tokenCache string + tokenExpiration time.Time +} + +func (ta *tokenAuthorizer) ping(endpoint string) ([]authorizationChallenge, error) { + req, err := http.NewRequest("GET", endpoint, nil) + if err != nil { + return nil, err + } + + resp, err := ta.client().Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var supportsV2 bool +HeaderLoop: + for _, supportedVersions := range resp.Header[http.CanonicalHeaderKey("Docker-Distribution-API-Version")] { + for _, versionName := range strings.Fields(supportedVersions) { + if versionName == "registry/2.0" { + supportsV2 = true + break HeaderLoop + } + } + } + + if !supportsV2 { + return nil, fmt.Errorf("%s does not appear to be a v2 registry endpoint", endpoint) + } + + if resp.StatusCode == http.StatusUnauthorized { + // Parse the WWW-Authenticate Header and store the challenges + // on this endpoint object. + return parseAuthHeader(resp.Header), nil + } else if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unable to get valid ping response: %d", resp.StatusCode) + } + + return nil, nil +} + +func (ta *tokenAuthorizer) Authorize(req *http.Request) error { + v2Root := strings.Index(req.URL.Path, "/v2/") + if v2Root == -1 { + return nil + } + + ping := url.URL{ + Host: req.URL.Host, + Scheme: req.URL.Scheme, + Path: req.URL.Path[:v2Root+4], + } + + pingEndpoint := ping.String() + + challenges, ok := ta.challenges[pingEndpoint] + if !ok { + var err error + challenges, err = ta.ping(pingEndpoint) + if err != nil { + return err + } + ta.challenges[pingEndpoint] = challenges + } + + return ta.setAuth(challenges, req) +} + +func (ta *tokenAuthorizer) client() *http.Client { + // TODO(dmcgowan): Use same transport which has properly configured TLS + return &http.Client{Transport: &Transport{ExtraHeader: ta.header}} +} + +func (ta *tokenAuthorizer) setAuth(challenges []authorizationChallenge, req *http.Request) error { + var useBasic bool + for _, challenge := range challenges { + switch strings.ToLower(challenge.Scheme) { + case "basic": + useBasic = true + case "bearer": + if err := ta.refreshToken(challenge); err != nil { + return err + } + + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", ta.tokenCache)) + + return nil + default: + //log.Infof("Unsupported auth scheme: %q", challenge.Scheme) + } + } + + // Only use basic when no token auth challenges found + if useBasic { + if ta.creds != nil { + username, password := ta.creds.Basic(req.URL) + if username != "" && password != "" { + req.SetBasicAuth(username, password) + return nil + } + } + return errors.New("no basic auth credentials") + } + + return nil +} + +func (ta *tokenAuthorizer) refreshToken(challenge authorizationChallenge) error { + ta.tokenLock.Lock() + defer ta.tokenLock.Unlock() + now := time.Now() + if now.After(ta.tokenExpiration) { + token, err := ta.fetchToken(challenge) + if err != nil { + return err + } + ta.tokenCache = token + ta.tokenExpiration = now.Add(time.Minute) + } + + return nil +} + +type tokenResponse struct { + Token string `json:"token"` +} + +func (ta *tokenAuthorizer) fetchToken(challenge authorizationChallenge) (token string, err error) { + //log.Debugf("Getting bearer token with %s for %s", challenge.Parameters, ta.auth.Username) + params := map[string]string{} + for k, v := range challenge.Parameters { + params[k] = v + } + params["scope"] = ta.scope.String() + + realm, ok := params["realm"] + if !ok { + return "", errors.New("no realm specified for token auth challenge") + } + + realmURL, err := url.Parse(realm) + if err != nil { + return "", fmt.Errorf("invalid token auth challenge realm: %s", err) + } + + // TODO(dmcgowan): Handle empty scheme + + req, err := http.NewRequest("GET", realmURL.String(), nil) + if err != nil { + return "", err + } + + reqParams := req.URL.Query() + service := params["service"] + scope := params["scope"] + + if service != "" { + reqParams.Add("service", service) + } + + for _, scopeField := range strings.Fields(scope) { + reqParams.Add("scope", scopeField) + } + + if ta.creds != nil { + username, password := ta.creds.Basic(realmURL) + if username != "" && password != "" { + reqParams.Add("account", username) + req.SetBasicAuth(username, password) + } + } + + req.URL.RawQuery = reqParams.Encode() + + resp, err := ta.client().Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("token auth attempt for registry: %s request failed with status: %d %s", req.URL, resp.StatusCode, http.StatusText(resp.StatusCode)) + } + + decoder := json.NewDecoder(resp.Body) + + tr := new(tokenResponse) + if err = decoder.Decode(tr); err != nil { + return "", fmt.Errorf("unable to decode token response: %s", err) + } + + if tr.Token == "" { + return "", errors.New("authorization server did not include a token in the response") + } + + return tr.Token, nil +} diff --git a/registry/client/endpoint_test.go b/registry/client/session_test.go similarity index 79% rename from registry/client/endpoint_test.go rename to registry/client/session_test.go index 42bdc357..87e1e66e 100644 --- a/registry/client/endpoint_test.go +++ b/registry/client/session_test.go @@ -30,7 +30,7 @@ func (w *testAuthenticationWrapper) ServeHTTP(rw http.ResponseWriter, r *http.Re w.next.ServeHTTP(rw, r) } -func testServerWithAuth(rrm testutil.RequestResponseMap, authenticate string, authCheck func(string) bool) (*RepositoryEndpoint, func()) { +func testServerWithAuth(rrm testutil.RequestResponseMap, authenticate string, authCheck func(string) bool) (string, func()) { h := testutil.NewHandler(rrm) wrapper := &testAuthenticationWrapper{ @@ -43,8 +43,7 @@ func testServerWithAuth(rrm testutil.RequestResponseMap, authenticate string, au } s := httptest.NewServer(wrapper) - e := RepositoryEndpoint{Endpoint: s.URL, Mirror: false} - return &e, s.Close + return s.URL, s.Close } type testCredentialStore struct { @@ -62,6 +61,16 @@ func TestEndpointAuthorizeToken(t *testing.T) { repo2 := "other/registry" scope1 := fmt.Sprintf("repository:%s:pull,push", repo1) scope2 := fmt.Sprintf("repository:%s:pull,push", repo2) + tokenScope1 := TokenScope{ + Resource: "repository", + Scope: repo1, + Actions: []string{"pull", "push"}, + } + tokenScope2 := TokenScope{ + Resource: "repository", + Scope: repo2, + Actions: []string{"pull", "push"}, + } tokenMap := testutil.RequestResponseMap([]testutil.RequestResponseMapping{ { @@ -92,7 +101,7 @@ func TestEndpointAuthorizeToken(t *testing.T) { { Request: testutil.Request{ Method: "GET", - Route: "/hello", + Route: "/v2/hello", }, Response: testutil.Response{ StatusCode: http.StatusAccepted, @@ -100,19 +109,23 @@ func TestEndpointAuthorizeToken(t *testing.T) { }, }) - authenicate := fmt.Sprintf("Bearer realm=%q,service=%q", te.Endpoint+"/token", service) + authenicate := fmt.Sprintf("Bearer realm=%q,service=%q", te+"/token", service) validCheck := func(a string) bool { return a == "Bearer statictoken" } e, c := testServerWithAuth(m, authenicate, validCheck) defer c() - client, err := e.HTTPClient(repo1) + repo1Config := &RepositoryConfig{ + AuthSource: NewTokenAuthorizer(nil, nil, tokenScope1), + } + + client, err := repo1Config.HTTPClient() if err != nil { t.Fatalf("Error creating http client: %s", err) } - req, _ := http.NewRequest("GET", e.Endpoint+"/hello", nil) + req, _ := http.NewRequest("GET", e+"/v2/hello", nil) resp, err := client.Do(req) if err != nil { t.Fatalf("Error sending get request: %s", err) @@ -128,12 +141,15 @@ func TestEndpointAuthorizeToken(t *testing.T) { e2, c2 := testServerWithAuth(m, authenicate, badCheck) defer c2() - client2, err := e2.HTTPClient(repo2) + repo2Config := &RepositoryConfig{ + AuthSource: NewTokenAuthorizer(nil, nil, tokenScope2), + } + client2, err := repo2Config.HTTPClient() if err != nil { t.Fatalf("Error creating http client: %s", err) } - req, _ = http.NewRequest("GET", e.Endpoint+"/hello", nil) + req, _ = http.NewRequest("GET", e2+"/v2/hello", nil) resp, err = client2.Do(req) if err != nil { t.Fatalf("Error sending get request: %s", err) @@ -155,6 +171,11 @@ func TestEndpointAuthorizeTokenBasic(t *testing.T) { scope := fmt.Sprintf("repository:%s:pull,push", repo) username := "tokenuser" password := "superSecretPa$$word" + tokenScope := TokenScope{ + Resource: "repository", + Scope: repo, + Actions: []string{"pull", "push"}, + } tokenMap := testutil.RequestResponseMap([]testutil.RequestResponseMapping{ { @@ -180,7 +201,7 @@ func TestEndpointAuthorizeTokenBasic(t *testing.T) { { Request: testutil.Request{ Method: "GET", - Route: "/hello", + Route: "/v2/hello", }, Response: testutil.Response{ StatusCode: http.StatusAccepted, @@ -188,24 +209,27 @@ func TestEndpointAuthorizeTokenBasic(t *testing.T) { }, }) - authenicate2 := fmt.Sprintf("Bearer realm=%q,service=%q", te.Endpoint+"/token", service) + authenicate2 := fmt.Sprintf("Bearer realm=%q,service=%q", te+"/token", service) bearerCheck := func(a string) bool { return a == "Bearer statictoken" } e, c := testServerWithAuth(m, authenicate2, bearerCheck) defer c() - e.Credentials = &testCredentialStore{ + creds := &testCredentialStore{ username: username, password: password, } + repoConfig := &RepositoryConfig{ + AuthSource: NewTokenAuthorizer(creds, nil, tokenScope), + } - client, err := e.HTTPClient(repo) + client, err := repoConfig.HTTPClient() if err != nil { t.Fatalf("Error creating http client: %s", err) } - req, _ := http.NewRequest("GET", e.Endpoint+"/hello", nil) + req, _ := http.NewRequest("GET", e+"/v2/hello", nil) resp, err := client.Do(req) if err != nil { t.Fatalf("Error sending get request: %s", err) @@ -221,7 +245,7 @@ func TestEndpointAuthorizeBasic(t *testing.T) { { Request: testutil.Request{ Method: "GET", - Route: "/hello", + Route: "/v2/hello", }, Response: testutil.Response{ StatusCode: http.StatusAccepted, @@ -237,17 +261,20 @@ func TestEndpointAuthorizeBasic(t *testing.T) { } e, c := testServerWithAuth(m, authenicate, validCheck) defer c() - e.Credentials = &testCredentialStore{ + creds := &testCredentialStore{ username: username, password: password, } + repoConfig := &RepositoryConfig{ + AuthSource: NewTokenAuthorizer(creds, nil, TokenScope{}), + } - client, err := e.HTTPClient("test/repo/basic") + client, err := repoConfig.HTTPClient() if err != nil { t.Fatalf("Error creating http client: %s", err) } - req, _ := http.NewRequest("GET", e.Endpoint+"/hello", nil) + req, _ := http.NewRequest("GET", e+"/v2/hello", nil) resp, err := client.Do(req) if err != nil { t.Fatalf("Error sending get request: %s", err) diff --git a/registry/client/token.go b/registry/client/token.go deleted file mode 100644 index 6439e01e..00000000 --- a/registry/client/token.go +++ /dev/null @@ -1,78 +0,0 @@ -package client - -import ( - "encoding/json" - "errors" - "fmt" - "net/http" - "net/url" - "strings" -) - -type tokenResponse struct { - Token string `json:"token"` -} - -func getToken(creds CredentialStore, params map[string]string, client *http.Client) (token string, err error) { - realm, ok := params["realm"] - if !ok { - return "", errors.New("no realm specified for token auth challenge") - } - - realmURL, err := url.Parse(realm) - if err != nil { - return "", fmt.Errorf("invalid token auth challenge realm: %s", err) - } - - // TODO(dmcgowan): Handle empty scheme - - req, err := http.NewRequest("GET", realmURL.String(), nil) - if err != nil { - return "", err - } - - reqParams := req.URL.Query() - service := params["service"] - scope := params["scope"] - - if service != "" { - reqParams.Add("service", service) - } - - for _, scopeField := range strings.Fields(scope) { - reqParams.Add("scope", scopeField) - } - - if creds != nil { - username, password := creds.Basic(realmURL) - if username != "" && password != "" { - reqParams.Add("account", username) - req.SetBasicAuth(username, password) - } - } - - req.URL.RawQuery = reqParams.Encode() - - resp, err := client.Do(req) - if err != nil { - return "", err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("token auth attempt for registry: %s request failed with status: %d %s", req.URL, resp.StatusCode, http.StatusText(resp.StatusCode)) - } - - decoder := json.NewDecoder(resp.Body) - - tr := new(tokenResponse) - if err = decoder.Decode(tr); err != nil { - return "", fmt.Errorf("unable to decode token response: %s", err) - } - - if tr.Token == "" { - return "", errors.New("authorization server did not include a token in the response") - } - - return tr.Token, nil -} diff --git a/registry/client/transport.go b/registry/client/transport.go new file mode 100644 index 00000000..e92ba543 --- /dev/null +++ b/registry/client/transport.go @@ -0,0 +1,120 @@ +package client + +import ( + "io" + "net/http" + "sync" +) + +// Transport is an http.RoundTripper that makes registry HTTP requests, +// wrapping a base RoundTripper and adding an Authorization header +// from an Auth source +type Transport struct { + AuthSource Authorizer + ExtraHeader http.Header + + Base http.RoundTripper + + mu sync.Mutex // guards modReq + modReq map[*http.Request]*http.Request // original -> modified +} + +// RoundTrip authorizes and authenticates the request with an +// access token. If no token exists or token is expired, +// tries to refresh/fetch a new token. +func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) { + req2 := t.cloneRequest(req) + if t.AuthSource != nil { + if err := t.AuthSource.Authorize(req2); err != nil { + return nil, err + } + } + t.setModReq(req, req2) + res, err := t.base().RoundTrip(req2) + if err != nil { + t.setModReq(req, nil) + return nil, err + } + res.Body = &onEOFReader{ + rc: res.Body, + fn: func() { t.setModReq(req, nil) }, + } + return res, nil +} + +// CancelRequest cancels an in-flight request by closing its connection. +func (t *Transport) CancelRequest(req *http.Request) { + type canceler interface { + CancelRequest(*http.Request) + } + if cr, ok := t.base().(canceler); ok { + t.mu.Lock() + modReq := t.modReq[req] + delete(t.modReq, req) + t.mu.Unlock() + cr.CancelRequest(modReq) + } +} + +func (t *Transport) base() http.RoundTripper { + if t.Base != nil { + return t.Base + } + return http.DefaultTransport +} + +func (t *Transport) setModReq(orig, mod *http.Request) { + t.mu.Lock() + defer t.mu.Unlock() + if t.modReq == nil { + t.modReq = make(map[*http.Request]*http.Request) + } + if mod == nil { + delete(t.modReq, orig) + } else { + t.modReq[orig] = mod + } +} + +// cloneRequest returns a clone of the provided *http.Request. +// The clone is a shallow copy of the struct and its Header map. +func (t *Transport) cloneRequest(r *http.Request) *http.Request { + // shallow copy of the struct + r2 := new(http.Request) + *r2 = *r + // deep copy of the Header + r2.Header = make(http.Header, len(r.Header)) + for k, s := range r.Header { + r2.Header[k] = append([]string(nil), s...) + } + for k, s := range t.ExtraHeader { + r2.Header[k] = append(r2.Header[k], s...) + } + return r2 +} + +type onEOFReader struct { + rc io.ReadCloser + fn func() +} + +func (r *onEOFReader) Read(p []byte) (n int, err error) { + n, err = r.rc.Read(p) + if err == io.EOF { + r.runFunc() + } + return +} + +func (r *onEOFReader) Close() error { + err := r.rc.Close() + r.runFunc() + return err +} + +func (r *onEOFReader) runFunc() { + if fn := r.fn; fn != nil { + fn() + r.fn = nil + } +} From da05873b7c55113b3650638eb941a6dccc9ea720 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Fri, 8 May 2015 16:33:27 -0700 Subject: [PATCH 06/26] Use distribution context instead of google Signed-off-by: Derek McGowan (github: dmcgowan) --- registry/client/repository.go | 2 +- registry/client/repository_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/registry/client/repository.go b/registry/client/repository.go index d5f75bda..0cda0d83 100644 --- a/registry/client/repository.go +++ b/registry/client/repository.go @@ -14,8 +14,8 @@ import ( "github.com/docker/distribution/digest" "github.com/docker/distribution" + "github.com/docker/distribution/context" "github.com/docker/distribution/registry/api/v2" - "golang.org/x/net/context" ) // NewRepository creates a new Repository for the given repository name and endpoint diff --git a/registry/client/repository_test.go b/registry/client/repository_test.go index 1674213d..f53112dc 100644 --- a/registry/client/repository_test.go +++ b/registry/client/repository_test.go @@ -14,10 +14,10 @@ import ( "code.google.com/p/go-uuid/uuid" + "github.com/docker/distribution/context" "github.com/docker/distribution/digest" "github.com/docker/distribution/manifest" "github.com/docker/distribution/testutil" - "golang.org/x/net/context" ) func testServer(rrm testutil.RequestResponseMap) (string, func()) { From d92e5b10961e678b307abc4921881ee13bf38bf6 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Fri, 8 May 2015 17:40:30 -0700 Subject: [PATCH 07/26] Add tags implementation Signed-off-by: Derek McGowan (github: dmcgowan) --- registry/client/errors.go | 7 ++++ registry/client/layer_upload.go | 8 ++--- registry/client/repository.go | 56 ++++++++++++++++++++---------- registry/client/repository_test.go | 52 +++++++++++++++++++++++++++ 4 files changed, 98 insertions(+), 25 deletions(-) diff --git a/registry/client/errors.go b/registry/client/errors.go index e02b0f73..adb909d1 100644 --- a/registry/client/errors.go +++ b/registry/client/errors.go @@ -112,3 +112,10 @@ func parseHTTPErrorResponse(response *http.Response) error { } return &errors } + +func handleErrorResponse(resp *http.Response) error { + if resp.StatusCode >= 400 && resp.StatusCode < 500 { + return parseHTTPErrorResponse(resp) + } + return &UnexpectedHTTPStatusError{Status: resp.Status} +} diff --git a/registry/client/layer_upload.go b/registry/client/layer_upload.go index ce0794c2..02cc5162 100644 --- a/registry/client/layer_upload.go +++ b/registry/client/layer_upload.go @@ -26,14 +26,10 @@ type httpLayerUpload struct { } func (hlu *httpLayerUpload) handleErrorResponse(resp *http.Response) error { - switch { - case resp.StatusCode == http.StatusNotFound: + if resp.StatusCode == http.StatusNotFound { return &BlobUploadNotFoundError{Location: hlu.location} - case resp.StatusCode >= 400 && resp.StatusCode < 500: - return parseHTTPErrorResponse(resp) - default: - return &UnexpectedHTTPStatusError{Status: resp.Status} } + return handleErrorResponse(resp) } func (hlu *httpLayerUpload) ReadFrom(r io.Reader) (n int64, err error) { diff --git a/registry/client/repository.go b/registry/client/repository.go index 0cda0d83..c79c306b 100644 --- a/registry/client/repository.go +++ b/registry/client/repository.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "fmt" + "io/ioutil" "net/http" "net/url" "strconv" @@ -90,7 +91,36 @@ type manifests struct { } func (ms *manifests) Tags() ([]string, error) { - panic("not implemented") + u, err := ms.ub.BuildTagsURL(ms.name) + if err != nil { + return nil, err + } + + resp, err := ms.client.Get(u) + if err != nil { + return nil, err + } + + switch { + case resp.StatusCode == http.StatusOK: + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + tagsResponse := struct { + Tags []string `json:"tags"` + }{} + if err := json.Unmarshal(b, &tagsResponse); err != nil { + return nil, err + } + + return tagsResponse.Tags, nil + case resp.StatusCode == http.StatusNotFound: + return nil, nil + default: + return nil, handleErrorResponse(resp) + } } func (ms *manifests) Exists(dgst digest.Digest) (bool, error) { @@ -113,10 +143,8 @@ func (ms *manifests) ExistsByTag(tag string) (bool, error) { return true, nil case resp.StatusCode == http.StatusNotFound: return false, nil - case resp.StatusCode >= 400 && resp.StatusCode < 500: - return false, parseHTTPErrorResponse(resp) default: - return false, &UnexpectedHTTPStatusError{Status: resp.Status} + return false, handleErrorResponse(resp) } } @@ -146,10 +174,8 @@ func (ms *manifests) GetByTag(tag string) (*manifest.SignedManifest, error) { } return &sm, nil - case resp.StatusCode >= 400 && resp.StatusCode < 500: - return nil, parseHTTPErrorResponse(resp) default: - return nil, &UnexpectedHTTPStatusError{Status: resp.Status} + return nil, handleErrorResponse(resp) } } @@ -174,10 +200,8 @@ func (ms *manifests) Put(m *manifest.SignedManifest) error { case resp.StatusCode == http.StatusAccepted: // TODO(dmcgowan): Use or check digest header return nil - case resp.StatusCode >= 400 && resp.StatusCode < 500: - return parseHTTPErrorResponse(resp) default: - return &UnexpectedHTTPStatusError{Status: resp.Status} + return handleErrorResponse(resp) } } @@ -200,10 +224,8 @@ func (ms *manifests) Delete(dgst digest.Digest) error { switch { case resp.StatusCode == http.StatusOK: return nil - case resp.StatusCode >= 400 && resp.StatusCode < 500: - return parseHTTPErrorResponse(resp) default: - return &UnexpectedHTTPStatusError{Status: resp.Status} + return handleErrorResponse(resp) } } @@ -275,10 +297,8 @@ func (ls *layers) Upload() (distribution.LayerUpload, error) { startedAt: time.Now(), location: location, }, nil - case resp.StatusCode >= 400 && resp.StatusCode < 500: - return nil, parseHTTPErrorResponse(resp) default: - return nil, &UnexpectedHTTPStatusError{Status: resp.Status} + return nil, handleErrorResponse(resp) } } @@ -327,9 +347,7 @@ func (ls *layers) fetchLayer(dgst digest.Digest) (distribution.Layer, error) { BlobSum: dgst, }, } - case resp.StatusCode >= 400 && resp.StatusCode < 500: - return nil, parseHTTPErrorResponse(resp) default: - return nil, &UnexpectedHTTPStatusError{Status: resp.Status} + return nil, handleErrorResponse(resp) } } diff --git a/registry/client/repository_test.go b/registry/client/repository_test.go index f53112dc..fe8ffeb7 100644 --- a/registry/client/repository_test.go +++ b/registry/client/repository_test.go @@ -9,6 +9,7 @@ import ( "log" "net/http" "net/http/httptest" + "strings" "testing" "time" @@ -602,3 +603,54 @@ func TestManifestPut(t *testing.T) { // TODO(dmcgowan): Check for error cases } + +func TestManifestTags(t *testing.T) { + repo := "test.example.com/repo/tags/list" + tagsList := []byte(strings.TrimSpace(` +{ + "name": "test.example.com/repo/tags/list", + "tags": [ + "tag1", + "tag2", + "funtag" + ] +} + `)) + var m testutil.RequestResponseMap + addPing(&m) + m = append(m, testutil.RequestResponseMapping{ + Request: testutil.Request{ + Method: "GET", + Route: "/v2/" + repo + "/tags/list", + }, + Response: testutil.Response{ + StatusCode: http.StatusOK, + Body: tagsList, + Headers: http.Header(map[string][]string{ + "Content-Length": {fmt.Sprint(len(tagsList))}, + "Last-Modified": {time.Now().Add(-1 * time.Second).Format(time.ANSIC)}, + }), + }, + }) + + e, c := testServer(m) + defer c() + + r, err := NewRepository(context.Background(), repo, e, &RepositoryConfig{}) + if err != nil { + t.Fatal(err) + } + + ms := r.Manifests() + tags, err := ms.Tags() + if err != nil { + t.Fatal(err) + } + + if len(tags) != 3 { + t.Fatalf("Wrong number of tags returned: %d, expected 3", len(tags)) + } + // TODO(dmcgowan): Check array + + // TODO(dmcgowan): Check for error cases +} From 49f7f54d07e419c75ebc9f2f3c13ad35e564e742 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Mon, 11 May 2015 11:31:22 -0700 Subject: [PATCH 08/26] Create authentication handler Refactory authorizer to take a set of authentication handlers for different authentication schemes returned by an unauthorized HTTP requst. Signed-off-by: Derek McGowan (github: dmcgowan) --- registry/client/authchallenge.go | 6 +- registry/client/authchallenge_test.go | 21 +-- registry/client/errors.go | 35 ----- registry/client/layer.go | 2 +- registry/client/session.go | 205 ++++++++++++++++---------- 5 files changed, 143 insertions(+), 126 deletions(-) diff --git a/registry/client/authchallenge.go b/registry/client/authchallenge.go index a9cce3cc..49cf270e 100644 --- a/registry/client/authchallenge.go +++ b/registry/client/authchallenge.go @@ -54,12 +54,12 @@ func init() { } } -func parseAuthHeader(header http.Header) []authorizationChallenge { - var challenges []authorizationChallenge +func parseAuthHeader(header http.Header) map[string]authorizationChallenge { + challenges := map[string]authorizationChallenge{} for _, h := range header[http.CanonicalHeaderKey("WWW-Authenticate")] { v, p := parseValueAndParams(h) if v != "" { - challenges = append(challenges, authorizationChallenge{Scheme: v, Parameters: p}) + challenges[v] = authorizationChallenge{Scheme: v, Parameters: p} } } return challenges diff --git a/registry/client/authchallenge_test.go b/registry/client/authchallenge_test.go index bb3016ee..802c94f3 100644 --- a/registry/client/authchallenge_test.go +++ b/registry/client/authchallenge_test.go @@ -13,25 +13,26 @@ func TestAuthChallengeParse(t *testing.T) { if len(challenges) != 1 { t.Fatalf("Unexpected number of auth challenges: %d, expected 1", len(challenges)) } + challenge := challenges["bearer"] - if expected := "bearer"; challenges[0].Scheme != expected { - t.Fatalf("Unexpected scheme: %s, expected: %s", challenges[0].Scheme, expected) + if expected := "bearer"; challenge.Scheme != expected { + t.Fatalf("Unexpected scheme: %s, expected: %s", challenge.Scheme, expected) } - if expected := "https://auth.example.com/token"; challenges[0].Parameters["realm"] != expected { - t.Fatalf("Unexpected param: %s, expected: %s", challenges[0].Parameters["realm"], expected) + if expected := "https://auth.example.com/token"; challenge.Parameters["realm"] != expected { + t.Fatalf("Unexpected param: %s, expected: %s", challenge.Parameters["realm"], expected) } - if expected := "registry.example.com"; challenges[0].Parameters["service"] != expected { - t.Fatalf("Unexpected param: %s, expected: %s", challenges[0].Parameters["service"], expected) + if expected := "registry.example.com"; challenge.Parameters["service"] != expected { + t.Fatalf("Unexpected param: %s, expected: %s", challenge.Parameters["service"], expected) } - if expected := "fun"; challenges[0].Parameters["other"] != expected { - t.Fatalf("Unexpected param: %s, expected: %s", challenges[0].Parameters["other"], expected) + if expected := "fun"; challenge.Parameters["other"] != expected { + t.Fatalf("Unexpected param: %s, expected: %s", challenge.Parameters["other"], expected) } - if expected := "he\"llo"; challenges[0].Parameters["slashed"] != expected { - t.Fatalf("Unexpected param: %s, expected: %s", challenges[0].Parameters["slashed"], expected) + if expected := "he\"llo"; challenge.Parameters["slashed"] != expected { + t.Fatalf("Unexpected param: %s, expected: %s", challenge.Parameters["slashed"], expected) } } diff --git a/registry/client/errors.go b/registry/client/errors.go index adb909d1..2bb64a44 100644 --- a/registry/client/errors.go +++ b/registry/client/errors.go @@ -6,44 +6,9 @@ import ( "io/ioutil" "net/http" - "github.com/docker/distribution/digest" "github.com/docker/distribution/registry/api/v2" ) -// RepositoryNotFoundError is returned when making an operation against a -// repository that does not exist in the registry. -type RepositoryNotFoundError struct { - Name string -} - -func (e *RepositoryNotFoundError) Error() string { - return fmt.Sprintf("No repository found with Name: %s", e.Name) -} - -// ImageManifestNotFoundError is returned when making an operation against a -// given image manifest that does not exist in the registry. -type ImageManifestNotFoundError struct { - Name string - Tag string -} - -func (e *ImageManifestNotFoundError) Error() string { - return fmt.Sprintf("No manifest found with Name: %s, Tag: %s", - e.Name, e.Tag) -} - -// BlobNotFoundError is returned when making an operation against a given image -// layer that does not exist in the registry. -type BlobNotFoundError struct { - Name string - Digest digest.Digest -} - -func (e *BlobNotFoundError) Error() string { - return fmt.Sprintf("No blob found with Name: %s, Digest: %s", - e.Name, e.Digest) -} - // BlobUploadNotFoundError is returned when making a blob upload operation against an // invalid blob upload location url. // This may be the result of using a cancelled, completed, or stale upload diff --git a/registry/client/layer.go b/registry/client/layer.go index f61a9034..b6e1697d 100644 --- a/registry/client/layer.go +++ b/registry/client/layer.go @@ -48,7 +48,7 @@ func (hl *httpLayer) Read(p []byte) (n int, err error) { n, err = rd.Read(p) hl.offset += int64(n) - // Simulate io.EOR error if we reach filesize. + // Simulate io.EOF error if we reach filesize. if err == nil && hl.offset >= hl.size { err = io.EOF } diff --git a/registry/client/session.go b/registry/client/session.go index bd8abe0f..97e932ff 100644 --- a/registry/client/session.go +++ b/registry/client/session.go @@ -17,6 +17,13 @@ type Authorizer interface { Authorize(req *http.Request) error } +// AuthenticationHandler is an interface for authorizing a request from +// params from a "WWW-Authenicate" header for a single scheme. +type AuthenticationHandler interface { + Scheme() string + AuthorizeRequest(req *http.Request, params map[string]string) error +} + // CredentialStore is an interface for getting credentials for // a given URL type CredentialStore interface { @@ -48,18 +55,6 @@ func (rc *RepositoryConfig) HTTPClient() (*http.Client, error) { return client, nil } -// TokenScope represents the scope at which a token will be requested. -// This represents a specific action on a registry resource. -type TokenScope struct { - Resource string - Scope string - Actions []string -} - -func (ts TokenScope) String() string { - return fmt.Sprintf("%s:%s:%s", ts.Resource, ts.Scope, strings.Join(ts.Actions, ",")) -} - // NewTokenAuthorizer returns an authorizer which is capable of getting a token // from a token server. The expected authorization method will be discovered // by the authorizer, getting the token server endpoint from the URL being @@ -69,24 +64,37 @@ func (ts TokenScope) String() string { func NewTokenAuthorizer(creds CredentialStore, header http.Header, scope TokenScope) Authorizer { return &tokenAuthorizer{ header: header, - creds: creds, - scope: scope, - challenges: map[string][]authorizationChallenge{}, + challenges: map[string]map[string]authorizationChallenge{}, + handlers: []AuthenticationHandler{ + NewTokenHandler(creds, scope, header), + NewBasicHandler(creds), + }, + } +} + +// NewAuthorizer creates an authorizer which can handle multiple authentication +// schemes. The handlers are tried in order, the higher priority authentication +// methods should be first. +func NewAuthorizer(header http.Header, handlers ...AuthenticationHandler) Authorizer { + return &tokenAuthorizer{ + header: header, + challenges: map[string]map[string]authorizationChallenge{}, + handlers: handlers, } } type tokenAuthorizer struct { header http.Header - challenges map[string][]authorizationChallenge - creds CredentialStore - scope TokenScope - - tokenLock sync.Mutex - tokenCache string - tokenExpiration time.Time + challenges map[string]map[string]authorizationChallenge + handlers []AuthenticationHandler } -func (ta *tokenAuthorizer) ping(endpoint string) ([]authorizationChallenge, error) { +func (ta *tokenAuthorizer) client() *http.Client { + // TODO(dmcgowan): Use same transport which has properly configured TLS + return &http.Client{Transport: &Transport{ExtraHeader: ta.header}} +} + +func (ta *tokenAuthorizer) ping(endpoint string) (map[string]authorizationChallenge, error) { req, err := http.NewRequest("GET", endpoint, nil) if err != nil { return nil, err @@ -98,6 +106,7 @@ func (ta *tokenAuthorizer) ping(endpoint string) ([]authorizationChallenge, erro } defer resp.Body.Close() + // TODO(dmcgowan): Add version string which would allow skipping this section var supportsV2 bool HeaderLoop: for _, supportedVersions := range resp.Header[http.CanonicalHeaderKey("Docker-Distribution-API-Version")] { @@ -148,59 +157,80 @@ func (ta *tokenAuthorizer) Authorize(req *http.Request) error { ta.challenges[pingEndpoint] = challenges } - return ta.setAuth(challenges, req) -} - -func (ta *tokenAuthorizer) client() *http.Client { - // TODO(dmcgowan): Use same transport which has properly configured TLS - return &http.Client{Transport: &Transport{ExtraHeader: ta.header}} -} - -func (ta *tokenAuthorizer) setAuth(challenges []authorizationChallenge, req *http.Request) error { - var useBasic bool - for _, challenge := range challenges { - switch strings.ToLower(challenge.Scheme) { - case "basic": - useBasic = true - case "bearer": - if err := ta.refreshToken(challenge); err != nil { + for _, handler := range ta.handlers { + challenge, ok := challenges[handler.Scheme()] + if ok { + if err := handler.AuthorizeRequest(req, challenge.Parameters); err != nil { return err } - - req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", ta.tokenCache)) - - return nil - default: - //log.Infof("Unsupported auth scheme: %q", challenge.Scheme) } } - // Only use basic when no token auth challenges found - if useBasic { - if ta.creds != nil { - username, password := ta.creds.Basic(req.URL) - if username != "" && password != "" { - req.SetBasicAuth(username, password) - return nil - } - } - return errors.New("no basic auth credentials") - } - return nil } -func (ta *tokenAuthorizer) refreshToken(challenge authorizationChallenge) error { - ta.tokenLock.Lock() - defer ta.tokenLock.Unlock() +type tokenHandler struct { + header http.Header + creds CredentialStore + scope TokenScope + + tokenLock sync.Mutex + tokenCache string + tokenExpiration time.Time +} + +// TokenScope represents the scope at which a token will be requested. +// This represents a specific action on a registry resource. +type TokenScope struct { + Resource string + Scope string + Actions []string +} + +// NewTokenHandler creates a new AuthenicationHandler which supports +// fetching tokens from a remote token server. +func NewTokenHandler(creds CredentialStore, scope TokenScope, header http.Header) AuthenticationHandler { + return &tokenHandler{ + header: header, + creds: creds, + scope: scope, + } +} + +func (ts TokenScope) String() string { + return fmt.Sprintf("%s:%s:%s", ts.Resource, ts.Scope, strings.Join(ts.Actions, ",")) +} + +func (ts *tokenHandler) client() *http.Client { + // TODO(dmcgowan): Use same transport which has properly configured TLS + return &http.Client{Transport: &Transport{ExtraHeader: ts.header}} +} + +func (ts *tokenHandler) Scheme() string { + return "bearer" +} + +func (ts *tokenHandler) AuthorizeRequest(req *http.Request, params map[string]string) error { + if err := ts.refreshToken(params); err != nil { + return err + } + + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", ts.tokenCache)) + + return nil +} + +func (ts *tokenHandler) refreshToken(params map[string]string) error { + ts.tokenLock.Lock() + defer ts.tokenLock.Unlock() now := time.Now() - if now.After(ta.tokenExpiration) { - token, err := ta.fetchToken(challenge) + if now.After(ts.tokenExpiration) { + token, err := ts.fetchToken(params) if err != nil { return err } - ta.tokenCache = token - ta.tokenExpiration = now.Add(time.Minute) + ts.tokenCache = token + ts.tokenExpiration = now.Add(time.Minute) } return nil @@ -210,26 +240,20 @@ type tokenResponse struct { Token string `json:"token"` } -func (ta *tokenAuthorizer) fetchToken(challenge authorizationChallenge) (token string, err error) { +func (ts *tokenHandler) fetchToken(params map[string]string) (token string, err error) { //log.Debugf("Getting bearer token with %s for %s", challenge.Parameters, ta.auth.Username) - params := map[string]string{} - for k, v := range challenge.Parameters { - params[k] = v - } - params["scope"] = ta.scope.String() - realm, ok := params["realm"] if !ok { return "", errors.New("no realm specified for token auth challenge") } + // TODO(dmcgowan): Handle empty scheme + realmURL, err := url.Parse(realm) if err != nil { return "", fmt.Errorf("invalid token auth challenge realm: %s", err) } - // TODO(dmcgowan): Handle empty scheme - req, err := http.NewRequest("GET", realmURL.String(), nil) if err != nil { return "", err @@ -237,7 +261,7 @@ func (ta *tokenAuthorizer) fetchToken(challenge authorizationChallenge) (token s reqParams := req.URL.Query() service := params["service"] - scope := params["scope"] + scope := ts.scope.String() if service != "" { reqParams.Add("service", service) @@ -247,8 +271,8 @@ func (ta *tokenAuthorizer) fetchToken(challenge authorizationChallenge) (token s reqParams.Add("scope", scopeField) } - if ta.creds != nil { - username, password := ta.creds.Basic(realmURL) + if ts.creds != nil { + username, password := ts.creds.Basic(realmURL) if username != "" && password != "" { reqParams.Add("account", username) req.SetBasicAuth(username, password) @@ -257,7 +281,7 @@ func (ta *tokenAuthorizer) fetchToken(challenge authorizationChallenge) (token s req.URL.RawQuery = reqParams.Encode() - resp, err := ta.client().Do(req) + resp, err := ts.client().Do(req) if err != nil { return "", err } @@ -280,3 +304,30 @@ func (ta *tokenAuthorizer) fetchToken(challenge authorizationChallenge) (token s return tr.Token, nil } + +type basicHandler struct { + creds CredentialStore +} + +// NewBasicHandler creaters a new authentiation handler which adds +// basic authentication credentials to a request. +func NewBasicHandler(creds CredentialStore) AuthenticationHandler { + return &basicHandler{ + creds: creds, + } +} + +func (*basicHandler) Scheme() string { + return "basic" +} + +func (bh *basicHandler) AuthorizeRequest(req *http.Request, params map[string]string) error { + if bh.creds != nil { + username, password := bh.creds.Basic(req.URL) + if username != "" && password != "" { + req.SetBasicAuth(username, password) + return nil + } + } + return errors.New("no basic auth credentials") +} From 68d5ecf6bf9b05135f8dc8626752a28719a7c8d0 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Mon, 11 May 2015 16:39:12 -0700 Subject: [PATCH 09/26] Update ReadFrom to wrap reader in NopCloser Wrapping the reader in a NopCloser is necessary to prevent the http library from closing the input reader. Signed-off-by: Derek McGowan (github: dmcgowan) --- registry/client/layer_upload.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/registry/client/layer_upload.go b/registry/client/layer_upload.go index 02cc5162..18e5fbab 100644 --- a/registry/client/layer_upload.go +++ b/registry/client/layer_upload.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "io/ioutil" "net/http" "os" "time" @@ -33,7 +34,7 @@ func (hlu *httpLayerUpload) handleErrorResponse(resp *http.Response) error { } func (hlu *httpLayerUpload) ReadFrom(r io.Reader) (n int64, err error) { - req, err := http.NewRequest("PATCH", hlu.location, r) + req, err := http.NewRequest("PATCH", hlu.location, ioutil.NopCloser(r)) if err != nil { return 0, err } From c6b51970cde3b51c8a94663c1a00f90434bbc819 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Mon, 11 May 2015 18:11:08 -0700 Subject: [PATCH 10/26] Removed unused mirror flags Signed-off-by: Derek McGowan (github: dmcgowan) --- registry/client/repository.go | 2 -- registry/client/session.go | 7 ++++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/registry/client/repository.go b/registry/client/repository.go index c79c306b..e7fcfa9f 100644 --- a/registry/client/repository.go +++ b/registry/client/repository.go @@ -40,7 +40,6 @@ func NewRepository(ctx context.Context, name, endpoint string, repoConfig *Repos ub: ub, name: name, context: ctx, - mirror: repoConfig.AllowMirrors, }, nil } @@ -49,7 +48,6 @@ type repository struct { ub *v2.URLBuilder context context.Context name string - mirror bool } func (r *repository) Name() string { diff --git a/registry/client/session.go b/registry/client/session.go index 97e932ff..dd8e7d80 100644 --- a/registry/client/session.go +++ b/registry/client/session.go @@ -34,9 +34,10 @@ type CredentialStore interface { // RepositoryConfig holds the base configuration needed to communicate // with a registry including a method of authorization and HTTP headers. type RepositoryConfig struct { - Header http.Header - AuthSource Authorizer - AllowMirrors bool + Header http.Header + AuthSource Authorizer + + //TODO(dmcgowan): Add tls config } // HTTPClient returns a new HTTP client configured for this configuration From a15806ed9c30908bc1a1b69abe2e95285a551327 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Tue, 12 May 2015 12:04:18 -0700 Subject: [PATCH 11/26] Add base transport to interface Signed-off-by: Derek McGowan (github: dmcgowan) --- registry/client/session.go | 47 ++++++++++++++++++++------------- registry/client/session_test.go | 8 +++--- 2 files changed, 33 insertions(+), 22 deletions(-) diff --git a/registry/client/session.go b/registry/client/session.go index dd8e7d80..e4e92383 100644 --- a/registry/client/session.go +++ b/registry/client/session.go @@ -37,16 +37,15 @@ type RepositoryConfig struct { Header http.Header AuthSource Authorizer - //TODO(dmcgowan): Add tls config + BaseTransport http.RoundTripper } // HTTPClient returns a new HTTP client configured for this configuration func (rc *RepositoryConfig) HTTPClient() (*http.Client, error) { - // TODO(dmcgowan): create base http.Transport with proper TLS configuration - transport := &Transport{ ExtraHeader: rc.Header, AuthSource: rc.AuthSource, + Base: rc.BaseTransport, } client := &http.Client{ @@ -62,25 +61,27 @@ func (rc *RepositoryConfig) HTTPClient() (*http.Client, error) { // requested. Basic authentication may either be done to the token source or // directly with the requested endpoint depending on the endpoint's // WWW-Authenticate header. -func NewTokenAuthorizer(creds CredentialStore, header http.Header, scope TokenScope) Authorizer { +func NewTokenAuthorizer(creds CredentialStore, transport http.RoundTripper, header http.Header, scope TokenScope) Authorizer { return &tokenAuthorizer{ header: header, challenges: map[string]map[string]authorizationChallenge{}, handlers: []AuthenticationHandler{ - NewTokenHandler(creds, scope, header), + NewTokenHandler(transport, creds, scope, header), NewBasicHandler(creds), }, + transport: transport, } } // NewAuthorizer creates an authorizer which can handle multiple authentication // schemes. The handlers are tried in order, the higher priority authentication // methods should be first. -func NewAuthorizer(header http.Header, handlers ...AuthenticationHandler) Authorizer { +func NewAuthorizer(transport http.RoundTripper, header http.Header, handlers ...AuthenticationHandler) Authorizer { return &tokenAuthorizer{ header: header, challenges: map[string]map[string]authorizationChallenge{}, handlers: handlers, + transport: transport, } } @@ -88,11 +89,7 @@ type tokenAuthorizer struct { header http.Header challenges map[string]map[string]authorizationChallenge handlers []AuthenticationHandler -} - -func (ta *tokenAuthorizer) client() *http.Client { - // TODO(dmcgowan): Use same transport which has properly configured TLS - return &http.Client{Transport: &Transport{ExtraHeader: ta.header}} + transport http.RoundTripper } func (ta *tokenAuthorizer) ping(endpoint string) (map[string]authorizationChallenge, error) { @@ -101,7 +98,16 @@ func (ta *tokenAuthorizer) ping(endpoint string) (map[string]authorizationChalle return nil, err } - resp, err := ta.client().Do(req) + client := &http.Client{ + Transport: &Transport{ + ExtraHeader: ta.header, + Base: ta.transport, + }, + // Ping should fail fast + Timeout: 5 * time.Second, + } + + resp, err := client.Do(req) if err != nil { return nil, err } @@ -171,9 +177,10 @@ func (ta *tokenAuthorizer) Authorize(req *http.Request) error { } type tokenHandler struct { - header http.Header - creds CredentialStore - scope TokenScope + header http.Header + creds CredentialStore + scope TokenScope + transport http.RoundTripper tokenLock sync.Mutex tokenCache string @@ -190,7 +197,7 @@ type TokenScope struct { // NewTokenHandler creates a new AuthenicationHandler which supports // fetching tokens from a remote token server. -func NewTokenHandler(creds CredentialStore, scope TokenScope, header http.Header) AuthenticationHandler { +func NewTokenHandler(transport http.RoundTripper, creds CredentialStore, scope TokenScope, header http.Header) AuthenticationHandler { return &tokenHandler{ header: header, creds: creds, @@ -203,8 +210,12 @@ func (ts TokenScope) String() string { } func (ts *tokenHandler) client() *http.Client { - // TODO(dmcgowan): Use same transport which has properly configured TLS - return &http.Client{Transport: &Transport{ExtraHeader: ts.header}} + return &http.Client{ + Transport: &Transport{ + ExtraHeader: ts.header, + Base: ts.transport, + }, + } } func (ts *tokenHandler) Scheme() string { diff --git a/registry/client/session_test.go b/registry/client/session_test.go index 87e1e66e..ee306cf6 100644 --- a/registry/client/session_test.go +++ b/registry/client/session_test.go @@ -117,7 +117,7 @@ func TestEndpointAuthorizeToken(t *testing.T) { defer c() repo1Config := &RepositoryConfig{ - AuthSource: NewTokenAuthorizer(nil, nil, tokenScope1), + AuthSource: NewTokenAuthorizer(nil, nil, nil, tokenScope1), } client, err := repo1Config.HTTPClient() @@ -142,7 +142,7 @@ func TestEndpointAuthorizeToken(t *testing.T) { defer c2() repo2Config := &RepositoryConfig{ - AuthSource: NewTokenAuthorizer(nil, nil, tokenScope2), + AuthSource: NewTokenAuthorizer(nil, nil, nil, tokenScope2), } client2, err := repo2Config.HTTPClient() if err != nil { @@ -221,7 +221,7 @@ func TestEndpointAuthorizeTokenBasic(t *testing.T) { password: password, } repoConfig := &RepositoryConfig{ - AuthSource: NewTokenAuthorizer(creds, nil, tokenScope), + AuthSource: NewTokenAuthorizer(creds, nil, nil, tokenScope), } client, err := repoConfig.HTTPClient() @@ -266,7 +266,7 @@ func TestEndpointAuthorizeBasic(t *testing.T) { password: password, } repoConfig := &RepositoryConfig{ - AuthSource: NewTokenAuthorizer(creds, nil, TokenScope{}), + AuthSource: NewTokenAuthorizer(creds, nil, nil, TokenScope{}), } client, err := repoConfig.HTTPClient() From 468c5e79bae559164cb9d8b6c63533123fb98a3b Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Thu, 14 May 2015 09:54:23 -0700 Subject: [PATCH 12/26] Simplify configuration and transport Repository creation now just takes in an http.RoundTripper. Authenticated requests or requests which require additional headers should use the NewTransport function along with a request modifier (such an an authentication handler). Signed-off-by: Derek McGowan (github: dmcgowan) --- registry/client/layer_upload_test.go | 7 +- registry/client/repository.go | 9 +- registry/client/repository_test.go | 18 ++-- registry/client/session.go | 121 ++++++++------------------- registry/client/session_test.go | 37 ++------ registry/client/transport.go | 57 +++++++++---- 6 files changed, 95 insertions(+), 154 deletions(-) diff --git a/registry/client/layer_upload_test.go b/registry/client/layer_upload_test.go index 9e22cb7c..3879c867 100644 --- a/registry/client/layer_upload_test.go +++ b/registry/client/layer_upload_test.go @@ -124,13 +124,8 @@ func TestUploadReadFrom(t *testing.T) { e, c := testServer(m) defer c() - repoConfig := &RepositoryConfig{} - client, err := repoConfig.HTTPClient() - if err != nil { - t.Fatalf("Error creating client: %s", err) - } layerUpload := &httpLayerUpload{ - client: client, + client: &http.Client{}, } // Valid case diff --git a/registry/client/repository.go b/registry/client/repository.go index e7fcfa9f..0bd89b11 100644 --- a/registry/client/repository.go +++ b/registry/client/repository.go @@ -20,7 +20,7 @@ import ( ) // NewRepository creates a new Repository for the given repository name and endpoint -func NewRepository(ctx context.Context, name, endpoint string, repoConfig *RepositoryConfig) (distribution.Repository, error) { +func NewRepository(ctx context.Context, name, endpoint string, transport http.RoundTripper) (distribution.Repository, error) { if err := v2.ValidateRespositoryName(name); err != nil { return nil, err } @@ -30,9 +30,10 @@ func NewRepository(ctx context.Context, name, endpoint string, repoConfig *Repos return nil, err } - client, err := repoConfig.HTTPClient() - if err != nil { - return nil, err + client := &http.Client{ + Transport: transport, + Timeout: 1 * time.Minute, + // TODO(dmcgowan): create cookie jar } return &repository{ diff --git a/registry/client/repository_test.go b/registry/client/repository_test.go index fe8ffeb7..650391c4 100644 --- a/registry/client/repository_test.go +++ b/registry/client/repository_test.go @@ -97,7 +97,7 @@ func TestLayerFetch(t *testing.T) { e, c := testServer(m) defer c() - r, err := NewRepository(context.Background(), "test.example.com/repo1", e, &RepositoryConfig{}) + r, err := NewRepository(context.Background(), "test.example.com/repo1", e, nil) if err != nil { t.Fatal(err) } @@ -127,7 +127,7 @@ func TestLayerExists(t *testing.T) { e, c := testServer(m) defer c() - r, err := NewRepository(context.Background(), "test.example.com/repo1", e, &RepositoryConfig{}) + r, err := NewRepository(context.Background(), "test.example.com/repo1", e, nil) if err != nil { t.Fatal(err) } @@ -227,7 +227,7 @@ func TestLayerUploadChunked(t *testing.T) { e, c := testServer(m) defer c() - r, err := NewRepository(context.Background(), repo, e, &RepositoryConfig{}) + r, err := NewRepository(context.Background(), repo, e, nil) if err != nil { t.Fatal(err) } @@ -334,7 +334,7 @@ func TestLayerUploadMonolithic(t *testing.T) { e, c := testServer(m) defer c() - r, err := NewRepository(context.Background(), repo, e, &RepositoryConfig{}) + r, err := NewRepository(context.Background(), repo, e, nil) if err != nil { t.Fatal(err) } @@ -475,7 +475,7 @@ func TestManifestFetch(t *testing.T) { e, c := testServer(m) defer c() - r, err := NewRepository(context.Background(), repo, e, &RepositoryConfig{}) + r, err := NewRepository(context.Background(), repo, e, nil) if err != nil { t.Fatal(err) } @@ -508,7 +508,7 @@ func TestManifestFetchByTag(t *testing.T) { e, c := testServer(m) defer c() - r, err := NewRepository(context.Background(), repo, e, &RepositoryConfig{}) + r, err := NewRepository(context.Background(), repo, e, nil) if err != nil { t.Fatal(err) } @@ -553,7 +553,7 @@ func TestManifestDelete(t *testing.T) { e, c := testServer(m) defer c() - r, err := NewRepository(context.Background(), repo, e, &RepositoryConfig{}) + r, err := NewRepository(context.Background(), repo, e, nil) if err != nil { t.Fatal(err) } @@ -591,7 +591,7 @@ func TestManifestPut(t *testing.T) { e, c := testServer(m) defer c() - r, err := NewRepository(context.Background(), repo, e, &RepositoryConfig{}) + r, err := NewRepository(context.Background(), repo, e, nil) if err != nil { t.Fatal(err) } @@ -636,7 +636,7 @@ func TestManifestTags(t *testing.T) { e, c := testServer(m) defer c() - r, err := NewRepository(context.Background(), repo, e, &RepositoryConfig{}) + r, err := NewRepository(context.Background(), repo, e, nil) if err != nil { t.Fatal(err) } diff --git a/registry/client/session.go b/registry/client/session.go index e4e92383..41bb4f31 100644 --- a/registry/client/session.go +++ b/registry/client/session.go @@ -11,12 +11,6 @@ import ( "time" ) -// Authorizer is used to apply Authorization to an HTTP request -type Authorizer interface { - // Authorizer updates an HTTP request with the needed authorization - Authorize(req *http.Request) error -} - // AuthenticationHandler is an interface for authorizing a request from // params from a "WWW-Authenicate" header for a single scheme. type AuthenticationHandler interface { @@ -31,54 +25,11 @@ type CredentialStore interface { Basic(*url.URL) (string, string) } -// RepositoryConfig holds the base configuration needed to communicate -// with a registry including a method of authorization and HTTP headers. -type RepositoryConfig struct { - Header http.Header - AuthSource Authorizer - - BaseTransport http.RoundTripper -} - -// HTTPClient returns a new HTTP client configured for this configuration -func (rc *RepositoryConfig) HTTPClient() (*http.Client, error) { - transport := &Transport{ - ExtraHeader: rc.Header, - AuthSource: rc.AuthSource, - Base: rc.BaseTransport, - } - - client := &http.Client{ - Transport: transport, - } - - return client, nil -} - -// NewTokenAuthorizer returns an authorizer which is capable of getting a token -// from a token server. The expected authorization method will be discovered -// by the authorizer, getting the token server endpoint from the URL being -// requested. Basic authentication may either be done to the token source or -// directly with the requested endpoint depending on the endpoint's -// WWW-Authenticate header. -func NewTokenAuthorizer(creds CredentialStore, transport http.RoundTripper, header http.Header, scope TokenScope) Authorizer { - return &tokenAuthorizer{ - header: header, - challenges: map[string]map[string]authorizationChallenge{}, - handlers: []AuthenticationHandler{ - NewTokenHandler(transport, creds, scope, header), - NewBasicHandler(creds), - }, - transport: transport, - } -} - // NewAuthorizer creates an authorizer which can handle multiple authentication // schemes. The handlers are tried in order, the higher priority authentication // methods should be first. -func NewAuthorizer(transport http.RoundTripper, header http.Header, handlers ...AuthenticationHandler) Authorizer { +func NewAuthorizer(transport http.RoundTripper, handlers ...AuthenticationHandler) RequestModifier { return &tokenAuthorizer{ - header: header, challenges: map[string]map[string]authorizationChallenge{}, handlers: handlers, transport: transport, @@ -86,7 +37,6 @@ func NewAuthorizer(transport http.RoundTripper, header http.Header, handlers ... } type tokenAuthorizer struct { - header http.Header challenges map[string]map[string]authorizationChallenge handlers []AuthenticationHandler transport http.RoundTripper @@ -99,10 +49,7 @@ func (ta *tokenAuthorizer) ping(endpoint string) (map[string]authorizationChalle } client := &http.Client{ - Transport: &Transport{ - ExtraHeader: ta.header, - Base: ta.transport, - }, + Transport: ta.transport, // Ping should fail fast Timeout: 5 * time.Second, } @@ -140,7 +87,7 @@ HeaderLoop: return nil, nil } -func (ta *tokenAuthorizer) Authorize(req *http.Request) error { +func (ta *tokenAuthorizer) ModifyRequest(req *http.Request) error { v2Root := strings.Index(req.URL.Path, "/v2/") if v2Root == -1 { return nil @@ -195,54 +142,52 @@ type TokenScope struct { Actions []string } -// NewTokenHandler creates a new AuthenicationHandler which supports -// fetching tokens from a remote token server. -func NewTokenHandler(transport http.RoundTripper, creds CredentialStore, scope TokenScope, header http.Header) AuthenticationHandler { - return &tokenHandler{ - header: header, - creds: creds, - scope: scope, - } -} - func (ts TokenScope) String() string { return fmt.Sprintf("%s:%s:%s", ts.Resource, ts.Scope, strings.Join(ts.Actions, ",")) } -func (ts *tokenHandler) client() *http.Client { - return &http.Client{ - Transport: &Transport{ - ExtraHeader: ts.header, - Base: ts.transport, - }, +// NewTokenHandler creates a new AuthenicationHandler which supports +// fetching tokens from a remote token server. +func NewTokenHandler(transport http.RoundTripper, creds CredentialStore, scope TokenScope) AuthenticationHandler { + return &tokenHandler{ + transport: transport, + creds: creds, + scope: scope, } } -func (ts *tokenHandler) Scheme() string { +func (th *tokenHandler) client() *http.Client { + return &http.Client{ + Transport: th.transport, + Timeout: 15 * time.Second, + } +} + +func (th *tokenHandler) Scheme() string { return "bearer" } -func (ts *tokenHandler) AuthorizeRequest(req *http.Request, params map[string]string) error { - if err := ts.refreshToken(params); err != nil { +func (th *tokenHandler) AuthorizeRequest(req *http.Request, params map[string]string) error { + if err := th.refreshToken(params); err != nil { return err } - req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", ts.tokenCache)) + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", th.tokenCache)) return nil } -func (ts *tokenHandler) refreshToken(params map[string]string) error { - ts.tokenLock.Lock() - defer ts.tokenLock.Unlock() +func (th *tokenHandler) refreshToken(params map[string]string) error { + th.tokenLock.Lock() + defer th.tokenLock.Unlock() now := time.Now() - if now.After(ts.tokenExpiration) { - token, err := ts.fetchToken(params) + if now.After(th.tokenExpiration) { + token, err := th.fetchToken(params) if err != nil { return err } - ts.tokenCache = token - ts.tokenExpiration = now.Add(time.Minute) + th.tokenCache = token + th.tokenExpiration = now.Add(time.Minute) } return nil @@ -252,7 +197,7 @@ type tokenResponse struct { Token string `json:"token"` } -func (ts *tokenHandler) fetchToken(params map[string]string) (token string, err error) { +func (th *tokenHandler) fetchToken(params map[string]string) (token string, err error) { //log.Debugf("Getting bearer token with %s for %s", challenge.Parameters, ta.auth.Username) realm, ok := params["realm"] if !ok { @@ -273,7 +218,7 @@ func (ts *tokenHandler) fetchToken(params map[string]string) (token string, err reqParams := req.URL.Query() service := params["service"] - scope := ts.scope.String() + scope := th.scope.String() if service != "" { reqParams.Add("service", service) @@ -283,8 +228,8 @@ func (ts *tokenHandler) fetchToken(params map[string]string) (token string, err reqParams.Add("scope", scopeField) } - if ts.creds != nil { - username, password := ts.creds.Basic(realmURL) + if th.creds != nil { + username, password := th.creds.Basic(realmURL) if username != "" && password != "" { reqParams.Add("account", username) req.SetBasicAuth(username, password) @@ -293,7 +238,7 @@ func (ts *tokenHandler) fetchToken(params map[string]string) (token string, err req.URL.RawQuery = reqParams.Encode() - resp, err := ts.client().Do(req) + resp, err := th.client().Do(req) if err != nil { return "", err } diff --git a/registry/client/session_test.go b/registry/client/session_test.go index ee306cf6..cf8e546e 100644 --- a/registry/client/session_test.go +++ b/registry/client/session_test.go @@ -116,14 +116,8 @@ func TestEndpointAuthorizeToken(t *testing.T) { e, c := testServerWithAuth(m, authenicate, validCheck) defer c() - repo1Config := &RepositoryConfig{ - AuthSource: NewTokenAuthorizer(nil, nil, nil, tokenScope1), - } - - client, err := repo1Config.HTTPClient() - if err != nil { - t.Fatalf("Error creating http client: %s", err) - } + transport1 := NewTransport(nil, NewAuthorizer(nil, NewTokenHandler(nil, nil, tokenScope1))) + client := &http.Client{Transport: transport1} req, _ := http.NewRequest("GET", e+"/v2/hello", nil) resp, err := client.Do(req) @@ -141,13 +135,8 @@ func TestEndpointAuthorizeToken(t *testing.T) { e2, c2 := testServerWithAuth(m, authenicate, badCheck) defer c2() - repo2Config := &RepositoryConfig{ - AuthSource: NewTokenAuthorizer(nil, nil, nil, tokenScope2), - } - client2, err := repo2Config.HTTPClient() - if err != nil { - t.Fatalf("Error creating http client: %s", err) - } + transport2 := NewTransport(nil, NewAuthorizer(nil, NewTokenHandler(nil, nil, tokenScope2))) + client2 := &http.Client{Transport: transport2} req, _ = http.NewRequest("GET", e2+"/v2/hello", nil) resp, err = client2.Do(req) @@ -220,14 +209,9 @@ func TestEndpointAuthorizeTokenBasic(t *testing.T) { username: username, password: password, } - repoConfig := &RepositoryConfig{ - AuthSource: NewTokenAuthorizer(creds, nil, nil, tokenScope), - } - client, err := repoConfig.HTTPClient() - if err != nil { - t.Fatalf("Error creating http client: %s", err) - } + transport1 := NewTransport(nil, NewAuthorizer(nil, NewTokenHandler(nil, creds, tokenScope), NewBasicHandler(creds))) + client := &http.Client{Transport: transport1} req, _ := http.NewRequest("GET", e+"/v2/hello", nil) resp, err := client.Do(req) @@ -265,14 +249,9 @@ func TestEndpointAuthorizeBasic(t *testing.T) { username: username, password: password, } - repoConfig := &RepositoryConfig{ - AuthSource: NewTokenAuthorizer(creds, nil, nil, TokenScope{}), - } - client, err := repoConfig.HTTPClient() - if err != nil { - t.Fatalf("Error creating http client: %s", err) - } + transport1 := NewTransport(nil, NewAuthorizer(nil, NewBasicHandler(creds))) + client := &http.Client{Transport: transport1} req, _ := http.NewRequest("GET", e+"/v2/hello", nil) resp, err := client.Do(req) diff --git a/registry/client/transport.go b/registry/client/transport.go index e92ba543..0b241619 100644 --- a/registry/client/transport.go +++ b/registry/client/transport.go @@ -6,14 +6,36 @@ import ( "sync" ) -// Transport is an http.RoundTripper that makes registry HTTP requests, -// wrapping a base RoundTripper and adding an Authorization header -// from an Auth source -type Transport struct { - AuthSource Authorizer - ExtraHeader http.Header +type RequestModifier interface { + ModifyRequest(*http.Request) error +} - Base http.RoundTripper +type headerModifier http.Header + +func NewHeaderRequestModifier(header http.Header) RequestModifier { + return headerModifier(header) +} + +func (h headerModifier) ModifyRequest(req *http.Request) error { + for k, s := range http.Header(h) { + req.Header[k] = append(req.Header[k], s...) + } + + return nil +} + +func NewTransport(base http.RoundTripper, modifiers ...RequestModifier) http.RoundTripper { + return &transport{ + Modifiers: modifiers, + Base: base, + } +} + +// transport is an http.RoundTripper that makes HTTP requests after +// copying and modifying the request +type transport struct { + Modifiers []RequestModifier + Base http.RoundTripper mu sync.Mutex // guards modReq modReq map[*http.Request]*http.Request // original -> modified @@ -22,13 +44,14 @@ type Transport struct { // RoundTrip authorizes and authenticates the request with an // access token. If no token exists or token is expired, // tries to refresh/fetch a new token. -func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) { - req2 := t.cloneRequest(req) - if t.AuthSource != nil { - if err := t.AuthSource.Authorize(req2); err != nil { +func (t *transport) RoundTrip(req *http.Request) (*http.Response, error) { + req2 := cloneRequest(req) + for _, modifier := range t.Modifiers { + if err := modifier.ModifyRequest(req2); err != nil { return nil, err } } + t.setModReq(req, req2) res, err := t.base().RoundTrip(req2) if err != nil { @@ -43,7 +66,7 @@ func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) { } // CancelRequest cancels an in-flight request by closing its connection. -func (t *Transport) CancelRequest(req *http.Request) { +func (t *transport) CancelRequest(req *http.Request) { type canceler interface { CancelRequest(*http.Request) } @@ -56,14 +79,14 @@ func (t *Transport) CancelRequest(req *http.Request) { } } -func (t *Transport) base() http.RoundTripper { +func (t *transport) base() http.RoundTripper { if t.Base != nil { return t.Base } return http.DefaultTransport } -func (t *Transport) setModReq(orig, mod *http.Request) { +func (t *transport) setModReq(orig, mod *http.Request) { t.mu.Lock() defer t.mu.Unlock() if t.modReq == nil { @@ -78,7 +101,7 @@ func (t *Transport) setModReq(orig, mod *http.Request) { // cloneRequest returns a clone of the provided *http.Request. // The clone is a shallow copy of the struct and its Header map. -func (t *Transport) cloneRequest(r *http.Request) *http.Request { +func cloneRequest(r *http.Request) *http.Request { // shallow copy of the struct r2 := new(http.Request) *r2 = *r @@ -87,9 +110,7 @@ func (t *Transport) cloneRequest(r *http.Request) *http.Request { for k, s := range r.Header { r2.Header[k] = append([]string(nil), s...) } - for k, s := range t.ExtraHeader { - r2.Header[k] = append(r2.Header[k], s...) - } + return r2 } From fddeb1c8d5816edcf9b6503a5e69af1bd55a7383 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Thu, 14 May 2015 10:18:21 -0700 Subject: [PATCH 13/26] Add missing defer on Tags Signed-off-by: Derek McGowan (github: dmcgowan) --- registry/client/repository.go | 1 + 1 file changed, 1 insertion(+) diff --git a/registry/client/repository.go b/registry/client/repository.go index 0bd89b11..4055577d 100644 --- a/registry/client/repository.go +++ b/registry/client/repository.go @@ -99,6 +99,7 @@ func (ms *manifests) Tags() ([]string, error) { if err != nil { return nil, err } + defer resp.Body.Close() switch { case resp.StatusCode == http.StatusOK: From 9d64e461bea872ea073ef393bc279503d81415ba Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Fri, 15 May 2015 13:29:44 -0700 Subject: [PATCH 14/26] Update to use blob interfaces Signed-off-by: Derek McGowan (github: dmcgowan) --- registry/client/layer.go | 113 +++++++++++---------------- registry/client/layer_upload.go | 72 ++++++++--------- registry/client/layer_upload_test.go | 30 +++---- registry/client/repository.go | 103 ++++++++++++++---------- registry/client/repository_test.go | 85 +++++++++++--------- 5 files changed, 204 insertions(+), 199 deletions(-) diff --git a/registry/client/layer.go b/registry/client/layer.go index b6e1697d..e7c0039c 100644 --- a/registry/client/layer.go +++ b/registry/client/layer.go @@ -8,18 +8,15 @@ import ( "io/ioutil" "net/http" "os" - "time" + "github.com/docker/distribution" "github.com/docker/distribution/context" - "github.com/docker/distribution/digest" ) -type httpLayer struct { - *layers +type httpBlob struct { + *repository - size int64 - digest digest.Digest - createdAt time.Time + desc distribution.Descriptor rc io.ReadCloser // remote read closer brd *bufio.Reader // internal buffered io @@ -27,48 +24,40 @@ type httpLayer struct { err error } -func (hl *httpLayer) CreatedAt() time.Time { - return hl.createdAt -} - -func (hl *httpLayer) Digest() digest.Digest { - return hl.digest -} - -func (hl *httpLayer) Read(p []byte) (n int, err error) { - if hl.err != nil { - return 0, hl.err +func (hb *httpBlob) Read(p []byte) (n int, err error) { + if hb.err != nil { + return 0, hb.err } - rd, err := hl.reader() + rd, err := hb.reader() if err != nil { return 0, err } n, err = rd.Read(p) - hl.offset += int64(n) + hb.offset += int64(n) // Simulate io.EOF error if we reach filesize. - if err == nil && hl.offset >= hl.size { + if err == nil && hb.offset >= hb.desc.Length { err = io.EOF } return n, err } -func (hl *httpLayer) Seek(offset int64, whence int) (int64, error) { - if hl.err != nil { - return 0, hl.err +func (hb *httpBlob) Seek(offset int64, whence int) (int64, error) { + if hb.err != nil { + return 0, hb.err } var err error - newOffset := hl.offset + newOffset := hb.offset switch whence { case os.SEEK_CUR: newOffset += int64(offset) case os.SEEK_END: - newOffset = hl.size + int64(offset) + newOffset = hb.desc.Length + int64(offset) case os.SEEK_SET: newOffset = int64(offset) } @@ -76,60 +65,60 @@ func (hl *httpLayer) Seek(offset int64, whence int) (int64, error) { if newOffset < 0 { err = fmt.Errorf("cannot seek to negative position") } else { - if hl.offset != newOffset { - hl.reset() + if hb.offset != newOffset { + hb.reset() } // No problems, set the offset. - hl.offset = newOffset + hb.offset = newOffset } - return hl.offset, err + return hb.offset, err } -func (hl *httpLayer) Close() error { - if hl.err != nil { - return hl.err +func (hb *httpBlob) Close() error { + if hb.err != nil { + return hb.err } // close and release reader chain - if hl.rc != nil { - hl.rc.Close() + if hb.rc != nil { + hb.rc.Close() } - hl.rc = nil - hl.brd = nil + hb.rc = nil + hb.brd = nil - hl.err = fmt.Errorf("httpLayer: closed") + hb.err = fmt.Errorf("httpBlob: closed") return nil } -func (hl *httpLayer) reset() { - if hl.err != nil { +func (hb *httpBlob) reset() { + if hb.err != nil { return } - if hl.rc != nil { - hl.rc.Close() - hl.rc = nil + if hb.rc != nil { + hb.rc.Close() + hb.rc = nil } } -func (hl *httpLayer) reader() (io.Reader, error) { - if hl.err != nil { - return nil, hl.err +func (hb *httpBlob) reader() (io.Reader, error) { + if hb.err != nil { + return nil, hb.err } - if hl.rc != nil { - return hl.brd, nil + if hb.rc != nil { + return hb.brd, nil } // If the offset is great than or equal to size, return a empty, noop reader. - if hl.offset >= hl.size { + if hb.offset >= hb.desc.Length { return ioutil.NopCloser(bytes.NewReader([]byte{})), nil } - blobURL, err := hl.ub.BuildBlobURL(hl.name, hl.digest) + blobURL, err := hb.ub.BuildBlobURL(hb.name, hb.desc.Digest) if err != nil { return nil, err } @@ -139,40 +128,32 @@ func (hl *httpLayer) reader() (io.Reader, error) { return nil, err } - if hl.offset > 0 { + if hb.offset > 0 { // TODO(stevvooe): Get this working correctly. // If we are at different offset, issue a range request from there. req.Header.Add("Range", fmt.Sprintf("1-")) - context.GetLogger(hl.context).Infof("Range: %s", req.Header.Get("Range")) + context.GetLogger(hb.context).Infof("Range: %s", req.Header.Get("Range")) } - resp, err := hl.client.Do(req) + resp, err := hb.client.Do(req) if err != nil { return nil, err } switch { case resp.StatusCode == 200: - hl.rc = resp.Body + hb.rc = resp.Body default: defer resp.Body.Close() return nil, fmt.Errorf("unexpected status resolving reader: %v", resp.Status) } - if hl.brd == nil { - hl.brd = bufio.NewReader(hl.rc) + if hb.brd == nil { + hb.brd = bufio.NewReader(hb.rc) } else { - hl.brd.Reset(hl.rc) + hb.brd.Reset(hb.rc) } - return hl.brd, nil -} - -func (hl *httpLayer) Length() int64 { - return hl.size -} - -func (hl *httpLayer) Handler(r *http.Request) (http.Handler, error) { - panic("Not implemented") + return hb.brd, nil } diff --git a/registry/client/layer_upload.go b/registry/client/layer_upload.go index 18e5fbab..3697ef8c 100644 --- a/registry/client/layer_upload.go +++ b/registry/client/layer_upload.go @@ -11,10 +11,10 @@ import ( "time" "github.com/docker/distribution" - "github.com/docker/distribution/digest" + "github.com/docker/distribution/context" ) -type httpLayerUpload struct { +type httpBlobUpload struct { repo distribution.Repository client *http.Client @@ -26,32 +26,32 @@ type httpLayerUpload struct { closed bool } -func (hlu *httpLayerUpload) handleErrorResponse(resp *http.Response) error { +func (hbu *httpBlobUpload) handleErrorResponse(resp *http.Response) error { if resp.StatusCode == http.StatusNotFound { - return &BlobUploadNotFoundError{Location: hlu.location} + return &BlobUploadNotFoundError{Location: hbu.location} } return handleErrorResponse(resp) } -func (hlu *httpLayerUpload) ReadFrom(r io.Reader) (n int64, err error) { - req, err := http.NewRequest("PATCH", hlu.location, ioutil.NopCloser(r)) +func (hbu *httpBlobUpload) ReadFrom(r io.Reader) (n int64, err error) { + req, err := http.NewRequest("PATCH", hbu.location, ioutil.NopCloser(r)) if err != nil { return 0, err } defer req.Body.Close() - resp, err := hlu.client.Do(req) + resp, err := hbu.client.Do(req) if err != nil { return 0, err } if resp.StatusCode != http.StatusAccepted { - return 0, hlu.handleErrorResponse(resp) + return 0, hbu.handleErrorResponse(resp) } // TODO(dmcgowan): Validate headers - hlu.uuid = resp.Header.Get("Docker-Upload-UUID") - hlu.location, err = sanitizeLocation(resp.Header.Get("Location"), hlu.location) + hbu.uuid = resp.Header.Get("Docker-Upload-UUID") + hbu.location, err = sanitizeLocation(resp.Header.Get("Location"), hbu.location) if err != nil { return 0, err } @@ -67,27 +67,27 @@ func (hlu *httpLayerUpload) ReadFrom(r io.Reader) (n int64, err error) { } -func (hlu *httpLayerUpload) Write(p []byte) (n int, err error) { - req, err := http.NewRequest("PATCH", hlu.location, bytes.NewReader(p)) +func (hbu *httpBlobUpload) Write(p []byte) (n int, err error) { + req, err := http.NewRequest("PATCH", hbu.location, bytes.NewReader(p)) if err != nil { return 0, err } - req.Header.Set("Content-Range", fmt.Sprintf("%d-%d", hlu.offset, hlu.offset+int64(len(p)-1))) + req.Header.Set("Content-Range", fmt.Sprintf("%d-%d", hbu.offset, hbu.offset+int64(len(p)-1))) req.Header.Set("Content-Length", fmt.Sprintf("%d", len(p))) req.Header.Set("Content-Type", "application/octet-stream") - resp, err := hlu.client.Do(req) + resp, err := hbu.client.Do(req) if err != nil { return 0, err } if resp.StatusCode != http.StatusAccepted { - return 0, hlu.handleErrorResponse(resp) + return 0, hbu.handleErrorResponse(resp) } // TODO(dmcgowan): Validate headers - hlu.uuid = resp.Header.Get("Docker-Upload-UUID") - hlu.location, err = sanitizeLocation(resp.Header.Get("Location"), hlu.location) + hbu.uuid = resp.Header.Get("Docker-Upload-UUID") + hbu.location, err = sanitizeLocation(resp.Header.Get("Location"), hbu.location) if err != nil { return 0, err } @@ -103,8 +103,8 @@ func (hlu *httpLayerUpload) Write(p []byte) (n int, err error) { } -func (hlu *httpLayerUpload) Seek(offset int64, whence int) (int64, error) { - newOffset := hlu.offset +func (hbu *httpBlobUpload) Seek(offset int64, whence int) (int64, error) { + newOffset := hbu.offset switch whence { case os.SEEK_CUR: @@ -115,47 +115,47 @@ func (hlu *httpLayerUpload) Seek(offset int64, whence int) (int64, error) { newOffset = int64(offset) } - hlu.offset = newOffset + hbu.offset = newOffset - return hlu.offset, nil + return hbu.offset, nil } -func (hlu *httpLayerUpload) UUID() string { - return hlu.uuid +func (hbu *httpBlobUpload) ID() string { + return hbu.uuid } -func (hlu *httpLayerUpload) StartedAt() time.Time { - return hlu.startedAt +func (hbu *httpBlobUpload) StartedAt() time.Time { + return hbu.startedAt } -func (hlu *httpLayerUpload) Finish(digest digest.Digest) (distribution.Layer, error) { +func (hbu *httpBlobUpload) Commit(ctx context.Context, desc distribution.Descriptor) (distribution.Descriptor, error) { // TODO(dmcgowan): Check if already finished, if so just fetch - req, err := http.NewRequest("PUT", hlu.location, nil) + req, err := http.NewRequest("PUT", hbu.location, nil) if err != nil { - return nil, err + return distribution.Descriptor{}, err } values := req.URL.Query() - values.Set("digest", digest.String()) + values.Set("digest", desc.Digest.String()) req.URL.RawQuery = values.Encode() - resp, err := hlu.client.Do(req) + resp, err := hbu.client.Do(req) if err != nil { - return nil, err + return distribution.Descriptor{}, err } if resp.StatusCode != http.StatusCreated { - return nil, hlu.handleErrorResponse(resp) + return distribution.Descriptor{}, hbu.handleErrorResponse(resp) } - return hlu.repo.Layers().Fetch(digest) + return hbu.repo.Blobs(ctx).Stat(ctx, desc.Digest) } -func (hlu *httpLayerUpload) Cancel() error { +func (hbu *httpBlobUpload) Rollback(ctx context.Context) error { panic("not implemented") } -func (hlu *httpLayerUpload) Close() error { - hlu.closed = true +func (hbu *httpBlobUpload) Close() error { + hbu.closed = true return nil } diff --git a/registry/client/layer_upload_test.go b/registry/client/layer_upload_test.go index 3879c867..2e4edc45 100644 --- a/registry/client/layer_upload_test.go +++ b/registry/client/layer_upload_test.go @@ -11,8 +11,8 @@ import ( "github.com/docker/distribution/testutil" ) -// Test implements distribution.LayerUpload -var _ distribution.LayerUpload = &httpLayerUpload{} +// Test implements distribution.BlobWriter +var _ distribution.BlobWriter = &httpBlobUpload{} func TestUploadReadFrom(t *testing.T) { _, b := newRandomBlob(64) @@ -124,13 +124,13 @@ func TestUploadReadFrom(t *testing.T) { e, c := testServer(m) defer c() - layerUpload := &httpLayerUpload{ + blobUpload := &httpBlobUpload{ client: &http.Client{}, } // Valid case - layerUpload.location = e + locationPath - n, err := layerUpload.ReadFrom(bytes.NewReader(b)) + blobUpload.location = e + locationPath + n, err := blobUpload.ReadFrom(bytes.NewReader(b)) if err != nil { t.Fatalf("Error calling ReadFrom: %s", err) } @@ -139,15 +139,15 @@ func TestUploadReadFrom(t *testing.T) { } // Bad range - layerUpload.location = e + locationPath - _, err = layerUpload.ReadFrom(bytes.NewReader(b)) + blobUpload.location = e + locationPath + _, err = blobUpload.ReadFrom(bytes.NewReader(b)) if err == nil { t.Fatalf("Expected error when bad range received") } // 404 - layerUpload.location = e + locationPath - _, err = layerUpload.ReadFrom(bytes.NewReader(b)) + blobUpload.location = e + locationPath + _, err = blobUpload.ReadFrom(bytes.NewReader(b)) if err == nil { t.Fatalf("Expected error when not found") } @@ -158,8 +158,8 @@ func TestUploadReadFrom(t *testing.T) { } // 400 valid json - layerUpload.location = e + locationPath - _, err = layerUpload.ReadFrom(bytes.NewReader(b)) + blobUpload.location = e + locationPath + _, err = blobUpload.ReadFrom(bytes.NewReader(b)) if err == nil { t.Fatalf("Expected error when not found") } @@ -181,8 +181,8 @@ func TestUploadReadFrom(t *testing.T) { } // 400 invalid json - layerUpload.location = e + locationPath - _, err = layerUpload.ReadFrom(bytes.NewReader(b)) + blobUpload.location = e + locationPath + _, err = blobUpload.ReadFrom(bytes.NewReader(b)) if err == nil { t.Fatalf("Expected error when not found") } @@ -196,8 +196,8 @@ func TestUploadReadFrom(t *testing.T) { } // 500 - layerUpload.location = e + locationPath - _, err = layerUpload.ReadFrom(bytes.NewReader(b)) + blobUpload.location = e + locationPath + _, err = blobUpload.ReadFrom(bytes.NewReader(b)) if err == nil { t.Fatalf("Expected error when not found") } diff --git a/registry/client/repository.go b/registry/client/repository.go index 4055577d..940ae1df 100644 --- a/registry/client/repository.go +++ b/registry/client/repository.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "fmt" + "io" "io/ioutil" "net/http" "net/url" @@ -55,8 +56,8 @@ func (r *repository) Name() string { return r.name } -func (r *repository) Layers() distribution.LayerService { - return &layers{ +func (r *repository) Blobs(ctx context.Context) distribution.BlobService { + return &blobs{ repository: r, } } @@ -229,7 +230,7 @@ func (ms *manifests) Delete(dgst digest.Digest) error { } } -type layers struct { +type blobs struct { *repository } @@ -254,25 +255,55 @@ func sanitizeLocation(location, source string) (string, error) { return location, nil } -func (ls *layers) Exists(dgst digest.Digest) (bool, error) { - _, err := ls.fetchLayer(dgst) +func (ls *blobs) Get(ctx context.Context, dgst digest.Digest) ([]byte, error) { + desc, err := ls.Stat(ctx, dgst) if err != nil { - switch err := err.(type) { - case distribution.ErrUnknownLayer: - return false, nil - default: - return false, err - } + return nil, err + } + reader, err := ls.Open(ctx, desc) + if err != nil { + return nil, err + } + defer reader.Close() + + return ioutil.ReadAll(reader) +} + +func (ls *blobs) Open(ctx context.Context, desc distribution.Descriptor) (distribution.ReadSeekCloser, error) { + return &httpBlob{ + repository: ls.repository, + desc: desc, + }, nil +} + +func (ls *blobs) ServeBlob(ctx context.Context, w http.ResponseWriter, r *http.Request, desc distribution.Descriptor) error { + return nil +} + +func (ls *blobs) Put(ctx context.Context, mediaType string, p []byte) (distribution.Descriptor, error) { + writer, err := ls.Writer(ctx) + if err != nil { + return distribution.Descriptor{}, err + } + dgstr := digest.NewCanonicalDigester() + n, err := io.Copy(writer, io.TeeReader(bytes.NewReader(p), dgstr)) + if err != nil { + return distribution.Descriptor{}, err + } + if n < int64(len(p)) { + return distribution.Descriptor{}, fmt.Errorf("short copy: wrote %d of %d", n, len(p)) } - return true, nil + desc := distribution.Descriptor{ + MediaType: mediaType, + Length: int64(len(p)), + Digest: dgstr.Digest(), + } + + return writer.Commit(ctx, desc) } -func (ls *layers) Fetch(dgst digest.Digest) (distribution.Layer, error) { - return ls.fetchLayer(dgst) -} - -func (ls *layers) Upload() (distribution.LayerUpload, error) { +func (ls *blobs) Writer(ctx context.Context) (distribution.BlobWriter, error) { u, err := ls.ub.BuildBlobUploadURL(ls.name) resp, err := ls.client.Post(u, "", nil) @@ -290,7 +321,7 @@ func (ls *layers) Upload() (distribution.LayerUpload, error) { return nil, err } - return &httpLayerUpload{ + return &httpBlobUpload{ repo: ls.repository, client: ls.client, uuid: uuid, @@ -302,19 +333,19 @@ func (ls *layers) Upload() (distribution.LayerUpload, error) { } } -func (ls *layers) Resume(uuid string) (distribution.LayerUpload, error) { +func (ls *blobs) Resume(ctx context.Context, id string) (distribution.BlobWriter, error) { panic("not implemented") } -func (ls *layers) fetchLayer(dgst digest.Digest) (distribution.Layer, error) { +func (ls *blobs) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) { u, err := ls.ub.BuildBlobURL(ls.name, dgst) if err != nil { - return nil, err + return distribution.Descriptor{}, err } resp, err := ls.client.Head(u) if err != nil { - return nil, err + return distribution.Descriptor{}, err } defer resp.Body.Close() @@ -323,31 +354,17 @@ func (ls *layers) fetchLayer(dgst digest.Digest) (distribution.Layer, error) { lengthHeader := resp.Header.Get("Content-Length") length, err := strconv.ParseInt(lengthHeader, 10, 64) if err != nil { - return nil, fmt.Errorf("error parsing content-length: %v", err) + return distribution.Descriptor{}, fmt.Errorf("error parsing content-length: %v", err) } - var t time.Time - lastModified := resp.Header.Get("Last-Modified") - if lastModified != "" { - t, err = http.ParseTime(lastModified) - if err != nil { - return nil, fmt.Errorf("error parsing last-modified: %v", err) - } - } - - return &httpLayer{ - layers: ls, - size: length, - digest: dgst, - createdAt: t, + return distribution.Descriptor{ + MediaType: resp.Header.Get("Content-Type"), + Length: length, + Digest: dgst, }, nil case resp.StatusCode == http.StatusNotFound: - return nil, distribution.ErrUnknownLayer{ - FSLayer: manifest.FSLayer{ - BlobSum: dgst, - }, - } + return distribution.Descriptor{}, distribution.ErrBlobUnknown default: - return nil, handleErrorResponse(resp) + return distribution.Descriptor{}, handleErrorResponse(resp) } } diff --git a/registry/client/repository_test.go b/registry/client/repository_test.go index 650391c4..514f3ee2 100644 --- a/registry/client/repository_test.go +++ b/registry/client/repository_test.go @@ -5,7 +5,6 @@ import ( "crypto/rand" "encoding/json" "fmt" - "io/ioutil" "log" "net/http" "net/http/httptest" @@ -15,6 +14,7 @@ import ( "code.google.com/p/go-uuid/uuid" + "github.com/docker/distribution" "github.com/docker/distribution/context" "github.com/docker/distribution/digest" "github.com/docker/distribution/manifest" @@ -88,7 +88,7 @@ func addPing(m *testutil.RequestResponseMap) { }) } -func TestLayerFetch(t *testing.T) { +func TestBlobFetch(t *testing.T) { d1, b1 := newRandomBlob(1024) var m testutil.RequestResponseMap addTestFetch("test.example.com/repo1", d1, b1, &m) @@ -97,17 +97,14 @@ func TestLayerFetch(t *testing.T) { e, c := testServer(m) defer c() - r, err := NewRepository(context.Background(), "test.example.com/repo1", e, nil) + ctx := context.Background() + r, err := NewRepository(ctx, "test.example.com/repo1", e, nil) if err != nil { t.Fatal(err) } - l := r.Layers() + l := r.Blobs(ctx) - layer, err := l.Fetch(d1) - if err != nil { - t.Fatal(err) - } - b, err := ioutil.ReadAll(layer) + b, err := l.Get(ctx, d1) if err != nil { t.Fatal(err) } @@ -118,7 +115,7 @@ func TestLayerFetch(t *testing.T) { // TODO(dmcgowan): Test error cases } -func TestLayerExists(t *testing.T) { +func TestBlobExists(t *testing.T) { d1, b1 := newRandomBlob(1024) var m testutil.RequestResponseMap addTestFetch("test.example.com/repo1", d1, b1, &m) @@ -127,24 +124,30 @@ func TestLayerExists(t *testing.T) { e, c := testServer(m) defer c() - r, err := NewRepository(context.Background(), "test.example.com/repo1", e, nil) + ctx := context.Background() + r, err := NewRepository(ctx, "test.example.com/repo1", e, nil) if err != nil { t.Fatal(err) } - l := r.Layers() + l := r.Blobs(ctx) - ok, err := l.Exists(d1) + stat, err := l.Stat(ctx, d1) if err != nil { t.Fatal(err) } - if !ok { - t.Fatalf("Blob does not exist: %s", d1) + + if stat.Digest != d1 { + t.Fatalf("Unexpected digest: %s, expected %s", stat.Digest, d1) } - // TODO(dmcgowan): Test error cases + if stat.Length != int64(len(b1)) { + t.Fatalf("Unexpected length: %d, expected %d", stat.Length, len(b1)) + } + + // TODO(dmcgowan): Test error cases and ErrBlobUnknown case } -func TestLayerUploadChunked(t *testing.T) { +func TestBlobUploadChunked(t *testing.T) { dgst, b1 := newRandomBlob(1024) var m testutil.RequestResponseMap addPing(&m) @@ -227,19 +230,20 @@ func TestLayerUploadChunked(t *testing.T) { e, c := testServer(m) defer c() - r, err := NewRepository(context.Background(), repo, e, nil) + ctx := context.Background() + r, err := NewRepository(ctx, repo, e, nil) if err != nil { t.Fatal(err) } - l := r.Layers() + l := r.Blobs(ctx) - upload, err := l.Upload() + upload, err := l.Writer(ctx) if err != nil { t.Fatal(err) } - if upload.UUID() != uuids[0] { - log.Fatalf("Unexpected UUID %s; expected %s", upload.UUID(), uuids[0]) + if upload.ID() != uuids[0] { + log.Fatalf("Unexpected UUID %s; expected %s", upload.ID(), uuids[0]) } for _, chunk := range chunks { @@ -252,17 +256,20 @@ func TestLayerUploadChunked(t *testing.T) { } } - layer, err := upload.Finish(dgst) + blob, err := upload.Commit(ctx, distribution.Descriptor{ + Digest: dgst, + Length: int64(len(b1)), + }) if err != nil { t.Fatal(err) } - if layer.Length() != int64(len(b1)) { - t.Fatalf("Unexpected layer size: %d; expected: %d", layer.Length(), len(b1)) + if blob.Length != int64(len(b1)) { + t.Fatalf("Unexpected blob size: %d; expected: %d", blob.Length, len(b1)) } } -func TestLayerUploadMonolithic(t *testing.T) { +func TestBlobUploadMonolithic(t *testing.T) { dgst, b1 := newRandomBlob(1024) var m testutil.RequestResponseMap addPing(&m) @@ -334,19 +341,20 @@ func TestLayerUploadMonolithic(t *testing.T) { e, c := testServer(m) defer c() - r, err := NewRepository(context.Background(), repo, e, nil) + ctx := context.Background() + r, err := NewRepository(ctx, repo, e, nil) if err != nil { t.Fatal(err) } - l := r.Layers() + l := r.Blobs(ctx) - upload, err := l.Upload() + upload, err := l.Writer(ctx) if err != nil { t.Fatal(err) } - if upload.UUID() != uploadID { - log.Fatalf("Unexpected UUID %s; expected %s", upload.UUID(), uploadID) + if upload.ID() != uploadID { + log.Fatalf("Unexpected UUID %s; expected %s", upload.ID(), uploadID) } n, err := upload.ReadFrom(bytes.NewReader(b1)) @@ -357,20 +365,19 @@ func TestLayerUploadMonolithic(t *testing.T) { t.Fatalf("Unexpected ReadFrom length: %d; expected: %d", n, len(b1)) } - layer, err := upload.Finish(dgst) + blob, err := upload.Commit(ctx, distribution.Descriptor{ + Digest: dgst, + Length: int64(len(b1)), + }) if err != nil { t.Fatal(err) } - if layer.Length() != int64(len(b1)) { - t.Fatalf("Unexpected layer size: %d; expected: %d", layer.Length(), len(b1)) + if blob.Length != int64(len(b1)) { + t.Fatalf("Unexpected blob size: %d; expected: %d", blob.Length, len(b1)) } } -func TestLayerUploadResume(t *testing.T) { - // TODO(dmcgowan): implement -} - func newRandomSchema1Manifest(name, tag string, blobCount int) (*manifest.SignedManifest, digest.Digest) { blobs := make([]manifest.FSLayer, blobCount) history := make([]manifest.History, blobCount) @@ -447,7 +454,7 @@ func checkEqualManifest(m1, m2 *manifest.SignedManifest) error { return fmt.Errorf("tag does not match %q != %q", m1.Tag, m2.Tag) } if len(m1.FSLayers) != len(m2.FSLayers) { - return fmt.Errorf("fs layer length does not match %d != %d", len(m1.FSLayers), len(m2.FSLayers)) + return fmt.Errorf("fs blob length does not match %d != %d", len(m1.FSLayers), len(m2.FSLayers)) } for i := range m1.FSLayers { if m1.FSLayers[i].BlobSum != m2.FSLayers[i].BlobSum { From 70074b2286156d22b7c15b6208fbcdafe993b5d1 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Fri, 15 May 2015 13:31:28 -0700 Subject: [PATCH 15/26] Rename layer files to blob Signed-off-by: Derek McGowan (github: dmcgowan) --- registry/client/{layer.go => blob.go} | 0 registry/client/{layer_upload.go => blob_writer.go} | 0 registry/client/{layer_upload_test.go => blob_writer_test.go} | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename registry/client/{layer.go => blob.go} (100%) rename registry/client/{layer_upload.go => blob_writer.go} (100%) rename registry/client/{layer_upload_test.go => blob_writer_test.go} (100%) diff --git a/registry/client/layer.go b/registry/client/blob.go similarity index 100% rename from registry/client/layer.go rename to registry/client/blob.go diff --git a/registry/client/layer_upload.go b/registry/client/blob_writer.go similarity index 100% rename from registry/client/layer_upload.go rename to registry/client/blob_writer.go diff --git a/registry/client/layer_upload_test.go b/registry/client/blob_writer_test.go similarity index 100% rename from registry/client/layer_upload_test.go rename to registry/client/blob_writer_test.go From fdf7c8ff158400718404dc57e2dbb36459e04e55 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Fri, 15 May 2015 15:54:04 -0700 Subject: [PATCH 16/26] Open cache interface Signed-off-by: Derek McGowan (github: dmcgowan) --- registry/storage/blobcachemetrics.go | 60 +++++++++++++ .../cache/cachedblobdescriptorstore.go | 80 ++++++++++++++++++ registry/storage/cachedblobdescriptorstore.go | 84 ------------------- registry/storage/registry.go | 10 +-- 4 files changed, 142 insertions(+), 92 deletions(-) create mode 100644 registry/storage/blobcachemetrics.go create mode 100644 registry/storage/cache/cachedblobdescriptorstore.go delete mode 100644 registry/storage/cachedblobdescriptorstore.go diff --git a/registry/storage/blobcachemetrics.go b/registry/storage/blobcachemetrics.go new file mode 100644 index 00000000..fad0a77a --- /dev/null +++ b/registry/storage/blobcachemetrics.go @@ -0,0 +1,60 @@ +package storage + +import ( + "expvar" + "sync/atomic" + + "github.com/docker/distribution/registry/storage/cache" +) + +type blobStatCollector struct { + metrics cache.Metrics +} + +func (bsc *blobStatCollector) Hit() { + atomic.AddUint64(&bsc.metrics.Requests, 1) + atomic.AddUint64(&bsc.metrics.Hits, 1) +} + +func (bsc *blobStatCollector) Miss() { + atomic.AddUint64(&bsc.metrics.Requests, 1) + atomic.AddUint64(&bsc.metrics.Misses, 1) +} + +func (bsc *blobStatCollector) Metrics() cache.Metrics { + return bsc.metrics +} + +// blobStatterCacheMetrics keeps track of cache metrics for blob descriptor +// cache requests. Note this is kept globally and made available via expvar. +// For more detailed metrics, its recommend to instrument a particular cache +// implementation. +var blobStatterCacheMetrics cache.MetricsTracker = &blobStatCollector{} + +func init() { + registry := expvar.Get("registry") + if registry == nil { + registry = expvar.NewMap("registry") + } + + cache := registry.(*expvar.Map).Get("cache") + if cache == nil { + cache = &expvar.Map{} + cache.(*expvar.Map).Init() + registry.(*expvar.Map).Set("cache", cache) + } + + storage := cache.(*expvar.Map).Get("storage") + if storage == nil { + storage = &expvar.Map{} + storage.(*expvar.Map).Init() + cache.(*expvar.Map).Set("storage", storage) + } + + storage.(*expvar.Map).Set("blobdescriptor", expvar.Func(func() interface{} { + // no need for synchronous access: the increments are atomic and + // during reading, we don't care if the data is up to date. The + // numbers will always *eventually* be reported correctly. + return blobStatterCacheMetrics + })) +} diff --git a/registry/storage/cache/cachedblobdescriptorstore.go b/registry/storage/cache/cachedblobdescriptorstore.go new file mode 100644 index 00000000..a095b19a --- /dev/null +++ b/registry/storage/cache/cachedblobdescriptorstore.go @@ -0,0 +1,80 @@ +package cache + +import ( + "github.com/docker/distribution/context" + "github.com/docker/distribution/digest" + + "github.com/docker/distribution" +) + +// Metrics is used to hold metric counters +// related to the number of times a cache was +// hit or missed. +type Metrics struct { + Requests uint64 + Hits uint64 + Misses uint64 +} + +// MetricsTracker represents a metric tracker +// which simply counts the number of hits and misses. +type MetricsTracker interface { + Hit() + Miss() + Metrics() Metrics +} + +type cachedBlobStatter struct { + cache distribution.BlobDescriptorService + backend distribution.BlobStatter + tracker MetricsTracker +} + +// NewCachedBlobStatter creates a new statter which prefers a cache and +// falls back to a backend. +func NewCachedBlobStatter(cache distribution.BlobDescriptorService, backend distribution.BlobStatter) distribution.BlobStatter { + return &cachedBlobStatter{ + cache: cache, + backend: backend, + } +} + +// NewCachedBlobStatterWithMetrics creates a new statter which prefers a cache and +// falls back to a backend. Hits and misses will send to the tracker. +func NewCachedBlobStatterWithMetrics(cache distribution.BlobDescriptorService, backend distribution.BlobStatter, tracker MetricsTracker) distribution.BlobStatter { + return &cachedBlobStatter{ + cache: cache, + backend: backend, + tracker: tracker, + } +} + +func (cbds *cachedBlobStatter) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) { + desc, err := cbds.cache.Stat(ctx, dgst) + if err != nil { + if err != distribution.ErrBlobUnknown { + context.GetLogger(ctx).Errorf("error retrieving descriptor from cache: %v", err) + } + + goto fallback + } + + if cbds.tracker != nil { + cbds.tracker.Hit() + } + return desc, nil +fallback: + if cbds.tracker != nil { + cbds.tracker.Miss() + } + desc, err = cbds.backend.Stat(ctx, dgst) + if err != nil { + return desc, err + } + + if err := cbds.cache.SetDescriptor(ctx, dgst, desc); err != nil { + context.GetLogger(ctx).Errorf("error adding descriptor %v to cache: %v", desc.Digest, err) + } + + return desc, err +} diff --git a/registry/storage/cachedblobdescriptorstore.go b/registry/storage/cachedblobdescriptorstore.go deleted file mode 100644 index a0ccd067..00000000 --- a/registry/storage/cachedblobdescriptorstore.go +++ /dev/null @@ -1,84 +0,0 @@ -package storage - -import ( - "expvar" - "sync/atomic" - - "github.com/docker/distribution/context" - "github.com/docker/distribution/digest" - - "github.com/docker/distribution" -) - -type cachedBlobStatter struct { - cache distribution.BlobDescriptorService - backend distribution.BlobStatter -} - -func (cbds *cachedBlobStatter) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) { - atomic.AddUint64(&blobStatterCacheMetrics.Stat.Requests, 1) - desc, err := cbds.cache.Stat(ctx, dgst) - if err != nil { - if err != distribution.ErrBlobUnknown { - context.GetLogger(ctx).Errorf("error retrieving descriptor from cache: %v", err) - } - - goto fallback - } - - atomic.AddUint64(&blobStatterCacheMetrics.Stat.Hits, 1) - return desc, nil -fallback: - atomic.AddUint64(&blobStatterCacheMetrics.Stat.Misses, 1) - desc, err = cbds.backend.Stat(ctx, dgst) - if err != nil { - return desc, err - } - - if err := cbds.cache.SetDescriptor(ctx, dgst, desc); err != nil { - context.GetLogger(ctx).Errorf("error adding descriptor %v to cache: %v", desc.Digest, err) - } - - return desc, err -} - -// blobStatterCacheMetrics keeps track of cache metrics for blob descriptor -// cache requests. Note this is kept globally and made available via expvar. -// For more detailed metrics, its recommend to instrument a particular cache -// implementation. -var blobStatterCacheMetrics struct { - // Stat tracks calls to the caches. - Stat struct { - Requests uint64 - Hits uint64 - Misses uint64 - } -} - -func init() { - registry := expvar.Get("registry") - if registry == nil { - registry = expvar.NewMap("registry") - } - - cache := registry.(*expvar.Map).Get("cache") - if cache == nil { - cache = &expvar.Map{} - cache.(*expvar.Map).Init() - registry.(*expvar.Map).Set("cache", cache) - } - - storage := cache.(*expvar.Map).Get("storage") - if storage == nil { - storage = &expvar.Map{} - storage.(*expvar.Map).Init() - cache.(*expvar.Map).Set("storage", storage) - } - - storage.(*expvar.Map).Set("blobdescriptor", expvar.Func(func() interface{} { - // no need for synchronous access: the increments are atomic and - // during reading, we don't care if the data is up to date. The - // numbers will always *eventually* be reported correctly. - return blobStatterCacheMetrics - })) -} diff --git a/registry/storage/registry.go b/registry/storage/registry.go index 659c789e..cc223727 100644 --- a/registry/storage/registry.go +++ b/registry/storage/registry.go @@ -29,10 +29,7 @@ func NewRegistryWithDriver(ctx context.Context, driver storagedriver.StorageDriv } if blobDescriptorCacheProvider != nil { - statter = &cachedBlobStatter{ - cache: blobDescriptorCacheProvider, - backend: statter, - } + statter = cache.NewCachedBlobStatter(blobDescriptorCacheProvider, statter) } bs := &blobStore{ @@ -143,10 +140,7 @@ func (repo *repository) Blobs(ctx context.Context) distribution.BlobStore { } if repo.descriptorCache != nil { - statter = &cachedBlobStatter{ - cache: repo.descriptorCache, - backend: statter, - } + statter = cache.NewCachedBlobStatter(repo.descriptorCache, statter) } return &linkedBlobStore{ From 98836d6267e9369cf6e17fee35a801f1b0f7b6bd Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Fri, 15 May 2015 16:25:00 -0700 Subject: [PATCH 17/26] Update to track refactor updates Added use of cache blob statter Signed-off-by: Derek McGowan (github: dmcgowan) --- registry/client/blob.go | 159 ---------------------------- registry/client/blob_writer.go | 2 +- registry/client/http_reader.go | 164 +++++++++++++++++++++++++++++ registry/client/repository.go | 45 ++++++-- registry/client/repository_test.go | 4 +- 5 files changed, 201 insertions(+), 173 deletions(-) delete mode 100644 registry/client/blob.go create mode 100644 registry/client/http_reader.go diff --git a/registry/client/blob.go b/registry/client/blob.go deleted file mode 100644 index e7c0039c..00000000 --- a/registry/client/blob.go +++ /dev/null @@ -1,159 +0,0 @@ -package client - -import ( - "bufio" - "bytes" - "fmt" - "io" - "io/ioutil" - "net/http" - "os" - - "github.com/docker/distribution" - "github.com/docker/distribution/context" -) - -type httpBlob struct { - *repository - - desc distribution.Descriptor - - rc io.ReadCloser // remote read closer - brd *bufio.Reader // internal buffered io - offset int64 - err error -} - -func (hb *httpBlob) Read(p []byte) (n int, err error) { - if hb.err != nil { - return 0, hb.err - } - - rd, err := hb.reader() - if err != nil { - return 0, err - } - - n, err = rd.Read(p) - hb.offset += int64(n) - - // Simulate io.EOF error if we reach filesize. - if err == nil && hb.offset >= hb.desc.Length { - err = io.EOF - } - - return n, err -} - -func (hb *httpBlob) Seek(offset int64, whence int) (int64, error) { - if hb.err != nil { - return 0, hb.err - } - - var err error - newOffset := hb.offset - - switch whence { - case os.SEEK_CUR: - newOffset += int64(offset) - case os.SEEK_END: - newOffset = hb.desc.Length + int64(offset) - case os.SEEK_SET: - newOffset = int64(offset) - } - - if newOffset < 0 { - err = fmt.Errorf("cannot seek to negative position") - } else { - if hb.offset != newOffset { - hb.reset() - } - - // No problems, set the offset. - hb.offset = newOffset - } - - return hb.offset, err -} - -func (hb *httpBlob) Close() error { - if hb.err != nil { - return hb.err - } - - // close and release reader chain - if hb.rc != nil { - hb.rc.Close() - } - - hb.rc = nil - hb.brd = nil - - hb.err = fmt.Errorf("httpBlob: closed") - - return nil -} - -func (hb *httpBlob) reset() { - if hb.err != nil { - return - } - if hb.rc != nil { - hb.rc.Close() - hb.rc = nil - } -} - -func (hb *httpBlob) reader() (io.Reader, error) { - if hb.err != nil { - return nil, hb.err - } - - if hb.rc != nil { - return hb.brd, nil - } - - // If the offset is great than or equal to size, return a empty, noop reader. - if hb.offset >= hb.desc.Length { - return ioutil.NopCloser(bytes.NewReader([]byte{})), nil - } - - blobURL, err := hb.ub.BuildBlobURL(hb.name, hb.desc.Digest) - if err != nil { - return nil, err - } - - req, err := http.NewRequest("GET", blobURL, nil) - if err != nil { - return nil, err - } - - if hb.offset > 0 { - // TODO(stevvooe): Get this working correctly. - - // If we are at different offset, issue a range request from there. - req.Header.Add("Range", fmt.Sprintf("1-")) - context.GetLogger(hb.context).Infof("Range: %s", req.Header.Get("Range")) - } - - resp, err := hb.client.Do(req) - if err != nil { - return nil, err - } - - switch { - case resp.StatusCode == 200: - hb.rc = resp.Body - default: - defer resp.Body.Close() - return nil, fmt.Errorf("unexpected status resolving reader: %v", resp.Status) - } - - if hb.brd == nil { - hb.brd = bufio.NewReader(hb.rc) - } else { - hb.brd.Reset(hb.rc) - } - - return hb.brd, nil -} diff --git a/registry/client/blob_writer.go b/registry/client/blob_writer.go index 3697ef8c..44151167 100644 --- a/registry/client/blob_writer.go +++ b/registry/client/blob_writer.go @@ -151,7 +151,7 @@ func (hbu *httpBlobUpload) Commit(ctx context.Context, desc distribution.Descrip return hbu.repo.Blobs(ctx).Stat(ctx, desc.Digest) } -func (hbu *httpBlobUpload) Rollback(ctx context.Context) error { +func (hbu *httpBlobUpload) Cancel(ctx context.Context) error { panic("not implemented") } diff --git a/registry/client/http_reader.go b/registry/client/http_reader.go new file mode 100644 index 00000000..22f9bfbc --- /dev/null +++ b/registry/client/http_reader.go @@ -0,0 +1,164 @@ +package client + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + + "github.com/docker/distribution" +) + +func NewHTTPReadSeeker(client *http.Client, url string, size int64) distribution.ReadSeekCloser { + return &httpReadSeeker{ + client: client, + url: url, + size: size, + } +} + +type httpReadSeeker struct { + client *http.Client + url string + + size int64 + + rc io.ReadCloser // remote read closer + brd *bufio.Reader // internal buffered io + offset int64 + err error +} + +func (hrs *httpReadSeeker) Read(p []byte) (n int, err error) { + if hrs.err != nil { + return 0, hrs.err + } + + rd, err := hrs.reader() + if err != nil { + return 0, err + } + + n, err = rd.Read(p) + hrs.offset += int64(n) + + // Simulate io.EOF error if we reach filesize. + if err == nil && hrs.offset >= hrs.size { + err = io.EOF + } + + return n, err +} + +func (hrs *httpReadSeeker) Seek(offset int64, whence int) (int64, error) { + if hrs.err != nil { + return 0, hrs.err + } + + var err error + newOffset := hrs.offset + + switch whence { + case os.SEEK_CUR: + newOffset += int64(offset) + case os.SEEK_END: + newOffset = hrs.size + int64(offset) + case os.SEEK_SET: + newOffset = int64(offset) + } + + if newOffset < 0 { + err = errors.New("cannot seek to negative position") + } else { + if hrs.offset != newOffset { + hrs.reset() + } + + // No problems, set the offset. + hrs.offset = newOffset + } + + return hrs.offset, err +} + +func (hrs *httpReadSeeker) Close() error { + if hrs.err != nil { + return hrs.err + } + + // close and release reader chain + if hrs.rc != nil { + hrs.rc.Close() + } + + hrs.rc = nil + hrs.brd = nil + + hrs.err = errors.New("httpLayer: closed") + + return nil +} + +func (hrs *httpReadSeeker) reset() { + if hrs.err != nil { + return + } + if hrs.rc != nil { + hrs.rc.Close() + hrs.rc = nil + } +} + +func (hrs *httpReadSeeker) reader() (io.Reader, error) { + if hrs.err != nil { + return nil, hrs.err + } + + if hrs.rc != nil { + return hrs.brd, nil + } + + // If the offset is great than or equal to size, return a empty, noop reader. + if hrs.offset >= hrs.size { + return ioutil.NopCloser(bytes.NewReader([]byte{})), nil + } + + req, err := http.NewRequest("GET", hrs.url, nil) + if err != nil { + return nil, err + } + + if hrs.offset > 0 { + // TODO(stevvooe): Get this working correctly. + + // If we are at different offset, issue a range request from there. + req.Header.Add("Range", "1-") + // TODO: get context in here + // context.GetLogger(hrs.context).Infof("Range: %s", req.Header.Get("Range")) + } + + resp, err := hrs.client.Do(req) + if err != nil { + return nil, err + } + + switch { + case resp.StatusCode == 200: + hrs.rc = resp.Body + default: + defer resp.Body.Close() + return nil, fmt.Errorf("unexpected status resolving reader: %v", resp.Status) + } + + if hrs.brd == nil { + hrs.brd = bufio.NewReader(hrs.rc) + } else { + hrs.brd.Reset(hrs.rc) + } + + return hrs.brd, nil +} diff --git a/registry/client/repository.go b/registry/client/repository.go index 940ae1df..61dcf0f4 100644 --- a/registry/client/repository.go +++ b/registry/client/repository.go @@ -18,6 +18,7 @@ import ( "github.com/docker/distribution" "github.com/docker/distribution/context" "github.com/docker/distribution/registry/api/v2" + "github.com/docker/distribution/registry/storage/cache" ) // NewRepository creates a new Repository for the given repository name and endpoint @@ -56,9 +57,13 @@ func (r *repository) Name() string { return r.name } -func (r *repository) Blobs(ctx context.Context) distribution.BlobService { +func (r *repository) Blobs(ctx context.Context) distribution.BlobStore { + statter := &blobStatter{ + repository: r, + } return &blobs{ repository: r, + statter: cache.NewCachedBlobStatter(cache.NewInMemoryBlobDescriptorCacheProvider(), statter), } } @@ -232,6 +237,8 @@ func (ms *manifests) Delete(dgst digest.Digest) error { type blobs struct { *repository + + statter distribution.BlobStatter } func sanitizeLocation(location, source string) (string, error) { @@ -255,12 +262,17 @@ func sanitizeLocation(location, source string) (string, error) { return location, nil } +func (ls *blobs) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) { + return ls.statter.Stat(ctx, dgst) + +} + func (ls *blobs) Get(ctx context.Context, dgst digest.Digest) ([]byte, error) { desc, err := ls.Stat(ctx, dgst) if err != nil { return nil, err } - reader, err := ls.Open(ctx, desc) + reader, err := ls.Open(ctx, desc.Digest) if err != nil { return nil, err } @@ -269,19 +281,26 @@ func (ls *blobs) Get(ctx context.Context, dgst digest.Digest) ([]byte, error) { return ioutil.ReadAll(reader) } -func (ls *blobs) Open(ctx context.Context, desc distribution.Descriptor) (distribution.ReadSeekCloser, error) { - return &httpBlob{ - repository: ls.repository, - desc: desc, - }, nil +func (ls *blobs) Open(ctx context.Context, dgst digest.Digest) (distribution.ReadSeekCloser, error) { + stat, err := ls.statter.Stat(ctx, dgst) + if err != nil { + return nil, err + } + + blobURL, err := ls.ub.BuildBlobURL(ls.Name(), stat.Digest) + if err != nil { + return nil, err + } + + return NewHTTPReadSeeker(ls.repository.client, blobURL, stat.Length), nil } -func (ls *blobs) ServeBlob(ctx context.Context, w http.ResponseWriter, r *http.Request, desc distribution.Descriptor) error { +func (ls *blobs) ServeBlob(ctx context.Context, w http.ResponseWriter, r *http.Request, dgst digest.Digest) error { return nil } func (ls *blobs) Put(ctx context.Context, mediaType string, p []byte) (distribution.Descriptor, error) { - writer, err := ls.Writer(ctx) + writer, err := ls.Create(ctx) if err != nil { return distribution.Descriptor{}, err } @@ -303,7 +322,7 @@ func (ls *blobs) Put(ctx context.Context, mediaType string, p []byte) (distribut return writer.Commit(ctx, desc) } -func (ls *blobs) Writer(ctx context.Context) (distribution.BlobWriter, error) { +func (ls *blobs) Create(ctx context.Context) (distribution.BlobWriter, error) { u, err := ls.ub.BuildBlobUploadURL(ls.name) resp, err := ls.client.Post(u, "", nil) @@ -337,7 +356,11 @@ func (ls *blobs) Resume(ctx context.Context, id string) (distribution.BlobWriter panic("not implemented") } -func (ls *blobs) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) { +type blobStatter struct { + *repository +} + +func (ls *blobStatter) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) { u, err := ls.ub.BuildBlobURL(ls.name, dgst) if err != nil { return distribution.Descriptor{}, err diff --git a/registry/client/repository_test.go b/registry/client/repository_test.go index 514f3ee2..f0f40316 100644 --- a/registry/client/repository_test.go +++ b/registry/client/repository_test.go @@ -237,7 +237,7 @@ func TestBlobUploadChunked(t *testing.T) { } l := r.Blobs(ctx) - upload, err := l.Writer(ctx) + upload, err := l.Create(ctx) if err != nil { t.Fatal(err) } @@ -348,7 +348,7 @@ func TestBlobUploadMonolithic(t *testing.T) { } l := r.Blobs(ctx) - upload, err := l.Writer(ctx) + upload, err := l.Create(ctx) if err != nil { t.Fatal(err) } From 94e375c5d1c28b6cdb56a820de0ee87c3fd5a355 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Fri, 15 May 2015 16:34:00 -0700 Subject: [PATCH 18/26] Remove unused and duplicate error types Signed-off-by: Derek McGowan (github: dmcgowan) --- registry/client/blob_writer.go | 2 +- registry/client/blob_writer_test.go | 6 ++---- registry/client/errors.go | 28 ---------------------------- 3 files changed, 3 insertions(+), 33 deletions(-) diff --git a/registry/client/blob_writer.go b/registry/client/blob_writer.go index 44151167..06ca8738 100644 --- a/registry/client/blob_writer.go +++ b/registry/client/blob_writer.go @@ -28,7 +28,7 @@ type httpBlobUpload struct { func (hbu *httpBlobUpload) handleErrorResponse(resp *http.Response) error { if resp.StatusCode == http.StatusNotFound { - return &BlobUploadNotFoundError{Location: hbu.location} + return distribution.ErrBlobUploadUnknown } return handleErrorResponse(resp) } diff --git a/registry/client/blob_writer_test.go b/registry/client/blob_writer_test.go index 2e4edc45..0cc20da4 100644 --- a/registry/client/blob_writer_test.go +++ b/registry/client/blob_writer_test.go @@ -151,10 +151,8 @@ func TestUploadReadFrom(t *testing.T) { if err == nil { t.Fatalf("Expected error when not found") } - if blobErr, ok := err.(*BlobUploadNotFoundError); !ok { - t.Fatalf("Wrong error type %T: %s", err, err) - } else if expected := e + locationPath; blobErr.Location != expected { - t.Fatalf("Unexpected location: %s, expected %s", blobErr.Location, expected) + if err != distribution.ErrBlobUploadUnknown { + t.Fatalf("Wrong error thrown: %s, expected", err, distribution.ErrBlobUploadUnknown) } // 400 valid json diff --git a/registry/client/errors.go b/registry/client/errors.go index 2bb64a44..c4296fa3 100644 --- a/registry/client/errors.go +++ b/registry/client/errors.go @@ -9,34 +9,6 @@ import ( "github.com/docker/distribution/registry/api/v2" ) -// BlobUploadNotFoundError is returned when making a blob upload operation against an -// invalid blob upload location url. -// This may be the result of using a cancelled, completed, or stale upload -// location. -type BlobUploadNotFoundError struct { - Location string -} - -func (e *BlobUploadNotFoundError) Error() string { - return fmt.Sprintf("No blob upload found at Location: %s", e.Location) -} - -// BlobUploadInvalidRangeError is returned when attempting to upload an image -// blob chunk that is out of order. -// This provides the known BlobSize and LastValidRange which can be used to -// resume the upload. -type BlobUploadInvalidRangeError struct { - Location string - LastValidRange int - BlobSize int -} - -func (e *BlobUploadInvalidRangeError) Error() string { - return fmt.Sprintf( - "Invalid range provided for upload at Location: %s. Last Valid Range: %d, Blob Size: %d", - e.Location, e.LastValidRange, e.BlobSize) -} - // UnexpectedHTTPStatusError is returned when an unexpected HTTP status is // returned when making a registry api call. type UnexpectedHTTPStatusError struct { From 28744542244a8a3c56a7c95bdd74a40095e1d796 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Fri, 15 May 2015 16:50:17 -0700 Subject: [PATCH 19/26] Create client transport package Signed-off-by: Derek McGowan (github: dmcgowan) --- registry/client/repository.go | 9 +++++---- registry/client/{ => transport}/authchallenge.go | 2 +- registry/client/{ => transport}/authchallenge_test.go | 2 +- registry/client/{ => transport}/http_reader.go | 2 +- registry/client/{ => transport}/session.go | 2 +- registry/client/{ => transport}/session_test.go | 8 +++++++- registry/client/{ => transport}/transport.go | 2 +- 7 files changed, 17 insertions(+), 10 deletions(-) rename registry/client/{ => transport}/authchallenge.go (99%) rename registry/client/{ => transport}/authchallenge_test.go (98%) rename registry/client/{ => transport}/http_reader.go (99%) rename registry/client/{ => transport}/session.go (99%) rename registry/client/{ => transport}/session_test.go (97%) rename registry/client/{ => transport}/transport.go (99%) diff --git a/registry/client/repository.go b/registry/client/repository.go index 61dcf0f4..788e7904 100644 --- a/registry/client/repository.go +++ b/registry/client/repository.go @@ -18,16 +18,17 @@ import ( "github.com/docker/distribution" "github.com/docker/distribution/context" "github.com/docker/distribution/registry/api/v2" + "github.com/docker/distribution/registry/client/transport" "github.com/docker/distribution/registry/storage/cache" ) -// NewRepository creates a new Repository for the given repository name and endpoint -func NewRepository(ctx context.Context, name, endpoint string, transport http.RoundTripper) (distribution.Repository, error) { +// NewRepository creates a new Repository for the given repository name and base URL +func NewRepository(ctx context.Context, name, baseURL string, transport http.RoundTripper) (distribution.Repository, error) { if err := v2.ValidateRespositoryName(name); err != nil { return nil, err } - ub, err := v2.NewURLBuilderFromString(endpoint) + ub, err := v2.NewURLBuilderFromString(baseURL) if err != nil { return nil, err } @@ -292,7 +293,7 @@ func (ls *blobs) Open(ctx context.Context, dgst digest.Digest) (distribution.Rea return nil, err } - return NewHTTPReadSeeker(ls.repository.client, blobURL, stat.Length), nil + return transport.NewHTTPReadSeeker(ls.repository.client, blobURL, stat.Length), nil } func (ls *blobs) ServeBlob(ctx context.Context, w http.ResponseWriter, r *http.Request, dgst digest.Digest) error { diff --git a/registry/client/authchallenge.go b/registry/client/transport/authchallenge.go similarity index 99% rename from registry/client/authchallenge.go rename to registry/client/transport/authchallenge.go index 49cf270e..fffd560b 100644 --- a/registry/client/authchallenge.go +++ b/registry/client/transport/authchallenge.go @@ -1,4 +1,4 @@ -package client +package transport import ( "net/http" diff --git a/registry/client/authchallenge_test.go b/registry/client/transport/authchallenge_test.go similarity index 98% rename from registry/client/authchallenge_test.go rename to registry/client/transport/authchallenge_test.go index 802c94f3..45c932b9 100644 --- a/registry/client/authchallenge_test.go +++ b/registry/client/transport/authchallenge_test.go @@ -1,4 +1,4 @@ -package client +package transport import ( "net/http" diff --git a/registry/client/http_reader.go b/registry/client/transport/http_reader.go similarity index 99% rename from registry/client/http_reader.go rename to registry/client/transport/http_reader.go index 22f9bfbc..de728a96 100644 --- a/registry/client/http_reader.go +++ b/registry/client/transport/http_reader.go @@ -1,4 +1,4 @@ -package client +package transport import ( "bufio" diff --git a/registry/client/session.go b/registry/client/transport/session.go similarity index 99% rename from registry/client/session.go rename to registry/client/transport/session.go index 41bb4f31..670be1ba 100644 --- a/registry/client/session.go +++ b/registry/client/transport/session.go @@ -1,4 +1,4 @@ -package client +package transport import ( "encoding/json" diff --git a/registry/client/session_test.go b/registry/client/transport/session_test.go similarity index 97% rename from registry/client/session_test.go rename to registry/client/transport/session_test.go index cf8e546e..374d6e79 100644 --- a/registry/client/session_test.go +++ b/registry/client/transport/session_test.go @@ -1,4 +1,4 @@ -package client +package transport import ( "encoding/base64" @@ -11,6 +11,12 @@ import ( "github.com/docker/distribution/testutil" ) +func testServer(rrm testutil.RequestResponseMap) (string, func()) { + h := testutil.NewHandler(rrm) + s := httptest.NewServer(h) + return s.URL, s.Close +} + type testAuthenticationWrapper struct { headers http.Header authCheck func(string) bool diff --git a/registry/client/transport.go b/registry/client/transport/transport.go similarity index 99% rename from registry/client/transport.go rename to registry/client/transport/transport.go index 0b241619..c8cfbb19 100644 --- a/registry/client/transport.go +++ b/registry/client/transport/transport.go @@ -1,4 +1,4 @@ -package client +package transport import ( "io" From 006ddd8283013d392d0e19bdf1adda5bac6612fe Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Fri, 15 May 2015 17:37:32 -0700 Subject: [PATCH 20/26] Lint and documentation fixes Signed-off-by: Derek McGowan (github: dmcgowan) --- registry/client/blob_writer_test.go | 2 +- registry/client/transport/http_reader.go | 3 +++ registry/client/transport/transport.go | 6 ++++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/registry/client/blob_writer_test.go b/registry/client/blob_writer_test.go index 0cc20da4..4d2ae862 100644 --- a/registry/client/blob_writer_test.go +++ b/registry/client/blob_writer_test.go @@ -152,7 +152,7 @@ func TestUploadReadFrom(t *testing.T) { t.Fatalf("Expected error when not found") } if err != distribution.ErrBlobUploadUnknown { - t.Fatalf("Wrong error thrown: %s, expected", err, distribution.ErrBlobUploadUnknown) + t.Fatalf("Wrong error thrown: %s, expected %s", err, distribution.ErrBlobUploadUnknown) } // 400 valid json diff --git a/registry/client/transport/http_reader.go b/registry/client/transport/http_reader.go index de728a96..d10d37e0 100644 --- a/registry/client/transport/http_reader.go +++ b/registry/client/transport/http_reader.go @@ -13,6 +13,9 @@ import ( "github.com/docker/distribution" ) +// NewHTTPReadSeeker handles reading from an HTTP endpoint using a GET +// request. When seeking and starting a read from a non-zero offset +// the a "Range" header will be added which sets the offset. func NewHTTPReadSeeker(client *http.Client, url string, size int64) distribution.ReadSeekCloser { return &httpReadSeeker{ client: client, diff --git a/registry/client/transport/transport.go b/registry/client/transport/transport.go index c8cfbb19..30e45fab 100644 --- a/registry/client/transport/transport.go +++ b/registry/client/transport/transport.go @@ -6,12 +6,16 @@ import ( "sync" ) +// RequestModifier represents an object which will do an inplace +// modification of an HTTP request. type RequestModifier interface { ModifyRequest(*http.Request) error } type headerModifier http.Header +// NewHeaderRequestModifier returns a new RequestModifier which will +// add the given headers to a request. func NewHeaderRequestModifier(header http.Header) RequestModifier { return headerModifier(header) } @@ -24,6 +28,8 @@ func (h headerModifier) ModifyRequest(req *http.Request) error { return nil } +// NewTransport creates a new transport which will apply modifiers to +// the request on a RoundTrip call. func NewTransport(base http.RoundTripper, modifiers ...RequestModifier) http.RoundTripper { return &transport{ Modifiers: modifiers, From a3276fcc5be14c6d3f41e87604158b81ff3654d6 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Tue, 19 May 2015 19:18:30 -0700 Subject: [PATCH 21/26] Feedback update Update comments and TODOs Fix switch style Updated parse http response to take in reader Add Cancel implementation Update blobstore variable name Signed-off-by: Derek McGowan (github: dmcgowan) --- registry/client/blob_writer.go | 23 ++++++-- registry/client/blob_writer_test.go | 10 ---- registry/client/errors.go | 7 ++- registry/client/repository.go | 90 +++++++++++++++-------------- 4 files changed, 68 insertions(+), 62 deletions(-) diff --git a/registry/client/blob_writer.go b/registry/client/blob_writer.go index 06ca8738..55223520 100644 --- a/registry/client/blob_writer.go +++ b/registry/client/blob_writer.go @@ -2,7 +2,6 @@ package client import ( "bytes" - "errors" "fmt" "io" "io/ioutil" @@ -49,7 +48,6 @@ func (hbu *httpBlobUpload) ReadFrom(r io.Reader) (n int64, err error) { return 0, hbu.handleErrorResponse(resp) } - // TODO(dmcgowan): Validate headers hbu.uuid = resp.Header.Get("Docker-Upload-UUID") hbu.location, err = sanitizeLocation(resp.Header.Get("Location"), hbu.location) if err != nil { @@ -85,7 +83,6 @@ func (hbu *httpBlobUpload) Write(p []byte) (n int, err error) { return 0, hbu.handleErrorResponse(resp) } - // TODO(dmcgowan): Validate headers hbu.uuid = resp.Header.Get("Docker-Upload-UUID") hbu.location, err = sanitizeLocation(resp.Header.Get("Location"), hbu.location) if err != nil { @@ -110,7 +107,7 @@ func (hbu *httpBlobUpload) Seek(offset int64, whence int) (int64, error) { case os.SEEK_CUR: newOffset += int64(offset) case os.SEEK_END: - return newOffset, errors.New("Cannot seek from end on incomplete upload") + newOffset += int64(offset) case os.SEEK_SET: newOffset = int64(offset) } @@ -143,6 +140,7 @@ func (hbu *httpBlobUpload) Commit(ctx context.Context, desc distribution.Descrip if err != nil { return distribution.Descriptor{}, err } + defer resp.Body.Close() if resp.StatusCode != http.StatusCreated { return distribution.Descriptor{}, hbu.handleErrorResponse(resp) @@ -152,7 +150,22 @@ func (hbu *httpBlobUpload) Commit(ctx context.Context, desc distribution.Descrip } func (hbu *httpBlobUpload) Cancel(ctx context.Context) error { - panic("not implemented") + req, err := http.NewRequest("DELETE", hbu.location, nil) + if err != nil { + return err + } + resp, err := hbu.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + switch resp.StatusCode { + case http.StatusNoContent, http.StatusNotFound: + return nil + default: + return hbu.handleErrorResponse(resp) + } } func (hbu *httpBlobUpload) Close() error { diff --git a/registry/client/blob_writer_test.go b/registry/client/blob_writer_test.go index 4d2ae862..674d6e01 100644 --- a/registry/client/blob_writer_test.go +++ b/registry/client/blob_writer_test.go @@ -205,13 +205,3 @@ func TestUploadReadFrom(t *testing.T) { t.Fatalf("Unexpected response status: %s, expected %s", uploadErr.Status, expected) } } - -//repo distribution.Repository -//client *http.Client - -//uuid string -//startedAt time.Time - -//location string // always the last value of the location header. -//offset int64 -//closed bool diff --git a/registry/client/errors.go b/registry/client/errors.go index c4296fa3..c6c802a2 100644 --- a/registry/client/errors.go +++ b/registry/client/errors.go @@ -3,6 +3,7 @@ package client import ( "encoding/json" "fmt" + "io" "io/ioutil" "net/http" @@ -34,9 +35,9 @@ func (e *UnexpectedHTTPResponseError) Error() string { return fmt.Sprintf("Error parsing HTTP response: %s: %q", e.ParseErr.Error(), shortenedResponse) } -func parseHTTPErrorResponse(response *http.Response) error { +func parseHTTPErrorResponse(r io.Reader) error { var errors v2.Errors - body, err := ioutil.ReadAll(response.Body) + body, err := ioutil.ReadAll(r) if err != nil { return err } @@ -52,7 +53,7 @@ func parseHTTPErrorResponse(response *http.Response) error { func handleErrorResponse(resp *http.Response) error { if resp.StatusCode >= 400 && resp.StatusCode < 500 { - return parseHTTPErrorResponse(resp) + return parseHTTPErrorResponse(resp.Body) } return &UnexpectedHTTPStatusError{Status: resp.Status} } diff --git a/registry/client/repository.go b/registry/client/repository.go index 788e7904..123ef6ce 100644 --- a/registry/client/repository.go +++ b/registry/client/repository.go @@ -11,12 +11,10 @@ import ( "strconv" "time" - "github.com/docker/distribution/manifest" - - "github.com/docker/distribution/digest" - "github.com/docker/distribution" "github.com/docker/distribution/context" + "github.com/docker/distribution/digest" + "github.com/docker/distribution/manifest" "github.com/docker/distribution/registry/api/v2" "github.com/docker/distribution/registry/client/transport" "github.com/docker/distribution/registry/storage/cache" @@ -108,8 +106,8 @@ func (ms *manifests) Tags() ([]string, error) { } defer resp.Body.Close() - switch { - case resp.StatusCode == http.StatusOK: + switch resp.StatusCode { + case http.StatusOK: b, err := ioutil.ReadAll(resp.Body) if err != nil { return nil, err @@ -123,7 +121,7 @@ func (ms *manifests) Tags() ([]string, error) { } return tagsResponse.Tags, nil - case resp.StatusCode == http.StatusNotFound: + case http.StatusNotFound: return nil, nil default: return nil, handleErrorResponse(resp) @@ -131,6 +129,8 @@ func (ms *manifests) Tags() ([]string, error) { } func (ms *manifests) Exists(dgst digest.Digest) (bool, error) { + // Call by Tag endpoint since the API uses the same + // URL endpoint for tags and digests. return ms.ExistsByTag(dgst.String()) } @@ -145,10 +145,10 @@ func (ms *manifests) ExistsByTag(tag string) (bool, error) { return false, err } - switch { - case resp.StatusCode == http.StatusOK: + switch resp.StatusCode { + case http.StatusOK: return true, nil - case resp.StatusCode == http.StatusNotFound: + case http.StatusNotFound: return false, nil default: return false, handleErrorResponse(resp) @@ -156,6 +156,8 @@ func (ms *manifests) ExistsByTag(tag string) (bool, error) { } func (ms *manifests) Get(dgst digest.Digest) (*manifest.SignedManifest, error) { + // Call by Tag endpoint since the API uses the same + // URL endpoint for tags and digests. return ms.GetByTag(dgst.String()) } @@ -171,8 +173,8 @@ func (ms *manifests) GetByTag(tag string) (*manifest.SignedManifest, error) { } defer resp.Body.Close() - switch { - case resp.StatusCode == http.StatusOK: + switch resp.StatusCode { + case http.StatusOK: var sm manifest.SignedManifest decoder := json.NewDecoder(resp.Body) @@ -203,9 +205,9 @@ func (ms *manifests) Put(m *manifest.SignedManifest) error { } defer resp.Body.Close() - switch { - case resp.StatusCode == http.StatusAccepted: - // TODO(dmcgowan): Use or check digest header + switch resp.StatusCode { + case http.StatusAccepted: + // TODO(dmcgowan): make use of digest header return nil default: return handleErrorResponse(resp) @@ -228,8 +230,8 @@ func (ms *manifests) Delete(dgst digest.Digest) error { } defer resp.Body.Close() - switch { - case resp.StatusCode == http.StatusOK: + switch resp.StatusCode { + case http.StatusOK: return nil default: return handleErrorResponse(resp) @@ -263,17 +265,17 @@ func sanitizeLocation(location, source string) (string, error) { return location, nil } -func (ls *blobs) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) { - return ls.statter.Stat(ctx, dgst) +func (bs *blobs) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) { + return bs.statter.Stat(ctx, dgst) } -func (ls *blobs) Get(ctx context.Context, dgst digest.Digest) ([]byte, error) { - desc, err := ls.Stat(ctx, dgst) +func (bs *blobs) Get(ctx context.Context, dgst digest.Digest) ([]byte, error) { + desc, err := bs.Stat(ctx, dgst) if err != nil { return nil, err } - reader, err := ls.Open(ctx, desc.Digest) + reader, err := bs.Open(ctx, desc.Digest) if err != nil { return nil, err } @@ -282,26 +284,26 @@ func (ls *blobs) Get(ctx context.Context, dgst digest.Digest) ([]byte, error) { return ioutil.ReadAll(reader) } -func (ls *blobs) Open(ctx context.Context, dgst digest.Digest) (distribution.ReadSeekCloser, error) { - stat, err := ls.statter.Stat(ctx, dgst) +func (bs *blobs) Open(ctx context.Context, dgst digest.Digest) (distribution.ReadSeekCloser, error) { + stat, err := bs.statter.Stat(ctx, dgst) if err != nil { return nil, err } - blobURL, err := ls.ub.BuildBlobURL(ls.Name(), stat.Digest) + blobURL, err := bs.ub.BuildBlobURL(bs.Name(), stat.Digest) if err != nil { return nil, err } - return transport.NewHTTPReadSeeker(ls.repository.client, blobURL, stat.Length), nil + return transport.NewHTTPReadSeeker(bs.repository.client, blobURL, stat.Length), nil } -func (ls *blobs) ServeBlob(ctx context.Context, w http.ResponseWriter, r *http.Request, dgst digest.Digest) error { - return nil +func (bs *blobs) ServeBlob(ctx context.Context, w http.ResponseWriter, r *http.Request, dgst digest.Digest) error { + panic("not implemented") } -func (ls *blobs) Put(ctx context.Context, mediaType string, p []byte) (distribution.Descriptor, error) { - writer, err := ls.Create(ctx) +func (bs *blobs) Put(ctx context.Context, mediaType string, p []byte) (distribution.Descriptor, error) { + writer, err := bs.Create(ctx) if err != nil { return distribution.Descriptor{}, err } @@ -323,17 +325,17 @@ func (ls *blobs) Put(ctx context.Context, mediaType string, p []byte) (distribut return writer.Commit(ctx, desc) } -func (ls *blobs) Create(ctx context.Context) (distribution.BlobWriter, error) { - u, err := ls.ub.BuildBlobUploadURL(ls.name) +func (bs *blobs) Create(ctx context.Context) (distribution.BlobWriter, error) { + u, err := bs.ub.BuildBlobUploadURL(bs.name) - resp, err := ls.client.Post(u, "", nil) + resp, err := bs.client.Post(u, "", nil) if err != nil { return nil, err } defer resp.Body.Close() - switch { - case resp.StatusCode == http.StatusAccepted: + switch resp.StatusCode { + case http.StatusAccepted: // TODO(dmcgowan): Check for invalid UUID uuid := resp.Header.Get("Docker-Upload-UUID") location, err := sanitizeLocation(resp.Header.Get("Location"), u) @@ -342,8 +344,8 @@ func (ls *blobs) Create(ctx context.Context) (distribution.BlobWriter, error) { } return &httpBlobUpload{ - repo: ls.repository, - client: ls.client, + repo: bs.repository, + client: bs.client, uuid: uuid, startedAt: time.Now(), location: location, @@ -353,7 +355,7 @@ func (ls *blobs) Create(ctx context.Context) (distribution.BlobWriter, error) { } } -func (ls *blobs) Resume(ctx context.Context, id string) (distribution.BlobWriter, error) { +func (bs *blobs) Resume(ctx context.Context, id string) (distribution.BlobWriter, error) { panic("not implemented") } @@ -361,20 +363,20 @@ type blobStatter struct { *repository } -func (ls *blobStatter) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) { - u, err := ls.ub.BuildBlobURL(ls.name, dgst) +func (bs *blobStatter) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) { + u, err := bs.ub.BuildBlobURL(bs.name, dgst) if err != nil { return distribution.Descriptor{}, err } - resp, err := ls.client.Head(u) + resp, err := bs.client.Head(u) if err != nil { return distribution.Descriptor{}, err } defer resp.Body.Close() - switch { - case resp.StatusCode == http.StatusOK: + switch resp.StatusCode { + case http.StatusOK: lengthHeader := resp.Header.Get("Content-Length") length, err := strconv.ParseInt(lengthHeader, 10, 64) if err != nil { @@ -386,7 +388,7 @@ func (ls *blobStatter) Stat(ctx context.Context, dgst digest.Digest) (distributi Length: length, Digest: dgst, }, nil - case resp.StatusCode == http.StatusNotFound: + case http.StatusNotFound: return distribution.Descriptor{}, distribution.ErrBlobUnknown default: return distribution.Descriptor{}, handleErrorResponse(resp) From 13894e8736eb37801433d3f50d09e3d7b56d9fcd Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Tue, 19 May 2015 19:56:27 -0700 Subject: [PATCH 22/26] Break down type dependencies Each type no longer requires holding a reference to repository. Added implementation for signatures get. Signed-off-by: Derek McGowan (github: dmcgowan) --- registry/client/blob_writer.go | 6 ++--- registry/client/repository.go | 42 +++++++++++++++++++++++----------- 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/registry/client/blob_writer.go b/registry/client/blob_writer.go index 55223520..9ebd4183 100644 --- a/registry/client/blob_writer.go +++ b/registry/client/blob_writer.go @@ -14,8 +14,8 @@ import ( ) type httpBlobUpload struct { - repo distribution.Repository - client *http.Client + statter distribution.BlobStatter + client *http.Client uuid string startedAt time.Time @@ -146,7 +146,7 @@ func (hbu *httpBlobUpload) Commit(ctx context.Context, desc distribution.Descrip return distribution.Descriptor{}, hbu.handleErrorResponse(resp) } - return hbu.repo.Blobs(ctx).Stat(ctx, desc.Digest) + return hbu.statter.Stat(ctx, desc.Digest) } func (hbu *httpBlobUpload) Cancel(ctx context.Context) error { diff --git a/registry/client/repository.go b/registry/client/repository.go index 123ef6ce..a1117ac2 100644 --- a/registry/client/repository.go +++ b/registry/client/repository.go @@ -58,32 +58,42 @@ func (r *repository) Name() string { func (r *repository) Blobs(ctx context.Context) distribution.BlobStore { statter := &blobStatter{ - repository: r, + name: r.Name(), + ub: r.ub, + client: r.client, } return &blobs{ - repository: r, - statter: cache.NewCachedBlobStatter(cache.NewInMemoryBlobDescriptorCacheProvider(), statter), + name: r.Name(), + ub: r.ub, + client: r.client, + statter: cache.NewCachedBlobStatter(cache.NewInMemoryBlobDescriptorCacheProvider(), statter), } } func (r *repository) Manifests() distribution.ManifestService { return &manifests{ - repository: r, + name: r.Name(), + ub: r.ub, + client: r.client, } } func (r *repository) Signatures() distribution.SignatureService { return &signatures{ - repository: r, + manifests: r.Manifests(), } } type signatures struct { - *repository + manifests distribution.ManifestService } func (s *signatures) Get(dgst digest.Digest) ([][]byte, error) { - panic("not implemented") + m, err := s.manifests.Get(dgst) + if err != nil { + return nil, err + } + return m.Signatures() } func (s *signatures) Put(dgst digest.Digest, signatures ...[]byte) error { @@ -91,7 +101,9 @@ func (s *signatures) Put(dgst digest.Digest, signatures ...[]byte) error { } type manifests struct { - *repository + name string + ub *v2.URLBuilder + client *http.Client } func (ms *manifests) Tags() ([]string, error) { @@ -239,7 +251,9 @@ func (ms *manifests) Delete(dgst digest.Digest) error { } type blobs struct { - *repository + name string + ub *v2.URLBuilder + client *http.Client statter distribution.BlobStatter } @@ -290,12 +304,12 @@ func (bs *blobs) Open(ctx context.Context, dgst digest.Digest) (distribution.Rea return nil, err } - blobURL, err := bs.ub.BuildBlobURL(bs.Name(), stat.Digest) + blobURL, err := bs.ub.BuildBlobURL(bs.name, stat.Digest) if err != nil { return nil, err } - return transport.NewHTTPReadSeeker(bs.repository.client, blobURL, stat.Length), nil + return transport.NewHTTPReadSeeker(bs.client, blobURL, stat.Length), nil } func (bs *blobs) ServeBlob(ctx context.Context, w http.ResponseWriter, r *http.Request, dgst digest.Digest) error { @@ -344,7 +358,7 @@ func (bs *blobs) Create(ctx context.Context) (distribution.BlobWriter, error) { } return &httpBlobUpload{ - repo: bs.repository, + statter: bs.statter, client: bs.client, uuid: uuid, startedAt: time.Now(), @@ -360,7 +374,9 @@ func (bs *blobs) Resume(ctx context.Context, id string) (distribution.BlobWriter } type blobStatter struct { - *repository + name string + ub *v2.URLBuilder + client *http.Client } func (bs *blobStatter) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) { From 572ff64d2169b96cd8cc3cdb69dc172cad7c1e62 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 20 May 2015 10:05:44 -0700 Subject: [PATCH 23/26] Add unauthorized error check Add check for unauthorized error code and explicitly set the error code if the content could not be parsed. Updated repository test for unauthorized tests and nit feedback. Signed-off-by: Derek McGowan (github: dmcgowan) --- registry/client/errors.go | 11 ++++ registry/client/repository_test.go | 82 ++++++++++++++++++------------ 2 files changed, 61 insertions(+), 32 deletions(-) diff --git a/registry/client/errors.go b/registry/client/errors.go index c6c802a2..e6ad5f51 100644 --- a/registry/client/errors.go +++ b/registry/client/errors.go @@ -52,6 +52,17 @@ func parseHTTPErrorResponse(r io.Reader) error { } func handleErrorResponse(resp *http.Response) error { + if resp.StatusCode == 401 { + err := parseHTTPErrorResponse(resp.Body) + if uErr, ok := err.(*UnexpectedHTTPResponseError); ok { + return &v2.Error{ + Code: v2.ErrorCodeUnauthorized, + Message: "401 Unauthorized", + Detail: uErr.Response, + } + } + return err + } if resp.StatusCode >= 400 && resp.StatusCode < 500 { return parseHTTPErrorResponse(resp.Body) } diff --git a/registry/client/repository_test.go b/registry/client/repository_test.go index f0f40316..9530bd37 100644 --- a/registry/client/repository_test.go +++ b/registry/client/repository_test.go @@ -18,6 +18,7 @@ import ( "github.com/docker/distribution/context" "github.com/docker/distribution/digest" "github.com/docker/distribution/manifest" + "github.com/docker/distribution/registry/api/v2" "github.com/docker/distribution/testutil" ) @@ -73,26 +74,10 @@ func addTestFetch(repo string, dgst digest.Digest, content []byte, m *testutil.R }) } -func addPing(m *testutil.RequestResponseMap) { - *m = append(*m, testutil.RequestResponseMapping{ - Request: testutil.Request{ - Method: "GET", - Route: "/v2/", - }, - Response: testutil.Response{ - StatusCode: http.StatusOK, - Headers: http.Header(map[string][]string{ - "Docker-Distribution-API-Version": {"registry/2.0"}, - }), - }, - }) -} - func TestBlobFetch(t *testing.T) { d1, b1 := newRandomBlob(1024) var m testutil.RequestResponseMap addTestFetch("test.example.com/repo1", d1, b1, &m) - addPing(&m) e, c := testServer(m) defer c() @@ -112,14 +97,13 @@ func TestBlobFetch(t *testing.T) { t.Fatalf("Wrong bytes values fetched: [%d]byte != [%d]byte", len(b), len(b1)) } - // TODO(dmcgowan): Test error cases + // TODO(dmcgowan): Test for unknown blob case } func TestBlobExists(t *testing.T) { d1, b1 := newRandomBlob(1024) var m testutil.RequestResponseMap addTestFetch("test.example.com/repo1", d1, b1, &m) - addPing(&m) e, c := testServer(m) defer c() @@ -150,7 +134,6 @@ func TestBlobExists(t *testing.T) { func TestBlobUploadChunked(t *testing.T) { dgst, b1 := newRandomBlob(1024) var m testutil.RequestResponseMap - addPing(&m) chunks := [][]byte{ b1[0:256], b1[256:512], @@ -272,7 +255,6 @@ func TestBlobUploadChunked(t *testing.T) { func TestBlobUploadMonolithic(t *testing.T) { dgst, b1 := newRandomBlob(1024) var m testutil.RequestResponseMap - addPing(&m) repo := "test.example.com/uploadrepo" uploadID := uuid.New() m = append(m, testutil.RequestResponseMapping{ @@ -378,7 +360,7 @@ func TestBlobUploadMonolithic(t *testing.T) { } } -func newRandomSchema1Manifest(name, tag string, blobCount int) (*manifest.SignedManifest, digest.Digest) { +func newRandomSchemaV1Manifest(name, tag string, blobCount int) (*manifest.SignedManifest, digest.Digest) { blobs := make([]manifest.FSLayer, blobCount) history := make([]manifest.History, blobCount) @@ -474,9 +456,8 @@ func checkEqualManifest(m1, m2 *manifest.SignedManifest) error { func TestManifestFetch(t *testing.T) { repo := "test.example.com/repo" - m1, dgst := newRandomSchema1Manifest(repo, "latest", 6) + m1, dgst := newRandomSchemaV1Manifest(repo, "latest", 6) var m testutil.RequestResponseMap - addPing(&m) addTestManifest(repo, dgst.String(), m1.Raw, &m) e, c := testServer(m) @@ -507,9 +488,8 @@ func TestManifestFetch(t *testing.T) { func TestManifestFetchByTag(t *testing.T) { repo := "test.example.com/repo/by/tag" - m1, _ := newRandomSchema1Manifest(repo, "latest", 6) + m1, _ := newRandomSchemaV1Manifest(repo, "latest", 6) var m testutil.RequestResponseMap - addPing(&m) addTestManifest(repo, "latest", m1.Raw, &m) e, c := testServer(m) @@ -540,10 +520,9 @@ func TestManifestFetchByTag(t *testing.T) { func TestManifestDelete(t *testing.T) { repo := "test.example.com/repo/delete" - _, dgst1 := newRandomSchema1Manifest(repo, "latest", 6) - _, dgst2 := newRandomSchema1Manifest(repo, "latest", 6) + _, dgst1 := newRandomSchemaV1Manifest(repo, "latest", 6) + _, dgst2 := newRandomSchemaV1Manifest(repo, "latest", 6) var m testutil.RequestResponseMap - addPing(&m) m = append(m, testutil.RequestResponseMapping{ Request: testutil.Request{ Method: "DELETE", @@ -577,9 +556,8 @@ func TestManifestDelete(t *testing.T) { func TestManifestPut(t *testing.T) { repo := "test.example.com/repo/delete" - m1, dgst := newRandomSchema1Manifest(repo, "other", 6) + m1, dgst := newRandomSchemaV1Manifest(repo, "other", 6) var m testutil.RequestResponseMap - addPing(&m) m = append(m, testutil.RequestResponseMapping{ Request: testutil.Request{ Method: "PUT", @@ -608,7 +586,7 @@ func TestManifestPut(t *testing.T) { t.Fatal(err) } - // TODO(dmcgowan): Check for error cases + // TODO(dmcgowan): Check for invalid input error } func TestManifestTags(t *testing.T) { @@ -624,7 +602,6 @@ func TestManifestTags(t *testing.T) { } `)) var m testutil.RequestResponseMap - addPing(&m) m = append(m, testutil.RequestResponseMapping{ Request: testutil.Request{ Method: "GET", @@ -661,3 +638,44 @@ func TestManifestTags(t *testing.T) { // TODO(dmcgowan): Check for error cases } + +func TestManifestUnauthorized(t *testing.T) { + repo := "test.example.com/repo" + _, dgst := newRandomSchemaV1Manifest(repo, "latest", 6) + var m testutil.RequestResponseMap + + m = append(m, testutil.RequestResponseMapping{ + Request: testutil.Request{ + Method: "GET", + Route: "/v2/" + repo + "/manifests/" + dgst.String(), + }, + Response: testutil.Response{ + StatusCode: http.StatusUnauthorized, + Body: []byte("garbage"), + }, + }) + + e, c := testServer(m) + defer c() + + r, err := NewRepository(context.Background(), repo, e, nil) + if err != nil { + t.Fatal(err) + } + ms := r.Manifests() + + _, err = ms.Get(dgst) + if err == nil { + t.Fatal("Expected error fetching manifest") + } + v2Err, ok := err.(*v2.Error) + if !ok { + t.Fatalf("Unexpected error type: %#v", err) + } + if v2Err.Code != v2.ErrorCodeUnauthorized { + t.Fatalf("Unexpected error code: %s", v2Err.Code.String()) + } + if expected := "401 Unauthorized"; v2Err.Message != expected { + t.Fatalf("Unexpected message value: %s, expected %s", v2Err.Message, expected) + } +} From c7f774736831bfc7fbf248c1155493ab87eda9d8 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 20 May 2015 10:09:37 -0700 Subject: [PATCH 24/26] Update transport package to sever distribution dependency The transport package no longer requires importing distribution for the ReadSeekCloser, instead declares its own. Added comments on the Authenication handler in session. Added todo on http seek reader to highlight its lack of belonging to the client transport. Signed-off-by: Derek McGowan (github: dmcgowan) --- registry/client/transport/http_reader.go | 11 ++++++++--- registry/client/transport/session.go | 5 +++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/registry/client/transport/http_reader.go b/registry/client/transport/http_reader.go index d10d37e0..e351bdfe 100644 --- a/registry/client/transport/http_reader.go +++ b/registry/client/transport/http_reader.go @@ -9,14 +9,19 @@ import ( "io/ioutil" "net/http" "os" - - "github.com/docker/distribution" ) +// ReadSeekCloser combines io.ReadSeeker with io.Closer. +type ReadSeekCloser interface { + io.ReadSeeker + io.Closer +} + // NewHTTPReadSeeker handles reading from an HTTP endpoint using a GET // request. When seeking and starting a read from a non-zero offset // the a "Range" header will be added which sets the offset. -func NewHTTPReadSeeker(client *http.Client, url string, size int64) distribution.ReadSeekCloser { +// TODO(dmcgowan): Move this into a separate utility package +func NewHTTPReadSeeker(client *http.Client, url string, size int64) ReadSeekCloser { return &httpReadSeeker{ client: client, url: url, diff --git a/registry/client/transport/session.go b/registry/client/transport/session.go index 670be1ba..5086c021 100644 --- a/registry/client/transport/session.go +++ b/registry/client/transport/session.go @@ -14,7 +14,12 @@ import ( // AuthenticationHandler is an interface for authorizing a request from // params from a "WWW-Authenicate" header for a single scheme. type AuthenticationHandler interface { + // Scheme returns the scheme as expected from the "WWW-Authenicate" header. Scheme() string + + // AuthorizeRequest adds the authorization header to a request (if needed) + // using the parameters from "WWW-Authenticate" method. The parameters + // values depend on the scheme. AuthorizeRequest(req *http.Request, params map[string]string) error } From 49369ffe9a7d693127cbab42b18c5ed2b505fa2b Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 20 May 2015 13:35:23 -0700 Subject: [PATCH 25/26] Only do auth checks for endpoints starting with v2 Changes behavior so ping doesn't happen if /v2/ is anywhere in a request path, but instead only at the beginning. This fixes attempts to ping on redirected URLs. Signed-off-by: Derek McGowan (github: dmcgowan) --- registry/client/transport/session.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/registry/client/transport/session.go b/registry/client/transport/session.go index 5086c021..90c8082c 100644 --- a/registry/client/transport/session.go +++ b/registry/client/transport/session.go @@ -94,7 +94,9 @@ HeaderLoop: func (ta *tokenAuthorizer) ModifyRequest(req *http.Request) error { v2Root := strings.Index(req.URL.Path, "/v2/") - if v2Root == -1 { + // Test if /v2/ does not exist or not at beginning + // TODO(dmcgowan) support v2 endpoints which have a prefix before /v2/ + if v2Root == -1 || v2Root > 0 { return nil } From 68c1ceac9590c9c4fcda1c932d967ca37c96801f Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 20 May 2015 14:55:59 -0700 Subject: [PATCH 26/26] Remove error message shortening Signed-off-by: Derek McGowan (github: dmcgowan) --- registry/client/errors.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/registry/client/errors.go b/registry/client/errors.go index e6ad5f51..2638055d 100644 --- a/registry/client/errors.go +++ b/registry/client/errors.go @@ -28,11 +28,7 @@ type UnexpectedHTTPResponseError struct { } func (e *UnexpectedHTTPResponseError) Error() string { - shortenedResponse := string(e.Response) - if len(shortenedResponse) > 15 { - shortenedResponse = shortenedResponse[:12] + "..." - } - return fmt.Sprintf("Error parsing HTTP response: %s: %q", e.ParseErr.Error(), shortenedResponse) + return fmt.Sprintf("Error parsing HTTP response: %s: %q", e.ParseErr.Error(), string(e.Response)) } func parseHTTPErrorResponse(r io.Reader) error {