From 6b400cd63c203065dcf2f73256ec3caee012243b Mon Sep 17 00:00:00 2001 From: Josh Hawn Date: Thu, 11 Dec 2014 17:55:15 -0800 Subject: [PATCH 01/15] Adds support for v2 registry login summary of changes: registry/auth.go - More logging around the login functions - split Login() out to handle different code paths for v1 (unchanged logic) and v2 (does not currently do account creation) - handling for either basic or token based login attempts registry/authchallenge.go - New File - credit to Brian Bland (github: BrianBland) - handles parsing of WWW-Authenticate response headers registry/endpoint.go - EVEN MOAR LOGGING - Many edits throught to make the coad less dense. Sparse code is more readable code. - slit Ping() out to handle different code paths for v1 (unchanged logic) and v2. - Updated Endpoint struct type to include an entry for authorization challenges discovered during ping of a v2 registry. - If registry endpoint version is unknown, v2 code path is first attempted, then fallback to v1 upon failure. registry/service.go - STILL MOAR LOGGING - simplified the logic around starting the 'auth' job. registry/session.go - updated use of a registry.Endpoint struct field. registry/token.go - New File - Handles getting token from the parameters of a token auth challenge. - Modified from function written by Brian Bland (see above credit). registry/types.go - Removed 'DefaultAPIVersion' in lieu of 'APIVersionUnknown = 0'` Docker-DCO-1.1-Signed-off-by: Josh Hawn (github: jlhawn) --- docs/auth.go | 114 +++++++++++++++++++++++++++++- docs/authchallenge.go | 150 +++++++++++++++++++++++++++++++++++++++ docs/endpoint.go | 158 +++++++++++++++++++++++++++++++----------- docs/endpoint_test.go | 6 +- docs/service.go | 42 +++++++---- docs/session.go | 2 +- docs/token.go | 70 +++++++++++++++++++ docs/types.go | 5 +- 8 files changed, 484 insertions(+), 63 deletions(-) create mode 100644 docs/authchallenge.go create mode 100644 docs/token.go diff --git a/docs/auth.go b/docs/auth.go index 102078d7a..2044236cf 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -11,6 +11,7 @@ import ( "path" "strings" + log "github.com/Sirupsen/logrus" "github.com/docker/docker/utils" ) @@ -144,8 +145,18 @@ func SaveConfig(configFile *ConfigFile) error { return nil } -// try to register/login to the registry server -func Login(authConfig *AuthConfig, factory *utils.HTTPRequestFactory) (string, error) { +// Login tries to register/login to the registry server. +func Login(authConfig *AuthConfig, registryEndpoint *Endpoint, factory *utils.HTTPRequestFactory) (string, error) { + // Separates the v2 registry login logic from the v1 logic. + if registryEndpoint.Version == APIVersion2 { + return loginV2(authConfig, registryEndpoint, factory) + } + + return loginV1(authConfig, registryEndpoint, factory) +} + +// loginV1 tries to register/login to the v1 registry server. +func loginV1(authConfig *AuthConfig, registryEndpoint *Endpoint, factory *utils.HTTPRequestFactory) (string, error) { var ( status string reqBody []byte @@ -161,6 +172,8 @@ func Login(authConfig *AuthConfig, factory *utils.HTTPRequestFactory) (string, e serverAddress = authConfig.ServerAddress ) + log.Debugf("attempting v1 login to registry endpoint %s", registryEndpoint) + if serverAddress == "" { return "", fmt.Errorf("Server Error: Server Address not set.") } @@ -253,6 +266,103 @@ func Login(authConfig *AuthConfig, factory *utils.HTTPRequestFactory) (string, e return status, nil } +// loginV2 tries to login to the v2 registry server. The given registry endpoint has been +// pinged or setup with a list of authorization challenges. Each of these challenges are +// tried until one of them succeeds. Currently supported challenge schemes are: +// HTTP Basic Authorization +// Token Authorization with a separate token issuing server +// NOTE: the v2 logic does not attempt to create a user account if one doesn't exist. For +// now, users should create their account through other means like directly from a web page +// served by the v2 registry service provider. Whether this will be supported in the future +// is to be determined. +func loginV2(authConfig *AuthConfig, registryEndpoint *Endpoint, factory *utils.HTTPRequestFactory) (string, error) { + log.Debugf("attempting v2 login to registry endpoint %s", registryEndpoint) + + client := &http.Client{ + Transport: &http.Transport{ + DisableKeepAlives: true, + Proxy: http.ProxyFromEnvironment, + }, + CheckRedirect: AddRequiredHeadersToRedirectedRequests, + } + + var ( + err error + allErrors []error + ) + + for _, challenge := range registryEndpoint.AuthChallenges { + log.Debugf("trying %q auth challenge with params %s", challenge.Scheme, challenge.Parameters) + + switch strings.ToLower(challenge.Scheme) { + case "basic": + err = tryV2BasicAuthLogin(authConfig, challenge.Parameters, registryEndpoint, client, factory) + case "bearer": + err = tryV2TokenAuthLogin(authConfig, challenge.Parameters, registryEndpoint, client, factory) + default: + // Unsupported challenge types are explicitly skipped. + err = fmt.Errorf("unsupported auth scheme: %q", challenge.Scheme) + } + + if err == nil { + return "Login Succeeded", nil + } + + log.Debugf("error trying auth challenge %q: %s", challenge.Scheme, err) + + allErrors = append(allErrors, err) + } + + return "", fmt.Errorf("no successful auth challenge for %s - errors: %s", registryEndpoint, allErrors) +} + +func tryV2BasicAuthLogin(authConfig *AuthConfig, params map[string]string, registryEndpoint *Endpoint, client *http.Client, factory *utils.HTTPRequestFactory) error { + req, err := factory.NewRequest("GET", registryEndpoint.Path(""), nil) + if err != nil { + return err + } + + req.SetBasicAuth(authConfig.Username, authConfig.Password) + + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("basic auth attempt to %s realm %q failed with status: %d %s", registryEndpoint, params["realm"], resp.StatusCode, http.StatusText(resp.StatusCode)) + } + + return nil +} + +func tryV2TokenAuthLogin(authConfig *AuthConfig, params map[string]string, registryEndpoint *Endpoint, client *http.Client, factory *utils.HTTPRequestFactory) error { + token, err := getToken(authConfig.Username, authConfig.Password, params, registryEndpoint, client, factory) + if err != nil { + return err + } + + req, err := factory.NewRequest("GET", registryEndpoint.Path(""), nil) + if err != nil { + return err + } + + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("token auth attempt to %s realm %q failed with status: %d %s", registryEndpoint, params["realm"], resp.StatusCode, http.StatusText(resp.StatusCode)) + } + + return nil +} + // this method matches a auth configuration to a server address or a url func (config *ConfigFile) ResolveAuthConfig(index *IndexInfo) AuthConfig { configKey := index.GetAuthConfigKey() diff --git a/docs/authchallenge.go b/docs/authchallenge.go new file mode 100644 index 000000000..e300d82a0 --- /dev/null +++ b/docs/authchallenge.go @@ -0,0 +1,150 @@ +package registry + +import ( + "net/http" + "strings" +) + +// Octet types from RFC 2616. +type octetType byte + +// AuthorizationChallenge carries information +// from a WWW-Authenticate response header. +type AuthorizationChallenge struct { + Scheme string + Parameters map[string]string +} + +var octetTypes [256]octetType + +const ( + isToken octetType = 1 << iota + isSpace +) + +func init() { + // OCTET = + // CHAR = + // CTL = + // CR = + // LF = + // SP = + // HT = + // <"> = + // CRLF = CR LF + // LWS = [CRLF] 1*( SP | HT ) + // TEXT = + // separators = "(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\" | <"> + // | "/" | "[" | "]" | "?" | "=" | "{" | "}" | SP | HT + // token = 1* + // qdtext = > + + for c := 0; c < 256; c++ { + var t octetType + isCtl := c <= 31 || c == 127 + isChar := 0 <= c && c <= 127 + isSeparator := strings.IndexRune(" \t\"(),/:;<=>?@[]\\{}", rune(c)) >= 0 + if strings.IndexRune(" \t\r\n", rune(c)) >= 0 { + t |= isSpace + } + if isChar && !isCtl && !isSeparator { + t |= isToken + } + octetTypes[c] = t + } +} + +func parseAuthHeader(header http.Header) []*AuthorizationChallenge { + var challenges []*AuthorizationChallenge + for _, h := range header[http.CanonicalHeaderKey("WWW-Authenticate")] { + v, p := parseValueAndParams(h) + if v != "" { + challenges = append(challenges, &AuthorizationChallenge{Scheme: v, Parameters: p}) + } + } + return challenges +} + +func parseValueAndParams(header string) (value string, params map[string]string) { + params = make(map[string]string) + value, s := expectToken(header) + if value == "" { + return + } + value = strings.ToLower(value) + s = "," + skipSpace(s) + for strings.HasPrefix(s, ",") { + var pkey string + pkey, s = expectToken(skipSpace(s[1:])) + if pkey == "" { + return + } + if !strings.HasPrefix(s, "=") { + return + } + var pvalue string + pvalue, s = expectTokenOrQuoted(s[1:]) + if pvalue == "" { + return + } + pkey = strings.ToLower(pkey) + params[pkey] = pvalue + s = skipSpace(s) + } + return +} + +func skipSpace(s string) (rest string) { + i := 0 + for ; i < len(s); i++ { + if octetTypes[s[i]]&isSpace == 0 { + break + } + } + return s[i:] +} + +func expectToken(s string) (token, rest string) { + i := 0 + for ; i < len(s); i++ { + if octetTypes[s[i]]&isToken == 0 { + break + } + } + return s[:i], s[i:] +} + +func expectTokenOrQuoted(s string) (value string, rest string) { + if !strings.HasPrefix(s, "\"") { + return expectToken(s) + } + s = s[1:] + for i := 0; i < len(s); i++ { + switch s[i] { + case '"': + return s[:i], s[i+1:] + case '\\': + p := make([]byte, len(s)-1) + j := copy(p, s[:i]) + escape := true + for i = i + i; i < len(s); i++ { + b := s[i] + switch { + case escape: + escape = false + p[j] = b + j++ + case b == '\\': + escape = true + case b == '"': + return string(p[:j]), s[i+1:] + default: + p[j] = b + j++ + } + } + return "", "" + } + } + return "", "" +} diff --git a/docs/endpoint.go b/docs/endpoint.go index 95680c5ef..5c5b05200 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -15,28 +15,31 @@ import ( // for mocking in unit tests var lookupIP = net.LookupIP -// scans string for api version in the URL path. returns the trimmed hostname, if version found, string and API version. -func scanForAPIVersion(hostname string) (string, APIVersion) { +// scans string for api version in the URL path. returns the trimmed address, if version found, string and API version. +func scanForAPIVersion(address string) (string, APIVersion) { var ( chunks []string apiVersionStr string ) - if strings.HasSuffix(hostname, "/") { - chunks = strings.Split(hostname[:len(hostname)-1], "/") - apiVersionStr = chunks[len(chunks)-1] - } else { - chunks = strings.Split(hostname, "/") - apiVersionStr = chunks[len(chunks)-1] + + if strings.HasSuffix(address, "/") { + address = address[:len(address)-1] } + + chunks = strings.Split(address, "/") + apiVersionStr = chunks[len(chunks)-1] + for k, v := range apiVersions { if apiVersionStr == v { - hostname = strings.Join(chunks[:len(chunks)-1], "/") - return hostname, k + address = strings.Join(chunks[:len(chunks)-1], "/") + return address, k } } - return hostname, DefaultAPIVersion + + return address, APIVersionUnknown } +// NewEndpoint parses the given address to return a registry endpoint. func NewEndpoint(index *IndexInfo) (*Endpoint, error) { // *TODO: Allow per-registry configuration of endpoints. endpoint, err := newEndpoint(index.GetAuthConfigKey(), index.Secure) @@ -44,81 +47,124 @@ func NewEndpoint(index *IndexInfo) (*Endpoint, error) { return nil, err } + log.Debugf("pinging registry endpoint %s", endpoint) + // Try HTTPS ping to registry endpoint.URL.Scheme = "https" if _, err := endpoint.Ping(); err != nil { - - //TODO: triggering highland build can be done there without "failing" - if index.Secure { // If registry is secure and HTTPS failed, show user the error and tell them about `--insecure-registry` // in case that's what they need. DO NOT accept unknown CA certificates, and DO NOT fallback to HTTP. - return nil, fmt.Errorf("Invalid registry endpoint %s: %v. If this private registry supports only HTTP or HTTPS with an unknown CA certificate, please add `--insecure-registry %s` to the daemon's arguments. In the case of HTTPS, if you have access to the registry's CA certificate, no need for the flag; simply place the CA certificate at /etc/docker/certs.d/%s/ca.crt", endpoint, err, endpoint.URL.Host, endpoint.URL.Host) + return nil, fmt.Errorf("invalid registry endpoint %s: %v. If this private registry supports only HTTP or HTTPS with an unknown CA certificate, please add `--insecure-registry %s` to the daemon's arguments. In the case of HTTPS, if you have access to the registry's CA certificate, no need for the flag; simply place the CA certificate at /etc/docker/certs.d/%s/ca.crt", endpoint, err, endpoint.URL.Host, endpoint.URL.Host) } // If registry is insecure and HTTPS failed, fallback to HTTP. log.Debugf("Error from registry %q marked as insecure: %v. Insecurely falling back to HTTP", endpoint, err) endpoint.URL.Scheme = "http" - _, err2 := endpoint.Ping() - if err2 == nil { + + var err2 error + if _, err2 = endpoint.Ping(); err2 == nil { return endpoint, nil } - return nil, fmt.Errorf("Invalid registry endpoint %q. HTTPS attempt: %v. HTTP attempt: %v", endpoint, err, err2) + return nil, fmt.Errorf("invalid registry endpoint %q. HTTPS attempt: %v. HTTP attempt: %v", endpoint, err, err2) } return endpoint, nil } -func newEndpoint(hostname string, secure bool) (*Endpoint, error) { + +func newEndpoint(address string, secure bool) (*Endpoint, error) { var ( - endpoint = Endpoint{} - trimmedHostname string - err error + endpoint = new(Endpoint) + trimmedAddress string + err error ) - if !strings.HasPrefix(hostname, "http") { - hostname = "https://" + hostname + + if !strings.HasPrefix(address, "http") { + address = "https://" + address } - trimmedHostname, endpoint.Version = scanForAPIVersion(hostname) - endpoint.URL, err = url.Parse(trimmedHostname) - if err != nil { + + trimmedAddress, endpoint.Version = scanForAPIVersion(address) + + if endpoint.URL, err = url.Parse(trimmedAddress); err != nil { return nil, err } - endpoint.secure = secure - return &endpoint, nil + endpoint.IsSecure = secure + return endpoint, nil } func (repoInfo *RepositoryInfo) GetEndpoint() (*Endpoint, error) { return NewEndpoint(repoInfo.Index) } +// Endpoint stores basic information about a registry endpoint. type Endpoint struct { - URL *url.URL - Version APIVersion - secure bool + URL *url.URL + Version APIVersion + IsSecure bool + AuthChallenges []*AuthorizationChallenge } // Get the formated URL for the root of this registry Endpoint -func (e Endpoint) String() string { - return fmt.Sprintf("%s/v%d/", e.URL.String(), e.Version) +func (e *Endpoint) String() string { + return fmt.Sprintf("%s/v%d/", e.URL, e.Version) } -func (e Endpoint) VersionString(version APIVersion) string { - return fmt.Sprintf("%s/v%d/", e.URL.String(), version) +// VersionString returns a formatted string of this +// endpoint address using the given API Version. +func (e *Endpoint) VersionString(version APIVersion) string { + return fmt.Sprintf("%s/v%d/", e.URL, version) } -func (e Endpoint) Ping() (RegistryInfo, error) { +// Path returns a formatted string for the URL +// of this endpoint with the given path appended. +func (e *Endpoint) Path(path string) string { + return fmt.Sprintf("%s/v%d/%s", e.URL, e.Version, path) +} + +func (e *Endpoint) Ping() (RegistryInfo, error) { + // The ping logic to use is determined by the registry endpoint version. + switch e.Version { + case APIVersion1: + return e.pingV1() + case APIVersion2: + return e.pingV2() + } + + // APIVersionUnknown + // We should try v2 first... + e.Version = APIVersion2 + regInfo, errV2 := e.pingV2() + if errV2 == nil { + return regInfo, nil + } + + // ... then fallback to v1. + e.Version = APIVersion1 + regInfo, errV1 := e.pingV1() + if errV1 == nil { + return regInfo, nil + } + + e.Version = APIVersionUnknown + return RegistryInfo{}, fmt.Errorf("unable to ping registry endpoint %s\nv2 ping attempt failed with error: %s\n v1 ping attempt failed with error: %s", e, errV2, errV1) +} + +func (e *Endpoint) pingV1() (RegistryInfo, error) { + log.Debugf("attempting v1 ping for registry endpoint %s", e) + if e.String() == IndexServerAddress() { - // Skip the check, we now this one is valid + // Skip the check, we know this one is valid // (and we never want to fallback to http in case of error) return RegistryInfo{Standalone: false}, nil } - req, err := http.NewRequest("GET", e.String()+"_ping", nil) + req, err := http.NewRequest("GET", e.Path("_ping"), nil) if err != nil { return RegistryInfo{Standalone: false}, err } - resp, _, err := doRequest(req, nil, ConnectTimeout, e.secure) + resp, _, err := doRequest(req, nil, ConnectTimeout, e.IsSecure) if err != nil { return RegistryInfo{Standalone: false}, err } @@ -127,7 +173,7 @@ func (e Endpoint) Ping() (RegistryInfo, error) { jsonString, err := ioutil.ReadAll(resp.Body) if err != nil { - return RegistryInfo{Standalone: false}, fmt.Errorf("Error while reading the http response: %s", err) + return RegistryInfo{Standalone: false}, fmt.Errorf("error while reading the http response: %s", err) } // If the header is absent, we assume true for compatibility with earlier @@ -157,3 +203,33 @@ func (e Endpoint) Ping() (RegistryInfo, error) { log.Debugf("RegistryInfo.Standalone: %t", info.Standalone) return info, nil } + +func (e *Endpoint) pingV2() (RegistryInfo, error) { + log.Debugf("attempting v2 ping for registry endpoint %s", e) + + req, err := http.NewRequest("GET", e.Path(""), nil) + if err != nil { + return RegistryInfo{}, err + } + + resp, _, err := doRequest(req, nil, ConnectTimeout, e.IsSecure) + if err != nil { + return RegistryInfo{}, err + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusOK { + // It would seem that no authentication/authorization is required. + // So we don't need to parse/add any authorization schemes. + return RegistryInfo{Standalone: true}, nil + } + + if resp.StatusCode == http.StatusUnauthorized { + // Parse the WWW-Authenticate Header and store the challenges + // on this endpoint object. + e.AuthChallenges = parseAuthHeader(resp.Header) + return RegistryInfo{}, nil + } + + return RegistryInfo{}, fmt.Errorf("v2 registry endpoint returned status %d: %q", resp.StatusCode, http.StatusText(resp.StatusCode)) +} diff --git a/docs/endpoint_test.go b/docs/endpoint_test.go index b691a4fb9..f6489034f 100644 --- a/docs/endpoint_test.go +++ b/docs/endpoint_test.go @@ -8,8 +8,10 @@ func TestEndpointParse(t *testing.T) { expected string }{ {IndexServerAddress(), IndexServerAddress()}, - {"http://0.0.0.0:5000", "http://0.0.0.0:5000/v1/"}, - {"0.0.0.0:5000", "https://0.0.0.0:5000/v1/"}, + {"http://0.0.0.0:5000/v1/", "http://0.0.0.0:5000/v1/"}, + {"http://0.0.0.0:5000/v2/", "http://0.0.0.0:5000/v2/"}, + {"http://0.0.0.0:5000", "http://0.0.0.0:5000/v0/"}, + {"0.0.0.0:5000", "https://0.0.0.0:5000/v0/"}, } for _, td := range testData { e, err := newEndpoint(td.str, false) diff --git a/docs/service.go b/docs/service.go index c34e38423..048340224 100644 --- a/docs/service.go +++ b/docs/service.go @@ -1,6 +1,7 @@ package registry import ( + log "github.com/Sirupsen/logrus" "github.com/docker/docker/engine" ) @@ -38,28 +39,39 @@ func (s *Service) Install(eng *engine.Engine) error { // and returns OK if authentication was sucessful. // It can be used to verify the validity of a client's credentials. func (s *Service) Auth(job *engine.Job) engine.Status { - var authConfig = new(AuthConfig) + var ( + authConfig = new(AuthConfig) + endpoint *Endpoint + index *IndexInfo + status string + err error + ) job.GetenvJson("authConfig", authConfig) - if authConfig.ServerAddress != "" { - index, err := ResolveIndexInfo(job, authConfig.ServerAddress) - if err != nil { - return job.Error(err) - } - if !index.Official { - endpoint, err := NewEndpoint(index) - if err != nil { - return job.Error(err) - } - authConfig.ServerAddress = endpoint.String() - } + addr := authConfig.ServerAddress + if addr == "" { + // Use the official registry address if not specified. + addr = IndexServerAddress() } - status, err := Login(authConfig, HTTPRequestFactory(nil)) - if err != nil { + if index, err = ResolveIndexInfo(job, addr); err != nil { return job.Error(err) } + + if endpoint, err = NewEndpoint(index); err != nil { + log.Errorf("unable to get new registry endpoint: %s", err) + return job.Error(err) + } + + authConfig.ServerAddress = endpoint.String() + + if status, err = Login(authConfig, endpoint, HTTPRequestFactory(nil)); err != nil { + log.Errorf("unable to login against registry endpoint %s: %s", endpoint, err) + return job.Error(err) + } + + log.Infof("successful registry login for endpoint %s: %s", endpoint, status) job.Printf("%s\n", status) return engine.StatusOK diff --git a/docs/session.go b/docs/session.go index 781a91b15..b1980e1ae 100644 --- a/docs/session.go +++ b/docs/session.go @@ -65,7 +65,7 @@ func NewSession(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, endpo } func (r *Session) doRequest(req *http.Request) (*http.Response, *http.Client, error) { - return doRequest(req, r.jar, r.timeout, r.indexEndpoint.secure) + return doRequest(req, r.jar, r.timeout, r.indexEndpoint.IsSecure) } // Retrieve the history of a given image from the Registry. diff --git a/docs/token.go b/docs/token.go new file mode 100644 index 000000000..0403734f8 --- /dev/null +++ b/docs/token.go @@ -0,0 +1,70 @@ +package registry + +import ( + "errors" + "fmt" + "net/http" + "net/url" + "strings" + + "github.com/docker/docker/utils" +) + +func getToken(username, password string, params map[string]string, registryEndpoint *Endpoint, client *http.Client, factory *utils.HTTPRequestFactory) (token string, err error) { + realm, ok := params["realm"] + if !ok { + return "", errors.New("no realm specified for token auth challenge") + } + + realmURL, err := url.Parse(realm) + if err != nil { + return "", fmt.Errorf("invalid token auth challenge realm: %s", err) + } + + if realmURL.Scheme == "" { + if registryEndpoint.IsSecure { + realmURL.Scheme = "https" + } else { + realmURL.Scheme = "http" + } + } + + req, err := factory.NewRequest("GET", realmURL.String(), nil) + if err != nil { + return "", err + } + + reqParams := req.URL.Query() + service := params["service"] + scope := params["scope"] + + if service != "" { + reqParams.Add("service", service) + } + + for _, scopeField := range strings.Fields(scope) { + reqParams.Add("scope", scopeField) + } + + reqParams.Add("account", username) + + req.URL.RawQuery = reqParams.Encode() + req.SetBasicAuth(username, password) + + resp, err := client.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + + if !(resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusNoContent) { + return "", fmt.Errorf("token auth attempt for registry %s: %s request failed with status: %d %s", registryEndpoint, req.URL, resp.StatusCode, http.StatusText(resp.StatusCode)) + } + + token = resp.Header.Get("X-Auth-Token") + if token == "" { + return "", errors.New("token server did not include a token in the response header") + } + + return token, nil +} diff --git a/docs/types.go b/docs/types.go index fbbc0e709..bd0bf8b75 100644 --- a/docs/types.go +++ b/docs/types.go @@ -55,14 +55,15 @@ func (av APIVersion) String() string { return apiVersions[av] } -var DefaultAPIVersion APIVersion = APIVersion1 var apiVersions = map[APIVersion]string{ 1: "v1", 2: "v2", } +// API Version identifiers. const ( - APIVersion1 = iota + 1 + APIVersionUnknown = iota + APIVersion1 APIVersion2 ) From 06d0ef4179ee2d489018adac3800a491891d2336 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Tue, 30 Sep 2014 17:03:57 -0700 Subject: [PATCH 02/15] Push flow Signed-off-by: Derek McGowan (github: dmcgowan) --- docs/session_v2.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/session_v2.go b/docs/session_v2.go index 20e9e2ee9..0498bf702 100644 --- a/docs/session_v2.go +++ b/docs/session_v2.go @@ -267,7 +267,7 @@ func (r *Session) GetV2ImageBlobReader(imageName, sumType, sum string, token []s // Push the image to the server for storage. // 'layer' is an uncompressed reader of the blob to be pushed. // The server will generate it's own checksum calculation. -func (r *Session) PutV2ImageBlob(imageName, sumType string, blobRdr io.Reader, token []string) (serverChecksum string, err error) { +func (r *Session) PutV2ImageBlob(imageName, sumType, sumStr string, blobRdr io.Reader, token []string) (serverChecksum string, err error) { vars := map[string]string{ "imagename": imageName, "sumtype": sumType, @@ -285,6 +285,7 @@ func (r *Session) PutV2ImageBlob(imageName, sumType string, blobRdr io.Reader, t return "", err } setTokenAuth(req, token) + req.Header.Set("X-Tarsum", sumStr) res, _, err := r.doRequest(req) if err != nil { return "", err @@ -309,6 +310,10 @@ func (r *Session) PutV2ImageBlob(imageName, sumType string, blobRdr io.Reader, t return "", fmt.Errorf("unable to decode PutV2ImageBlob JSON response: %s", err) } + if sumInfo.Checksum != sumStr { + return "", fmt.Errorf("failed checksum comparison. serverChecksum: %q, localChecksum: %q", sumInfo.Checksum, sumStr) + } + // XXX this is a json struct from the registry, with its checksum return sumInfo.Checksum, nil } From 24895820bd88f05bd38c041995ea4ca91b88aa35 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Fri, 14 Nov 2014 16:22:06 -0800 Subject: [PATCH 03/15] Update push to use mount blob endpoint Using mount blob prevents repushing images which have already been uploaded Signed-off-by: Derek McGowan (github: dmcgowan) --- docs/session_v2.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/session_v2.go b/docs/session_v2.go index 0498bf702..86d0c228a 100644 --- a/docs/session_v2.go +++ b/docs/session_v2.go @@ -34,7 +34,7 @@ func newV2RegistryRouter() *mux.Router { v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9._+-]+}").Name("uploadBlob") // Mounting a blob in an image - v2Router.Path("/mountblob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9._+-]+}/{sum:[a-fA-F0-9]{4,}}").Name("mountBlob") + v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9._+-]+}/{sum:[a-fA-F0-9]{4,}}").Name("mountBlob") return router } @@ -184,7 +184,7 @@ func (r *Session) PostV2ImageMountBlob(imageName, sumType, sum string, token []s case 200: // return something indicating no push needed return true, nil - case 300: + case 404: // return something indicating blob push needed return false, nil } From e256a0e0bc06aca81812960b7509d0fa76356ac5 Mon Sep 17 00:00:00 2001 From: Josh Hawn Date: Fri, 12 Dec 2014 13:30:12 -0800 Subject: [PATCH 04/15] Update token response handling Registry authorization token is now taken from the response body rather than the repsonse header. Docker-DCO-1.1-Signed-off-by: Josh Hawn (github: jlhawn) --- docs/token.go | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/docs/token.go b/docs/token.go index 0403734f8..250486304 100644 --- a/docs/token.go +++ b/docs/token.go @@ -1,6 +1,7 @@ package registry import ( + "encoding/json" "errors" "fmt" "net/http" @@ -10,6 +11,10 @@ import ( "github.com/docker/docker/utils" ) +type tokenResponse struct { + Token string `json:"token"` +} + func getToken(username, password string, params map[string]string, registryEndpoint *Endpoint, client *http.Client, factory *utils.HTTPRequestFactory) (token string, err error) { realm, ok := params["realm"] if !ok { @@ -57,14 +62,20 @@ func getToken(username, password string, params map[string]string, registryEndpo } defer resp.Body.Close() - if !(resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusNoContent) { + if resp.StatusCode != http.StatusOK { return "", fmt.Errorf("token auth attempt for registry %s: %s request failed with status: %d %s", registryEndpoint, req.URL, resp.StatusCode, http.StatusText(resp.StatusCode)) } - token = resp.Header.Get("X-Auth-Token") - if token == "" { - return "", errors.New("token server did not include a token in the response header") + decoder := json.NewDecoder(resp.Body) + + tr := new(tokenResponse) + if err = decoder.Decode(tr); err != nil { + return "", fmt.Errorf("unable to decode token response: %s", err) } - return token, nil + if tr.Token == "" { + return "", errors.New("authorization server did not include a token in the response") + } + + return tr.Token, nil } From 2fcad2a10fa5463bdad44b243f5c257455cb9e14 Mon Sep 17 00:00:00 2001 From: Stephen J Day Date: Fri, 12 Dec 2014 11:27:22 -0800 Subject: [PATCH 05/15] Registry V2 HTTP route and error code definitions This package, ported from next-generation docker regsitry, includes route and error definitions. These facilitate compliant V2 client implementation. The portions of the HTTP API that are included in this package are considered to be locked down and should only be changed through a careful change proposal. Descriptor definitions package layout may change without affecting API behavior until the exported Go API is ready to be locked down. When the new registry stabilizes and becomes the master branch, this package can be vendored from the registry. Signed-off-by: Stephen J Day --- docs/v2/descriptors.go | 144 ++++++++++++++++++++++++++++++++ docs/v2/doc.go | 13 +++ docs/v2/errors.go | 185 +++++++++++++++++++++++++++++++++++++++++ docs/v2/errors_test.go | 165 ++++++++++++++++++++++++++++++++++++ docs/v2/routes.go | 69 +++++++++++++++ docs/v2/routes_test.go | 184 ++++++++++++++++++++++++++++++++++++++++ docs/v2/urls.go | 165 ++++++++++++++++++++++++++++++++++++ docs/v2/urls_test.go | 100 ++++++++++++++++++++++ 8 files changed, 1025 insertions(+) create mode 100644 docs/v2/descriptors.go create mode 100644 docs/v2/doc.go create mode 100644 docs/v2/errors.go create mode 100644 docs/v2/errors_test.go create mode 100644 docs/v2/routes.go create mode 100644 docs/v2/routes_test.go create mode 100644 docs/v2/urls.go create mode 100644 docs/v2/urls_test.go diff --git a/docs/v2/descriptors.go b/docs/v2/descriptors.go new file mode 100644 index 000000000..68d182411 --- /dev/null +++ b/docs/v2/descriptors.go @@ -0,0 +1,144 @@ +package v2 + +import "net/http" + +// TODO(stevvooe): Add route descriptors for each named route, along with +// accepted methods, parameters, returned status codes and error codes. + +// ErrorDescriptor provides relevant information about a given error code. +type ErrorDescriptor struct { + // Code is the error code that this descriptor describes. + Code ErrorCode + + // Value provides a unique, string key, often captilized with + // underscores, to identify the error code. This value is used as the + // keyed value when serializing api errors. + Value string + + // Message is a short, human readable decription of the error condition + // included in API responses. + Message string + + // Description provides a complete account of the errors purpose, suitable + // for use in documentation. + Description string + + // HTTPStatusCodes provides a list of status under which this error + // condition may arise. If it is empty, the error condition may be seen + // for any status code. + HTTPStatusCodes []int +} + +// ErrorDescriptors provides a list of HTTP API Error codes that may be +// encountered when interacting with the registry API. +var ErrorDescriptors = []ErrorDescriptor{ + { + Code: ErrorCodeUnknown, + Value: "UNKNOWN", + Message: "unknown error", + Description: `Generic error returned when the error does not have an + API classification.`, + }, + { + Code: ErrorCodeDigestInvalid, + Value: "DIGEST_INVALID", + Message: "provided digest did not match uploaded content", + Description: `When a blob is uploaded, the registry will check that + the content matches the digest provided by the client. The error may + include a detail structure with the key "digest", including the + invalid digest string. This error may also be returned when a manifest + includes an invalid layer digest.`, + HTTPStatusCodes: []int{http.StatusBadRequest, http.StatusNotFound}, + }, + { + Code: ErrorCodeSizeInvalid, + Value: "SIZE_INVALID", + Message: "provided length did not match content length", + Description: `When a layer is uploaded, the provided size will be + checked against the uploaded content. If they do not match, this error + will be returned.`, + HTTPStatusCodes: []int{http.StatusBadRequest}, + }, + { + Code: ErrorCodeNameInvalid, + Value: "NAME_INVALID", + Message: "manifest name did not match URI", + Description: `During a manifest upload, if the name in the manifest + does not match the uri name, this error will be returned.`, + HTTPStatusCodes: []int{http.StatusBadRequest, http.StatusNotFound}, + }, + { + Code: ErrorCodeTagInvalid, + Value: "TAG_INVALID", + Message: "manifest tag did not match URI", + Description: `During a manifest upload, if the tag in the manifest + does not match the uri tag, this error will be returned.`, + HTTPStatusCodes: []int{http.StatusBadRequest, http.StatusNotFound}, + }, + { + Code: ErrorCodeNameUnknown, + Value: "NAME_UNKNOWN", + Message: "repository name not known to registry", + Description: `This is returned if the name used during an operation is + unknown to the registry.`, + HTTPStatusCodes: []int{http.StatusNotFound}, + }, + { + Code: ErrorCodeManifestUnknown, + Value: "MANIFEST_UNKNOWN", + Message: "manifest unknown", + Description: `This error is returned when the manifest, identified by + name and tag is unknown to the repository.`, + HTTPStatusCodes: []int{http.StatusNotFound}, + }, + { + Code: ErrorCodeManifestInvalid, + Value: "MANIFEST_INVALID", + Message: "manifest invalid", + Description: `During upload, manifests undergo several checks ensuring + validity. If those checks fail, this error may be returned, unless a + more specific error is included. The detail will contain information + the failed validation.`, + HTTPStatusCodes: []int{http.StatusBadRequest}, + }, + { + Code: ErrorCodeManifestUnverified, + Value: "MANIFEST_UNVERIFIED", + Message: "manifest failed signature verification", + Description: `During manifest upload, if the manifest fails signature + verification, this error will be returned.`, + HTTPStatusCodes: []int{http.StatusBadRequest}, + }, + { + Code: ErrorCodeBlobUnknown, + Value: "BLOB_UNKNOWN", + Message: "blob unknown to registry", + Description: `This error may be returned when a blob is unknown to the + registry in a specified repository. This can be returned with a + standard get or if a manifest references an unknown layer during + upload.`, + HTTPStatusCodes: []int{http.StatusBadRequest, http.StatusNotFound}, + }, + + { + Code: ErrorCodeBlobUploadUnknown, + Value: "BLOB_UPLOAD_UNKNOWN", + Message: "blob upload unknown to registry", + Description: `If a blob upload has been cancelled or was never + started, this error code may be returned.`, + HTTPStatusCodes: []int{http.StatusNotFound}, + }, +} + +var errorCodeToDescriptors map[ErrorCode]ErrorDescriptor +var idToDescriptors map[string]ErrorDescriptor + +func init() { + errorCodeToDescriptors = make(map[ErrorCode]ErrorDescriptor, len(ErrorDescriptors)) + idToDescriptors = make(map[string]ErrorDescriptor, len(ErrorDescriptors)) + + for _, descriptor := range ErrorDescriptors { + errorCodeToDescriptors[descriptor.Code] = descriptor + idToDescriptors[descriptor.Value] = descriptor + } +} diff --git a/docs/v2/doc.go b/docs/v2/doc.go new file mode 100644 index 000000000..30fe2271a --- /dev/null +++ b/docs/v2/doc.go @@ -0,0 +1,13 @@ +// Package v2 describes routes, urls and the error codes used in the Docker +// Registry JSON HTTP API V2. In addition to declarations, descriptors are +// provided for routes and error codes that can be used for implementation and +// automatically generating documentation. +// +// Definitions here are considered to be locked down for the V2 registry api. +// Any changes must be considered carefully and should not proceed without a +// change proposal. +// +// Currently, while the HTTP API definitions are considered stable, the Go API +// exports are considered unstable. Go API consumers should take care when +// relying on these definitions until this message is deleted. +package v2 diff --git a/docs/v2/errors.go b/docs/v2/errors.go new file mode 100644 index 000000000..8c85d3a97 --- /dev/null +++ b/docs/v2/errors.go @@ -0,0 +1,185 @@ +package v2 + +import ( + "fmt" + "strings" +) + +// ErrorCode represents the error type. The errors are serialized via strings +// and the integer format may change and should *never* be exported. +type ErrorCode int + +const ( + // ErrorCodeUnknown is a catch-all for errors not defined below. + ErrorCodeUnknown ErrorCode = iota + + // ErrorCodeDigestInvalid is returned when uploading a blob if the + // provided digest does not match the blob contents. + ErrorCodeDigestInvalid + + // ErrorCodeSizeInvalid is returned when uploading a blob if the provided + // size does not match the content length. + ErrorCodeSizeInvalid + + // ErrorCodeNameInvalid is returned when the name in the manifest does not + // match the provided name. + ErrorCodeNameInvalid + + // ErrorCodeTagInvalid is returned when the tag in the manifest does not + // match the provided tag. + ErrorCodeTagInvalid + + // ErrorCodeNameUnknown when the repository name is not known. + ErrorCodeNameUnknown + + // ErrorCodeManifestUnknown returned when image manifest is unknown. + ErrorCodeManifestUnknown + + // ErrorCodeManifestInvalid returned when an image manifest is invalid, + // typically during a PUT operation. This error encompasses all errors + // encountered during manifest validation that aren't signature errors. + ErrorCodeManifestInvalid + + // ErrorCodeManifestUnverified is returned when the manifest fails + // signature verfication. + ErrorCodeManifestUnverified + + // ErrorCodeBlobUnknown is returned when a blob is unknown to the + // registry. This can happen when the manifest references a nonexistent + // layer or the result is not found by a blob fetch. + ErrorCodeBlobUnknown + + // ErrorCodeBlobUploadUnknown is returned when an upload is unknown. + ErrorCodeBlobUploadUnknown +) + +// ParseErrorCode attempts to parse the error code string, returning +// ErrorCodeUnknown if the error is not known. +func ParseErrorCode(s string) ErrorCode { + desc, ok := idToDescriptors[s] + + if !ok { + return ErrorCodeUnknown + } + + return desc.Code +} + +// Descriptor returns the descriptor for the error code. +func (ec ErrorCode) Descriptor() ErrorDescriptor { + d, ok := errorCodeToDescriptors[ec] + + if !ok { + return ErrorCodeUnknown.Descriptor() + } + + return d +} + +// String returns the canonical identifier for this error code. +func (ec ErrorCode) String() string { + return ec.Descriptor().Value +} + +// Message returned the human-readable error message for this error code. +func (ec ErrorCode) Message() string { + return ec.Descriptor().Message +} + +// MarshalText encodes the receiver into UTF-8-encoded text and returns the +// result. +func (ec ErrorCode) MarshalText() (text []byte, err error) { + return []byte(ec.String()), nil +} + +// UnmarshalText decodes the form generated by MarshalText. +func (ec *ErrorCode) UnmarshalText(text []byte) error { + desc, ok := idToDescriptors[string(text)] + + if !ok { + desc = ErrorCodeUnknown.Descriptor() + } + + *ec = desc.Code + + return nil +} + +// Error provides a wrapper around ErrorCode with extra Details provided. +type Error struct { + Code ErrorCode `json:"code"` + Message string `json:"message,omitempty"` + Detail interface{} `json:"detail,omitempty"` +} + +// Error returns a human readable representation of the error. +func (e Error) Error() string { + return fmt.Sprintf("%s: %s", + strings.ToLower(strings.Replace(e.Code.String(), "_", " ", -1)), + e.Message) +} + +// Errors provides the envelope for multiple errors and a few sugar methods +// for use within the application. +type Errors struct { + Errors []Error `json:"errors,omitempty"` +} + +// Push pushes an error on to the error stack, with the optional detail +// argument. It is a programming error (ie panic) to push more than one +// detail at a time. +func (errs *Errors) Push(code ErrorCode, details ...interface{}) { + if len(details) > 1 { + panic("please specify zero or one detail items for this error") + } + + var detail interface{} + if len(details) > 0 { + detail = details[0] + } + + if err, ok := detail.(error); ok { + detail = err.Error() + } + + errs.PushErr(Error{ + Code: code, + Message: code.Message(), + Detail: detail, + }) +} + +// PushErr pushes an error interface onto the error stack. +func (errs *Errors) PushErr(err error) { + switch err.(type) { + case Error: + errs.Errors = append(errs.Errors, err.(Error)) + default: + errs.Errors = append(errs.Errors, Error{Message: err.Error()}) + } +} + +func (errs *Errors) Error() string { + switch errs.Len() { + case 0: + return "" + case 1: + return errs.Errors[0].Error() + default: + msg := "errors:\n" + for _, err := range errs.Errors { + msg += err.Error() + "\n" + } + return msg + } +} + +// Clear clears the errors. +func (errs *Errors) Clear() { + errs.Errors = errs.Errors[:0] +} + +// Len returns the current number of errors. +func (errs *Errors) Len() int { + return len(errs.Errors) +} diff --git a/docs/v2/errors_test.go b/docs/v2/errors_test.go new file mode 100644 index 000000000..d2fc091ac --- /dev/null +++ b/docs/v2/errors_test.go @@ -0,0 +1,165 @@ +package v2 + +import ( + "encoding/json" + "reflect" + "testing" + + "github.com/docker/docker-registry/digest" +) + +// TestErrorCodes ensures that error code format, mappings and +// marshaling/unmarshaling. round trips are stable. +func TestErrorCodes(t *testing.T) { + for _, desc := range ErrorDescriptors { + if desc.Code.String() != desc.Value { + t.Fatalf("error code string incorrect: %q != %q", desc.Code.String(), desc.Value) + } + + if desc.Code.Message() != desc.Message { + t.Fatalf("incorrect message for error code %v: %q != %q", desc.Code, desc.Code.Message(), desc.Message) + } + + // Serialize the error code using the json library to ensure that we + // get a string and it works round trip. + p, err := json.Marshal(desc.Code) + + if err != nil { + t.Fatalf("error marshaling error code %v: %v", desc.Code, err) + } + + if len(p) <= 0 { + t.Fatalf("expected content in marshaled before for error code %v", desc.Code) + } + + // First, unmarshal to interface and ensure we have a string. + var ecUnspecified interface{} + if err := json.Unmarshal(p, &ecUnspecified); err != nil { + t.Fatalf("error unmarshaling error code %v: %v", desc.Code, err) + } + + if _, ok := ecUnspecified.(string); !ok { + t.Fatalf("expected a string for error code %v on unmarshal got a %T", desc.Code, ecUnspecified) + } + + // Now, unmarshal with the error code type and ensure they are equal + var ecUnmarshaled ErrorCode + if err := json.Unmarshal(p, &ecUnmarshaled); err != nil { + t.Fatalf("error unmarshaling error code %v: %v", desc.Code, err) + } + + if ecUnmarshaled != desc.Code { + t.Fatalf("unexpected error code during error code marshal/unmarshal: %v != %v", ecUnmarshaled, desc.Code) + } + } +} + +// TestErrorsManagement does a quick check of the Errors type to ensure that +// members are properly pushed and marshaled. +func TestErrorsManagement(t *testing.T) { + var errs Errors + + errs.Push(ErrorCodeDigestInvalid) + errs.Push(ErrorCodeBlobUnknown, + map[string]digest.Digest{"digest": "sometestblobsumdoesntmatter"}) + + p, err := json.Marshal(errs) + + if err != nil { + t.Fatalf("error marashaling errors: %v", err) + } + + expectedJSON := "{\"errors\":[{\"code\":\"DIGEST_INVALID\",\"message\":\"provided digest did not match uploaded content\"},{\"code\":\"BLOB_UNKNOWN\",\"message\":\"blob unknown to registry\",\"detail\":{\"digest\":\"sometestblobsumdoesntmatter\"}}]}" + + if string(p) != expectedJSON { + t.Fatalf("unexpected json: %q != %q", string(p), expectedJSON) + } + + errs.Clear() + errs.Push(ErrorCodeUnknown) + expectedJSON = "{\"errors\":[{\"code\":\"UNKNOWN\",\"message\":\"unknown error\"}]}" + p, err = json.Marshal(errs) + + if err != nil { + t.Fatalf("error marashaling errors: %v", err) + } + + if string(p) != expectedJSON { + t.Fatalf("unexpected json: %q != %q", string(p), expectedJSON) + } +} + +// TestMarshalUnmarshal ensures that api errors can round trip through json +// without losing information. +func TestMarshalUnmarshal(t *testing.T) { + + var errors Errors + + for _, testcase := range []struct { + description string + err Error + }{ + { + description: "unknown error", + err: Error{ + + Code: ErrorCodeUnknown, + Message: ErrorCodeUnknown.Descriptor().Message, + }, + }, + { + description: "unknown manifest", + err: Error{ + Code: ErrorCodeManifestUnknown, + Message: ErrorCodeManifestUnknown.Descriptor().Message, + }, + }, + { + description: "unknown manifest", + err: Error{ + Code: ErrorCodeBlobUnknown, + Message: ErrorCodeBlobUnknown.Descriptor().Message, + Detail: map[string]interface{}{"digest": "asdfqwerqwerqwerqwer"}, + }, + }, + } { + fatalf := func(format string, args ...interface{}) { + t.Fatalf(testcase.description+": "+format, args...) + } + + unexpectedErr := func(err error) { + fatalf("unexpected error: %v", err) + } + + p, err := json.Marshal(testcase.err) + if err != nil { + unexpectedErr(err) + } + + var unmarshaled Error + if err := json.Unmarshal(p, &unmarshaled); err != nil { + unexpectedErr(err) + } + + if !reflect.DeepEqual(unmarshaled, testcase.err) { + fatalf("errors not equal after round trip: %#v != %#v", unmarshaled, testcase.err) + } + + // Roll everything up into an error response envelope. + errors.PushErr(testcase.err) + } + + p, err := json.Marshal(errors) + if err != nil { + t.Fatalf("unexpected error marshaling error envelope: %v", err) + } + + var unmarshaled Errors + if err := json.Unmarshal(p, &unmarshaled); err != nil { + t.Fatalf("unexpected error unmarshaling error envelope: %v", err) + } + + if !reflect.DeepEqual(unmarshaled, errors) { + t.Fatalf("errors not equal after round trip: %#v != %#v", unmarshaled, errors) + } +} diff --git a/docs/v2/routes.go b/docs/v2/routes.go new file mode 100644 index 000000000..7ebe61d66 --- /dev/null +++ b/docs/v2/routes.go @@ -0,0 +1,69 @@ +package v2 + +import ( + "github.com/docker/docker-registry/common" + "github.com/gorilla/mux" +) + +// The following are definitions of the name under which all V2 routes are +// registered. These symbols can be used to look up a route based on the name. +const ( + RouteNameBase = "base" + RouteNameManifest = "manifest" + RouteNameTags = "tags" + RouteNameBlob = "blob" + RouteNameBlobUpload = "blob-upload" + RouteNameBlobUploadChunk = "blob-upload-chunk" +) + +var allEndpoints = []string{ + RouteNameManifest, + RouteNameTags, + RouteNameBlob, + RouteNameBlobUpload, + RouteNameBlobUploadChunk, +} + +// Router builds a gorilla router with named routes for the various API +// methods. This can be used directly by both server implementations and +// clients. +func Router() *mux.Router { + router := mux.NewRouter(). + StrictSlash(true) + + // GET /v2/ Check Check that the registry implements API version 2(.1) + router. + Path("/v2/"). + Name(RouteNameBase) + + // GET /v2//manifest/ Image Manifest Fetch the image manifest identified by name and tag. + // PUT /v2//manifest/ Image Manifest Upload the image manifest identified by name and tag. + // DELETE /v2//manifest/ Image Manifest Delete the image identified by name and tag. + router. + Path("/v2/{name:" + common.RepositoryNameRegexp.String() + "}/manifests/{tag:" + common.TagNameRegexp.String() + "}"). + Name(RouteNameManifest) + + // GET /v2//tags/list Tags Fetch the tags under the repository identified by name. + router. + Path("/v2/{name:" + common.RepositoryNameRegexp.String() + "}/tags/list"). + Name(RouteNameTags) + + // GET /v2//blob/ Layer Fetch the blob identified by digest. + router. + Path("/v2/{name:" + common.RepositoryNameRegexp.String() + "}/blobs/{digest:[a-zA-Z0-9-_+.]+:[a-zA-Z0-9-_+.=]+}"). + Name(RouteNameBlob) + + // POST /v2//blob/upload/ Layer Upload Initiate an upload of the layer identified by tarsum. + router. + Path("/v2/{name:" + common.RepositoryNameRegexp.String() + "}/blobs/uploads/"). + Name(RouteNameBlobUpload) + + // GET /v2//blob/upload/ Layer Upload Get the status of the upload identified by tarsum and uuid. + // PUT /v2//blob/upload/ Layer Upload Upload all or a chunk of the upload identified by tarsum and uuid. + // DELETE /v2//blob/upload/ Layer Upload Cancel the upload identified by layer and uuid + router. + Path("/v2/{name:" + common.RepositoryNameRegexp.String() + "}/blobs/uploads/{uuid}"). + Name(RouteNameBlobUploadChunk) + + return router +} diff --git a/docs/v2/routes_test.go b/docs/v2/routes_test.go new file mode 100644 index 000000000..9969ebcc4 --- /dev/null +++ b/docs/v2/routes_test.go @@ -0,0 +1,184 @@ +package v2 + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "reflect" + "testing" + + "github.com/gorilla/mux" +) + +type routeTestCase struct { + RequestURI string + Vars map[string]string + RouteName string + StatusCode int +} + +// TestRouter registers a test handler with all the routes and ensures that +// each route returns the expected path variables. Not method verification is +// present. This not meant to be exhaustive but as check to ensure that the +// expected variables are extracted. +// +// This may go away as the application structure comes together. +func TestRouter(t *testing.T) { + + router := Router() + + testHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + testCase := routeTestCase{ + RequestURI: r.RequestURI, + Vars: mux.Vars(r), + RouteName: mux.CurrentRoute(r).GetName(), + } + + enc := json.NewEncoder(w) + + if err := enc.Encode(testCase); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + }) + + // Startup test server + server := httptest.NewServer(router) + + for _, testcase := range []routeTestCase{ + { + RouteName: RouteNameBase, + RequestURI: "/v2/", + Vars: map[string]string{}, + }, + { + RouteName: RouteNameManifest, + RequestURI: "/v2/foo/bar/manifests/tag", + Vars: map[string]string{ + "name": "foo/bar", + "tag": "tag", + }, + }, + { + RouteName: RouteNameTags, + RequestURI: "/v2/foo/bar/tags/list", + Vars: map[string]string{ + "name": "foo/bar", + }, + }, + { + RouteName: RouteNameBlob, + RequestURI: "/v2/foo/bar/blobs/tarsum.dev+foo:abcdef0919234", + Vars: map[string]string{ + "name": "foo/bar", + "digest": "tarsum.dev+foo:abcdef0919234", + }, + }, + { + RouteName: RouteNameBlob, + RequestURI: "/v2/foo/bar/blobs/sha256:abcdef0919234", + Vars: map[string]string{ + "name": "foo/bar", + "digest": "sha256:abcdef0919234", + }, + }, + { + RouteName: RouteNameBlobUpload, + RequestURI: "/v2/foo/bar/blobs/uploads/", + Vars: map[string]string{ + "name": "foo/bar", + }, + }, + { + RouteName: RouteNameBlobUploadChunk, + RequestURI: "/v2/foo/bar/blobs/uploads/uuid", + Vars: map[string]string{ + "name": "foo/bar", + "uuid": "uuid", + }, + }, + { + RouteName: RouteNameBlobUploadChunk, + RequestURI: "/v2/foo/bar/blobs/uploads/D95306FA-FAD3-4E36-8D41-CF1C93EF8286", + Vars: map[string]string{ + "name": "foo/bar", + "uuid": "D95306FA-FAD3-4E36-8D41-CF1C93EF8286", + }, + }, + { + RouteName: RouteNameBlobUploadChunk, + RequestURI: "/v2/foo/bar/blobs/uploads/RDk1MzA2RkEtRkFEMy00RTM2LThENDEtQ0YxQzkzRUY4Mjg2IA==", + Vars: map[string]string{ + "name": "foo/bar", + "uuid": "RDk1MzA2RkEtRkFEMy00RTM2LThENDEtQ0YxQzkzRUY4Mjg2IA==", + }, + }, + { + // Check ambiguity: ensure we can distinguish between tags for + // "foo/bar/image/image" and image for "foo/bar/image" with tag + // "tags" + RouteName: RouteNameManifest, + RequestURI: "/v2/foo/bar/manifests/manifests/tags", + Vars: map[string]string{ + "name": "foo/bar/manifests", + "tag": "tags", + }, + }, + { + // This case presents an ambiguity between foo/bar with tag="tags" + // and list tags for "foo/bar/manifest" + RouteName: RouteNameTags, + RequestURI: "/v2/foo/bar/manifests/tags/list", + Vars: map[string]string{ + "name": "foo/bar/manifests", + }, + }, + { + RouteName: RouteNameBlobUploadChunk, + RequestURI: "/v2/foo/../../blob/uploads/D95306FA-FAD3-4E36-8D41-CF1C93EF8286", + StatusCode: http.StatusNotFound, + }, + } { + // Register the endpoint + router.GetRoute(testcase.RouteName).Handler(testHandler) + u := server.URL + testcase.RequestURI + + resp, err := http.Get(u) + + if err != nil { + t.Fatalf("error issuing get request: %v", err) + } + + if testcase.StatusCode == 0 { + // Override default, zero-value + testcase.StatusCode = http.StatusOK + } + + if resp.StatusCode != testcase.StatusCode { + t.Fatalf("unexpected status for %s: %v %v", u, resp.Status, resp.StatusCode) + } + + if testcase.StatusCode != http.StatusOK { + // We don't care about json response. + continue + } + + dec := json.NewDecoder(resp.Body) + + var actualRouteInfo routeTestCase + if err := dec.Decode(&actualRouteInfo); err != nil { + t.Fatalf("error reading json response: %v", err) + } + // Needs to be set out of band + actualRouteInfo.StatusCode = resp.StatusCode + + if actualRouteInfo.RouteName != testcase.RouteName { + t.Fatalf("incorrect route %q matched, expected %q", actualRouteInfo.RouteName, testcase.RouteName) + } + + if !reflect.DeepEqual(actualRouteInfo, testcase) { + t.Fatalf("actual does not equal expected: %#v != %#v", actualRouteInfo, testcase) + } + } + +} diff --git a/docs/v2/urls.go b/docs/v2/urls.go new file mode 100644 index 000000000..72f44299a --- /dev/null +++ b/docs/v2/urls.go @@ -0,0 +1,165 @@ +package v2 + +import ( + "net/http" + "net/url" + + "github.com/docker/docker-registry/digest" + "github.com/gorilla/mux" +) + +// URLBuilder creates registry API urls from a single base endpoint. It can be +// used to create urls for use in a registry client or server. +// +// All urls will be created from the given base, including the api version. +// For example, if a root of "/foo/" is provided, urls generated will be fall +// under "/foo/v2/...". Most application will only provide a schema, host and +// port, such as "https://localhost:5000/". +type URLBuilder struct { + root *url.URL // url root (ie http://localhost/) + router *mux.Router +} + +// NewURLBuilder creates a URLBuilder with provided root url object. +func NewURLBuilder(root *url.URL) *URLBuilder { + return &URLBuilder{ + root: root, + router: Router(), + } +} + +// NewURLBuilderFromString workes identically to NewURLBuilder except it takes +// a string argument for the root, returning an error if it is not a valid +// url. +func NewURLBuilderFromString(root string) (*URLBuilder, error) { + u, err := url.Parse(root) + if err != nil { + return nil, err + } + + return NewURLBuilder(u), nil +} + +// NewURLBuilderFromRequest uses information from an *http.Request to +// construct the root url. +func NewURLBuilderFromRequest(r *http.Request) *URLBuilder { + u := &url.URL{ + Scheme: r.URL.Scheme, + Host: r.Host, + } + + return NewURLBuilder(u) +} + +// BuildBaseURL constructs a base url for the API, typically just "/v2/". +func (ub *URLBuilder) BuildBaseURL() (string, error) { + route := ub.cloneRoute(RouteNameBase) + + baseURL, err := route.URL() + if err != nil { + return "", err + } + + return baseURL.String(), nil +} + +// BuildTagsURL constructs a url to list the tags in the named repository. +func (ub *URLBuilder) BuildTagsURL(name string) (string, error) { + route := ub.cloneRoute(RouteNameTags) + + tagsURL, err := route.URL("name", name) + if err != nil { + return "", err + } + + return tagsURL.String(), nil +} + +// BuildManifestURL constructs a url for the manifest identified by name and tag. +func (ub *URLBuilder) BuildManifestURL(name, tag string) (string, error) { + route := ub.cloneRoute(RouteNameManifest) + + manifestURL, err := route.URL("name", name, "tag", tag) + if err != nil { + return "", err + } + + return manifestURL.String(), nil +} + +// BuildBlobURL constructs the url for the blob identified by name and dgst. +func (ub *URLBuilder) BuildBlobURL(name string, dgst digest.Digest) (string, error) { + route := ub.cloneRoute(RouteNameBlob) + + layerURL, err := route.URL("name", name, "digest", dgst.String()) + if err != nil { + return "", err + } + + return layerURL.String(), nil +} + +// BuildBlobUploadURL constructs a url to begin a blob upload in the +// repository identified by name. +func (ub *URLBuilder) BuildBlobUploadURL(name string, values ...url.Values) (string, error) { + route := ub.cloneRoute(RouteNameBlobUpload) + + uploadURL, err := route.URL("name", name) + if err != nil { + return "", err + } + + return appendValuesURL(uploadURL, values...).String(), nil +} + +// BuildBlobUploadChunkURL constructs a url for the upload identified by uuid, +// including any url values. This should generally not be used by clients, as +// this url is provided by server implementations during the blob upload +// process. +func (ub *URLBuilder) BuildBlobUploadChunkURL(name, uuid string, values ...url.Values) (string, error) { + route := ub.cloneRoute(RouteNameBlobUploadChunk) + + uploadURL, err := route.URL("name", name, "uuid", uuid) + if err != nil { + return "", err + } + + return appendValuesURL(uploadURL, values...).String(), nil +} + +// clondedRoute returns a clone of the named route from the router. Routes +// must be cloned to avoid modifying them during url generation. +func (ub *URLBuilder) cloneRoute(name string) *mux.Route { + route := new(mux.Route) + *route = *ub.router.GetRoute(name) // clone the route + + return route. + Schemes(ub.root.Scheme). + Host(ub.root.Host) +} + +// appendValuesURL appends the parameters to the url. +func appendValuesURL(u *url.URL, values ...url.Values) *url.URL { + merged := u.Query() + + for _, v := range values { + for k, vv := range v { + merged[k] = append(merged[k], vv...) + } + } + + u.RawQuery = merged.Encode() + return u +} + +// appendValues appends the parameters to the url. Panics if the string is not +// a url. +func appendValues(u string, values ...url.Values) string { + up, err := url.Parse(u) + + if err != nil { + panic(err) // should never happen + } + + return appendValuesURL(up, values...).String() +} diff --git a/docs/v2/urls_test.go b/docs/v2/urls_test.go new file mode 100644 index 000000000..a9590dba9 --- /dev/null +++ b/docs/v2/urls_test.go @@ -0,0 +1,100 @@ +package v2 + +import ( + "net/url" + "testing" +) + +type urlBuilderTestCase struct { + description string + expected string + build func() (string, error) +} + +// TestURLBuilder tests the various url building functions, ensuring they are +// returning the expected values. +func TestURLBuilder(t *testing.T) { + + root := "http://localhost:5000/" + urlBuilder, err := NewURLBuilderFromString(root) + if err != nil { + t.Fatalf("unexpected error creating urlbuilder: %v", err) + } + + for _, testcase := range []struct { + description string + expected string + build func() (string, error) + }{ + { + description: "test base url", + expected: "http://localhost:5000/v2/", + build: urlBuilder.BuildBaseURL, + }, + { + description: "test tags url", + expected: "http://localhost:5000/v2/foo/bar/tags/list", + build: func() (string, error) { + return urlBuilder.BuildTagsURL("foo/bar") + }, + }, + { + description: "test manifest url", + expected: "http://localhost:5000/v2/foo/bar/manifests/tag", + build: func() (string, error) { + return urlBuilder.BuildManifestURL("foo/bar", "tag") + }, + }, + { + description: "build blob url", + expected: "http://localhost:5000/v2/foo/bar/blobs/tarsum.v1+sha256:abcdef0123456789", + build: func() (string, error) { + return urlBuilder.BuildBlobURL("foo/bar", "tarsum.v1+sha256:abcdef0123456789") + }, + }, + { + description: "build blob upload url", + expected: "http://localhost:5000/v2/foo/bar/blobs/uploads/", + build: func() (string, error) { + return urlBuilder.BuildBlobUploadURL("foo/bar") + }, + }, + { + description: "build blob upload url with digest and size", + expected: "http://localhost:5000/v2/foo/bar/blobs/uploads/?digest=tarsum.v1%2Bsha256%3Aabcdef0123456789&size=10000", + build: func() (string, error) { + return urlBuilder.BuildBlobUploadURL("foo/bar", url.Values{ + "size": []string{"10000"}, + "digest": []string{"tarsum.v1+sha256:abcdef0123456789"}, + }) + }, + }, + { + description: "build blob upload chunk url", + expected: "http://localhost:5000/v2/foo/bar/blobs/uploads/uuid-part", + build: func() (string, error) { + return urlBuilder.BuildBlobUploadChunkURL("foo/bar", "uuid-part") + }, + }, + { + description: "build blob upload chunk url with digest and size", + expected: "http://localhost:5000/v2/foo/bar/blobs/uploads/uuid-part?digest=tarsum.v1%2Bsha256%3Aabcdef0123456789&size=10000", + build: func() (string, error) { + return urlBuilder.BuildBlobUploadChunkURL("foo/bar", "uuid-part", url.Values{ + "size": []string{"10000"}, + "digest": []string{"tarsum.v1+sha256:abcdef0123456789"}, + }) + }, + }, + } { + u, err := testcase.build() + if err != nil { + t.Fatalf("%s: error building url: %v", testcase.description, err) + } + + if u != testcase.expected { + t.Fatalf("%s: %q != %q", testcase.description, u, testcase.expected) + } + } + +} From ee1e1abb15a46b325595eab68276f30543a68e92 Mon Sep 17 00:00:00 2001 From: Stephen J Day Date: Mon, 15 Dec 2014 12:42:52 -0800 Subject: [PATCH 06/15] Remove dependencies on registry packages Because docker core cannot vendor non-master Go dependencies, we need to remove dependencies on registry package. The definition of digest.Digest has been changed to a string and the regular expressions have been ported from docker-registry/common library. We'll likely change this be dependent on the registry in the future when the API stabilizies and use of the master branch becomes the norm. Signed-off-by: Stephen J Day --- docs/v2/errors_test.go | 4 +--- docs/v2/regexp.go | 19 +++++++++++++++++++ docs/v2/routes.go | 15 ++++++--------- docs/v2/urls.go | 5 ++--- 4 files changed, 28 insertions(+), 15 deletions(-) create mode 100644 docs/v2/regexp.go diff --git a/docs/v2/errors_test.go b/docs/v2/errors_test.go index d2fc091ac..4a80cdfe2 100644 --- a/docs/v2/errors_test.go +++ b/docs/v2/errors_test.go @@ -4,8 +4,6 @@ import ( "encoding/json" "reflect" "testing" - - "github.com/docker/docker-registry/digest" ) // TestErrorCodes ensures that error code format, mappings and @@ -61,7 +59,7 @@ func TestErrorsManagement(t *testing.T) { errs.Push(ErrorCodeDigestInvalid) errs.Push(ErrorCodeBlobUnknown, - map[string]digest.Digest{"digest": "sometestblobsumdoesntmatter"}) + map[string]string{"digest": "sometestblobsumdoesntmatter"}) p, err := json.Marshal(errs) diff --git a/docs/v2/regexp.go b/docs/v2/regexp.go new file mode 100644 index 000000000..b7e95b9ff --- /dev/null +++ b/docs/v2/regexp.go @@ -0,0 +1,19 @@ +package v2 + +import "regexp" + +// This file defines regular expressions for use in route definition. These +// are also defined in the registry code base. Until they are in a common, +// shared location, and exported, they must be repeated here. + +// RepositoryNameComponentRegexp restricts registtry path components names to +// start with at least two letters or numbers, with following parts able to +// separated by one period, dash or underscore. +var RepositoryNameComponentRegexp = regexp.MustCompile(`[a-z0-9]+(?:[._-][a-z0-9]+)*`) + +// RepositoryNameRegexp builds on RepositoryNameComponentRegexp to allow 2 to +// 5 path components, separated by a forward slash. +var RepositoryNameRegexp = regexp.MustCompile(`(?:` + RepositoryNameComponentRegexp.String() + `/){1,4}` + RepositoryNameComponentRegexp.String()) + +// TagNameRegexp matches valid tag names. From docker/docker:graph/tags.go. +var TagNameRegexp = regexp.MustCompile(`[\w][\w.-]{0,127}`) diff --git a/docs/v2/routes.go b/docs/v2/routes.go index 7ebe61d66..08f36e2f7 100644 --- a/docs/v2/routes.go +++ b/docs/v2/routes.go @@ -1,9 +1,6 @@ package v2 -import ( - "github.com/docker/docker-registry/common" - "github.com/gorilla/mux" -) +import "github.com/gorilla/mux" // The following are definitions of the name under which all V2 routes are // registered. These symbols can be used to look up a route based on the name. @@ -40,29 +37,29 @@ func Router() *mux.Router { // PUT /v2//manifest/ Image Manifest Upload the image manifest identified by name and tag. // DELETE /v2//manifest/ Image Manifest Delete the image identified by name and tag. router. - Path("/v2/{name:" + common.RepositoryNameRegexp.String() + "}/manifests/{tag:" + common.TagNameRegexp.String() + "}"). + Path("/v2/{name:" + RepositoryNameRegexp.String() + "}/manifests/{tag:" + TagNameRegexp.String() + "}"). Name(RouteNameManifest) // GET /v2//tags/list Tags Fetch the tags under the repository identified by name. router. - Path("/v2/{name:" + common.RepositoryNameRegexp.String() + "}/tags/list"). + Path("/v2/{name:" + RepositoryNameRegexp.String() + "}/tags/list"). Name(RouteNameTags) // GET /v2//blob/ Layer Fetch the blob identified by digest. router. - Path("/v2/{name:" + common.RepositoryNameRegexp.String() + "}/blobs/{digest:[a-zA-Z0-9-_+.]+:[a-zA-Z0-9-_+.=]+}"). + Path("/v2/{name:" + RepositoryNameRegexp.String() + "}/blobs/{digest:[a-zA-Z0-9-_+.]+:[a-zA-Z0-9-_+.=]+}"). Name(RouteNameBlob) // POST /v2//blob/upload/ Layer Upload Initiate an upload of the layer identified by tarsum. router. - Path("/v2/{name:" + common.RepositoryNameRegexp.String() + "}/blobs/uploads/"). + Path("/v2/{name:" + RepositoryNameRegexp.String() + "}/blobs/uploads/"). Name(RouteNameBlobUpload) // GET /v2//blob/upload/ Layer Upload Get the status of the upload identified by tarsum and uuid. // PUT /v2//blob/upload/ Layer Upload Upload all or a chunk of the upload identified by tarsum and uuid. // DELETE /v2//blob/upload/ Layer Upload Cancel the upload identified by layer and uuid router. - Path("/v2/{name:" + common.RepositoryNameRegexp.String() + "}/blobs/uploads/{uuid}"). + Path("/v2/{name:" + RepositoryNameRegexp.String() + "}/blobs/uploads/{uuid}"). Name(RouteNameBlobUploadChunk) return router diff --git a/docs/v2/urls.go b/docs/v2/urls.go index 72f44299a..19ef06fa1 100644 --- a/docs/v2/urls.go +++ b/docs/v2/urls.go @@ -4,7 +4,6 @@ import ( "net/http" "net/url" - "github.com/docker/docker-registry/digest" "github.com/gorilla/mux" ) @@ -88,10 +87,10 @@ func (ub *URLBuilder) BuildManifestURL(name, tag string) (string, error) { } // BuildBlobURL constructs the url for the blob identified by name and dgst. -func (ub *URLBuilder) BuildBlobURL(name string, dgst digest.Digest) (string, error) { +func (ub *URLBuilder) BuildBlobURL(name string, dgst string) (string, error) { route := ub.cloneRoute(RouteNameBlob) - layerURL, err := route.URL("name", name, "digest", dgst.String()) + layerURL, err := route.URL("name", name, "digest", dgst) if err != nil { return "", err } From 751a1a8dd0e1985eb921e888be3b027b7a6bfadb Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Tue, 16 Dec 2014 16:57:37 -0800 Subject: [PATCH 07/15] Update push and pull to registry 2.1 specification Signed-off-by: Derek McGowan --- docs/auth.go | 53 ++++++++ docs/session_v2.go | 298 +++++++++++++++------------------------------ 2 files changed, 151 insertions(+), 200 deletions(-) diff --git a/docs/auth.go b/docs/auth.go index 2044236cf..b138fb530 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -37,6 +37,59 @@ type ConfigFile struct { rootPath string } +type RequestAuthorization struct { + Token string + Username string + Password string +} + +func NewRequestAuthorization(authConfig *AuthConfig, registryEndpoint *Endpoint, resource, scope string, actions []string) (*RequestAuthorization, error) { + var auth RequestAuthorization + + client := &http.Client{ + Transport: &http.Transport{ + DisableKeepAlives: true, + Proxy: http.ProxyFromEnvironment, + }, + CheckRedirect: AddRequiredHeadersToRedirectedRequests, + } + factory := HTTPRequestFactory(nil) + + for _, challenge := range registryEndpoint.AuthChallenges { + log.Debugf("Using %q auth challenge with params %s for %s", challenge.Scheme, challenge.Parameters, authConfig.Username) + + switch strings.ToLower(challenge.Scheme) { + case "basic": + auth.Username = authConfig.Username + auth.Password = authConfig.Password + case "bearer": + params := map[string]string{} + for k, v := range challenge.Parameters { + params[k] = v + } + params["scope"] = fmt.Sprintf("%s:%s:%s", resource, scope, strings.Join(actions, ",")) + token, err := getToken(authConfig.Username, authConfig.Password, params, registryEndpoint, client, factory) + if err != nil { + return nil, err + } + + auth.Token = token + default: + log.Infof("Unsupported auth scheme: %q", challenge.Scheme) + } + } + + return &auth, nil +} + +func (auth *RequestAuthorization) Authorize(req *http.Request) { + if auth.Token != "" { + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", auth.Token)) + } else if auth.Username != "" && auth.Password != "" { + req.SetBasicAuth(auth.Username, auth.Password) + } +} + // create a base64 encoded auth string to store in config func encodeAuth(authConfig *AuthConfig) string { authStr := authConfig.Username + ":" + authConfig.Password diff --git a/docs/session_v2.go b/docs/session_v2.go index 86d0c228a..407c5f3a2 100644 --- a/docs/session_v2.go +++ b/docs/session_v2.go @@ -9,100 +9,34 @@ import ( "strconv" log "github.com/Sirupsen/logrus" + "github.com/docker/docker/registry/v2" "github.com/docker/docker/utils" - "github.com/gorilla/mux" ) -func newV2RegistryRouter() *mux.Router { - router := mux.NewRouter() +var registryURLBuilder *v2.URLBuilder - v2Router := router.PathPrefix("/v2/").Subrouter() - - // Version Info - v2Router.Path("/version").Name("version") - - // Image Manifests - v2Router.Path("/manifest/{imagename:[a-z0-9-._/]+}/{tagname:[a-zA-Z0-9-._]+}").Name("manifests") - - // List Image Tags - v2Router.Path("/tags/{imagename:[a-z0-9-._/]+}").Name("tags") - - // Download a blob - v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9._+-]+}/{sum:[a-fA-F0-9]{4,}}").Name("downloadBlob") - - // Upload a blob - v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9._+-]+}").Name("uploadBlob") - - // Mounting a blob in an image - v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9._+-]+}/{sum:[a-fA-F0-9]{4,}}").Name("mountBlob") - - return router -} - -// APIVersion2 /v2/ -var v2HTTPRoutes = newV2RegistryRouter() - -func getV2URL(e *Endpoint, routeName string, vars map[string]string) (*url.URL, error) { - route := v2HTTPRoutes.Get(routeName) - if route == nil { - return nil, fmt.Errorf("unknown regisry v2 route name: %q", routeName) - } - - varReplace := make([]string, 0, len(vars)*2) - for key, val := range vars { - varReplace = append(varReplace, key, val) - } - - routePath, err := route.URLPath(varReplace...) - if err != nil { - return nil, fmt.Errorf("unable to make registry route %q with vars %v: %s", routeName, vars, err) - } +func init() { u, err := url.Parse(REGISTRYSERVER) if err != nil { - return nil, fmt.Errorf("invalid registry url: %s", err) + panic(fmt.Errorf("invalid registry url: %s", err)) } - - return &url.URL{ - Scheme: u.Scheme, - Host: u.Host, - Path: routePath.Path, - }, nil + registryURLBuilder = v2.NewURLBuilder(u) } -// V2 Provenance POC +func getV2Builder(e *Endpoint) *v2.URLBuilder { + return registryURLBuilder +} -func (r *Session) GetV2Version(token []string) (*RegistryInfo, error) { - routeURL, err := getV2URL(r.indexEndpoint, "version", nil) - if err != nil { - return nil, err +// GetV2Authorization gets the authorization needed to the given image +// If readonly access is requested, then only the authorization may +// only be used for Get operations. +func (r *Session) GetV2Authorization(imageName string, readOnly bool) (*RequestAuthorization, error) { + scopes := []string{"pull"} + if !readOnly { + scopes = append(scopes, "push") } - method := "GET" - log.Debugf("[registry] Calling %q %s", method, routeURL.String()) - - req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil) - if err != nil { - return nil, err - } - setTokenAuth(req, token) - res, _, err := r.doRequest(req) - if err != nil { - return nil, err - } - defer res.Body.Close() - if res.StatusCode != 200 { - return nil, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d fetching Version", res.StatusCode), res) - } - - decoder := json.NewDecoder(res.Body) - versionInfo := new(RegistryInfo) - - err = decoder.Decode(versionInfo) - if err != nil { - return nil, fmt.Errorf("unable to decode GetV2Version JSON response: %s", err) - } - - return versionInfo, nil + return NewRequestAuthorization(r.GetAuthConfig(true), r.indexEndpoint, "repository", imageName, scopes) } // @@ -112,25 +46,20 @@ func (r *Session) GetV2Version(token []string) (*RegistryInfo, error) { // 1.c) if anything else, err // 2) PUT the created/signed manifest // -func (r *Session) GetV2ImageManifest(imageName, tagName string, token []string) ([]byte, error) { - vars := map[string]string{ - "imagename": imageName, - "tagname": tagName, - } - - routeURL, err := getV2URL(r.indexEndpoint, "manifests", vars) +func (r *Session) GetV2ImageManifest(imageName, tagName string, auth *RequestAuthorization) ([]byte, error) { + routeURL, err := getV2Builder(r.indexEndpoint).BuildManifestURL(imageName, tagName) if err != nil { return nil, err } method := "GET" - log.Debugf("[registry] Calling %q %s", method, routeURL.String()) + log.Debugf("[registry] Calling %q %s", method, routeURL) - req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil) + req, err := r.reqFactory.NewRequest(method, routeURL, nil) if err != nil { return nil, err } - setTokenAuth(req, token) + auth.Authorize(req) res, _, err := r.doRequest(req) if err != nil { return nil, err @@ -155,26 +84,20 @@ func (r *Session) GetV2ImageManifest(imageName, tagName string, token []string) // - Succeeded to mount for this image scope // - Failed with no error (So continue to Push the Blob) // - Failed with error -func (r *Session) PostV2ImageMountBlob(imageName, sumType, sum string, token []string) (bool, error) { - vars := map[string]string{ - "imagename": imageName, - "sumtype": sumType, - "sum": sum, - } - - routeURL, err := getV2URL(r.indexEndpoint, "mountBlob", vars) +func (r *Session) PostV2ImageMountBlob(imageName, sumType, sum string, auth *RequestAuthorization) (bool, error) { + routeURL, err := getV2Builder(r.indexEndpoint).BuildBlobURL(imageName, sumType+":"+sum) if err != nil { return false, err } - method := "POST" - log.Debugf("[registry] Calling %q %s", method, routeURL.String()) + method := "HEAD" + log.Debugf("[registry] Calling %q %s", method, routeURL) - req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil) + req, err := r.reqFactory.NewRequest(method, routeURL, nil) if err != nil { return false, err } - setTokenAuth(req, token) + auth.Authorize(req) res, _, err := r.doRequest(req) if err != nil { return false, err @@ -191,25 +114,19 @@ func (r *Session) PostV2ImageMountBlob(imageName, sumType, sum string, token []s return false, fmt.Errorf("Failed to mount %q - %s:%s : %d", imageName, sumType, sum, res.StatusCode) } -func (r *Session) GetV2ImageBlob(imageName, sumType, sum string, blobWrtr io.Writer, token []string) error { - vars := map[string]string{ - "imagename": imageName, - "sumtype": sumType, - "sum": sum, - } - - routeURL, err := getV2URL(r.indexEndpoint, "downloadBlob", vars) +func (r *Session) GetV2ImageBlob(imageName, sumType, sum string, blobWrtr io.Writer, auth *RequestAuthorization) error { + routeURL, err := getV2Builder(r.indexEndpoint).BuildBlobURL(imageName, sumType+":"+sum) if err != nil { return err } method := "GET" - log.Debugf("[registry] Calling %q %s", method, routeURL.String()) - req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil) + log.Debugf("[registry] Calling %q %s", method, routeURL) + req, err := r.reqFactory.NewRequest(method, routeURL, nil) if err != nil { return err } - setTokenAuth(req, token) + auth.Authorize(req) res, _, err := r.doRequest(req) if err != nil { return err @@ -226,25 +143,19 @@ func (r *Session) GetV2ImageBlob(imageName, sumType, sum string, blobWrtr io.Wri return err } -func (r *Session) GetV2ImageBlobReader(imageName, sumType, sum string, token []string) (io.ReadCloser, int64, error) { - vars := map[string]string{ - "imagename": imageName, - "sumtype": sumType, - "sum": sum, - } - - routeURL, err := getV2URL(r.indexEndpoint, "downloadBlob", vars) +func (r *Session) GetV2ImageBlobReader(imageName, sumType, sum string, auth *RequestAuthorization) (io.ReadCloser, int64, error) { + routeURL, err := getV2Builder(r.indexEndpoint).BuildBlobURL(imageName, sumType+":"+sum) if err != nil { return nil, 0, err } method := "GET" - log.Debugf("[registry] Calling %q %s", method, routeURL.String()) - req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil) + log.Debugf("[registry] Calling %q %s", method, routeURL) + req, err := r.reqFactory.NewRequest(method, routeURL, nil) if err != nil { return nil, 0, err } - setTokenAuth(req, token) + auth.Authorize(req) res, _, err := r.doRequest(req) if err != nil { return nil, 0, err @@ -267,85 +178,76 @@ func (r *Session) GetV2ImageBlobReader(imageName, sumType, sum string, token []s // Push the image to the server for storage. // 'layer' is an uncompressed reader of the blob to be pushed. // The server will generate it's own checksum calculation. -func (r *Session) PutV2ImageBlob(imageName, sumType, sumStr string, blobRdr io.Reader, token []string) (serverChecksum string, err error) { - vars := map[string]string{ - "imagename": imageName, - "sumtype": sumType, +func (r *Session) PutV2ImageBlob(imageName, sumType, sumStr string, blobRdr io.Reader, auth *RequestAuthorization) error { + routeURL, err := getV2Builder(r.indexEndpoint).BuildBlobUploadURL(imageName) + if err != nil { + return err } - routeURL, err := getV2URL(r.indexEndpoint, "uploadBlob", vars) + log.Debugf("[registry] Calling %q %s", "POST", routeURL) + req, err := r.reqFactory.NewRequest("POST", routeURL, nil) if err != nil { - return "", err + return err } - method := "PUT" - log.Debugf("[registry] Calling %q %s", method, routeURL.String()) - req, err := r.reqFactory.NewRequest(method, routeURL.String(), blobRdr) - if err != nil { - return "", err - } - setTokenAuth(req, token) - req.Header.Set("X-Tarsum", sumStr) + auth.Authorize(req) res, _, err := r.doRequest(req) if err != nil { - return "", err + return err + } + location := res.Header.Get("Location") + + method := "PUT" + log.Debugf("[registry] Calling %q %s", method, location) + req, err = r.reqFactory.NewRequest(method, location, blobRdr) + if err != nil { + return err + } + queryParams := url.Values{} + queryParams.Add("digest", sumType+":"+sumStr) + req.URL.RawQuery = queryParams.Encode() + auth.Authorize(req) + res, _, err = r.doRequest(req) + if err != nil { + return err } defer res.Body.Close() - if res.StatusCode != 201 { - if res.StatusCode == 401 { - return "", errLoginRequired - } - return "", utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s blob", res.StatusCode, imageName), res) - } - type sumReturn struct { - Checksum string `json:"checksum"` - } - - decoder := json.NewDecoder(res.Body) - var sumInfo sumReturn - - err = decoder.Decode(&sumInfo) - if err != nil { - return "", fmt.Errorf("unable to decode PutV2ImageBlob JSON response: %s", err) - } - - if sumInfo.Checksum != sumStr { - return "", fmt.Errorf("failed checksum comparison. serverChecksum: %q, localChecksum: %q", sumInfo.Checksum, sumStr) - } - - // XXX this is a json struct from the registry, with its checksum - return sumInfo.Checksum, nil -} - -// Finally Push the (signed) manifest of the blobs we've just pushed -func (r *Session) PutV2ImageManifest(imageName, tagName string, manifestRdr io.Reader, token []string) error { - vars := map[string]string{ - "imagename": imageName, - "tagname": tagName, - } - - routeURL, err := getV2URL(r.indexEndpoint, "manifests", vars) - if err != nil { - return err - } - - method := "PUT" - log.Debugf("[registry] Calling %q %s", method, routeURL.String()) - req, err := r.reqFactory.NewRequest(method, routeURL.String(), manifestRdr) - if err != nil { - return err - } - setTokenAuth(req, token) - res, _, err := r.doRequest(req) - if err != nil { - return err - } - res.Body.Close() if res.StatusCode != 201 { if res.StatusCode == 401 { return errLoginRequired } + return utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s blob", res.StatusCode, imageName), res) + } + + return nil +} + +// Finally Push the (signed) manifest of the blobs we've just pushed +func (r *Session) PutV2ImageManifest(imageName, tagName string, manifestRdr io.Reader, auth *RequestAuthorization) error { + routeURL, err := getV2Builder(r.indexEndpoint).BuildManifestURL(imageName, tagName) + if err != nil { + return err + } + + method := "PUT" + log.Debugf("[registry] Calling %q %s", method, routeURL) + req, err := r.reqFactory.NewRequest(method, routeURL, manifestRdr) + if err != nil { + return err + } + auth.Authorize(req) + res, _, err := r.doRequest(req) + if err != nil { + return err + } + b, _ := ioutil.ReadAll(res.Body) + res.Body.Close() + if res.StatusCode != 200 { + if res.StatusCode == 401 { + return errLoginRequired + } + log.Debugf("Unexpected response from server: %q %#v", b, res.Header) return utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s:%s manifest", res.StatusCode, imageName, tagName), res) } @@ -353,24 +255,20 @@ func (r *Session) PutV2ImageManifest(imageName, tagName string, manifestRdr io.R } // Given a repository name, returns a json array of string tags -func (r *Session) GetV2RemoteTags(imageName string, token []string) ([]string, error) { - vars := map[string]string{ - "imagename": imageName, - } - - routeURL, err := getV2URL(r.indexEndpoint, "tags", vars) +func (r *Session) GetV2RemoteTags(imageName string, auth *RequestAuthorization) ([]string, error) { + routeURL, err := getV2Builder(r.indexEndpoint).BuildTagsURL(imageName) if err != nil { return nil, err } method := "GET" - log.Debugf("[registry] Calling %q %s", method, routeURL.String()) + log.Debugf("[registry] Calling %q %s", method, routeURL) - req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil) + req, err := r.reqFactory.NewRequest(method, routeURL, nil) if err != nil { return nil, err } - setTokenAuth(req, token) + auth.Authorize(req) res, _, err := r.doRequest(req) if err != nil { return nil, err From 6f36ce3a0183e750adef930ae4a4cb8e7d47683c Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Fri, 19 Dec 2014 14:44:18 -0800 Subject: [PATCH 08/15] Allow private V2 registry endpoints Signed-off-by: Derek McGowan --- docs/config.go | 2 +- docs/endpoint.go | 2 ++ docs/session_v2.go | 32 +++++++++++++++++++------------- 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/docs/config.go b/docs/config.go index b5652b15d..4d13aaea3 100644 --- a/docs/config.go +++ b/docs/config.go @@ -23,7 +23,7 @@ type Options struct { const ( // Only used for user auth + account creation INDEXSERVER = "https://index.docker.io/v1/" - REGISTRYSERVER = "https://registry-1.docker.io/v1/" + REGISTRYSERVER = "https://registry-1.docker.io/v2/" INDEXNAME = "docker.io" // INDEXSERVER = "https://registry-stage.hub.docker.com/v1/" diff --git a/docs/endpoint.go b/docs/endpoint.go index 5c5b05200..9a783f1f0 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -10,6 +10,7 @@ import ( "strings" log "github.com/Sirupsen/logrus" + "github.com/docker/docker/registry/v2" ) // for mocking in unit tests @@ -103,6 +104,7 @@ type Endpoint struct { Version APIVersion IsSecure bool AuthChallenges []*AuthorizationChallenge + URLBuilder *v2.URLBuilder } // Get the formated URL for the root of this registry Endpoint diff --git a/docs/session_v2.go b/docs/session_v2.go index 407c5f3a2..2304a6134 100644 --- a/docs/session_v2.go +++ b/docs/session_v2.go @@ -13,30 +13,36 @@ import ( "github.com/docker/docker/utils" ) -var registryURLBuilder *v2.URLBuilder - -func init() { - u, err := url.Parse(REGISTRYSERVER) - if err != nil { - panic(fmt.Errorf("invalid registry url: %s", err)) - } - registryURLBuilder = v2.NewURLBuilder(u) -} - func getV2Builder(e *Endpoint) *v2.URLBuilder { - return registryURLBuilder + if e.URLBuilder == nil { + e.URLBuilder = v2.NewURLBuilder(e.URL) + } + return e.URLBuilder } // GetV2Authorization gets the authorization needed to the given image // If readonly access is requested, then only the authorization may // only be used for Get operations. -func (r *Session) GetV2Authorization(imageName string, readOnly bool) (*RequestAuthorization, error) { +func (r *Session) GetV2Authorization(imageName string, readOnly bool) (auth *RequestAuthorization, err error) { scopes := []string{"pull"} if !readOnly { scopes = append(scopes, "push") } - return NewRequestAuthorization(r.GetAuthConfig(true), r.indexEndpoint, "repository", imageName, scopes) + var registry *Endpoint + if r.indexEndpoint.URL.Host == IndexServerURL.Host { + registry, err = NewEndpoint(REGISTRYSERVER, nil) + if err != nil { + return + } + } else { + registry = r.indexEndpoint + } + registry.URLBuilder = v2.NewURLBuilder(registry.URL) + r.indexEndpoint = registry + + log.Debugf("Getting authorization for %s %s", imageName, scopes) + return NewRequestAuthorization(r.GetAuthConfig(true), registry, "repository", imageName, scopes) } // From 22c7328529311bfa6c38b67df6156658e2a2f411 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Fri, 19 Dec 2014 16:14:04 -0800 Subject: [PATCH 09/15] Get token on each request Signed-off-by: Derek McGowan --- docs/auth.go | 60 ++++++++++++++++++++++++++++------------------ docs/session_v2.go | 34 +++++++++++++++++++------- 2 files changed, 62 insertions(+), 32 deletions(-) diff --git a/docs/auth.go b/docs/auth.go index b138fb530..1e1c7ddb8 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -38,56 +38,70 @@ type ConfigFile struct { } type RequestAuthorization struct { - Token string - Username string - Password string + authConfig *AuthConfig + registryEndpoint *Endpoint + resource string + scope string + actions []string } -func NewRequestAuthorization(authConfig *AuthConfig, registryEndpoint *Endpoint, resource, scope string, actions []string) (*RequestAuthorization, error) { - var auth RequestAuthorization +func NewRequestAuthorization(authConfig *AuthConfig, registryEndpoint *Endpoint, resource, scope string, actions []string) *RequestAuthorization { + return &RequestAuthorization{ + authConfig: authConfig, + registryEndpoint: registryEndpoint, + resource: resource, + scope: scope, + actions: actions, + } +} +func (auth *RequestAuthorization) getToken() (string, error) { + // TODO check if already has token and before expiration client := &http.Client{ Transport: &http.Transport{ DisableKeepAlives: true, - Proxy: http.ProxyFromEnvironment, - }, + Proxy: http.ProxyFromEnvironment}, CheckRedirect: AddRequiredHeadersToRedirectedRequests, } factory := HTTPRequestFactory(nil) - for _, challenge := range registryEndpoint.AuthChallenges { - log.Debugf("Using %q auth challenge with params %s for %s", challenge.Scheme, challenge.Parameters, authConfig.Username) - + for _, challenge := range auth.registryEndpoint.AuthChallenges { switch strings.ToLower(challenge.Scheme) { case "basic": - auth.Username = authConfig.Username - auth.Password = authConfig.Password + // no token necessary case "bearer": + log.Debugf("Getting bearer token with %s for %s", challenge.Parameters, auth.authConfig.Username) params := map[string]string{} for k, v := range challenge.Parameters { params[k] = v } - params["scope"] = fmt.Sprintf("%s:%s:%s", resource, scope, strings.Join(actions, ",")) - token, err := getToken(authConfig.Username, authConfig.Password, params, registryEndpoint, client, factory) + params["scope"] = fmt.Sprintf("%s:%s:%s", auth.resource, auth.scope, strings.Join(auth.actions, ",")) + token, err := getToken(auth.authConfig.Username, auth.authConfig.Password, params, auth.registryEndpoint, client, factory) if err != nil { - return nil, err + return "", err } + // TODO cache token and set expiration to one minute from now - auth.Token = token + return token, nil default: log.Infof("Unsupported auth scheme: %q", challenge.Scheme) } } - - return &auth, nil + // TODO no expiration, do not reattempt to get a token + return "", nil } -func (auth *RequestAuthorization) Authorize(req *http.Request) { - if auth.Token != "" { - req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", auth.Token)) - } else if auth.Username != "" && auth.Password != "" { - req.SetBasicAuth(auth.Username, auth.Password) +func (auth *RequestAuthorization) Authorize(req *http.Request) error { + token, err := auth.getToken() + if err != nil { + return err } + if token != "" { + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + } else if auth.authConfig.Username != "" && auth.authConfig.Password != "" { + req.SetBasicAuth(auth.authConfig.Username, auth.authConfig.Password) + } + return nil } // create a base64 encoded auth string to store in config diff --git a/docs/session_v2.go b/docs/session_v2.go index 2304a6134..491cd2c6e 100644 --- a/docs/session_v2.go +++ b/docs/session_v2.go @@ -42,7 +42,7 @@ func (r *Session) GetV2Authorization(imageName string, readOnly bool) (auth *Req r.indexEndpoint = registry log.Debugf("Getting authorization for %s %s", imageName, scopes) - return NewRequestAuthorization(r.GetAuthConfig(true), registry, "repository", imageName, scopes) + return NewRequestAuthorization(r.GetAuthConfig(true), registry, "repository", imageName, scopes), nil } // @@ -65,7 +65,9 @@ func (r *Session) GetV2ImageManifest(imageName, tagName string, auth *RequestAut if err != nil { return nil, err } - auth.Authorize(req) + if err := auth.Authorize(req) { + return nil, err + } res, _, err := r.doRequest(req) if err != nil { return nil, err @@ -103,7 +105,9 @@ func (r *Session) PostV2ImageMountBlob(imageName, sumType, sum string, auth *Req if err != nil { return false, err } - auth.Authorize(req) + if err := auth.Authorize(req) { + return nil, err + } res, _, err := r.doRequest(req) if err != nil { return false, err @@ -132,7 +136,9 @@ func (r *Session) GetV2ImageBlob(imageName, sumType, sum string, blobWrtr io.Wri if err != nil { return err } - auth.Authorize(req) + if err := auth.Authorize(req) { + return nil, err + } res, _, err := r.doRequest(req) if err != nil { return err @@ -161,7 +167,9 @@ func (r *Session) GetV2ImageBlobReader(imageName, sumType, sum string, auth *Req if err != nil { return nil, 0, err } - auth.Authorize(req) + if err := auth.Authorize(req) { + return nil, err + } res, _, err := r.doRequest(req) if err != nil { return nil, 0, err @@ -196,7 +204,9 @@ func (r *Session) PutV2ImageBlob(imageName, sumType, sumStr string, blobRdr io.R return err } - auth.Authorize(req) + if err := auth.Authorize(req) { + return nil, err + } res, _, err := r.doRequest(req) if err != nil { return err @@ -212,7 +222,9 @@ func (r *Session) PutV2ImageBlob(imageName, sumType, sumStr string, blobRdr io.R queryParams := url.Values{} queryParams.Add("digest", sumType+":"+sumStr) req.URL.RawQuery = queryParams.Encode() - auth.Authorize(req) + if err := auth.Authorize(req) { + return nil, err + } res, _, err = r.doRequest(req) if err != nil { return err @@ -242,7 +254,9 @@ func (r *Session) PutV2ImageManifest(imageName, tagName string, manifestRdr io.R if err != nil { return err } - auth.Authorize(req) + if err := auth.Authorize(req) { + return nil, err + } res, _, err := r.doRequest(req) if err != nil { return err @@ -274,7 +288,9 @@ func (r *Session) GetV2RemoteTags(imageName string, auth *RequestAuthorization) if err != nil { return nil, err } - auth.Authorize(req) + if err := auth.Authorize(req) { + return nil, err + } res, _, err := r.doRequest(req) if err != nil { return nil, err From 6f09abd5c97aa1c8efc491b4fb0eb73c73a53a8f Mon Sep 17 00:00:00 2001 From: Stephen J Day Date: Mon, 22 Dec 2014 14:58:08 -0800 Subject: [PATCH 10/15] Correctly check and propagate errors in v2 session Signed-off-by: Stephen J Day --- docs/session_v2.go | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/session_v2.go b/docs/session_v2.go index 491cd2c6e..411df46e3 100644 --- a/docs/session_v2.go +++ b/docs/session_v2.go @@ -65,7 +65,7 @@ func (r *Session) GetV2ImageManifest(imageName, tagName string, auth *RequestAut if err != nil { return nil, err } - if err := auth.Authorize(req) { + if err := auth.Authorize(req); err != nil { return nil, err } res, _, err := r.doRequest(req) @@ -105,8 +105,8 @@ func (r *Session) PostV2ImageMountBlob(imageName, sumType, sum string, auth *Req if err != nil { return false, err } - if err := auth.Authorize(req) { - return nil, err + if err := auth.Authorize(req); err != nil { + return false, err } res, _, err := r.doRequest(req) if err != nil { @@ -136,8 +136,8 @@ func (r *Session) GetV2ImageBlob(imageName, sumType, sum string, blobWrtr io.Wri if err != nil { return err } - if err := auth.Authorize(req) { - return nil, err + if err := auth.Authorize(req); err != nil { + return err } res, _, err := r.doRequest(req) if err != nil { @@ -167,8 +167,8 @@ func (r *Session) GetV2ImageBlobReader(imageName, sumType, sum string, auth *Req if err != nil { return nil, 0, err } - if err := auth.Authorize(req) { - return nil, err + if err := auth.Authorize(req); err != nil { + return nil, 0, err } res, _, err := r.doRequest(req) if err != nil { @@ -204,8 +204,8 @@ func (r *Session) PutV2ImageBlob(imageName, sumType, sumStr string, blobRdr io.R return err } - if err := auth.Authorize(req) { - return nil, err + if err := auth.Authorize(req); err != nil { + return err } res, _, err := r.doRequest(req) if err != nil { @@ -222,8 +222,8 @@ func (r *Session) PutV2ImageBlob(imageName, sumType, sumStr string, blobRdr io.R queryParams := url.Values{} queryParams.Add("digest", sumType+":"+sumStr) req.URL.RawQuery = queryParams.Encode() - if err := auth.Authorize(req) { - return nil, err + if err := auth.Authorize(req); err != nil { + return err } res, _, err = r.doRequest(req) if err != nil { @@ -254,8 +254,8 @@ func (r *Session) PutV2ImageManifest(imageName, tagName string, manifestRdr io.R if err != nil { return err } - if err := auth.Authorize(req) { - return nil, err + if err := auth.Authorize(req); err != nil { + return err } res, _, err := r.doRequest(req) if err != nil { @@ -288,7 +288,7 @@ func (r *Session) GetV2RemoteTags(imageName string, auth *RequestAuthorization) if err != nil { return nil, err } - if err := auth.Authorize(req) { + if err := auth.Authorize(req); err != nil { return nil, err } res, _, err := r.doRequest(req) From 826bde851b3c0962c9075d123e9c3a00d14dc883 Mon Sep 17 00:00:00 2001 From: Josh Hawn Date: Tue, 23 Dec 2014 13:40:06 -0800 Subject: [PATCH 11/15] Add Tarsum Calculation during v2 Pull operation While the v2 pull operation is writing the body of the layer blob to disk it now computes the tarsum checksum of the archive before extracting it to the backend storage driver. If the checksum does not match that from the image manifest an error is raised. Also adds more debug logging to the pull operation and fixes existing test cases which were failing. Adds a reverse lookup constructor to the tarsum package so that you can get a tarsum object using a checksum label. Docker-DCO-1.1-Signed-off-by: Josh Hawn (github: jlhawn) --- docs/endpoint.go | 17 ++++++++++++----- docs/session_v2.go | 8 ++++++-- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/docs/endpoint.go b/docs/endpoint.go index 9a783f1f0..9ca9ed8b9 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -47,16 +47,23 @@ func NewEndpoint(index *IndexInfo) (*Endpoint, error) { if err != nil { return nil, err } + if err := validateEndpoint(endpoint); err != nil { + return nil, err + } + return endpoint, nil +} + +func validateEndpoint(endpoint *Endpoint) error { log.Debugf("pinging registry endpoint %s", endpoint) // Try HTTPS ping to registry endpoint.URL.Scheme = "https" if _, err := endpoint.Ping(); err != nil { - if index.Secure { + if endpoint.IsSecure { // If registry is secure and HTTPS failed, show user the error and tell them about `--insecure-registry` // in case that's what they need. DO NOT accept unknown CA certificates, and DO NOT fallback to HTTP. - return nil, fmt.Errorf("invalid registry endpoint %s: %v. If this private registry supports only HTTP or HTTPS with an unknown CA certificate, please add `--insecure-registry %s` to the daemon's arguments. In the case of HTTPS, if you have access to the registry's CA certificate, no need for the flag; simply place the CA certificate at /etc/docker/certs.d/%s/ca.crt", endpoint, err, endpoint.URL.Host, endpoint.URL.Host) + return fmt.Errorf("invalid registry endpoint %s: %v. If this private registry supports only HTTP or HTTPS with an unknown CA certificate, please add `--insecure-registry %s` to the daemon's arguments. In the case of HTTPS, if you have access to the registry's CA certificate, no need for the flag; simply place the CA certificate at /etc/docker/certs.d/%s/ca.crt", endpoint, err, endpoint.URL.Host, endpoint.URL.Host) } // If registry is insecure and HTTPS failed, fallback to HTTP. @@ -65,13 +72,13 @@ func NewEndpoint(index *IndexInfo) (*Endpoint, error) { var err2 error if _, err2 = endpoint.Ping(); err2 == nil { - return endpoint, nil + return nil } - return nil, fmt.Errorf("invalid registry endpoint %q. HTTPS attempt: %v. HTTP attempt: %v", endpoint, err, err2) + return fmt.Errorf("invalid registry endpoint %q. HTTPS attempt: %v. HTTP attempt: %v", endpoint, err, err2) } - return endpoint, nil + return nil } func newEndpoint(address string, secure bool) (*Endpoint, error) { diff --git a/docs/session_v2.go b/docs/session_v2.go index 411df46e3..031122dcf 100644 --- a/docs/session_v2.go +++ b/docs/session_v2.go @@ -30,8 +30,12 @@ func (r *Session) GetV2Authorization(imageName string, readOnly bool) (auth *Req } var registry *Endpoint - if r.indexEndpoint.URL.Host == IndexServerURL.Host { - registry, err = NewEndpoint(REGISTRYSERVER, nil) + if r.indexEndpoint.String() == IndexServerAddress() { + registry, err = newEndpoint(REGISTRYSERVER, true) + if err != nil { + return + } + err = validateEndpoint(registry) if err != nil { return } From e5744a3bad5de7d92d0fdc3e14f7f0f17466987e Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Fri, 2 Jan 2015 11:13:11 -0800 Subject: [PATCH 12/15] Refactor from feedback Signed-off-by: Derek McGowan (github: dmcgowan) --- docs/session_v2.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/session_v2.go b/docs/session_v2.go index 031122dcf..0e03f4a9c 100644 --- a/docs/session_v2.go +++ b/docs/session_v2.go @@ -5,7 +5,6 @@ import ( "fmt" "io" "io/ioutil" - "net/url" "strconv" log "github.com/Sirupsen/logrus" @@ -223,7 +222,7 @@ func (r *Session) PutV2ImageBlob(imageName, sumType, sumStr string, blobRdr io.R if err != nil { return err } - queryParams := url.Values{} + queryParams := req.URL.Query() queryParams.Add("digest", sumType+":"+sumStr) req.URL.RawQuery = queryParams.Encode() if err := auth.Authorize(req); err != nil { From 735a112415ee94795dd3594ae978d45ef6e5b36a Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 7 Jan 2015 15:55:29 -0800 Subject: [PATCH 13/15] Fix list tags Signed-off-by: Derek McGowan (github: dmcgowan) --- docs/session_v2.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/session_v2.go b/docs/session_v2.go index 0e03f4a9c..b08f4cf0d 100644 --- a/docs/session_v2.go +++ b/docs/session_v2.go @@ -277,6 +277,11 @@ func (r *Session) PutV2ImageManifest(imageName, tagName string, manifestRdr io.R return nil } +type remoteTags struct { + name string + tags []string +} + // Given a repository name, returns a json array of string tags func (r *Session) GetV2RemoteTags(imageName string, auth *RequestAuthorization) ([]string, error) { routeURL, err := getV2Builder(r.indexEndpoint).BuildTagsURL(imageName) @@ -309,10 +314,10 @@ func (r *Session) GetV2RemoteTags(imageName string, auth *RequestAuthorization) } decoder := json.NewDecoder(res.Body) - var tags []string - err = decoder.Decode(&tags) + var remote remoteTags + err = decoder.Decode(&remote) if err != nil { return nil, fmt.Errorf("Error while decoding the http response: %s", err) } - return tags, nil + return remote.tags, nil } From 5bf94a6438b6619aef7d711b438e8fa0323ca88c Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 14 Jan 2015 16:46:31 -0800 Subject: [PATCH 14/15] Cleanup v2 session to require endpoint Signed-off-by: Derek McGowan (github: dmcgowan) --- docs/session_v2.go | 76 +++++++++++++++++++++++++--------------------- 1 file changed, 42 insertions(+), 34 deletions(-) diff --git a/docs/session_v2.go b/docs/session_v2.go index b08f4cf0d..11b96bd65 100644 --- a/docs/session_v2.go +++ b/docs/session_v2.go @@ -19,33 +19,41 @@ func getV2Builder(e *Endpoint) *v2.URLBuilder { return e.URLBuilder } +func (r *Session) V2RegistryEndpoint(index *IndexInfo) (ep *Endpoint, err error) { + // TODO check if should use Mirror + if index.Official { + ep, err = newEndpoint(REGISTRYSERVER, true) + if err != nil { + return + } + err = validateEndpoint(ep) + if err != nil { + return + } + } else if r.indexEndpoint.String() == index.GetAuthConfigKey() { + ep = r.indexEndpoint + } else { + ep, err = NewEndpoint(index) + if err != nil { + return + } + } + + ep.URLBuilder = v2.NewURLBuilder(ep.URL) + return +} + // GetV2Authorization gets the authorization needed to the given image // If readonly access is requested, then only the authorization may // only be used for Get operations. -func (r *Session) GetV2Authorization(imageName string, readOnly bool) (auth *RequestAuthorization, err error) { +func (r *Session) GetV2Authorization(ep *Endpoint, imageName string, readOnly bool) (auth *RequestAuthorization, err error) { scopes := []string{"pull"} if !readOnly { scopes = append(scopes, "push") } - var registry *Endpoint - if r.indexEndpoint.String() == IndexServerAddress() { - registry, err = newEndpoint(REGISTRYSERVER, true) - if err != nil { - return - } - err = validateEndpoint(registry) - if err != nil { - return - } - } else { - registry = r.indexEndpoint - } - registry.URLBuilder = v2.NewURLBuilder(registry.URL) - r.indexEndpoint = registry - log.Debugf("Getting authorization for %s %s", imageName, scopes) - return NewRequestAuthorization(r.GetAuthConfig(true), registry, "repository", imageName, scopes), nil + return NewRequestAuthorization(r.GetAuthConfig(true), ep, "repository", imageName, scopes), nil } // @@ -55,8 +63,8 @@ func (r *Session) GetV2Authorization(imageName string, readOnly bool) (auth *Req // 1.c) if anything else, err // 2) PUT the created/signed manifest // -func (r *Session) GetV2ImageManifest(imageName, tagName string, auth *RequestAuthorization) ([]byte, error) { - routeURL, err := getV2Builder(r.indexEndpoint).BuildManifestURL(imageName, tagName) +func (r *Session) GetV2ImageManifest(ep *Endpoint, imageName, tagName string, auth *RequestAuthorization) ([]byte, error) { + routeURL, err := getV2Builder(ep).BuildManifestURL(imageName, tagName) if err != nil { return nil, err } @@ -92,11 +100,11 @@ func (r *Session) GetV2ImageManifest(imageName, tagName string, auth *RequestAut return buf, nil } -// - Succeeded to mount for this image scope -// - Failed with no error (So continue to Push the Blob) +// - Succeeded to head image blob (already exists) +// - Failed with no error (continue to Push the Blob) // - Failed with error -func (r *Session) PostV2ImageMountBlob(imageName, sumType, sum string, auth *RequestAuthorization) (bool, error) { - routeURL, err := getV2Builder(r.indexEndpoint).BuildBlobURL(imageName, sumType+":"+sum) +func (r *Session) HeadV2ImageBlob(ep *Endpoint, imageName, sumType, sum string, auth *RequestAuthorization) (bool, error) { + routeURL, err := getV2Builder(ep).BuildBlobURL(imageName, sumType+":"+sum) if err != nil { return false, err } @@ -127,8 +135,8 @@ func (r *Session) PostV2ImageMountBlob(imageName, sumType, sum string, auth *Req return false, fmt.Errorf("Failed to mount %q - %s:%s : %d", imageName, sumType, sum, res.StatusCode) } -func (r *Session) GetV2ImageBlob(imageName, sumType, sum string, blobWrtr io.Writer, auth *RequestAuthorization) error { - routeURL, err := getV2Builder(r.indexEndpoint).BuildBlobURL(imageName, sumType+":"+sum) +func (r *Session) GetV2ImageBlob(ep *Endpoint, imageName, sumType, sum string, blobWrtr io.Writer, auth *RequestAuthorization) error { + routeURL, err := getV2Builder(ep).BuildBlobURL(imageName, sumType+":"+sum) if err != nil { return err } @@ -158,8 +166,8 @@ func (r *Session) GetV2ImageBlob(imageName, sumType, sum string, blobWrtr io.Wri return err } -func (r *Session) GetV2ImageBlobReader(imageName, sumType, sum string, auth *RequestAuthorization) (io.ReadCloser, int64, error) { - routeURL, err := getV2Builder(r.indexEndpoint).BuildBlobURL(imageName, sumType+":"+sum) +func (r *Session) GetV2ImageBlobReader(ep *Endpoint, imageName, sumType, sum string, auth *RequestAuthorization) (io.ReadCloser, int64, error) { + routeURL, err := getV2Builder(ep).BuildBlobURL(imageName, sumType+":"+sum) if err != nil { return nil, 0, err } @@ -195,8 +203,8 @@ func (r *Session) GetV2ImageBlobReader(imageName, sumType, sum string, auth *Req // Push the image to the server for storage. // 'layer' is an uncompressed reader of the blob to be pushed. // The server will generate it's own checksum calculation. -func (r *Session) PutV2ImageBlob(imageName, sumType, sumStr string, blobRdr io.Reader, auth *RequestAuthorization) error { - routeURL, err := getV2Builder(r.indexEndpoint).BuildBlobUploadURL(imageName) +func (r *Session) PutV2ImageBlob(ep *Endpoint, imageName, sumType, sumStr string, blobRdr io.Reader, auth *RequestAuthorization) error { + routeURL, err := getV2Builder(ep).BuildBlobUploadURL(imageName) if err != nil { return err } @@ -245,8 +253,8 @@ func (r *Session) PutV2ImageBlob(imageName, sumType, sumStr string, blobRdr io.R } // Finally Push the (signed) manifest of the blobs we've just pushed -func (r *Session) PutV2ImageManifest(imageName, tagName string, manifestRdr io.Reader, auth *RequestAuthorization) error { - routeURL, err := getV2Builder(r.indexEndpoint).BuildManifestURL(imageName, tagName) +func (r *Session) PutV2ImageManifest(ep *Endpoint, imageName, tagName string, manifestRdr io.Reader, auth *RequestAuthorization) error { + routeURL, err := getV2Builder(ep).BuildManifestURL(imageName, tagName) if err != nil { return err } @@ -283,8 +291,8 @@ type remoteTags struct { } // Given a repository name, returns a json array of string tags -func (r *Session) GetV2RemoteTags(imageName string, auth *RequestAuthorization) ([]string, error) { - routeURL, err := getV2Builder(r.indexEndpoint).BuildTagsURL(imageName) +func (r *Session) GetV2RemoteTags(ep *Endpoint, imageName string, auth *RequestAuthorization) ([]string, error) { + routeURL, err := getV2Builder(ep).BuildTagsURL(imageName) if err != nil { return nil, err } From 9c24fc93adf4bdc666575f1ca2bb530d7a10c8ac Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Thu, 15 Jan 2015 13:06:52 -0800 Subject: [PATCH 15/15] Add token cache Token cache prevents the need to get a new token for every registry interaction. Since the tokens are short lived, the cache expires after only a minute. Signed-off-by: Derek McGowan (github: dmcgowan) --- docs/auth.go | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/docs/auth.go b/docs/auth.go index 1e1c7ddb8..1ce99805f 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -10,6 +10,8 @@ import ( "os" "path" "strings" + "sync" + "time" log "github.com/Sirupsen/logrus" "github.com/docker/docker/utils" @@ -43,6 +45,10 @@ type RequestAuthorization struct { resource string scope string actions []string + + tokenLock sync.Mutex + tokenCache string + tokenExpiration time.Time } func NewRequestAuthorization(authConfig *AuthConfig, registryEndpoint *Endpoint, resource, scope string, actions []string) *RequestAuthorization { @@ -56,7 +62,14 @@ func NewRequestAuthorization(authConfig *AuthConfig, registryEndpoint *Endpoint, } func (auth *RequestAuthorization) getToken() (string, error) { - // TODO check if already has token and before expiration + auth.tokenLock.Lock() + defer auth.tokenLock.Unlock() + now := time.Now() + if now.Before(auth.tokenExpiration) { + log.Debugf("Using cached token for %s", auth.authConfig.Username) + return auth.tokenCache, nil + } + client := &http.Client{ Transport: &http.Transport{ DisableKeepAlives: true, @@ -80,14 +93,18 @@ func (auth *RequestAuthorization) getToken() (string, error) { if err != nil { return "", err } - // TODO cache token and set expiration to one minute from now + auth.tokenCache = token + auth.tokenExpiration = now.Add(time.Minute) return token, nil default: log.Infof("Unsupported auth scheme: %q", challenge.Scheme) } } - // TODO no expiration, do not reattempt to get a token + + // Do not expire cache since there are no challenges which use a token + auth.tokenExpiration = time.Now().Add(time.Hour * 24) + return "", nil }