From ec6d1d60201a3f6a3efd3684db85c6505f57602d Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Fri, 5 Jul 2013 12:20:58 -0700 Subject: [PATCH 1/5] Adding support for nicer URLs to support standalone registry (+ some registry code cleaning) --- docs/registry.go | 141 +++++++++++++++++++++++++++-------------------- 1 file changed, 81 insertions(+), 60 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 622c09b3..e9d7b2b8 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -12,18 +12,70 @@ import ( "net/http" "net/http/cookiejar" "net/url" + "regexp" "strconv" "strings" ) var ErrAlreadyExists = errors.New("Image already exists") -func UrlScheme() string { - u, err := url.Parse(auth.IndexServerAddress()) +func pingRegistryEndpoint(endpoint string) error { + // FIXME: implement the check to discover if it should be http or https + resp, err := http.Get(endpoint) if err != nil { - return "https" + return err } - return u.Scheme + if resp.Header.Get("X-Docker-Registry-Version") == "" { + return errors.New("This does not look like a Registry server (\"X-Docker-Registry-Version\" header not found in the response)") + } + return nil +} + +func validateRepositoryName(namespace, name string) error { + validNamespace := regexp.MustCompile(`^([a-z0-9_]{4,30})$`) + if !validNamespace.MatchString(namespace) { + return fmt.Errorf("Invalid namespace name (%s), only [a-z0-9_] are allowed, size between 4 and 30", namespace) + } + validRepo := regexp.MustCompile(`^([a-zA-Z0-9-_.]+)$`) + if !validRepo.MatchString(name) { + return fmt.Errorf("Invalid repository name (%s), only [a-zA-Z0-9-_.] are allowed", name) + } + return nil +} + +// Resolves a repository name to a endpoint + name +func ResolveRepositoryName(reposName string) (string, string, error) { + nameParts := strings.SplitN(reposName, "/", 2) + if !strings.Contains(nameParts[0], ".") { + // This is a Docker Index repos (ex: samalba/hipache or ubuntu) + var err error + if len(nameParts) < 2 { + err = validateRepositoryName("library", nameParts[0]) + } else { + err = validateRepositoryName(nameParts[0], nameParts[1]) + } + return "https://index.docker.io/v1/", reposName, err + } + if len(nameParts) < 2 { + // There is a dot in repos name (and no registry address) + // Is it a Registry address without repos name? + return "", "", errors.New("Invalid repository name (ex: \"registry.domain.tld/myrepos\")") + } + n := strings.LastIndex(reposName, "/") + hostname := nameParts[0] + path := reposName[len(nameParts[0]):n] + reposName = reposName[n+1:] + endpoint := fmt.Sprintf("https://%s%s/v1/", hostname, path) + if err := pingRegistryEndpoint(endpoint); err != nil { + utils.Debugf("Registry %s does not work (%s), falling back to http", endpoint, err) + endpoint = fmt.Sprintf("http://%s%s/v1/", hostname, path) + if err = pingRegistryEndpoint(endpoint); err != nil { + //TODO: triggering highland build can be done there without "failing" + return "", "", errors.New("Invalid Registry endpoint: " + err.Error()) + } + } + err := validateRepositoryName("library", reposName) + return endpoint, reposName, err } func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { @@ -36,7 +88,7 @@ func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { // Retrieve the history of a given image from the Registry. // Return a list of the parent's json (requested image included) func (r *Registry) GetRemoteHistory(imgId, registry string, token []string) ([]string, error) { - req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/ancestry", nil) + req, err := http.NewRequest("GET", registry+"images/"+imgId+"/ancestry", nil) if err != nil { return nil, err } @@ -67,7 +119,7 @@ func (r *Registry) GetRemoteHistory(imgId, registry string, token []string) ([]s func (r *Registry) LookupRemoteImage(imgId, registry string, token []string) bool { rt := &http.Transport{Proxy: http.ProxyFromEnvironment} - req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/json", nil) + req, err := http.NewRequest("GET", registry+"images/"+imgId+"/json", nil) if err != nil { return false } @@ -79,44 +131,10 @@ func (r *Registry) LookupRemoteImage(imgId, registry string, token []string) boo return res.StatusCode == 200 } -func (r *Registry) getImagesInRepository(repository string, authConfig *auth.AuthConfig) ([]map[string]string, error) { - u := auth.IndexServerAddress() + "/repositories/" + repository + "/images" - req, err := http.NewRequest("GET", u, nil) - if err != nil { - return nil, err - } - if authConfig != nil && len(authConfig.Username) > 0 { - req.SetBasicAuth(authConfig.Username, authConfig.Password) - } - res, err := r.client.Do(req) - if err != nil { - return nil, err - } - defer res.Body.Close() - - // Repository doesn't exist yet - if res.StatusCode == 404 { - return nil, nil - } - - jsonData, err := ioutil.ReadAll(res.Body) - if err != nil { - return nil, err - } - - imageList := []map[string]string{} - if err := json.Unmarshal(jsonData, &imageList); err != nil { - utils.Debugf("Body: %s (%s)\n", res.Body, u) - return nil, err - } - - return imageList, nil -} - // Retrieve an image from the Registry. func (r *Registry) GetRemoteImageJSON(imgId, registry string, token []string) ([]byte, int, error) { // Get the JSON - req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/json", nil) + req, err := http.NewRequest("GET", registry+"images/"+imgId+"/json", nil) if err != nil { return nil, -1, fmt.Errorf("Failed to download json: %s", err) } @@ -143,7 +161,7 @@ func (r *Registry) GetRemoteImageJSON(imgId, registry string, token []string) ([ } func (r *Registry) GetRemoteImageLayer(imgId, registry string, token []string) (io.ReadCloser, error) { - req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/layer", nil) + req, err := http.NewRequest("GET", registry+"images/"+imgId+"/layer", nil) if err != nil { return nil, fmt.Errorf("Error while getting from the server: %s\n", err) } @@ -162,10 +180,7 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ repository = "library/" + repository } for _, host := range registries { - endpoint := fmt.Sprintf("%s/v1/repositories/%s/tags", host, repository) - if !(strings.HasPrefix(endpoint, "http://") || strings.HasPrefix(endpoint, "https://")) { - endpoint = fmt.Sprintf("%s://%s", UrlScheme(), endpoint) - } + endpoint := fmt.Sprintf("%srepositories/%s/tags", host, repository) req, err := r.opaqueRequest("GET", endpoint, nil) if err != nil { return nil, err @@ -198,8 +213,8 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ return nil, fmt.Errorf("Could not reach any registry endpoint") } -func (r *Registry) GetRepositoryData(remote string) (*RepositoryData, error) { - repositoryTarget := auth.IndexServerAddress() + "/repositories/" + remote + "/images" +func (r *Registry) GetRepositoryData(indexEp, remote string) (*RepositoryData, error) { + repositoryTarget := fmt.Sprintf("%srepositories/%s/images", indexEp, remote) req, err := r.opaqueRequest("GET", repositoryTarget, nil) if err != nil { @@ -230,8 +245,12 @@ func (r *Registry) GetRepositoryData(remote string) (*RepositoryData, error) { } var endpoints []string + var urlScheme = indexEp[:strings.Index(indexEp, ":")] if res.Header.Get("X-Docker-Endpoints") != "" { - endpoints = res.Header["X-Docker-Endpoints"] + // The Registry's URL scheme has to match the Index' + for _, ep := range res.Header["X-Docker-Endpoints"] { + endpoints = append(endpoints, fmt.Sprintf("%s://%s/v1/", urlScheme, ep)) + } } else { return nil, fmt.Errorf("Index response didn't contain any endpoints") } @@ -260,9 +279,8 @@ func (r *Registry) GetRepositoryData(remote string) (*RepositoryData, error) { // Push a local image to the registry func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, registry string, token []string) error { - registry = registry + "/v1" // FIXME: try json with UTF8 - req, err := http.NewRequest("PUT", registry+"/images/"+imgData.ID+"/json", strings.NewReader(string(jsonRaw))) + req, err := http.NewRequest("PUT", registry+"images/"+imgData.ID+"/json", strings.NewReader(string(jsonRaw))) if err != nil { return err } @@ -296,8 +314,7 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis } func (r *Registry) PushImageLayerRegistry(imgId string, layer io.Reader, registry string, token []string) error { - registry = registry + "/v1" - req, err := http.NewRequest("PUT", registry+"/images/"+imgId+"/layer", layer) + req, err := http.NewRequest("PUT", registry+"images/"+imgId+"/layer", layer) if err != nil { return err } @@ -334,9 +351,8 @@ func (r *Registry) opaqueRequest(method, urlStr string, body io.Reader) (*http.R func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token []string) error { // "jsonify" the string revision = "\"" + revision + "\"" - registry = registry + "/v1" - req, err := r.opaqueRequest("PUT", registry+"/repositories/"+remote+"/tags/"+tag, strings.NewReader(revision)) + req, err := r.opaqueRequest("PUT", registry+"repositories/"+remote+"/tags/"+tag, strings.NewReader(revision)) if err != nil { return err } @@ -354,7 +370,7 @@ func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token return nil } -func (r *Registry) PushImageJSONIndex(remote string, imgList []*ImgData, validate bool, regs []string) (*RepositoryData, error) { +func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData, validate bool, regs []string) (*RepositoryData, error) { imgListJSON, err := json.Marshal(imgList) if err != nil { return nil, err @@ -364,9 +380,10 @@ func (r *Registry) PushImageJSONIndex(remote string, imgList []*ImgData, validat suffix = "images" } + u := fmt.Sprintf("%srepositories/%s/%s", indexEp, remote, suffix) + utils.Debugf("PUT %s", u) utils.Debugf("Image list pushed to index:\n%s\n", imgListJSON) - - req, err := r.opaqueRequest("PUT", auth.IndexServerAddress()+"/repositories/"+remote+"/"+suffix, bytes.NewReader(imgListJSON)) + req, err := r.opaqueRequest("PUT", u, bytes.NewReader(imgListJSON)) if err != nil { return nil, err } @@ -404,6 +421,7 @@ func (r *Registry) PushImageJSONIndex(remote string, imgList []*ImgData, validat } var tokens, endpoints []string + var urlScheme = indexEp[:strings.Index(indexEp, ":")] if !validate { if res.StatusCode != 200 && res.StatusCode != 201 { errBody, err := ioutil.ReadAll(res.Body) @@ -420,7 +438,10 @@ func (r *Registry) PushImageJSONIndex(remote string, imgList []*ImgData, validat } if res.Header.Get("X-Docker-Endpoints") != "" { - endpoints = res.Header["X-Docker-Endpoints"] + // The Registry's URL scheme has to match the Index' + for _, ep := range res.Header["X-Docker-Endpoints"] { + endpoints = append(endpoints, fmt.Sprintf("%s://%s/v1/", urlScheme, ep)) + } } else { return nil, fmt.Errorf("Index response didn't contain any endpoints") } @@ -442,7 +463,7 @@ func (r *Registry) PushImageJSONIndex(remote string, imgList []*ImgData, validat } func (r *Registry) SearchRepositories(term string) (*SearchResults, error) { - u := auth.IndexServerAddress() + "/search?q=" + url.QueryEscape(term) + u := auth.IndexServerAddress() + "search?q=" + url.QueryEscape(term) req, err := http.NewRequest("GET", u, nil) if err != nil { return nil, err From c6068feffab5e2351a0bc7a395173103be531829 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Fri, 5 Jul 2013 12:37:07 -0700 Subject: [PATCH 2/5] Restoring old changeset lost by previous merge --- docs/registry.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index ee473493..4bd2a5ad 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -87,8 +87,8 @@ func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { // Retrieve the history of a given image from the Registry. // Return a list of the parent's json (requested image included) -func (r *Registry) GetRemoteHistory(imgId, registry string, token []string) ([]string, error) { - req, err := http.NewRequest("GET", registry+"images/"+imgId+"/ancestry", nil) +func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]string, error) { + req, err := http.NewRequest("GET", registry+"images/"+imgID+"/ancestry", nil) if err != nil { return nil, err } @@ -119,7 +119,7 @@ func (r *Registry) GetRemoteHistory(imgId, registry string, token []string) ([]s func (r *Registry) LookupRemoteImage(imgID, registry string, token []string) bool { rt := &http.Transport{Proxy: http.ProxyFromEnvironment} - req, err := http.NewRequest("GET", registry+"images/"+imgId+"/json", nil) + req, err := http.NewRequest("GET", registry+"images/"+imgID+"/json", nil) if err != nil { return false } @@ -134,7 +134,7 @@ func (r *Registry) LookupRemoteImage(imgID, registry string, token []string) boo // Retrieve an image from the Registry. func (r *Registry) GetRemoteImageJSON(imgID, registry string, token []string) ([]byte, int, error) { // Get the JSON - req, err := http.NewRequest("GET", registry+"images/"+imgId+"/json", nil) + req, err := http.NewRequest("GET", registry+"images/"+imgID+"/json", nil) if err != nil { return nil, -1, fmt.Errorf("Failed to download json: %s", err) } @@ -160,8 +160,8 @@ func (r *Registry) GetRemoteImageJSON(imgID, registry string, token []string) ([ return jsonString, imageSize, nil } -func (r *Registry) GetRemoteImageLayer(imgId, registry string, token []string) (io.ReadCloser, error) { - req, err := http.NewRequest("GET", registry+"images/"+imgId+"/layer", nil) +func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string) (io.ReadCloser, error) { + req, err := http.NewRequest("GET", registry+"images/"+imgID+"/layer", nil) if err != nil { return nil, fmt.Errorf("Error while getting from the server: %s\n", err) } @@ -313,8 +313,8 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis return nil } -func (r *Registry) PushImageLayerRegistry(imgId string, layer io.Reader, registry string, token []string) error { - req, err := http.NewRequest("PUT", registry+"images/"+imgId+"/layer", layer) +func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registry string, token []string) error { + req, err := http.NewRequest("PUT", registry+"images/"+imgID+"/layer", layer) if err != nil { return err } From 16fa043e344eafa67121ffea9ed0032081653f59 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Fri, 5 Jul 2013 14:30:43 -0700 Subject: [PATCH 3/5] Allowing namespaces in standalone registry --- docs/registry.go | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 4bd2a5ad..72521a31 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -20,8 +20,7 @@ import ( var ErrAlreadyExists = errors.New("Image already exists") func pingRegistryEndpoint(endpoint string) error { - // FIXME: implement the check to discover if it should be http or https - resp, err := http.Get(endpoint) + resp, err := http.Get(endpoint + "/_ping") if err != nil { return err } @@ -31,7 +30,19 @@ func pingRegistryEndpoint(endpoint string) error { return nil } -func validateRepositoryName(namespace, name string) error { +func validateRepositoryName(repositoryName string) error { + var ( + namespace string + name string + ) + nameParts := strings.SplitN(repositoryName, "/", 2) + if len(nameParts) < 2 { + namespace = "library" + name = nameParts[0] + } else { + namespace = nameParts[0] + name = nameParts[1] + } validNamespace := regexp.MustCompile(`^([a-z0-9_]{4,30})$`) if !validNamespace.MatchString(namespace) { return fmt.Errorf("Invalid namespace name (%s), only [a-z0-9_] are allowed, size between 4 and 30", namespace) @@ -48,12 +59,7 @@ func ResolveRepositoryName(reposName string) (string, string, error) { nameParts := strings.SplitN(reposName, "/", 2) if !strings.Contains(nameParts[0], ".") { // This is a Docker Index repos (ex: samalba/hipache or ubuntu) - var err error - if len(nameParts) < 2 { - err = validateRepositoryName("library", nameParts[0]) - } else { - err = validateRepositoryName(nameParts[0], nameParts[1]) - } + err := validateRepositoryName(reposName) return "https://index.docker.io/v1/", reposName, err } if len(nameParts) < 2 { @@ -61,20 +67,18 @@ func ResolveRepositoryName(reposName string) (string, string, error) { // Is it a Registry address without repos name? return "", "", errors.New("Invalid repository name (ex: \"registry.domain.tld/myrepos\")") } - n := strings.LastIndex(reposName, "/") hostname := nameParts[0] - path := reposName[len(nameParts[0]):n] - reposName = reposName[n+1:] - endpoint := fmt.Sprintf("https://%s%s/v1/", hostname, path) + reposName = nameParts[1] + endpoint := fmt.Sprintf("https://%s/v1/", hostname) if err := pingRegistryEndpoint(endpoint); err != nil { utils.Debugf("Registry %s does not work (%s), falling back to http", endpoint, err) - endpoint = fmt.Sprintf("http://%s%s/v1/", hostname, path) + endpoint = fmt.Sprintf("http://%s/v1/", hostname) if err = pingRegistryEndpoint(endpoint); err != nil { //TODO: triggering highland build can be done there without "failing" return "", "", errors.New("Invalid Registry endpoint: " + err.Error()) } } - err := validateRepositoryName("library", reposName) + err := validateRepositoryName(reposName) return endpoint, reposName, err } From 98060903a9d86f96a9ed96c64a310d3c947910d2 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Fri, 5 Jul 2013 14:55:48 -0700 Subject: [PATCH 4/5] Fixed ping URL --- docs/registry.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/registry.go b/docs/registry.go index 72521a31..730fcf6e 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -20,7 +20,7 @@ import ( var ErrAlreadyExists = errors.New("Image already exists") func pingRegistryEndpoint(endpoint string) error { - resp, err := http.Get(endpoint + "/_ping") + resp, err := http.Get(endpoint + "_ping") if err != nil { return err } From 67115ec4794a5d45fcaff332a732350f9e233b55 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Fri, 5 Jul 2013 14:56:56 -0700 Subject: [PATCH 5/5] fmt.Errorf instead of errors.New --- docs/registry.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/registry.go b/docs/registry.go index 730fcf6e..c458f616 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -65,7 +65,7 @@ func ResolveRepositoryName(reposName string) (string, string, error) { if len(nameParts) < 2 { // There is a dot in repos name (and no registry address) // Is it a Registry address without repos name? - return "", "", errors.New("Invalid repository name (ex: \"registry.domain.tld/myrepos\")") + return "", "", fmt.Errorf("Invalid repository name (ex: \"registry.domain.tld/myrepos\")") } hostname := nameParts[0] reposName = nameParts[1]