From 7bfdb6d495506c95768177fde302d19763a39932 Mon Sep 17 00:00:00 2001 From: unclejack Date: Mon, 6 Oct 2014 22:34:39 +0300 Subject: [PATCH 01/26] registry: lint Docker-DCO-1.1-Signed-off-by: Cristian Staretu (github: unclejack) --- docs/auth.go | 7 +++---- docs/endpoint.go | 4 ++-- docs/registry_mock_test.go | 22 ++++++++++---------- docs/registry_test.go | 41 ++++++++++++++++++++------------------ docs/session.go | 5 +++-- 5 files changed, 41 insertions(+), 38 deletions(-) diff --git a/docs/auth.go b/docs/auth.go index 906a37dd..ba370f4b 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -224,12 +224,11 @@ func Login(authConfig *AuthConfig, factory *utils.HTTPRequestFactory) (string, e return "", fmt.Errorf("Login: Account is not Active. Please check your e-mail for a confirmation link.") } return "", fmt.Errorf("Login: Account is not Active. Please see the documentation of the registry %s for instructions how to activate it.", serverAddress) - } else { - return "", fmt.Errorf("Login: %s (Code: %d; Headers: %s)", body, resp.StatusCode, resp.Header) } - } else { - return "", fmt.Errorf("Registration: %s", reqBody) + return "", fmt.Errorf("Login: %s (Code: %d; Headers: %s)", body, resp.StatusCode, resp.Header) } + return "", fmt.Errorf("Registration: %s", reqBody) + } else if reqStatusCode == 401 { // This case would happen with private registries where /v1/users is // protected, so people can use `docker login` as an auth check. diff --git a/docs/endpoint.go b/docs/endpoint.go index 5313a807..58311d32 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -13,7 +13,7 @@ import ( ) // scans string for api version in the URL path. returns the trimmed hostname, if version found, string and API version. -func scanForApiVersion(hostname string) (string, APIVersion) { +func scanForAPIVersion(hostname string) (string, APIVersion) { var ( chunks []string apiVersionStr string @@ -43,7 +43,7 @@ func NewEndpoint(hostname string) (*Endpoint, error) { if !strings.HasPrefix(hostname, "http") { hostname = "https://" + hostname } - trimmedHostname, endpoint.Version = scanForApiVersion(hostname) + trimmedHostname, endpoint.Version = scanForAPIVersion(hostname) endpoint.URL, err = url.Parse(trimmedHostname) if err != nil { return nil, err diff --git a/docs/registry_mock_test.go b/docs/registry_mock_test.go index 379dc78f..967d8b26 100644 --- a/docs/registry_mock_test.go +++ b/docs/registry_mock_test.go @@ -19,7 +19,7 @@ import ( ) var ( - testHttpServer *httptest.Server + testHTTPServer *httptest.Server testLayers = map[string]map[string]string{ "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20": { "json": `{"id":"77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20", @@ -99,7 +99,7 @@ func init() { // /v2/ r.HandleFunc("/v2/version", handlerGetPing).Methods("GET") - testHttpServer = httptest.NewServer(handlerAccessLog(r)) + testHTTPServer = httptest.NewServer(handlerAccessLog(r)) } func handlerAccessLog(handler http.Handler) http.Handler { @@ -111,7 +111,7 @@ func handlerAccessLog(handler http.Handler) http.Handler { } func makeURL(req string) string { - return testHttpServer.URL + req + return testHTTPServer.URL + req } func writeHeaders(w http.ResponseWriter) { @@ -198,8 +198,8 @@ func handlerGetImage(w http.ResponseWriter, r *http.Request) { return } writeHeaders(w) - layer_size := len(layer["layer"]) - w.Header().Add("X-Docker-Size", strconv.Itoa(layer_size)) + layerSize := len(layer["layer"]) + w.Header().Add("X-Docker-Size", strconv.Itoa(layerSize)) io.WriteString(w, layer[vars["action"]]) } @@ -208,16 +208,16 @@ func handlerPutImage(w http.ResponseWriter, r *http.Request) { return } vars := mux.Vars(r) - image_id := vars["image_id"] + imageID := vars["image_id"] action := vars["action"] - layer, exists := testLayers[image_id] + layer, exists := testLayers[imageID] if !exists { if action != "json" { http.NotFound(w, r) return } layer = make(map[string]string) - testLayers[image_id] = layer + testLayers[imageID] = layer } if checksum := r.Header.Get("X-Docker-Checksum"); checksum != "" { if checksum != layer["checksum_simple"] && checksum != layer["checksum_tarsum"] { @@ -301,7 +301,7 @@ func handlerUsers(w http.ResponseWriter, r *http.Request) { } func handlerImages(w http.ResponseWriter, r *http.Request) { - u, _ := url.Parse(testHttpServer.URL) + u, _ := url.Parse(testHTTPServer.URL) w.Header().Add("X-Docker-Endpoints", fmt.Sprintf("%s , %s ", u.Host, "test.example.com")) w.Header().Add("X-Docker-Token", fmt.Sprintf("FAKE-SESSION-%d", time.Now().UnixNano())) if r.Method == "PUT" { @@ -317,9 +317,9 @@ func handlerImages(w http.ResponseWriter, r *http.Request) { return } images := []map[string]string{} - for image_id, layer := range testLayers { + for imageID, layer := range testLayers { image := make(map[string]string) - image["id"] = image_id + image["id"] = imageID image["checksum"] = layer["checksum_tarsum"] image["Tag"] = "latest" images = append(images, image) diff --git a/docs/registry_test.go b/docs/registry_test.go index ab417812..fdf714e8 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -11,9 +11,12 @@ import ( ) var ( - IMAGE_ID = "42d718c941f5c532ac049bf0b0ab53f0062f09a03afd4aa4a02c098e46032b9d" - TOKEN = []string{"fake-token"} - REPO = "foo42/bar" + token = []string{"fake-token"} +) + +const ( + imageID = "42d718c941f5c532ac049bf0b0ab53f0062f09a03afd4aa4a02c098e46032b9d" + REPO = "foo42/bar" ) func spawnTestRegistrySession(t *testing.T) *Session { @@ -43,27 +46,27 @@ func TestPingRegistryEndpoint(t *testing.T) { func TestGetRemoteHistory(t *testing.T) { r := spawnTestRegistrySession(t) - hist, err := r.GetRemoteHistory(IMAGE_ID, makeURL("/v1/"), TOKEN) + hist, err := r.GetRemoteHistory(imageID, makeURL("/v1/"), token) if err != nil { t.Fatal(err) } assertEqual(t, len(hist), 2, "Expected 2 images in history") - assertEqual(t, hist[0], IMAGE_ID, "Expected "+IMAGE_ID+"as first ancestry") + assertEqual(t, hist[0], imageID, "Expected "+imageID+"as first ancestry") assertEqual(t, hist[1], "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20", "Unexpected second ancestry") } func TestLookupRemoteImage(t *testing.T) { r := spawnTestRegistrySession(t) - found := r.LookupRemoteImage(IMAGE_ID, makeURL("/v1/"), TOKEN) + found := r.LookupRemoteImage(imageID, makeURL("/v1/"), token) assertEqual(t, found, true, "Expected remote lookup to succeed") - found = r.LookupRemoteImage("abcdef", makeURL("/v1/"), TOKEN) + found = r.LookupRemoteImage("abcdef", makeURL("/v1/"), token) assertEqual(t, found, false, "Expected remote lookup to fail") } func TestGetRemoteImageJSON(t *testing.T) { r := spawnTestRegistrySession(t) - json, size, err := r.GetRemoteImageJSON(IMAGE_ID, makeURL("/v1/"), TOKEN) + json, size, err := r.GetRemoteImageJSON(imageID, makeURL("/v1/"), token) if err != nil { t.Fatal(err) } @@ -72,7 +75,7 @@ func TestGetRemoteImageJSON(t *testing.T) { t.Fatal("Expected non-empty json") } - _, _, err = r.GetRemoteImageJSON("abcdef", makeURL("/v1/"), TOKEN) + _, _, err = r.GetRemoteImageJSON("abcdef", makeURL("/v1/"), token) if err == nil { t.Fatal("Expected image not found error") } @@ -80,7 +83,7 @@ func TestGetRemoteImageJSON(t *testing.T) { func TestGetRemoteImageLayer(t *testing.T) { r := spawnTestRegistrySession(t) - data, err := r.GetRemoteImageLayer(IMAGE_ID, makeURL("/v1/"), TOKEN, 0) + data, err := r.GetRemoteImageLayer(imageID, makeURL("/v1/"), token, 0) if err != nil { t.Fatal(err) } @@ -88,7 +91,7 @@ func TestGetRemoteImageLayer(t *testing.T) { t.Fatal("Expected non-nil data result") } - _, err = r.GetRemoteImageLayer("abcdef", makeURL("/v1/"), TOKEN, 0) + _, err = r.GetRemoteImageLayer("abcdef", makeURL("/v1/"), token, 0) if err == nil { t.Fatal("Expected image not found error") } @@ -96,14 +99,14 @@ func TestGetRemoteImageLayer(t *testing.T) { func TestGetRemoteTags(t *testing.T) { r := spawnTestRegistrySession(t) - tags, err := r.GetRemoteTags([]string{makeURL("/v1/")}, REPO, TOKEN) + tags, err := r.GetRemoteTags([]string{makeURL("/v1/")}, REPO, token) if err != nil { t.Fatal(err) } assertEqual(t, len(tags), 1, "Expected one tag") - assertEqual(t, tags["latest"], IMAGE_ID, "Expected tag latest to map to "+IMAGE_ID) + assertEqual(t, tags["latest"], imageID, "Expected tag latest to map to "+imageID) - _, err = r.GetRemoteTags([]string{makeURL("/v1/")}, "foo42/baz", TOKEN) + _, err = r.GetRemoteTags([]string{makeURL("/v1/")}, "foo42/baz", token) if err == nil { t.Fatal("Expected error when fetching tags for bogus repo") } @@ -111,11 +114,11 @@ func TestGetRemoteTags(t *testing.T) { func TestGetRepositoryData(t *testing.T) { r := spawnTestRegistrySession(t) - parsedUrl, err := url.Parse(makeURL("/v1/")) + parsedURL, err := url.Parse(makeURL("/v1/")) if err != nil { t.Fatal(err) } - host := "http://" + parsedUrl.Host + "/v1/" + host := "http://" + parsedURL.Host + "/v1/" data, err := r.GetRepositoryData("foo42/bar") if err != nil { t.Fatal(err) @@ -137,7 +140,7 @@ func TestPushImageJSONRegistry(t *testing.T) { Checksum: "sha256:1ac330d56e05eef6d438586545ceff7550d3bdcb6b19961f12c5ba714ee1bb37", } - err := r.PushImageJSONRegistry(imgData, []byte{0x42, 0xdf, 0x0}, makeURL("/v1/"), TOKEN) + err := r.PushImageJSONRegistry(imgData, []byte{0x42, 0xdf, 0x0}, makeURL("/v1/"), token) if err != nil { t.Fatal(err) } @@ -146,7 +149,7 @@ func TestPushImageJSONRegistry(t *testing.T) { func TestPushImageLayerRegistry(t *testing.T) { r := spawnTestRegistrySession(t) layer := strings.NewReader("") - _, _, err := r.PushImageLayerRegistry(IMAGE_ID, layer, makeURL("/v1/"), TOKEN, []byte{}) + _, _, err := r.PushImageLayerRegistry(imageID, layer, makeURL("/v1/"), token, []byte{}) if err != nil { t.Fatal(err) } @@ -180,7 +183,7 @@ func TestResolveRepositoryName(t *testing.T) { func TestPushRegistryTag(t *testing.T) { r := spawnTestRegistrySession(t) - err := r.PushRegistryTag("foo42/bar", IMAGE_ID, "stable", makeURL("/v1/"), TOKEN) + err := r.PushRegistryTag("foo42/bar", imageID, "stable", makeURL("/v1/"), token) if err != nil { t.Fatal(err) } diff --git a/docs/session.go b/docs/session.go index 5067b8d5..ff0be343 100644 --- a/docs/session.go +++ b/docs/session.go @@ -3,6 +3,7 @@ package registry import ( "bytes" "crypto/sha256" + // this is required for some certificates _ "crypto/sha512" "encoding/hex" "encoding/json" @@ -243,11 +244,11 @@ func (r *Session) GetRemoteTags(registries []string, repository string, token [] func buildEndpointsList(headers []string, indexEp string) ([]string, error) { var endpoints []string - parsedUrl, err := url.Parse(indexEp) + parsedURL, err := url.Parse(indexEp) if err != nil { return nil, err } - var urlScheme = parsedUrl.Scheme + var urlScheme = parsedURL.Scheme // The Registry's URL scheme has to match the Index' for _, ep := range headers { epList := strings.Split(ep, ",") From 20867c3b1ffe5e4bca669295ad2d76b53f2ca672 Mon Sep 17 00:00:00 2001 From: "Daniel, Dao Quang Minh" Date: Wed, 15 Oct 2014 22:39:51 -0400 Subject: [PATCH 02/26] Avoid fallback to SSL protocols < TLS1.0 Signed-off-by: Tibor Vass Docker-DCO-1.1-Signed-off-by: Daniel, Dao Quang Minh (github: dqminh) --- docs/registry.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/registry.go b/docs/registry.go index fd74b751..0c648a94 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -36,7 +36,11 @@ const ( ) func newClient(jar http.CookieJar, roots *x509.CertPool, cert *tls.Certificate, timeout TimeoutType) *http.Client { - tlsConfig := tls.Config{RootCAs: roots} + tlsConfig := tls.Config{ + RootCAs: roots, + // Avoid fallback to SSL protocols < TLS1.0 + MinVersion: tls.VersionTLS10, + } if cert != nil { tlsConfig.Certificates = append(tlsConfig.Certificates, *cert) From 3a6fe4c5c9bc8119c5468b3d1fa8b7046d87ca8f Mon Sep 17 00:00:00 2001 From: Dan Walsh Date: Tue, 14 Oct 2014 09:19:45 -0400 Subject: [PATCH 03/26] On Red Hat Registry Servers we return 404 on certification errors. We do this to prevent leakage of information, we don't want people to be able to probe for existing content. According to RFC 2616, "This status code (404) is commonly used when the server does not wish to reveal exactly why the request has been refused, or when no other response i is applicable." https://www.ietf.org/rfc/rfc2616.txt 10.4.4 403 Forbidden The server understood the request, but is refusing to fulfill it. Authorization will not help and the request SHOULD NOT be repeated. If the request method was not HEAD and the server wishes to make public why the request has not been fulfilled, it SHOULD describe the reason for the refusal in the entity. If the server does not wish to make this information available to the client, the status code 404 (Not Found) can be used instead. 10.4.5 404 Not Found The server has not found anything matching the Request-URI. No indication is given of whether the condition is temporary or permanent. The 410 (Gone) status code SHOULD be used if the server knows, through some internally configurable mechanism, that an old resource is permanently unavailable and has no forwarding address. This status code is commonly used when the server does not wish to reveal exactly why the request has been refused, or when no other response is applicable. When docker is running through its certificates, it should continue trying with a new certificate even if it gets back a 404 error code. Docker-DCO-1.1-Signed-off-by: Dan Walsh (github: rhatdan) --- docs/registry.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/registry.go b/docs/registry.go index 0c648a94..d1315ed4 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -147,7 +147,10 @@ func doRequest(req *http.Request, jar http.CookieJar, timeout TimeoutType) (*htt client := newClient(jar, pool, cert, timeout) res, err := client.Do(req) // If this is the last cert, otherwise, continue to next cert if 403 or 5xx - if i == len(certs)-1 || err == nil && res.StatusCode != 403 && res.StatusCode < 500 { + if i == len(certs)-1 || err == nil && + res.StatusCode != 403 && + res.StatusCode != 404 && + res.StatusCode < 500 { return res, client, err } } From ef57ab120c604d1d52d402ef58ce0b33a67c0253 Mon Sep 17 00:00:00 2001 From: Gleb M Borisov Date: Tue, 21 Oct 2014 03:45:45 +0400 Subject: [PATCH 04/26] Use dual-stack Dialer when talking to registy Signed-off-by: Gleb M. Borisov --- docs/registry.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index d1315ed4..0b3ec12b 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -56,7 +56,9 @@ func newClient(jar http.CookieJar, roots *x509.CertPool, cert *tls.Certificate, case ConnectTimeout: httpTransport.Dial = func(proto string, addr string) (net.Conn, error) { // Set the connect timeout to 5 seconds - conn, err := net.DialTimeout(proto, addr, 5*time.Second) + d := net.Dialer{Timeout: 5 * time.Second, DualStack: true} + + conn, err := d.Dial(proto, addr) if err != nil { return nil, err } @@ -66,7 +68,9 @@ func newClient(jar http.CookieJar, roots *x509.CertPool, cert *tls.Certificate, } case ReceiveTimeout: httpTransport.Dial = func(proto string, addr string) (net.Conn, error) { - conn, err := net.Dial(proto, addr) + d := net.Dialer{DualStack: true} + + conn, err := d.Dial(proto, addr) if err != nil { return nil, err } From bcbb7e0c416f95d35c5709df2049858dffedc358 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Fri, 3 Oct 2014 15:46:42 -0400 Subject: [PATCH 05/26] registry/endpoint: make it testable Signed-off-by: Vincent Batts --- docs/endpoint.go | 27 +++++++++++++++++---------- docs/endpoint_test.go | 27 +++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 10 deletions(-) create mode 100644 docs/endpoint_test.go diff --git a/docs/endpoint.go b/docs/endpoint.go index 58311d32..99f52578 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -35,16 +35,7 @@ func scanForAPIVersion(hostname string) (string, APIVersion) { } func NewEndpoint(hostname string) (*Endpoint, error) { - var ( - endpoint Endpoint - trimmedHostname string - err error - ) - if !strings.HasPrefix(hostname, "http") { - hostname = "https://" + hostname - } - trimmedHostname, endpoint.Version = scanForAPIVersion(hostname) - endpoint.URL, err = url.Parse(trimmedHostname) + endpoint, err := newEndpoint(hostname) if err != nil { return nil, err } @@ -59,6 +50,22 @@ func NewEndpoint(hostname string) (*Endpoint, error) { } } + return endpoint, nil +} +func newEndpoint(hostname string) (*Endpoint, error) { + var ( + endpoint Endpoint + trimmedHostname string + err error + ) + if !strings.HasPrefix(hostname, "http") { + hostname = "https://" + hostname + } + trimmedHostname, endpoint.Version = scanForAPIVersion(hostname) + endpoint.URL, err = url.Parse(trimmedHostname) + if err != nil { + return nil, err + } return &endpoint, nil } diff --git a/docs/endpoint_test.go b/docs/endpoint_test.go new file mode 100644 index 00000000..0ec1220d --- /dev/null +++ b/docs/endpoint_test.go @@ -0,0 +1,27 @@ +package registry + +import "testing" + +func TestEndpointParse(t *testing.T) { + testData := []struct { + str string + expected string + }{ + {IndexServerAddress(), IndexServerAddress()}, + {"http://0.0.0.0:5000", "http://0.0.0.0:5000/v1/"}, + {"0.0.0.0:5000", "https://0.0.0.0:5000/v1/"}, + } + for _, td := range testData { + e, err := newEndpoint(td.str) + if err != nil { + t.Errorf("%q: %s", td.str, err) + } + if e == nil { + t.Logf("something's fishy, endpoint for %q is nil", td.str) + continue + } + if e.String() != td.expected { + t.Errorf("expected %q, got %q", td.expected, e.String()) + } + } +} From 32654af8b6e538aaf0a4f4261ced50de1a4fe806 Mon Sep 17 00:00:00 2001 From: Alexandr Morozov Date: Fri, 24 Oct 2014 10:12:35 -0700 Subject: [PATCH 06/26] Use logrus everywhere for logging Fixed #8761 Signed-off-by: Alexandr Morozov --- docs/endpoint.go | 2 +- docs/registry_mock_test.go | 2 +- docs/session.go | 2 +- docs/session_v2.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/endpoint.go b/docs/endpoint.go index 58311d32..05b5c08b 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -9,7 +9,7 @@ import ( "net/url" "strings" - "github.com/docker/docker/pkg/log" + log "github.com/Sirupsen/logrus" ) // scans string for api version in the URL path. returns the trimmed hostname, if version found, string and API version. diff --git a/docs/registry_mock_test.go b/docs/registry_mock_test.go index 967d8b26..02884c62 100644 --- a/docs/registry_mock_test.go +++ b/docs/registry_mock_test.go @@ -15,7 +15,7 @@ import ( "github.com/gorilla/mux" - "github.com/docker/docker/pkg/log" + log "github.com/Sirupsen/logrus" ) var ( diff --git a/docs/session.go b/docs/session.go index ff0be343..de97db3a 100644 --- a/docs/session.go +++ b/docs/session.go @@ -18,7 +18,7 @@ import ( "time" "github.com/docker/docker/pkg/httputils" - "github.com/docker/docker/pkg/log" + log "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/tarsum" "github.com/docker/docker/utils" ) diff --git a/docs/session_v2.go b/docs/session_v2.go index c0bc19b3..20e9e2ee 100644 --- a/docs/session_v2.go +++ b/docs/session_v2.go @@ -8,7 +8,7 @@ import ( "net/url" "strconv" - "github.com/docker/docker/pkg/log" + log "github.com/Sirupsen/logrus" "github.com/docker/docker/utils" "github.com/gorilla/mux" ) From 0827b71157a1d2afdca5d6fdf3f0b3b44efca826 Mon Sep 17 00:00:00 2001 From: Alexandr Morozov Date: Fri, 24 Oct 2014 15:11:48 -0700 Subject: [PATCH 07/26] Mass gofmt Signed-off-by: Alexandr Morozov --- docs/session.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/session.go b/docs/session.go index de97db3a..0c5f0139 100644 --- a/docs/session.go +++ b/docs/session.go @@ -17,8 +17,8 @@ import ( "strings" "time" - "github.com/docker/docker/pkg/httputils" log "github.com/Sirupsen/logrus" + "github.com/docker/docker/pkg/httputils" "github.com/docker/docker/pkg/tarsum" "github.com/docker/docker/utils" ) From 1a8edd0d55316d28c46bcac559565d9440d5695f Mon Sep 17 00:00:00 2001 From: Igor Dolzhikov Date: Tue, 28 Oct 2014 01:04:36 +0600 Subject: [PATCH 08/26] excluding unused transformation to []byte Signed-off-by: Igor Dolzhikov --- docs/session.go | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/docs/session.go b/docs/session.go index 0c5f0139..8dbf1362 100644 --- a/docs/session.go +++ b/docs/session.go @@ -230,11 +230,7 @@ func (r *Session) GetRemoteTags(registries []string, repository string, token [] } result := make(map[string]string) - rawJSON, err := ioutil.ReadAll(res.Body) - if err != nil { - return nil, err - } - if err := json.Unmarshal(rawJSON, &result); err != nil { + if err := json.NewDecoder(res.Body).Decode(&result); err != nil { return nil, err } return result, nil @@ -305,12 +301,8 @@ func (r *Session) GetRepositoryData(remote string) (*RepositoryData, error) { endpoints = append(endpoints, fmt.Sprintf("%s://%s/v1/", r.indexEndpoint.URL.Scheme, req.URL.Host)) } - checksumsJSON, err := ioutil.ReadAll(res.Body) - if err != nil { - return nil, err - } remoteChecksums := []*ImgData{} - if err := json.Unmarshal(checksumsJSON, &remoteChecksums); err != nil { + if err := json.NewDecoder(res.Body).Decode(&remoteChecksums); err != nil { return nil, err } @@ -590,12 +582,8 @@ func (r *Session) SearchRepositories(term string) (*SearchResults, error) { if res.StatusCode != 200 { return nil, utils.NewHTTPRequestError(fmt.Sprintf("Unexepected status code %d", res.StatusCode), res) } - rawData, err := ioutil.ReadAll(res.Body) - if err != nil { - return nil, err - } result := new(SearchResults) - err = json.Unmarshal(rawData, result) + err = json.NewDecoder(res.Body).Decode(result) return result, err } From 22f87eb9bed22ed4aba4de0146bcba2288df1a87 Mon Sep 17 00:00:00 2001 From: Jessica Frazelle Date: Tue, 28 Oct 2014 17:42:03 -0700 Subject: [PATCH 09/26] Fix error on successful login. Docker-DCO-1.1-Signed-off-by: Jessica Frazelle (github: jfrazelle) --- docs/auth.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/auth.go b/docs/auth.go index c9067e7a..1b111795 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -219,7 +219,7 @@ func Login(authConfig *AuthConfig, factory *utils.HTTPRequestFactory) (string, e return "", err } if resp.StatusCode == 200 { - status = "Login Succeeded" + return "Login Succeeded", nil } else if resp.StatusCode == 401 { return "", fmt.Errorf("Wrong login/password, please try again") } else if resp.StatusCode == 403 { @@ -247,7 +247,7 @@ func Login(authConfig *AuthConfig, factory *utils.HTTPRequestFactory) (string, e return "", err } if resp.StatusCode == 200 { - status = "Login Succeeded" + return "Login Succeeded", nil } else if resp.StatusCode == 401 { return "", fmt.Errorf("Wrong login/password, please try again") } else { From 034c1cfb9dcd6ba1b7a242ebfbabde80858a91fc Mon Sep 17 00:00:00 2001 From: unclejack Date: Sat, 16 Aug 2014 13:27:04 +0300 Subject: [PATCH 10/26] make http usage for registry explicit Docker-DCO-1.1-Signed-off-by: Cristian Staretu (github: unclejack) Conflicts: daemon/config.go daemon/daemon.go graph/pull.go graph/push.go graph/tags.go registry/registry.go registry/service.go --- docs/registry.go | 49 ++++++++++++++++++++++++++++++++++++++++++++++++ docs/service.go | 2 +- 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/docs/registry.go b/docs/registry.go index 0b3ec12b..8599d368 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -213,6 +213,55 @@ func ResolveRepositoryName(reposName string) (string, string, error) { return hostname, reposName, nil } +// this method expands the registry name as used in the prefix of a repo +// to a full url. if it already is a url, there will be no change. +func ExpandAndVerifyRegistryUrl(hostname string, secure bool) (endpoint string, err error) { + if strings.HasPrefix(hostname, "http:") || strings.HasPrefix(hostname, "https:") { + // if there is no slash after https:// (8 characters) then we have no path in the url + if strings.LastIndex(hostname, "/") < 9 { + // there is no path given. Expand with default path + hostname = hostname + "/v1/" + } + if _, err := pingRegistryEndpoint(hostname); err != nil { + return "", errors.New("Invalid Registry endpoint: " + err.Error()) + } + return hostname, nil + } + + // use HTTPS if secure, otherwise use HTTP + if secure { + endpoint = fmt.Sprintf("https://%s/v1/", hostname) + } else { + endpoint = fmt.Sprintf("http://%s/v1/", hostname) + } + _, err = pingRegistryEndpoint(endpoint) + if err != nil { + //TODO: triggering highland build can be done there without "failing" + err = fmt.Errorf("Invalid registry endpoint '%s': %s ", endpoint, err) + if secure { + err = fmt.Errorf("%s. If this private registry supports only HTTP, please add `--insecure-registry %s` to the daemon's arguments.", err, hostname) + } + return "", err + } + return endpoint, nil +} + +// this method verifies if the provided hostname is part of the list of +// insecure registries and returns false if HTTP should be used +func IsSecure(hostname string, insecureRegistries []string) (secure bool) { + secure = true + for _, h := range insecureRegistries { + if hostname == h { + secure = false + break + } + } + if hostname == IndexServerAddress() { + secure = true + } + return +} + func trustedLocation(req *http.Request) bool { var ( trusteds = []string{"docker.com", "docker.io"} diff --git a/docs/service.go b/docs/service.go index f7b35300..334e7c2e 100644 --- a/docs/service.go +++ b/docs/service.go @@ -40,7 +40,7 @@ func (s *Service) Auth(job *engine.Job) engine.Status { job.GetenvJson("authConfig", authConfig) // TODO: this is only done here because auth and registry need to be merged into one pkg if addr := authConfig.ServerAddress; addr != "" && addr != IndexServerAddress() { - endpoint, err := NewEndpoint(addr) + endpoint, err := NewEndpoint(addr, true) if err != nil { return job.Error(err) } From 50e11c9d8ea20e950280757d909857cdecda1951 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Tue, 19 Aug 2014 11:54:42 -0700 Subject: [PATCH 11/26] Refactor IsSecure change Fix issue with restoring the tag store and setting static configuration from the daemon. i.e. the field on the TagStore struct must be made internal or the json.Unmarshal in restore will overwrite the insecure registries to be an empty struct. Signed-off-by: Michael Crosby Conflicts: graph/pull.go graph/push.go graph/tags.go --- docs/registry.go | 44 +++++++++++++++++++------------------------- 1 file changed, 19 insertions(+), 25 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 8599d368..78899681 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -215,51 +215,45 @@ func ResolveRepositoryName(reposName string) (string, string, error) { // this method expands the registry name as used in the prefix of a repo // to a full url. if it already is a url, there will be no change. -func ExpandAndVerifyRegistryUrl(hostname string, secure bool) (endpoint string, err error) { - if strings.HasPrefix(hostname, "http:") || strings.HasPrefix(hostname, "https:") { - // if there is no slash after https:// (8 characters) then we have no path in the url - if strings.LastIndex(hostname, "/") < 9 { - // there is no path given. Expand with default path - hostname = hostname + "/v1/" - } - if _, err := pingRegistryEndpoint(hostname); err != nil { - return "", errors.New("Invalid Registry endpoint: " + err.Error()) - } +func ExpandAndVerifyRegistryUrl(hostname string, secure bool) (string, error) { + if hostname == IndexServerAddress() { return hostname, nil } - // use HTTPS if secure, otherwise use HTTP + endpoint := fmt.Sprintf("http://%s/v1/", hostname) + if secure { endpoint = fmt.Sprintf("https://%s/v1/", hostname) - } else { - endpoint = fmt.Sprintf("http://%s/v1/", hostname) } - _, err = pingRegistryEndpoint(endpoint) - if err != nil { + + if _, oerr := pingRegistryEndpoint(endpoint); oerr != nil { //TODO: triggering highland build can be done there without "failing" - err = fmt.Errorf("Invalid registry endpoint '%s': %s ", endpoint, err) + err := fmt.Errorf("Invalid registry endpoint '%s': %s ", endpoint, oerr) + if secure { - err = fmt.Errorf("%s. If this private registry supports only HTTP, please add `--insecure-registry %s` to the daemon's arguments.", err, hostname) + err = fmt.Errorf("%s. If this private registry supports only HTTP, please add `--insecure-registry %s` to the daemon's arguments.", oerr, hostname) } + return "", err } + return endpoint, nil } // this method verifies if the provided hostname is part of the list of // insecure registries and returns false if HTTP should be used -func IsSecure(hostname string, insecureRegistries []string) (secure bool) { - secure = true +func IsSecure(hostname string, insecureRegistries []string) bool { + if hostname == IndexServerAddress() { + return true + } + for _, h := range insecureRegistries { if hostname == h { - secure = false - break + return false } } - if hostname == IndexServerAddress() { - secure = true - } - return + + return true } func trustedLocation(req *http.Request) bool { From 552c17d618033c0f07a4d063ce0d261db214bcb4 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Wed, 20 Aug 2014 08:31:24 -0700 Subject: [PATCH 12/26] Don't hard code true for auth job Signed-off-by: Michael Crosby Conflicts: registry/service.go --- docs/service.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/service.go b/docs/service.go index 334e7c2e..890837ca 100644 --- a/docs/service.go +++ b/docs/service.go @@ -13,12 +13,15 @@ import ( // 'pull': Download images from any registry (TODO) // 'push': Upload images to any registry (TODO) type Service struct { + insecureRegistries []string } // NewService returns a new instance of Service ready to be // installed no an engine. -func NewService() *Service { - return &Service{} +func NewService(insecureRegistries []string) *Service { + return &Service{ + insecureRegistries: insecureRegistries, + } } // Install installs registry capabilities to eng. @@ -32,15 +35,12 @@ func (s *Service) Install(eng *engine.Engine) error { // and returns OK if authentication was sucessful. // It can be used to verify the validity of a client's credentials. func (s *Service) Auth(job *engine.Job) engine.Status { - var ( - err error - authConfig = &AuthConfig{} - ) + var authConfig = new(AuthConfig) job.GetenvJson("authConfig", authConfig) - // TODO: this is only done here because auth and registry need to be merged into one pkg + if addr := authConfig.ServerAddress; addr != "" && addr != IndexServerAddress() { - endpoint, err := NewEndpoint(addr, true) + endpoint, err := NewEndpoint(addr, IsSecure(addr, s.insecureRegistries)) if err != nil { return job.Error(err) } @@ -49,11 +49,11 @@ func (s *Service) Auth(job *engine.Job) engine.Status { } authConfig.ServerAddress = endpoint.String() } - status, err := Login(authConfig, HTTPRequestFactory(nil)) - if err != nil { + + if _, err := Login(authConfig, HTTPRequestFactory(nil)); err != nil { return job.Error(err) } - job.Printf("%s\n", status) + return engine.StatusOK } From 1b72e0234eb0ccc1769059fcdf2d89369e5c7963 Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Fri, 10 Oct 2014 23:22:12 -0400 Subject: [PATCH 13/26] Do not verify certificate when using --insecure-registry on an HTTPS registry Signed-off-by: Tibor Vass Conflicts: registry/registry.go registry/registry_test.go registry/service.go registry/session.go Conflicts: registry/endpoint.go registry/registry.go --- docs/endpoint.go | 51 +++++++++++---- docs/endpoint_test.go | 2 +- docs/registry.go | 143 +++++++++++++++++------------------------- docs/registry_test.go | 4 +- docs/service.go | 5 +- docs/session.go | 2 +- 6 files changed, 104 insertions(+), 103 deletions(-) diff --git a/docs/endpoint.go b/docs/endpoint.go index 639c9970..88dbeafd 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -2,7 +2,6 @@ package registry import ( "encoding/json" - "errors" "fmt" "io/ioutil" "net/http" @@ -34,27 +33,40 @@ func scanForAPIVersion(hostname string) (string, APIVersion) { return hostname, DefaultAPIVersion } -func NewEndpoint(hostname string) (*Endpoint, error) { - endpoint, err := newEndpoint(hostname) +func NewEndpoint(hostname string, secure bool) (*Endpoint, error) { + endpoint, err := newEndpoint(hostname, secure) if err != nil { return nil, err } + // Try HTTPS ping to registry endpoint.URL.Scheme = "https" if _, err := endpoint.Ping(); err != nil { - log.Debugf("Registry %s does not work (%s), falling back to http", endpoint, err) - // TODO: Check if http fallback is enabled - endpoint.URL.Scheme = "http" - if _, err = endpoint.Ping(); err != nil { - return nil, errors.New("Invalid Registry endpoint: " + err.Error()) + + //TODO: triggering highland build can be done there without "failing" + + if secure { + // If registry is secure and HTTPS failed, show user the error and tell them about `--insecure-registry` + // in case that's what they need. DO NOT accept unknown CA certificates, and DO NOT fallback to HTTP. + return nil, fmt.Errorf("Invalid registry endpoint %s: %v. If this private registry supports only HTTP or HTTPS with an unknown CA certificate, please add `--insecure-registry %s` to the daemon's arguments. In the case of HTTPS, if you have access to the registry's CA certificate, no need for the flag; simply place the CA certificate at /etc/docker/certs.d/%s/ca.crt", endpoint, err, endpoint.URL.Host, endpoint.URL.Host) } + + // If registry is insecure and HTTPS failed, fallback to HTTP. + log.Debugf("Error from registry %q marked as insecure: %v. Insecurely falling back to HTTP", endpoint, err) + endpoint.URL.Scheme = "http" + _, err2 := endpoint.Ping() + if err2 == nil { + return endpoint, nil + } + + return nil, fmt.Errorf("Invalid registry endpoint %q. HTTPS attempt: %v. HTTP attempt: %v", endpoint, err, err2) } return endpoint, nil } -func newEndpoint(hostname string) (*Endpoint, error) { +func newEndpoint(hostname string, secure bool) (*Endpoint, error) { var ( - endpoint Endpoint + endpoint = Endpoint{secure: secure} trimmedHostname string err error ) @@ -72,6 +84,7 @@ func newEndpoint(hostname string) (*Endpoint, error) { type Endpoint struct { URL *url.URL Version APIVersion + secure bool } // Get the formated URL for the root of this registry Endpoint @@ -95,7 +108,7 @@ func (e Endpoint) Ping() (RegistryInfo, error) { return RegistryInfo{Standalone: false}, err } - resp, _, err := doRequest(req, nil, ConnectTimeout) + resp, _, err := doRequest(req, nil, ConnectTimeout, e.secure) if err != nil { return RegistryInfo{Standalone: false}, err } @@ -134,3 +147,19 @@ func (e Endpoint) Ping() (RegistryInfo, error) { log.Debugf("RegistryInfo.Standalone: %t", info.Standalone) return info, nil } + +// IsSecure returns false if the provided hostname is part of the list of insecure registries. +// Insecure registries accept HTTP and/or accept HTTPS with certificates from unknown CAs. +func IsSecure(hostname string, insecureRegistries []string) bool { + if hostname == IndexServerAddress() { + return true + } + + for _, h := range insecureRegistries { + if hostname == h { + return false + } + } + + return true +} diff --git a/docs/endpoint_test.go b/docs/endpoint_test.go index 0ec1220d..def5e0d7 100644 --- a/docs/endpoint_test.go +++ b/docs/endpoint_test.go @@ -12,7 +12,7 @@ func TestEndpointParse(t *testing.T) { {"0.0.0.0:5000", "https://0.0.0.0:5000/v1/"}, } for _, td := range testData { - e, err := newEndpoint(td.str) + e, err := newEndpoint(td.str, true) if err != nil { t.Errorf("%q: %s", td.str, err) } diff --git a/docs/registry.go b/docs/registry.go index 78899681..8d436374 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -14,6 +14,7 @@ import ( "strings" "time" + log "github.com/Sirupsen/logrus" "github.com/docker/docker/utils" ) @@ -35,7 +36,7 @@ const ( ConnectTimeout ) -func newClient(jar http.CookieJar, roots *x509.CertPool, cert *tls.Certificate, timeout TimeoutType) *http.Client { +func newClient(jar http.CookieJar, roots *x509.CertPool, cert *tls.Certificate, timeout TimeoutType, secure bool) *http.Client { tlsConfig := tls.Config{ RootCAs: roots, // Avoid fallback to SSL protocols < TLS1.0 @@ -46,6 +47,10 @@ func newClient(jar http.CookieJar, roots *x509.CertPool, cert *tls.Certificate, tlsConfig.Certificates = append(tlsConfig.Certificates, *cert) } + if !secure { + tlsConfig.InsecureSkipVerify = true + } + httpTransport := &http.Transport{ DisableKeepAlives: true, Proxy: http.ProxyFromEnvironment, @@ -86,69 +91,76 @@ func newClient(jar http.CookieJar, roots *x509.CertPool, cert *tls.Certificate, } } -func doRequest(req *http.Request, jar http.CookieJar, timeout TimeoutType) (*http.Response, *http.Client, error) { - hasFile := func(files []os.FileInfo, name string) bool { - for _, f := range files { - if f.Name() == name { - return true - } - } - return false - } - - hostDir := path.Join("/etc/docker/certs.d", req.URL.Host) - fs, err := ioutil.ReadDir(hostDir) - if err != nil && !os.IsNotExist(err) { - return nil, nil, err - } - +func doRequest(req *http.Request, jar http.CookieJar, timeout TimeoutType, secure bool) (*http.Response, *http.Client, error) { var ( pool *x509.CertPool certs []*tls.Certificate ) - for _, f := range fs { - if strings.HasSuffix(f.Name(), ".crt") { - if pool == nil { - pool = x509.NewCertPool() + if secure && req.URL.Scheme == "https" { + hasFile := func(files []os.FileInfo, name string) bool { + for _, f := range files { + if f.Name() == name { + return true + } } - data, err := ioutil.ReadFile(path.Join(hostDir, f.Name())) - if err != nil { - return nil, nil, err - } - pool.AppendCertsFromPEM(data) + return false } - if strings.HasSuffix(f.Name(), ".cert") { - certName := f.Name() - keyName := certName[:len(certName)-5] + ".key" - if !hasFile(fs, keyName) { - return nil, nil, fmt.Errorf("Missing key %s for certificate %s", keyName, certName) - } - cert, err := tls.LoadX509KeyPair(path.Join(hostDir, certName), path.Join(hostDir, keyName)) - if err != nil { - return nil, nil, err - } - certs = append(certs, &cert) + + hostDir := path.Join("/etc/docker/certs.d", req.URL.Host) + log.Debugf("hostDir: %s", hostDir) + fs, err := ioutil.ReadDir(hostDir) + if err != nil && !os.IsNotExist(err) { + return nil, nil, err } - if strings.HasSuffix(f.Name(), ".key") { - keyName := f.Name() - certName := keyName[:len(keyName)-4] + ".cert" - if !hasFile(fs, certName) { - return nil, nil, fmt.Errorf("Missing certificate %s for key %s", certName, keyName) + + for _, f := range fs { + if strings.HasSuffix(f.Name(), ".crt") { + if pool == nil { + pool = x509.NewCertPool() + } + log.Debugf("crt: %s", hostDir+"/"+f.Name()) + data, err := ioutil.ReadFile(path.Join(hostDir, f.Name())) + if err != nil { + return nil, nil, err + } + pool.AppendCertsFromPEM(data) + } + if strings.HasSuffix(f.Name(), ".cert") { + certName := f.Name() + keyName := certName[:len(certName)-5] + ".key" + log.Debugf("cert: %s", hostDir+"/"+f.Name()) + if !hasFile(fs, keyName) { + return nil, nil, fmt.Errorf("Missing key %s for certificate %s", keyName, certName) + } + cert, err := tls.LoadX509KeyPair(path.Join(hostDir, certName), path.Join(hostDir, keyName)) + if err != nil { + return nil, nil, err + } + certs = append(certs, &cert) + } + if strings.HasSuffix(f.Name(), ".key") { + keyName := f.Name() + certName := keyName[:len(keyName)-4] + ".cert" + log.Debugf("key: %s", hostDir+"/"+f.Name()) + if !hasFile(fs, certName) { + return nil, nil, fmt.Errorf("Missing certificate %s for key %s", certName, keyName) + } } } } if len(certs) == 0 { - client := newClient(jar, pool, nil, timeout) + client := newClient(jar, pool, nil, timeout, secure) res, err := client.Do(req) if err != nil { return nil, nil, err } return res, client, nil } + for i, cert := range certs { - client := newClient(jar, pool, cert, timeout) + client := newClient(jar, pool, cert, timeout, secure) res, err := client.Do(req) // If this is the last cert, otherwise, continue to next cert if 403 or 5xx if i == len(certs)-1 || err == nil && @@ -213,49 +225,6 @@ func ResolveRepositoryName(reposName string) (string, string, error) { return hostname, reposName, nil } -// this method expands the registry name as used in the prefix of a repo -// to a full url. if it already is a url, there will be no change. -func ExpandAndVerifyRegistryUrl(hostname string, secure bool) (string, error) { - if hostname == IndexServerAddress() { - return hostname, nil - } - - endpoint := fmt.Sprintf("http://%s/v1/", hostname) - - if secure { - endpoint = fmt.Sprintf("https://%s/v1/", hostname) - } - - if _, oerr := pingRegistryEndpoint(endpoint); oerr != nil { - //TODO: triggering highland build can be done there without "failing" - err := fmt.Errorf("Invalid registry endpoint '%s': %s ", endpoint, oerr) - - if secure { - err = fmt.Errorf("%s. If this private registry supports only HTTP, please add `--insecure-registry %s` to the daemon's arguments.", oerr, hostname) - } - - return "", err - } - - return endpoint, nil -} - -// this method verifies if the provided hostname is part of the list of -// insecure registries and returns false if HTTP should be used -func IsSecure(hostname string, insecureRegistries []string) bool { - if hostname == IndexServerAddress() { - return true - } - - for _, h := range insecureRegistries { - if hostname == h { - return false - } - } - - return true -} - func trustedLocation(req *http.Request) bool { var ( trusteds = []string{"docker.com", "docker.io"} diff --git a/docs/registry_test.go b/docs/registry_test.go index fdf714e8..23aef6c3 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -21,7 +21,7 @@ const ( func spawnTestRegistrySession(t *testing.T) *Session { authConfig := &AuthConfig{} - endpoint, err := NewEndpoint(makeURL("/v1/")) + endpoint, err := NewEndpoint(makeURL("/v1/"), false) if err != nil { t.Fatal(err) } @@ -33,7 +33,7 @@ func spawnTestRegistrySession(t *testing.T) *Session { } func TestPingRegistryEndpoint(t *testing.T) { - ep, err := NewEndpoint(makeURL("/v1/")) + ep, err := NewEndpoint(makeURL("/v1/"), false) if err != nil { t.Fatal(err) } diff --git a/docs/service.go b/docs/service.go index 890837ca..32274f40 100644 --- a/docs/service.go +++ b/docs/service.go @@ -89,7 +89,10 @@ func (s *Service) Search(job *engine.Job) engine.Status { if err != nil { return job.Error(err) } - endpoint, err := NewEndpoint(hostname) + + secure := IsSecure(hostname, s.insecureRegistries) + + endpoint, err := NewEndpoint(hostname, secure) if err != nil { return job.Error(err) } diff --git a/docs/session.go b/docs/session.go index 8dbf1362..ba6df358 100644 --- a/docs/session.go +++ b/docs/session.go @@ -65,7 +65,7 @@ func NewSession(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, endpo } func (r *Session) doRequest(req *http.Request) (*http.Response, *http.Client, error) { - return doRequest(req, r.jar, r.timeout) + return doRequest(req, r.jar, r.timeout, r.indexEndpoint.secure) } // Retrieve the history of a given image from the Registry. From 47a494e0fd9d7b2d4aa31ce1bf48ed9992407f56 Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Tue, 28 Oct 2014 21:20:30 -0400 Subject: [PATCH 14/26] Fix login command Signed-off-by: Tibor Vass --- docs/service.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/service.go b/docs/service.go index 32274f40..7051d934 100644 --- a/docs/service.go +++ b/docs/service.go @@ -50,9 +50,11 @@ func (s *Service) Auth(job *engine.Job) engine.Status { authConfig.ServerAddress = endpoint.String() } - if _, err := Login(authConfig, HTTPRequestFactory(nil)); err != nil { + status, err := Login(authConfig, HTTPRequestFactory(nil)) + if err != nil { return job.Error(err) } + job.Printf("%s\n", status) return engine.StatusOK } From 7dd4199fe8055877979a51426a2ff50e5e04c0f5 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Thu, 9 Oct 2014 13:52:30 -0400 Subject: [PATCH 15/26] registry: don't iterate through certs the golang tls.Conn does a fine job of that. http://golang.org/src/pkg/crypto/tls/handshake_client.go?#L334 Signed-off-by: Vincent Batts --- docs/registry.go | 29 ++++++++--------------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 8d436374..e1d22b09 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -36,15 +36,12 @@ const ( ConnectTimeout ) -func newClient(jar http.CookieJar, roots *x509.CertPool, cert *tls.Certificate, timeout TimeoutType, secure bool) *http.Client { +func newClient(jar http.CookieJar, roots *x509.CertPool, certs []tls.Certificate, timeout TimeoutType, secure bool) *http.Client { tlsConfig := tls.Config{ RootCAs: roots, // Avoid fallback to SSL protocols < TLS1.0 - MinVersion: tls.VersionTLS10, - } - - if cert != nil { - tlsConfig.Certificates = append(tlsConfig.Certificates, *cert) + MinVersion: tls.VersionTLS10, + Certificates: certs, } if !secure { @@ -94,7 +91,7 @@ func newClient(jar http.CookieJar, roots *x509.CertPool, cert *tls.Certificate, func doRequest(req *http.Request, jar http.CookieJar, timeout TimeoutType, secure bool) (*http.Response, *http.Client, error) { var ( pool *x509.CertPool - certs []*tls.Certificate + certs []tls.Certificate ) if secure && req.URL.Scheme == "https" { @@ -137,7 +134,7 @@ func doRequest(req *http.Request, jar http.CookieJar, timeout TimeoutType, secur if err != nil { return nil, nil, err } - certs = append(certs, &cert) + certs = append(certs, cert) } if strings.HasSuffix(f.Name(), ".key") { keyName := f.Name() @@ -159,19 +156,9 @@ func doRequest(req *http.Request, jar http.CookieJar, timeout TimeoutType, secur return res, client, nil } - for i, cert := range certs { - client := newClient(jar, pool, cert, timeout, secure) - res, err := client.Do(req) - // If this is the last cert, otherwise, continue to next cert if 403 or 5xx - if i == len(certs)-1 || err == nil && - res.StatusCode != 403 && - res.StatusCode != 404 && - res.StatusCode < 500 { - return res, client, err - } - } - - return nil, nil, nil + client := newClient(jar, pool, certs, timeout, secure) + res, err := client.Do(req) + return res, client, err } func validateRepositoryName(repositoryName string) error { From cd246befe28710ed350c94996e3ee887518a9a4b Mon Sep 17 00:00:00 2001 From: Johan Euphrosine Date: Tue, 11 Nov 2014 11:01:49 -0800 Subject: [PATCH 16/26] registry: add tests for IsSecure Signed-off-by: Johan Euphrosine --- docs/registry_test.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/registry_test.go b/docs/registry_test.go index 23aef6c3..f7b5168b 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -319,3 +319,23 @@ func TestAddRequiredHeadersToRedirectedRequests(t *testing.T) { } } } + +func TestIsSecure(t *testing.T) { + tests := []struct { + addr string + insecureRegistries []string + expected bool + }{ + {"example.com", []string{}, true}, + {"example.com", []string{"example.com"}, false}, + {"localhost", []string{"localhost:5000"}, true}, + {"localhost:5000", []string{"localhost:5000"}, false}, + {"localhost", []string{"example.com"}, true}, + {"127.0.0.1:5000", []string{"127.0.0.1:5000"}, false}, + } + for _, tt := range tests { + if sec := IsSecure(tt.addr, tt.insecureRegistries); sec != tt.expected { + t.Errorf("IsSecure failed for %q %v, expected %v got %v", tt.addr, tt.insecureRegistries, tt.expected, sec) + } + } +} From 8582d04393a05b074c35915f612a69c376564be3 Mon Sep 17 00:00:00 2001 From: Johan Euphrosine Date: Fri, 31 Oct 2014 13:00:49 -0700 Subject: [PATCH 17/26] registry: default --insecure-registry to localhost and 127.0.0.1 Signed-off-by: Johan Euphrosine --- docs/endpoint.go | 12 +++++++++++- docs/registry_test.go | 21 +++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/docs/endpoint.go b/docs/endpoint.go index 88dbeafd..cb96cb4f 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "io/ioutil" + "net" "net/http" "net/url" "strings" @@ -154,7 +155,16 @@ func IsSecure(hostname string, insecureRegistries []string) bool { if hostname == IndexServerAddress() { return true } - + if len(insecureRegistries) == 0 { + host, _, err := net.SplitHostPort(hostname) + if err != nil { + host = hostname + } + if host == "127.0.0.1" || host == "localhost" { + return false + } + return true + } for _, h := range insecureRegistries { if hostname == h { return false diff --git a/docs/registry_test.go b/docs/registry_test.go index f7b5168b..7191acea 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -339,3 +339,24 @@ func TestIsSecure(t *testing.T) { } } } + +func TestIsSecure(t *testing.T) { + tests := []struct { + addr string + insecureRegistries []string + expected bool + }{ + {"localhost", []string{}, false}, + {"localhost:5000", []string{}, false}, + {"127.0.0.1", []string{}, false}, + {"localhost", []string{"example.com"}, true}, + {"127.0.0.1", []string{"example.com"}, true}, + {"example.com", []string{}, true}, + {"example.com", []string{"example.com"}, false}, + } + for _, tt := range tests { + if sec := IsSecure(tt.addr, tt.insecureRegistries); sec != tt.expected { + t.Errorf("IsSecure failed for %q %v, expected %v got %v", tt.addr, tt.insecureRegistries, tt.expected, sec) + } + } +} From 524aa8b1a68ae467315ff26ea9128aa3b996a5d8 Mon Sep 17 00:00:00 2001 From: Erik Hollensbe Date: Wed, 12 Nov 2014 09:08:45 -0800 Subject: [PATCH 18/26] registry: always treat 127.0.0.1 as insecure for all cases anytime anywhere Docker-DCO-1.1-Signed-off-by: Erik Hollensbe (github: erikh) --- docs/endpoint.go | 20 +++++++++++++------- docs/registry_test.go | 24 ++++++------------------ 2 files changed, 19 insertions(+), 25 deletions(-) diff --git a/docs/endpoint.go b/docs/endpoint.go index cb96cb4f..0d0749d7 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -152,19 +152,25 @@ func (e Endpoint) Ping() (RegistryInfo, error) { // IsSecure returns false if the provided hostname is part of the list of insecure registries. // Insecure registries accept HTTP and/or accept HTTPS with certificates from unknown CAs. func IsSecure(hostname string, insecureRegistries []string) bool { + if hostname == IndexServerAddress() { return true } + + host, _, err := net.SplitHostPort(hostname) + + if err != nil { + host = hostname + } + + if host == "127.0.0.1" || host == "localhost" { + return false + } + if len(insecureRegistries) == 0 { - host, _, err := net.SplitHostPort(hostname) - if err != nil { - host = hostname - } - if host == "127.0.0.1" || host == "localhost" { - return false - } return true } + for _, h := range insecureRegistries { if hostname == h { return false diff --git a/docs/registry_test.go b/docs/registry_test.go index 7191acea..032c9fbf 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -328,31 +328,19 @@ func TestIsSecure(t *testing.T) { }{ {"example.com", []string{}, true}, {"example.com", []string{"example.com"}, false}, - {"localhost", []string{"localhost:5000"}, true}, + {"localhost", []string{"localhost:5000"}, false}, {"localhost:5000", []string{"localhost:5000"}, false}, - {"localhost", []string{"example.com"}, true}, + {"localhost", []string{"example.com"}, false}, {"127.0.0.1:5000", []string{"127.0.0.1:5000"}, false}, - } - for _, tt := range tests { - if sec := IsSecure(tt.addr, tt.insecureRegistries); sec != tt.expected { - t.Errorf("IsSecure failed for %q %v, expected %v got %v", tt.addr, tt.insecureRegistries, tt.expected, sec) - } - } -} - -func TestIsSecure(t *testing.T) { - tests := []struct { - addr string - insecureRegistries []string - expected bool - }{ {"localhost", []string{}, false}, {"localhost:5000", []string{}, false}, {"127.0.0.1", []string{}, false}, - {"localhost", []string{"example.com"}, true}, - {"127.0.0.1", []string{"example.com"}, true}, + {"localhost", []string{"example.com"}, false}, + {"127.0.0.1", []string{"example.com"}, false}, {"example.com", []string{}, true}, {"example.com", []string{"example.com"}, false}, + {"127.0.0.1", []string{"example.com"}, false}, + {"127.0.0.1:5000", []string{"example.com"}, false}, } for _, tt := range tests { if sec := IsSecure(tt.addr, tt.insecureRegistries); sec != tt.expected { From 80255ff22489d42040e00e25ee9e0b388ae78ef2 Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Tue, 11 Nov 2014 17:37:44 -0500 Subject: [PATCH 19/26] registry: refactor registry.IsSecure calls into registry.NewEndpoint Signed-off-by: Tibor Vass --- docs/endpoint.go | 16 +++++++++------- docs/endpoint_test.go | 2 +- docs/registry_test.go | 4 ++-- docs/service.go | 6 ++---- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/endpoint.go b/docs/endpoint.go index 0d0749d7..390eec2e 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -34,12 +34,15 @@ func scanForAPIVersion(hostname string) (string, APIVersion) { return hostname, DefaultAPIVersion } -func NewEndpoint(hostname string, secure bool) (*Endpoint, error) { - endpoint, err := newEndpoint(hostname, secure) +func NewEndpoint(hostname string, insecureRegistries []string) (*Endpoint, error) { + endpoint, err := newEndpoint(hostname) if err != nil { return nil, err } + secure := isSecure(endpoint.URL.Host, insecureRegistries) + endpoint.secure = secure + // Try HTTPS ping to registry endpoint.URL.Scheme = "https" if _, err := endpoint.Ping(); err != nil { @@ -65,9 +68,9 @@ func NewEndpoint(hostname string, secure bool) (*Endpoint, error) { return endpoint, nil } -func newEndpoint(hostname string, secure bool) (*Endpoint, error) { +func newEndpoint(hostname string) (*Endpoint, error) { var ( - endpoint = Endpoint{secure: secure} + endpoint = Endpoint{secure: true} trimmedHostname string err error ) @@ -149,10 +152,9 @@ func (e Endpoint) Ping() (RegistryInfo, error) { return info, nil } -// IsSecure returns false if the provided hostname is part of the list of insecure registries. +// isSecure returns false if the provided hostname is part of the list of insecure registries. // Insecure registries accept HTTP and/or accept HTTPS with certificates from unknown CAs. -func IsSecure(hostname string, insecureRegistries []string) bool { - +func isSecure(hostname string, insecureRegistries []string) bool { if hostname == IndexServerAddress() { return true } diff --git a/docs/endpoint_test.go b/docs/endpoint_test.go index def5e0d7..0ec1220d 100644 --- a/docs/endpoint_test.go +++ b/docs/endpoint_test.go @@ -12,7 +12,7 @@ func TestEndpointParse(t *testing.T) { {"0.0.0.0:5000", "https://0.0.0.0:5000/v1/"}, } for _, td := range testData { - e, err := newEndpoint(td.str, true) + e, err := newEndpoint(td.str) if err != nil { t.Errorf("%q: %s", td.str, err) } diff --git a/docs/registry_test.go b/docs/registry_test.go index 032c9fbf..8bc6a351 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -343,8 +343,8 @@ func TestIsSecure(t *testing.T) { {"127.0.0.1:5000", []string{"example.com"}, false}, } for _, tt := range tests { - if sec := IsSecure(tt.addr, tt.insecureRegistries); sec != tt.expected { - t.Errorf("IsSecure failed for %q %v, expected %v got %v", tt.addr, tt.insecureRegistries, tt.expected, sec) + if sec := isSecure(tt.addr, tt.insecureRegistries); sec != tt.expected { + t.Errorf("isSecure failed for %q %v, expected %v got %v", tt.addr, tt.insecureRegistries, tt.expected, sec) } } } diff --git a/docs/service.go b/docs/service.go index 7051d934..53e8278b 100644 --- a/docs/service.go +++ b/docs/service.go @@ -40,7 +40,7 @@ func (s *Service) Auth(job *engine.Job) engine.Status { job.GetenvJson("authConfig", authConfig) if addr := authConfig.ServerAddress; addr != "" && addr != IndexServerAddress() { - endpoint, err := NewEndpoint(addr, IsSecure(addr, s.insecureRegistries)) + endpoint, err := NewEndpoint(addr, s.insecureRegistries) if err != nil { return job.Error(err) } @@ -92,9 +92,7 @@ func (s *Service) Search(job *engine.Job) engine.Status { return job.Error(err) } - secure := IsSecure(hostname, s.insecureRegistries) - - endpoint, err := NewEndpoint(hostname, secure) + endpoint, err := NewEndpoint(hostname, s.insecureRegistries) if err != nil { return job.Error(err) } From cca910e878f20c842f599377470015d770784d99 Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Tue, 11 Nov 2014 20:08:59 -0500 Subject: [PATCH 20/26] Put mock registry address in insecureRegistries for unit tests Signed-off-by: Tibor Vass --- docs/registry_mock_test.go | 10 ++++++++-- docs/registry_test.go | 4 ++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/docs/registry_mock_test.go b/docs/registry_mock_test.go index 02884c62..1c710e21 100644 --- a/docs/registry_mock_test.go +++ b/docs/registry_mock_test.go @@ -19,8 +19,9 @@ import ( ) var ( - testHTTPServer *httptest.Server - testLayers = map[string]map[string]string{ + testHTTPServer *httptest.Server + insecureRegistries []string + testLayers = map[string]map[string]string{ "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20": { "json": `{"id":"77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20", "comment":"test base image","created":"2013-03-23T12:53:11.10432-07:00", @@ -100,6 +101,11 @@ func init() { r.HandleFunc("/v2/version", handlerGetPing).Methods("GET") testHTTPServer = httptest.NewServer(handlerAccessLog(r)) + URL, err := url.Parse(testHTTPServer.URL) + if err != nil { + panic(err) + } + insecureRegistries = []string{URL.Host} } func handlerAccessLog(handler http.Handler) http.Handler { diff --git a/docs/registry_test.go b/docs/registry_test.go index 8bc6a351..37dedc2a 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -21,7 +21,7 @@ const ( func spawnTestRegistrySession(t *testing.T) *Session { authConfig := &AuthConfig{} - endpoint, err := NewEndpoint(makeURL("/v1/"), false) + endpoint, err := NewEndpoint(makeURL("/v1/"), insecureRegistries) if err != nil { t.Fatal(err) } @@ -33,7 +33,7 @@ func spawnTestRegistrySession(t *testing.T) *Session { } func TestPingRegistryEndpoint(t *testing.T) { - ep, err := NewEndpoint(makeURL("/v1/"), false) + ep, err := NewEndpoint(makeURL("/v1/"), insecureRegistries) if err != nil { t.Fatal(err) } From f0920e61bfbbaa692b4a7ff5148e67317ad087b3 Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Thu, 13 Nov 2014 06:56:36 -0800 Subject: [PATCH 21/26] registry: parse INDEXSERVERADDRESS into a URL for easier check in isSecure Signed-off-by: Tibor Vass --- docs/auth.go | 10 ++++++++++ docs/endpoint.go | 14 ++++++-------- docs/endpoint_test.go | 2 +- docs/registry_test.go | 1 + 4 files changed, 18 insertions(+), 9 deletions(-) diff --git a/docs/auth.go b/docs/auth.go index 1b111795..a22d0b88 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -7,6 +7,7 @@ import ( "fmt" "io/ioutil" "net/http" + "net/url" "os" "path" "strings" @@ -27,8 +28,17 @@ const ( var ( ErrConfigFileMissing = errors.New("The Auth config file is missing") + IndexServerURL *url.URL ) +func init() { + url, err := url.Parse(INDEXSERVER) + if err != nil { + panic(err) + } + IndexServerURL = url +} + type AuthConfig struct { Username string `json:"username,omitempty"` Password string `json:"password,omitempty"` diff --git a/docs/endpoint.go b/docs/endpoint.go index 390eec2e..bd23c302 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -35,21 +35,18 @@ func scanForAPIVersion(hostname string) (string, APIVersion) { } func NewEndpoint(hostname string, insecureRegistries []string) (*Endpoint, error) { - endpoint, err := newEndpoint(hostname) + endpoint, err := newEndpoint(hostname, insecureRegistries) if err != nil { return nil, err } - secure := isSecure(endpoint.URL.Host, insecureRegistries) - endpoint.secure = secure - // Try HTTPS ping to registry endpoint.URL.Scheme = "https" if _, err := endpoint.Ping(); err != nil { //TODO: triggering highland build can be done there without "failing" - if secure { + if endpoint.secure { // If registry is secure and HTTPS failed, show user the error and tell them about `--insecure-registry` // in case that's what they need. DO NOT accept unknown CA certificates, and DO NOT fallback to HTTP. return nil, fmt.Errorf("Invalid registry endpoint %s: %v. If this private registry supports only HTTP or HTTPS with an unknown CA certificate, please add `--insecure-registry %s` to the daemon's arguments. In the case of HTTPS, if you have access to the registry's CA certificate, no need for the flag; simply place the CA certificate at /etc/docker/certs.d/%s/ca.crt", endpoint, err, endpoint.URL.Host, endpoint.URL.Host) @@ -68,9 +65,9 @@ func NewEndpoint(hostname string, insecureRegistries []string) (*Endpoint, error return endpoint, nil } -func newEndpoint(hostname string) (*Endpoint, error) { +func newEndpoint(hostname string, insecureRegistries []string) (*Endpoint, error) { var ( - endpoint = Endpoint{secure: true} + endpoint = Endpoint{} trimmedHostname string err error ) @@ -82,6 +79,7 @@ func newEndpoint(hostname string) (*Endpoint, error) { if err != nil { return nil, err } + endpoint.secure = isSecure(endpoint.URL.Host, insecureRegistries) return &endpoint, nil } @@ -155,7 +153,7 @@ func (e Endpoint) Ping() (RegistryInfo, error) { // isSecure returns false if the provided hostname is part of the list of insecure registries. // Insecure registries accept HTTP and/or accept HTTPS with certificates from unknown CAs. func isSecure(hostname string, insecureRegistries []string) bool { - if hostname == IndexServerAddress() { + if hostname == IndexServerURL.Host { return true } diff --git a/docs/endpoint_test.go b/docs/endpoint_test.go index 0ec1220d..54105ec1 100644 --- a/docs/endpoint_test.go +++ b/docs/endpoint_test.go @@ -12,7 +12,7 @@ func TestEndpointParse(t *testing.T) { {"0.0.0.0:5000", "https://0.0.0.0:5000/v1/"}, } for _, td := range testData { - e, err := newEndpoint(td.str) + e, err := newEndpoint(td.str, insecureRegistries) if err != nil { t.Errorf("%q: %s", td.str, err) } diff --git a/docs/registry_test.go b/docs/registry_test.go index 37dedc2a..3e0950ef 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -326,6 +326,7 @@ func TestIsSecure(t *testing.T) { insecureRegistries []string expected bool }{ + {IndexServerURL.Host, nil, true}, {"example.com", []string{}, true}, {"example.com", []string{"example.com"}, false}, {"localhost", []string{"localhost:5000"}, false}, From ae0ebb9d074e8fe05a0bfc7c057a5d0222df5128 Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Tue, 11 Nov 2014 16:31:15 -0500 Subject: [PATCH 22/26] Add the possibility of specifying a subnet for --insecure-registry Signed-off-by: Tibor Vass --- docs/endpoint.go | 60 +++++++++++++++++++++++++++++--------- docs/registry_mock_test.go | 26 +++++++++++++++++ docs/registry_test.go | 19 ++++++++---- 3 files changed, 85 insertions(+), 20 deletions(-) diff --git a/docs/endpoint.go b/docs/endpoint.go index bd23c302..c485a13d 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -12,6 +12,9 @@ import ( log "github.com/Sirupsen/logrus" ) +// for mocking in unit tests +var lookupIP = net.LookupIP + // scans string for api version in the URL path. returns the trimmed hostname, if version found, string and API version. func scanForAPIVersion(hostname string) (string, APIVersion) { var ( @@ -79,7 +82,10 @@ func newEndpoint(hostname string, insecureRegistries []string) (*Endpoint, error if err != nil { return nil, err } - endpoint.secure = isSecure(endpoint.URL.Host, insecureRegistries) + endpoint.secure, err = isSecure(endpoint.URL.Host, insecureRegistries) + if err != nil { + return nil, err + } return &endpoint, nil } @@ -152,30 +158,56 @@ func (e Endpoint) Ping() (RegistryInfo, error) { // isSecure returns false if the provided hostname is part of the list of insecure registries. // Insecure registries accept HTTP and/or accept HTTPS with certificates from unknown CAs. -func isSecure(hostname string, insecureRegistries []string) bool { +// +// The list of insecure registries can contain an element with CIDR notation to specify a whole subnet. +// If the subnet contains one of the IPs of the registry specified by hostname, the latter is considered +// insecure. +// +// hostname should be a URL.Host (`host:port` or `host`) +func isSecure(hostname string, insecureRegistries []string) (bool, error) { if hostname == IndexServerURL.Host { - return true + return true, nil } host, _, err := net.SplitHostPort(hostname) - if err != nil { + // assume hostname is of the form `host` without the port and go on. host = hostname } - - if host == "127.0.0.1" || host == "localhost" { - return false + addrs, err := lookupIP(host) + if err != nil { + ip := net.ParseIP(host) + if ip == nil { + // if resolving `host` fails, error out, since host is to be net.Dial-ed anyway + return true, fmt.Errorf("issecure: could not resolve %q: %v", host, err) + } + addrs = []net.IP{ip} + } + if len(addrs) == 0 { + return true, fmt.Errorf("issecure: could not resolve %q", host) } - if len(insecureRegistries) == 0 { - return true - } + for _, addr := range addrs { + for _, r := range insecureRegistries { + // hostname matches insecure registry + if hostname == r { + return false, nil + } - for _, h := range insecureRegistries { - if hostname == h { - return false + // now assume a CIDR was passed to --insecure-registry + _, ipnet, err := net.ParseCIDR(r) + if err != nil { + // if could not parse it as a CIDR, even after removing + // assume it's not a CIDR and go on with the next candidate + continue + } + + // check if the addr falls in the subnet + if ipnet.Contains(addr) { + return false, nil + } } } - return true + return true, nil } diff --git a/docs/registry_mock_test.go b/docs/registry_mock_test.go index 1c710e21..887d2ef6 100644 --- a/docs/registry_mock_test.go +++ b/docs/registry_mock_test.go @@ -2,9 +2,11 @@ package registry import ( "encoding/json" + "errors" "fmt" "io" "io/ioutil" + "net" "net/http" "net/http/httptest" "net/url" @@ -80,6 +82,11 @@ var ( "latest": "42d718c941f5c532ac049bf0b0ab53f0062f09a03afd4aa4a02c098e46032b9d", }, } + mockHosts = map[string][]net.IP{ + "": {net.ParseIP("0.0.0.0")}, + "localhost": {net.ParseIP("127.0.0.1"), net.ParseIP("::1")}, + "example.com": {net.ParseIP("42.42.42.42")}, + } ) func init() { @@ -106,6 +113,25 @@ func init() { panic(err) } insecureRegistries = []string{URL.Host} + + // override net.LookupIP + lookupIP = func(host string) ([]net.IP, error) { + if host == "127.0.0.1" { + // I believe in future Go versions this will fail, so let's fix it later + return net.LookupIP(host) + } + for h, addrs := range mockHosts { + if host == h { + return addrs, nil + } + for _, addr := range addrs { + if addr.String() == host { + return []net.IP{addr}, nil + } + } + } + return nil, errors.New("lookup: no such host") + } } func handlerAccessLog(handler http.Handler) http.Handler { diff --git a/docs/registry_test.go b/docs/registry_test.go index 3e0950ef..d24a5f57 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -333,19 +333,26 @@ func TestIsSecure(t *testing.T) { {"localhost:5000", []string{"localhost:5000"}, false}, {"localhost", []string{"example.com"}, false}, {"127.0.0.1:5000", []string{"127.0.0.1:5000"}, false}, - {"localhost", []string{}, false}, - {"localhost:5000", []string{}, false}, - {"127.0.0.1", []string{}, false}, + {"localhost", nil, false}, + {"localhost:5000", nil, false}, + {"127.0.0.1", nil, false}, {"localhost", []string{"example.com"}, false}, {"127.0.0.1", []string{"example.com"}, false}, - {"example.com", []string{}, true}, + {"example.com", nil, true}, {"example.com", []string{"example.com"}, false}, {"127.0.0.1", []string{"example.com"}, false}, {"127.0.0.1:5000", []string{"example.com"}, false}, + {"example.com:5000", []string{"42.42.0.0/16"}, false}, + {"example.com", []string{"42.42.0.0/16"}, false}, + {"example.com:5000", []string{"42.42.42.42/8"}, false}, + {"127.0.0.1:5000", []string{"127.0.0.0/8"}, false}, + {"42.42.42.42:5000", []string{"42.1.1.1/8"}, false}, } for _, tt := range tests { - if sec := isSecure(tt.addr, tt.insecureRegistries); sec != tt.expected { - t.Errorf("isSecure failed for %q %v, expected %v got %v", tt.addr, tt.insecureRegistries, tt.expected, sec) + // TODO: remove this once we remove localhost insecure by default + insecureRegistries := append(tt.insecureRegistries, "127.0.0.0/8") + if sec, err := isSecure(tt.addr, insecureRegistries); err != nil || sec != tt.expected { + t.Fatalf("isSecure failed for %q %v, expected %v got %v. Error: %v", tt.addr, insecureRegistries, tt.expected, sec, err) } } } From df85a0f700b264dafcbfb3fd4f8d3e5d7eb64930 Mon Sep 17 00:00:00 2001 From: Vaidas Jablonskis Date: Sat, 22 Nov 2014 23:21:47 +0000 Subject: [PATCH 23/26] registry: fix ServerAddress setting This ensures that ServerAddress is set, while previously it was getting set after configFile.Configs. Signed-off-by: Vaidas Jablonskis --- docs/auth.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/auth.go b/docs/auth.go index a22d0b88..42760640 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -126,8 +126,8 @@ func LoadConfig(rootPath string) (*ConfigFile, error) { return &configFile, err } authConfig.Auth = "" - configFile.Configs[k] = authConfig authConfig.ServerAddress = k + configFile.Configs[k] = authConfig } } return &configFile, nil From 3911c8b8dc8bcde030a5ad178bf69263b4b42fe8 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Wed, 10 Dec 2014 17:37:31 -0800 Subject: [PATCH 24/26] Prevent loop with var overshadowing Incase of a 3xx redirect the var was being overshowed and ever changed causing an infinite loop. Fixes #9480 Signed-off-by: Michael Crosby --- docs/session.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/session.go b/docs/session.go index ba6df358..2658ec1a 100644 --- a/docs/session.go +++ b/docs/session.go @@ -505,7 +505,7 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate // Redirect if necessary for res.StatusCode >= 300 && res.StatusCode < 400 { log.Debugf("Redirected to %s", res.Header.Get("Location")) - req, err = r.reqFactory.NewRequest("PUT", res.Header.Get("Location"), bytes.NewReader(imgListJSON)) + req, err := r.reqFactory.NewRequest("PUT", res.Header.Get("Location"), bytes.NewReader(imgListJSON)) if err != nil { return nil, err } @@ -515,10 +515,11 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate if validate { req.Header["X-Docker-Endpoints"] = regs } - res, _, err := r.doRequest(req) + redirect, _, err := r.doRequest(req) if err != nil { return nil, err } + res = redirect defer res.Body.Close() } From 6ad54e3df6085f905d742168f676620f80b422ef Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Wed, 10 Dec 2014 18:08:40 -0800 Subject: [PATCH 25/26] Refactor put image function's redirect loop Signed-off-by: Michael Crosby --- docs/session.go | 64 ++++++++++++++++++++++++++----------------------- 1 file changed, 34 insertions(+), 30 deletions(-) diff --git a/docs/session.go b/docs/session.go index 2658ec1a..4b2f5522 100644 --- a/docs/session.go +++ b/docs/session.go @@ -462,7 +462,6 @@ func (r *Session) PushRegistryTag(remote, revision, tag, registry string, token func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate bool, regs []string) (*RepositoryData, error) { cleanImgList := []*ImgData{} - if validate { for _, elem := range imgList { if elem.Checksum != "" { @@ -484,44 +483,28 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate u := fmt.Sprintf("%srepositories/%s/%s", r.indexEndpoint.VersionString(1), remote, suffix) log.Debugf("[registry] PUT %s", u) log.Debugf("Image list pushed to index:\n%s", imgListJSON) - req, err := r.reqFactory.NewRequest("PUT", u, bytes.NewReader(imgListJSON)) - if err != nil { - return nil, err + headers := map[string][]string{ + "Content-type": {"application/json"}, + "X-Docker-Token": {"true"}, } - req.Header.Add("Content-type", "application/json") - req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) - req.ContentLength = int64(len(imgListJSON)) - req.Header.Set("X-Docker-Token", "true") if validate { - req.Header["X-Docker-Endpoints"] = regs + headers["X-Docker-Endpoints"] = regs } - res, _, err := r.doRequest(req) - if err != nil { - return nil, err - } - defer res.Body.Close() - // Redirect if necessary - for res.StatusCode >= 300 && res.StatusCode < 400 { - log.Debugf("Redirected to %s", res.Header.Get("Location")) - req, err := r.reqFactory.NewRequest("PUT", res.Header.Get("Location"), bytes.NewReader(imgListJSON)) - if err != nil { + var res *http.Response + for { + if res, err = r.putImageRequest(u, headers, imgListJSON); err != nil { return nil, err } - req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) - req.ContentLength = int64(len(imgListJSON)) - req.Header.Set("X-Docker-Token", "true") - if validate { - req.Header["X-Docker-Endpoints"] = regs + if !shouldRedirect(res) { + break } - redirect, _, err := r.doRequest(req) - if err != nil { - return nil, err - } - res = redirect - defer res.Body.Close() + res.Body.Close() + u = res.Header.Get("Location") + log.Debugf("Redirected to %s", u) } + defer res.Body.Close() var tokens, endpoints []string if !validate { @@ -564,6 +547,27 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate }, nil } +func (r *Session) putImageRequest(u string, headers map[string][]string, body []byte) (*http.Response, error) { + req, err := r.reqFactory.NewRequest("PUT", u, bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) + req.ContentLength = int64(len(body)) + for k, v := range headers { + req.Header[k] = v + } + response, _, err := r.doRequest(req) + if err != nil { + return nil, err + } + return response, nil +} + +func shouldRedirect(response *http.Response) bool { + return response.StatusCode >= 300 && response.StatusCode < 400 +} + func (r *Session) SearchRepositories(term string) (*SearchResults, error) { log.Debugf("Index server: %s", r.indexEndpoint) u := r.indexEndpoint.VersionString(1) + "search?q=" + url.QueryEscape(term) From fdd4f4f2d14705f172e8ae8f7110662de621cf08 Mon Sep 17 00:00:00 2001 From: unclejack Date: Thu, 27 Nov 2014 23:55:03 +0200 Subject: [PATCH 26/26] validate image ID properly & before load Signed-off-by: Cristian Staretu Conflicts: graph/load.go --- docs/registry.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index e1d22b09..d503a63d 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -23,7 +23,6 @@ var ( ErrInvalidRepositoryName = errors.New("Invalid repository name (ex: \"registry.domain.tld/myrepos\")") ErrDoesNotExist = errors.New("Image does not exist") errLoginRequired = errors.New("Authentication is required.") - validHex = regexp.MustCompile(`^([a-f0-9]{64})$`) validNamespace = regexp.MustCompile(`^([a-z0-9_]{4,30})$`) validRepo = regexp.MustCompile(`^([a-z0-9-_.]+)$`) ) @@ -171,7 +170,8 @@ func validateRepositoryName(repositoryName string) error { namespace = "library" name = nameParts[0] - if validHex.MatchString(name) { + // the repository name must not be a valid image ID + if err := utils.ValidateID(name); err == nil { return fmt.Errorf("Invalid repository name (%s), cannot specify 64-byte hexadecimal strings", name) } } else {