From d95235cc502ea2067d3aaed24b79e4fd578c45ab Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Wed, 4 Dec 2013 15:03:51 +0100 Subject: [PATCH 01/16] Add support for client certificates for registries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This lets you specify custom client TLS certificates and CA root for a specific registry hostname. Docker will then verify the registry against the CA and present the client cert when talking to that registry. This allows the registry to verify that the client has a proper key, indicating that the client is allowed to access the images. A custom cert is configured by creating a directory in /etc/docker/certs.d with the same name as the registry hostname. Inside this directory all *.crt files are added as CA Roots (if none exists, the system default is used) and pair of files .key and .cert indicate a custom certificate to present to the registry. If there are multiple certificates each one will be tried in alphabetical order, proceeding to the next if we get a 403 of 5xx response. So, an example setup would be: /etc/docker/certs.d/ └── localhost ├── client.cert ├── client.key └── localhost.crt A simple way to test this setup is to use an apache server to host a registry. Just copy a registry tree into the apache root, here is an example one containing the busybox image: http://people.gnome.org/~alexl/v1.tar.gz Then add this conf file as /etc/httpd/conf.d/registry.conf: # This must be in the root context, otherwise it causes a re-negotiation # which is not supported by the tls implementation in go SSLVerifyClient optional_no_ca Action cert-protected /cgi-bin/cert.cgi SetHandler cert-protected Header set x-docker-registry-version "0.6.2" SetEnvIf Host (.*) custom_host=$1 Header set X-Docker-Endpoints "%{custom_host}e" And this as /var/www/cgi-bin/cert.cgi #!/bin/bash if [ "$HTTPS" != "on" ]; then echo "Status: 403 Not using SSL" echo "x-docker-registry-version: 0.6.2" echo exit 0 fi if [ "$SSL_CLIENT_VERIFY" == "NONE" ]; then echo "Status: 403 Client certificate invalid" echo "x-docker-registry-version: 0.6.2" echo exit 0 fi echo "Content-length: $(stat --printf='%s' $PATH_TRANSLATED)" echo "x-docker-registry-version: 0.6.2" echo "X-Docker-Endpoints: $SERVER_NAME" echo "X-Docker-Size: 0" echo cat $PATH_TRANSLATED This will return 403 for all accessed to /v1 unless *any* client cert is presented. Obviously a real implementation would verify more details about the certificate. Example client certs can be generated with: openssl genrsa -out client.key 1024 openssl req -new -x509 -text -key client.key -out client.cert Docker-DCO-1.1-Signed-off-by: Alexander Larsson (github: alexlarsson) --- docs/registry.go | 227 ++++++++++++++++++++++++++++++++++++----------- 1 file changed, 174 insertions(+), 53 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 24c55125c..748636dca 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -4,6 +4,8 @@ import ( "bytes" "crypto/sha256" _ "crypto/sha512" + "crypto/tls" + "crypto/x509" "encoding/json" "errors" "fmt" @@ -13,6 +15,8 @@ import ( "net/http" "net/http/cookiejar" "net/url" + "os" + "path" "regexp" "runtime" "strconv" @@ -29,31 +33,155 @@ var ( errLoginRequired = errors.New("Authentication is required.") ) +type TimeoutType uint32 + +const ( + NoTimeout TimeoutType = iota + ReceiveTimeout + ConnectTimeout +) + +func newClient(jar http.CookieJar, roots *x509.CertPool, cert *tls.Certificate, timeout TimeoutType) *http.Client { + tlsConfig := tls.Config{RootCAs: roots} + + if cert != nil { + tlsConfig.Certificates = append(tlsConfig.Certificates, *cert) + } + + httpTransport := &http.Transport{ + DisableKeepAlives: true, + Proxy: http.ProxyFromEnvironment, + TLSClientConfig: &tlsConfig, + } + + switch timeout { + case ConnectTimeout: + httpTransport.Dial = func(proto string, addr string) (net.Conn, error) { + // Set the connect timeout to 5 seconds + conn, err := net.DialTimeout(proto, addr, 5*time.Second) + if err != nil { + return nil, err + } + // Set the recv timeout to 10 seconds + conn.SetDeadline(time.Now().Add(10 * time.Second)) + return conn, nil + } + case ReceiveTimeout: + httpTransport.Dial = func(proto string, addr string) (net.Conn, error) { + conn, err := net.Dial(proto, addr) + if err != nil { + return nil, err + } + conn = utils.NewTimeoutConn(conn, 1*time.Minute) + return conn, nil + } + } + + return &http.Client{ + Transport: httpTransport, + CheckRedirect: AddRequiredHeadersToRedirectedRequests, + Jar: jar, + } +} + +func doRequest(req *http.Request, jar http.CookieJar, timeout TimeoutType) (*http.Response, *http.Client, error) { + hasFile := func(files []os.FileInfo, name string) bool { + for _, f := range files { + if f.Name() == name { + return true + } + } + return false + } + + hostDir := path.Join("/etc/docker/certs.d", req.URL.Host) + fs, err := ioutil.ReadDir(hostDir) + if err != nil && !os.IsNotExist(err) { + return nil, nil, err + } + + var ( + pool *x509.CertPool + certs []*tls.Certificate + ) + + for _, f := range fs { + if strings.HasSuffix(f.Name(), ".crt") { + if pool == nil { + pool = x509.NewCertPool() + } + data, err := ioutil.ReadFile(path.Join(hostDir, f.Name())) + if err != nil { + return nil, nil, err + } else { + pool.AppendCertsFromPEM(data) + } + } + if strings.HasSuffix(f.Name(), ".cert") { + certName := f.Name() + keyName := certName[:len(certName)-5] + ".key" + if !hasFile(fs, keyName) { + return nil, nil, fmt.Errorf("Missing key %s for certificate %s", keyName, certName) + } else { + cert, err := tls.LoadX509KeyPair(path.Join(hostDir, certName), path.Join(hostDir, keyName)) + if err != nil { + return nil, nil, err + } + certs = append(certs, &cert) + } + } + if strings.HasSuffix(f.Name(), ".key") { + keyName := f.Name() + certName := keyName[:len(keyName)-4] + ".cert" + if !hasFile(fs, certName) { + return nil, nil, fmt.Errorf("Missing certificate %s for key %s", certName, keyName) + } + } + } + + if len(certs) == 0 { + client := newClient(jar, pool, nil, timeout) + res, err := client.Do(req) + if err != nil { + return nil, nil, err + } + return res, client, nil + } else { + for i, cert := range certs { + client := newClient(jar, pool, cert, timeout) + res, err := client.Do(req) + if i == len(certs)-1 { + // If this is the last cert, always return the result + return res, client, err + } else { + // Otherwise, continue to next cert if 403 or 5xx + if err == nil && res.StatusCode != 403 && !(res.StatusCode >= 500 && res.StatusCode < 600) { + return res, client, err + } + } + } + } + + return nil, nil, nil +} + func pingRegistryEndpoint(endpoint string) (RegistryInfo, error) { if endpoint == IndexServerAddress() { // Skip the check, we now this one is valid // (and we never want to fallback to http in case of error) return RegistryInfo{Standalone: false}, nil } - httpDial := func(proto string, addr string) (net.Conn, error) { - // Set the connect timeout to 5 seconds - conn, err := net.DialTimeout(proto, addr, 5*time.Second) - if err != nil { - return nil, err - } - // Set the recv timeout to 10 seconds - conn.SetDeadline(time.Now().Add(10 * time.Second)) - return conn, nil - } - httpTransport := &http.Transport{ - Dial: httpDial, - Proxy: http.ProxyFromEnvironment, - } - client := &http.Client{Transport: httpTransport} - resp, err := client.Get(endpoint + "_ping") + + req, err := http.NewRequest("GET", endpoint+"_ping", nil) if err != nil { return RegistryInfo{Standalone: false}, err } + + resp, _, err := doRequest(req, nil, ConnectTimeout) + if err != nil { + return RegistryInfo{Standalone: false}, err + } + defer resp.Body.Close() jsonString, err := ioutil.ReadAll(resp.Body) @@ -171,6 +299,10 @@ func setTokenAuth(req *http.Request, token []string) { } } +func (r *Registry) doRequest(req *http.Request) (*http.Response, *http.Client, error) { + return doRequest(req, r.jar, r.timeout) +} + // Retrieve the history of a given image from the Registry. // Return a list of the parent's json (requested image included) func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]string, error) { @@ -179,7 +311,7 @@ func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]s return nil, err } setTokenAuth(req, token) - res, err := r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { return nil, err } @@ -214,7 +346,7 @@ func (r *Registry) LookupRemoteImage(imgID, registry string, token []string) boo return false } setTokenAuth(req, token) - res, err := r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { utils.Errorf("Error in LookupRemoteImage %s", err) return false @@ -231,7 +363,7 @@ func (r *Registry) GetRemoteImageJSON(imgID, registry string, token []string) ([ return nil, -1, fmt.Errorf("Failed to download json: %s", err) } setTokenAuth(req, token) - res, err := r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { return nil, -1, fmt.Errorf("Failed to download json: %s", err) } @@ -260,6 +392,7 @@ func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string, i var ( retries = 5 headRes *http.Response + client *http.Client hasResume bool = false imageURL = fmt.Sprintf("%simages/%s/layer", registry, imgID) ) @@ -267,9 +400,10 @@ func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string, i if err != nil { return nil, fmt.Errorf("Error while getting from the server: %s\n", err) } + setTokenAuth(headReq, token) for i := 1; i <= retries; i++ { - headRes, err = r.client.Do(headReq) + headRes, client, err = r.doRequest(headReq) if err != nil && i == retries { return nil, fmt.Errorf("Eror while making head request: %s\n", err) } else if err != nil { @@ -290,10 +424,10 @@ func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string, i setTokenAuth(req, token) if hasResume { utils.Debugf("server supports resume") - return utils.ResumableRequestReader(r.client, req, 5, imgSize), nil + return utils.ResumableRequestReader(client, req, 5, imgSize), nil } utils.Debugf("server doesn't support resume") - res, err := r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { return nil, err } @@ -319,7 +453,7 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ return nil, err } setTokenAuth(req, token) - res, err := r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { return nil, err } @@ -380,7 +514,7 @@ func (r *Registry) GetRepositoryData(remote string) (*RepositoryData, error) { } req.Header.Set("X-Docker-Token", "true") - res, err := r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { return nil, err } @@ -448,13 +582,13 @@ func (r *Registry) PushImageChecksumRegistry(imgData *ImgData, registry string, req.Header.Set("X-Docker-Checksum", imgData.Checksum) req.Header.Set("X-Docker-Checksum-Payload", imgData.ChecksumPayload) - res, err := r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { return fmt.Errorf("Failed to upload metadata: %s", err) } defer res.Body.Close() if len(res.Cookies()) > 0 { - r.client.Jar.SetCookies(req.URL, res.Cookies()) + r.jar.SetCookies(req.URL, res.Cookies()) } if res.StatusCode != 200 { errBody, err := ioutil.ReadAll(res.Body) @@ -484,7 +618,7 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis req.Header.Add("Content-type", "application/json") setTokenAuth(req, token) - res, err := r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { return fmt.Errorf("Failed to upload metadata: %s", err) } @@ -525,7 +659,7 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr req.ContentLength = -1 req.TransferEncoding = []string{"chunked"} setTokenAuth(req, token) - res, err := r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { return "", "", fmt.Errorf("Failed to upload layer: %s", err) } @@ -562,7 +696,7 @@ func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token req.Header.Add("Content-type", "application/json") setTokenAuth(req, token) req.ContentLength = int64(len(revision)) - res, err := r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { return err } @@ -610,7 +744,7 @@ func (r *Registry) PushImageJSONIndex(remote string, imgList []*ImgData, validat req.Header["X-Docker-Endpoints"] = regs } - res, err := r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { return nil, err } @@ -629,7 +763,7 @@ func (r *Registry) PushImageJSONIndex(remote string, imgList []*ImgData, validat if validate { req.Header["X-Docker-Endpoints"] = regs } - res, err = r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { return nil, err } @@ -688,7 +822,7 @@ func (r *Registry) SearchRepositories(term string) (*SearchResults, error) { req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) } req.Header.Set("X-Docker-Token", "true") - res, err := r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { return nil, err } @@ -750,10 +884,11 @@ type RegistryInfo struct { } type Registry struct { - client *http.Client authConfig *AuthConfig reqFactory *utils.HTTPRequestFactory indexEndpoint string + jar *cookiejar.Jar + timeout TimeoutType } func trustedLocation(req *http.Request) bool { @@ -791,30 +926,16 @@ func AddRequiredHeadersToRedirectedRequests(req *http.Request, via []*http.Reque } func NewRegistry(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, indexEndpoint string, timeout bool) (r *Registry, err error) { - httpTransport := &http.Transport{ - DisableKeepAlives: true, - Proxy: http.ProxyFromEnvironment, - } - if timeout { - httpTransport.Dial = func(proto string, addr string) (net.Conn, error) { - conn, err := net.Dial(proto, addr) - if err != nil { - return nil, err - } - conn = utils.NewTimeoutConn(conn, 1*time.Minute) - return conn, nil - } - } r = &Registry{ - authConfig: authConfig, - client: &http.Client{ - Transport: httpTransport, - CheckRedirect: AddRequiredHeadersToRedirectedRequests, - }, + authConfig: authConfig, indexEndpoint: indexEndpoint, } - r.client.Jar, err = cookiejar.New(nil) + if timeout { + r.timeout = ReceiveTimeout + } + + r.jar, err = cookiejar.New(nil) if err != nil { return nil, err } From 19b4616baa3e502e3d630e8f7dd7709560457fc5 Mon Sep 17 00:00:00 2001 From: Gabor Nagy Date: Wed, 16 Jul 2014 12:22:13 +0200 Subject: [PATCH 02/16] Add Content-Type header in PushImageLayerRegistry Docker-DCO-1.1-Signed-off-by: Gabor Nagy (github: Aigeruth) --- docs/registry.go | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/registry.go b/docs/registry.go index 24c55125c..c8d000823 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -522,6 +522,7 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr if err != nil { return "", "", err } + req.Header.Add("Content-Type", "application/octet-stream") req.ContentLength = -1 req.TransferEncoding = []string{"chunked"} setTokenAuth(req, token) From 78a499ac67b8f96c6aa1cb6ec4fd781d36f14c18 Mon Sep 17 00:00:00 2001 From: unclejack Date: Fri, 27 Jun 2014 15:10:30 +0300 Subject: [PATCH 03/16] get layer: remove HEAD req & pass down response Docker-DCO-1.1-Signed-off-by: Cristian Staretu (github: unclejack) --- docs/registry.go | 56 ++++++++++++++++++++---------------------------- 1 file changed, 23 insertions(+), 33 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 748636dca..57795f1c3 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -390,52 +390,42 @@ func (r *Registry) GetRemoteImageJSON(imgID, registry string, token []string) ([ func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string, imgSize int64) (io.ReadCloser, error) { var ( - retries = 5 - headRes *http.Response - client *http.Client - hasResume bool = false - imageURL = fmt.Sprintf("%simages/%s/layer", registry, imgID) + retries = 5 + client *http.Client + res *http.Response + imageURL = fmt.Sprintf("%simages/%s/layer", registry, imgID) ) - headReq, err := r.reqFactory.NewRequest("HEAD", imageURL, nil) - if err != nil { - return nil, fmt.Errorf("Error while getting from the server: %s\n", err) - } - - setTokenAuth(headReq, token) - for i := 1; i <= retries; i++ { - headRes, client, err = r.doRequest(headReq) - if err != nil && i == retries { - return nil, fmt.Errorf("Eror while making head request: %s\n", err) - } else if err != nil { - time.Sleep(time.Duration(i) * 5 * time.Second) - continue - } - break - } - - if headRes.Header.Get("Accept-Ranges") == "bytes" && imgSize > 0 { - hasResume = true - } req, err := r.reqFactory.NewRequest("GET", imageURL, nil) if err != nil { return nil, fmt.Errorf("Error while getting from the server: %s\n", err) } setTokenAuth(req, token) - if hasResume { - utils.Debugf("server supports resume") - return utils.ResumableRequestReader(client, req, 5, imgSize), nil - } - utils.Debugf("server doesn't support resume") - res, _, err := r.doRequest(req) - if err != nil { - return nil, err + for i := 1; i <= retries; i++ { + res, client, err = r.doRequest(req) + if err != nil { + res.Body.Close() + if i == retries { + return nil, fmt.Errorf("Server error: Status %d while fetching image layer (%s)", + res.StatusCode, imgID) + } + time.Sleep(time.Duration(i) * 5 * time.Second) + continue + } + break } + if res.StatusCode != 200 { res.Body.Close() return nil, fmt.Errorf("Server error: Status %d while fetching image layer (%s)", res.StatusCode, imgID) } + + if res.Header.Get("Accept-Ranges") == "bytes" && imgSize > 0 { + utils.Debugf("server supports resume") + return utils.ResumableRequestReaderWithInitialResponse(client, req, 5, imgSize, res), nil + } + utils.Debugf("server doesn't support resume") return res.Body, nil } From 6365d94ef4a6f1a38611b9f2f04fc6bc8ad4fa99 Mon Sep 17 00:00:00 2001 From: Olivier Gambier Date: Tue, 22 Jul 2014 01:26:14 +0200 Subject: [PATCH 04/16] Joining registry maintainers Docker-DCO-1.1-Signed-off-by: Olivier Gambier (github: dmp42) --- docs/MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/MAINTAINERS b/docs/MAINTAINERS index af791fb40..6ed4e9d65 100644 --- a/docs/MAINTAINERS +++ b/docs/MAINTAINERS @@ -2,3 +2,4 @@ Sam Alba (@samalba) Joffrey Fuhrer (@shin-) Ken Cochrane (@kencochrane) Vincent Batts (@vbatts) +Olivier Gambier (@dmp42) From 822f8c1b5277c61d5e16777724b03094853f862d Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Thu, 24 Jul 2014 22:19:50 +0000 Subject: [PATCH 05/16] update go import path and libcontainer Docker-DCO-1.1-Signed-off-by: Victor Vieux (github: vieux) --- docs/MAINTAINERS | 6 +++--- docs/auth.go | 4 ++-- docs/registry.go | 4 ++-- docs/registry_mock_test.go | 2 +- docs/registry_test.go | 4 ++-- docs/service.go | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/MAINTAINERS b/docs/MAINTAINERS index 6ed4e9d65..fdb03ed57 100644 --- a/docs/MAINTAINERS +++ b/docs/MAINTAINERS @@ -1,5 +1,5 @@ -Sam Alba (@samalba) -Joffrey Fuhrer (@shin-) -Ken Cochrane (@kencochrane) +Sam Alba (@samalba) +Joffrey Fuhrer (@shin-) +Ken Cochrane (@kencochrane) Vincent Batts (@vbatts) Olivier Gambier (@dmp42) diff --git a/docs/auth.go b/docs/auth.go index 7384efbad..906a37dde 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -11,7 +11,7 @@ import ( "path" "strings" - "github.com/dotcloud/docker/utils" + "github.com/docker/docker/utils" ) // Where we store the config file @@ -20,7 +20,7 @@ const CONFIGFILE = ".dockercfg" // Only used for user auth + account creation const INDEXSERVER = "https://index.docker.io/v1/" -//const INDEXSERVER = "https://indexstaging-docker.dotcloud.com/v1/" +//const INDEXSERVER = "https://registry-stage.hub.docker.com/v1/" var ( ErrConfigFileMissing = errors.New("The Auth config file is missing") diff --git a/docs/registry.go b/docs/registry.go index 974e7fb9f..567cd9f70 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -23,8 +23,8 @@ import ( "strings" "time" - "github.com/dotcloud/docker/dockerversion" - "github.com/dotcloud/docker/utils" + "github.com/docker/docker/dockerversion" + "github.com/docker/docker/utils" ) var ( diff --git a/docs/registry_mock_test.go b/docs/registry_mock_test.go index 6b0075131..1a622228e 100644 --- a/docs/registry_mock_test.go +++ b/docs/registry_mock_test.go @@ -3,7 +3,7 @@ package registry import ( "encoding/json" "fmt" - "github.com/dotcloud/docker/utils" + "github.com/docker/docker/utils" "github.com/gorilla/mux" "io" "io/ioutil" diff --git a/docs/registry_test.go b/docs/registry_test.go index 5cec05950..12dc7a28a 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -7,7 +7,7 @@ import ( "strings" "testing" - "github.com/dotcloud/docker/utils" + "github.com/docker/docker/utils" ) var ( @@ -145,7 +145,7 @@ func TestPushImageLayerRegistry(t *testing.T) { } func TestResolveRepositoryName(t *testing.T) { - _, _, err := ResolveRepositoryName("https://github.com/dotcloud/docker") + _, _, err := ResolveRepositoryName("https://github.com/docker/docker") assertEqual(t, err, ErrInvalidRepositoryName, "Expected error invalid repo name") ep, repo, err := ResolveRepositoryName("fooo/bar") if err != nil { diff --git a/docs/service.go b/docs/service.go index 89a4baa72..d2775e3cd 100644 --- a/docs/service.go +++ b/docs/service.go @@ -1,7 +1,7 @@ package registry import ( - "github.com/dotcloud/docker/engine" + "github.com/docker/docker/engine" ) // Service exposes registry capabilities in the standard Engine From 775ca3caa33ca2976aef1a8e9abf5a9dd25075d7 Mon Sep 17 00:00:00 2001 From: unclejack Date: Mon, 28 Jul 2014 18:01:21 +0300 Subject: [PATCH 06/16] move resumablerequestreader to pkg Docker-DCO-1.1-Signed-off-by: Cristian Staretu (github: unclejack) --- docs/registry.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/registry.go b/docs/registry.go index 567cd9f70..9563c3b28 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -24,6 +24,7 @@ import ( "time" "github.com/docker/docker/dockerversion" + "github.com/docker/docker/pkg/httputils" "github.com/docker/docker/utils" ) @@ -423,7 +424,7 @@ func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string, i if res.Header.Get("Accept-Ranges") == "bytes" && imgSize > 0 { utils.Debugf("server supports resume") - return utils.ResumableRequestReaderWithInitialResponse(client, req, 5, imgSize, res), nil + return httputils.ResumableRequestReaderWithInitialResponse(client, req, 5, imgSize, res), nil } utils.Debugf("server doesn't support resume") return res.Body, nil From 052128c4fc159a7b5f1927a523e7c2826e71fe8f Mon Sep 17 00:00:00 2001 From: Erik Hollensbe Date: Mon, 28 Jul 2014 17:23:38 -0700 Subject: [PATCH 07/16] Move parsing functions to pkg/parsers and the specific kernel handling functions to pkg/parsers/kernel, and parsing filters to pkg/parsers/filter. Adjust imports and package references. Docker-DCO-1.1-Signed-off-by: Erik Hollensbe (github: erikh) --- docs/registry.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/registry.go b/docs/registry.go index 9563c3b28..0d4f2b3cc 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -25,6 +25,7 @@ import ( "github.com/docker/docker/dockerversion" "github.com/docker/docker/pkg/httputils" + "github.com/docker/docker/pkg/parsers/kernel" "github.com/docker/docker/utils" ) @@ -956,7 +957,7 @@ func HTTPRequestFactory(metaHeaders map[string][]string) *utils.HTTPRequestFacto httpVersion = append(httpVersion, &simpleVersionInfo{"docker", dockerversion.VERSION}) httpVersion = append(httpVersion, &simpleVersionInfo{"go", runtime.Version()}) httpVersion = append(httpVersion, &simpleVersionInfo{"git-commit", dockerversion.GITCOMMIT}) - if kernelVersion, err := utils.GetKernelVersion(); err == nil { + if kernelVersion, err := kernel.GetKernelVersion(); err == nil { httpVersion = append(httpVersion, &simpleVersionInfo{"kernel", kernelVersion.String()}) } httpVersion = append(httpVersion, &simpleVersionInfo{"os", runtime.GOOS}) From 7f2dca77d4cb830a321dd2e16d0d01121da29321 Mon Sep 17 00:00:00 2001 From: Erik Hollensbe Date: Wed, 30 Jul 2014 06:42:12 -0700 Subject: [PATCH 08/16] utils/tarsum* -> pkg/tarsum Docker-DCO-1.1-Signed-off-by: Erik Hollensbe (github: erikh) --- docs/registry.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/registry.go b/docs/registry.go index 0d4f2b3cc..106a51814 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -26,6 +26,7 @@ import ( "github.com/docker/docker/dockerversion" "github.com/docker/docker/pkg/httputils" "github.com/docker/docker/pkg/parsers/kernel" + "github.com/docker/docker/pkg/tarsum" "github.com/docker/docker/utils" ) @@ -638,7 +639,7 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgID+"/layer") - tarsumLayer := &utils.TarSum{Reader: layer} + tarsumLayer := &tarsum.TarSum{Reader: layer} h := sha256.New() h.Write(jsonRaw) h.Write([]byte{'\n'}) From 47261aa8cf7aadda67c2d554ee7985f0852d663e Mon Sep 17 00:00:00 2001 From: Erik Hollensbe Date: Wed, 30 Jul 2014 09:28:42 -0700 Subject: [PATCH 09/16] Remove CheckSum from utils; replace with a TeeReader Docker-DCO-1.1-Signed-off-by: Erik Hollensbe (github: erikh) --- docs/registry.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 0d4f2b3cc..9035ce90e 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -6,6 +6,7 @@ import ( _ "crypto/sha512" "crypto/tls" "crypto/x509" + "encoding/hex" "encoding/json" "errors" "fmt" @@ -642,7 +643,7 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr h := sha256.New() h.Write(jsonRaw) h.Write([]byte{'\n'}) - checksumLayer := &utils.CheckSum{Reader: tarsumLayer, Hash: h} + checksumLayer := io.TeeReader(tarsumLayer, h) req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgID+"/layer", checksumLayer) if err != nil { @@ -671,7 +672,7 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr return "", "", utils.NewHTTPRequestError(fmt.Sprintf("Received HTTP code %d while uploading layer: %s", res.StatusCode, errBody), res) } - checksumPayload = "sha256:" + checksumLayer.Sum() + checksumPayload = "sha256:" + hex.EncodeToString(h.Sum(nil)) return tarsumLayer.Sum(jsonRaw), checksumPayload, nil } From d768343cbe275f34be77d71a2c7c22da256d63dd Mon Sep 17 00:00:00 2001 From: Daniel Menet Date: Sat, 9 Aug 2014 09:16:54 +0200 Subject: [PATCH 10/16] Enable `docker search` on private docker registry. The cli interface works similar to other registry related commands: docker search foo ... searches for foo on the official hub docker search localhost:5000/foo ... does the same for the private reg at localhost:5000 Signed-off-by: Daniel Menet --- docs/service.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/service.go b/docs/service.go index d2775e3cd..8c2c0cc7b 100644 --- a/docs/service.go +++ b/docs/service.go @@ -82,7 +82,11 @@ func (s *Service) Search(job *engine.Job) engine.Status { job.GetenvJson("authConfig", authConfig) job.GetenvJson("metaHeaders", metaHeaders) - r, err := NewRegistry(authConfig, HTTPRequestFactory(metaHeaders), IndexServerAddress(), true) + hostname, term, err := ResolveRepositoryName(term) + if err != nil { + return job.Error(err) + } + r, err := NewRegistry(authConfig, HTTPRequestFactory(metaHeaders), hostname, true) if err != nil { return job.Error(err) } From 94c52da6c0934c6b8e0423c2764ac9cd26edda40 Mon Sep 17 00:00:00 2001 From: Daniel Menet Date: Sun, 10 Aug 2014 11:48:34 +0200 Subject: [PATCH 11/16] Expand hostname before passing it to NewRegistry() Signed-off-by: Daniel Menet --- docs/service.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/service.go b/docs/service.go index 8c2c0cc7b..96fc3f48d 100644 --- a/docs/service.go +++ b/docs/service.go @@ -86,6 +86,10 @@ func (s *Service) Search(job *engine.Job) engine.Status { if err != nil { return job.Error(err) } + hostname, err = ExpandAndVerifyRegistryUrl(hostname) + if err != nil { + return job.Error(err) + } r, err := NewRegistry(authConfig, HTTPRequestFactory(metaHeaders), hostname, true) if err != nil { return job.Error(err) From 7ef3a5bc73e68b0638aa5794d52d20416fa00fde Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Thu, 7 Aug 2014 10:43:06 -0400 Subject: [PATCH 12/16] registry.Registry -> registry.Session renaming this struct to more clearly be session, as that is what it handles. Splitting out files for easier readability. Signed-off-by: Vincent Batts --- docs/httpfactory.go | 46 +++ docs/registry.go | 672 ------------------------------------------ docs/registry_test.go | 26 +- docs/service.go | 2 +- docs/session.go | 611 ++++++++++++++++++++++++++++++++++++++ docs/types.go | 33 +++ 6 files changed, 704 insertions(+), 686 deletions(-) create mode 100644 docs/httpfactory.go create mode 100644 docs/session.go create mode 100644 docs/types.go diff --git a/docs/httpfactory.go b/docs/httpfactory.go new file mode 100644 index 000000000..4c7843609 --- /dev/null +++ b/docs/httpfactory.go @@ -0,0 +1,46 @@ +package registry + +import ( + "runtime" + + "github.com/docker/docker/dockerversion" + "github.com/docker/docker/pkg/parsers/kernel" + "github.com/docker/docker/utils" +) + +func HTTPRequestFactory(metaHeaders map[string][]string) *utils.HTTPRequestFactory { + // FIXME: this replicates the 'info' job. + httpVersion := make([]utils.VersionInfo, 0, 4) + httpVersion = append(httpVersion, &simpleVersionInfo{"docker", dockerversion.VERSION}) + httpVersion = append(httpVersion, &simpleVersionInfo{"go", runtime.Version()}) + httpVersion = append(httpVersion, &simpleVersionInfo{"git-commit", dockerversion.GITCOMMIT}) + if kernelVersion, err := kernel.GetKernelVersion(); err == nil { + httpVersion = append(httpVersion, &simpleVersionInfo{"kernel", kernelVersion.String()}) + } + httpVersion = append(httpVersion, &simpleVersionInfo{"os", runtime.GOOS}) + httpVersion = append(httpVersion, &simpleVersionInfo{"arch", runtime.GOARCH}) + ud := utils.NewHTTPUserAgentDecorator(httpVersion...) + md := &utils.HTTPMetaHeadersDecorator{ + Headers: metaHeaders, + } + factory := utils.NewHTTPRequestFactory(ud, md) + return factory +} + +// simpleVersionInfo is a simple implementation of +// the interface VersionInfo, which is used +// to provide version information for some product, +// component, etc. It stores the product name and the version +// in string and returns them on calls to Name() and Version(). +type simpleVersionInfo struct { + name string + version string +} + +func (v *simpleVersionInfo) Name() string { + return v.name +} + +func (v *simpleVersionInfo) Version() string { + return v.version +} diff --git a/docs/registry.go b/docs/registry.go index a590bb5f0..14b8f6d5b 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -1,33 +1,20 @@ package registry import ( - "bytes" - "crypto/sha256" - _ "crypto/sha512" "crypto/tls" "crypto/x509" - "encoding/hex" "encoding/json" "errors" "fmt" - "io" "io/ioutil" "net" "net/http" - "net/http/cookiejar" - "net/url" "os" "path" "regexp" - "runtime" - "strconv" "strings" "time" - "github.com/docker/docker/dockerversion" - "github.com/docker/docker/pkg/httputils" - "github.com/docker/docker/pkg/parsers/kernel" - "github.com/docker/docker/pkg/tarsum" "github.com/docker/docker/utils" ) @@ -297,595 +284,6 @@ func ExpandAndVerifyRegistryUrl(hostname string) (string, error) { return endpoint, nil } -func setTokenAuth(req *http.Request, token []string) { - if req.Header.Get("Authorization") == "" { // Don't override - req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) - } -} - -func (r *Registry) doRequest(req *http.Request) (*http.Response, *http.Client, error) { - return doRequest(req, r.jar, r.timeout) -} - -// Retrieve the history of a given image from the Registry. -// Return a list of the parent's json (requested image included) -func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]string, error) { - req, err := r.reqFactory.NewRequest("GET", registry+"images/"+imgID+"/ancestry", nil) - if err != nil { - return nil, err - } - setTokenAuth(req, token) - res, _, err := r.doRequest(req) - if err != nil { - return nil, err - } - defer res.Body.Close() - if res.StatusCode != 200 { - if res.StatusCode == 401 { - return nil, errLoginRequired - } - return nil, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch remote history for %s", res.StatusCode, imgID), res) - } - - jsonString, err := ioutil.ReadAll(res.Body) - if err != nil { - return nil, fmt.Errorf("Error while reading the http response: %s", err) - } - - utils.Debugf("Ancestry: %s", jsonString) - history := new([]string) - if err := json.Unmarshal(jsonString, history); err != nil { - return nil, err - } - return *history, nil -} - -// Check if an image exists in the Registry -// TODO: This method should return the errors instead of masking them and returning false -func (r *Registry) LookupRemoteImage(imgID, registry string, token []string) bool { - - req, err := r.reqFactory.NewRequest("GET", registry+"images/"+imgID+"/json", nil) - if err != nil { - utils.Errorf("Error in LookupRemoteImage %s", err) - return false - } - setTokenAuth(req, token) - res, _, err := r.doRequest(req) - if err != nil { - utils.Errorf("Error in LookupRemoteImage %s", err) - return false - } - res.Body.Close() - return res.StatusCode == 200 -} - -// Retrieve an image from the Registry. -func (r *Registry) GetRemoteImageJSON(imgID, registry string, token []string) ([]byte, int, error) { - // Get the JSON - req, err := r.reqFactory.NewRequest("GET", registry+"images/"+imgID+"/json", nil) - if err != nil { - return nil, -1, fmt.Errorf("Failed to download json: %s", err) - } - setTokenAuth(req, token) - res, _, err := r.doRequest(req) - if err != nil { - return nil, -1, fmt.Errorf("Failed to download json: %s", err) - } - defer res.Body.Close() - if res.StatusCode != 200 { - return nil, -1, utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d", res.StatusCode), res) - } - - // if the size header is not present, then set it to '-1' - imageSize := -1 - if hdr := res.Header.Get("X-Docker-Size"); hdr != "" { - imageSize, err = strconv.Atoi(hdr) - if err != nil { - return nil, -1, err - } - } - - jsonString, err := ioutil.ReadAll(res.Body) - if err != nil { - return nil, -1, fmt.Errorf("Failed to parse downloaded json: %s (%s)", err, jsonString) - } - return jsonString, imageSize, nil -} - -func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string, imgSize int64) (io.ReadCloser, error) { - var ( - retries = 5 - client *http.Client - res *http.Response - imageURL = fmt.Sprintf("%simages/%s/layer", registry, imgID) - ) - - req, err := r.reqFactory.NewRequest("GET", imageURL, nil) - if err != nil { - return nil, fmt.Errorf("Error while getting from the server: %s\n", err) - } - setTokenAuth(req, token) - for i := 1; i <= retries; i++ { - res, client, err = r.doRequest(req) - if err != nil { - res.Body.Close() - if i == retries { - return nil, fmt.Errorf("Server error: Status %d while fetching image layer (%s)", - res.StatusCode, imgID) - } - time.Sleep(time.Duration(i) * 5 * time.Second) - continue - } - break - } - - if res.StatusCode != 200 { - res.Body.Close() - return nil, fmt.Errorf("Server error: Status %d while fetching image layer (%s)", - res.StatusCode, imgID) - } - - if res.Header.Get("Accept-Ranges") == "bytes" && imgSize > 0 { - utils.Debugf("server supports resume") - return httputils.ResumableRequestReaderWithInitialResponse(client, req, 5, imgSize, res), nil - } - utils.Debugf("server doesn't support resume") - return res.Body, nil -} - -func (r *Registry) GetRemoteTags(registries []string, repository string, token []string) (map[string]string, error) { - if strings.Count(repository, "/") == 0 { - // This will be removed once the Registry supports auto-resolution on - // the "library" namespace - repository = "library/" + repository - } - for _, host := range registries { - endpoint := fmt.Sprintf("%srepositories/%s/tags", host, repository) - req, err := r.reqFactory.NewRequest("GET", endpoint, nil) - - if err != nil { - return nil, err - } - setTokenAuth(req, token) - res, _, err := r.doRequest(req) - if err != nil { - return nil, err - } - - utils.Debugf("Got status code %d from %s", res.StatusCode, endpoint) - defer res.Body.Close() - - if res.StatusCode != 200 && res.StatusCode != 404 { - continue - } else if res.StatusCode == 404 { - return nil, fmt.Errorf("Repository not found") - } - - result := make(map[string]string) - rawJSON, err := ioutil.ReadAll(res.Body) - if err != nil { - return nil, err - } - if err := json.Unmarshal(rawJSON, &result); err != nil { - return nil, err - } - return result, nil - } - return nil, fmt.Errorf("Could not reach any registry endpoint") -} - -func buildEndpointsList(headers []string, indexEp string) ([]string, error) { - var endpoints []string - parsedUrl, err := url.Parse(indexEp) - if err != nil { - return nil, err - } - var urlScheme = parsedUrl.Scheme - // The Registry's URL scheme has to match the Index' - for _, ep := range headers { - epList := strings.Split(ep, ",") - for _, epListElement := range epList { - endpoints = append( - endpoints, - fmt.Sprintf("%s://%s/v1/", urlScheme, strings.TrimSpace(epListElement))) - } - } - return endpoints, nil -} - -func (r *Registry) GetRepositoryData(remote string) (*RepositoryData, error) { - indexEp := r.indexEndpoint - repositoryTarget := fmt.Sprintf("%srepositories/%s/images", indexEp, remote) - - utils.Debugf("[registry] Calling GET %s", repositoryTarget) - - req, err := r.reqFactory.NewRequest("GET", repositoryTarget, nil) - if err != nil { - return nil, err - } - if r.authConfig != nil && len(r.authConfig.Username) > 0 { - req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) - } - req.Header.Set("X-Docker-Token", "true") - - res, _, err := r.doRequest(req) - if err != nil { - return nil, err - } - defer res.Body.Close() - if res.StatusCode == 401 { - return nil, errLoginRequired - } - // TODO: Right now we're ignoring checksums in the response body. - // In the future, we need to use them to check image validity. - if res.StatusCode != 200 { - return nil, utils.NewHTTPRequestError(fmt.Sprintf("HTTP code: %d", res.StatusCode), res) - } - - var tokens []string - if res.Header.Get("X-Docker-Token") != "" { - tokens = res.Header["X-Docker-Token"] - } - - var endpoints []string - if res.Header.Get("X-Docker-Endpoints") != "" { - endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], indexEp) - if err != nil { - return nil, err - } - } else { - // Assume the endpoint is on the same host - u, err := url.Parse(indexEp) - if err != nil { - return nil, err - } - endpoints = append(endpoints, fmt.Sprintf("%s://%s/v1/", u.Scheme, req.URL.Host)) - } - - checksumsJSON, err := ioutil.ReadAll(res.Body) - if err != nil { - return nil, err - } - remoteChecksums := []*ImgData{} - if err := json.Unmarshal(checksumsJSON, &remoteChecksums); err != nil { - return nil, err - } - - // Forge a better object from the retrieved data - imgsData := make(map[string]*ImgData) - for _, elem := range remoteChecksums { - imgsData[elem.ID] = elem - } - - return &RepositoryData{ - ImgList: imgsData, - Endpoints: endpoints, - Tokens: tokens, - }, nil -} - -func (r *Registry) PushImageChecksumRegistry(imgData *ImgData, registry string, token []string) error { - - utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgData.ID+"/checksum") - - req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgData.ID+"/checksum", nil) - if err != nil { - return err - } - setTokenAuth(req, token) - req.Header.Set("X-Docker-Checksum", imgData.Checksum) - req.Header.Set("X-Docker-Checksum-Payload", imgData.ChecksumPayload) - - res, _, err := r.doRequest(req) - if err != nil { - return fmt.Errorf("Failed to upload metadata: %s", err) - } - defer res.Body.Close() - if len(res.Cookies()) > 0 { - r.jar.SetCookies(req.URL, res.Cookies()) - } - if res.StatusCode != 200 { - errBody, err := ioutil.ReadAll(res.Body) - if err != nil { - return fmt.Errorf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err) - } - var jsonBody map[string]string - if err := json.Unmarshal(errBody, &jsonBody); err != nil { - errBody = []byte(err.Error()) - } else if jsonBody["error"] == "Image already exists" { - return ErrAlreadyExists - } - return fmt.Errorf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody) - } - return nil -} - -// Push a local image to the registry -func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, registry string, token []string) error { - - utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgData.ID+"/json") - - req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgData.ID+"/json", bytes.NewReader(jsonRaw)) - if err != nil { - return err - } - req.Header.Add("Content-type", "application/json") - setTokenAuth(req, token) - - res, _, err := r.doRequest(req) - if err != nil { - return fmt.Errorf("Failed to upload metadata: %s", err) - } - defer res.Body.Close() - if res.StatusCode == 401 && strings.HasPrefix(registry, "http://") { - return utils.NewHTTPRequestError("HTTP code 401, Docker will not send auth headers over HTTP.", res) - } - if res.StatusCode != 200 { - errBody, err := ioutil.ReadAll(res.Body) - if err != nil { - return utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err), res) - } - var jsonBody map[string]string - if err := json.Unmarshal(errBody, &jsonBody); err != nil { - errBody = []byte(err.Error()) - } else if jsonBody["error"] == "Image already exists" { - return ErrAlreadyExists - } - return utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody), res) - } - return nil -} - -func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registry string, token []string, jsonRaw []byte) (checksum string, checksumPayload string, err error) { - - utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgID+"/layer") - - tarsumLayer := &tarsum.TarSum{Reader: layer} - h := sha256.New() - h.Write(jsonRaw) - h.Write([]byte{'\n'}) - checksumLayer := io.TeeReader(tarsumLayer, h) - - req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgID+"/layer", checksumLayer) - if err != nil { - return "", "", err - } - req.Header.Add("Content-Type", "application/octet-stream") - req.ContentLength = -1 - req.TransferEncoding = []string{"chunked"} - setTokenAuth(req, token) - res, _, err := r.doRequest(req) - if err != nil { - return "", "", fmt.Errorf("Failed to upload layer: %s", err) - } - if rc, ok := layer.(io.Closer); ok { - if err := rc.Close(); err != nil { - return "", "", err - } - } - defer res.Body.Close() - - if res.StatusCode != 200 { - errBody, err := ioutil.ReadAll(res.Body) - if err != nil { - return "", "", utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err), res) - } - return "", "", utils.NewHTTPRequestError(fmt.Sprintf("Received HTTP code %d while uploading layer: %s", res.StatusCode, errBody), res) - } - - checksumPayload = "sha256:" + hex.EncodeToString(h.Sum(nil)) - return tarsumLayer.Sum(jsonRaw), checksumPayload, nil -} - -// push a tag on the registry. -// Remote has the format '/ -func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token []string) error { - // "jsonify" the string - revision = "\"" + revision + "\"" - path := fmt.Sprintf("repositories/%s/tags/%s", remote, tag) - - req, err := r.reqFactory.NewRequest("PUT", registry+path, strings.NewReader(revision)) - if err != nil { - return err - } - req.Header.Add("Content-type", "application/json") - setTokenAuth(req, token) - req.ContentLength = int64(len(revision)) - res, _, err := r.doRequest(req) - if err != nil { - return err - } - res.Body.Close() - if res.StatusCode != 200 && res.StatusCode != 201 { - return utils.NewHTTPRequestError(fmt.Sprintf("Internal server error: %d trying to push tag %s on %s", res.StatusCode, tag, remote), res) - } - return nil -} - -func (r *Registry) PushImageJSONIndex(remote string, imgList []*ImgData, validate bool, regs []string) (*RepositoryData, error) { - cleanImgList := []*ImgData{} - indexEp := r.indexEndpoint - - if validate { - for _, elem := range imgList { - if elem.Checksum != "" { - cleanImgList = append(cleanImgList, elem) - } - } - } else { - cleanImgList = imgList - } - - imgListJSON, err := json.Marshal(cleanImgList) - if err != nil { - return nil, err - } - var suffix string - if validate { - suffix = "images" - } - u := fmt.Sprintf("%srepositories/%s/%s", indexEp, remote, suffix) - utils.Debugf("[registry] PUT %s", u) - utils.Debugf("Image list pushed to index:\n%s", imgListJSON) - req, err := r.reqFactory.NewRequest("PUT", u, bytes.NewReader(imgListJSON)) - if err != nil { - return nil, err - } - req.Header.Add("Content-type", "application/json") - req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) - req.ContentLength = int64(len(imgListJSON)) - req.Header.Set("X-Docker-Token", "true") - if validate { - req.Header["X-Docker-Endpoints"] = regs - } - - res, _, err := r.doRequest(req) - if err != nil { - return nil, err - } - defer res.Body.Close() - - // Redirect if necessary - for res.StatusCode >= 300 && res.StatusCode < 400 { - utils.Debugf("Redirected to %s", res.Header.Get("Location")) - req, err = r.reqFactory.NewRequest("PUT", res.Header.Get("Location"), bytes.NewReader(imgListJSON)) - if err != nil { - return nil, err - } - req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) - req.ContentLength = int64(len(imgListJSON)) - req.Header.Set("X-Docker-Token", "true") - if validate { - req.Header["X-Docker-Endpoints"] = regs - } - res, _, err := r.doRequest(req) - if err != nil { - return nil, err - } - defer res.Body.Close() - } - - var tokens, endpoints []string - if !validate { - if res.StatusCode != 200 && res.StatusCode != 201 { - errBody, err := ioutil.ReadAll(res.Body) - if err != nil { - return nil, err - } - return nil, utils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push repository %s: %s", res.StatusCode, remote, errBody), res) - } - if res.Header.Get("X-Docker-Token") != "" { - tokens = res.Header["X-Docker-Token"] - utils.Debugf("Auth token: %v", tokens) - } else { - return nil, fmt.Errorf("Index response didn't contain an access token") - } - - if res.Header.Get("X-Docker-Endpoints") != "" { - endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], indexEp) - if err != nil { - return nil, err - } - } else { - return nil, fmt.Errorf("Index response didn't contain any endpoints") - } - } - if validate { - if res.StatusCode != 204 { - errBody, err := ioutil.ReadAll(res.Body) - if err != nil { - return nil, err - } - return nil, utils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push checksums %s: %s", res.StatusCode, remote, errBody), res) - } - } - - return &RepositoryData{ - Tokens: tokens, - Endpoints: endpoints, - }, nil -} - -func (r *Registry) SearchRepositories(term string) (*SearchResults, error) { - utils.Debugf("Index server: %s", r.indexEndpoint) - u := r.indexEndpoint + "search?q=" + url.QueryEscape(term) - req, err := r.reqFactory.NewRequest("GET", u, nil) - if err != nil { - return nil, err - } - if r.authConfig != nil && len(r.authConfig.Username) > 0 { - req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) - } - req.Header.Set("X-Docker-Token", "true") - res, _, err := r.doRequest(req) - if err != nil { - return nil, err - } - defer res.Body.Close() - if res.StatusCode != 200 { - return nil, utils.NewHTTPRequestError(fmt.Sprintf("Unexepected status code %d", res.StatusCode), res) - } - rawData, err := ioutil.ReadAll(res.Body) - if err != nil { - return nil, err - } - result := new(SearchResults) - err = json.Unmarshal(rawData, result) - return result, err -} - -func (r *Registry) GetAuthConfig(withPasswd bool) *AuthConfig { - password := "" - if withPasswd { - password = r.authConfig.Password - } - return &AuthConfig{ - Username: r.authConfig.Username, - Password: password, - Email: r.authConfig.Email, - } -} - -type SearchResult struct { - StarCount int `json:"star_count"` - IsOfficial bool `json:"is_official"` - Name string `json:"name"` - IsTrusted bool `json:"is_trusted"` - Description string `json:"description"` -} - -type SearchResults struct { - Query string `json:"query"` - NumResults int `json:"num_results"` - Results []SearchResult `json:"results"` -} - -type RepositoryData struct { - ImgList map[string]*ImgData - Endpoints []string - Tokens []string -} - -type ImgData struct { - ID string `json:"id"` - Checksum string `json:"checksum,omitempty"` - ChecksumPayload string `json:"-"` - Tag string `json:",omitempty"` -} - -type RegistryInfo struct { - Version string `json:"version"` - Standalone bool `json:"standalone"` -} - -type Registry struct { - authConfig *AuthConfig - reqFactory *utils.HTTPRequestFactory - indexEndpoint string - jar *cookiejar.Jar - timeout TimeoutType -} - func trustedLocation(req *http.Request) bool { var ( trusteds = []string{"docker.com", "docker.io"} @@ -919,73 +317,3 @@ func AddRequiredHeadersToRedirectedRequests(req *http.Request, via []*http.Reque } return nil } - -func NewRegistry(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, indexEndpoint string, timeout bool) (r *Registry, err error) { - r = &Registry{ - authConfig: authConfig, - indexEndpoint: indexEndpoint, - } - - if timeout { - r.timeout = ReceiveTimeout - } - - r.jar, err = cookiejar.New(nil) - if err != nil { - return nil, err - } - - // If we're working with a standalone private registry over HTTPS, send Basic Auth headers - // alongside our requests. - if indexEndpoint != IndexServerAddress() && strings.HasPrefix(indexEndpoint, "https://") { - info, err := pingRegistryEndpoint(indexEndpoint) - if err != nil { - return nil, err - } - if info.Standalone { - utils.Debugf("Endpoint %s is eligible for private registry registry. Enabling decorator.", indexEndpoint) - dec := utils.NewHTTPAuthDecorator(authConfig.Username, authConfig.Password) - factory.AddDecorator(dec) - } - } - - r.reqFactory = factory - return r, nil -} - -func HTTPRequestFactory(metaHeaders map[string][]string) *utils.HTTPRequestFactory { - // FIXME: this replicates the 'info' job. - httpVersion := make([]utils.VersionInfo, 0, 4) - httpVersion = append(httpVersion, &simpleVersionInfo{"docker", dockerversion.VERSION}) - httpVersion = append(httpVersion, &simpleVersionInfo{"go", runtime.Version()}) - httpVersion = append(httpVersion, &simpleVersionInfo{"git-commit", dockerversion.GITCOMMIT}) - if kernelVersion, err := kernel.GetKernelVersion(); err == nil { - httpVersion = append(httpVersion, &simpleVersionInfo{"kernel", kernelVersion.String()}) - } - httpVersion = append(httpVersion, &simpleVersionInfo{"os", runtime.GOOS}) - httpVersion = append(httpVersion, &simpleVersionInfo{"arch", runtime.GOARCH}) - ud := utils.NewHTTPUserAgentDecorator(httpVersion...) - md := &utils.HTTPMetaHeadersDecorator{ - Headers: metaHeaders, - } - factory := utils.NewHTTPRequestFactory(ud, md) - return factory -} - -// simpleVersionInfo is a simple implementation of -// the interface VersionInfo, which is used -// to provide version information for some product, -// component, etc. It stores the product name and the version -// in string and returns them on calls to Name() and Version(). -type simpleVersionInfo struct { - name string - version string -} - -func (v *simpleVersionInfo) Name() string { - return v.name -} - -func (v *simpleVersionInfo) Version() string { - return v.version -} diff --git a/docs/registry_test.go b/docs/registry_test.go index 12dc7a28a..303879e8d 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -16,9 +16,9 @@ var ( REPO = "foo42/bar" ) -func spawnTestRegistry(t *testing.T) *Registry { +func spawnTestRegistrySession(t *testing.T) *Session { authConfig := &AuthConfig{} - r, err := NewRegistry(authConfig, utils.NewHTTPRequestFactory(), makeURL("/v1/"), true) + r, err := NewSession(authConfig, utils.NewHTTPRequestFactory(), makeURL("/v1/"), true) if err != nil { t.Fatal(err) } @@ -34,7 +34,7 @@ func TestPingRegistryEndpoint(t *testing.T) { } func TestGetRemoteHistory(t *testing.T) { - r := spawnTestRegistry(t) + r := spawnTestRegistrySession(t) hist, err := r.GetRemoteHistory(IMAGE_ID, makeURL("/v1/"), TOKEN) if err != nil { t.Fatal(err) @@ -46,7 +46,7 @@ func TestGetRemoteHistory(t *testing.T) { } func TestLookupRemoteImage(t *testing.T) { - r := spawnTestRegistry(t) + r := spawnTestRegistrySession(t) found := r.LookupRemoteImage(IMAGE_ID, makeURL("/v1/"), TOKEN) assertEqual(t, found, true, "Expected remote lookup to succeed") found = r.LookupRemoteImage("abcdef", makeURL("/v1/"), TOKEN) @@ -54,7 +54,7 @@ func TestLookupRemoteImage(t *testing.T) { } func TestGetRemoteImageJSON(t *testing.T) { - r := spawnTestRegistry(t) + r := spawnTestRegistrySession(t) json, size, err := r.GetRemoteImageJSON(IMAGE_ID, makeURL("/v1/"), TOKEN) if err != nil { t.Fatal(err) @@ -71,7 +71,7 @@ func TestGetRemoteImageJSON(t *testing.T) { } func TestGetRemoteImageLayer(t *testing.T) { - r := spawnTestRegistry(t) + r := spawnTestRegistrySession(t) data, err := r.GetRemoteImageLayer(IMAGE_ID, makeURL("/v1/"), TOKEN, 0) if err != nil { t.Fatal(err) @@ -87,7 +87,7 @@ func TestGetRemoteImageLayer(t *testing.T) { } func TestGetRemoteTags(t *testing.T) { - r := spawnTestRegistry(t) + r := spawnTestRegistrySession(t) tags, err := r.GetRemoteTags([]string{makeURL("/v1/")}, REPO, TOKEN) if err != nil { t.Fatal(err) @@ -102,7 +102,7 @@ func TestGetRemoteTags(t *testing.T) { } func TestGetRepositoryData(t *testing.T) { - r := spawnTestRegistry(t) + r := spawnTestRegistrySession(t) parsedUrl, err := url.Parse(makeURL("/v1/")) if err != nil { t.Fatal(err) @@ -123,7 +123,7 @@ func TestGetRepositoryData(t *testing.T) { } func TestPushImageJSONRegistry(t *testing.T) { - r := spawnTestRegistry(t) + r := spawnTestRegistrySession(t) imgData := &ImgData{ ID: "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20", Checksum: "sha256:1ac330d56e05eef6d438586545ceff7550d3bdcb6b19961f12c5ba714ee1bb37", @@ -136,7 +136,7 @@ func TestPushImageJSONRegistry(t *testing.T) { } func TestPushImageLayerRegistry(t *testing.T) { - r := spawnTestRegistry(t) + r := spawnTestRegistrySession(t) layer := strings.NewReader("") _, _, err := r.PushImageLayerRegistry(IMAGE_ID, layer, makeURL("/v1/"), TOKEN, []byte{}) if err != nil { @@ -171,7 +171,7 @@ func TestResolveRepositoryName(t *testing.T) { } func TestPushRegistryTag(t *testing.T) { - r := spawnTestRegistry(t) + r := spawnTestRegistrySession(t) err := r.PushRegistryTag("foo42/bar", IMAGE_ID, "stable", makeURL("/v1/"), TOKEN) if err != nil { t.Fatal(err) @@ -179,7 +179,7 @@ func TestPushRegistryTag(t *testing.T) { } func TestPushImageJSONIndex(t *testing.T) { - r := spawnTestRegistry(t) + r := spawnTestRegistrySession(t) imgData := []*ImgData{ { ID: "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20", @@ -207,7 +207,7 @@ func TestPushImageJSONIndex(t *testing.T) { } func TestSearchRepositories(t *testing.T) { - r := spawnTestRegistry(t) + r := spawnTestRegistrySession(t) results, err := r.SearchRepositories("fakequery") if err != nil { t.Fatal(err) diff --git a/docs/service.go b/docs/service.go index d2775e3cd..29afd1639 100644 --- a/docs/service.go +++ b/docs/service.go @@ -82,7 +82,7 @@ func (s *Service) Search(job *engine.Job) engine.Status { job.GetenvJson("authConfig", authConfig) job.GetenvJson("metaHeaders", metaHeaders) - r, err := NewRegistry(authConfig, HTTPRequestFactory(metaHeaders), IndexServerAddress(), true) + r, err := NewSession(authConfig, HTTPRequestFactory(metaHeaders), IndexServerAddress(), true) if err != nil { return job.Error(err) } diff --git a/docs/session.go b/docs/session.go new file mode 100644 index 000000000..e60fbeb74 --- /dev/null +++ b/docs/session.go @@ -0,0 +1,611 @@ +package registry + +import ( + "bytes" + "crypto/sha256" + _ "crypto/sha512" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/http/cookiejar" + "net/url" + "strconv" + "strings" + "time" + + "github.com/docker/docker/pkg/httputils" + "github.com/docker/docker/pkg/tarsum" + "github.com/docker/docker/utils" +) + +type Session struct { + authConfig *AuthConfig + reqFactory *utils.HTTPRequestFactory + indexEndpoint string + jar *cookiejar.Jar + timeout TimeoutType +} + +func NewSession(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, indexEndpoint string, timeout bool) (r *Session, err error) { + r = &Session{ + authConfig: authConfig, + indexEndpoint: indexEndpoint, + } + + if timeout { + r.timeout = ReceiveTimeout + } + + r.jar, err = cookiejar.New(nil) + if err != nil { + return nil, err + } + + // If we're working with a standalone private registry over HTTPS, send Basic Auth headers + // alongside our requests. + if indexEndpoint != IndexServerAddress() && strings.HasPrefix(indexEndpoint, "https://") { + info, err := pingRegistryEndpoint(indexEndpoint) + if err != nil { + return nil, err + } + if info.Standalone { + utils.Debugf("Endpoint %s is eligible for private registry registry. Enabling decorator.", indexEndpoint) + dec := utils.NewHTTPAuthDecorator(authConfig.Username, authConfig.Password) + factory.AddDecorator(dec) + } + } + + r.reqFactory = factory + return r, nil +} + +func (r *Session) doRequest(req *http.Request) (*http.Response, *http.Client, error) { + return doRequest(req, r.jar, r.timeout) +} + +// Retrieve the history of a given image from the Registry. +// Return a list of the parent's json (requested image included) +func (r *Session) GetRemoteHistory(imgID, registry string, token []string) ([]string, error) { + req, err := r.reqFactory.NewRequest("GET", registry+"images/"+imgID+"/ancestry", nil) + if err != nil { + return nil, err + } + setTokenAuth(req, token) + res, _, err := r.doRequest(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + if res.StatusCode != 200 { + if res.StatusCode == 401 { + return nil, errLoginRequired + } + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch remote history for %s", res.StatusCode, imgID), res) + } + + jsonString, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, fmt.Errorf("Error while reading the http response: %s", err) + } + + utils.Debugf("Ancestry: %s", jsonString) + history := new([]string) + if err := json.Unmarshal(jsonString, history); err != nil { + return nil, err + } + return *history, nil +} + +// Check if an image exists in the Registry +// TODO: This method should return the errors instead of masking them and returning false +func (r *Session) LookupRemoteImage(imgID, registry string, token []string) bool { + + req, err := r.reqFactory.NewRequest("GET", registry+"images/"+imgID+"/json", nil) + if err != nil { + utils.Errorf("Error in LookupRemoteImage %s", err) + return false + } + setTokenAuth(req, token) + res, _, err := r.doRequest(req) + if err != nil { + utils.Errorf("Error in LookupRemoteImage %s", err) + return false + } + res.Body.Close() + return res.StatusCode == 200 +} + +// Retrieve an image from the Registry. +func (r *Session) GetRemoteImageJSON(imgID, registry string, token []string) ([]byte, int, error) { + // Get the JSON + req, err := r.reqFactory.NewRequest("GET", registry+"images/"+imgID+"/json", nil) + if err != nil { + return nil, -1, fmt.Errorf("Failed to download json: %s", err) + } + setTokenAuth(req, token) + res, _, err := r.doRequest(req) + if err != nil { + return nil, -1, fmt.Errorf("Failed to download json: %s", err) + } + defer res.Body.Close() + if res.StatusCode != 200 { + return nil, -1, utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d", res.StatusCode), res) + } + // if the size header is not present, then set it to '-1' + imageSize := -1 + if hdr := res.Header.Get("X-Docker-Size"); hdr != "" { + imageSize, err = strconv.Atoi(hdr) + if err != nil { + return nil, -1, err + } + } + + jsonString, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, -1, fmt.Errorf("Failed to parse downloaded json: %s (%s)", err, jsonString) + } + return jsonString, imageSize, nil +} + +func (r *Session) GetRemoteImageLayer(imgID, registry string, token []string, imgSize int64) (io.ReadCloser, error) { + var ( + retries = 5 + client *http.Client + res *http.Response + imageURL = fmt.Sprintf("%simages/%s/layer", registry, imgID) + ) + + req, err := r.reqFactory.NewRequest("GET", imageURL, nil) + if err != nil { + return nil, fmt.Errorf("Error while getting from the server: %s\n", err) + } + setTokenAuth(req, token) + for i := 1; i <= retries; i++ { + res, client, err = r.doRequest(req) + if err != nil { + res.Body.Close() + if i == retries { + return nil, fmt.Errorf("Server error: Status %d while fetching image layer (%s)", + res.StatusCode, imgID) + } + time.Sleep(time.Duration(i) * 5 * time.Second) + continue + } + break + } + + if res.StatusCode != 200 { + res.Body.Close() + return nil, fmt.Errorf("Server error: Status %d while fetching image layer (%s)", + res.StatusCode, imgID) + } + + if res.Header.Get("Accept-Ranges") == "bytes" && imgSize > 0 { + utils.Debugf("server supports resume") + return httputils.ResumableRequestReaderWithInitialResponse(client, req, 5, imgSize, res), nil + } + utils.Debugf("server doesn't support resume") + return res.Body, nil +} + +func (r *Session) GetRemoteTags(registries []string, repository string, token []string) (map[string]string, error) { + if strings.Count(repository, "/") == 0 { + // This will be removed once the Registry supports auto-resolution on + // the "library" namespace + repository = "library/" + repository + } + for _, host := range registries { + endpoint := fmt.Sprintf("%srepositories/%s/tags", host, repository) + req, err := r.reqFactory.NewRequest("GET", endpoint, nil) + + if err != nil { + return nil, err + } + setTokenAuth(req, token) + res, _, err := r.doRequest(req) + if err != nil { + return nil, err + } + + utils.Debugf("Got status code %d from %s", res.StatusCode, endpoint) + defer res.Body.Close() + + if res.StatusCode != 200 && res.StatusCode != 404 { + continue + } else if res.StatusCode == 404 { + return nil, fmt.Errorf("Repository not found") + } + + result := make(map[string]string) + rawJSON, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + if err := json.Unmarshal(rawJSON, &result); err != nil { + return nil, err + } + return result, nil + } + return nil, fmt.Errorf("Could not reach any registry endpoint") +} + +func buildEndpointsList(headers []string, indexEp string) ([]string, error) { + var endpoints []string + parsedUrl, err := url.Parse(indexEp) + if err != nil { + return nil, err + } + var urlScheme = parsedUrl.Scheme + // The Registry's URL scheme has to match the Index' + for _, ep := range headers { + epList := strings.Split(ep, ",") + for _, epListElement := range epList { + endpoints = append( + endpoints, + fmt.Sprintf("%s://%s/v1/", urlScheme, strings.TrimSpace(epListElement))) + } + } + return endpoints, nil +} + +func (r *Session) GetRepositoryData(remote string) (*RepositoryData, error) { + indexEp := r.indexEndpoint + repositoryTarget := fmt.Sprintf("%srepositories/%s/images", indexEp, remote) + + utils.Debugf("[registry] Calling GET %s", repositoryTarget) + + req, err := r.reqFactory.NewRequest("GET", repositoryTarget, nil) + if err != nil { + return nil, err + } + if r.authConfig != nil && len(r.authConfig.Username) > 0 { + req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) + } + req.Header.Set("X-Docker-Token", "true") + + res, _, err := r.doRequest(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + if res.StatusCode == 401 { + return nil, errLoginRequired + } + // TODO: Right now we're ignoring checksums in the response body. + // In the future, we need to use them to check image validity. + if res.StatusCode != 200 { + return nil, utils.NewHTTPRequestError(fmt.Sprintf("HTTP code: %d", res.StatusCode), res) + } + + var tokens []string + if res.Header.Get("X-Docker-Token") != "" { + tokens = res.Header["X-Docker-Token"] + } + + var endpoints []string + if res.Header.Get("X-Docker-Endpoints") != "" { + endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], indexEp) + if err != nil { + return nil, err + } + } else { + // Assume the endpoint is on the same host + u, err := url.Parse(indexEp) + if err != nil { + return nil, err + } + endpoints = append(endpoints, fmt.Sprintf("%s://%s/v1/", u.Scheme, req.URL.Host)) + } + + checksumsJSON, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + remoteChecksums := []*ImgData{} + if err := json.Unmarshal(checksumsJSON, &remoteChecksums); err != nil { + return nil, err + } + + // Forge a better object from the retrieved data + imgsData := make(map[string]*ImgData) + for _, elem := range remoteChecksums { + imgsData[elem.ID] = elem + } + + return &RepositoryData{ + ImgList: imgsData, + Endpoints: endpoints, + Tokens: tokens, + }, nil +} + +func (r *Session) PushImageChecksumRegistry(imgData *ImgData, registry string, token []string) error { + + utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgData.ID+"/checksum") + + req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgData.ID+"/checksum", nil) + if err != nil { + return err + } + setTokenAuth(req, token) + req.Header.Set("X-Docker-Checksum", imgData.Checksum) + req.Header.Set("X-Docker-Checksum-Payload", imgData.ChecksumPayload) + + res, _, err := r.doRequest(req) + if err != nil { + return fmt.Errorf("Failed to upload metadata: %s", err) + } + defer res.Body.Close() + if len(res.Cookies()) > 0 { + r.jar.SetCookies(req.URL, res.Cookies()) + } + if res.StatusCode != 200 { + errBody, err := ioutil.ReadAll(res.Body) + if err != nil { + return fmt.Errorf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err) + } + var jsonBody map[string]string + if err := json.Unmarshal(errBody, &jsonBody); err != nil { + errBody = []byte(err.Error()) + } else if jsonBody["error"] == "Image already exists" { + return ErrAlreadyExists + } + return fmt.Errorf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody) + } + return nil +} + +// Push a local image to the registry +func (r *Session) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, registry string, token []string) error { + + utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgData.ID+"/json") + + req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgData.ID+"/json", bytes.NewReader(jsonRaw)) + if err != nil { + return err + } + req.Header.Add("Content-type", "application/json") + setTokenAuth(req, token) + + res, _, err := r.doRequest(req) + if err != nil { + return fmt.Errorf("Failed to upload metadata: %s", err) + } + defer res.Body.Close() + if res.StatusCode == 401 && strings.HasPrefix(registry, "http://") { + return utils.NewHTTPRequestError("HTTP code 401, Docker will not send auth headers over HTTP.", res) + } + if res.StatusCode != 200 { + errBody, err := ioutil.ReadAll(res.Body) + if err != nil { + return utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err), res) + } + var jsonBody map[string]string + if err := json.Unmarshal(errBody, &jsonBody); err != nil { + errBody = []byte(err.Error()) + } else if jsonBody["error"] == "Image already exists" { + return ErrAlreadyExists + } + return utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody), res) + } + return nil +} + +func (r *Session) PushImageLayerRegistry(imgID string, layer io.Reader, registry string, token []string, jsonRaw []byte) (checksum string, checksumPayload string, err error) { + + utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgID+"/layer") + + tarsumLayer := &tarsum.TarSum{Reader: layer} + h := sha256.New() + h.Write(jsonRaw) + h.Write([]byte{'\n'}) + checksumLayer := io.TeeReader(tarsumLayer, h) + + req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgID+"/layer", checksumLayer) + if err != nil { + return "", "", err + } + req.Header.Add("Content-Type", "application/octet-stream") + req.ContentLength = -1 + req.TransferEncoding = []string{"chunked"} + setTokenAuth(req, token) + res, _, err := r.doRequest(req) + if err != nil { + return "", "", fmt.Errorf("Failed to upload layer: %s", err) + } + if rc, ok := layer.(io.Closer); ok { + if err := rc.Close(); err != nil { + return "", "", err + } + } + defer res.Body.Close() + + if res.StatusCode != 200 { + errBody, err := ioutil.ReadAll(res.Body) + if err != nil { + return "", "", utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err), res) + } + return "", "", utils.NewHTTPRequestError(fmt.Sprintf("Received HTTP code %d while uploading layer: %s", res.StatusCode, errBody), res) + } + + checksumPayload = "sha256:" + hex.EncodeToString(h.Sum(nil)) + return tarsumLayer.Sum(jsonRaw), checksumPayload, nil +} + +// push a tag on the registry. +// Remote has the format '/ +func (r *Session) PushRegistryTag(remote, revision, tag, registry string, token []string) error { + // "jsonify" the string + revision = "\"" + revision + "\"" + path := fmt.Sprintf("repositories/%s/tags/%s", remote, tag) + + req, err := r.reqFactory.NewRequest("PUT", registry+path, strings.NewReader(revision)) + if err != nil { + return err + } + req.Header.Add("Content-type", "application/json") + setTokenAuth(req, token) + req.ContentLength = int64(len(revision)) + res, _, err := r.doRequest(req) + if err != nil { + return err + } + res.Body.Close() + if res.StatusCode != 200 && res.StatusCode != 201 { + return utils.NewHTTPRequestError(fmt.Sprintf("Internal server error: %d trying to push tag %s on %s", res.StatusCode, tag, remote), res) + } + return nil +} + +func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate bool, regs []string) (*RepositoryData, error) { + cleanImgList := []*ImgData{} + indexEp := r.indexEndpoint + + if validate { + for _, elem := range imgList { + if elem.Checksum != "" { + cleanImgList = append(cleanImgList, elem) + } + } + } else { + cleanImgList = imgList + } + + imgListJSON, err := json.Marshal(cleanImgList) + if err != nil { + return nil, err + } + var suffix string + if validate { + suffix = "images" + } + u := fmt.Sprintf("%srepositories/%s/%s", indexEp, remote, suffix) + utils.Debugf("[registry] PUT %s", u) + utils.Debugf("Image list pushed to index:\n%s", imgListJSON) + req, err := r.reqFactory.NewRequest("PUT", u, bytes.NewReader(imgListJSON)) + if err != nil { + return nil, err + } + req.Header.Add("Content-type", "application/json") + req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) + req.ContentLength = int64(len(imgListJSON)) + req.Header.Set("X-Docker-Token", "true") + if validate { + req.Header["X-Docker-Endpoints"] = regs + } + + res, _, err := r.doRequest(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + + // Redirect if necessary + for res.StatusCode >= 300 && res.StatusCode < 400 { + utils.Debugf("Redirected to %s", res.Header.Get("Location")) + req, err = r.reqFactory.NewRequest("PUT", res.Header.Get("Location"), bytes.NewReader(imgListJSON)) + if err != nil { + return nil, err + } + req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) + req.ContentLength = int64(len(imgListJSON)) + req.Header.Set("X-Docker-Token", "true") + if validate { + req.Header["X-Docker-Endpoints"] = regs + } + res, _, err := r.doRequest(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + } + + var tokens, endpoints []string + if !validate { + if res.StatusCode != 200 && res.StatusCode != 201 { + errBody, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push repository %s: %s", res.StatusCode, remote, errBody), res) + } + if res.Header.Get("X-Docker-Token") != "" { + tokens = res.Header["X-Docker-Token"] + utils.Debugf("Auth token: %v", tokens) + } else { + return nil, fmt.Errorf("Index response didn't contain an access token") + } + + if res.Header.Get("X-Docker-Endpoints") != "" { + endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], indexEp) + if err != nil { + return nil, err + } + } else { + return nil, fmt.Errorf("Index response didn't contain any endpoints") + } + } + if validate { + if res.StatusCode != 204 { + errBody, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push checksums %s: %s", res.StatusCode, remote, errBody), res) + } + } + + return &RepositoryData{ + Tokens: tokens, + Endpoints: endpoints, + }, nil +} + +func (r *Session) SearchRepositories(term string) (*SearchResults, error) { + utils.Debugf("Index server: %s", r.indexEndpoint) + u := r.indexEndpoint + "search?q=" + url.QueryEscape(term) + req, err := r.reqFactory.NewRequest("GET", u, nil) + if err != nil { + return nil, err + } + if r.authConfig != nil && len(r.authConfig.Username) > 0 { + req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) + } + req.Header.Set("X-Docker-Token", "true") + res, _, err := r.doRequest(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + if res.StatusCode != 200 { + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Unexepected status code %d", res.StatusCode), res) + } + rawData, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + result := new(SearchResults) + err = json.Unmarshal(rawData, result) + return result, err +} + +func (r *Session) GetAuthConfig(withPasswd bool) *AuthConfig { + password := "" + if withPasswd { + password = r.authConfig.Password + } + return &AuthConfig{ + Username: r.authConfig.Username, + Password: password, + Email: r.authConfig.Email, + } +} + +func setTokenAuth(req *http.Request, token []string) { + if req.Header.Get("Authorization") == "" { // Don't override + req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + } +} diff --git a/docs/types.go b/docs/types.go new file mode 100644 index 000000000..70d55e42f --- /dev/null +++ b/docs/types.go @@ -0,0 +1,33 @@ +package registry + +type SearchResult struct { + StarCount int `json:"star_count"` + IsOfficial bool `json:"is_official"` + Name string `json:"name"` + IsTrusted bool `json:"is_trusted"` + Description string `json:"description"` +} + +type SearchResults struct { + Query string `json:"query"` + NumResults int `json:"num_results"` + Results []SearchResult `json:"results"` +} + +type RepositoryData struct { + ImgList map[string]*ImgData + Endpoints []string + Tokens []string +} + +type ImgData struct { + ID string `json:"id"` + Checksum string `json:"checksum,omitempty"` + ChecksumPayload string `json:"-"` + Tag string `json:",omitempty"` +} + +type RegistryInfo struct { + Version string `json:"version"` + Standalone bool `json:"standalone"` +} From 2a7cf96c8fbf7c18b96f20fd89a45e1dd6732f6f Mon Sep 17 00:00:00 2001 From: Josiah Kiehl Date: Thu, 24 Jul 2014 13:37:44 -0700 Subject: [PATCH 13/16] Extract log utils into pkg/log Docker-DCO-1.1-Signed-off-by: Josiah Kiehl (github: capoferro) --- docs/registry.go | 13 +++++++------ docs/registry_mock_test.go | 8 +++++--- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/docs/registry.go b/docs/registry.go index 14b8f6d5b..9c76aca9f 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -15,6 +15,7 @@ import ( "strings" "time" + "github.com/docker/docker/pkg/log" "github.com/docker/docker/utils" ) @@ -186,17 +187,17 @@ func pingRegistryEndpoint(endpoint string) (RegistryInfo, error) { Standalone: true, } if err := json.Unmarshal(jsonString, &info); err != nil { - utils.Debugf("Error unmarshalling the _ping RegistryInfo: %s", err) + log.Debugf("Error unmarshalling the _ping RegistryInfo: %s", err) // don't stop here. Just assume sane defaults } if hdr := resp.Header.Get("X-Docker-Registry-Version"); hdr != "" { - utils.Debugf("Registry version header: '%s'", hdr) + log.Debugf("Registry version header: '%s'", hdr) info.Version = hdr } - utils.Debugf("RegistryInfo.Version: %q", info.Version) + log.Debugf("RegistryInfo.Version: %q", info.Version) standalone := resp.Header.Get("X-Docker-Registry-Standalone") - utils.Debugf("Registry standalone header: '%s'", standalone) + log.Debugf("Registry standalone header: '%s'", standalone) // Accepted values are "true" (case-insensitive) and "1". if strings.EqualFold(standalone, "true") || standalone == "1" { info.Standalone = true @@ -204,7 +205,7 @@ func pingRegistryEndpoint(endpoint string) (RegistryInfo, error) { // there is a header set, and it is not "true" or "1", so assume fails info.Standalone = false } - utils.Debugf("RegistryInfo.Standalone: %q", info.Standalone) + log.Debugf("RegistryInfo.Standalone: %q", info.Standalone) return info, nil } @@ -274,7 +275,7 @@ func ExpandAndVerifyRegistryUrl(hostname string) (string, error) { } endpoint := fmt.Sprintf("https://%s/v1/", hostname) if _, err := pingRegistryEndpoint(endpoint); err != nil { - utils.Debugf("Registry %s does not work (%s), falling back to http", endpoint, err) + log.Debugf("Registry %s does not work (%s), falling back to http", endpoint, err) endpoint = fmt.Sprintf("http://%s/v1/", hostname) if _, err = pingRegistryEndpoint(endpoint); err != nil { //TODO: triggering highland build can be done there without "failing" diff --git a/docs/registry_mock_test.go b/docs/registry_mock_test.go index 1a622228e..2b4cd9dea 100644 --- a/docs/registry_mock_test.go +++ b/docs/registry_mock_test.go @@ -3,8 +3,6 @@ package registry import ( "encoding/json" "fmt" - "github.com/docker/docker/utils" - "github.com/gorilla/mux" "io" "io/ioutil" "net/http" @@ -14,6 +12,10 @@ import ( "strings" "testing" "time" + + "github.com/gorilla/mux" + + "github.com/docker/docker/pkg/log" ) var ( @@ -96,7 +98,7 @@ func init() { func handlerAccessLog(handler http.Handler) http.Handler { logHandler := func(w http.ResponseWriter, r *http.Request) { - utils.Debugf("%s \"%s %s\"", r.RemoteAddr, r.Method, r.URL) + log.Debugf("%s \"%s %s\"", r.RemoteAddr, r.Method, r.URL) handler.ServeHTTP(w, r) } return http.HandlerFunc(logHandler) From 94ff3f3e4d08c1da67b54b176ad2df96fcf21fc1 Mon Sep 17 00:00:00 2001 From: Erik Hollensbe Date: Wed, 13 Aug 2014 15:13:21 -0700 Subject: [PATCH 14/16] move utils.Fataler to pkg/log.Fataler Docker-DCO-1.1-Signed-off-by: Erik Hollensbe (github: erikh) --- docs/session.go | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/docs/session.go b/docs/session.go index e60fbeb74..82b931f26 100644 --- a/docs/session.go +++ b/docs/session.go @@ -17,6 +17,7 @@ import ( "time" "github.com/docker/docker/pkg/httputils" + "github.com/docker/docker/pkg/log" "github.com/docker/docker/pkg/tarsum" "github.com/docker/docker/utils" ) @@ -52,7 +53,7 @@ func NewSession(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, index return nil, err } if info.Standalone { - utils.Debugf("Endpoint %s is eligible for private registry registry. Enabling decorator.", indexEndpoint) + log.Debugf("Endpoint %s is eligible for private registry registry. Enabling decorator.", indexEndpoint) dec := utils.NewHTTPAuthDecorator(authConfig.Username, authConfig.Password) factory.AddDecorator(dec) } @@ -91,7 +92,7 @@ func (r *Session) GetRemoteHistory(imgID, registry string, token []string) ([]st return nil, fmt.Errorf("Error while reading the http response: %s", err) } - utils.Debugf("Ancestry: %s", jsonString) + log.Debugf("Ancestry: %s", jsonString) history := new([]string) if err := json.Unmarshal(jsonString, history); err != nil { return nil, err @@ -105,13 +106,13 @@ func (r *Session) LookupRemoteImage(imgID, registry string, token []string) bool req, err := r.reqFactory.NewRequest("GET", registry+"images/"+imgID+"/json", nil) if err != nil { - utils.Errorf("Error in LookupRemoteImage %s", err) + log.Errorf("Error in LookupRemoteImage %s", err) return false } setTokenAuth(req, token) res, _, err := r.doRequest(req) if err != nil { - utils.Errorf("Error in LookupRemoteImage %s", err) + log.Errorf("Error in LookupRemoteImage %s", err) return false } res.Body.Close() @@ -184,10 +185,10 @@ func (r *Session) GetRemoteImageLayer(imgID, registry string, token []string, im } if res.Header.Get("Accept-Ranges") == "bytes" && imgSize > 0 { - utils.Debugf("server supports resume") + log.Debugf("server supports resume") return httputils.ResumableRequestReaderWithInitialResponse(client, req, 5, imgSize, res), nil } - utils.Debugf("server doesn't support resume") + log.Debugf("server doesn't support resume") return res.Body, nil } @@ -210,7 +211,7 @@ func (r *Session) GetRemoteTags(registries []string, repository string, token [] return nil, err } - utils.Debugf("Got status code %d from %s", res.StatusCode, endpoint) + log.Debugf("Got status code %d from %s", res.StatusCode, endpoint) defer res.Body.Close() if res.StatusCode != 200 && res.StatusCode != 404 { @@ -255,7 +256,7 @@ func (r *Session) GetRepositoryData(remote string) (*RepositoryData, error) { indexEp := r.indexEndpoint repositoryTarget := fmt.Sprintf("%srepositories/%s/images", indexEp, remote) - utils.Debugf("[registry] Calling GET %s", repositoryTarget) + log.Debugf("[registry] Calling GET %s", repositoryTarget) req, err := r.reqFactory.NewRequest("GET", repositoryTarget, nil) if err != nil { @@ -324,7 +325,7 @@ func (r *Session) GetRepositoryData(remote string) (*RepositoryData, error) { func (r *Session) PushImageChecksumRegistry(imgData *ImgData, registry string, token []string) error { - utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgData.ID+"/checksum") + log.Debugf("[registry] Calling PUT %s", registry+"images/"+imgData.ID+"/checksum") req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgData.ID+"/checksum", nil) if err != nil { @@ -361,7 +362,7 @@ func (r *Session) PushImageChecksumRegistry(imgData *ImgData, registry string, t // Push a local image to the registry func (r *Session) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, registry string, token []string) error { - utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgData.ID+"/json") + log.Debugf("[registry] Calling PUT %s", registry+"images/"+imgData.ID+"/json") req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgData.ID+"/json", bytes.NewReader(jsonRaw)) if err != nil { @@ -396,7 +397,7 @@ func (r *Session) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regist func (r *Session) PushImageLayerRegistry(imgID string, layer io.Reader, registry string, token []string, jsonRaw []byte) (checksum string, checksumPayload string, err error) { - utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgID+"/layer") + log.Debugf("[registry] Calling PUT %s", registry+"images/"+imgID+"/layer") tarsumLayer := &tarsum.TarSum{Reader: layer} h := sha256.New() @@ -483,8 +484,8 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate suffix = "images" } u := fmt.Sprintf("%srepositories/%s/%s", indexEp, remote, suffix) - utils.Debugf("[registry] PUT %s", u) - utils.Debugf("Image list pushed to index:\n%s", imgListJSON) + log.Debugf("[registry] PUT %s", u) + log.Debugf("Image list pushed to index:\n%s", imgListJSON) req, err := r.reqFactory.NewRequest("PUT", u, bytes.NewReader(imgListJSON)) if err != nil { return nil, err @@ -505,7 +506,7 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate // Redirect if necessary for res.StatusCode >= 300 && res.StatusCode < 400 { - utils.Debugf("Redirected to %s", res.Header.Get("Location")) + log.Debugf("Redirected to %s", res.Header.Get("Location")) req, err = r.reqFactory.NewRequest("PUT", res.Header.Get("Location"), bytes.NewReader(imgListJSON)) if err != nil { return nil, err @@ -534,7 +535,7 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate } if res.Header.Get("X-Docker-Token") != "" { tokens = res.Header["X-Docker-Token"] - utils.Debugf("Auth token: %v", tokens) + log.Debugf("Auth token: %v", tokens) } else { return nil, fmt.Errorf("Index response didn't contain an access token") } @@ -565,7 +566,7 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate } func (r *Session) SearchRepositories(term string) (*SearchResults, error) { - utils.Debugf("Index server: %s", r.indexEndpoint) + log.Debugf("Index server: %s", r.indexEndpoint) u := r.indexEndpoint + "search?q=" + url.QueryEscape(term) req, err := r.reqFactory.NewRequest("GET", u, nil) if err != nil { From 744919be3d5dde9abe48ef242c4134e8b4cd1898 Mon Sep 17 00:00:00 2001 From: Daniel Menet Date: Sat, 9 Aug 2014 09:16:54 +0200 Subject: [PATCH 15/16] Enable `docker search` on private docker registry. The cli interface works similar to other registry related commands: docker search foo ... searches for foo on the official hub docker search localhost:5000/foo ... does the same for the private reg at localhost:5000 Signed-off-by: Daniel Menet --- docs/service.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/service.go b/docs/service.go index 29afd1639..2d493f7e2 100644 --- a/docs/service.go +++ b/docs/service.go @@ -82,6 +82,14 @@ func (s *Service) Search(job *engine.Job) engine.Status { job.GetenvJson("authConfig", authConfig) job.GetenvJson("metaHeaders", metaHeaders) + hostname, term, err := ResolveRepositoryName(term) + if err != nil { + return job.Error(err) + } + hostname, err = ExpandAndVerifyRegistryUrl(hostname) + if err != nil { + return job.Error(err) + } r, err := NewSession(authConfig, HTTPRequestFactory(metaHeaders), IndexServerAddress(), true) if err != nil { return job.Error(err) From 283fba482103906eacfd1068a202c5e97b45f818 Mon Sep 17 00:00:00 2001 From: Daniel Menet Date: Sun, 10 Aug 2014 11:48:34 +0200 Subject: [PATCH 16/16] Expand hostname before passing it to NewRegistry() Signed-off-by: Daniel Menet --- docs/service.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/service.go b/docs/service.go index 2d493f7e2..0e6f1bda9 100644 --- a/docs/service.go +++ b/docs/service.go @@ -90,7 +90,7 @@ func (s *Service) Search(job *engine.Job) engine.Status { if err != nil { return job.Error(err) } - r, err := NewSession(authConfig, HTTPRequestFactory(metaHeaders), IndexServerAddress(), true) + r, err := NewSession(authConfig, HTTPRequestFactory(metaHeaders), hostname, true) if err != nil { return job.Error(err) }