diff --git a/docs/auth.go b/docs/auth.go index 65991d696..7111ede9b 100644 --- a/docs/auth.go +++ b/docs/auth.go @@ -125,7 +125,7 @@ func loginV1(authConfig *cliconfig.AuthConfig, registryEndpoint *Endpoint) (stri return "", fmt.Errorf("Server Error: Server Address not set.") } - loginAgainstOfficialIndex := serverAddress == IndexServerAddress() + loginAgainstOfficialIndex := serverAddress == INDEXSERVER // to avoid sending the server address to the server it should be removed before being marshalled authCopy := *authConfig diff --git a/docs/auth_test.go b/docs/auth_test.go index 71b963a1f..5f54add30 100644 --- a/docs/auth_test.go +++ b/docs/auth_test.go @@ -37,7 +37,7 @@ func setupTempConfigFile() (*cliconfig.ConfigFile, error) { root = filepath.Join(root, cliconfig.CONFIGFILE) configFile := cliconfig.NewConfigFile(root) - for _, registry := range []string{"testIndex", IndexServerAddress()} { + for _, registry := range []string{"testIndex", INDEXSERVER} { configFile.AuthConfigs[registry] = cliconfig.AuthConfig{ Username: "docker-user", Password: "docker-pass", @@ -82,7 +82,7 @@ func TestResolveAuthConfigIndexServer(t *testing.T) { } defer os.RemoveAll(configFile.Filename()) - indexConfig := configFile.AuthConfigs[IndexServerAddress()] + indexConfig := configFile.AuthConfigs[INDEXSERVER] officialIndex := &IndexInfo{ Official: true, @@ -92,10 +92,10 @@ func TestResolveAuthConfigIndexServer(t *testing.T) { } resolved := ResolveAuthConfig(configFile, officialIndex) - assertEqual(t, resolved, indexConfig, "Expected ResolveAuthConfig to return IndexServerAddress()") + assertEqual(t, resolved, indexConfig, "Expected ResolveAuthConfig to return INDEXSERVER") resolved = ResolveAuthConfig(configFile, privateIndex) - assertNotEqual(t, resolved, indexConfig, "Expected ResolveAuthConfig to not return IndexServerAddress()") + assertNotEqual(t, resolved, indexConfig, "Expected ResolveAuthConfig to not return INDEXSERVER") } func TestResolveAuthConfigFullURL(t *testing.T) { @@ -120,7 +120,7 @@ func TestResolveAuthConfigFullURL(t *testing.T) { Password: "baz-pass", Email: "baz@example.com", } - configFile.AuthConfigs[IndexServerAddress()] = officialAuth + configFile.AuthConfigs[INDEXSERVER] = officialAuth expectedAuths := map[string]cliconfig.AuthConfig{ "registry.example.com": registryAuth, diff --git a/docs/config.go b/docs/config.go index 105ec61d6..333f1c46c 100644 --- a/docs/config.go +++ b/docs/config.go @@ -21,9 +21,16 @@ type Options struct { } const ( + DEFAULT_NAMESPACE = "docker.io" + DEFAULT_V2_REGISTRY = "https://registry-1.docker.io" + DEFAULT_REGISTRY_VERSION_HEADER = "Docker-Distribution-Api-Version" + DEFAULT_V1_REGISTRY = "https://index.docker.io" + + CERTS_DIR = "/etc/docker/certs.d" + // Only used for user auth + account creation - INDEXSERVER = "https://index.docker.io/v1/" - REGISTRYSERVER = "https://registry-1.docker.io/v2/" + REGISTRYSERVER = DEFAULT_V2_REGISTRY + INDEXSERVER = DEFAULT_V1_REGISTRY + "/v1/" INDEXNAME = "docker.io" // INDEXSERVER = "https://registry-stage.hub.docker.com/v1/" @@ -34,14 +41,6 @@ var ( emptyServiceConfig = NewServiceConfig(nil) ) -func IndexServerAddress() string { - return INDEXSERVER -} - -func IndexServerName() string { - return INDEXNAME -} - // InstallFlags adds command-line options to the top-level flag parser for // the current process. func (options *Options) InstallFlags() { @@ -72,6 +71,7 @@ func (ipnet *netIPNet) UnmarshalJSON(b []byte) (err error) { type ServiceConfig struct { InsecureRegistryCIDRs []*netIPNet `json:"InsecureRegistryCIDRs"` IndexConfigs map[string]*IndexInfo `json:"IndexConfigs"` + Mirrors []string } // NewServiceConfig returns a new instance of ServiceConfig @@ -93,6 +93,9 @@ func NewServiceConfig(options *Options) *ServiceConfig { config := &ServiceConfig{ InsecureRegistryCIDRs: make([]*netIPNet, 0), IndexConfigs: make(map[string]*IndexInfo, 0), + // Hack: Bypass setting the mirrors to IndexConfigs since they are going away + // and Mirrors are only for the official registry anyways. + Mirrors: options.Mirrors.GetAll(), } // Split --insecure-registry into CIDR and registry-specific settings. for _, r := range options.InsecureRegistries.GetAll() { @@ -113,9 +116,9 @@ func NewServiceConfig(options *Options) *ServiceConfig { } // Configure public registry. - config.IndexConfigs[IndexServerName()] = &IndexInfo{ - Name: IndexServerName(), - Mirrors: options.Mirrors.GetAll(), + config.IndexConfigs[INDEXNAME] = &IndexInfo{ + Name: INDEXNAME, + Mirrors: config.Mirrors, Secure: true, Official: true, } @@ -193,8 +196,8 @@ func ValidateMirror(val string) (string, error) { // ValidateIndexName validates an index name. func ValidateIndexName(val string) (string, error) { // 'index.docker.io' => 'docker.io' - if val == "index."+IndexServerName() { - val = IndexServerName() + if val == "index."+INDEXNAME { + val = INDEXNAME } if strings.HasPrefix(val, "-") || strings.HasSuffix(val, "-") { return "", fmt.Errorf("Invalid index name (%s). Cannot begin or end with a hyphen.", val) @@ -264,7 +267,7 @@ func (config *ServiceConfig) NewIndexInfo(indexName string) (*IndexInfo, error) // index as the AuthConfig key, and uses the (host)name[:port] for private indexes. func (index *IndexInfo) GetAuthConfigKey() string { if index.Official { - return IndexServerAddress() + return INDEXSERVER } return index.Name } @@ -277,7 +280,7 @@ func splitReposName(reposName string) (string, string) { !strings.Contains(nameParts[0], ":") && nameParts[0] != "localhost") { // This is a Docker Index repos (ex: samalba/hipache or ubuntu) // 'docker.io' - indexName = IndexServerName() + indexName = INDEXNAME remoteName = reposName } else { indexName = nameParts[0] diff --git a/docs/endpoint.go b/docs/endpoint.go index ce92668f4..17443543e 100644 --- a/docs/endpoint.go +++ b/docs/endpoint.go @@ -1,6 +1,7 @@ package registry import ( + "crypto/tls" "encoding/json" "fmt" "io/ioutil" @@ -11,7 +12,8 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/distribution/registry/api/v2" - "github.com/docker/docker/pkg/transport" + "github.com/docker/distribution/registry/client/transport" + "github.com/docker/docker/pkg/tlsconfig" ) // for mocking in unit tests @@ -44,7 +46,9 @@ func scanForAPIVersion(address string) (string, APIVersion) { // NewEndpoint parses the given address to return a registry endpoint. func NewEndpoint(index *IndexInfo, metaHeaders http.Header) (*Endpoint, error) { // *TODO: Allow per-registry configuration of endpoints. - endpoint, err := newEndpoint(index.GetAuthConfigKey(), index.Secure, metaHeaders) + tlsConfig := tlsconfig.ServerDefault + tlsConfig.InsecureSkipVerify = !index.Secure + endpoint, err := newEndpoint(index.GetAuthConfigKey(), &tlsConfig, metaHeaders) if err != nil { return nil, err } @@ -82,7 +86,7 @@ func validateEndpoint(endpoint *Endpoint) error { return nil } -func newEndpoint(address string, secure bool, metaHeaders http.Header) (*Endpoint, error) { +func newEndpoint(address string, tlsConfig *tls.Config, metaHeaders http.Header) (*Endpoint, error) { var ( endpoint = new(Endpoint) trimmedAddress string @@ -93,13 +97,16 @@ func newEndpoint(address string, secure bool, metaHeaders http.Header) (*Endpoin address = "https://" + address } + endpoint.IsSecure = (tlsConfig == nil || !tlsConfig.InsecureSkipVerify) + trimmedAddress, endpoint.Version = scanForAPIVersion(address) if endpoint.URL, err = url.Parse(trimmedAddress); err != nil { return nil, err } - endpoint.IsSecure = secure - tr := NewTransport(ConnectTimeout, endpoint.IsSecure) + + // TODO(tiborvass): make sure a ConnectTimeout transport is used + tr := NewTransport(tlsConfig) endpoint.client = HTTPClient(transport.NewTransport(tr, DockerHeaders(metaHeaders)...)) return endpoint, nil } @@ -166,7 +173,7 @@ func (e *Endpoint) Ping() (RegistryInfo, error) { func (e *Endpoint) pingV1() (RegistryInfo, error) { logrus.Debugf("attempting v1 ping for registry endpoint %s", e) - if e.String() == IndexServerAddress() { + if e.String() == INDEXSERVER { // Skip the check, we know this one is valid // (and we never want to fallback to http in case of error) return RegistryInfo{Standalone: false}, nil diff --git a/docs/endpoint_test.go b/docs/endpoint_test.go index 6f67867bb..a04f9a036 100644 --- a/docs/endpoint_test.go +++ b/docs/endpoint_test.go @@ -12,14 +12,14 @@ func TestEndpointParse(t *testing.T) { str string expected string }{ - {IndexServerAddress(), IndexServerAddress()}, + {INDEXSERVER, INDEXSERVER}, {"http://0.0.0.0:5000/v1/", "http://0.0.0.0:5000/v1/"}, {"http://0.0.0.0:5000/v2/", "http://0.0.0.0:5000/v2/"}, {"http://0.0.0.0:5000", "http://0.0.0.0:5000/v0/"}, {"0.0.0.0:5000", "https://0.0.0.0:5000/v0/"}, } for _, td := range testData { - e, err := newEndpoint(td.str, false, nil) + e, err := newEndpoint(td.str, nil, nil) if err != nil { t.Errorf("%q: %s", td.str, err) } @@ -60,7 +60,7 @@ func TestValidateEndpointAmbiguousAPIVersion(t *testing.T) { testEndpoint := Endpoint{ URL: testServerURL, Version: APIVersionUnknown, - client: HTTPClient(NewTransport(ConnectTimeout, false)), + client: HTTPClient(NewTransport(nil)), } if err = validateEndpoint(&testEndpoint); err != nil { diff --git a/docs/registry.go b/docs/registry.go index fb08e5bdf..74a0ad5f1 100644 --- a/docs/registry.go +++ b/docs/registry.go @@ -2,26 +2,21 @@ package registry import ( "crypto/tls" - "crypto/x509" "errors" - "fmt" - "io/ioutil" "net" "net/http" "os" - "path" - "path/filepath" "runtime" "strings" - "sync" "time" "github.com/Sirupsen/logrus" + "github.com/docker/distribution/registry/api/errcode" + "github.com/docker/distribution/registry/api/v2" + "github.com/docker/distribution/registry/client/transport" "github.com/docker/docker/autogen/dockerversion" "github.com/docker/docker/pkg/parsers/kernel" - "github.com/docker/docker/pkg/timeoutconn" "github.com/docker/docker/pkg/tlsconfig" - "github.com/docker/docker/pkg/transport" "github.com/docker/docker/pkg/useragent" ) @@ -57,135 +52,13 @@ func init() { dockerUserAgent = useragent.AppendVersions("", httpVersion...) } -type httpsRequestModifier struct { - mu sync.Mutex - tlsConfig *tls.Config -} - -// DRAGONS(tiborvass): If someone wonders why do we set tlsconfig in a roundtrip, -// it's because it's so as to match the current behavior in master: we generate the -// certpool on every-goddam-request. It's not great, but it allows people to just put -// the certs in /etc/docker/certs.d/.../ and let docker "pick it up" immediately. Would -// prefer an fsnotify implementation, but that was out of scope of my refactoring. -func (m *httpsRequestModifier) ModifyRequest(req *http.Request) error { - var ( - roots *x509.CertPool - certs []tls.Certificate - hostDir string - ) - - if req.URL.Scheme == "https" { - hasFile := func(files []os.FileInfo, name string) bool { - for _, f := range files { - if f.Name() == name { - return true - } - } - return false - } - - if runtime.GOOS == "windows" { - hostDir = path.Join(os.TempDir(), "/docker/certs.d", req.URL.Host) - } else { - hostDir = path.Join("/etc/docker/certs.d", req.URL.Host) - } - logrus.Debugf("hostDir: %s", hostDir) - fs, err := ioutil.ReadDir(hostDir) - if err != nil && !os.IsNotExist(err) { - return nil - } - - for _, f := range fs { - if strings.HasSuffix(f.Name(), ".crt") { - if roots == nil { - roots = x509.NewCertPool() - } - logrus.Debugf("crt: %s", hostDir+"/"+f.Name()) - data, err := ioutil.ReadFile(filepath.Join(hostDir, f.Name())) - if err != nil { - return err - } - roots.AppendCertsFromPEM(data) - } - if strings.HasSuffix(f.Name(), ".cert") { - certName := f.Name() - keyName := certName[:len(certName)-5] + ".key" - logrus.Debugf("cert: %s", hostDir+"/"+f.Name()) - if !hasFile(fs, keyName) { - return fmt.Errorf("Missing key %s for certificate %s", keyName, certName) - } - cert, err := tls.LoadX509KeyPair(filepath.Join(hostDir, certName), path.Join(hostDir, keyName)) - if err != nil { - return err - } - certs = append(certs, cert) - } - if strings.HasSuffix(f.Name(), ".key") { - keyName := f.Name() - certName := keyName[:len(keyName)-4] + ".cert" - logrus.Debugf("key: %s", hostDir+"/"+f.Name()) - if !hasFile(fs, certName) { - return fmt.Errorf("Missing certificate %s for key %s", certName, keyName) - } - } - } - m.mu.Lock() - m.tlsConfig.RootCAs = roots - m.tlsConfig.Certificates = certs - m.mu.Unlock() - } - return nil -} - -func NewTransport(timeout TimeoutType, secure bool) http.RoundTripper { - tlsConfig := &tls.Config{ - // Avoid fallback to SSL protocols < TLS1.0 - MinVersion: tls.VersionTLS10, - InsecureSkipVerify: !secure, - CipherSuites: tlsconfig.DefaultServerAcceptedCiphers, - } - - tr := &http.Transport{ - DisableKeepAlives: true, - Proxy: http.ProxyFromEnvironment, - TLSClientConfig: tlsConfig, - } - - switch timeout { - case ConnectTimeout: - tr.Dial = func(proto string, addr string) (net.Conn, error) { - // Set the connect timeout to 30 seconds to allow for slower connection - // times... - d := net.Dialer{Timeout: 30 * time.Second, DualStack: true} - - conn, err := d.Dial(proto, addr) - if err != nil { - return nil, err - } - // Set the recv timeout to 10 seconds - conn.SetDeadline(time.Now().Add(10 * time.Second)) - return conn, nil - } - case ReceiveTimeout: - tr.Dial = func(proto string, addr string) (net.Conn, error) { - d := net.Dialer{DualStack: true} - - conn, err := d.Dial(proto, addr) - if err != nil { - return nil, err - } - conn = timeoutconn.New(conn, 1*time.Minute) - return conn, nil +func hasFile(files []os.FileInfo, name string) bool { + for _, f := range files { + if f.Name() == name { + return true } } - - if secure { - // note: httpsTransport also handles http transport - // but for HTTPS, it sets up the certs - return transport.NewTransport(tr, &httpsRequestModifier{tlsConfig: tlsConfig}) - } - - return tr + return false } // DockerHeaders returns request modifiers that ensure requests have @@ -202,10 +75,6 @@ func DockerHeaders(metaHeaders http.Header) []transport.RequestModifier { } func HTTPClient(transport http.RoundTripper) *http.Client { - if transport == nil { - transport = NewTransport(ConnectTimeout, true) - } - return &http.Client{ Transport: transport, CheckRedirect: AddRequiredHeadersToRedirectedRequests, @@ -245,3 +114,52 @@ func AddRequiredHeadersToRedirectedRequests(req *http.Request, via []*http.Reque } return nil } + +func shouldV2Fallback(err errcode.Error) bool { + logrus.Debugf("v2 error: %T %v", err, err) + switch err.Code { + case v2.ErrorCodeUnauthorized, v2.ErrorCodeManifestUnknown: + return true + } + return false +} + +type ErrNoSupport struct{ Err error } + +func (e ErrNoSupport) Error() string { + if e.Err == nil { + return "not supported" + } + return e.Err.Error() +} + +func ContinueOnError(err error) bool { + switch v := err.(type) { + case errcode.Errors: + return ContinueOnError(v[0]) + case ErrNoSupport: + return ContinueOnError(v.Err) + case errcode.Error: + return shouldV2Fallback(v) + } + return false +} + +func NewTransport(tlsConfig *tls.Config) *http.Transport { + if tlsConfig == nil { + var cfg = tlsconfig.ServerDefault + tlsConfig = &cfg + } + return &http.Transport{ + Proxy: http.ProxyFromEnvironment, + Dial: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + }).Dial, + TLSHandshakeTimeout: 10 * time.Second, + TLSClientConfig: tlsConfig, + // TODO(dmcgowan): Call close idle connections when complete and use keep alive + DisableKeepAlives: true, + } +} diff --git a/docs/registry_mock_test.go b/docs/registry_mock_test.go index eab87d463..9217956ce 100644 --- a/docs/registry_mock_test.go +++ b/docs/registry_mock_test.go @@ -165,7 +165,7 @@ func makeHttpsIndex(req string) *IndexInfo { func makePublicIndex() *IndexInfo { index := &IndexInfo{ - Name: IndexServerAddress(), + Name: INDEXSERVER, Secure: true, Official: true, } diff --git a/docs/registry_test.go b/docs/registry_test.go index 52a2dc30f..4d17a62cb 100644 --- a/docs/registry_test.go +++ b/docs/registry_test.go @@ -8,8 +8,8 @@ import ( "strings" "testing" + "github.com/docker/distribution/registry/client/transport" "github.com/docker/docker/cliconfig" - "github.com/docker/docker/pkg/transport" ) var ( @@ -27,7 +27,7 @@ func spawnTestRegistrySession(t *testing.T) *Session { if err != nil { t.Fatal(err) } - var tr http.RoundTripper = debugTransport{NewTransport(ReceiveTimeout, endpoint.IsSecure), t.Log} + var tr http.RoundTripper = debugTransport{NewTransport(nil), t.Log} tr = transport.NewTransport(AuthTransport(tr, authConfig, false), DockerHeaders(nil)...) client := HTTPClient(tr) r, err := NewSession(client, authConfig, endpoint) @@ -332,7 +332,7 @@ func TestParseRepositoryInfo(t *testing.T) { expectedRepoInfos := map[string]RepositoryInfo{ "fooo/bar": { Index: &IndexInfo{ - Name: IndexServerName(), + Name: INDEXNAME, Official: true, }, RemoteName: "fooo/bar", @@ -342,7 +342,7 @@ func TestParseRepositoryInfo(t *testing.T) { }, "library/ubuntu": { Index: &IndexInfo{ - Name: IndexServerName(), + Name: INDEXNAME, Official: true, }, RemoteName: "library/ubuntu", @@ -352,7 +352,7 @@ func TestParseRepositoryInfo(t *testing.T) { }, "nonlibrary/ubuntu": { Index: &IndexInfo{ - Name: IndexServerName(), + Name: INDEXNAME, Official: true, }, RemoteName: "nonlibrary/ubuntu", @@ -362,7 +362,7 @@ func TestParseRepositoryInfo(t *testing.T) { }, "ubuntu": { Index: &IndexInfo{ - Name: IndexServerName(), + Name: INDEXNAME, Official: true, }, RemoteName: "library/ubuntu", @@ -372,7 +372,7 @@ func TestParseRepositoryInfo(t *testing.T) { }, "other/library": { Index: &IndexInfo{ - Name: IndexServerName(), + Name: INDEXNAME, Official: true, }, RemoteName: "other/library", @@ -480,9 +480,9 @@ func TestParseRepositoryInfo(t *testing.T) { CanonicalName: "localhost/privatebase", Official: false, }, - IndexServerName() + "/public/moonbase": { + INDEXNAME + "/public/moonbase": { Index: &IndexInfo{ - Name: IndexServerName(), + Name: INDEXNAME, Official: true, }, RemoteName: "public/moonbase", @@ -490,19 +490,9 @@ func TestParseRepositoryInfo(t *testing.T) { CanonicalName: "docker.io/public/moonbase", Official: false, }, - "index." + IndexServerName() + "/public/moonbase": { + "index." + INDEXNAME + "/public/moonbase": { Index: &IndexInfo{ - Name: IndexServerName(), - Official: true, - }, - RemoteName: "public/moonbase", - LocalName: "public/moonbase", - CanonicalName: "docker.io/public/moonbase", - Official: false, - }, - IndexServerName() + "/public/moonbase": { - Index: &IndexInfo{ - Name: IndexServerName(), + Name: INDEXNAME, Official: true, }, RemoteName: "public/moonbase", @@ -512,7 +502,7 @@ func TestParseRepositoryInfo(t *testing.T) { }, "ubuntu-12.04-base": { Index: &IndexInfo{ - Name: IndexServerName(), + Name: INDEXNAME, Official: true, }, RemoteName: "library/ubuntu-12.04-base", @@ -520,9 +510,9 @@ func TestParseRepositoryInfo(t *testing.T) { CanonicalName: "docker.io/library/ubuntu-12.04-base", Official: true, }, - IndexServerName() + "/ubuntu-12.04-base": { + INDEXNAME + "/ubuntu-12.04-base": { Index: &IndexInfo{ - Name: IndexServerName(), + Name: INDEXNAME, Official: true, }, RemoteName: "library/ubuntu-12.04-base", @@ -530,19 +520,9 @@ func TestParseRepositoryInfo(t *testing.T) { CanonicalName: "docker.io/library/ubuntu-12.04-base", Official: true, }, - IndexServerName() + "/ubuntu-12.04-base": { + "index." + INDEXNAME + "/ubuntu-12.04-base": { Index: &IndexInfo{ - Name: IndexServerName(), - Official: true, - }, - RemoteName: "library/ubuntu-12.04-base", - LocalName: "ubuntu-12.04-base", - CanonicalName: "docker.io/library/ubuntu-12.04-base", - Official: true, - }, - "index." + IndexServerName() + "/ubuntu-12.04-base": { - Index: &IndexInfo{ - Name: IndexServerName(), + Name: INDEXNAME, Official: true, }, RemoteName: "library/ubuntu-12.04-base", @@ -585,14 +565,14 @@ func TestNewIndexInfo(t *testing.T) { config := NewServiceConfig(nil) noMirrors := make([]string, 0) expectedIndexInfos := map[string]*IndexInfo{ - IndexServerName(): { - Name: IndexServerName(), + INDEXNAME: { + Name: INDEXNAME, Official: true, Secure: true, Mirrors: noMirrors, }, - "index." + IndexServerName(): { - Name: IndexServerName(), + "index." + INDEXNAME: { + Name: INDEXNAME, Official: true, Secure: true, Mirrors: noMirrors, @@ -616,14 +596,14 @@ func TestNewIndexInfo(t *testing.T) { config = makeServiceConfig(publicMirrors, []string{"example.com"}) expectedIndexInfos = map[string]*IndexInfo{ - IndexServerName(): { - Name: IndexServerName(), + INDEXNAME: { + Name: INDEXNAME, Official: true, Secure: true, Mirrors: publicMirrors, }, - "index." + IndexServerName(): { - Name: IndexServerName(), + "index." + INDEXNAME: { + Name: INDEXNAME, Official: true, Secure: true, Mirrors: publicMirrors, @@ -880,7 +860,7 @@ func TestIsSecureIndex(t *testing.T) { insecureRegistries []string expected bool }{ - {IndexServerName(), nil, true}, + {INDEXNAME, nil, true}, {"example.com", []string{}, true}, {"example.com", []string{"example.com"}, false}, {"localhost", []string{"localhost:5000"}, false}, diff --git a/docs/service.go b/docs/service.go index 681174927..8dda537a9 100644 --- a/docs/service.go +++ b/docs/service.go @@ -1,9 +1,19 @@ package registry import ( + "crypto/tls" + "crypto/x509" + "fmt" + "io/ioutil" "net/http" + "os" + "path/filepath" + "strings" + "github.com/Sirupsen/logrus" + "github.com/docker/distribution/registry/client/auth" "github.com/docker/docker/cliconfig" + "github.com/docker/docker/pkg/tlsconfig" ) type Service struct { @@ -25,7 +35,7 @@ func (s *Service) Auth(authConfig *cliconfig.AuthConfig) (string, error) { addr := authConfig.ServerAddress if addr == "" { // Use the official registry address if not specified. - addr = IndexServerAddress() + addr = INDEXSERVER } index, err := s.ResolveIndex(addr) if err != nil { @@ -69,3 +79,186 @@ func (s *Service) ResolveRepository(name string) (*RepositoryInfo, error) { func (s *Service) ResolveIndex(name string) (*IndexInfo, error) { return s.Config.NewIndexInfo(name) } + +type APIEndpoint struct { + Mirror bool + URL string + Version APIVersion + Official bool + TrimHostname bool + TLSConfig *tls.Config + VersionHeader string + Versions []auth.APIVersion +} + +func (e APIEndpoint) ToV1Endpoint(metaHeaders http.Header) (*Endpoint, error) { + return newEndpoint(e.URL, e.TLSConfig, metaHeaders) +} + +func (s *Service) TlsConfig(hostname string) (*tls.Config, error) { + // we construct a client tls config from server defaults + // PreferredServerCipherSuites should have no effect + tlsConfig := tlsconfig.ServerDefault + + isSecure := s.Config.isSecureIndex(hostname) + + tlsConfig.InsecureSkipVerify = !isSecure + + if isSecure { + hasFile := func(files []os.FileInfo, name string) bool { + for _, f := range files { + if f.Name() == name { + return true + } + } + return false + } + + hostDir := filepath.Join(CERTS_DIR, hostname) + logrus.Debugf("hostDir: %s", hostDir) + fs, err := ioutil.ReadDir(hostDir) + if err != nil && !os.IsNotExist(err) { + return nil, err + } + + for _, f := range fs { + if strings.HasSuffix(f.Name(), ".crt") { + if tlsConfig.RootCAs == nil { + // TODO(dmcgowan): Copy system pool + tlsConfig.RootCAs = x509.NewCertPool() + } + logrus.Debugf("crt: %s", filepath.Join(hostDir, f.Name())) + data, err := ioutil.ReadFile(filepath.Join(hostDir, f.Name())) + if err != nil { + return nil, err + } + tlsConfig.RootCAs.AppendCertsFromPEM(data) + } + if strings.HasSuffix(f.Name(), ".cert") { + certName := f.Name() + keyName := certName[:len(certName)-5] + ".key" + logrus.Debugf("cert: %s", filepath.Join(hostDir, f.Name())) + if !hasFile(fs, keyName) { + return nil, fmt.Errorf("Missing key %s for certificate %s", keyName, certName) + } + cert, err := tls.LoadX509KeyPair(filepath.Join(hostDir, certName), filepath.Join(hostDir, keyName)) + if err != nil { + return nil, err + } + tlsConfig.Certificates = append(tlsConfig.Certificates, cert) + } + if strings.HasSuffix(f.Name(), ".key") { + keyName := f.Name() + certName := keyName[:len(keyName)-4] + ".cert" + logrus.Debugf("key: %s", filepath.Join(hostDir, f.Name())) + if !hasFile(fs, certName) { + return nil, fmt.Errorf("Missing certificate %s for key %s", certName, keyName) + } + } + } + } + + return &tlsConfig, nil +} + +func (s *Service) LookupEndpoints(repoName string) (endpoints []APIEndpoint, err error) { + var cfg = tlsconfig.ServerDefault + tlsConfig := &cfg + if strings.HasPrefix(repoName, DEFAULT_NAMESPACE+"/") { + // v2 mirrors + for _, mirror := range s.Config.Mirrors { + endpoints = append(endpoints, APIEndpoint{ + URL: mirror, + // guess mirrors are v2 + Version: APIVersion2, + Mirror: true, + TrimHostname: true, + TLSConfig: tlsConfig, + }) + } + // v2 registry + endpoints = append(endpoints, APIEndpoint{ + URL: DEFAULT_V2_REGISTRY, + Version: APIVersion2, + Official: true, + TrimHostname: true, + TLSConfig: tlsConfig, + }) + // v1 mirrors + // TODO(tiborvass): shouldn't we remove v1 mirrors from here, since v1 mirrors are kinda special? + for _, mirror := range s.Config.Mirrors { + endpoints = append(endpoints, APIEndpoint{ + URL: mirror, + // guess mirrors are v1 + Version: APIVersion1, + Mirror: true, + TrimHostname: true, + TLSConfig: tlsConfig, + }) + } + // v1 registry + endpoints = append(endpoints, APIEndpoint{ + URL: DEFAULT_V1_REGISTRY, + Version: APIVersion1, + Official: true, + TrimHostname: true, + TLSConfig: tlsConfig, + }) + return endpoints, nil + } + + slashIndex := strings.IndexRune(repoName, '/') + if slashIndex <= 0 { + return nil, fmt.Errorf("invalid repo name: missing '/': %s", repoName) + } + hostname := repoName[:slashIndex] + + tlsConfig, err = s.TlsConfig(hostname) + if err != nil { + return nil, err + } + isSecure := !tlsConfig.InsecureSkipVerify + + v2Versions := []auth.APIVersion{ + { + Type: "registry", + Version: "2.0", + }, + } + endpoints = []APIEndpoint{ + { + URL: "https://" + hostname, + Version: APIVersion2, + TrimHostname: true, + TLSConfig: tlsConfig, + VersionHeader: DEFAULT_REGISTRY_VERSION_HEADER, + Versions: v2Versions, + }, + { + URL: "https://" + hostname, + Version: APIVersion1, + TrimHostname: true, + TLSConfig: tlsConfig, + }, + } + + if !isSecure { + endpoints = append(endpoints, APIEndpoint{ + URL: "http://" + hostname, + Version: APIVersion2, + TrimHostname: true, + // used to check if supposed to be secure via InsecureSkipVerify + TLSConfig: tlsConfig, + VersionHeader: DEFAULT_REGISTRY_VERSION_HEADER, + Versions: v2Versions, + }, APIEndpoint{ + URL: "http://" + hostname, + Version: APIVersion1, + TrimHostname: true, + // used to check if supposed to be secure via InsecureSkipVerify + TLSConfig: tlsConfig, + }) + } + + return endpoints, nil +} diff --git a/docs/session.go b/docs/session.go index 07195eb11..154c63e11 100644 --- a/docs/session.go +++ b/docs/session.go @@ -22,8 +22,8 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/docker/cliconfig" "github.com/docker/docker/pkg/httputils" + "github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/pkg/tarsum" - "github.com/docker/docker/pkg/transport" ) var ( @@ -73,6 +73,21 @@ func AuthTransport(base http.RoundTripper, authConfig *cliconfig.AuthConfig, alw } } +// cloneRequest returns a clone of the provided *http.Request. +// The clone is a shallow copy of the struct and its Header map. +func cloneRequest(r *http.Request) *http.Request { + // shallow copy of the struct + r2 := new(http.Request) + *r2 = *r + // deep copy of the Header + r2.Header = make(http.Header, len(r.Header)) + for k, s := range r.Header { + r2.Header[k] = append([]string(nil), s...) + } + + return r2 +} + func (tr *authTransport) RoundTrip(orig *http.Request) (*http.Response, error) { // Authorization should not be set on 302 redirect for untrusted locations. // This logic mirrors the behavior in AddRequiredHeadersToRedirectedRequests. @@ -83,7 +98,7 @@ func (tr *authTransport) RoundTrip(orig *http.Request) (*http.Response, error) { return tr.RoundTripper.RoundTrip(orig) } - req := transport.CloneRequest(orig) + req := cloneRequest(orig) tr.mu.Lock() tr.modReq[orig] = req tr.mu.Unlock() @@ -112,7 +127,7 @@ func (tr *authTransport) RoundTrip(orig *http.Request) (*http.Response, error) { if len(resp.Header["X-Docker-Token"]) > 0 { tr.token = resp.Header["X-Docker-Token"] } - resp.Body = &transport.OnEOFReader{ + resp.Body = &ioutils.OnEOFReader{ Rc: resp.Body, Fn: func() { tr.mu.Lock() @@ -149,12 +164,11 @@ func NewSession(client *http.Client, authConfig *cliconfig.AuthConfig, endpoint // If we're working with a standalone private registry over HTTPS, send Basic Auth headers // alongside all our requests. - if endpoint.VersionString(1) != IndexServerAddress() && endpoint.URL.Scheme == "https" { + if endpoint.VersionString(1) != INDEXSERVER && endpoint.URL.Scheme == "https" { info, err := endpoint.Ping() if err != nil { return nil, err } - if info.Standalone && authConfig != nil { logrus.Debugf("Endpoint %s is eligible for private registry. Enabling decorator.", endpoint.String()) alwaysSetBasicAuth = true @@ -250,7 +264,7 @@ func (r *Session) GetRemoteImageLayer(imgID, registry string, imgSize int64) (io if err != nil { return nil, fmt.Errorf("Error while getting from the server: %v", err) } - // TODO: why are we doing retries at this level? + // TODO(tiborvass): why are we doing retries at this level? // These retries should be generic to both v1 and v2 for i := 1; i <= retries; i++ { statusCode = 0 @@ -417,7 +431,7 @@ func (r *Session) GetRepositoryData(remote string) (*RepositoryData, error) { } // Forge a better object from the retrieved data - imgsData := make(map[string]*ImgData) + imgsData := make(map[string]*ImgData, len(remoteChecksums)) for _, elem := range remoteChecksums { imgsData[elem.ID] = elem } diff --git a/docs/session_v2.go b/docs/session_v2.go deleted file mode 100644 index f2b21df43..000000000 --- a/docs/session_v2.go +++ /dev/null @@ -1,414 +0,0 @@ -package registry - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "io/ioutil" - "net/http" - "strconv" - - "github.com/Sirupsen/logrus" - "github.com/docker/distribution/digest" - "github.com/docker/distribution/registry/api/v2" - "github.com/docker/docker/pkg/httputils" -) - -const DockerDigestHeader = "Docker-Content-Digest" - -func getV2Builder(e *Endpoint) *v2.URLBuilder { - if e.URLBuilder == nil { - e.URLBuilder = v2.NewURLBuilder(e.URL) - } - return e.URLBuilder -} - -func (r *Session) V2RegistryEndpoint(index *IndexInfo) (ep *Endpoint, err error) { - // TODO check if should use Mirror - if index.Official { - ep, err = newEndpoint(REGISTRYSERVER, true, nil) - if err != nil { - return - } - err = validateEndpoint(ep) - if err != nil { - return - } - } else if r.indexEndpoint.String() == index.GetAuthConfigKey() { - ep = r.indexEndpoint - } else { - ep, err = NewEndpoint(index, nil) - if err != nil { - return - } - } - - ep.URLBuilder = v2.NewURLBuilder(ep.URL) - return -} - -// GetV2Authorization gets the authorization needed to the given image -// If readonly access is requested, then the authorization may -// only be used for Get operations. -func (r *Session) GetV2Authorization(ep *Endpoint, imageName string, readOnly bool) (auth *RequestAuthorization, err error) { - scopes := []string{"pull"} - if !readOnly { - scopes = append(scopes, "push") - } - - logrus.Debugf("Getting authorization for %s %s", imageName, scopes) - return NewRequestAuthorization(r.GetAuthConfig(true), ep, "repository", imageName, scopes), nil -} - -// -// 1) Check if TarSum of each layer exists /v2/ -// 1.a) if 200, continue -// 1.b) if 300, then push the -// 1.c) if anything else, err -// 2) PUT the created/signed manifest -// - -// GetV2ImageManifest simply fetches the bytes of a manifest and the remote -// digest, if available in the request. Note that the application shouldn't -// rely on the untrusted remoteDigest, and should also verify against a -// locally provided digest, if applicable. -func (r *Session) GetV2ImageManifest(ep *Endpoint, imageName, tagName string, auth *RequestAuthorization) (remoteDigest digest.Digest, p []byte, err error) { - routeURL, err := getV2Builder(ep).BuildManifestURL(imageName, tagName) - if err != nil { - return "", nil, err - } - - method := "GET" - logrus.Debugf("[registry] Calling %q %s", method, routeURL) - - req, err := http.NewRequest(method, routeURL, nil) - if err != nil { - return "", nil, err - } - - if err := auth.Authorize(req); err != nil { - return "", nil, err - } - - res, err := r.client.Do(req) - if err != nil { - return "", nil, err - } - defer res.Body.Close() - - if res.StatusCode != 200 { - if res.StatusCode == 401 { - return "", nil, errLoginRequired - } else if res.StatusCode == 404 { - return "", nil, ErrDoesNotExist - } - return "", nil, httputils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch for %s:%s", res.StatusCode, imageName, tagName), res) - } - - p, err = ioutil.ReadAll(res.Body) - if err != nil { - return "", nil, fmt.Errorf("Error while reading the http response: %s", err) - } - - dgstHdr := res.Header.Get(DockerDigestHeader) - if dgstHdr != "" { - remoteDigest, err = digest.ParseDigest(dgstHdr) - if err != nil { - // NOTE(stevvooe): Including the remote digest is optional. We - // don't need to verify against it, but it is good practice. - remoteDigest = "" - logrus.Debugf("error parsing remote digest when fetching %v: %v", routeURL, err) - } - } - - return -} - -// - Succeeded to head image blob (already exists) -// - Failed with no error (continue to Push the Blob) -// - Failed with error -func (r *Session) HeadV2ImageBlob(ep *Endpoint, imageName string, dgst digest.Digest, auth *RequestAuthorization) (bool, error) { - routeURL, err := getV2Builder(ep).BuildBlobURL(imageName, dgst) - if err != nil { - return false, err - } - - method := "HEAD" - logrus.Debugf("[registry] Calling %q %s", method, routeURL) - - req, err := http.NewRequest(method, routeURL, nil) - if err != nil { - return false, err - } - if err := auth.Authorize(req); err != nil { - return false, err - } - res, err := r.client.Do(req) - if err != nil { - return false, err - } - res.Body.Close() // close early, since we're not needing a body on this call .. yet? - switch { - case res.StatusCode >= 200 && res.StatusCode < 400: - // return something indicating no push needed - return true, nil - case res.StatusCode == 401: - return false, errLoginRequired - case res.StatusCode == 404: - // return something indicating blob push needed - return false, nil - } - - return false, httputils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying head request for %s - %s", res.StatusCode, imageName, dgst), res) -} - -func (r *Session) GetV2ImageBlob(ep *Endpoint, imageName string, dgst digest.Digest, blobWrtr io.Writer, auth *RequestAuthorization) error { - routeURL, err := getV2Builder(ep).BuildBlobURL(imageName, dgst) - if err != nil { - return err - } - - method := "GET" - logrus.Debugf("[registry] Calling %q %s", method, routeURL) - req, err := http.NewRequest(method, routeURL, nil) - if err != nil { - return err - } - if err := auth.Authorize(req); err != nil { - return err - } - res, err := r.client.Do(req) - if err != nil { - return err - } - defer res.Body.Close() - if res.StatusCode != 200 { - if res.StatusCode == 401 { - return errLoginRequired - } - return httputils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to pull %s blob", res.StatusCode, imageName), res) - } - - _, err = io.Copy(blobWrtr, res.Body) - return err -} - -func (r *Session) GetV2ImageBlobReader(ep *Endpoint, imageName string, dgst digest.Digest, auth *RequestAuthorization) (io.ReadCloser, int64, error) { - routeURL, err := getV2Builder(ep).BuildBlobURL(imageName, dgst) - if err != nil { - return nil, 0, err - } - - method := "GET" - logrus.Debugf("[registry] Calling %q %s", method, routeURL) - req, err := http.NewRequest(method, routeURL, nil) - if err != nil { - return nil, 0, err - } - if err := auth.Authorize(req); err != nil { - return nil, 0, err - } - res, err := r.client.Do(req) - if err != nil { - return nil, 0, err - } - if res.StatusCode != 200 { - if res.StatusCode == 401 { - return nil, 0, errLoginRequired - } - return nil, 0, httputils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to pull %s blob - %s", res.StatusCode, imageName, dgst), res) - } - lenStr := res.Header.Get("Content-Length") - l, err := strconv.ParseInt(lenStr, 10, 64) - if err != nil { - return nil, 0, err - } - - return res.Body, l, err -} - -// Push the image to the server for storage. -// 'layer' is an uncompressed reader of the blob to be pushed. -// The server will generate it's own checksum calculation. -func (r *Session) PutV2ImageBlob(ep *Endpoint, imageName string, dgst digest.Digest, blobRdr io.Reader, auth *RequestAuthorization) error { - location, err := r.initiateBlobUpload(ep, imageName, auth) - if err != nil { - return err - } - - method := "PUT" - logrus.Debugf("[registry] Calling %q %s", method, location) - req, err := http.NewRequest(method, location, ioutil.NopCloser(blobRdr)) - if err != nil { - return err - } - queryParams := req.URL.Query() - queryParams.Add("digest", dgst.String()) - req.URL.RawQuery = queryParams.Encode() - if err := auth.Authorize(req); err != nil { - return err - } - res, err := r.client.Do(req) - if err != nil { - return err - } - defer res.Body.Close() - - if res.StatusCode != 201 { - if res.StatusCode == 401 { - return errLoginRequired - } - errBody, err := ioutil.ReadAll(res.Body) - if err != nil { - return err - } - logrus.Debugf("Unexpected response from server: %q %#v", errBody, res.Header) - return httputils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s blob - %s", res.StatusCode, imageName, dgst), res) - } - - return nil -} - -// initiateBlobUpload gets the blob upload location for the given image name. -func (r *Session) initiateBlobUpload(ep *Endpoint, imageName string, auth *RequestAuthorization) (location string, err error) { - routeURL, err := getV2Builder(ep).BuildBlobUploadURL(imageName) - if err != nil { - return "", err - } - - logrus.Debugf("[registry] Calling %q %s", "POST", routeURL) - req, err := http.NewRequest("POST", routeURL, nil) - if err != nil { - return "", err - } - - if err := auth.Authorize(req); err != nil { - return "", err - } - res, err := r.client.Do(req) - if err != nil { - return "", err - } - - if res.StatusCode != http.StatusAccepted { - if res.StatusCode == http.StatusUnauthorized { - return "", errLoginRequired - } - if res.StatusCode == http.StatusNotFound { - return "", ErrDoesNotExist - } - - errBody, err := ioutil.ReadAll(res.Body) - if err != nil { - return "", err - } - - logrus.Debugf("Unexpected response from server: %q %#v", errBody, res.Header) - return "", httputils.NewHTTPRequestError(fmt.Sprintf("Server error: unexpected %d response status trying to initiate upload of %s", res.StatusCode, imageName), res) - } - - if location = res.Header.Get("Location"); location == "" { - return "", fmt.Errorf("registry did not return a Location header for resumable blob upload for image %s", imageName) - } - - return -} - -// Finally Push the (signed) manifest of the blobs we've just pushed -func (r *Session) PutV2ImageManifest(ep *Endpoint, imageName, tagName string, signedManifest, rawManifest []byte, auth *RequestAuthorization) (digest.Digest, error) { - routeURL, err := getV2Builder(ep).BuildManifestURL(imageName, tagName) - if err != nil { - return "", err - } - - method := "PUT" - logrus.Debugf("[registry] Calling %q %s", method, routeURL) - req, err := http.NewRequest(method, routeURL, bytes.NewReader(signedManifest)) - if err != nil { - return "", err - } - if err := auth.Authorize(req); err != nil { - return "", err - } - res, err := r.client.Do(req) - if err != nil { - return "", err - } - defer res.Body.Close() - - // All 2xx and 3xx responses can be accepted for a put. - if res.StatusCode >= 400 { - if res.StatusCode == 401 { - return "", errLoginRequired - } - errBody, err := ioutil.ReadAll(res.Body) - if err != nil { - return "", err - } - logrus.Debugf("Unexpected response from server: %q %#v", errBody, res.Header) - return "", httputils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s:%s manifest", res.StatusCode, imageName, tagName), res) - } - - hdrDigest, err := digest.ParseDigest(res.Header.Get(DockerDigestHeader)) - if err != nil { - return "", fmt.Errorf("invalid manifest digest from registry: %s", err) - } - - dgstVerifier, err := digest.NewDigestVerifier(hdrDigest) - if err != nil { - return "", fmt.Errorf("invalid manifest digest from registry: %s", err) - } - - dgstVerifier.Write(rawManifest) - - if !dgstVerifier.Verified() { - computedDigest, _ := digest.FromBytes(rawManifest) - return "", fmt.Errorf("unable to verify manifest digest: registry has %q, computed %q", hdrDigest, computedDigest) - } - - return hdrDigest, nil -} - -type remoteTags struct { - Name string `json:"name"` - Tags []string `json:"tags"` -} - -// Given a repository name, returns a json array of string tags -func (r *Session) GetV2RemoteTags(ep *Endpoint, imageName string, auth *RequestAuthorization) ([]string, error) { - routeURL, err := getV2Builder(ep).BuildTagsURL(imageName) - if err != nil { - return nil, err - } - - method := "GET" - logrus.Debugf("[registry] Calling %q %s", method, routeURL) - - req, err := http.NewRequest(method, routeURL, nil) - if err != nil { - return nil, err - } - if err := auth.Authorize(req); err != nil { - return nil, err - } - res, err := r.client.Do(req) - if err != nil { - return nil, err - } - defer res.Body.Close() - if res.StatusCode != 200 { - if res.StatusCode == 401 { - return nil, errLoginRequired - } else if res.StatusCode == 404 { - return nil, ErrDoesNotExist - } - return nil, httputils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch for %s", res.StatusCode, imageName), res) - } - - var remote remoteTags - if err := json.NewDecoder(res.Body).Decode(&remote); err != nil { - return nil, fmt.Errorf("Error while decoding the http response: %s", err) - } - return remote.Tags, nil -}