Merge branch 'master' into bump_v0.11.0

Docker-DCO-1.1-Signed-off-by: Michael Crosby <michael@crosbymichael.com> (github: crosbymichael)

docs/registry.go

@@ -3,10 +3,10 @@ package registry
 import (
 	"bytes"
 	"crypto/sha256"
+	_ "crypto/sha512"
 	"encoding/json"
 	"errors"
 	"fmt"
-	"github.com/dotcloud/docker/utils"
 	"io"
 	"io/ioutil"
 	"net"
@@ -14,9 +14,13 @@ import (
 	"net/http/cookiejar"
 	"net/url"
 	"regexp"
+	"runtime"
 	"strconv"
 	"strings"
 	"time"
+
+	"github.com/dotcloud/docker/dockerversion"
+	"github.com/dotcloud/docker/utils"
 )
 
 var (
@@ -25,11 +29,11 @@ var (
 	errLoginRequired         = errors.New("Authentication is required.")
 )
 
-func pingRegistryEndpoint(endpoint string) (bool, error) {
+func pingRegistryEndpoint(endpoint string) (RegistryInfo, error) {
 	if endpoint == IndexServerAddress() {
 		// Skip the check, we now this one is valid
 		// (and we never want to fallback to http in case of error)
-		return false, nil
+		return RegistryInfo{Standalone: false}, nil
 	}
 	httpDial := func(proto string, addr string) (net.Conn, error) {
 		// Set the connect timeout to 5 seconds
@@ -48,26 +52,41 @@ func pingRegistryEndpoint(endpoint string) (bool, error) {
 	client := &http.Client{Transport: httpTransport}
 	resp, err := client.Get(endpoint + "_ping")
 	if err != nil {
-		return false, err
+		return RegistryInfo{Standalone: false}, err
 	}
 	defer resp.Body.Close()
 
-	if resp.Header.Get("X-Docker-Registry-Version") == "" {
+	jsonString, err := ioutil.ReadAll(resp.Body)
-		return false, errors.New("This does not look like a Registry server (\"X-Docker-Registry-Version\" header not found in the response)")
+	if err != nil {
+		return RegistryInfo{Standalone: false}, fmt.Errorf("Error while reading the http response: %s", err)
 	}
 
+	// If the header is absent, we assume true for compatibility with earlier
+	// versions of the registry. default to true
+	info := RegistryInfo{
+		Standalone: true,
+	}
+	if err := json.Unmarshal(jsonString, &info); err != nil {
+		utils.Debugf("Error unmarshalling the _ping RegistryInfo: %s", err)
+		// don't stop here. Just assume sane defaults
+	}
+	if hdr := resp.Header.Get("X-Docker-Registry-Version"); hdr != "" {
+		utils.Debugf("Registry version header: '%s'", hdr)
+		info.Version = hdr
+	}
+	utils.Debugf("RegistryInfo.Version: %q", info.Version)
+
 	standalone := resp.Header.Get("X-Docker-Registry-Standalone")
 	utils.Debugf("Registry standalone header: '%s'", standalone)
-	// If the header is absent, we assume true for compatibility with earlier
+	// Accepted values are "true" (case-insensitive) and "1".
-	// versions of the registry
+	if strings.EqualFold(standalone, "true") || standalone == "1" {
-	if standalone == "" {
+		info.Standalone = true
-		return true, nil
+	} else if len(standalone) > 0 {
-		// Accepted values are "true" (case-insensitive) and "1".
+		// there is a header set, and it is not "true" or "1", so assume fails
-	} else if strings.EqualFold(standalone, "true") || standalone == "1" {
+		info.Standalone = false
-		return true, nil
 	}
-	// Otherwise, not standalone
+	utils.Debugf("RegistryInfo.Standalone: %q", info.Standalone)
-	return false, nil
+	return info, nil
 }
 
 func validateRepositoryName(repositoryName string) error {
@@ -101,17 +120,12 @@ func ResolveRepositoryName(reposName string) (string, string, error) {
 		return "", "", ErrInvalidRepositoryName
 	}
 	nameParts := strings.SplitN(reposName, "/", 2)
-	if !strings.Contains(nameParts[0], ".") && !strings.Contains(nameParts[0], ":") &&
+	if len(nameParts) == 1 || (!strings.Contains(nameParts[0], ".") && !strings.Contains(nameParts[0], ":") &&
-		nameParts[0] != "localhost" {
+		nameParts[0] != "localhost") {
 		// This is a Docker Index repos (ex: samalba/hipache or ubuntu)
 		err := validateRepositoryName(reposName)
 		return IndexServerAddress(), reposName, err
 	}
-	if len(nameParts) < 2 {
-		// There is a dot in repos name (and no registry address)
-		// Is it a Registry address without repos name?
-		return "", "", ErrInvalidRepositoryName
-	}
 	hostname := nameParts[0]
 	reposName = nameParts[1]
 	if strings.Contains(hostname, "index.docker.io") {
@@ -226,9 +240,13 @@ func (r *Registry) GetRemoteImageJSON(imgID, registry string, token []string) ([
 		return nil, -1, utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d", res.StatusCode), res)
 	}
 
-	imageSize, err := strconv.Atoi(res.Header.Get("X-Docker-Size"))
+	// if the size header is not present, then set it to '-1'
-	if err != nil {
+	imageSize := -1
-		return nil, -1, err
+	if hdr := res.Header.Get("X-Docker-Size"); hdr != "" {
+		imageSize, err = strconv.Atoi(hdr)
+		if err != nil {
+			return nil, -1, err
+		}
 	}
 
 	jsonString, err := ioutil.ReadAll(res.Body)
@@ -297,6 +315,25 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [
 	return nil, fmt.Errorf("Could not reach any registry endpoint")
 }
 
+func buildEndpointsList(headers []string, indexEp string) ([]string, error) {
+	var endpoints []string
+	parsedUrl, err := url.Parse(indexEp)
+	if err != nil {
+		return nil, err
+	}
+	var urlScheme = parsedUrl.Scheme
+	// The Registry's URL scheme has to match the Index'
+	for _, ep := range headers {
+		epList := strings.Split(ep, ",")
+		for _, epListElement := range epList {
+			endpoints = append(
+				endpoints,
+				fmt.Sprintf("%s://%s/v1/", urlScheme, strings.TrimSpace(epListElement)))
+		}
+	}
+	return endpoints, nil
+}
+
 func (r *Registry) GetRepositoryData(remote string) (*RepositoryData, error) {
 	indexEp := r.indexEndpoint
 	repositoryTarget := fmt.Sprintf("%srepositories/%s/images", indexEp, remote)
@@ -332,14 +369,18 @@ func (r *Registry) GetRepositoryData(remote string) (*RepositoryData, error) {
 	}
 
 	var endpoints []string
-	var urlScheme = indexEp[:strings.Index(indexEp, ":")]
 	if res.Header.Get("X-Docker-Endpoints") != "" {
-		// The Registry's URL scheme has to match the Index'
+		endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], indexEp)
-		for _, ep := range res.Header["X-Docker-Endpoints"] {
+		if err != nil {
-			endpoints = append(endpoints, fmt.Sprintf("%s://%s/v1/", urlScheme, ep))
+			return nil, err
 		}
 	} else {
-		return nil, fmt.Errorf("Index response didn't contain any endpoints")
+		// Assume the endpoint is on the same host
+		u, err := url.Parse(indexEp)
+		if err != nil {
+			return nil, err
+		}
+		endpoints = append(endpoints, fmt.Sprintf("%s://%s/v1/", u.Scheme, req.URL.Host))
 	}
 
 	checksumsJSON, err := ioutil.ReadAll(res.Body)
@@ -565,7 +606,6 @@ func (r *Registry) PushImageJSONIndex(remote string, imgList []*ImgData, validat
 	}
 
 	var tokens, endpoints []string
-	var urlScheme = indexEp[:strings.Index(indexEp, ":")]
 	if !validate {
 		if res.StatusCode != 200 && res.StatusCode != 201 {
 			errBody, err := ioutil.ReadAll(res.Body)
@@ -582,9 +622,9 @@ func (r *Registry) PushImageJSONIndex(remote string, imgList []*ImgData, validat
 		}
 
 		if res.Header.Get("X-Docker-Endpoints") != "" {
-			// The Registry's URL scheme has to match the Index'
+			endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], indexEp)
-			for _, ep := range res.Header["X-Docker-Endpoints"] {
+			if err != nil {
-				endpoints = append(endpoints, fmt.Sprintf("%s://%s/v1/", urlScheme, ep))
+				return nil, err
 			}
 		} else {
 			return nil, fmt.Errorf("Index response didn't contain any endpoints")
@@ -673,6 +713,11 @@ type ImgData struct {
 	Tag             string `json:",omitempty"`
 }
 
+type RegistryInfo struct {
+	Version    string `json:"version"`
+	Standalone bool   `json:"standalone"`
+}
+
 type Registry struct {
 	client        *http.Client
 	authConfig    *AuthConfig
@@ -701,11 +746,11 @@ func NewRegistry(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, inde
 	// If we're working with a standalone private registry over HTTPS, send Basic Auth headers
 	// alongside our requests.
 	if indexEndpoint != IndexServerAddress() && strings.HasPrefix(indexEndpoint, "https://") {
-		standalone, err := pingRegistryEndpoint(indexEndpoint)
+		info, err := pingRegistryEndpoint(indexEndpoint)
 		if err != nil {
 			return nil, err
 		}
-		if standalone {
+		if info.Standalone {
 			utils.Debugf("Endpoint %s is eligible for private registry registry. Enabling decorator.", indexEndpoint)
 			dec := utils.NewHTTPAuthDecorator(authConfig.Username, authConfig.Password)
 			factory.AddDecorator(dec)
@@ -715,3 +760,40 @@ func NewRegistry(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, inde
 	r.reqFactory = factory
 	return r, nil
 }
+
+func HTTPRequestFactory(metaHeaders map[string][]string) *utils.HTTPRequestFactory {
+	// FIXME: this replicates the 'info' job.
+	httpVersion := make([]utils.VersionInfo, 0, 4)
+	httpVersion = append(httpVersion, &simpleVersionInfo{"docker", dockerversion.VERSION})
+	httpVersion = append(httpVersion, &simpleVersionInfo{"go", runtime.Version()})
+	httpVersion = append(httpVersion, &simpleVersionInfo{"git-commit", dockerversion.GITCOMMIT})
+	if kernelVersion, err := utils.GetKernelVersion(); err == nil {
+		httpVersion = append(httpVersion, &simpleVersionInfo{"kernel", kernelVersion.String()})
+	}
+	httpVersion = append(httpVersion, &simpleVersionInfo{"os", runtime.GOOS})
+	httpVersion = append(httpVersion, &simpleVersionInfo{"arch", runtime.GOARCH})
+	ud := utils.NewHTTPUserAgentDecorator(httpVersion...)
+	md := &utils.HTTPMetaHeadersDecorator{
+		Headers: metaHeaders,
+	}
+	factory := utils.NewHTTPRequestFactory(ud, md)
+	return factory
+}
+
+// simpleVersionInfo is a simple implementation of
+// the interface VersionInfo, which is used
+// to provide version information for some product,
+// component, etc. It stores the product name and the version
+// in string and returns them on calls to Name() and Version().
+type simpleVersionInfo struct {
+	name    string
+	version string
+}
+
+func (v *simpleVersionInfo) Name() string {
+	return v.name
+}
+
+func (v *simpleVersionInfo) Version() string {
+	return v.version
+}

docs/registry_mock_test.go

@@ -291,7 +291,7 @@ func handlerUsers(w http.ResponseWriter, r *http.Request) {
 
 func handlerImages(w http.ResponseWriter, r *http.Request) {
 	u, _ := url.Parse(testHttpServer.URL)
-	w.Header().Add("X-Docker-Endpoints", u.Host)
+	w.Header().Add("X-Docker-Endpoints", fmt.Sprintf("%s 	,  %s ", u.Host, "test.example.com"))
 	w.Header().Add("X-Docker-Token", fmt.Sprintf("FAKE-SESSION-%d", time.Now().UnixNano()))
 	if r.Method == "PUT" {
 		if strings.HasSuffix(r.URL.Path, "images") {

docs/registry_test.go

@@ -1,7 +1,9 @@
 package registry
 
 import (
+	"fmt"
 	"github.com/dotcloud/docker/utils"
+	"net/url"
 	"strings"
 	"testing"
 )
@@ -22,11 +24,11 @@ func spawnTestRegistry(t *testing.T) *Registry {
 }
 
 func TestPingRegistryEndpoint(t *testing.T) {
-	standalone, err := pingRegistryEndpoint(makeURL("/v1/"))
+	regInfo, err := pingRegistryEndpoint(makeURL("/v1/"))
 	if err != nil {
 		t.Fatal(err)
 	}
-	assertEqual(t, standalone, true, "Expected standalone to be true (default)")
+	assertEqual(t, regInfo.Standalone, true, "Expected standalone to be true (default)")
 }
 
 func TestGetRemoteHistory(t *testing.T) {
@@ -99,12 +101,23 @@ func TestGetRemoteTags(t *testing.T) {
 
 func TestGetRepositoryData(t *testing.T) {
 	r := spawnTestRegistry(t)
+	parsedUrl, err := url.Parse(makeURL("/v1/"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	host := "http://" + parsedUrl.Host + "/v1/"
 	data, err := r.GetRepositoryData("foo42/bar")
 	if err != nil {
 		t.Fatal(err)
 	}
 	assertEqual(t, len(data.ImgList), 2, "Expected 2 images in ImgList")
-	assertEqual(t, len(data.Endpoints), 1, "Expected one endpoint in Endpoints")
+	assertEqual(t, len(data.Endpoints), 2,
+		fmt.Sprintf("Expected 2 endpoints in Endpoints, found %d instead", len(data.Endpoints)))
+	assertEqual(t, data.Endpoints[0], host,
+		fmt.Sprintf("Expected first endpoint to be %s but found %s instead", host, data.Endpoints[0]))
+	assertEqual(t, data.Endpoints[1], "http://test.example.com/v1/",
+		fmt.Sprintf("Expected first endpoint to be http://test.example.com/v1/ but found %s instead", data.Endpoints[1]))
+
 }
 
 func TestPushImageJSONRegistry(t *testing.T) {
@@ -146,6 +159,13 @@ func TestResolveRepositoryName(t *testing.T) {
 	}
 	assertEqual(t, ep, u, "Expected endpoint to be "+u)
 	assertEqual(t, repo, "private/moonbase", "Expected endpoint to be private/moonbase")
+
+	ep, repo, err = ResolveRepositoryName("ubuntu-12.04-base")
+	if err != nil {
+		t.Fatal(err)
+	}
+	assertEqual(t, ep, IndexServerAddress(), "Expected endpoint to be "+IndexServerAddress())
+	assertEqual(t, repo, "ubuntu-12.04-base", "Expected endpoint to be ubuntu-12.04-base")
 }
 
 func TestPushRegistryTag(t *testing.T) {

docs/service.go

@@ -0,0 +1,104 @@
+package registry
+
+import (
+	"github.com/dotcloud/docker/engine"
+)
+
+// Service exposes registry capabilities in the standard Engine
+// interface. Once installed, it extends the engine with the
+// following calls:
+//
+//  'auth': Authenticate against the public registry
+//  'search': Search for images on the public registry
+//  'pull': Download images from any registry (TODO)
+//  'push': Upload images to any registry (TODO)
+type Service struct {
+}
+
+// NewService returns a new instance of Service ready to be
+// installed no an engine.
+func NewService() *Service {
+	return &Service{}
+}
+
+// Install installs registry capabilities to eng.
+func (s *Service) Install(eng *engine.Engine) error {
+	eng.Register("auth", s.Auth)
+	eng.Register("search", s.Search)
+	return nil
+}
+
+// Auth contacts the public registry with the provided credentials,
+// and returns OK if authentication was sucessful.
+// It can be used to verify the validity of a client's credentials.
+func (s *Service) Auth(job *engine.Job) engine.Status {
+	var (
+		err        error
+		authConfig = &AuthConfig{}
+	)
+
+	job.GetenvJson("authConfig", authConfig)
+	// TODO: this is only done here because auth and registry need to be merged into one pkg
+	if addr := authConfig.ServerAddress; addr != "" && addr != IndexServerAddress() {
+		addr, err = ExpandAndVerifyRegistryUrl(addr)
+		if err != nil {
+			return job.Error(err)
+		}
+		authConfig.ServerAddress = addr
+	}
+	status, err := Login(authConfig, HTTPRequestFactory(nil))
+	if err != nil {
+		return job.Error(err)
+	}
+	job.Printf("%s\n", status)
+	return engine.StatusOK
+}
+
+// Search queries the public registry for images matching the specified
+// search terms, and returns the results.
+//
+// Argument syntax: search TERM
+//
+// Option environment:
+//	'authConfig': json-encoded credentials to authenticate against the registry.
+//		The search extends to images only accessible via the credentials.
+//
+//	'metaHeaders': extra HTTP headers to include in the request to the registry.
+//		The headers should be passed as a json-encoded dictionary.
+//
+// Output:
+//	Results are sent as a collection of structured messages (using engine.Table).
+//	Each result is sent as a separate message.
+//	Results are ordered by number of stars on the public registry.
+func (s *Service) Search(job *engine.Job) engine.Status {
+	if n := len(job.Args); n != 1 {
+		return job.Errorf("Usage: %s TERM", job.Name)
+	}
+	var (
+		term        = job.Args[0]
+		metaHeaders = map[string][]string{}
+		authConfig  = &AuthConfig{}
+	)
+	job.GetenvJson("authConfig", authConfig)
+	job.GetenvJson("metaHeaders", metaHeaders)
+
+	r, err := NewRegistry(authConfig, HTTPRequestFactory(metaHeaders), IndexServerAddress())
+	if err != nil {
+		return job.Error(err)
+	}
+	results, err := r.SearchRepositories(term)
+	if err != nil {
+		return job.Error(err)
+	}
+	outs := engine.NewTable("star_count", 0)
+	for _, result := range results.Results {
+		out := &engine.Env{}
+		out.Import(result)
+		outs.Add(out)
+	}
+	outs.ReverseSort()
+	if _, err := outs.WriteListTo(job.Stdout); err != nil {
+		return job.Error(err)
+	}
+	return engine.StatusOK
+}