Merge pull request #1739 from cezarsa/master

[Swift] Expose EndpointType parameter in driver

Godeps/Godeps.json

@@ -1,7 +1,7 @@
 {
 	"ImportPath": "github.com/docker/distribution",
 	"GoVersion": "go1.6",
-	"GodepVersion": "v60",
+	"GodepVersion": "v70",
 	"Packages": [
 		"./..."
 	],
@@ -223,11 +223,11 @@
 		},
 		{
 			"ImportPath": "github.com/ncw/swift",
-			"Rev": "c54732e87b0b283d1baf0a18db689d0aea460ba3"
+			"Rev": "ce444d6d47c51d4dda9202cd38f5094dd8e27e86"
 		},
 		{
 			"ImportPath": "github.com/ncw/swift/swifttest",
-			"Rev": "c54732e87b0b283d1baf0a18db689d0aea460ba3"
+			"Rev": "ce444d6d47c51d4dda9202cd38f5094dd8e27e86"
 		},
 		{
 			"ImportPath": "github.com/spf13/cobra",

docs/storage-drivers/swift.md

@@ -187,169 +187,28 @@ An implementation of the `storagedriver.StorageDriver` interface that uses [Open
       The access key to generate temporary URLs. It is used by HP Cloud Object Storage in addition to the `secretkey` parameter.
     </td>
   </tr>
-</table>
+  <tr>
-
-
-<table>
-<tr>
-  <td>
-  <code>authurl</code>
-  </td>
-  <td>
-    <p>URL for obtaining an auth token.</p>
-  </td>
-</tr>
-<tr>
-  <td>
-  <code>username</code>
-  </td>
-  <td>
-  <p>
-  Your OpenStack user name.</p>
-  </p>
-  </td>
-</tr>
-<tr>
-  <td>
-  <code>password</code>
-  <p>
-  </td>
-  <td>
-  <p>
-  Your OpenStack password.
-  </p>
-  </td>
-</tr>
-<tr>
-  <td>
-  <code>container</code>
-  </td>
-  <td>
-  <p>
-	The name of your Swift container where you wish to store the registry's data. The driver creates the named container during its initialization.
-  </p>
-  </td>
-</tr>
-<tr>
-  <td>
-  <code>tenant</code>
-  </td>
-  <td>
-  <p>
-  Optionally, your OpenStack tenant name. You can either use <code>tenant</code> or <code>tenantid</code>.
-  </p>
-  </td>
-</tr>
-<tr>
-    <td>
-    <code>tenantid</code>
-    </td>
-    <td>
-    <p>
-    Optionally, your OpenStack tenant id. You can either use <code>tenant</code> or <code>tenantid</code>.
-    </p>
-    </td>
-</tr>
-<tr>
-    <td>
-    <code>domain</code>
-    </td>
-    <td>
-    <p>
-    Optionally, your OpenStack domain name for Identity v3 API. You can either use <code>domain</code> or <code>domainid</code>.
-    </p>
-    </td>
-</tr>
-<tr>
-    <td>
-    <code>domainid</code>
-    </td>
-    <td>
-    <p>
-    Optionally, your OpenStack domain id for Identity v3 API. You can either use <code>domain</code> or <code>domainid</code>.
-    </p>
-    </td>
-</tr>
-<tr>
-    <td>
-    <code>trustid</code>
-    </td>
-    <td>
-    <p>
-    Optionally, your OpenStack trust id for Identity v3 API.
-    </p>
-    </td>
-</tr>
-<tr>
-    <td>
-    <code>insecureskipverify</code>
-    </td>
-    <td>
-    <p>
-    Optionally, set <code>insecureskipverify</code> to true to skip TLS verification for your OpenStack provider. The driver uses false by default.
-    </p>
-    </td>
-</tr>
-<tr>
-    <td>
-    <code>region</code>
-    </td>
-    <td>
-    <p>
-    Optionally, specify the OpenStack region name in which you would like to store objects (for example <code>fr</code>).
-    </p>
-    </td>
-</tr>
-<tr>
     <td>
       <code>authversion</code>
     </td>
     <td>
-    <p>
+      no
-    Optionally, specify the OpenStack Auth's version,for example <code>3</code>. By default the driver will autodetect the auth's version from the AuthURL.
-    </p>
-    </td>
-</tr>
-<tr>
-    <td>
-    <code>chunksize</code>
     </td>
     <td>
-    <p>
+      Specify the OpenStack Auth's version,for example <code>3</code>. By default the driver will autodetect the auth's version from the AuthURL.
-    Optionally, specify the segment size for Dynamic Large Objects uploads (performed by WriteStream) to Swift. The default is 5 MB. You might experience better performance for larger chunk sizes depending on the speed of your connection to Swift.
-    </p>
     </td>
-</tr>
+  </tr>
-<tr>
+  <tr>
     <td>
-    <code>prefix</code>
+      <code>endpointtype</code>
     </td>
     <td>
-    <p>
+      no
-    Optionally, supply a prefix that will be applied to all Swift keys to allow you to segment data in your container if necessary. Defaults to the empty string which is the container's root.</p>
-    </p>
-    </td>
-</tr>
-<tr>
-    <td>
-    <code>secretkey</code>
     </td>
     <td>
-    <p>
+      The endpoint type used when connecting to swift. Possible values are `public`, `internal` and `admin`. Default is `public`.
-    Optionally, the secret key used to generate temporary URLs.</p>
-    </p>
     </td>
-</tr>
+  </tr>
-<tr>
-    <td>
-    <code>accesskey</code>
-    </td>
-    <td>
-    <p>
-    Optionally, the access key to generate temporary URLs. It is used by HP Cloud Object Storage in addition to the `secretkey` parameter.</p>
-    </p>
-    </td>
-</tr>
 </table>
 
 The features supported by the Swift server are queried by requesting the `/info` URL on the server. In case the administrator

registry/storage/driver/swift/swift.go

@@ -72,6 +72,7 @@ type Parameters struct {
 	AuthVersion         int
 	Container           string
 	Prefix              string
+	EndpointType        string
 	InsecureSkipVerify  bool
 	ChunkSize           int
 	SecretKey           string
@@ -182,6 +183,7 @@ func New(params Parameters) (*Driver, error) {
 		Domain:         params.Domain,
 		DomainId:       params.DomainID,
 		TrustId:        params.TrustID,
+		EndpointType:   swift.EndpointType(params.EndpointType),
 		Transport:      transport,
 		ConnectTimeout: 60 * time.Second,
 		Timeout:        15 * 60 * time.Second,

registry/storage/driver/swift/swift_test.go

@@ -34,6 +34,7 @@ func init() {
 		container          string
 		region             string
 		AuthVersion        int
+		endpointType       string
 		insecureSkipVerify bool
 		secretKey          string
 		accessKey          string
@@ -54,6 +55,7 @@ func init() {
 	container = os.Getenv("SWIFT_CONTAINER_NAME")
 	region = os.Getenv("SWIFT_REGION_NAME")
 	AuthVersion, _ = strconv.Atoi(os.Getenv("SWIFT_AUTH_VERSION"))
+	endpointType = os.Getenv("SWIFT_ENDPOINT_TYPE")
 	insecureSkipVerify, _ = strconv.ParseBool(os.Getenv("SWIFT_INSECURESKIPVERIFY"))
 	secretKey = os.Getenv("SWIFT_SECRET_KEY")
 	accessKey = os.Getenv("SWIFT_ACCESS_KEY")
@@ -90,6 +92,7 @@ func init() {
 			AuthVersion,
 			container,
 			root,
+			endpointType,
 			insecureSkipVerify,
 			defaultChunkSize,
 			secretKey,

vendor/github.com/ncw/swift/.travis.yml

@@ -4,8 +4,11 @@ sudo: false
 go:
   - 1.1.2
   - 1.2.2
-  - 1.3
+  - 1.3.3
+  - 1.4.2
+  - 1.5.1
   - tip
 
 script:
+  - test -z "$(go fmt ./...)"
   - go test

vendor/github.com/ncw/swift/README.md

@@ -9,7 +9,7 @@ See here for package docs
 
   http://godoc.org/github.com/ncw/swift
 
-[![Build Status](https://travis-ci.org/ncw/swift.png)](https://travis-ci.org/ncw/swift)
+[![Build Status](https://api.travis-ci.org/ncw/swift.svg?branch=master)](https://travis-ci.org/ncw/swift) [![GoDoc](https://godoc.org/github.com/ncw/swift?status.svg)](https://godoc.org/github.com/ncw/swift) 
 
 Install
 -------
@@ -134,3 +134,6 @@ Contributors
 - Fabian Ruff <fabian@progra.de>
 - Arturo Reuschenbach Puncernau <reuschenbach@gmail.com>
 - Petr Kotek <petr.kotek@bigcommerce.com>
+- Stefan Majewsky <stefan.majewsky@sap.com>
+- Cezar Sa Espinola <cezarsa@gmail.com>
+- Sam Gunaratne <samgzeit@gmail.com>

vendor/github.com/ncw/swift/auth.go

@@ -12,7 +12,9 @@ import (
 //
 // This encapsulates the different authentication schemes in use
 type Authenticator interface {
+	// Request creates an http.Request for the auth - return nil if not needed
 	Request(*Connection) (*http.Request, error)
+	// Response parses the http.Response
 	Response(resp *http.Response) error
 	// The public storage URL - set Internal to true to read
 	// internal/service net URL
@@ -23,6 +25,23 @@ type Authenticator interface {
 	CdnUrl() string
 }
 
+type CustomEndpointAuthenticator interface {
+	StorageUrlForEndpoint(endpointType EndpointType) string
+}
+
+type EndpointType string
+
+const (
+	// Use public URL as storage URL
+	EndpointTypePublic = EndpointType("public")
+
+	// Use internal URL as storage URL
+	EndpointTypeInternal = EndpointType("internal")
+
+	// Use admin URL as storage URL
+	EndpointTypeAdmin = EndpointType("admin")
+)
+
 // newAuth - create a new Authenticator from the AuthUrl
 //
 // A hint for AuthVersion can be provided
@@ -175,15 +194,20 @@ func (auth *v2Auth) Response(resp *http.Response) error {
 // Region if set or defaulting to the first one if not
 //
 // Returns "" if not found
-func (auth *v2Auth) endpointUrl(Type string, Internal bool) string {
+func (auth *v2Auth) endpointUrl(Type string, endpointType EndpointType) string {
 	for _, catalog := range auth.Auth.Access.ServiceCatalog {
 		if catalog.Type == Type {
 			for _, endpoint := range catalog.Endpoints {
 				if auth.Region == "" || (auth.Region == endpoint.Region) {
-					if Internal {
+					switch endpointType {
+					case EndpointTypeInternal:
 						return endpoint.InternalUrl
-					} else {
+					case EndpointTypePublic:
 						return endpoint.PublicUrl
+					case EndpointTypeAdmin:
+						return endpoint.AdminUrl
+					default:
+						return ""
 					}
 				}
 			}
@@ -197,7 +221,18 @@ func (auth *v2Auth) endpointUrl(Type string, Internal bool) string {
 // If Internal is true then it reads the private (internal / service
 // net) URL.
 func (auth *v2Auth) StorageUrl(Internal bool) string {
-	return auth.endpointUrl("object-store", Internal)
+	endpointType := EndpointTypePublic
+	if Internal {
+		endpointType = EndpointTypeInternal
+	}
+	return auth.StorageUrlForEndpoint(endpointType)
+}
+
+// v2 Authentication - read storage url
+//
+// Use the indicated endpointType to choose a URL.
+func (auth *v2Auth) StorageUrlForEndpoint(endpointType EndpointType) string {
+	return auth.endpointUrl("object-store", endpointType)
 }
 
 // v2 Authentication - read auth token
@@ -207,7 +242,7 @@ func (auth *v2Auth) Token() string {
 
 // v2 Authentication - read cdn url
 func (auth *v2Auth) CdnUrl() string {
-	return auth.endpointUrl("rax:object-cdn", false)
+	return auth.endpointUrl("rax:object-cdn", EndpointTypePublic)
 }
 
 // ------------------------------------------------------------
@@ -255,6 +290,7 @@ type v2AuthResponse struct {
 			Endpoints []struct {
 				InternalUrl string
 				PublicUrl   string
+				AdminUrl    string
 				Region      string
 				TenantId    string
 			}

vendor/github.com/ncw/swift/auth_v3.go

@@ -10,9 +10,6 @@ import (
 const (
 	v3AuthMethodToken        = "token"
 	v3AuthMethodPassword     = "password"
-	v3InterfacePublic        = "public"
-	v3InterfaceInternal      = "internal"
-	v3InterfaceAdmin         = "admin"
 	v3CatalogTypeObjectStore = "object-store"
 )
 
@@ -88,7 +85,8 @@ type v3AuthResponse struct {
 		Catalog []struct {
 			Id, Namem, Type string
 			Endpoints       []struct {
-				Id, Region_Id, Url, Region, Interface string
+				Id, Region_Id, Url, Region string
+				Interface                  EndpointType
 			}
 		}
 
@@ -107,11 +105,13 @@ type v3AuthResponse struct {
 }
 
 type v3Auth struct {
+	Region  string
 	Auth    *v3AuthResponse
 	Headers http.Header
 }
 
 func (auth *v3Auth) Request(c *Connection) (*http.Request, error) {
+	auth.Region = c.Region
 
 	var v3i interface{}
 
@@ -149,13 +149,18 @@ func (auth *v3Auth) Request(c *Connection) (*http.Request, error) {
 			v3.Auth.Scope.Project.Id = c.TenantId
 		} else if c.Tenant != "" {
 			v3.Auth.Scope.Project.Name = c.Tenant
-			var defaultDomain v3Domain
+			switch {
-			if c.Domain != "" {
+			case c.TenantDomain != "":
-				defaultDomain = v3Domain{Name: "Default"}
+				v3.Auth.Scope.Project.Domain = &v3Domain{Name: c.TenantDomain}
-			} else if c.DomainId != "" {
+			case c.TenantDomainId != "":
-				defaultDomain = v3Domain{Id: "Default"}
+				v3.Auth.Scope.Project.Domain = &v3Domain{Id: c.TenantDomainId}
+			case c.Domain != "":
+				v3.Auth.Scope.Project.Domain = &v3Domain{Name: c.Domain}
+			case c.DomainId != "":
+				v3.Auth.Scope.Project.Domain = &v3Domain{Id: c.DomainId}
+			default:
+				v3.Auth.Scope.Project.Domain = &v3Domain{Name: "Default"}
 			}
-			v3.Auth.Scope.Project.Domain = &defaultDomain
 		}
 	}
 
@@ -188,19 +193,13 @@ func (auth *v3Auth) Response(resp *http.Response) error {
 	return err
 }
 
-func (auth *v3Auth) endpointUrl(Type string, Internal bool) string {
+func (auth *v3Auth) endpointUrl(Type string, endpointType EndpointType) string {
 	for _, catalog := range auth.Auth.Token.Catalog {
 		if catalog.Type == Type {
 			for _, endpoint := range catalog.Endpoints {
-				if Internal {
+				if endpoint.Interface == endpointType && (auth.Region == "" || (auth.Region == endpoint.Region)) {
-					if endpoint.Interface == v3InterfaceInternal {
 					return endpoint.Url
 				}
-				} else {
-					if endpoint.Interface == v3InterfacePublic {
-						return endpoint.Url
-					}
-				}
 			}
 		}
 	}
@@ -208,7 +207,15 @@ func (auth *v3Auth) endpointUrl(Type string, Internal bool) string {
 }
 
 func (auth *v3Auth) StorageUrl(Internal bool) string {
-	return auth.endpointUrl(v3CatalogTypeObjectStore, Internal)
+	endpointType := EndpointTypePublic
+	if Internal {
+		endpointType = EndpointTypeInternal
+	}
+	return auth.StorageUrlForEndpoint(endpointType)
+}
+
+func (auth *v3Auth) StorageUrlForEndpoint(endpointType EndpointType) string {
+	return auth.endpointUrl("object-store", endpointType)
 }
 
 func (auth *v3Auth) Token() string {

vendor/github.com/ncw/swift/swift.go

@@ -92,11 +92,14 @@ type Connection struct {
 	UserAgent      string            // Http User agent (default goswift/1.0)
 	ConnectTimeout time.Duration     // Connect channel timeout (default 10s)
 	Timeout        time.Duration     // Data channel timeout (default 60s)
-	Region         string            // Region to use eg "LON", "ORD" - default is use first region (V2 auth only)
+	Region         string            // Region to use eg "LON", "ORD" - default is use first region (v2,v3 auth only)
-	AuthVersion    int               // Set to 1 or 2 or leave at 0 for autodetect
+	AuthVersion    int               // Set to 1, 2 or 3 or leave at 0 for autodetect
 	Internal       bool              // Set this to true to use the the internal / service network
-	Tenant         string            // Name of the tenant (v2 auth only)
+	Tenant         string            // Name of the tenant (v2,v3 auth only)
-	TenantId       string            // Id of the tenant (v2 auth only)
+	TenantId       string            // Id of the tenant (v2,v3 auth only)
+	EndpointType   EndpointType      // Endpoint type (v2,v3 auth only) (default is public URL unless Internal is set)
+	TenantDomain   string            // Name of the tenant's domain (v3 auth only), only needed if it differs from the user domain
+	TenantDomainId string            // Id of the tenant's domain (v3 auth only), only needed if it differs the from user domain
 	TrustId        string            // Id of the trust (v3 auth only)
 	Transport      http.RoundTripper `json:"-" xml:"-"` // Optional specialised http.Transport (eg. for Google Appengine)
 	// These are filled in after Authenticate is called as are the defaults for above
@@ -300,6 +303,7 @@ again:
 	if err != nil {
 		return
 	}
+	if req != nil {
 		timer := time.NewTimer(c.ConnectTimeout)
 		var resp *http.Response
 		resp, err = c.doTimeoutRequest(timer, req)
@@ -326,7 +330,12 @@ again:
 		if err != nil {
 			return
 		}
+	}
+	if customAuth, isCustom := c.Auth.(CustomEndpointAuthenticator); isCustom && c.EndpointType != "" {
+		c.StorageUrl = customAuth.StorageUrlForEndpoint(c.EndpointType)
+	} else {
 		c.StorageUrl = c.Auth.StorageUrl(c.Internal)
+	}
 	c.AuthToken = c.Auth.Token()
 	if !c.authenticated() {
 		err = newError(0, "Response didn't have storage url and auth token")
@@ -1424,13 +1433,13 @@ var _ io.Seeker = &ObjectOpenFile{}
 // will also check the length returned. No checking will be done if
 // you don't read all the contents.
 //
-// Note that objects with X-Object-Manifest set won't ever have their
+// Note that objects with X-Object-Manifest or X-Static-Large-Object
-// md5sum's checked as the md5sum reported on the object is actually
+// set won't ever have their md5sum's checked as the md5sum reported
-// the md5sum of the md5sums of the parts. This isn't very helpful to
+// on the object is actually the md5sum of the md5sums of the
-// detect a corrupted download as the size of the parts aren't known
+// parts. This isn't very helpful to detect a corrupted download as
-// without doing more operations.  If you want to ensure integrity of
+// the size of the parts aren't known without doing more operations.
-// an object with a manifest then you will need to download everything
+// If you want to ensure integrity of an object with a manifest then
-// in the manifest separately.
+// you will need to download everything in the manifest separately.
 //
 // headers["Content-Type"] will give the content type if desired.
 func (c *Connection) ObjectOpen(container string, objectName string, checkHash bool, h Headers) (file *ObjectOpenFile, headers Headers, err error) {
@@ -1445,8 +1454,8 @@ func (c *Connection) ObjectOpen(container string, objectName string, checkHash b
 	if err != nil {
 		return
 	}
-	// Can't check MD5 on an object with X-Object-Manifest set
+	// Can't check MD5 on an object with X-Object-Manifest or X-Static-Large-Object set
-	if checkHash && headers["X-Object-Manifest"] != "" {
+	if checkHash && (headers["X-Object-Manifest"] != "" || headers["X-Static-Large-Object"] != "") {
 		// log.Printf("swift: turning off md5 checking on object with manifest %v", objectName)
 		checkHash = false
 	}