dkirillov/distribution
1
0
Fork 0
forked from TrueCloudLab/distribution
Code Pull requests Activity

Merge pull request #1739 from cezarsa/master

Browse source 
[Swift] Expose EndpointType parameter in driver
This commit is contained in:
Richard Scothern 2016-06-15 10:33:48 -07:00 committed by GitHub
commit edd7cb5249
9 changed files with 140 additions and 218 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
Godeps/Godeps.json generated
View file

@ -1,7 +1,7 @@
{ {
"ImportPath": "github.com/docker/distribution", "ImportPath": "github.com/docker/distribution",
"GoVersion": "go1.6", "GoVersion": "go1.6",
"GodepVersion": "v60", "GodepVersion": "v70",
"Packages": [ "Packages": [
"./..." "./..."
], ],
@ -223,11 +223,11 @@
}, },
{ {
"ImportPath": "github.com/ncw/swift", "ImportPath": "github.com/ncw/swift",
"Rev": "c54732e87b0b283d1baf0a18db689d0aea460ba3" "Rev": "ce444d6d47c51d4dda9202cd38f5094dd8e27e86"
}, },
{ {
"ImportPath": "github.com/ncw/swift/swifttest", "ImportPath": "github.com/ncw/swift/swifttest",
"Rev": "c54732e87b0b283d1baf0a18db689d0aea460ba3" "Rev": "ce444d6d47c51d4dda9202cd38f5094dd8e27e86"
}, },
{ {
"ImportPath": "github.com/spf13/cobra", "ImportPath": "github.com/spf13/cobra",

161
docs/storage-drivers/swift.md
View file

@ -187,169 +187,28 @@ An implementation of the `storagedriver.StorageDriver` interface that uses [Open
The access key to generate temporary URLs. It is used by HP Cloud Object Storage in addition to the `secretkey` parameter. The access key to generate temporary URLs. It is used by HP Cloud Object Storage in addition to the `secretkey` parameter.
</td> </td>
</tr> </tr>
</table> <tr>
<table>
<tr>
<td>
<code>authurl</code>
</td>
<td>
<p>URL for obtaining an auth token.</p>
</td>
</tr>
<tr>
<td>
<code>username</code>
</td>
<td>
<p>
Your OpenStack user name.</p>
</p>
</td>
</tr>
<tr>
<td>
<code>password</code>
<p>
</td>
<td>
<p>
Your OpenStack password.
</p>
</td>
</tr>
<tr>
<td>
<code>container</code>
</td>
<td>
<p>
The name of your Swift container where you wish to store the registry's data. The driver creates the named container during its initialization.
</p>
</td>
</tr>
<tr>
<td>
<code>tenant</code>
</td>
<td>
<p>
Optionally, your OpenStack tenant name. You can either use <code>tenant</code> or <code>tenantid</code>.
</p>
</td>
</tr>
<tr>
<td> <td>
<code>tenantid</code> <code>authversion</code>
</td> </td>
<td> <td>
<p> no
Optionally, your OpenStack tenant id. You can either use <code>tenant</code> or <code>tenantid</code>.
</p>
</td>
</tr>
<tr>
<td>
<code>domain</code>
</td> </td>
<td> <td>
<p> Specify the OpenStack Auth's version,for example <code>3</code>. By default the driver will autodetect the auth's version from the AuthURL.
Optionally, your OpenStack domain name for Identity v3 API. You can either use <code>domain</code> or <code>domainid</code>.
</p>
</td> </td>
</tr> </tr>
<tr> <tr>
<td> <td>
<code>domainid</code> <code>endpointtype</code>
</td> </td>
<td> <td>
<p> no
Optionally, your OpenStack domain id for Identity v3 API. You can either use <code>domain</code> or <code>domainid</code>.
</p>
</td>
</tr>
<tr>
<td>
<code>trustid</code>
</td> </td>
<td> <td>
<p> The endpoint type used when connecting to swift. Possible values are `public`, `internal` and `admin`. Default is `public`.
Optionally, your OpenStack trust id for Identity v3 API.
</p>
</td> </td>
</tr> </tr>
<tr>
<td>
<code>insecureskipverify</code>
</td>
<td>
<p>
Optionally, set <code>insecureskipverify</code> to true to skip TLS verification for your OpenStack provider. The driver uses false by default.
</p>
</td>
</tr>
<tr>
<td>
<code>region</code>
</td>
<td>
<p>
Optionally, specify the OpenStack region name in which you would like to store objects (for example <code>fr</code>).
</p>
</td>
</tr>
<tr>
<td>
<code>authversion</code>
</td>
<td>
<p>
Optionally, specify the OpenStack Auth's version,for example <code>3</code>. By default the driver will autodetect the auth's version from the AuthURL.
</p>
</td>
</tr>
<tr>
<td>
<code>chunksize</code>
</td>
<td>
<p>
Optionally, specify the segment size for Dynamic Large Objects uploads (performed by WriteStream) to Swift. The default is 5 MB. You might experience better performance for larger chunk sizes depending on the speed of your connection to Swift.
</p>
</td>
</tr>
<tr>
<td>
<code>prefix</code>
</td>
<td>
<p>
Optionally, supply a prefix that will be applied to all Swift keys to allow you to segment data in your container if necessary. Defaults to the empty string which is the container's root.</p>
</p>
</td>
</tr>
<tr>
<td>
<code>secretkey</code>
</td>
<td>
<p>
Optionally, the secret key used to generate temporary URLs.</p>
</p>
</td>
</tr>
<tr>
<td>
<code>accesskey</code>
</td>
<td>
<p>
Optionally, the access key to generate temporary URLs. It is used by HP Cloud Object Storage in addition to the `secretkey` parameter.</p>
</p>
</td>
</tr>
</table> </table>
The features supported by the Swift server are queried by requesting the `/info` URL on the server. In case the administrator The features supported by the Swift server are queried by requesting the `/info` URL on the server. In case the administrator

2
registry/storage/driver/swift/swift.go
View file

@ -72,6 +72,7 @@ type Parameters struct {
AuthVersion int AuthVersion int
Container string Container string
Prefix string Prefix string
EndpointType string
InsecureSkipVerify bool InsecureSkipVerify bool
ChunkSize int ChunkSize int
SecretKey string SecretKey string
@ -182,6 +183,7 @@ func New(params Parameters) (*Driver, error) {
Domain: params.Domain, Domain: params.Domain,
DomainId: params.DomainID, DomainId: params.DomainID,
TrustId: params.TrustID, TrustId: params.TrustID,
EndpointType: swift.EndpointType(params.EndpointType),
Transport: transport, Transport: transport,
ConnectTimeout: 60 * time.Second, ConnectTimeout: 60 * time.Second,
Timeout: 15 * 60 * time.Second, Timeout: 15 * 60 * time.Second,

3
registry/storage/driver/swift/swift_test.go
View file

@ -34,6 +34,7 @@ func init() {
container string container string
region string region string
AuthVersion int AuthVersion int
endpointType string
insecureSkipVerify bool insecureSkipVerify bool
secretKey string secretKey string
accessKey string accessKey string
@ -54,6 +55,7 @@ func init() {
container = os.Getenv("SWIFT_CONTAINER_NAME") container = os.Getenv("SWIFT_CONTAINER_NAME")
region = os.Getenv("SWIFT_REGION_NAME") region = os.Getenv("SWIFT_REGION_NAME")
AuthVersion, _ = strconv.Atoi(os.Getenv("SWIFT_AUTH_VERSION")) AuthVersion, _ = strconv.Atoi(os.Getenv("SWIFT_AUTH_VERSION"))
endpointType = os.Getenv("SWIFT_ENDPOINT_TYPE")
insecureSkipVerify, _ = strconv.ParseBool(os.Getenv("SWIFT_INSECURESKIPVERIFY")) insecureSkipVerify, _ = strconv.ParseBool(os.Getenv("SWIFT_INSECURESKIPVERIFY"))
secretKey = os.Getenv("SWIFT_SECRET_KEY") secretKey = os.Getenv("SWIFT_SECRET_KEY")
accessKey = os.Getenv("SWIFT_ACCESS_KEY") accessKey = os.Getenv("SWIFT_ACCESS_KEY")
@ -90,6 +92,7 @@ func init() {
AuthVersion, AuthVersion,
container, container,
root, root,
endpointType,
insecureSkipVerify, insecureSkipVerify,
defaultChunkSize, defaultChunkSize,
secretKey, secretKey,

5
vendor/github.com/ncw/swift/.travis.yml generated vendored
View file

@ -4,8 +4,11 @@ sudo: false
go: go:
- 1.1.2 - 1.1.2
- 1.2.2 - 1.2.2
- 1.3 - 1.3.3
- 1.4.2
- 1.5.1
- tip - tip
script: script:
- test -z "$(go fmt ./...)"
- go test - go test

5
vendor/github.com/ncw/swift/README.md generated vendored
View file

@ -9,7 +9,7 @@ See here for package docs
http://godoc.org/github.com/ncw/swift http://godoc.org/github.com/ncw/swift
[![Build Status](https://travis-ci.org/ncw/swift.png)](https://travis-ci.org/ncw/swift) [![Build Status](https://api.travis-ci.org/ncw/swift.svg?branch=master)](https://travis-ci.org/ncw/swift) [![GoDoc](https://godoc.org/github.com/ncw/swift?status.svg)](https://godoc.org/github.com/ncw/swift)
Install Install
------- -------
@ -134,3 +134,6 @@ Contributors
- Fabian Ruff <fabian@progra.de> - Fabian Ruff <fabian@progra.de>
- Arturo Reuschenbach Puncernau <reuschenbach@gmail.com> - Arturo Reuschenbach Puncernau <reuschenbach@gmail.com>
- Petr Kotek <petr.kotek@bigcommerce.com> - Petr Kotek <petr.kotek@bigcommerce.com>
- Stefan Majewsky <stefan.majewsky@sap.com>
- Cezar Sa Espinola <cezarsa@gmail.com>
- Sam Gunaratne <samgzeit@gmail.com>

46
vendor/github.com/ncw/swift/auth.go generated vendored
View file

@ -12,7 +12,9 @@ import (
// //
// This encapsulates the different authentication schemes in use // This encapsulates the different authentication schemes in use
type Authenticator interface { type Authenticator interface {
// Request creates an http.Request for the auth - return nil if not needed
Request(*Connection) (*http.Request, error) Request(*Connection) (*http.Request, error)
// Response parses the http.Response
Response(resp *http.Response) error Response(resp *http.Response) error
// The public storage URL - set Internal to true to read // The public storage URL - set Internal to true to read
// internal/service net URL // internal/service net URL
@ -23,6 +25,23 @@ type Authenticator interface {
CdnUrl() string CdnUrl() string
} }
type CustomEndpointAuthenticator interface {
StorageUrlForEndpoint(endpointType EndpointType) string
}
type EndpointType string
const (
// Use public URL as storage URL
EndpointTypePublic = EndpointType("public")
// Use internal URL as storage URL
EndpointTypeInternal = EndpointType("internal")
// Use admin URL as storage URL
EndpointTypeAdmin = EndpointType("admin")
)
// newAuth - create a new Authenticator from the AuthUrl // newAuth - create a new Authenticator from the AuthUrl
// //
// A hint for AuthVersion can be provided // A hint for AuthVersion can be provided
@ -175,15 +194,20 @@ func (auth *v2Auth) Response(resp *http.Response) error {
// Region if set or defaulting to the first one if not // Region if set or defaulting to the first one if not
// //
// Returns "" if not found // Returns "" if not found
func (auth *v2Auth) endpointUrl(Type string, Internal bool) string { func (auth *v2Auth) endpointUrl(Type string, endpointType EndpointType) string {
for _, catalog := range auth.Auth.Access.ServiceCatalog { for _, catalog := range auth.Auth.Access.ServiceCatalog {
if catalog.Type == Type { if catalog.Type == Type {
for _, endpoint := range catalog.Endpoints { for _, endpoint := range catalog.Endpoints {
if auth.Region == "" || (auth.Region == endpoint.Region) { if auth.Region == "" || (auth.Region == endpoint.Region) {
if Internal { switch endpointType {
case EndpointTypeInternal:
return endpoint.InternalUrl return endpoint.InternalUrl
} else { case EndpointTypePublic:
return endpoint.PublicUrl return endpoint.PublicUrl
case EndpointTypeAdmin:
return endpoint.AdminUrl
default:
return ""
} }
} }
} }
@ -197,7 +221,18 @@ func (auth *v2Auth) endpointUrl(Type string, Internal bool) string {
// If Internal is true then it reads the private (internal / service // If Internal is true then it reads the private (internal / service
// net) URL. // net) URL.
func (auth *v2Auth) StorageUrl(Internal bool) string { func (auth *v2Auth) StorageUrl(Internal bool) string {
return auth.endpointUrl("object-store", Internal) endpointType := EndpointTypePublic
if Internal {
endpointType = EndpointTypeInternal
}
return auth.StorageUrlForEndpoint(endpointType)
}
// v2 Authentication - read storage url
//
// Use the indicated endpointType to choose a URL.
func (auth *v2Auth) StorageUrlForEndpoint(endpointType EndpointType) string {
return auth.endpointUrl("object-store", endpointType)
} }
// v2 Authentication - read auth token // v2 Authentication - read auth token
@ -207,7 +242,7 @@ func (auth *v2Auth) Token() string {
// v2 Authentication - read cdn url // v2 Authentication - read cdn url
func (auth *v2Auth) CdnUrl() string { func (auth *v2Auth) CdnUrl() string {
return auth.endpointUrl("rax:object-cdn", false) return auth.endpointUrl("rax:object-cdn", EndpointTypePublic)
} }
// ------------------------------------------------------------ // ------------------------------------------------------------
@ -255,6 +290,7 @@ type v2AuthResponse struct {
Endpoints []struct { Endpoints []struct {
InternalUrl string InternalUrl string
PublicUrl string PublicUrl string
AdminUrl string
Region string Region string
TenantId string TenantId string
} }

47
vendor/github.com/ncw/swift/auth_v3.go generated vendored
View file

@ -10,9 +10,6 @@ import (
const ( const (
v3AuthMethodToken = "token" v3AuthMethodToken = "token"
v3AuthMethodPassword = "password" v3AuthMethodPassword = "password"
v3InterfacePublic = "public"
v3InterfaceInternal = "internal"
v3InterfaceAdmin = "admin"
v3CatalogTypeObjectStore = "object-store" v3CatalogTypeObjectStore = "object-store"
) )
@ -88,7 +85,8 @@ type v3AuthResponse struct {
Catalog []struct { Catalog []struct {
Id, Namem, Type string Id, Namem, Type string
Endpoints []struct { Endpoints []struct {
Id, Region_Id, Url, Region, Interface string Id, Region_Id, Url, Region string
Interface EndpointType
} }
} }
@ -107,11 +105,13 @@ type v3AuthResponse struct {
} }
type v3Auth struct { type v3Auth struct {
Region string
Auth *v3AuthResponse Auth *v3AuthResponse
Headers http.Header Headers http.Header
} }
func (auth *v3Auth) Request(c *Connection) (*http.Request, error) { func (auth *v3Auth) Request(c *Connection) (*http.Request, error) {
auth.Region = c.Region
var v3i interface{} var v3i interface{}
@ -149,13 +149,18 @@ func (auth *v3Auth) Request(c *Connection) (*http.Request, error) {
v3.Auth.Scope.Project.Id = c.TenantId v3.Auth.Scope.Project.Id = c.TenantId
} else if c.Tenant != "" { } else if c.Tenant != "" {
v3.Auth.Scope.Project.Name = c.Tenant v3.Auth.Scope.Project.Name = c.Tenant
var defaultDomain v3Domain switch {
if c.Domain != "" { case c.TenantDomain != "":
defaultDomain = v3Domain{Name: "Default"} v3.Auth.Scope.Project.Domain = &v3Domain{Name: c.TenantDomain}
} else if c.DomainId != "" { case c.TenantDomainId != "":
defaultDomain = v3Domain{Id: "Default"} v3.Auth.Scope.Project.Domain = &v3Domain{Id: c.TenantDomainId}
case c.Domain != "":
v3.Auth.Scope.Project.Domain = &v3Domain{Name: c.Domain}
case c.DomainId != "":
v3.Auth.Scope.Project.Domain = &v3Domain{Id: c.DomainId}
default:
v3.Auth.Scope.Project.Domain = &v3Domain{Name: "Default"}
} }
v3.Auth.Scope.Project.Domain = &defaultDomain
} }
} }
@ -188,18 +193,12 @@ func (auth *v3Auth) Response(resp *http.Response) error {
return err return err
} }
func (auth *v3Auth) endpointUrl(Type string, Internal bool) string { func (auth *v3Auth) endpointUrl(Type string, endpointType EndpointType) string {
for _, catalog := range auth.Auth.Token.Catalog { for _, catalog := range auth.Auth.Token.Catalog {
if catalog.Type == Type { if catalog.Type == Type {
for _, endpoint := range catalog.Endpoints { for _, endpoint := range catalog.Endpoints {
if Internal { if endpoint.Interface == endpointType && (auth.Region == "" || (auth.Region == endpoint.Region)) {
if endpoint.Interface == v3InterfaceInternal { return endpoint.Url
return endpoint.Url
}
} else {
if endpoint.Interface == v3InterfacePublic {
return endpoint.Url
}
} }
} }
} }
@ -208,7 +207,15 @@ func (auth *v3Auth) endpointUrl(Type string, Internal bool) string {
} }
func (auth *v3Auth) StorageUrl(Internal bool) string { func (auth *v3Auth) StorageUrl(Internal bool) string {
return auth.endpointUrl(v3CatalogTypeObjectStore, Internal) endpointType := EndpointTypePublic
if Internal {
endpointType = EndpointTypeInternal
}
return auth.StorageUrlForEndpoint(endpointType)
}
func (auth *v3Auth) StorageUrlForEndpoint(endpointType EndpointType) string {
return auth.endpointUrl("object-store", endpointType)
} }
func (auth *v3Auth) Token() string { func (auth *v3Auth) Token() string {

83
vendor/github.com/ncw/swift/swift.go generated vendored
View file

@ -92,11 +92,14 @@ type Connection struct {
UserAgent string // Http User agent (default goswift/1.0) UserAgent string // Http User agent (default goswift/1.0)
ConnectTimeout time.Duration // Connect channel timeout (default 10s) ConnectTimeout time.Duration // Connect channel timeout (default 10s)
Timeout time.Duration // Data channel timeout (default 60s) Timeout time.Duration // Data channel timeout (default 60s)
Region string // Region to use eg "LON", "ORD" - default is use first region (V2 auth only) Region string // Region to use eg "LON", "ORD" - default is use first region (v2,v3 auth only)
AuthVersion int // Set to 1 or 2 or leave at 0 for autodetect AuthVersion int // Set to 1, 2 or 3 or leave at 0 for autodetect
Internal bool // Set this to true to use the the internal / service network Internal bool // Set this to true to use the the internal / service network
Tenant string // Name of the tenant (v2 auth only) Tenant string // Name of the tenant (v2,v3 auth only)
TenantId string // Id of the tenant (v2 auth only) TenantId string // Id of the tenant (v2,v3 auth only)
EndpointType EndpointType // Endpoint type (v2,v3 auth only) (default is public URL unless Internal is set)
TenantDomain string // Name of the tenant's domain (v3 auth only), only needed if it differs from the user domain
TenantDomainId string // Id of the tenant's domain (v3 auth only), only needed if it differs the from user domain
TrustId string // Id of the trust (v3 auth only) TrustId string // Id of the trust (v3 auth only)
Transport http.RoundTripper `json:"-" xml:"-"` // Optional specialised http.Transport (eg. for Google Appengine) Transport http.RoundTripper `json:"-" xml:"-"` // Optional specialised http.Transport (eg. for Google Appengine)
// These are filled in after Authenticate is called as are the defaults for above // These are filled in after Authenticate is called as are the defaults for above
@ -300,33 +303,39 @@ again:
if err != nil { if err != nil {
return return
} }
timer := time.NewTimer(c.ConnectTimeout) if req != nil {
var resp *http.Response timer := time.NewTimer(c.ConnectTimeout)
resp, err = c.doTimeoutRequest(timer, req) var resp *http.Response
if err != nil { resp, err = c.doTimeoutRequest(timer, req)
return if err != nil {
} return
defer func() { }
checkClose(resp.Body, &err) defer func() {
// Flush the auth connection - we don't want to keep checkClose(resp.Body, &err)
// it open if keepalives were enabled // Flush the auth connection - we don't want to keep
flushKeepaliveConnections(c.Transport) // it open if keepalives were enabled
}() flushKeepaliveConnections(c.Transport)
if err = c.parseHeaders(resp, authErrorMap); err != nil { }()
// Try again for a limited number of times on if err = c.parseHeaders(resp, authErrorMap); err != nil {
// AuthorizationFailed or BadRequest. This allows us // Try again for a limited number of times on
// to try some alternate forms of the request // AuthorizationFailed or BadRequest. This allows us
if (err == AuthorizationFailed || err == BadRequest) && retries > 0 { // to try some alternate forms of the request
retries-- if (err == AuthorizationFailed || err == BadRequest) && retries > 0 {
goto again retries--
goto again
}
return
}
err = c.Auth.Response(resp)
if err != nil {
return
} }
return
} }
err = c.Auth.Response(resp) if customAuth, isCustom := c.Auth.(CustomEndpointAuthenticator); isCustom && c.EndpointType != "" {
if err != nil { c.StorageUrl = customAuth.StorageUrlForEndpoint(c.EndpointType)
return } else {
c.StorageUrl = c.Auth.StorageUrl(c.Internal)
} }
c.StorageUrl = c.Auth.StorageUrl(c.Internal)
c.AuthToken = c.Auth.Token() c.AuthToken = c.Auth.Token()
if !c.authenticated() { if !c.authenticated() {
err = newError(0, "Response didn't have storage url and auth token") err = newError(0, "Response didn't have storage url and auth token")
@ -1424,13 +1433,13 @@ var _ io.Seeker = &ObjectOpenFile{}
// will also check the length returned. No checking will be done if // will also check the length returned. No checking will be done if
// you don't read all the contents. // you don't read all the contents.
// //
// Note that objects with X-Object-Manifest set won't ever have their // Note that objects with X-Object-Manifest or X-Static-Large-Object
// md5sum's checked as the md5sum reported on the object is actually // set won't ever have their md5sum's checked as the md5sum reported
// the md5sum of the md5sums of the parts. This isn't very helpful to // on the object is actually the md5sum of the md5sums of the
// detect a corrupted download as the size of the parts aren't known // parts. This isn't very helpful to detect a corrupted download as
// without doing more operations. If you want to ensure integrity of // the size of the parts aren't known without doing more operations.
// an object with a manifest then you will need to download everything // If you want to ensure integrity of an object with a manifest then
// in the manifest separately. // you will need to download everything in the manifest separately.
// //
// headers["Content-Type"] will give the content type if desired. // headers["Content-Type"] will give the content type if desired.
func (c *Connection) ObjectOpen(container string, objectName string, checkHash bool, h Headers) (file *ObjectOpenFile, headers Headers, err error) { func (c *Connection) ObjectOpen(container string, objectName string, checkHash bool, h Headers) (file *ObjectOpenFile, headers Headers, err error) {
@ -1445,8 +1454,8 @@ func (c *Connection) ObjectOpen(container string, objectName string, checkHash b
if err != nil { if err != nil {
return return
} }
// Can't check MD5 on an object with X-Object-Manifest set // Can't check MD5 on an object with X-Object-Manifest or X-Static-Large-Object set
if checkHash && headers["X-Object-Manifest"] != "" { if checkHash && (headers["X-Object-Manifest"] != "" || headers["X-Static-Large-Object"] != "") {
// log.Printf("swift: turning off md5 checking on object with manifest %v", objectName) // log.Printf("swift: turning off md5 checking on object with manifest %v", objectName)
checkHash = false checkHash = false
} }