From 3ff9f9b9ccc268ea4c32e8d405007ee0b1f63237 Mon Sep 17 00:00:00 2001 From: davidli Date: Mon, 8 Jun 2015 13:21:46 +0800 Subject: [PATCH] Update swift lib for adding the support of Openstack Identity v3 API Signed-off-by: Li Wenquan --- Godeps/Godeps.json | 2 +- .../src/github.com/ncw/swift/README.md | 23 +- .../src/github.com/ncw/swift/auth.go | 6 +- .../src/github.com/ncw/swift/auth_v3.go | 207 ++++++++++++++++ .../src/github.com/ncw/swift/swift.go | 2 + .../src/github.com/ncw/swift/swift_test.go | 223 +++++++++++++++--- .../github.com/ncw/swift/swifttest/server.go | 10 +- 7 files changed, 434 insertions(+), 39 deletions(-) create mode 100644 Godeps/_workspace/src/github.com/ncw/swift/auth_v3.go diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 453932ca8..d40401810 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -87,7 +87,7 @@ }, { "ImportPath": "github.com/ncw/swift", - "Rev": "021f1ecdb0940ce5c64ce0e27928d9680f85f291" + "Rev": "22c8fa9fb5ba145b4d4e2cebb027e84b1a7b1296" }, { "ImportPath": "github.com/yvasiyarov/go-metrics", diff --git a/Godeps/_workspace/src/github.com/ncw/swift/README.md b/Godeps/_workspace/src/github.com/ncw/swift/README.md index 379457a4c..993560eab 100644 --- a/Godeps/_workspace/src/github.com/ncw/swift/README.md +++ b/Godeps/_workspace/src/github.com/ncw/swift/README.md @@ -34,6 +34,7 @@ Here is a short example from the docs UserName: "user", ApiKey: "key", AuthUrl: "auth_url", + Domain: "domain", // Name of the domain (v3 auth only) Tenant: "tenant", // Name of the tenant (v2 auth only) } // Authenticate @@ -45,7 +46,7 @@ Here is a short example from the docs containers, err := c.ContainerNames(nil) fmt.Println(containers) // etc... - + Additions --------- @@ -69,6 +70,25 @@ And optionally these if using v2 authentication export SWIFT_TENANT='TenantName' export SWIFT_TENANT_ID='TenantId' +And optionally these if using v3 authentication + + export SWIFT_TENANT='TenantName' + export SWIFT_TENANT_ID='TenantId' + export SWIFT_API_DOMAIN_ID='domain id' + export SWIFT_API_DOMAIN='domain name' + +And optionally this if you want to skip server certificate validation + + export SWIFT_AUTH_INSECURE=1 + +And optionally this to configure the connect channel timeout, in seconds + + export SWIFT_CONNECTION_CHANNEL_TIMEOUT=60 + +And optionally this to configure the data channel timeout, in seconds + + export SWIFT_DATA_CHANNEL_TIMEOUT=60 + Then run the tests with `go test` License @@ -105,3 +125,4 @@ Contributors - lsowen - Sylvain Baubeau - Chris Kastorff +- Dai HaoJun diff --git a/Godeps/_workspace/src/github.com/ncw/swift/auth.go b/Godeps/_workspace/src/github.com/ncw/swift/auth.go index 8d2f54eaf..ca35d2379 100644 --- a/Godeps/_workspace/src/github.com/ncw/swift/auth.go +++ b/Godeps/_workspace/src/github.com/ncw/swift/auth.go @@ -29,7 +29,9 @@ type Authenticator interface { func newAuth(c *Connection) (Authenticator, error) { AuthVersion := c.AuthVersion if AuthVersion == 0 { - if strings.Contains(c.AuthUrl, "v2") { + if strings.Contains(c.AuthUrl, "v3") { + AuthVersion = 3 + } else if strings.Contains(c.AuthUrl, "v2") { AuthVersion = 2 } else if strings.Contains(c.AuthUrl, "v1") { AuthVersion = 1 @@ -47,6 +49,8 @@ func newAuth(c *Connection) (Authenticator, error) { // this is just an optimization. useApiKey: len(c.ApiKey) >= 32, }, nil + case 3: + return &v3Auth{}, nil } return nil, newErrorf(500, "Auth Version %d not supported", AuthVersion) } diff --git a/Godeps/_workspace/src/github.com/ncw/swift/auth_v3.go b/Godeps/_workspace/src/github.com/ncw/swift/auth_v3.go new file mode 100644 index 000000000..efcb77e5f --- /dev/null +++ b/Godeps/_workspace/src/github.com/ncw/swift/auth_v3.go @@ -0,0 +1,207 @@ +package swift + +import ( + "bytes" + "encoding/json" + "net/http" + "strings" +) + +const ( + v3AuthMethodToken = "token" + v3AuthMethodPassword = "password" + v3InterfacePublic = "public" + v3InterfaceInternal = "internal" + v3InterfaceAdmin = "admin" + v3CatalogTypeObjectStore = "object-store" +) + +// V3 Authentication request +// http://docs.openstack.org/developer/keystone/api_curl_examples.html +// http://developer.openstack.org/api-ref-identity-v3.html +type v3AuthRequest struct { + Auth struct { + Identity struct { + Methods []string `json:"methods"` + Password *v3AuthPassword `json:"password,omitempty"` + Token *v3AuthToken `json:"token,omitempty"` + } `json:"identity"` + Scope *v3Scope `json:"scope,omitempty"` + } `json:"auth"` +} + +type v3Scope struct { + Project *v3Project `json:"project,omitempty"` + Domain *v3Domain `json:"domain,omitempty"` +} + +type v3Domain struct { + Id string `json:"id,omitempty"` + Name string `json:"name,omitempty"` +} + +type v3Project struct { + Name string `json:"name,omitempty"` + Id string `json:"id,omitempty"` + Domain *v3Domain `json:"domain,omitempty"` +} + +type v3User struct { + Domain *v3Domain `json:"domain,omitempty"` + Id string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Password string `json:"password,omitempty"` +} + +type v3AuthToken struct { + Id string `json:"id"` +} + +type v3AuthPassword struct { + User v3User `json:"user"` +} + +// V3 Authentication response +type v3AuthResponse struct { + Token struct { + Expires_At, Issued_At string + Methods []string + Roles []map[string]string + + Project struct { + Domain struct { + Id, Name string + } + Id, Name string + } + + Catalog []struct { + Id, Namem, Type string + Endpoints []struct { + Id, Region_Id, Url, Region, Interface string + } + } + + User struct { + Id, Name string + Domain struct { + Id, Name string + Links struct { + Self string + } + } + } + + Audit_Ids []string + } +} + +type v3Auth struct { + Auth *v3AuthResponse + Headers http.Header +} + +func (auth *v3Auth) Request(c *Connection) (*http.Request, error) { + + var v3i interface{} + + v3 := v3AuthRequest{} + + if c.UserName == "" { + v3.Auth.Identity.Methods = []string{v3AuthMethodToken} + v3.Auth.Identity.Token = &v3AuthToken{Id: c.ApiKey} + } else { + v3.Auth.Identity.Methods = []string{v3AuthMethodPassword} + v3.Auth.Identity.Password = &v3AuthPassword{ + User: v3User{ + Name: c.UserName, + Password: c.ApiKey, + }, + } + + var domain *v3Domain + + if c.Domain != "" { + domain = &v3Domain{Name: c.Domain} + } else if c.DomainId != "" { + domain = &v3Domain{Id: c.DomainId} + } + v3.Auth.Identity.Password.User.Domain = domain + } + + if c.TenantId != "" || c.Tenant != "" { + + v3.Auth.Scope = &v3Scope{Project: &v3Project{}} + + if c.TenantId != "" { + v3.Auth.Scope.Project.Id = c.TenantId + } else if c.Tenant != "" { + v3.Auth.Scope.Project.Name = c.Tenant + var defaultDomain v3Domain + if c.Domain != "" { + defaultDomain = v3Domain{Name: "Default"} + } else if c.DomainId != "" { + defaultDomain = v3Domain{Id: "Default"} + } + v3.Auth.Scope.Project.Domain = &defaultDomain + } + } + + v3i = v3 + + body, err := json.Marshal(v3i) + + if err != nil { + return nil, err + } + + url := c.AuthUrl + if !strings.HasSuffix(url, "/") { + url += "/" + } + url += "tokens" + req, err := http.NewRequest("POST", url, bytes.NewBuffer(body)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + return req, nil +} + +func (auth *v3Auth) Response(resp *http.Response) error { + auth.Auth = &v3AuthResponse{} + auth.Headers = resp.Header + err := readJson(resp, auth.Auth) + return err +} + +func (auth *v3Auth) endpointUrl(Type string, Internal bool) string { + for _, catalog := range auth.Auth.Token.Catalog { + if catalog.Type == Type { + for _, endpoint := range catalog.Endpoints { + if Internal { + if endpoint.Interface == v3InterfaceInternal { + return endpoint.Url + } + } else { + if endpoint.Interface == v3InterfacePublic { + return endpoint.Url + } + } + } + } + } + return "" +} + +func (auth *v3Auth) StorageUrl(Internal bool) string { + return auth.endpointUrl(v3CatalogTypeObjectStore, Internal) +} + +func (auth *v3Auth) Token() string { + return auth.Headers.Get("X-Subject-Token") +} + +func (auth *v3Auth) CdnUrl() string { + return "" +} diff --git a/Godeps/_workspace/src/github.com/ncw/swift/swift.go b/Godeps/_workspace/src/github.com/ncw/swift/swift.go index 5def1d673..db8eba106 100644 --- a/Godeps/_workspace/src/github.com/ncw/swift/swift.go +++ b/Godeps/_workspace/src/github.com/ncw/swift/swift.go @@ -80,6 +80,8 @@ const ( type Connection struct { // Parameters - fill these in before calling Authenticate // They are all optional except UserName, ApiKey and AuthUrl + Domain string // User's domain name + DomainId string // User's domain Id UserName string // UserName for api ApiKey string // Key for api access AuthUrl string // Auth URL diff --git a/Godeps/_workspace/src/github.com/ncw/swift/swift_test.go b/Godeps/_workspace/src/github.com/ncw/swift/swift_test.go index 0dc8a4af8..57f7d9e86 100644 --- a/Godeps/_workspace/src/github.com/ncw/swift/swift_test.go +++ b/Godeps/_workspace/src/github.com/ncw/swift/swift_test.go @@ -16,22 +16,24 @@ import ( "archive/tar" "bytes" "crypto/md5" + "crypto/tls" "encoding/json" "encoding/xml" "fmt" + "github.com/ncw/swift" + "github.com/ncw/swift/swifttest" "io" "net/http" "os" + "strconv" + "strings" "sync" "testing" "time" - - "github.com/ncw/swift" - "github.com/ncw/swift/swifttest" ) var ( - c swift.Connection + c *swift.Connection srv *swifttest.SwiftServer m1 = swift.Metadata{"Hello": "1", "potato-Salad": "2"} m2 = swift.Metadata{"hello": "", "potato-salad": ""} @@ -54,36 +56,106 @@ const ( type someTransport struct{ http.Transport } -func TestTransport(t *testing.T) { +func makeConnection() (*swift.Connection, error) { var err error + UserName := os.Getenv("SWIFT_API_USER") ApiKey := os.Getenv("SWIFT_API_KEY") AuthUrl := os.Getenv("SWIFT_AUTH_URL") + + Insecure := os.Getenv("SWIFT_AUTH_INSECURE") + ConnectionChannelTimeout := os.Getenv("SWIFT_CONNECTION_CHANNEL_TIMEOUT") + DataChannelTimeout := os.Getenv("SWIFT_DATA_CHANNEL_TIMEOUT") + if UserName == "" || ApiKey == "" || AuthUrl == "" { + if srv != nil { + srv.Close() + } srv, err = swifttest.NewSwiftServer("localhost") if err != nil { - t.Fatal("Failed to create server", err) + return nil, err } + UserName = "swifttest" ApiKey = "swifttest" AuthUrl = srv.AuthURL } - tr := &someTransport{Transport: http.Transport{MaxIdleConnsPerHost: 2048}} - ct := swift.Connection{ + + transport := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + MaxIdleConnsPerHost: 2048, + } + if Insecure == "1" { + transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + } + + c := swift.Connection{ UserName: UserName, ApiKey: ApiKey, AuthUrl: AuthUrl, - Tenant: os.Getenv("SWIFT_TENANT"), - TenantId: os.Getenv("SWIFT_TENANT_ID"), - Transport: tr, + Transport: transport, ConnectTimeout: 60 * time.Second, Timeout: 60 * time.Second, } - err = ct.Authenticate() + + var timeout int64 + if ConnectionChannelTimeout != "" { + timeout, err = strconv.ParseInt(ConnectionChannelTimeout, 10, 32) + if err == nil { + c.ConnectTimeout = time.Duration(timeout) * time.Second + } + } + + if DataChannelTimeout != "" { + timeout, err = strconv.ParseInt(DataChannelTimeout, 10, 32) + if err == nil { + c.Timeout = time.Duration(timeout) * time.Second + } + } + + return &c, nil +} + +func isV3Api() bool { + AuthUrl := os.Getenv("SWIFT_AUTH_URL") + return strings.Contains(AuthUrl, "v3") +} + +func TestTransport(t *testing.T) { + var err error + + c, err = makeConnection() + if err != nil { + t.Fatal("Failed to create server", err) + } + + tr := &someTransport{ + Transport: http.Transport{ + MaxIdleConnsPerHost: 2048, + }, + } + + Insecure := os.Getenv("SWIFT_AUTH_INSECURE") + + if Insecure == "1" { + tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + } + + c.Transport = tr + + if isV3Api() { + c.Tenant = os.Getenv("SWIFT_TENANT") + c.Domain = os.Getenv("SWIFT_API_DOMAIN") + } else { + c.Tenant = os.Getenv("SWIFT_TENANT") + c.TenantId = os.Getenv("SWIFT_TENANT_ID") + } + + err = c.Authenticate() if err != nil { t.Fatal("Auth failed", err) } - if !ct.Authenticated() { + if !c.Authenticated() { t.Fatal("Not authenticated") } if srv != nil { @@ -92,27 +164,116 @@ func TestTransport(t *testing.T) { } // The following Test functions are run in order - this one must come before the others! -func TestAuthenticate(t *testing.T) { +func TestV1V2Authenticate(t *testing.T) { var err error - UserName := os.Getenv("SWIFT_API_USER") - ApiKey := os.Getenv("SWIFT_API_KEY") - AuthUrl := os.Getenv("SWIFT_AUTH_URL") - if UserName == "" || ApiKey == "" || AuthUrl == "" { - srv, err = swifttest.NewSwiftServer("localhost") - if err != nil { - t.Fatal("Failed to create server", err) - } - UserName = "swifttest" - ApiKey = "swifttest" - AuthUrl = srv.AuthURL + + if isV3Api() { + return } - c = swift.Connection{ - UserName: UserName, - ApiKey: ApiKey, - AuthUrl: AuthUrl, - Tenant: os.Getenv("SWIFT_TENANT"), - TenantId: os.Getenv("SWIFT_TENANT_ID"), + + c, err = makeConnection() + if err != nil { + t.Fatal("Failed to create server", err) } + + c.Tenant = os.Getenv("SWIFT_TENANT") + c.TenantId = os.Getenv("SWIFT_TENANT_ID") + + err = c.Authenticate() + if err != nil { + t.Fatal("Auth failed", err) + } + if !c.Authenticated() { + t.Fatal("Not authenticated") + } +} + +func TestV3AuthenticateWithDomainNameAndTenantId(t *testing.T) { + var err error + if !isV3Api() { + return + } + + c, err = makeConnection() + if err != nil { + t.Fatal("Failed to create server", err) + } + + c.TenantId = os.Getenv("SWIFT_TENANT_ID") + c.Domain = os.Getenv("SWIFT_API_DOMAIN") + + err = c.Authenticate() + if err != nil { + t.Fatal("Auth failed", err) + } + if !c.Authenticated() { + t.Fatal("Not authenticated") + } +} + +func TestV3AuthenticateWithDomainIdAndTenantId(t *testing.T) { + var err error + + if !isV3Api() { + return + } + + c, err = makeConnection() + if err != nil { + t.Fatal("Failed to create server", err) + } + + c.TenantId = os.Getenv("SWIFT_TENANT_ID") + c.DomainId = os.Getenv("SWIFT_API_DOMAIN_ID") + + err = c.Authenticate() + if err != nil { + t.Fatal("Auth failed", err) + } + if !c.Authenticated() { + t.Fatal("Not authenticated") + } +} + +func TestV3AuthenticateWithDomainNameAndTenantName(t *testing.T) { + var err error + + if !isV3Api() { + return + } + + c, err = makeConnection() + if err != nil { + t.Fatal("Failed to create server", err) + } + + c.Tenant = os.Getenv("SWIFT_TENANT") + c.Domain = os.Getenv("SWIFT_API_DOMAIN") + + err = c.Authenticate() + if err != nil { + t.Fatal("Auth failed", err) + } + if !c.Authenticated() { + t.Fatal("Not authenticated") + } +} + +func TestV3AuthenticateWithDomainIdAndTenantName(t *testing.T) { + var err error + + if !isV3Api() { + return + } + + c, err = makeConnection() + if err != nil { + t.Fatal("Failed to create server", err) + } + + c.Tenant = os.Getenv("SWIFT_TENANT") + c.DomainId = os.Getenv("SWIFT_API_DOMAIN_ID") + err = c.Authenticate() if err != nil { t.Fatal("Auth failed", err) diff --git a/Godeps/_workspace/src/github.com/ncw/swift/swifttest/server.go b/Godeps/_workspace/src/github.com/ncw/swift/swifttest/server.go index 3e38efd1d..78c07da41 100644 --- a/Godeps/_workspace/src/github.com/ncw/swift/swifttest/server.go +++ b/Godeps/_workspace/src/github.com/ncw/swift/swifttest/server.go @@ -444,7 +444,7 @@ func (objr objectResource) get(a *action) interface{} { if start >= cursor+length { continue } - segments = append(segments, bytes.NewReader(obj.data[max(0, start - cursor):])) + segments = append(segments, bytes.NewReader(obj.data[max(0, start-cursor):])) cursor += length } } @@ -452,7 +452,7 @@ func (objr objectResource) get(a *action) interface{} { if end == -1 { end = size } - reader = io.LimitReader(io.MultiReader(segments...), int64(end - start)) + reader = io.LimitReader(io.MultiReader(segments...), int64(end-start)) } else { if end == -1 { end = len(obj.data) @@ -461,7 +461,7 @@ func (objr objectResource) get(a *action) interface{} { reader = bytes.NewReader(obj.data[start:end]) } - h.Set("Content-Length", fmt.Sprint(end - start)) + h.Set("Content-Length", fmt.Sprint(end-start)) h.Set("ETag", hex.EncodeToString(etag)) h.Set("Last-Modified", obj.mtime.Format(http.TimeFormat)) @@ -519,7 +519,7 @@ func (objr objectResource) put(a *action) interface{} { var content_type string if content_type = a.req.Header.Get("Content-Type"); content_type == "" { - content_type := mime.TypeByExtension(obj.name) + content_type = mime.TypeByExtension(obj.name) if content_type == "" { content_type = "application/octet-stream" } @@ -880,6 +880,6 @@ func NewSwiftServer(address string) (*SwiftServer, error) { return server, nil } -func (srv SwiftServer) Close() { +func (srv *SwiftServer) Close() { srv.Listener.Close() }