From ddb689ef8b29ed2f15a49d56649380cd263e39b4 Mon Sep 17 00:00:00 2001 From: Hua Wang Date: Mon, 24 Aug 2015 23:08:33 +0800 Subject: [PATCH] Add TrustId parameter to swift driver github/ncw/swift has added support for trust, so let's add it. Signed-off-by: Hua Wang --- Godeps/Godeps.json | 2 +- .../src/github.com/ncw/swift/.travis.yml | 1 + .../src/github.com/ncw/swift/README.md | 6 + .../src/github.com/ncw/swift/auth_v3.go | 18 ++- .../src/github.com/ncw/swift/swift.go | 20 ++- .../src/github.com/ncw/swift/swift_test.go | 66 +++++++++- .../github.com/ncw/swift/swifttest/server.go | 122 ++++++++++++------ docs/configuration.md | 11 ++ registry/storage/driver/swift/swift.go | 2 + registry/storage/driver/swift/swift_test.go | 3 + 10 files changed, 201 insertions(+), 50 deletions(-) diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 355596dff..8f1ecf0dc 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -96,7 +96,7 @@ }, { "ImportPath": "github.com/ncw/swift", - "Rev": "22c8fa9fb5ba145b4d4e2cebb027e84b1a7b1296" + "Rev": "ca8cbbde50d4e12dd8ad70b1bd66589ae98efc5c" }, { "ImportPath": "github.com/yvasiyarov/go-metrics", diff --git a/Godeps/_workspace/src/github.com/ncw/swift/.travis.yml b/Godeps/_workspace/src/github.com/ncw/swift/.travis.yml index 391b1d191..4e7c3b3b2 100644 --- a/Godeps/_workspace/src/github.com/ncw/swift/.travis.yml +++ b/Godeps/_workspace/src/github.com/ncw/swift/.travis.yml @@ -1,4 +1,5 @@ language: go +sudo: false go: - 1.1.2 diff --git a/Godeps/_workspace/src/github.com/ncw/swift/README.md b/Godeps/_workspace/src/github.com/ncw/swift/README.md index 993560eab..7d9b1be13 100644 --- a/Godeps/_workspace/src/github.com/ncw/swift/README.md +++ b/Godeps/_workspace/src/github.com/ncw/swift/README.md @@ -77,6 +77,10 @@ And optionally these if using v3 authentication export SWIFT_API_DOMAIN_ID='domain id' export SWIFT_API_DOMAIN='domain name' +And optionally these if using v3 trust + + export SWIFT_TRUST_ID='TrustId' + And optionally this if you want to skip server certificate validation export SWIFT_AUTH_INSECURE=1 @@ -126,3 +130,5 @@ Contributors - Sylvain Baubeau - Chris Kastorff - Dai HaoJun +- Hua Wang +- Fabian Ruff diff --git a/Godeps/_workspace/src/github.com/ncw/swift/auth_v3.go b/Godeps/_workspace/src/github.com/ncw/swift/auth_v3.go index efcb77e5f..7c375d224 100644 --- a/Godeps/_workspace/src/github.com/ncw/swift/auth_v3.go +++ b/Godeps/_workspace/src/github.com/ncw/swift/auth_v3.go @@ -33,6 +33,7 @@ type v3AuthRequest struct { type v3Scope struct { Project *v3Project `json:"project,omitempty"` Domain *v3Domain `json:"domain,omitempty"` + Trust *v3Trust `json:"OS-TRUST:trust,omitempty"` } type v3Domain struct { @@ -46,6 +47,10 @@ type v3Project struct { Domain *v3Domain `json:"domain,omitempty"` } +type v3Trust struct { + Id string `json:"id"` +} + type v3User struct { Domain *v3Domain `json:"domain,omitempty"` Id string `json:"id,omitempty"` @@ -66,7 +71,12 @@ type v3AuthResponse struct { Token struct { Expires_At, Issued_At string Methods []string - Roles []map[string]string + Roles []struct { + Id, Name string + Links struct { + Self string + } + } Project struct { Domain struct { @@ -129,7 +139,9 @@ func (auth *v3Auth) Request(c *Connection) (*http.Request, error) { v3.Auth.Identity.Password.User.Domain = domain } - if c.TenantId != "" || c.Tenant != "" { + if c.TrustId != "" { + v3.Auth.Scope = &v3Scope{Trust: &v3Trust{Id: c.TrustId}} + } else if c.TenantId != "" || c.Tenant != "" { v3.Auth.Scope = &v3Scope{Project: &v3Project{}} @@ -159,7 +171,7 @@ func (auth *v3Auth) Request(c *Connection) (*http.Request, error) { if !strings.HasSuffix(url, "/") { url += "/" } - url += "tokens" + url += "auth/tokens" req, err := http.NewRequest("POST", url, bytes.NewBuffer(body)) if err != nil { return nil, err diff --git a/Godeps/_workspace/src/github.com/ncw/swift/swift.go b/Godeps/_workspace/src/github.com/ncw/swift/swift.go index db8eba106..003e10535 100644 --- a/Godeps/_workspace/src/github.com/ncw/swift/swift.go +++ b/Godeps/_workspace/src/github.com/ncw/swift/swift.go @@ -3,7 +3,10 @@ package swift import ( "bufio" "bytes" + "crypto/hmac" "crypto/md5" + "crypto/sha1" + "encoding/hex" "encoding/json" "fmt" "hash" @@ -94,6 +97,7 @@ type Connection struct { Internal bool // Set this to true to use the the internal / service network Tenant string // Name of the tenant (v2 auth only) TenantId string // Id of the tenant (v2 auth only) + TrustId string // Id of the trust (v3 auth only) Transport http.RoundTripper `json:"-" xml:"-"` // Optional specialised http.Transport (eg. for Google Appengine) // These are filled in after Authenticate is called as are the defaults for above StorageUrl string @@ -1422,8 +1426,10 @@ func (c *Connection) ObjectOpen(container string, objectName string, checkHash b file.body = io.TeeReader(resp.Body, file.hash) } // Read Content-Length - file.length, err = getInt64FromHeader(resp, "Content-Length") - file.lengthOk = (err == nil) + if resp.Header.Get("Content-Length") != "" { + file.length, err = getInt64FromHeader(resp, "Content-Length") + file.lengthOk = (err == nil) + } return } @@ -1479,6 +1485,16 @@ func (c *Connection) ObjectDelete(container string, objectName string) error { return err } +// ObjectTempUrl returns a temporary URL for an object +func (c *Connection) ObjectTempUrl(container string, objectName string, secretKey string, method string, expires time.Time) string { + mac := hmac.New(sha1.New, []byte(secretKey)) + prefix, _ := url.Parse(c.StorageUrl) + body := fmt.Sprintf("%s\n%d\n%s/%s/%s", method, expires.Unix(), prefix.Path, container, objectName) + mac.Write([]byte(body)) + sig := hex.EncodeToString(mac.Sum(nil)) + return fmt.Sprintf("%s/%s/%s?temp_url_sig=%s&temp_url_expires=%d", c.StorageUrl, container, objectName, sig, expires.Unix()) +} + // parseResponseStatus parses string like "200 OK" and returns Error. // // For status codes beween 200 and 299, this returns nil. diff --git a/Godeps/_workspace/src/github.com/ncw/swift/swift_test.go b/Godeps/_workspace/src/github.com/ncw/swift/swift_test.go index 57f7d9e86..87bc7a5d7 100644 --- a/Godeps/_workspace/src/github.com/ncw/swift/swift_test.go +++ b/Godeps/_workspace/src/github.com/ncw/swift/swift_test.go @@ -20,9 +20,8 @@ import ( "encoding/json" "encoding/xml" "fmt" - "github.com/ncw/swift" - "github.com/ncw/swift/swifttest" "io" + "io/ioutil" "net/http" "os" "strconv" @@ -30,6 +29,9 @@ import ( "sync" "testing" "time" + + "github.com/ncw/swift" + "github.com/ncw/swift/swifttest" ) var ( @@ -52,6 +54,7 @@ const ( CONTENT_SIZE = int64(len(CONTENTS)) CONTENT_MD5 = "827ccb0eea8a706c4c34a16891f84e7b" EMPTY_MD5 = "d41d8cd98f00b204e9800998ecf8427e" + SECRET_KEY = "b3968d0207b54ece87cccc06515a89d4" ) type someTransport struct{ http.Transport } @@ -211,6 +214,28 @@ func TestV3AuthenticateWithDomainNameAndTenantId(t *testing.T) { } } +func TestV3TrustWithTrustId(t *testing.T) { + var err error + if !isV3Api() { + return + } + + c, err = makeConnection() + if err != nil { + t.Fatal("Failed to create server", err) + } + + c.TrustId = os.Getenv("SWIFT_TRUST_ID") + + err = c.Authenticate() + if err != nil { + t.Fatal("Auth failed", err) + } + if !c.Authenticated() { + t.Fatal("Not authenticated") + } +} + func TestV3AuthenticateWithDomainIdAndTenantId(t *testing.T) { var err error @@ -1441,6 +1466,43 @@ func TestObjectDifficultName(t *testing.T) { } } +func TestTempUrl(t *testing.T) { + err := c.ObjectPutBytes(CONTAINER, OBJECT, []byte(CONTENTS), "") + if err != nil { + t.Fatal(err) + } + + m := swift.Metadata{} + m["temp-url-key"] = SECRET_KEY + err = c.AccountUpdate(m.AccountHeaders()) + if err != nil { + t.Fatal(err) + } + + expiresTime := time.Now().Add(20 * time.Minute) + tempUrl := c.ObjectTempUrl(CONTAINER, OBJECT, SECRET_KEY, "GET", expiresTime) + resp, err := http.Get(tempUrl) + if err != nil { + t.Fatal("Failed to retrieve file from temporary url") + } + if resp.StatusCode == 401 { + t.Log("Server doesn't support tempurl") + } else if resp.StatusCode != 200 { + t.Fatal("HTTP Error retrieving file from temporary url", resp.StatusCode) + } else { + if content, err := ioutil.ReadAll(resp.Body); err != nil || string(content) != CONTENTS { + t.Error("Bad content", err) + } + } + + resp.Body.Close() + err = c.ObjectDelete(CONTAINER, OBJECT) + if err != nil { + t.Fatal(err) + } + +} + func TestContainerDelete(t *testing.T) { err := c.ContainerDelete(CONTAINER) if err != nil { diff --git a/Godeps/_workspace/src/github.com/ncw/swift/swifttest/server.go b/Godeps/_workspace/src/github.com/ncw/swift/swifttest/server.go index 78c07da41..587c9bad6 100644 --- a/Godeps/_workspace/src/github.com/ncw/swift/swifttest/server.go +++ b/Godeps/_workspace/src/github.com/ncw/swift/swifttest/server.go @@ -8,8 +8,10 @@ package swifttest import ( "bytes" + "crypto/hmac" "crypto/md5" "crypto/rand" + "crypto/sha1" "encoding/hex" "encoding/json" "fmt" @@ -33,19 +35,19 @@ import ( ) const ( - DEBUG = false + DEBUG = false + TEST_ACCOUNT = "swifttest" ) type SwiftServer struct { - t *testing.T - reqId int - mu sync.Mutex - Listener net.Listener - AuthURL string - URL string - Containers map[string]*container - Accounts map[string]*account - Sessions map[string]*session + t *testing.T + reqId int + mu sync.Mutex + Listener net.Listener + AuthURL string + URL string + Accounts map[string]*account + Sessions map[string]*session } // The Folder type represents a container stored in an account @@ -96,7 +98,8 @@ type metadata struct { type account struct { swift.Account metadata - password string + password string + Containers map[string]*container } type object struct { @@ -294,8 +297,8 @@ func (r containerResource) delete(a *action) interface{} { if len(b.objects) > 0 { fatalf(409, "Conflict", "The container you tried to delete is not empty") } - delete(a.srv.Containers, b.name) - a.user.Containers-- + delete(a.user.Containers, b.name) + a.user.Account.Containers-- return nil } @@ -316,8 +319,8 @@ func (r containerResource) put(a *action) interface{} { }, } r.container.setMetadata(a, "container") - a.srv.Containers[r.name] = r.container - a.user.Containers++ + a.user.Containers[r.name] = r.container + a.user.Account.Containers++ } return nil @@ -430,7 +433,7 @@ func (objr objectResource) get(a *action) interface{} { if manifest, ok := obj.meta["X-Object-Manifest"]; ok { var segments []io.Reader components := strings.SplitN(manifest[0], "/", 2) - segContainer := a.srv.Containers[components[0]] + segContainer := a.user.Containers[components[0]] prefix := components[1] resp := segContainer.list("", "", prefix, "") sum := md5.New() @@ -575,7 +578,7 @@ func (objr objectResource) copy(a *action) interface{} { objr2 objectResource ) - destURL, _ := url.Parse("/v1/AUTH_tk/" + destination) + destURL, _ := url.Parse("/v1/AUTH_" + TEST_ACCOUNT + "/" + destination) r := a.srv.resourceForURL(destURL) switch t := r.(type) { case objectResource: @@ -665,16 +668,35 @@ func (s *SwiftServer) serveHTTP(w http.ResponseWriter, req *http.Request) { panic(notAuthorized()) } - key := req.Header.Get("x-auth-token") - session, ok := s.Sessions[key[7:]] - if !ok { - panic(notAuthorized()) - } - - a.user = s.Accounts[session.username] - r = s.resourceForURL(req.URL) + key := req.Header.Get("x-auth-token") + if key == "" { + secretKey := "" + signature := req.URL.Query().Get("temp_url_sig") + expires := req.URL.Query().Get("temp_url_expires") + accountName, _, _, _ := s.parseURL(req.URL) + if account, ok := s.Accounts[accountName]; ok { + secretKey = account.meta.Get("X-Account-Meta-Temp-Url-Key") + } + + mac := hmac.New(sha1.New, []byte(secretKey)) + body := fmt.Sprintf("%s\n%s\n%s", req.Method, expires, req.URL.Path) + mac.Write([]byte(body)) + expectedSignature := hex.EncodeToString(mac.Sum(nil)) + + if signature != expectedSignature { + panic(notAuthorized()) + } + } else { + session, ok := s.Sessions[key[7:]] + if !ok { + panic(notAuthorized()) + } + + a.user = s.Accounts[session.username] + } + switch req.Method { case "PUT": resp = r.put(a) @@ -712,22 +734,38 @@ func jsonMarshal(w io.Writer, x interface{}) { } } -var pathRegexp = regexp.MustCompile("/v1/AUTH_[a-zA-Z0-9]+(/([^/]+)(/(.*))?)?") +var pathRegexp = regexp.MustCompile("/v1/AUTH_([a-zA-Z0-9]+)(/([^/]+)(/(.*))?)?") + +func (srv *SwiftServer) parseURL(u *url.URL) (account string, container string, object string, err error) { + m := pathRegexp.FindStringSubmatch(u.Path) + if m == nil { + return "", "", "", fmt.Errorf("Couldn't parse the specified URI") + } + account = m[1] + container = m[3] + object = m[5] + return +} // resourceForURL returns a resource object for the given URL. func (srv *SwiftServer) resourceForURL(u *url.URL) (r resource) { - m := pathRegexp.FindStringSubmatch(u.Path) - if m == nil { - fatalf(404, "InvalidURI", "Couldn't parse the specified URI") + accountName, containerName, objectName, err := srv.parseURL(u) + + if err != nil { + fatalf(404, "InvalidURI", err.Error()) } - containerName := m[2] - objectName := m[4] + + account, ok := srv.Accounts[accountName] + if !ok { + fatalf(404, "NoSuchAccount", "The specified account does not exist") + } + if containerName == "" { return rootResource{} } b := containerResource{ name: containerName, - container: srv.Containers[containerName], + container: account.Containers[containerName], } if objectName == "" { @@ -780,7 +818,7 @@ func (rootResource) get(a *action) interface{} { h := a.w.Header() h.Set("X-Account-Bytes-Used", strconv.Itoa(int(a.user.BytesUsed))) - h.Set("X-Account-Container-Count", strconv.Itoa(int(a.user.Containers))) + h.Set("X-Account-Container-Count", strconv.Itoa(int(a.user.Account.Containers))) h.Set("X-Account-Object-Count", strconv.Itoa(int(a.user.Objects))) // add metadata @@ -792,7 +830,7 @@ func (rootResource) get(a *action) interface{} { var tmp orderedContainers // first get all matching objects and arrange them in alphabetical order. - for _, container := range a.srv.Containers { + for _, container := range a.user.Containers { if strings.HasPrefix(container.name, prefix) { tmp = append(tmp, container) } @@ -858,19 +896,19 @@ func NewSwiftServer(address string) (*SwiftServer, error) { } server := &SwiftServer{ - Listener: l, - AuthURL: "http://" + l.Addr().String() + "/v1.0", - URL: "http://" + l.Addr().String() + "/v1", - Containers: make(map[string]*container), - Accounts: make(map[string]*account), - Sessions: make(map[string]*session), + Listener: l, + AuthURL: "http://" + l.Addr().String() + "/v1.0", + URL: "http://" + l.Addr().String() + "/v1", + Accounts: make(map[string]*account), + Sessions: make(map[string]*session), } - server.Accounts["swifttest"] = &account{ - password: "swifttest", + server.Accounts[TEST_ACCOUNT] = &account{ + password: TEST_ACCOUNT, metadata: metadata{ meta: make(http.Header), }, + Containers: make(map[string]*container), } go http.Serve(l, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { diff --git a/docs/configuration.md b/docs/configuration.md index 0e79f1808..cfaabcf2f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -778,6 +778,17 @@ This storage backend uses Openstack Swift object storage. Your Openstack domain id for Identity v3 API. + + + trustid + + + no + + + Your Openstack trust id for Identity v3 API. + + insecureskipverify diff --git a/registry/storage/driver/swift/swift.go b/registry/storage/driver/swift/swift.go index 0921ccc03..38c41b3dd 100644 --- a/registry/storage/driver/swift/swift.go +++ b/registry/storage/driver/swift/swift.go @@ -61,6 +61,7 @@ type Parameters struct { TenantID string Domain string DomainID string + TrustID string Region string Container string Prefix string @@ -156,6 +157,7 @@ func New(params Parameters) (*Driver, error) { TenantId: params.TenantID, Domain: params.Domain, DomainId: params.DomainID, + TrustId: params.TrustID, Transport: transport, ConnectTimeout: 60 * time.Second, Timeout: 15 * 60 * time.Second, diff --git a/registry/storage/driver/swift/swift_test.go b/registry/storage/driver/swift/swift_test.go index 6be2238a5..705c26312 100644 --- a/registry/storage/driver/swift/swift_test.go +++ b/registry/storage/driver/swift/swift_test.go @@ -29,6 +29,7 @@ func init() { tenantID string domain string domainID string + trustID string container string region string insecureSkipVerify bool @@ -42,6 +43,7 @@ func init() { tenantID = os.Getenv("SWIFT_TENANT_ID") domain = os.Getenv("SWIFT_DOMAIN_NAME") domainID = os.Getenv("SWIFT_DOMAIN_ID") + trustID = os.Getenv("SWIFT_TRUST_ID") container = os.Getenv("SWIFT_CONTAINER_NAME") region = os.Getenv("SWIFT_REGION_NAME") insecureSkipVerify, _ = strconv.ParseBool(os.Getenv("SWIFT_INSECURESKIPVERIFY")) @@ -71,6 +73,7 @@ func init() { tenantID, domain, domainID, + trustID, region, container, root,