Merge pull request #946 from humble00/master

Add TrustId parameter to swift driver

Godeps/Godeps.json

@@ -96,7 +96,7 @@
 		},
 		{
 			"ImportPath": "github.com/ncw/swift",
-			"Rev": "22c8fa9fb5ba145b4d4e2cebb027e84b1a7b1296"
+			"Rev": "ca8cbbde50d4e12dd8ad70b1bd66589ae98efc5c"
 		},
 		{
 			"ImportPath": "github.com/yvasiyarov/go-metrics",

Godeps/_workspace/src/github.com/ncw/swift/.travis.yml

@@ -1,4 +1,5 @@
 language: go
+sudo: false
 
 go:
   - 1.1.2

Godeps/_workspace/src/github.com/ncw/swift/README.md

@@ -77,6 +77,10 @@ And optionally these if using v3 authentication
     export SWIFT_API_DOMAIN_ID='domain id'
     export SWIFT_API_DOMAIN='domain name'
 
+And optionally these if using v3 trust
+
+    export SWIFT_TRUST_ID='TrustId'
+
 And optionally this if you want to skip server certificate validation
 
     export SWIFT_AUTH_INSECURE=1
@@ -126,3 +130,5 @@ Contributors
 - Sylvain Baubeau <sbaubeau@redhat.com>
 - Chris Kastorff <encryptio@gmail.com>
 - Dai HaoJun <haojun.dai@hp.com>
+- Hua Wang <wanghua.humble@gmail.com>
+- Fabian Ruff <fabian@progra.de>

Godeps/_workspace/src/github.com/ncw/swift/auth_v3.go

@@ -33,6 +33,7 @@ type v3AuthRequest struct {
 type v3Scope struct {
 	Project *v3Project `json:"project,omitempty"`
 	Domain  *v3Domain  `json:"domain,omitempty"`
+	Trust   *v3Trust   `json:"OS-TRUST:trust,omitempty"`
 }
 
 type v3Domain struct {
@@ -46,6 +47,10 @@ type v3Project struct {
 	Domain *v3Domain `json:"domain,omitempty"`
 }
 
+type v3Trust struct {
+	Id string `json:"id"`
+}
+
 type v3User struct {
 	Domain   *v3Domain `json:"domain,omitempty"`
 	Id       string    `json:"id,omitempty"`
@@ -66,7 +71,12 @@ type v3AuthResponse struct {
 	Token struct {
 		Expires_At, Issued_At string
 		Methods               []string
-		Roles                 []map[string]string
+		Roles                 []struct {
+			Id, Name string
+			Links    struct {
+				Self string
+			}
+		}
 
 		Project struct {
 			Domain struct {
@@ -129,7 +139,9 @@ func (auth *v3Auth) Request(c *Connection) (*http.Request, error) {
 		v3.Auth.Identity.Password.User.Domain = domain
 	}
 
-	if c.TenantId != "" || c.Tenant != "" {
+	if c.TrustId != "" {
+		v3.Auth.Scope = &v3Scope{Trust: &v3Trust{Id: c.TrustId}}
+	} else if c.TenantId != "" || c.Tenant != "" {
 
 		v3.Auth.Scope = &v3Scope{Project: &v3Project{}}
 
@@ -159,7 +171,7 @@ func (auth *v3Auth) Request(c *Connection) (*http.Request, error) {
 	if !strings.HasSuffix(url, "/") {
 		url += "/"
 	}
-	url += "tokens"
+	url += "auth/tokens"
 	req, err := http.NewRequest("POST", url, bytes.NewBuffer(body))
 	if err != nil {
 		return nil, err

Godeps/_workspace/src/github.com/ncw/swift/swift.go

@@ -3,7 +3,10 @@ package swift
 import (
 	"bufio"
 	"bytes"
+	"crypto/hmac"
 	"crypto/md5"
+	"crypto/sha1"
+	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"hash"
@@ -94,6 +97,7 @@ type Connection struct {
 	Internal       bool              // Set this to true to use the the internal / service network
 	Tenant         string            // Name of the tenant (v2 auth only)
 	TenantId       string            // Id of the tenant (v2 auth only)
+	TrustId        string            // Id of the trust (v3 auth only)
 	Transport      http.RoundTripper `json:"-" xml:"-"` // Optional specialised http.Transport (eg. for Google Appengine)
 	// These are filled in after Authenticate is called as are the defaults for above
 	StorageUrl string
@@ -1422,8 +1426,10 @@ func (c *Connection) ObjectOpen(container string, objectName string, checkHash b
 		file.body = io.TeeReader(resp.Body, file.hash)
 	}
 	// Read Content-Length
+	if resp.Header.Get("Content-Length") != "" {
 		file.length, err = getInt64FromHeader(resp, "Content-Length")
 		file.lengthOk = (err == nil)
+	}
 	return
 }
 
@@ -1479,6 +1485,16 @@ func (c *Connection) ObjectDelete(container string, objectName string) error {
 	return err
 }
 
+// ObjectTempUrl returns a temporary URL for an object
+func (c *Connection) ObjectTempUrl(container string, objectName string, secretKey string, method string, expires time.Time) string {
+	mac := hmac.New(sha1.New, []byte(secretKey))
+	prefix, _ := url.Parse(c.StorageUrl)
+	body := fmt.Sprintf("%s\n%d\n%s/%s/%s", method, expires.Unix(), prefix.Path, container, objectName)
+	mac.Write([]byte(body))
+	sig := hex.EncodeToString(mac.Sum(nil))
+	return fmt.Sprintf("%s/%s/%s?temp_url_sig=%s&temp_url_expires=%d", c.StorageUrl, container, objectName, sig, expires.Unix())
+}
+
 // parseResponseStatus parses string like "200 OK" and returns Error.
 //
 // For status codes beween 200 and 299, this returns nil.

Godeps/_workspace/src/github.com/ncw/swift/swift_test.go

@@ -20,9 +20,8 @@ import (
 	"encoding/json"
 	"encoding/xml"
 	"fmt"
-	"github.com/ncw/swift"
-	"github.com/ncw/swift/swifttest"
 	"io"
+	"io/ioutil"
 	"net/http"
 	"os"
 	"strconv"
@@ -30,6 +29,9 @@ import (
 	"sync"
 	"testing"
 	"time"
+
+	"github.com/ncw/swift"
+	"github.com/ncw/swift/swifttest"
 )
 
 var (
@@ -52,6 +54,7 @@ const (
 	CONTENT_SIZE       = int64(len(CONTENTS))
 	CONTENT_MD5        = "827ccb0eea8a706c4c34a16891f84e7b"
 	EMPTY_MD5          = "d41d8cd98f00b204e9800998ecf8427e"
+	SECRET_KEY         = "b3968d0207b54ece87cccc06515a89d4"
 )
 
 type someTransport struct{ http.Transport }
@@ -211,6 +214,28 @@ func TestV3AuthenticateWithDomainNameAndTenantId(t *testing.T) {
 	}
 }
 
+func TestV3TrustWithTrustId(t *testing.T) {
+	var err error
+	if !isV3Api() {
+		return
+	}
+
+	c, err = makeConnection()
+	if err != nil {
+		t.Fatal("Failed to create server", err)
+	}
+
+	c.TrustId = os.Getenv("SWIFT_TRUST_ID")
+
+	err = c.Authenticate()
+	if err != nil {
+		t.Fatal("Auth failed", err)
+	}
+	if !c.Authenticated() {
+		t.Fatal("Not authenticated")
+	}
+}
+
 func TestV3AuthenticateWithDomainIdAndTenantId(t *testing.T) {
 	var err error
 
@@ -1441,6 +1466,43 @@ func TestObjectDifficultName(t *testing.T) {
 	}
 }
 
+func TestTempUrl(t *testing.T) {
+	err := c.ObjectPutBytes(CONTAINER, OBJECT, []byte(CONTENTS), "")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	m := swift.Metadata{}
+	m["temp-url-key"] = SECRET_KEY
+	err = c.AccountUpdate(m.AccountHeaders())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expiresTime := time.Now().Add(20 * time.Minute)
+	tempUrl := c.ObjectTempUrl(CONTAINER, OBJECT, SECRET_KEY, "GET", expiresTime)
+	resp, err := http.Get(tempUrl)
+	if err != nil {
+		t.Fatal("Failed to retrieve file from temporary url")
+	}
+	if resp.StatusCode == 401 {
+		t.Log("Server doesn't support tempurl")
+	} else if resp.StatusCode != 200 {
+		t.Fatal("HTTP Error retrieving file from temporary url", resp.StatusCode)
+	} else {
+		if content, err := ioutil.ReadAll(resp.Body); err != nil || string(content) != CONTENTS {
+			t.Error("Bad content", err)
+		}
+	}
+
+	resp.Body.Close()
+	err = c.ObjectDelete(CONTAINER, OBJECT)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+}
+
 func TestContainerDelete(t *testing.T) {
 	err := c.ContainerDelete(CONTAINER)
 	if err != nil {

Godeps/_workspace/src/github.com/ncw/swift/swifttest/server.go

@@ -8,8 +8,10 @@ package swifttest
 
 import (
 	"bytes"
+	"crypto/hmac"
 	"crypto/md5"
 	"crypto/rand"
+	"crypto/sha1"
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
@@ -34,6 +36,7 @@ import (
 
 const (
 	DEBUG        = false
+	TEST_ACCOUNT = "swifttest"
 )
 
 type SwiftServer struct {
@@ -43,7 +46,6 @@ type SwiftServer struct {
 	Listener net.Listener
 	AuthURL  string
 	URL      string
-	Containers map[string]*container
 	Accounts map[string]*account
 	Sessions map[string]*session
 }
@@ -97,6 +99,7 @@ type account struct {
 	swift.Account
 	metadata
 	password   string
+	Containers map[string]*container
 }
 
 type object struct {
@@ -294,8 +297,8 @@ func (r containerResource) delete(a *action) interface{} {
 	if len(b.objects) > 0 {
 		fatalf(409, "Conflict", "The container you tried to delete is not empty")
 	}
-	delete(a.srv.Containers, b.name)
-	a.user.Containers--
+	delete(a.user.Containers, b.name)
+	a.user.Account.Containers--
 	return nil
 }
 
@@ -316,8 +319,8 @@ func (r containerResource) put(a *action) interface{} {
 			},
 		}
 		r.container.setMetadata(a, "container")
-		a.srv.Containers[r.name] = r.container
-		a.user.Containers++
+		a.user.Containers[r.name] = r.container
+		a.user.Account.Containers++
 	}
 
 	return nil
@@ -430,7 +433,7 @@ func (objr objectResource) get(a *action) interface{} {
 	if manifest, ok := obj.meta["X-Object-Manifest"]; ok {
 		var segments []io.Reader
 		components := strings.SplitN(manifest[0], "/", 2)
-		segContainer := a.srv.Containers[components[0]]
+		segContainer := a.user.Containers[components[0]]
 		prefix := components[1]
 		resp := segContainer.list("", "", prefix, "")
 		sum := md5.New()
@@ -575,7 +578,7 @@ func (objr objectResource) copy(a *action) interface{} {
 		objr2 objectResource
 	)
 
-	destURL, _ := url.Parse("/v1/AUTH_tk/" + destination)
+	destURL, _ := url.Parse("/v1/AUTH_" + TEST_ACCOUNT + "/" + destination)
 	r := a.srv.resourceForURL(destURL)
 	switch t := r.(type) {
 	case objectResource:
@@ -665,15 +668,34 @@ func (s *SwiftServer) serveHTTP(w http.ResponseWriter, req *http.Request) {
 		panic(notAuthorized())
 	}
 
+	r = s.resourceForURL(req.URL)
+
 	key := req.Header.Get("x-auth-token")
+	if key == "" {
+		secretKey := ""
+		signature := req.URL.Query().Get("temp_url_sig")
+		expires := req.URL.Query().Get("temp_url_expires")
+		accountName, _, _, _ := s.parseURL(req.URL)
+		if account, ok := s.Accounts[accountName]; ok {
+			secretKey = account.meta.Get("X-Account-Meta-Temp-Url-Key")
+		}
+
+		mac := hmac.New(sha1.New, []byte(secretKey))
+		body := fmt.Sprintf("%s\n%s\n%s", req.Method, expires, req.URL.Path)
+		mac.Write([]byte(body))
+		expectedSignature := hex.EncodeToString(mac.Sum(nil))
+
+		if signature != expectedSignature {
+			panic(notAuthorized())
+		}
+	} else {
 		session, ok := s.Sessions[key[7:]]
 		if !ok {
 			panic(notAuthorized())
 		}
 
 		a.user = s.Accounts[session.username]
-
-	r = s.resourceForURL(req.URL)
+	}
 
 	switch req.Method {
 	case "PUT":
@@ -712,22 +734,38 @@ func jsonMarshal(w io.Writer, x interface{}) {
 	}
 }
 
-var pathRegexp = regexp.MustCompile("/v1/AUTH_[a-zA-Z0-9]+(/([^/]+)(/(.*))?)?")
+var pathRegexp = regexp.MustCompile("/v1/AUTH_([a-zA-Z0-9]+)(/([^/]+)(/(.*))?)?")
+
+func (srv *SwiftServer) parseURL(u *url.URL) (account string, container string, object string, err error) {
+	m := pathRegexp.FindStringSubmatch(u.Path)
+	if m == nil {
+		return "", "", "", fmt.Errorf("Couldn't parse the specified URI")
+	}
+	account = m[1]
+	container = m[3]
+	object = m[5]
+	return
+}
 
 // resourceForURL returns a resource object for the given URL.
 func (srv *SwiftServer) resourceForURL(u *url.URL) (r resource) {
-	m := pathRegexp.FindStringSubmatch(u.Path)
-	if m == nil {
-		fatalf(404, "InvalidURI", "Couldn't parse the specified URI")
+	accountName, containerName, objectName, err := srv.parseURL(u)
+
+	if err != nil {
+		fatalf(404, "InvalidURI", err.Error())
 	}
-	containerName := m[2]
-	objectName := m[4]
+
+	account, ok := srv.Accounts[accountName]
+	if !ok {
+		fatalf(404, "NoSuchAccount", "The specified account does not exist")
+	}
+
 	if containerName == "" {
 		return rootResource{}
 	}
 	b := containerResource{
 		name:      containerName,
-		container: srv.Containers[containerName],
+		container: account.Containers[containerName],
 	}
 
 	if objectName == "" {
@@ -780,7 +818,7 @@ func (rootResource) get(a *action) interface{} {
 	h := a.w.Header()
 
 	h.Set("X-Account-Bytes-Used", strconv.Itoa(int(a.user.BytesUsed)))
-	h.Set("X-Account-Container-Count", strconv.Itoa(int(a.user.Containers)))
+	h.Set("X-Account-Container-Count", strconv.Itoa(int(a.user.Account.Containers)))
 	h.Set("X-Account-Object-Count", strconv.Itoa(int(a.user.Objects)))
 
 	// add metadata
@@ -792,7 +830,7 @@ func (rootResource) get(a *action) interface{} {
 
 	var tmp orderedContainers
 	// first get all matching objects and arrange them in alphabetical order.
-	for _, container := range a.srv.Containers {
+	for _, container := range a.user.Containers {
 		if strings.HasPrefix(container.name, prefix) {
 			tmp = append(tmp, container)
 		}
@@ -861,16 +899,16 @@ func NewSwiftServer(address string) (*SwiftServer, error) {
 		Listener: l,
 		AuthURL:  "http://" + l.Addr().String() + "/v1.0",
 		URL:      "http://" + l.Addr().String() + "/v1",
-		Containers: make(map[string]*container),
 		Accounts: make(map[string]*account),
 		Sessions: make(map[string]*session),
 	}
 
-	server.Accounts["swifttest"] = &account{
-		password: "swifttest",
+	server.Accounts[TEST_ACCOUNT] = &account{
+		password: TEST_ACCOUNT,
 		metadata: metadata{
 			meta: make(http.Header),
 		},
+		Containers: make(map[string]*container),
 	}
 
 	go http.Serve(l, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {

docs/configuration.md

@@ -778,6 +778,17 @@ This storage backend uses Openstack Swift object storage.
       Your Openstack domain id for Identity v3 API.
     </td>
   </tr>
+  <tr>
+    <td>
+      <code>trustid</code>
+    </td>
+    <td>
+      no
+    </td>
+    <td>
+      Your Openstack trust id for Identity v3 API.
+    </td>
+  </tr>
   <tr>
     <td>
       <code>insecureskipverify</code>

registry/storage/driver/swift/swift.go

@@ -61,6 +61,7 @@ type Parameters struct {
 	TenantID           string
 	Domain             string
 	DomainID           string
+	TrustID            string
 	Region             string
 	Container          string
 	Prefix             string
@@ -156,6 +157,7 @@ func New(params Parameters) (*Driver, error) {
 		TenantId:       params.TenantID,
 		Domain:         params.Domain,
 		DomainId:       params.DomainID,
+		TrustId:        params.TrustID,
 		Transport:      transport,
 		ConnectTimeout: 60 * time.Second,
 		Timeout:        15 * 60 * time.Second,

registry/storage/driver/swift/swift_test.go

@@ -29,6 +29,7 @@ func init() {
 		tenantID           string
 		domain             string
 		domainID           string
+		trustID            string
 		container          string
 		region             string
 		insecureSkipVerify bool
@@ -42,6 +43,7 @@ func init() {
 	tenantID = os.Getenv("SWIFT_TENANT_ID")
 	domain = os.Getenv("SWIFT_DOMAIN_NAME")
 	domainID = os.Getenv("SWIFT_DOMAIN_ID")
+	trustID = os.Getenv("SWIFT_TRUST_ID")
 	container = os.Getenv("SWIFT_CONTAINER_NAME")
 	region = os.Getenv("SWIFT_REGION_NAME")
 	insecureSkipVerify, _ = strconv.ParseBool(os.Getenv("SWIFT_INSECURESKIPVERIFY"))
@@ -71,6 +73,7 @@ func init() {
 			tenantID,
 			domain,
 			domainID,
+			trustID,
 			region,
 			container,
 			root,