Merge pull request #1114 from lebauce/swift-temp-url

Redirect support in Swift driver

Godeps/Godeps.json

@@ -122,7 +122,7 @@
 		},
 		{
 			"ImportPath": "github.com/ncw/swift",
-			"Rev": "ca8cbbde50d4e12dd8ad70b1bd66589ae98efc5c"
+			"Rev": "c54732e87b0b283d1baf0a18db689d0aea460ba3"
 		},
 		{
 			"ImportPath": "github.com/noahdesu/go-ceph/rados",

Godeps/_workspace/src/github.com/ncw/swift/README.md

@@ -132,3 +132,5 @@ Contributors
 - Dai HaoJun <haojun.dai@hp.com>
 - Hua Wang <wanghua.humble@gmail.com>
 - Fabian Ruff <fabian@progra.de>
+- Arturo Reuschenbach Puncernau <reuschenbach@gmail.com>
+- Petr Kotek <petr.kotek@bigcommerce.com>

Godeps/_workspace/src/github.com/ncw/swift/auth.go

@@ -156,6 +156,7 @@ func (auth *v2Auth) Request(c *Connection) (*http.Request, error) {
 		return nil, err
 	}
 	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("User-Agent", c.UserAgent)
 	return req, nil
 }
 

Godeps/_workspace/src/github.com/ncw/swift/auth_v3.go

@@ -177,6 +177,7 @@ func (auth *v3Auth) Request(c *Connection) (*http.Request, error) {
 		return nil, err
 	}
 	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("User-Agent", c.UserAgent)
 	return req, nil
 }
 

Godeps/_workspace/src/github.com/ncw/swift/swift.go

@@ -392,6 +392,26 @@ func (c *Connection) authenticated() bool {
 	return c.StorageUrl != "" && c.AuthToken != ""
 }
 
+// SwiftInfo contains the JSON object returned by Swift when the /info
+// route is queried. The object contains, among others, the Swift version,
+// the enabled middlewares and their configuration
+type SwiftInfo map[string]interface{}
+
+// Discover Swift configuration by doing a request against /info
+func (c *Connection) QueryInfo() (infos SwiftInfo, err error) {
+	infoUrl, err := url.Parse(c.StorageUrl)
+	if err != nil {
+		return nil, err
+	}
+	infoUrl.Path = path.Join(infoUrl.Path, "..", "..", "info")
+	resp, err := http.Get(infoUrl.String())
+	if err == nil {
+		err = readJson(resp, &infos)
+		return infos, err
+	}
+	return nil, err
+}
+
 // RequestOpts contains parameters for Connection.storage.
 type RequestOpts struct {
 	Container  string
@@ -418,6 +438,10 @@ type RequestOpts struct {
 // resp.Body.Close() must be called on it, unless noResponse is set in
 // which case the body will be closed in this function
 //
+// If "Content-Length" is set in p.Headers it will be used - this can
+// be used to override the default chunked transfer encoding for
+// uploads.
+//
 // This will Authenticate if necessary, and re-authenticate if it
 // receives a 401 error which means the token has expired
 //
@@ -433,8 +457,9 @@ func (c *Connection) Call(targetUrl string, p RequestOpts) (resp *http.Response,
 	var req *http.Request
 	for {
 		var authToken string
-		targetUrl, authToken, err = c.getUrlAndAuthToken(targetUrl, p.OnReAuth)
+		if targetUrl, authToken, err = c.getUrlAndAuthToken(targetUrl, p.OnReAuth); err != nil {
-
+			return //authentication failure
+		}
 		var URL *url.URL
 		URL, err = url.Parse(targetUrl)
 		if err != nil {
@@ -460,18 +485,27 @@ func (c *Connection) Call(targetUrl string, p RequestOpts) (resp *http.Response,
 		}
 		if p.Headers != nil {
 			for k, v := range p.Headers {
-				req.Header.Add(k, v)
+				// Set ContentLength in req if the user passed it in in the headers
+				if k == "Content-Length" {
+					contentLength, err := strconv.ParseInt(v, 10, 64)
+					if err != nil {
+						return nil, nil, fmt.Errorf("Invalid %q header %q: %v", k, v, err)
+					}
+					req.ContentLength = contentLength
+				} else {
+					req.Header.Add(k, v)
+				}
 			}
 		}
-		req.Header.Add("User-Agent", DefaultUserAgent)
+		req.Header.Add("User-Agent", c.UserAgent)
 		req.Header.Add("X-Auth-Token", authToken)
 		resp, err = c.doTimeoutRequest(timer, req)
 		if err != nil {
-			if p.Operation == "HEAD" || p.Operation == "GET" {
+			if (p.Operation == "HEAD" || p.Operation == "GET") && retries > 0 {
 				retries--
 				continue
 			}
-			return
+			return nil, nil, err
 		}
 		// Check to see if token has expired
 		if resp.StatusCode == 401 && retries > 0 {
@@ -566,7 +600,8 @@ func readJson(resp *http.Response, result interface{}) (err error) {
 // ContainersOpts is options for Containers() and ContainerNames()
 type ContainersOpts struct {
 	Limit     int     // For an integer value n, limits the number of results to at most n values.
-	Marker    string  // Given a string value x, return object names greater in value than the specified marker.
+	Prefix    string  // Given a string value x, return container names matching the specified prefix.
+	Marker    string  // Given a string value x, return container names greater in value than the specified marker.
 	EndMarker string  // Given a string value x, return container names less in value than the specified marker.
 	Headers   Headers // Any additional HTTP headers - can be nil
 }
@@ -579,6 +614,9 @@ func (opts *ContainersOpts) parse() (url.Values, Headers) {
 		if opts.Limit > 0 {
 			v.Set("limit", strconv.Itoa(opts.Limit))
 		}
+		if opts.Prefix != "" {
+			v.Set("prefix", opts.Prefix)
+		}
 		if opts.Marker != "" {
 			v.Set("marker", opts.Marker)
 		}

Godeps/_workspace/src/github.com/ncw/swift/swift_test.go

@@ -65,6 +65,7 @@ func makeConnection() (*swift.Connection, error) {
 	UserName := os.Getenv("SWIFT_API_USER")
 	ApiKey := os.Getenv("SWIFT_API_KEY")
 	AuthUrl := os.Getenv("SWIFT_AUTH_URL")
+	Region := os.Getenv("SWIFT_REGION_NAME")
 
 	Insecure := os.Getenv("SWIFT_AUTH_INSECURE")
 	ConnectionChannelTimeout := os.Getenv("SWIFT_CONNECTION_CHANNEL_TIMEOUT")
@@ -96,6 +97,7 @@ func makeConnection() (*swift.Connection, error) {
 		UserName:       UserName,
 		ApiKey:         ApiKey,
 		AuthUrl:        AuthUrl,
+		Region:         Region,
 		Transport:      transport,
 		ConnectTimeout: 60 * time.Second,
 		Timeout:        60 * time.Second,
@@ -601,6 +603,45 @@ func TestObjectPutString(t *testing.T) {
 	}
 }
 
+func TestObjectPut(t *testing.T) {
+	headers := swift.Headers{}
+
+	// Set content size incorrectly - should produce an error
+	headers["Content-Length"] = strconv.FormatInt(CONTENT_SIZE-1, 10)
+	contents := bytes.NewBufferString(CONTENTS)
+	h, err := c.ObjectPut(CONTAINER, OBJECT, contents, true, CONTENT_MD5, "text/plain", headers)
+	if err == nil {
+		t.Fatal("Expecting error but didn't get one")
+	}
+
+	// Now set content size correctly
+	contents = bytes.NewBufferString(CONTENTS)
+	headers["Content-Length"] = strconv.FormatInt(CONTENT_SIZE, 10)
+	h, err = c.ObjectPut(CONTAINER, OBJECT, contents, true, CONTENT_MD5, "text/plain", headers)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if h["Etag"] != CONTENT_MD5 {
+		t.Errorf("Bad Etag want %q got %q", CONTENT_MD5, h["Etag"])
+	}
+
+	// Fetch object info and compare
+	info, _, err := c.Object(CONTAINER, OBJECT)
+	if err != nil {
+		t.Error(err)
+	}
+	if info.ContentType != "text/plain" {
+		t.Error("Bad content type", info.ContentType)
+	}
+	if info.Bytes != CONTENT_SIZE {
+		t.Error("Bad length")
+	}
+	if info.Hash != CONTENT_MD5 {
+		t.Error("Bad length")
+	}
+}
+
 func TestObjectEmpty(t *testing.T) {
 	err := c.ObjectPutString(CONTAINER, EMPTYOBJECT, "", "")
 	if err != nil {
@@ -1493,6 +1534,14 @@ func TestTempUrl(t *testing.T) {
 		if content, err := ioutil.ReadAll(resp.Body); err != nil || string(content) != CONTENTS {
 			t.Error("Bad content", err)
 		}
+
+		resp, err := http.Post(tempUrl, "image/jpeg", bytes.NewReader([]byte(CONTENTS)))
+		if err != nil {
+			t.Fatal("Failed to retrieve file from temporary url")
+		}
+		if resp.StatusCode != 401 {
+			t.Fatal("Expecting server to forbid access to object")
+		}
 	}
 
 	resp.Body.Close()
@@ -1500,7 +1549,17 @@ func TestTempUrl(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+}
 
+func TestQueryInfo(t *testing.T) {
+	infos, err := c.QueryInfo()
+	if err != nil {
+		t.Log("Server doesn't support querying info")
+		return
+	}
+	if _, ok := infos["swift"]; !ok {
+		t.Fatal("No 'swift' section found in configuration")
+	}
 }
 
 func TestContainerDelete(t *testing.T) {

Godeps/_workspace/src/github.com/ncw/swift/swifttest/server.go

@@ -443,7 +443,7 @@ func (objr objectResource) get(a *action) interface{} {
 			if obj, ok := item.(*object); ok {
 				length := len(obj.data)
 				size += length
-				sum.Write([]byte(components[0] + "/" + obj.name + "\n"))
+				sum.Write([]byte(hex.EncodeToString(obj.checksum)))
 				if start >= cursor+length {
 					continue
 				}
@@ -668,24 +668,42 @@ func (s *SwiftServer) serveHTTP(w http.ResponseWriter, req *http.Request) {
 		panic(notAuthorized())
 	}
 
+	if req.URL.String() == "/info" {
+		jsonMarshal(w, &swift.SwiftInfo{
+			"swift": map[string]interface{}{
+				"version": "1.2",
+			},
+			"tempurl": map[string]interface{}{
+				"methods": []string{"GET", "HEAD", "PUT"},
+			},
+		})
+		return
+	}
+
 	r = s.resourceForURL(req.URL)
 
 	key := req.Header.Get("x-auth-token")
-	if key == "" {
+	signature := req.URL.Query().Get("temp_url_sig")
-		secretKey := ""
+	expires := req.URL.Query().Get("temp_url_expires")
-		signature := req.URL.Query().Get("temp_url_sig")
+	if key == "" && signature != "" && expires != "" {
-		expires := req.URL.Query().Get("temp_url_expires")
 		accountName, _, _, _ := s.parseURL(req.URL)
+		secretKey := ""
 		if account, ok := s.Accounts[accountName]; ok {
 			secretKey = account.meta.Get("X-Account-Meta-Temp-Url-Key")
 		}
 
-		mac := hmac.New(sha1.New, []byte(secretKey))
+		get_hmac := func(method string) string {
-		body := fmt.Sprintf("%s\n%s\n%s", req.Method, expires, req.URL.Path)
+			mac := hmac.New(sha1.New, []byte(secretKey))
-		mac.Write([]byte(body))
+			body := fmt.Sprintf("%s\n%s\n%s", method, expires, req.URL.Path)
-		expectedSignature := hex.EncodeToString(mac.Sum(nil))
+			mac.Write([]byte(body))
+			return hex.EncodeToString(mac.Sum(nil))
+		}
 
-		if signature != expectedSignature {
+		if req.Method == "HEAD" {
+			if signature != get_hmac("GET") && signature != get_hmac("POST") && signature != get_hmac("PUT") {
+				panic(notAuthorized())
+			}
+		} else if signature != get_hmac(req.Method) {
 			panic(notAuthorized())
 		}
 	} else {

docs/storage-drivers/swift.md

@@ -93,6 +93,16 @@ An implementation of the `storagedriver.StorageDriver` interface that uses [Open
     </p>
     </td>
 </tr>
+<tr>
+    <td>
+    <code>trustid</code>
+    </td>
+    <td>
+    <p>
+    Optionally, your OpenStack trust id for Identity v3 API.
+    </p>
+    </td>
+</tr>
 <tr>
     <td>
     <code>insecureskipverify</code>
@@ -133,4 +143,58 @@ An implementation of the `storagedriver.StorageDriver` interface that uses [Open
     </p>
     </td>
 </tr>
+<tr>
+    <td>
+    <code>secretkey</code>
+    </td>
+    <td>
+    <p>
+    Optionally, the secret key used to generate temporary URLs.</p>
+    </p>
+    </td>
+</tr>
+<tr>
+    <td>
+    <code>accesskey</code>
+    </td>
+    <td>
+    <p>
+    Optionally, the access key to generate temporary URLs. It is used by HP Cloud Object Storage in addition to the `secretkey` parameter.</p>
+    </p>
+    </td>
+</tr>
+</table>
+
+The features supported by the Swift server are queried by requesting the `/info` URL on the server. In case the administrator
+disabled that feature, the configuration file can specify the following optional parameters :
+
+<table>
+<tr>
+    <td>
+    <code>tempurlcontainerkey</code>
+    </td>
+    <td>
+    <p>
+    Specify whether to use container secret key to generate temporary URL when set to true, or the account secret key otherwise.</p>
+    </p>
+    </td>
+</tr>
+<tr>
+    <td>
+    <code>tempurlmethods</code>
+    </td>
+    <td>
+    <p>
+    Array of HTTP methods that are supported by the TempURL middleware of the Swift server. Example:</p>
+    <code>
+    - tempurlmethods:
+      - GET
+      - PUT
+      - HEAD
+      - POST
+      - DELETE
+    </code>
+    </p>
+    </td>
+</tr>
 </table>

registry/storage/driver/swift/swift.go

@@ -7,9 +7,6 @@
 // It supports both TempAuth authentication and Keystone authentication
 // (up to version 3).
 //
-// Since Swift has no concept of directories (directories are an abstration),
-// empty objects are created with the MIME type application/vnd.swift.directory.
-//
 // As Swift has a limit on the size of a single uploaded object (by default
 // this is 5GB), the driver makes use of the Swift Large Object Support
 // (http://docs.openstack.org/developer/swift/overview_large_objects.html).
@@ -24,12 +21,11 @@ import (
 	"crypto/sha1"
 	"crypto/tls"
 	"encoding/hex"
-	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"net/http"
-	gopath "path"
+	"net/url"
 	"strconv"
 	"strings"
 	"time"
@@ -54,22 +50,34 @@ const minChunkSize = 1 << 20
 
 // Parameters A struct that encapsulates all of the driver parameters after all values have been set
 type Parameters struct {
-	Username           string
+	Username            string
-	Password           string
+	Password            string
-	AuthURL            string
+	AuthURL             string
-	Tenant             string
+	Tenant              string
-	TenantID           string
+	TenantID            string
-	Domain             string
+	Domain              string
-	DomainID           string
+	DomainID            string
-	TrustID            string
+	TrustID             string
-	Region             string
+	Region              string
-	Container          string
+	Container           string
-	Prefix             string
+	Prefix              string
-	InsecureSkipVerify bool
+	InsecureSkipVerify  bool
-	ChunkSize          int
+	ChunkSize           int
+	SecretKey           string
+	AccessKey           string
+	TempURLContainerKey bool
+	TempURLMethods      []string
 }
 
-type swiftInfo map[string]interface{}
+// swiftInfo maps the JSON structure returned by Swift /info endpoint
+type swiftInfo struct {
+	Swift struct {
+		Version string `mapstructure:"version"`
+	}
+	Tempurl struct {
+		Methods []string `mapstructure:"methods"`
+	}
+}
 
 func init() {
 	factory.Register(driverName, &swiftDriverFactory{})
@@ -83,11 +91,15 @@ func (factory *swiftDriverFactory) Create(parameters map[string]interface{}) (st
 }
 
 type driver struct {
-	Conn              swift.Connection
+	Conn                swift.Connection
-	Container         string
+	Container           string
-	Prefix            string
+	Prefix              string
-	BulkDeleteSupport bool
+	BulkDeleteSupport   bool
-	ChunkSize         int
+	ChunkSize           int
+	SecretKey           string
+	AccessKey           string
+	TempURLContainerKey bool
+	TempURLMethods      []string
 }
 
 type baseEmbed struct {
@@ -176,11 +188,65 @@ func New(params Parameters) (*Driver, error) {
 	}
 
 	d := &driver{
-		Conn:              ct,
+		Conn:           ct,
-		Container:         params.Container,
+		Container:      params.Container,
-		Prefix:            params.Prefix,
+		Prefix:         params.Prefix,
-		BulkDeleteSupport: detectBulkDelete(params.AuthURL),
+		ChunkSize:      params.ChunkSize,
-		ChunkSize:         params.ChunkSize,
+		TempURLMethods: make([]string, 0),
+		AccessKey:      params.AccessKey,
+	}
+
+	info := swiftInfo{}
+	if config, err := d.Conn.QueryInfo(); err == nil {
+		_, d.BulkDeleteSupport = config["bulk_delete"]
+
+		if err := mapstructure.Decode(config, &info); err == nil {
+			d.TempURLContainerKey = info.Swift.Version >= "2.3.0"
+			d.TempURLMethods = info.Tempurl.Methods
+		}
+	} else {
+		d.TempURLContainerKey = params.TempURLContainerKey
+		d.TempURLMethods = params.TempURLMethods
+	}
+
+	if len(d.TempURLMethods) > 0 {
+		secretKey := params.SecretKey
+		if secretKey == "" {
+			secretKey, _ = generateSecret()
+		}
+
+		// Since Swift 2.2.2, we can now set secret keys on containers
+		// in addition to the account secret keys. Use them in preference.
+		if d.TempURLContainerKey {
+			_, containerHeaders, err := d.Conn.Container(d.Container)
+			if err != nil {
+				return nil, fmt.Errorf("Failed to fetch container info %s (%s)", d.Container, err)
+			}
+
+			d.SecretKey = containerHeaders["X-Container-Meta-Temp-Url-Key"]
+			if d.SecretKey == "" || (params.SecretKey != "" && d.SecretKey != params.SecretKey) {
+				m := swift.Metadata{}
+				m["temp-url-key"] = secretKey
+				if d.Conn.ContainerUpdate(d.Container, m.ContainerHeaders()); err == nil {
+					d.SecretKey = secretKey
+				}
+			}
+		} else {
+			// Use the account secret key
+			_, accountHeaders, err := d.Conn.Account()
+			if err != nil {
+				return nil, fmt.Errorf("Failed to fetch account info (%s)", err)
+			}
+
+			d.SecretKey = accountHeaders["X-Account-Meta-Temp-Url-Key"]
+			if d.SecretKey == "" || (params.SecretKey != "" && d.SecretKey != params.SecretKey) {
+				m := swift.Metadata{}
+				m["temp-url-key"] = secretKey
+				if err := d.Conn.AccountUpdate(m.AccountHeaders()); err == nil {
+					d.SecretKey = secretKey
+				}
+			}
+		}
 	}
 
 	return &Driver{
@@ -590,9 +656,58 @@ func (d *driver) Delete(ctx context.Context, path string) error {
 }
 
 // URLFor returns a URL which may be used to retrieve the content stored at the given path.
-// May return an UnsupportedMethodErr in certain StorageDriver implementations.
 func (d *driver) URLFor(ctx context.Context, path string, options map[string]interface{}) (string, error) {
-	return "", storagedriver.ErrUnsupportedMethod
+	if d.SecretKey == "" {
+		return "", storagedriver.ErrUnsupportedMethod
+	}
+
+	methodString := "GET"
+	method, ok := options["method"]
+	if ok {
+		if methodString, ok = method.(string); !ok {
+			return "", storagedriver.ErrUnsupportedMethod
+		}
+	}
+
+	if methodString == "HEAD" {
+		// A "HEAD" request on a temporary URL is allowed if the
+		// signature was generated with "GET", "POST" or "PUT"
+		methodString = "GET"
+	}
+
+	supported := false
+	for _, method := range d.TempURLMethods {
+		if method == methodString {
+			supported = true
+			break
+		}
+	}
+
+	if !supported {
+		return "", storagedriver.ErrUnsupportedMethod
+	}
+
+	expiresTime := time.Now().Add(20 * time.Minute)
+	expires, ok := options["expiry"]
+	if ok {
+		et, ok := expires.(time.Time)
+		if ok {
+			expiresTime = et
+		}
+	}
+
+	tempURL := d.Conn.ObjectTempUrl(d.Container, d.swiftPath(path), d.SecretKey, methodString, expiresTime)
+
+	if d.AccessKey != "" {
+		// On HP Cloud, the signature must be in the form of tenant_id:access_key:signature
+		url, _ := url.Parse(tempURL)
+		query := url.Query()
+		query.Set("temp_url_sig", fmt.Sprintf("%s:%s:%s", d.Conn.TenantId, d.AccessKey, query.Get("temp_url_sig")))
+		url.RawQuery = query.Encode()
+		tempURL = url.String()
+	}
+
+	return tempURL, nil
 }
 
 func (d *driver) swiftPath(path string) string {
@@ -640,19 +755,6 @@ func (d *driver) createManifest(path string, segments string) error {
 	return nil
 }
 
-func detectBulkDelete(authURL string) (bulkDelete bool) {
-	resp, err := http.Get(gopath.Join(authURL, "..", "..") + "/info")
-	if err == nil {
-		defer resp.Body.Close()
-		decoder := json.NewDecoder(resp.Body)
-		var infos swiftInfo
-		if decoder.Decode(&infos) == nil {
-			_, bulkDelete = infos["bulk_delete"]
-		}
-	}
-	return
-}
-
 func parseManifest(manifest string) (container string, prefix string) {
 	components := strings.SplitN(manifest, "/", 2)
 	container = components[0]
@@ -661,3 +763,11 @@ func parseManifest(manifest string) (container string, prefix string) {
 	}
 	return container, prefix
 }
+
+func generateSecret() (string, error) {
+	var secretBytes [32]byte
+	if _, err := rand.Read(secretBytes[:]); err != nil {
+		return "", fmt.Errorf("could not generate random bytes for Swift secret key: %v", err)
+	}
+	return hex.EncodeToString(secretBytes[:]), nil
+}

registry/storage/driver/swift/swift_test.go

@@ -4,6 +4,7 @@ import (
 	"io/ioutil"
 	"os"
 	"strconv"
+	"strings"
 	"testing"
 
 	"github.com/ncw/swift/swifttest"
@@ -33,8 +34,13 @@ func init() {
 		container          string
 		region             string
 		insecureSkipVerify bool
-		swiftServer        *swifttest.SwiftServer
+		secretKey          string
-		err                error
+		accessKey          string
+		containerKey       bool
+		tempURLMethods     []string
+
+		swiftServer *swifttest.SwiftServer
+		err         error
 	)
 	username = os.Getenv("SWIFT_USERNAME")
 	password = os.Getenv("SWIFT_PASSWORD")
@@ -47,6 +53,10 @@ func init() {
 	container = os.Getenv("SWIFT_CONTAINER_NAME")
 	region = os.Getenv("SWIFT_REGION_NAME")
 	insecureSkipVerify, _ = strconv.ParseBool(os.Getenv("SWIFT_INSECURESKIPVERIFY"))
+	secretKey = os.Getenv("SWIFT_SECRET_KEY")
+	accessKey = os.Getenv("SWIFT_ACCESS_KEY")
+	containerKey, _ = strconv.ParseBool(os.Getenv("SWIFT_TEMPURL_CONTAINERKEY"))
+	tempURLMethods = strings.Split(os.Getenv("SWIFT_TEMPURL_METHODS"), ",")
 
 	if username == "" || password == "" || authURL == "" || container == "" {
 		if swiftServer, err = swifttest.NewSwiftServer("localhost"); err != nil {
@@ -79,6 +89,10 @@ func init() {
 			root,
 			insecureSkipVerify,
 			defaultChunkSize,
+			secretKey,
+			accessKey,
+			containerKey,
+			tempURLMethods,
 		}
 
 		return New(parameters)