Add support for temporary URL for Swift driver

Signed-off-by: Sylvain Baubeau <sbaubeau@redhat.com>
This commit is contained in:
Sylvain Baubeau 2015-11-03 09:59:50 +01:00
parent 7759153f2f
commit 7c3281861f
3 changed files with 234 additions and 46 deletions

View file

@ -93,6 +93,16 @@ An implementation of the `storagedriver.StorageDriver` interface that uses [Open
</p> </p>
</td> </td>
</tr> </tr>
<tr>
<td>
<code>trustid</code>
</td>
<td>
<p>
Optionally, your OpenStack trust id for Identity v3 API.
</p>
</td>
</tr>
<tr> <tr>
<td> <td>
<code>insecureskipverify</code> <code>insecureskipverify</code>
@ -133,4 +143,58 @@ An implementation of the `storagedriver.StorageDriver` interface that uses [Open
</p> </p>
</td> </td>
</tr> </tr>
<tr>
<td>
<code>secretkey</code>
</td>
<td>
<p>
Optionally, the secret key used to generate temporary URLs.</p>
</p>
</td>
</tr>
<tr>
<td>
<code>accesskey</code>
</td>
<td>
<p>
Optionally, the access key to generate temporary URLs. It is used by HP Cloud Object Storage in addition to the `secretkey` parameter.</p>
</p>
</td>
</tr>
</table>
The features supported by the Swift server are queried by requesting the `/info` URL on the server. In case the administrator
disabled that feature, the configuration file can specify the following optional parameters :
<table>
<tr>
<td>
<code>tempurlcontainerkey</code>
</td>
<td>
<p>
Specify whether to use container secret key to generate temporary URL when set to true, or the account secret key otherwise.</p>
</p>
</td>
</tr>
<tr>
<td>
<code>tempurlmethods</code>
</td>
<td>
<p>
Array of HTTP methods that are supported by the TempURL middleware of the Swift server. Example:</p>
<code>
- tempurlmethods:
- GET
- PUT
- HEAD
- POST
- DELETE
</code>
</p>
</td>
</tr>
</table> </table>

View file

@ -7,9 +7,6 @@
// It supports both TempAuth authentication and Keystone authentication // It supports both TempAuth authentication and Keystone authentication
// (up to version 3). // (up to version 3).
// //
// Since Swift has no concept of directories (directories are an abstration),
// empty objects are created with the MIME type application/vnd.swift.directory.
//
// As Swift has a limit on the size of a single uploaded object (by default // As Swift has a limit on the size of a single uploaded object (by default
// this is 5GB), the driver makes use of the Swift Large Object Support // this is 5GB), the driver makes use of the Swift Large Object Support
// (http://docs.openstack.org/developer/swift/overview_large_objects.html). // (http://docs.openstack.org/developer/swift/overview_large_objects.html).
@ -24,12 +21,11 @@ import (
"crypto/sha1" "crypto/sha1"
"crypto/tls" "crypto/tls"
"encoding/hex" "encoding/hex"
"encoding/json"
"fmt" "fmt"
"io" "io"
"io/ioutil" "io/ioutil"
"net/http" "net/http"
gopath "path" "net/url"
"strconv" "strconv"
"strings" "strings"
"time" "time"
@ -54,22 +50,34 @@ const minChunkSize = 1 << 20
// Parameters A struct that encapsulates all of the driver parameters after all values have been set // Parameters A struct that encapsulates all of the driver parameters after all values have been set
type Parameters struct { type Parameters struct {
Username string Username string
Password string Password string
AuthURL string AuthURL string
Tenant string Tenant string
TenantID string TenantID string
Domain string Domain string
DomainID string DomainID string
TrustID string TrustID string
Region string Region string
Container string Container string
Prefix string Prefix string
InsecureSkipVerify bool InsecureSkipVerify bool
ChunkSize int ChunkSize int
SecretKey string
AccessKey string
TempURLContainerKey bool
TempURLMethods []string
} }
type swiftInfo map[string]interface{} // swiftInfo maps the JSON structure returned by Swift /info endpoint
type swiftInfo struct {
Swift struct {
Version string `mapstructure:"version"`
}
Tempurl struct {
Methods []string `mapstructure:"methods"`
}
}
func init() { func init() {
factory.Register(driverName, &swiftDriverFactory{}) factory.Register(driverName, &swiftDriverFactory{})
@ -83,11 +91,15 @@ func (factory *swiftDriverFactory) Create(parameters map[string]interface{}) (st
} }
type driver struct { type driver struct {
Conn swift.Connection Conn swift.Connection
Container string Container string
Prefix string Prefix string
BulkDeleteSupport bool BulkDeleteSupport bool
ChunkSize int ChunkSize int
SecretKey string
AccessKey string
TempURLContainerKey bool
TempURLMethods []string
} }
type baseEmbed struct { type baseEmbed struct {
@ -176,11 +188,65 @@ func New(params Parameters) (*Driver, error) {
} }
d := &driver{ d := &driver{
Conn: ct, Conn: ct,
Container: params.Container, Container: params.Container,
Prefix: params.Prefix, Prefix: params.Prefix,
BulkDeleteSupport: detectBulkDelete(params.AuthURL), ChunkSize: params.ChunkSize,
ChunkSize: params.ChunkSize, TempURLMethods: make([]string, 0),
AccessKey: params.AccessKey,
}
info := swiftInfo{}
if config, err := d.Conn.QueryInfo(); err == nil {
_, d.BulkDeleteSupport = config["bulk_delete"]
if err := mapstructure.Decode(config, &info); err == nil {
d.TempURLContainerKey = info.Swift.Version >= "2.3.0"
d.TempURLMethods = info.Tempurl.Methods
}
} else {
d.TempURLContainerKey = params.TempURLContainerKey
d.TempURLMethods = params.TempURLMethods
}
if len(d.TempURLMethods) > 0 {
secretKey := params.SecretKey
if secretKey == "" {
secretKey, _ = generateSecret()
}
// Since Swift 2.2.2, we can now set secret keys on containers
// in addition to the account secret keys. Use them in preference.
if d.TempURLContainerKey {
_, containerHeaders, err := d.Conn.Container(d.Container)
if err != nil {
return nil, fmt.Errorf("Failed to fetch container info %s (%s)", d.Container, err)
}
d.SecretKey = containerHeaders["X-Container-Meta-Temp-Url-Key"]
if d.SecretKey == "" || (params.SecretKey != "" && d.SecretKey != params.SecretKey) {
m := swift.Metadata{}
m["temp-url-key"] = secretKey
if d.Conn.ContainerUpdate(d.Container, m.ContainerHeaders()); err == nil {
d.SecretKey = secretKey
}
}
} else {
// Use the account secret key
_, accountHeaders, err := d.Conn.Account()
if err != nil {
return nil, fmt.Errorf("Failed to fetch account info (%s)", err)
}
d.SecretKey = accountHeaders["X-Account-Meta-Temp-Url-Key"]
if d.SecretKey == "" || (params.SecretKey != "" && d.SecretKey != params.SecretKey) {
m := swift.Metadata{}
m["temp-url-key"] = secretKey
if err := d.Conn.AccountUpdate(m.AccountHeaders()); err == nil {
d.SecretKey = secretKey
}
}
}
} }
return &Driver{ return &Driver{
@ -590,9 +656,58 @@ func (d *driver) Delete(ctx context.Context, path string) error {
} }
// URLFor returns a URL which may be used to retrieve the content stored at the given path. // URLFor returns a URL which may be used to retrieve the content stored at the given path.
// May return an UnsupportedMethodErr in certain StorageDriver implementations.
func (d *driver) URLFor(ctx context.Context, path string, options map[string]interface{}) (string, error) { func (d *driver) URLFor(ctx context.Context, path string, options map[string]interface{}) (string, error) {
return "", storagedriver.ErrUnsupportedMethod if d.SecretKey == "" {
return "", storagedriver.ErrUnsupportedMethod
}
methodString := "GET"
method, ok := options["method"]
if ok {
if methodString, ok = method.(string); !ok {
return "", storagedriver.ErrUnsupportedMethod
}
}
if methodString == "HEAD" {
// A "HEAD" request on a temporary URL is allowed if the
// signature was generated with "GET", "POST" or "PUT"
methodString = "GET"
}
supported := false
for _, method := range d.TempURLMethods {
if method == methodString {
supported = true
break
}
}
if !supported {
return "", storagedriver.ErrUnsupportedMethod
}
expiresTime := time.Now().Add(20 * time.Minute)
expires, ok := options["expiry"]
if ok {
et, ok := expires.(time.Time)
if ok {
expiresTime = et
}
}
tempURL := d.Conn.ObjectTempUrl(d.Container, d.swiftPath(path), d.SecretKey, methodString, expiresTime)
if d.AccessKey != "" {
// On HP Cloud, the signature must be in the form of tenant_id:access_key:signature
url, _ := url.Parse(tempURL)
query := url.Query()
query.Set("temp_url_sig", fmt.Sprintf("%s:%s:%s", d.Conn.TenantId, d.AccessKey, query.Get("temp_url_sig")))
url.RawQuery = query.Encode()
tempURL = url.String()
}
return tempURL, nil
} }
func (d *driver) swiftPath(path string) string { func (d *driver) swiftPath(path string) string {
@ -640,19 +755,6 @@ func (d *driver) createManifest(path string, segments string) error {
return nil return nil
} }
func detectBulkDelete(authURL string) (bulkDelete bool) {
resp, err := http.Get(gopath.Join(authURL, "..", "..") + "/info")
if err == nil {
defer resp.Body.Close()
decoder := json.NewDecoder(resp.Body)
var infos swiftInfo
if decoder.Decode(&infos) == nil {
_, bulkDelete = infos["bulk_delete"]
}
}
return
}
func parseManifest(manifest string) (container string, prefix string) { func parseManifest(manifest string) (container string, prefix string) {
components := strings.SplitN(manifest, "/", 2) components := strings.SplitN(manifest, "/", 2)
container = components[0] container = components[0]
@ -661,3 +763,11 @@ func parseManifest(manifest string) (container string, prefix string) {
} }
return container, prefix return container, prefix
} }
func generateSecret() (string, error) {
var secretBytes [32]byte
if _, err := rand.Read(secretBytes[:]); err != nil {
return "", fmt.Errorf("could not generate random bytes for Swift secret key: %v", err)
}
return hex.EncodeToString(secretBytes[:]), nil
}

View file

@ -4,6 +4,7 @@ import (
"io/ioutil" "io/ioutil"
"os" "os"
"strconv" "strconv"
"strings"
"testing" "testing"
"github.com/ncw/swift/swifttest" "github.com/ncw/swift/swifttest"
@ -33,8 +34,13 @@ func init() {
container string container string
region string region string
insecureSkipVerify bool insecureSkipVerify bool
swiftServer *swifttest.SwiftServer secretKey string
err error accessKey string
containerKey bool
tempURLMethods []string
swiftServer *swifttest.SwiftServer
err error
) )
username = os.Getenv("SWIFT_USERNAME") username = os.Getenv("SWIFT_USERNAME")
password = os.Getenv("SWIFT_PASSWORD") password = os.Getenv("SWIFT_PASSWORD")
@ -47,6 +53,10 @@ func init() {
container = os.Getenv("SWIFT_CONTAINER_NAME") container = os.Getenv("SWIFT_CONTAINER_NAME")
region = os.Getenv("SWIFT_REGION_NAME") region = os.Getenv("SWIFT_REGION_NAME")
insecureSkipVerify, _ = strconv.ParseBool(os.Getenv("SWIFT_INSECURESKIPVERIFY")) insecureSkipVerify, _ = strconv.ParseBool(os.Getenv("SWIFT_INSECURESKIPVERIFY"))
secretKey = os.Getenv("SWIFT_SECRET_KEY")
accessKey = os.Getenv("SWIFT_ACCESS_KEY")
containerKey, _ = strconv.ParseBool(os.Getenv("SWIFT_TEMPURL_CONTAINERKEY"))
tempURLMethods = strings.Split(os.Getenv("SWIFT_TEMPURL_METHODS"), ",")
if username == "" || password == "" || authURL == "" || container == "" { if username == "" || password == "" || authURL == "" || container == "" {
if swiftServer, err = swifttest.NewSwiftServer("localhost"); err != nil { if swiftServer, err = swifttest.NewSwiftServer("localhost"); err != nil {
@ -79,6 +89,10 @@ func init() {
root, root,
insecureSkipVerify, insecureSkipVerify,
defaultChunkSize, defaultChunkSize,
secretKey,
accessKey,
containerKey,
tempURLMethods,
} }
return New(parameters) return New(parameters)