TrueCloudLab/distribution
13
0
Fork 3
Code Issues 3 Pull requests Projects Releases 2 Packages Wiki Activity Actions

Merge pull request #1114 from lebauce/swift-temp-url

Browse source 
Redirect support in Swift driver
This commit is contained in:
Richard Scothern 2015-11-03 09:34:01 -08:00
commit a9da0e5100
10 changed files with 371 additions and 64 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Godeps/Godeps.json generated
View file

@ -122,7 +122,7 @@
},
{
"ImportPath": "github.com/ncw/swift",
"Rev": "ca8cbbde50d4e12dd8ad70b1bd66589ae98efc5c"
"Rev": "c54732e87b0b283d1baf0a18db689d0aea460ba3"
},
{
"ImportPath": "github.com/noahdesu/go-ceph/rados",

2
Godeps/_workspace/src/github.com/ncw/swift/README.md generated vendored
View file

@ -132,3 +132,5 @@ Contributors
- Dai HaoJun <haojun.dai@hp.com>
- Hua Wang <wanghua.humble@gmail.com>
- Fabian Ruff <fabian@progra.de>
- Arturo Reuschenbach Puncernau <reuschenbach@gmail.com>
- Petr Kotek <petr.kotek@bigcommerce.com>

1
Godeps/_workspace/src/github.com/ncw/swift/auth.go generated vendored
View file

@ -156,6 +156,7 @@ func (auth *v2Auth) Request(c *Connection) (*http.Request, error) {
return nil, err
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("User-Agent", c.UserAgent)
return req, nil
}

1
Godeps/_workspace/src/github.com/ncw/swift/auth_v3.go generated vendored
View file

@ -177,6 +177,7 @@ func (auth *v3Auth) Request(c *Connection) (*http.Request, error) {
return nil, err
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("User-Agent", c.UserAgent)
return req, nil
}

52
Godeps/_workspace/src/github.com/ncw/swift/swift.go generated vendored
View file

@ -392,6 +392,26 @@ func (c *Connection) authenticated() bool {
return c.StorageUrl != "" && c.AuthToken != ""
}
// SwiftInfo contains the JSON object returned by Swift when the /info
// route is queried. The object contains, among others, the Swift version,
// the enabled middlewares and their configuration
type SwiftInfo map[string]interface{}
// Discover Swift configuration by doing a request against /info
func (c *Connection) QueryInfo() (infos SwiftInfo, err error) {
infoUrl, err := url.Parse(c.StorageUrl)
if err != nil {
return nil, err
}
infoUrl.Path = path.Join(infoUrl.Path, "..", "..", "info")
resp, err := http.Get(infoUrl.String())
if err == nil {
err = readJson(resp, &infos)
return infos, err
}
return nil, err
}
// RequestOpts contains parameters for Connection.storage.
type RequestOpts struct {
Container string
@ -418,6 +438,10 @@ type RequestOpts struct {
// resp.Body.Close() must be called on it, unless noResponse is set in
// which case the body will be closed in this function
//
// If "Content-Length" is set in p.Headers it will be used - this can
// be used to override the default chunked transfer encoding for
// uploads.
//
// This will Authenticate if necessary, and re-authenticate if it
// receives a 401 error which means the token has expired
//
@ -433,8 +457,9 @@ func (c *Connection) Call(targetUrl string, p RequestOpts) (resp *http.Response,
var req *http.Request
for {
var authToken string
targetUrl, authToken, err = c.getUrlAndAuthToken(targetUrl, p.OnReAuth)
if targetUrl, authToken, err = c.getUrlAndAuthToken(targetUrl, p.OnReAuth); err != nil {
return //authentication failure
}
var URL *url.URL
URL, err = url.Parse(targetUrl)
if err != nil {
@ -460,18 +485,27 @@ func (c *Connection) Call(targetUrl string, p RequestOpts) (resp *http.Response,
}
if p.Headers != nil {
for k, v := range p.Headers {
req.Header.Add(k, v)
// Set ContentLength in req if the user passed it in in the headers
if k == "Content-Length" {
contentLength, err := strconv.ParseInt(v, 10, 64)
if err != nil {
return nil, nil, fmt.Errorf("Invalid %q header %q: %v", k, v, err)
}
req.ContentLength = contentLength
} else {
req.Header.Add(k, v)
}
}
}
req.Header.Add("User-Agent", DefaultUserAgent)
req.Header.Add("User-Agent", c.UserAgent)
req.Header.Add("X-Auth-Token", authToken)
resp, err = c.doTimeoutRequest(timer, req)
if err != nil {
if p.Operation == "HEAD" || p.Operation == "GET" {
if (p.Operation == "HEAD" || p.Operation == "GET") && retries > 0 {
retries--
continue
}
return
return nil, nil, err
}
// Check to see if token has expired
if resp.StatusCode == 401 && retries > 0 {
@ -566,7 +600,8 @@ func readJson(resp *http.Response, result interface{}) (err error) {
// ContainersOpts is options for Containers() and ContainerNames()
type ContainersOpts struct {
Limit int // For an integer value n, limits the number of results to at most n values.
Marker string // Given a string value x, return object names greater in value than the specified marker.
Prefix string // Given a string value x, return container names matching the specified prefix.
Marker string // Given a string value x, return container names greater in value than the specified marker.
EndMarker string // Given a string value x, return container names less in value than the specified marker.
Headers Headers // Any additional HTTP headers - can be nil
}
@ -579,6 +614,9 @@ func (opts *ContainersOpts) parse() (url.Values, Headers) {
if opts.Limit > 0 {
v.Set("limit", strconv.Itoa(opts.Limit))
}
if opts.Prefix != "" {
v.Set("prefix", opts.Prefix)
}
if opts.Marker != "" {
v.Set("marker", opts.Marker)
}

59
Godeps/_workspace/src/github.com/ncw/swift/swift_test.go generated vendored
View file

@ -65,6 +65,7 @@ func makeConnection() (*swift.Connection, error) {
UserName := os.Getenv("SWIFT_API_USER")
ApiKey := os.Getenv("SWIFT_API_KEY")
AuthUrl := os.Getenv("SWIFT_AUTH_URL")
Region := os.Getenv("SWIFT_REGION_NAME")
Insecure := os.Getenv("SWIFT_AUTH_INSECURE")
ConnectionChannelTimeout := os.Getenv("SWIFT_CONNECTION_CHANNEL_TIMEOUT")
@ -96,6 +97,7 @@ func makeConnection() (*swift.Connection, error) {
UserName: UserName,
ApiKey: ApiKey,
AuthUrl: AuthUrl,
Region: Region,
Transport: transport,
ConnectTimeout: 60 * time.Second,
Timeout: 60 * time.Second,
@ -601,6 +603,45 @@ func TestObjectPutString(t *testing.T) {
}
}
func TestObjectPut(t *testing.T) {
headers := swift.Headers{}
// Set content size incorrectly - should produce an error
headers["Content-Length"] = strconv.FormatInt(CONTENT_SIZE-1, 10)
contents := bytes.NewBufferString(CONTENTS)
h, err := c.ObjectPut(CONTAINER, OBJECT, contents, true, CONTENT_MD5, "text/plain", headers)
if err == nil {
t.Fatal("Expecting error but didn't get one")
}
// Now set content size correctly
contents = bytes.NewBufferString(CONTENTS)
headers["Content-Length"] = strconv.FormatInt(CONTENT_SIZE, 10)
h, err = c.ObjectPut(CONTAINER, OBJECT, contents, true, CONTENT_MD5, "text/plain", headers)
if err != nil {
t.Fatal(err)
}
if h["Etag"] != CONTENT_MD5 {
t.Errorf("Bad Etag want %q got %q", CONTENT_MD5, h["Etag"])
}
// Fetch object info and compare
info, _, err := c.Object(CONTAINER, OBJECT)
if err != nil {
t.Error(err)
}
if info.ContentType != "text/plain" {
t.Error("Bad content type", info.ContentType)
}
if info.Bytes != CONTENT_SIZE {
t.Error("Bad length")
}
if info.Hash != CONTENT_MD5 {
t.Error("Bad length")
}
}
func TestObjectEmpty(t *testing.T) {
err := c.ObjectPutString(CONTAINER, EMPTYOBJECT, "", "")
if err != nil {
@ -1493,6 +1534,14 @@ func TestTempUrl(t *testing.T) {
if content, err := ioutil.ReadAll(resp.Body); err != nil || string(content) != CONTENTS {
t.Error("Bad content", err)
}
resp, err := http.Post(tempUrl, "image/jpeg", bytes.NewReader([]byte(CONTENTS)))
if err != nil {
t.Fatal("Failed to retrieve file from temporary url")
}
if resp.StatusCode != 401 {
t.Fatal("Expecting server to forbid access to object")
}
}
resp.Body.Close()
@ -1500,7 +1549,17 @@ func TestTempUrl(t *testing.T) {
if err != nil {
t.Fatal(err)
}
}
func TestQueryInfo(t *testing.T) {
infos, err := c.QueryInfo()
if err != nil {
t.Log("Server doesn't support querying info")
return
}
if _, ok := infos["swift"]; !ok {
t.Fatal("No 'swift' section found in configuration")
}
}
func TestContainerDelete(t *testing.T) {

38
Godeps/_workspace/src/github.com/ncw/swift/swifttest/server.go generated vendored
View file

@ -443,7 +443,7 @@ func (objr objectResource) get(a *action) interface{} {
if obj, ok := item.(*object); ok {
length := len(obj.data)
size += length
sum.Write([]byte(components[0] + "/" + obj.name + "\n"))
sum.Write([]byte(hex.EncodeToString(obj.checksum)))
if start >= cursor+length {
continue
}
@ -668,24 +668,42 @@ func (s *SwiftServer) serveHTTP(w http.ResponseWriter, req *http.Request) {
panic(notAuthorized())
}
if req.URL.String() == "/info" {
jsonMarshal(w, &swift.SwiftInfo{
"swift": map[string]interface{}{
"version": "1.2",
},
"tempurl": map[string]interface{}{
"methods": []string{"GET", "HEAD", "PUT"},
},
})
return
}
r = s.resourceForURL(req.URL)
key := req.Header.Get("x-auth-token")
if key == "" {
secretKey := ""
signature := req.URL.Query().Get("temp_url_sig")
expires := req.URL.Query().Get("temp_url_expires")
signature := req.URL.Query().Get("temp_url_sig")
expires := req.URL.Query().Get("temp_url_expires")
if key == "" && signature != "" && expires != "" {
accountName, _, _, _ := s.parseURL(req.URL)
secretKey := ""
if account, ok := s.Accounts[accountName]; ok {
secretKey = account.meta.Get("X-Account-Meta-Temp-Url-Key")
}
mac := hmac.New(sha1.New, []byte(secretKey))
body := fmt.Sprintf("%s\n%s\n%s", req.Method, expires, req.URL.Path)
mac.Write([]byte(body))
expectedSignature := hex.EncodeToString(mac.Sum(nil))
get_hmac := func(method string) string {
mac := hmac.New(sha1.New, []byte(secretKey))
body := fmt.Sprintf("%s\n%s\n%s", method, expires, req.URL.Path)
mac.Write([]byte(body))
return hex.EncodeToString(mac.Sum(nil))
}
if signature != expectedSignature {
if req.Method == "HEAD" {
if signature != get_hmac("GET") && signature != get_hmac("POST") && signature != get_hmac("PUT") {
panic(notAuthorized())
}
} else if signature != get_hmac(req.Method) {
panic(notAuthorized())
}
} else {

64
docs/storage-drivers/swift.md
View file

@ -93,6 +93,16 @@ An implementation of the `storagedriver.StorageDriver` interface that uses [Open
</p>
</td>
</tr>
<tr>
<td>
<code>trustid</code>
</td>
<td>
<p>
Optionally, your OpenStack trust id for Identity v3 API.
</p>
</td>
</tr>
<tr>
<td>
<code>insecureskipverify</code>
@ -133,4 +143,58 @@ An implementation of the `storagedriver.StorageDriver` interface that uses [Open
</p>
</td>
</tr>
<tr>
<td>
<code>secretkey</code>
</td>
<td>
<p>
Optionally, the secret key used to generate temporary URLs.</p>
</p>
</td>
</tr>
<tr>
<td>
<code>accesskey</code>
</td>
<td>
<p>
Optionally, the access key to generate temporary URLs. It is used by HP Cloud Object Storage in addition to the `secretkey` parameter.</p>
</p>
</td>
</tr>
</table>
The features supported by the Swift server are queried by requesting the `/info` URL on the server. In case the administrator
disabled that feature, the configuration file can specify the following optional parameters :
<table>
<tr>
<td>
<code>tempurlcontainerkey</code>
</td>
<td>
<p>
Specify whether to use container secret key to generate temporary URL when set to true, or the account secret key otherwise.</p>
</p>
</td>
</tr>
<tr>
<td>
<code>tempurlmethods</code>
</td>
<td>
<p>
Array of HTTP methods that are supported by the TempURL middleware of the Swift server. Example:</p>
<code>
- tempurlmethods:
- GET
- PUT
- HEAD
- POST
- DELETE
</code>
</p>
</td>
</tr>
</table>

198
registry/storage/driver/swift/swift.go
View file

@ -7,9 +7,6 @@
// It supports both TempAuth authentication and Keystone authentication
// (up to version 3).
//
// Since Swift has no concept of directories (directories are an abstration),
// empty objects are created with the MIME type application/vnd.swift.directory.
//
// As Swift has a limit on the size of a single uploaded object (by default
// this is 5GB), the driver makes use of the Swift Large Object Support
// (http://docs.openstack.org/developer/swift/overview_large_objects.html).
@ -24,12 +21,11 @@ import (
"crypto/sha1"
"crypto/tls"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
gopath "path"
"net/url"
"strconv"
"strings"
"time"
@ -54,22 +50,34 @@ const minChunkSize = 1 << 20
// Parameters A struct that encapsulates all of the driver parameters after all values have been set
type Parameters struct {
Username string
Password string
AuthURL string
Tenant string
TenantID string
Domain string
DomainID string
TrustID string
Region string
Container string
Prefix string
InsecureSkipVerify bool
ChunkSize int
Username string
Password string
AuthURL string
Tenant string
TenantID string
Domain string
DomainID string
TrustID string
Region string
Container string
Prefix string
InsecureSkipVerify bool
ChunkSize int
SecretKey string
AccessKey string
TempURLContainerKey bool
TempURLMethods []string
}
type swiftInfo map[string]interface{}
// swiftInfo maps the JSON structure returned by Swift /info endpoint
type swiftInfo struct {
Swift struct {
Version string `mapstructure:"version"`
}
Tempurl struct {
Methods []string `mapstructure:"methods"`
}
}
func init() {
factory.Register(driverName, &swiftDriverFactory{})
@ -83,11 +91,15 @@ func (factory *swiftDriverFactory) Create(parameters map[string]interface{}) (st
}
type driver struct {
Conn swift.Connection
Container string
Prefix string
BulkDeleteSupport bool
ChunkSize int
Conn swift.Connection
Container string
Prefix string
BulkDeleteSupport bool
ChunkSize int
SecretKey string
AccessKey string
TempURLContainerKey bool
TempURLMethods []string
}
type baseEmbed struct {
@ -176,11 +188,65 @@ func New(params Parameters) (*Driver, error) {
}
d := &driver{
Conn: ct,
Container: params.Container,
Prefix: params.Prefix,
BulkDeleteSupport: detectBulkDelete(params.AuthURL),
ChunkSize: params.ChunkSize,
Conn: ct,
Container: params.Container,
Prefix: params.Prefix,
ChunkSize: params.ChunkSize,
TempURLMethods: make([]string, 0),
AccessKey: params.AccessKey,
}
info := swiftInfo{}
if config, err := d.Conn.QueryInfo(); err == nil {
_, d.BulkDeleteSupport = config["bulk_delete"]
if err := mapstructure.Decode(config, &info); err == nil {
d.TempURLContainerKey = info.Swift.Version >= "2.3.0"
d.TempURLMethods = info.Tempurl.Methods
}
} else {
d.TempURLContainerKey = params.TempURLContainerKey
d.TempURLMethods = params.TempURLMethods
}
if len(d.TempURLMethods) > 0 {
secretKey := params.SecretKey
if secretKey == "" {
secretKey, _ = generateSecret()
}
// Since Swift 2.2.2, we can now set secret keys on containers
// in addition to the account secret keys. Use them in preference.
if d.TempURLContainerKey {
_, containerHeaders, err := d.Conn.Container(d.Container)
if err != nil {
return nil, fmt.Errorf("Failed to fetch container info %s (%s)", d.Container, err)
}
d.SecretKey = containerHeaders["X-Container-Meta-Temp-Url-Key"]
if d.SecretKey == "" || (params.SecretKey != "" && d.SecretKey != params.SecretKey) {
m := swift.Metadata{}
m["temp-url-key"] = secretKey
if d.Conn.ContainerUpdate(d.Container, m.ContainerHeaders()); err == nil {
d.SecretKey = secretKey
}
}
} else {
// Use the account secret key
_, accountHeaders, err := d.Conn.Account()
if err != nil {
return nil, fmt.Errorf("Failed to fetch account info (%s)", err)
}
d.SecretKey = accountHeaders["X-Account-Meta-Temp-Url-Key"]
if d.SecretKey == "" || (params.SecretKey != "" && d.SecretKey != params.SecretKey) {
m := swift.Metadata{}
m["temp-url-key"] = secretKey
if err := d.Conn.AccountUpdate(m.AccountHeaders()); err == nil {
d.SecretKey = secretKey
}
}
}
}
return &Driver{
@ -590,9 +656,58 @@ func (d *driver) Delete(ctx context.Context, path string) error {
}
// URLFor returns a URL which may be used to retrieve the content stored at the given path.
// May return an UnsupportedMethodErr in certain StorageDriver implementations.
func (d *driver) URLFor(ctx context.Context, path string, options map[string]interface{}) (string, error) {
return "", storagedriver.ErrUnsupportedMethod
if d.SecretKey == "" {
return "", storagedriver.ErrUnsupportedMethod
}
methodString := "GET"
method, ok := options["method"]
if ok {
if methodString, ok = method.(string); !ok {
return "", storagedriver.ErrUnsupportedMethod
}
}
if methodString == "HEAD" {
// A "HEAD" request on a temporary URL is allowed if the
// signature was generated with "GET", "POST" or "PUT"
methodString = "GET"
}
supported := false
for _, method := range d.TempURLMethods {
if method == methodString {
supported = true
break
}
}
if !supported {
return "", storagedriver.ErrUnsupportedMethod
}
expiresTime := time.Now().Add(20 * time.Minute)
expires, ok := options["expiry"]
if ok {
et, ok := expires.(time.Time)
if ok {
expiresTime = et
}
}
tempURL := d.Conn.ObjectTempUrl(d.Container, d.swiftPath(path), d.SecretKey, methodString, expiresTime)
if d.AccessKey != "" {
// On HP Cloud, the signature must be in the form of tenant_id:access_key:signature
url, _ := url.Parse(tempURL)
query := url.Query()
query.Set("temp_url_sig", fmt.Sprintf("%s:%s:%s", d.Conn.TenantId, d.AccessKey, query.Get("temp_url_sig")))
url.RawQuery = query.Encode()
tempURL = url.String()
}
return tempURL, nil
}
func (d *driver) swiftPath(path string) string {
@ -640,19 +755,6 @@ func (d *driver) createManifest(path string, segments string) error {
return nil
}
func detectBulkDelete(authURL string) (bulkDelete bool) {
resp, err := http.Get(gopath.Join(authURL, "..", "..") + "/info")
if err == nil {
defer resp.Body.Close()
decoder := json.NewDecoder(resp.Body)
var infos swiftInfo
if decoder.Decode(&infos) == nil {
_, bulkDelete = infos["bulk_delete"]
}
}
return
}
func parseManifest(manifest string) (container string, prefix string) {
components := strings.SplitN(manifest, "/", 2)
container = components[0]
@ -661,3 +763,11 @@ func parseManifest(manifest string) (container string, prefix string) {
}
return container, prefix
}
func generateSecret() (string, error) {
var secretBytes [32]byte
if _, err := rand.Read(secretBytes[:]); err != nil {
return "", fmt.Errorf("could not generate random bytes for Swift secret key: %v", err)
}
return hex.EncodeToString(secretBytes[:]), nil
}

18
registry/storage/driver/swift/swift_test.go
View file

@ -4,6 +4,7 @@ import (
"io/ioutil"
"os"
"strconv"
"strings"
"testing"
"github.com/ncw/swift/swifttest"
@ -33,8 +34,13 @@ func init() {
container string
region string
insecureSkipVerify bool
swiftServer *swifttest.SwiftServer
err error
secretKey string
accessKey string
containerKey bool
tempURLMethods []string
swiftServer *swifttest.SwiftServer
err error
)
username = os.Getenv("SWIFT_USERNAME")
password = os.Getenv("SWIFT_PASSWORD")
@ -47,6 +53,10 @@ func init() {
container = os.Getenv("SWIFT_CONTAINER_NAME")
region = os.Getenv("SWIFT_REGION_NAME")
insecureSkipVerify, _ = strconv.ParseBool(os.Getenv("SWIFT_INSECURESKIPVERIFY"))
secretKey = os.Getenv("SWIFT_SECRET_KEY")
accessKey = os.Getenv("SWIFT_ACCESS_KEY")
containerKey, _ = strconv.ParseBool(os.Getenv("SWIFT_TEMPURL_CONTAINERKEY"))
tempURLMethods = strings.Split(os.Getenv("SWIFT_TEMPURL_METHODS"), ",")
if username == "" || password == "" || authURL == "" || container == "" {
if swiftServer, err = swifttest.NewSwiftServer("localhost"); err != nil {
@ -79,6 +89,10 @@ func init() {
root,
insecureSkipVerify,
defaultChunkSize,
secretKey,
accessKey,
containerKey,
tempURLMethods,
}
return New(parameters)