forked from TrueCloudLab/distribution
Update Godeps for Aliyun OSS
Signed-off-by: Li Yi <denverdino@gmail.com>
This commit is contained in:
parent
c3b42db014
commit
38b23c8dff
18 changed files with 2830 additions and 0 deletions
8
Godeps/Godeps.json
generated
8
Godeps/Godeps.json
generated
|
@ -44,6 +44,14 @@
|
|||
"Comment": "1.2.0-66-g6086d79",
|
||||
"Rev": "6086d7927ec35315964d9fea46df6c04e6d697c1"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/denverdino/aliyungo/oss",
|
||||
"Rev": "17d1e888c907ffdbd875f37500f3d130ce2ee6eb"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/denverdino/aliyungo/util",
|
||||
"Rev": "17d1e888c907ffdbd875f37500f3d130ce2ee6eb"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/docker/docker/pkg/tarsum",
|
||||
"Comment": "v1.4.1-3932-gb63ec6e",
|
||||
|
|
1252
Godeps/_workspace/src/github.com/denverdino/aliyungo/oss/client.go
generated
vendored
Normal file
1252
Godeps/_workspace/src/github.com/denverdino/aliyungo/oss/client.go
generated
vendored
Normal file
File diff suppressed because it is too large
Load diff
211
Godeps/_workspace/src/github.com/denverdino/aliyungo/oss/client_test.go
generated
vendored
Normal file
211
Godeps/_workspace/src/github.com/denverdino/aliyungo/oss/client_test.go
generated
vendored
Normal file
|
@ -0,0 +1,211 @@
|
|||
package oss_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io/ioutil"
|
||||
//"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/denverdino/aliyungo/oss"
|
||||
)
|
||||
|
||||
var (
|
||||
//If you test on ECS, you can set the internal param to true
|
||||
client = oss.NewOSSClient(TestRegion, false, TestAccessKeyId, TestAccessKeySecret, false)
|
||||
)
|
||||
|
||||
func TestCreateBucket(t *testing.T) {
|
||||
|
||||
b := client.Bucket(TestBucket)
|
||||
err := b.PutBucket(oss.Private)
|
||||
if err != nil {
|
||||
t.Errorf("Failed for PutBucket: %v", err)
|
||||
}
|
||||
t.Log("Wait a while for bucket creation ...")
|
||||
time.Sleep(10 * time.Second)
|
||||
}
|
||||
|
||||
func TestHead(t *testing.T) {
|
||||
|
||||
b := client.Bucket(TestBucket)
|
||||
_, err := b.Head("name", nil)
|
||||
|
||||
if err == nil {
|
||||
t.Errorf("Failed for Head: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPutObject(t *testing.T) {
|
||||
const DISPOSITION = "attachment; filename=\"0x1a2b3c.jpg\""
|
||||
|
||||
b := client.Bucket(TestBucket)
|
||||
err := b.Put("name", []byte("content"), "content-type", oss.Private, oss.Options{ContentDisposition: DISPOSITION})
|
||||
if err != nil {
|
||||
t.Errorf("Failed for Put: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGet(t *testing.T) {
|
||||
|
||||
b := client.Bucket(TestBucket)
|
||||
data, err := b.Get("name")
|
||||
|
||||
if err != nil || string(data) != "content" {
|
||||
t.Errorf("Failed for Get: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestURL(t *testing.T) {
|
||||
|
||||
b := client.Bucket(TestBucket)
|
||||
url := b.URL("name")
|
||||
|
||||
t.Log("URL: ", url)
|
||||
// /c.Assert(req.URL.Path, check.Equals, "/denverdino_test/name")
|
||||
}
|
||||
|
||||
func TestGetReader(t *testing.T) {
|
||||
|
||||
b := client.Bucket(TestBucket)
|
||||
rc, err := b.GetReader("name")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed for GetReader: %v", err)
|
||||
}
|
||||
data, err := ioutil.ReadAll(rc)
|
||||
rc.Close()
|
||||
if err != nil || string(data) != "content" {
|
||||
t.Errorf("Failed for ReadAll: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func aTestGetNotFound(t *testing.T) {
|
||||
|
||||
b := client.Bucket("non-existent-bucket")
|
||||
_, err := b.Get("non-existent")
|
||||
if err == nil {
|
||||
t.Fatalf("Failed for TestGetNotFound: %v", err)
|
||||
}
|
||||
ossErr, _ := err.(*oss.Error)
|
||||
if ossErr.StatusCode != 404 || ossErr.BucketName != "non-existent-bucket" {
|
||||
t.Errorf("Failed for TestGetNotFound: %v", err)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestPutCopy(t *testing.T) {
|
||||
b := client.Bucket(TestBucket)
|
||||
t.Log("Source: ", b.Path("name"))
|
||||
res, err := b.PutCopy("newname", oss.Private, oss.CopyOptions{},
|
||||
b.Path("name"))
|
||||
if err == nil {
|
||||
t.Logf("Copy result: %v", res)
|
||||
} else {
|
||||
t.Errorf("Failed for PutCopy: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestList(t *testing.T) {
|
||||
|
||||
b := client.Bucket(TestBucket)
|
||||
|
||||
data, err := b.List("n", "", "", 0)
|
||||
if err != nil || len(data.Contents) != 2 {
|
||||
t.Errorf("Failed for List: %v", err)
|
||||
} else {
|
||||
t.Logf("Contents = %++v", data)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListWithDelimiter(t *testing.T) {
|
||||
|
||||
b := client.Bucket(TestBucket)
|
||||
|
||||
data, err := b.List("photos/2006/", "/", "some-marker", 1000)
|
||||
if err != nil || len(data.Contents) != 0 {
|
||||
t.Errorf("Failed for List: %v", err)
|
||||
} else {
|
||||
t.Logf("Contents = %++v", data)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestPutReader(t *testing.T) {
|
||||
|
||||
b := client.Bucket(TestBucket)
|
||||
buf := bytes.NewBufferString("content")
|
||||
err := b.PutReader("name", buf, int64(buf.Len()), "content-type", oss.Private, oss.Options{})
|
||||
if err != nil {
|
||||
t.Errorf("Failed for PutReader: %v", err)
|
||||
}
|
||||
TestGetReader(t)
|
||||
}
|
||||
|
||||
func TestExists(t *testing.T) {
|
||||
|
||||
b := client.Bucket(TestBucket)
|
||||
result, err := b.Exists("name")
|
||||
if err != nil || result != true {
|
||||
t.Errorf("Failed for Exists: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocation(t *testing.T) {
|
||||
b := client.Bucket(TestBucket)
|
||||
result, err := b.Location()
|
||||
|
||||
if err != nil || result != string(TestRegion) {
|
||||
t.Errorf("Failed for Location: %v %s", err, result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestACL(t *testing.T) {
|
||||
b := client.Bucket(TestBucket)
|
||||
result, err := b.ACL()
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("Failed for ACL: %v", err)
|
||||
} else {
|
||||
t.Logf("AccessControlPolicy: %++v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelObject(t *testing.T) {
|
||||
|
||||
b := client.Bucket(TestBucket)
|
||||
err := b.Del("name")
|
||||
if err != nil {
|
||||
t.Errorf("Failed for Del: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelMultiObjects(t *testing.T) {
|
||||
|
||||
b := client.Bucket(TestBucket)
|
||||
objects := []oss.Object{oss.Object{Key: "newname"}}
|
||||
err := b.DelMulti(oss.Delete{
|
||||
Quiet: false,
|
||||
Objects: objects,
|
||||
})
|
||||
if err != nil {
|
||||
t.Errorf("Failed for DelMulti: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetService(t *testing.T) {
|
||||
bucketList, err := client.GetService()
|
||||
if err != nil {
|
||||
t.Errorf("Unable to get service: %v", err)
|
||||
} else {
|
||||
t.Logf("GetService: %++v", bucketList)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelBucket(t *testing.T) {
|
||||
|
||||
b := client.Bucket(TestBucket)
|
||||
err := b.DelBucket()
|
||||
if err != nil {
|
||||
t.Errorf("Failed for DelBucket: %v", err)
|
||||
}
|
||||
}
|
14
Godeps/_workspace/src/github.com/denverdino/aliyungo/oss/config_test.go
generated
vendored
Normal file
14
Godeps/_workspace/src/github.com/denverdino/aliyungo/oss/config_test.go
generated
vendored
Normal file
|
@ -0,0 +1,14 @@
|
|||
package oss_test
|
||||
|
||||
import (
|
||||
"github.com/denverdino/aliyungo/oss"
|
||||
)
|
||||
|
||||
//Modify with your Access Key Id and Access Key Secret
|
||||
const (
|
||||
TestAccessKeyId = "MY_ACCESS_KEY_ID"
|
||||
TestAccessKeySecret = "MY_ACCESS_KEY_ID"
|
||||
TestIAmRich = false
|
||||
TestRegion = oss.Beijing
|
||||
TestBucket = "denverdino"
|
||||
)
|
23
Godeps/_workspace/src/github.com/denverdino/aliyungo/oss/export.go
generated
vendored
Normal file
23
Godeps/_workspace/src/github.com/denverdino/aliyungo/oss/export.go
generated
vendored
Normal file
|
@ -0,0 +1,23 @@
|
|||
package oss
|
||||
|
||||
import (
|
||||
"github.com/denverdino/aliyungo/util"
|
||||
)
|
||||
|
||||
var originalStrategy = attempts
|
||||
|
||||
func SetAttemptStrategy(s *util.AttemptStrategy) {
|
||||
if s == nil {
|
||||
attempts = originalStrategy
|
||||
} else {
|
||||
attempts = *s
|
||||
}
|
||||
}
|
||||
|
||||
func SetListPartsMax(n int) {
|
||||
listPartsMax = n
|
||||
}
|
||||
|
||||
func SetListMultiMax(n int) {
|
||||
listMultiMax = n
|
||||
}
|
460
Godeps/_workspace/src/github.com/denverdino/aliyungo/oss/multi.go
generated
vendored
Normal file
460
Godeps/_workspace/src/github.com/denverdino/aliyungo/oss/multi.go
generated
vendored
Normal file
|
@ -0,0 +1,460 @@
|
|||
package oss
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/md5"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"io"
|
||||
//"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Multi represents an unfinished multipart upload.
|
||||
//
|
||||
// Multipart uploads allow sending big objects in smaller chunks.
|
||||
// After all parts have been sent, the upload must be explicitly
|
||||
// completed by calling Complete with the list of parts.
|
||||
|
||||
type Multi struct {
|
||||
Bucket *Bucket
|
||||
Key string
|
||||
UploadId string
|
||||
}
|
||||
|
||||
// That's the default. Here just for testing.
|
||||
var listMultiMax = 1000
|
||||
|
||||
type listMultiResp struct {
|
||||
NextKeyMarker string
|
||||
NextUploadIdMarker string
|
||||
IsTruncated bool
|
||||
Upload []Multi
|
||||
CommonPrefixes []string `xml:"CommonPrefixes>Prefix"`
|
||||
}
|
||||
|
||||
// ListMulti returns the list of unfinished multipart uploads in b.
|
||||
//
|
||||
// The prefix parameter limits the response to keys that begin with the
|
||||
// specified prefix. You can use prefixes to separate a bucket into different
|
||||
// groupings of keys (to get the feeling of folders, for example).
|
||||
//
|
||||
// The delim parameter causes the response to group all of the keys that
|
||||
// share a common prefix up to the next delimiter in a single entry within
|
||||
// the CommonPrefixes field. You can use delimiters to separate a bucket
|
||||
// into different groupings of keys, similar to how folders would work.
|
||||
//
|
||||
func (b *Bucket) ListMulti(prefix, delim string) (multis []*Multi, prefixes []string, err error) {
|
||||
params := make(url.Values)
|
||||
params.Set("uploads", "")
|
||||
params.Set("max-uploads", strconv.FormatInt(int64(listMultiMax), 10))
|
||||
params.Set("prefix", prefix)
|
||||
params.Set("delimiter", delim)
|
||||
|
||||
for attempt := attempts.Start(); attempt.Next(); {
|
||||
req := &request{
|
||||
method: "GET",
|
||||
bucket: b.Name,
|
||||
params: params,
|
||||
}
|
||||
var resp listMultiResp
|
||||
err := b.Client.query(req, &resp)
|
||||
if shouldRetry(err) && attempt.HasNext() {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
for i := range resp.Upload {
|
||||
multi := &resp.Upload[i]
|
||||
multi.Bucket = b
|
||||
multis = append(multis, multi)
|
||||
}
|
||||
prefixes = append(prefixes, resp.CommonPrefixes...)
|
||||
if !resp.IsTruncated {
|
||||
return multis, prefixes, nil
|
||||
}
|
||||
params.Set("key-marker", resp.NextKeyMarker)
|
||||
params.Set("upload-id-marker", resp.NextUploadIdMarker)
|
||||
attempt = attempts.Start() // Last request worked.
|
||||
}
|
||||
panic("unreachable")
|
||||
}
|
||||
|
||||
// Multi returns a multipart upload handler for the provided key
|
||||
// inside b. If a multipart upload exists for key, it is returned,
|
||||
// otherwise a new multipart upload is initiated with contType and perm.
|
||||
func (b *Bucket) Multi(key, contType string, perm ACL, options Options) (*Multi, error) {
|
||||
multis, _, err := b.ListMulti(key, "")
|
||||
if err != nil && !hasCode(err, "NoSuchUpload") {
|
||||
return nil, err
|
||||
}
|
||||
for _, m := range multis {
|
||||
if m.Key == key {
|
||||
return m, nil
|
||||
}
|
||||
}
|
||||
return b.InitMulti(key, contType, perm, options)
|
||||
}
|
||||
|
||||
// InitMulti initializes a new multipart upload at the provided
|
||||
// key inside b and returns a value for manipulating it.
|
||||
//
|
||||
func (b *Bucket) InitMulti(key string, contType string, perm ACL, options Options) (*Multi, error) {
|
||||
headers := make(http.Header)
|
||||
headers.Set("Content-Length", "0")
|
||||
headers.Set("Content-Type", contType)
|
||||
headers.Set("x-oss-acl", string(perm))
|
||||
|
||||
options.addHeaders(headers)
|
||||
params := make(url.Values)
|
||||
params.Set("uploads", "")
|
||||
req := &request{
|
||||
method: "POST",
|
||||
bucket: b.Name,
|
||||
path: key,
|
||||
headers: headers,
|
||||
params: params,
|
||||
}
|
||||
var err error
|
||||
var resp struct {
|
||||
UploadId string `xml:"UploadId"`
|
||||
}
|
||||
for attempt := attempts.Start(); attempt.Next(); {
|
||||
err = b.Client.query(req, &resp)
|
||||
if !shouldRetry(err) {
|
||||
break
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Multi{Bucket: b, Key: key, UploadId: resp.UploadId}, nil
|
||||
}
|
||||
|
||||
func (m *Multi) PutPartCopy(n int, options CopyOptions, source string) (*CopyObjectResult, Part, error) {
|
||||
headers := make(http.Header)
|
||||
headers.Set("x-oss-copy-source", source)
|
||||
|
||||
options.addHeaders(headers)
|
||||
params := make(url.Values)
|
||||
params.Set("uploadId", m.UploadId)
|
||||
params.Set("partNumber", strconv.FormatInt(int64(n), 10))
|
||||
|
||||
sourceBucket := m.Bucket.Client.Bucket(strings.TrimRight(strings.Split(source, "/")[1], "/"))
|
||||
//log.Println("source: ", source)
|
||||
//log.Println("sourceBucket: ", sourceBucket.Name)
|
||||
//log.Println("HEAD: ", strings.Split(source, "/")[2])
|
||||
sourceMeta, err := sourceBucket.Head(strings.Split(source, "/")[2], nil)
|
||||
if err != nil {
|
||||
return nil, Part{}, err
|
||||
}
|
||||
|
||||
for attempt := attempts.Start(); attempt.Next(); {
|
||||
req := &request{
|
||||
method: "PUT",
|
||||
bucket: m.Bucket.Name,
|
||||
path: m.Key,
|
||||
headers: headers,
|
||||
params: params,
|
||||
}
|
||||
resp := &CopyObjectResult{}
|
||||
err = m.Bucket.Client.query(req, resp)
|
||||
if shouldRetry(err) && attempt.HasNext() {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, Part{}, err
|
||||
}
|
||||
if resp.ETag == "" {
|
||||
return nil, Part{}, errors.New("part upload succeeded with no ETag")
|
||||
}
|
||||
return resp, Part{n, resp.ETag, sourceMeta.ContentLength}, nil
|
||||
}
|
||||
panic("unreachable")
|
||||
}
|
||||
|
||||
// PutPart sends part n of the multipart upload, reading all the content from r.
|
||||
// Each part, except for the last one, must be at least 5MB in size.
|
||||
//
|
||||
func (m *Multi) PutPart(n int, r io.ReadSeeker) (Part, error) {
|
||||
partSize, _, md5b64, err := seekerInfo(r)
|
||||
if err != nil {
|
||||
return Part{}, err
|
||||
}
|
||||
return m.putPart(n, r, partSize, md5b64)
|
||||
}
|
||||
|
||||
func (m *Multi) putPart(n int, r io.ReadSeeker, partSize int64, md5b64 string) (Part, error) {
|
||||
headers := make(http.Header)
|
||||
headers.Set("Content-Length", strconv.FormatInt(partSize, 10))
|
||||
headers.Set("Content-MD5", md5b64)
|
||||
|
||||
params := make(url.Values)
|
||||
params.Set("uploadId", m.UploadId)
|
||||
params.Set("partNumber", strconv.FormatInt(int64(n), 10))
|
||||
|
||||
for attempt := attempts.Start(); attempt.Next(); {
|
||||
_, err := r.Seek(0, 0)
|
||||
if err != nil {
|
||||
return Part{}, err
|
||||
}
|
||||
req := &request{
|
||||
method: "PUT",
|
||||
bucket: m.Bucket.Name,
|
||||
path: m.Key,
|
||||
headers: headers,
|
||||
params: params,
|
||||
payload: r,
|
||||
}
|
||||
err = m.Bucket.Client.prepare(req)
|
||||
if err != nil {
|
||||
return Part{}, err
|
||||
}
|
||||
resp, err := m.Bucket.Client.run(req, nil)
|
||||
if shouldRetry(err) && attempt.HasNext() {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return Part{}, err
|
||||
}
|
||||
etag := resp.Header.Get("ETag")
|
||||
if etag == "" {
|
||||
return Part{}, errors.New("part upload succeeded with no ETag")
|
||||
}
|
||||
return Part{n, etag, partSize}, nil
|
||||
}
|
||||
panic("unreachable")
|
||||
}
|
||||
|
||||
func seekerInfo(r io.ReadSeeker) (size int64, md5hex string, md5b64 string, err error) {
|
||||
_, err = r.Seek(0, 0)
|
||||
if err != nil {
|
||||
return 0, "", "", err
|
||||
}
|
||||
digest := md5.New()
|
||||
size, err = io.Copy(digest, r)
|
||||
if err != nil {
|
||||
return 0, "", "", err
|
||||
}
|
||||
sum := digest.Sum(nil)
|
||||
md5hex = hex.EncodeToString(sum)
|
||||
md5b64 = base64.StdEncoding.EncodeToString(sum)
|
||||
return size, md5hex, md5b64, nil
|
||||
}
|
||||
|
||||
type Part struct {
|
||||
N int `xml:"PartNumber"`
|
||||
ETag string
|
||||
Size int64
|
||||
}
|
||||
|
||||
type partSlice []Part
|
||||
|
||||
func (s partSlice) Len() int { return len(s) }
|
||||
func (s partSlice) Less(i, j int) bool { return s[i].N < s[j].N }
|
||||
func (s partSlice) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
|
||||
|
||||
type listPartsResp struct {
|
||||
NextPartNumberMarker string
|
||||
IsTruncated bool
|
||||
Part []Part
|
||||
}
|
||||
|
||||
// That's the default. Here just for testing.
|
||||
var listPartsMax = 1000
|
||||
|
||||
// ListParts for backcompatability. See the documentation for ListPartsFull
|
||||
func (m *Multi) ListParts() ([]Part, error) {
|
||||
return m.ListPartsFull(0, listPartsMax)
|
||||
}
|
||||
|
||||
// ListPartsFull returns the list of previously uploaded parts in m,
|
||||
// ordered by part number (Only parts with higher part numbers than
|
||||
// partNumberMarker will be listed). Only up to maxParts parts will be
|
||||
// returned.
|
||||
//
|
||||
func (m *Multi) ListPartsFull(partNumberMarker int, maxParts int) ([]Part, error) {
|
||||
if maxParts > listPartsMax {
|
||||
maxParts = listPartsMax
|
||||
}
|
||||
|
||||
params := make(url.Values)
|
||||
params.Set("uploadId", m.UploadId)
|
||||
params.Set("max-parts", strconv.FormatInt(int64(maxParts), 10))
|
||||
params.Set("part-number-marker", strconv.FormatInt(int64(partNumberMarker), 10))
|
||||
|
||||
var parts partSlice
|
||||
for attempt := attempts.Start(); attempt.Next(); {
|
||||
req := &request{
|
||||
method: "GET",
|
||||
bucket: m.Bucket.Name,
|
||||
path: m.Key,
|
||||
params: params,
|
||||
}
|
||||
var resp listPartsResp
|
||||
err := m.Bucket.Client.query(req, &resp)
|
||||
if shouldRetry(err) && attempt.HasNext() {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
parts = append(parts, resp.Part...)
|
||||
if !resp.IsTruncated {
|
||||
sort.Sort(parts)
|
||||
return parts, nil
|
||||
}
|
||||
params.Set("part-number-marker", resp.NextPartNumberMarker)
|
||||
attempt = attempts.Start() // Last request worked.
|
||||
}
|
||||
panic("unreachable")
|
||||
}
|
||||
|
||||
type ReaderAtSeeker interface {
|
||||
io.ReaderAt
|
||||
io.ReadSeeker
|
||||
}
|
||||
|
||||
// PutAll sends all of r via a multipart upload with parts no larger
|
||||
// than partSize bytes, which must be set to at least 5MB.
|
||||
// Parts previously uploaded are either reused if their checksum
|
||||
// and size match the new part, or otherwise overwritten with the
|
||||
// new content.
|
||||
// PutAll returns all the parts of m (reused or not).
|
||||
func (m *Multi) PutAll(r ReaderAtSeeker, partSize int64) ([]Part, error) {
|
||||
old, err := m.ListParts()
|
||||
if err != nil && !hasCode(err, "NoSuchUpload") {
|
||||
return nil, err
|
||||
}
|
||||
reuse := 0 // Index of next old part to consider reusing.
|
||||
current := 1 // Part number of latest good part handled.
|
||||
totalSize, err := r.Seek(0, 2)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
first := true // Must send at least one empty part if the file is empty.
|
||||
var result []Part
|
||||
NextSection:
|
||||
for offset := int64(0); offset < totalSize || first; offset += partSize {
|
||||
first = false
|
||||
if offset+partSize > totalSize {
|
||||
partSize = totalSize - offset
|
||||
}
|
||||
section := io.NewSectionReader(r, offset, partSize)
|
||||
_, md5hex, md5b64, err := seekerInfo(section)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for reuse < len(old) && old[reuse].N <= current {
|
||||
// Looks like this part was already sent.
|
||||
part := &old[reuse]
|
||||
etag := `"` + md5hex + `"`
|
||||
if part.N == current && part.Size == partSize && part.ETag == etag {
|
||||
// Checksum matches. Reuse the old part.
|
||||
result = append(result, *part)
|
||||
current++
|
||||
continue NextSection
|
||||
}
|
||||
reuse++
|
||||
}
|
||||
|
||||
// Part wasn't found or doesn't match. Send it.
|
||||
part, err := m.putPart(current, section, partSize, md5b64)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, part)
|
||||
current++
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
type completeUpload struct {
|
||||
XMLName xml.Name `xml:"CompleteMultipartUpload"`
|
||||
Parts completeParts `xml:"Part"`
|
||||
}
|
||||
|
||||
type completePart struct {
|
||||
PartNumber int
|
||||
ETag string
|
||||
}
|
||||
|
||||
type completeParts []completePart
|
||||
|
||||
func (p completeParts) Len() int { return len(p) }
|
||||
func (p completeParts) Less(i, j int) bool { return p[i].PartNumber < p[j].PartNumber }
|
||||
func (p completeParts) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
|
||||
|
||||
// Complete assembles the given previously uploaded parts into the
|
||||
// final object. This operation may take several minutes.
|
||||
//
|
||||
func (m *Multi) Complete(parts []Part) error {
|
||||
params := make(url.Values)
|
||||
params.Set("uploadId", m.UploadId)
|
||||
|
||||
c := completeUpload{}
|
||||
for _, p := range parts {
|
||||
c.Parts = append(c.Parts, completePart{p.N, p.ETag})
|
||||
}
|
||||
sort.Sort(c.Parts)
|
||||
data, err := xml.Marshal(&c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for attempt := attempts.Start(); attempt.Next(); {
|
||||
req := &request{
|
||||
method: "POST",
|
||||
bucket: m.Bucket.Name,
|
||||
path: m.Key,
|
||||
params: params,
|
||||
payload: bytes.NewReader(data),
|
||||
}
|
||||
err := m.Bucket.Client.query(req, nil)
|
||||
if shouldRetry(err) && attempt.HasNext() {
|
||||
continue
|
||||
}
|
||||
return err
|
||||
}
|
||||
panic("unreachable")
|
||||
}
|
||||
|
||||
// Abort deletes an unifinished multipart upload and any previously
|
||||
// uploaded parts for it.
|
||||
//
|
||||
// After a multipart upload is aborted, no additional parts can be
|
||||
// uploaded using it. However, if any part uploads are currently in
|
||||
// progress, those part uploads might or might not succeed. As a result,
|
||||
// it might be necessary to abort a given multipart upload multiple
|
||||
// times in order to completely free all storage consumed by all parts.
|
||||
//
|
||||
// NOTE: If the described scenario happens to you, please report back to
|
||||
// the goamz authors with details. In the future such retrying should be
|
||||
// handled internally, but it's not clear what happens precisely (Is an
|
||||
// error returned? Is the issue completely undetectable?).
|
||||
//
|
||||
func (m *Multi) Abort() error {
|
||||
params := make(url.Values)
|
||||
params.Set("uploadId", m.UploadId)
|
||||
|
||||
for attempt := attempts.Start(); attempt.Next(); {
|
||||
req := &request{
|
||||
method: "DELETE",
|
||||
bucket: m.Bucket.Name,
|
||||
path: m.Key,
|
||||
params: params,
|
||||
}
|
||||
err := m.Bucket.Client.query(req, nil)
|
||||
if shouldRetry(err) && attempt.HasNext() {
|
||||
continue
|
||||
}
|
||||
return err
|
||||
}
|
||||
panic("unreachable")
|
||||
}
|
161
Godeps/_workspace/src/github.com/denverdino/aliyungo/oss/multi_test.go
generated
vendored
Normal file
161
Godeps/_workspace/src/github.com/denverdino/aliyungo/oss/multi_test.go
generated
vendored
Normal file
|
@ -0,0 +1,161 @@
|
|||
package oss_test
|
||||
|
||||
import (
|
||||
//"encoding/xml"
|
||||
"github.com/denverdino/aliyungo/oss"
|
||||
"testing"
|
||||
//"io"
|
||||
//"io/ioutil"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func TestCreateBucketMulti(t *testing.T) {
|
||||
TestCreateBucket(t)
|
||||
}
|
||||
|
||||
func TestInitMulti(t *testing.T) {
|
||||
b := client.Bucket(TestBucket)
|
||||
|
||||
metadata := make(map[string][]string)
|
||||
metadata["key1"] = []string{"value1"}
|
||||
metadata["key2"] = []string{"value2"}
|
||||
options := oss.Options{
|
||||
ServerSideEncryption: true,
|
||||
Meta: metadata,
|
||||
ContentEncoding: "text/utf8",
|
||||
CacheControl: "no-cache",
|
||||
ContentMD5: "0000000000000000",
|
||||
}
|
||||
|
||||
multi, err := b.InitMulti("multi", "text/plain", oss.Private, options)
|
||||
if err != nil {
|
||||
t.Errorf("Failed for InitMulti: %v", err)
|
||||
} else {
|
||||
t.Logf("InitMulti result: %++v", multi)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultiReturnOld(t *testing.T) {
|
||||
|
||||
b := client.Bucket(TestBucket)
|
||||
|
||||
multi, err := b.Multi("multi", "text/plain", oss.Private, oss.Options{})
|
||||
if err != nil {
|
||||
t.Errorf("Failed for Multi: %v", err)
|
||||
} else {
|
||||
t.Logf("Multi result: %++v", multi)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestPutPart(t *testing.T) {
|
||||
|
||||
b := client.Bucket(TestBucket)
|
||||
|
||||
multi, err := b.Multi("multi", "text/plain", oss.Private, oss.Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("Failed for Multi: %v", err)
|
||||
}
|
||||
|
||||
part, err := multi.PutPart(1, strings.NewReader("<part 1>"))
|
||||
if err != nil {
|
||||
t.Errorf("Failed for PutPart: %v", err)
|
||||
} else {
|
||||
t.Logf("PutPart result: %++v", part)
|
||||
}
|
||||
|
||||
}
|
||||
func TestPutPartCopy(t *testing.T) {
|
||||
|
||||
TestPutObject(t)
|
||||
|
||||
b := client.Bucket(TestBucket)
|
||||
|
||||
multi, err := b.Multi("multi", "text/plain", oss.Private, oss.Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("Failed for Multi: %v", err)
|
||||
}
|
||||
|
||||
res, part, err := multi.PutPartCopy(2, oss.CopyOptions{}, b.Path("name"))
|
||||
if err != nil {
|
||||
t.Errorf("Failed for PutPartCopy: %v", err)
|
||||
} else {
|
||||
t.Logf("PutPartCopy result: %++v %++v", part, res)
|
||||
}
|
||||
TestDelObject(t)
|
||||
}
|
||||
|
||||
func TestListParts(t *testing.T) {
|
||||
|
||||
b := client.Bucket(TestBucket)
|
||||
|
||||
multi, err := b.Multi("multi", "text/plain", oss.Private, oss.Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("Failed for Multi: %v", err)
|
||||
}
|
||||
|
||||
parts, err := multi.ListParts()
|
||||
if err != nil {
|
||||
t.Errorf("Failed for ListParts: %v", err)
|
||||
} else {
|
||||
t.Logf("ListParts result: %++v", parts)
|
||||
}
|
||||
}
|
||||
func TestListMulti(t *testing.T) {
|
||||
|
||||
b := client.Bucket(TestBucket)
|
||||
|
||||
multis, prefixes, err := b.ListMulti("", "/")
|
||||
if err != nil {
|
||||
t.Errorf("Failed for ListMulti: %v", err)
|
||||
} else {
|
||||
t.Logf("ListMulti result : %++v %++v", multis, prefixes)
|
||||
}
|
||||
}
|
||||
func TestMultiAbort(t *testing.T) {
|
||||
|
||||
b := client.Bucket(TestBucket)
|
||||
|
||||
multi, err := b.Multi("multi", "text/plain", oss.Private, oss.Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("Failed for Multi: %v", err)
|
||||
}
|
||||
|
||||
err = multi.Abort()
|
||||
if err != nil {
|
||||
t.Errorf("Failed for Abort: %v", err)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestPutAll(t *testing.T) {
|
||||
TestInitMulti(t)
|
||||
// Don't retry the NoSuchUpload error.
|
||||
b := client.Bucket(TestBucket)
|
||||
|
||||
multi, err := b.Multi("multi", "text/plain", oss.Private, oss.Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("Failed for Multi: %v", err)
|
||||
}
|
||||
|
||||
// Must send at least one part, so that completing it will work.
|
||||
parts, err := multi.PutAll(strings.NewReader("part1part2last"), 5)
|
||||
if err != nil {
|
||||
t.Errorf("Failed for PutAll: %v", err)
|
||||
} else {
|
||||
t.Logf("PutAll result: %++v", parts)
|
||||
}
|
||||
// // Must send at least one part, so that completing it will work.
|
||||
// err = multi.Complete(parts)
|
||||
// if err != nil {
|
||||
// t.Errorf("Failed for Complete: %v", err)
|
||||
// }
|
||||
err = multi.Abort()
|
||||
if err != nil {
|
||||
t.Errorf("Failed for Abort: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanUp(t *testing.T) {
|
||||
TestDelBucket(t)
|
||||
}
|
53
Godeps/_workspace/src/github.com/denverdino/aliyungo/oss/regions.go
generated
vendored
Normal file
53
Godeps/_workspace/src/github.com/denverdino/aliyungo/oss/regions.go
generated
vendored
Normal file
|
@ -0,0 +1,53 @@
|
|||
package oss
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// Region represents OSS region
|
||||
type Region string
|
||||
|
||||
// Constants of region definition
|
||||
const (
|
||||
Hangzhou = Region("oss-cn-hangzhou")
|
||||
Qingdao = Region("oss-cn-qingdao")
|
||||
Beijing = Region("oss-cn-beijing")
|
||||
Hongkong = Region("oss-cn-hongkong")
|
||||
Shenzhen = Region("oss-cn-shenzhen")
|
||||
USWest1 = Region("oss-us-west-1")
|
||||
DefaultRegion = Hangzhou
|
||||
)
|
||||
|
||||
// GetEndpoint returns endpoint of region
|
||||
func (r Region) GetEndpoint(internal bool, bucket string, secure bool) string {
|
||||
if internal {
|
||||
return r.GetInternalEndpoint(bucket, secure)
|
||||
}
|
||||
return r.GetInternetEndpoint(bucket, secure)
|
||||
}
|
||||
|
||||
func getProtocol(secure bool) string {
|
||||
protocol := "http"
|
||||
if secure {
|
||||
protocol = "https"
|
||||
}
|
||||
return protocol
|
||||
}
|
||||
|
||||
// GetInternetEndpoint returns internet endpoint of region
|
||||
func (r Region) GetInternetEndpoint(bucket string, secure bool) string {
|
||||
protocol := getProtocol(secure)
|
||||
if bucket == "" {
|
||||
return fmt.Sprintf("%s://oss.aliyuncs.com", protocol)
|
||||
}
|
||||
return fmt.Sprintf("%s://%s.%s.aliyuncs.com", protocol, bucket, string(r))
|
||||
}
|
||||
|
||||
// GetInternalEndpoint returns internal endpoint of region
|
||||
func (r Region) GetInternalEndpoint(bucket string, secure bool) string {
|
||||
protocol := getProtocol(secure)
|
||||
if bucket == "" {
|
||||
return fmt.Sprintf("%s://oss-internal.aliyuncs.com", protocol)
|
||||
}
|
||||
return fmt.Sprintf("%s://%s.%s-internal.aliyuncs.com", protocol, bucket, string(r))
|
||||
}
|
105
Godeps/_workspace/src/github.com/denverdino/aliyungo/oss/signature.go
generated
vendored
Normal file
105
Godeps/_workspace/src/github.com/denverdino/aliyungo/oss/signature.go
generated
vendored
Normal file
|
@ -0,0 +1,105 @@
|
|||
package oss
|
||||
|
||||
import (
|
||||
"github.com/denverdino/aliyungo/util"
|
||||
//"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const HeaderOSSPrefix = "x-oss-"
|
||||
|
||||
var ossParamsToSign = map[string]bool{
|
||||
"acl": true,
|
||||
"delete": true,
|
||||
"location": true,
|
||||
"logging": true,
|
||||
"notification": true,
|
||||
"partNumber": true,
|
||||
"policy": true,
|
||||
"requestPayment": true,
|
||||
"torrent": true,
|
||||
"uploadId": true,
|
||||
"uploads": true,
|
||||
"versionId": true,
|
||||
"versioning": true,
|
||||
"versions": true,
|
||||
"response-content-type": true,
|
||||
"response-content-language": true,
|
||||
"response-expires": true,
|
||||
"response-cache-control": true,
|
||||
"response-content-disposition": true,
|
||||
"response-content-encoding": true,
|
||||
}
|
||||
|
||||
func (client *Client) signRequest(request *request) {
|
||||
query := request.params
|
||||
|
||||
urlSignature := query.Get("OSSAccessKeyId") != ""
|
||||
|
||||
headers := request.headers
|
||||
contentMd5 := headers.Get("Content-Md5")
|
||||
contentType := headers.Get("Content-Type")
|
||||
date := ""
|
||||
if urlSignature {
|
||||
date = query.Get("Expires")
|
||||
} else {
|
||||
date = headers.Get("Date")
|
||||
}
|
||||
|
||||
resource := request.path
|
||||
if request.bucket != "" {
|
||||
resource = "/" + request.bucket + request.path
|
||||
}
|
||||
params := make(url.Values)
|
||||
for k, v := range query {
|
||||
if ossParamsToSign[k] {
|
||||
params[k] = v
|
||||
}
|
||||
}
|
||||
|
||||
if len(params) > 0 {
|
||||
resource = resource + "?" + util.Encode(params)
|
||||
}
|
||||
|
||||
canonicalizedResource := resource
|
||||
|
||||
_, canonicalizedHeader := canonicalizeHeader(headers)
|
||||
|
||||
stringToSign := request.method + "\n" + contentMd5 + "\n" + contentType + "\n" + date + "\n" + canonicalizedHeader + canonicalizedResource
|
||||
|
||||
//log.Println("stringToSign: ", stringToSign)
|
||||
signature := util.CreateSignature(stringToSign, client.AccessKeySecret)
|
||||
|
||||
if query.Get("OSSAccessKeyId") != "" {
|
||||
query.Set("Signature", signature)
|
||||
} else {
|
||||
headers.Set("Authorization", "OSS "+client.AccessKeyId+":"+signature)
|
||||
}
|
||||
}
|
||||
|
||||
//Have to break the abstraction to append keys with lower case.
|
||||
func canonicalizeHeader(headers http.Header) (newHeaders http.Header, result string) {
|
||||
var canonicalizedHeaders []string
|
||||
newHeaders = http.Header{}
|
||||
|
||||
for k, v := range headers {
|
||||
if lower := strings.ToLower(k); strings.HasPrefix(lower, HeaderOSSPrefix) {
|
||||
newHeaders[lower] = v
|
||||
canonicalizedHeaders = append(canonicalizedHeaders, lower)
|
||||
} else {
|
||||
newHeaders[k] = v
|
||||
}
|
||||
}
|
||||
|
||||
sort.Strings(canonicalizedHeaders)
|
||||
|
||||
var canonicalizedHeader string
|
||||
|
||||
for _, k := range canonicalizedHeaders {
|
||||
canonicalizedHeader += k + ":" + headers.Get(k) + "\n"
|
||||
}
|
||||
return newHeaders, canonicalizedHeader
|
||||
}
|
74
Godeps/_workspace/src/github.com/denverdino/aliyungo/util/attempt.go
generated
vendored
Normal file
74
Godeps/_workspace/src/github.com/denverdino/aliyungo/util/attempt.go
generated
vendored
Normal file
|
@ -0,0 +1,74 @@
|
|||
package util
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// AttemptStrategy represents a strategy for waiting for an action
|
||||
// to complete successfully. This is an internal type used by the
|
||||
// implementation of other goamz packages.
|
||||
type AttemptStrategy struct {
|
||||
Total time.Duration // total duration of attempt.
|
||||
Delay time.Duration // interval between each try in the burst.
|
||||
Min int // minimum number of retries; overrides Total
|
||||
}
|
||||
|
||||
type Attempt struct {
|
||||
strategy AttemptStrategy
|
||||
last time.Time
|
||||
end time.Time
|
||||
force bool
|
||||
count int
|
||||
}
|
||||
|
||||
// Start begins a new sequence of attempts for the given strategy.
|
||||
func (s AttemptStrategy) Start() *Attempt {
|
||||
now := time.Now()
|
||||
return &Attempt{
|
||||
strategy: s,
|
||||
last: now,
|
||||
end: now.Add(s.Total),
|
||||
force: true,
|
||||
}
|
||||
}
|
||||
|
||||
// Next waits until it is time to perform the next attempt or returns
|
||||
// false if it is time to stop trying.
|
||||
func (a *Attempt) Next() bool {
|
||||
now := time.Now()
|
||||
sleep := a.nextSleep(now)
|
||||
if !a.force && !now.Add(sleep).Before(a.end) && a.strategy.Min <= a.count {
|
||||
return false
|
||||
}
|
||||
a.force = false
|
||||
if sleep > 0 && a.count > 0 {
|
||||
time.Sleep(sleep)
|
||||
now = time.Now()
|
||||
}
|
||||
a.count++
|
||||
a.last = now
|
||||
return true
|
||||
}
|
||||
|
||||
func (a *Attempt) nextSleep(now time.Time) time.Duration {
|
||||
sleep := a.strategy.Delay - now.Sub(a.last)
|
||||
if sleep < 0 {
|
||||
return 0
|
||||
}
|
||||
return sleep
|
||||
}
|
||||
|
||||
// HasNext returns whether another attempt will be made if the current
|
||||
// one fails. If it returns true, the following call to Next is
|
||||
// guaranteed to return true.
|
||||
func (a *Attempt) HasNext() bool {
|
||||
if a.force || a.strategy.Min > a.count {
|
||||
return true
|
||||
}
|
||||
now := time.Now()
|
||||
if now.Add(a.nextSleep(now)).Before(a.end) {
|
||||
a.force = true
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
90
Godeps/_workspace/src/github.com/denverdino/aliyungo/util/attempt_test.go
generated
vendored
Normal file
90
Godeps/_workspace/src/github.com/denverdino/aliyungo/util/attempt_test.go
generated
vendored
Normal file
|
@ -0,0 +1,90 @@
|
|||
package util
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestAttemptTiming(t *testing.T) {
|
||||
testAttempt := AttemptStrategy{
|
||||
Total: 0.25e9,
|
||||
Delay: 0.1e9,
|
||||
}
|
||||
want := []time.Duration{0, 0.1e9, 0.2e9, 0.2e9}
|
||||
got := make([]time.Duration, 0, len(want)) // avoid allocation when testing timing
|
||||
t0 := time.Now()
|
||||
for a := testAttempt.Start(); a.Next(); {
|
||||
got = append(got, time.Now().Sub(t0))
|
||||
}
|
||||
got = append(got, time.Now().Sub(t0))
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("Failed!")
|
||||
}
|
||||
const margin = 0.01e9
|
||||
for i, got := range want {
|
||||
lo := want[i] - margin
|
||||
hi := want[i] + margin
|
||||
if got < lo || got > hi {
|
||||
t.Errorf("attempt %d want %g got %g", i, want[i].Seconds(), got.Seconds())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAttemptNextHasNext(t *testing.T) {
|
||||
a := AttemptStrategy{}.Start()
|
||||
if !a.Next() {
|
||||
t.Fatalf("Failed!")
|
||||
}
|
||||
if a.Next() {
|
||||
t.Fatalf("Failed!")
|
||||
}
|
||||
|
||||
a = AttemptStrategy{}.Start()
|
||||
if !a.Next() {
|
||||
t.Fatalf("Failed!")
|
||||
}
|
||||
if a.HasNext() {
|
||||
t.Fatalf("Failed!")
|
||||
}
|
||||
if a.Next() {
|
||||
t.Fatalf("Failed!")
|
||||
}
|
||||
a = AttemptStrategy{Total: 2e8}.Start()
|
||||
|
||||
if !a.Next() {
|
||||
t.Fatalf("Failed!")
|
||||
}
|
||||
if !a.HasNext() {
|
||||
t.Fatalf("Failed!")
|
||||
}
|
||||
time.Sleep(2e8)
|
||||
|
||||
if !a.HasNext() {
|
||||
t.Fatalf("Failed!")
|
||||
}
|
||||
if !a.Next() {
|
||||
t.Fatalf("Failed!")
|
||||
}
|
||||
if a.Next() {
|
||||
t.Fatalf("Failed!")
|
||||
}
|
||||
|
||||
a = AttemptStrategy{Total: 1e8, Min: 2}.Start()
|
||||
time.Sleep(1e8)
|
||||
|
||||
if !a.Next() {
|
||||
t.Fatalf("Failed!")
|
||||
}
|
||||
if !a.HasNext() {
|
||||
t.Fatalf("Failed!")
|
||||
}
|
||||
if !a.Next() {
|
||||
t.Fatalf("Failed!")
|
||||
}
|
||||
if a.HasNext() {
|
||||
t.Fatalf("Failed!")
|
||||
}
|
||||
if a.Next() {
|
||||
t.Fatalf("Failed!")
|
||||
}
|
||||
}
|
113
Godeps/_workspace/src/github.com/denverdino/aliyungo/util/encoding.go
generated
vendored
Normal file
113
Godeps/_workspace/src/github.com/denverdino/aliyungo/util/encoding.go
generated
vendored
Normal file
|
@ -0,0 +1,113 @@
|
|||
package util
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/url"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
//ConvertToQueryValues converts the struct to url.Values
|
||||
func ConvertToQueryValues(ifc interface{}) url.Values {
|
||||
values := url.Values{}
|
||||
SetQueryValues(ifc, &values)
|
||||
return values
|
||||
}
|
||||
|
||||
//SetQueryValues sets the struct to existing url.Values following ECS encoding rules
|
||||
func SetQueryValues(ifc interface{}, values *url.Values) {
|
||||
setQueryValues(ifc, values, "")
|
||||
}
|
||||
|
||||
func setQueryValues(i interface{}, values *url.Values, prefix string) {
|
||||
elem := reflect.ValueOf(i)
|
||||
if elem.Kind() == reflect.Ptr {
|
||||
elem = elem.Elem()
|
||||
}
|
||||
elemType := elem.Type()
|
||||
for i := 0; i < elem.NumField(); i++ {
|
||||
fieldName := elemType.Field(i).Name
|
||||
field := elem.Field(i)
|
||||
// TODO Use Tag for validation
|
||||
// tag := typ.Field(i).Tag.Get("tagname")
|
||||
kind := field.Kind()
|
||||
if (kind == reflect.Ptr || kind == reflect.Array || kind == reflect.Slice || kind == reflect.Map || kind == reflect.Chan) && field.IsNil() {
|
||||
continue
|
||||
}
|
||||
if kind == reflect.Ptr {
|
||||
field = field.Elem()
|
||||
}
|
||||
var value string
|
||||
switch field.Interface().(type) {
|
||||
case int, int8, int16, int32, int64:
|
||||
i := field.Int()
|
||||
if i != 0 {
|
||||
value = strconv.FormatInt(i, 10)
|
||||
}
|
||||
case uint, uint8, uint16, uint32, uint64:
|
||||
i := field.Uint()
|
||||
if i != 0 {
|
||||
value = strconv.FormatUint(i, 10)
|
||||
}
|
||||
case float32:
|
||||
value = strconv.FormatFloat(field.Float(), 'f', 4, 32)
|
||||
case float64:
|
||||
value = strconv.FormatFloat(field.Float(), 'f', 4, 64)
|
||||
case []byte:
|
||||
value = string(field.Bytes())
|
||||
case bool:
|
||||
value = strconv.FormatBool(field.Bool())
|
||||
case string:
|
||||
value = field.String()
|
||||
case []string:
|
||||
l := field.Len()
|
||||
if l > 0 {
|
||||
strArray := make([]string, l)
|
||||
for i := 0; i < l; i++ {
|
||||
strArray[i] = field.Index(i).String()
|
||||
}
|
||||
bytes, err := json.Marshal(strArray)
|
||||
if err == nil {
|
||||
value = string(bytes)
|
||||
} else {
|
||||
log.Printf("Failed to convert JSON: %v", err)
|
||||
}
|
||||
}
|
||||
case time.Time:
|
||||
t := field.Interface().(time.Time)
|
||||
value = GetISO8601TimeStamp(t)
|
||||
|
||||
default:
|
||||
if kind == reflect.Slice { //Array of structs
|
||||
l := field.Len()
|
||||
for j := 0; j < l; j++ {
|
||||
prefixName := fmt.Sprintf("%s.%d.", fieldName, (j + 1))
|
||||
ifc := field.Index(j).Interface()
|
||||
log.Printf("%s : %v", prefixName, ifc)
|
||||
if ifc != nil {
|
||||
setQueryValues(ifc, values, prefixName)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
ifc := field.Interface()
|
||||
if ifc != nil {
|
||||
SetQueryValues(ifc, values)
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
if value != "" {
|
||||
name := elemType.Field(i).Tag.Get("ArgName")
|
||||
if name == "" {
|
||||
name = fieldName
|
||||
}
|
||||
if prefix != "" {
|
||||
name = prefix + name
|
||||
}
|
||||
values.Set(name, value)
|
||||
}
|
||||
}
|
||||
}
|
46
Godeps/_workspace/src/github.com/denverdino/aliyungo/util/encoding_test.go
generated
vendored
Normal file
46
Godeps/_workspace/src/github.com/denverdino/aliyungo/util/encoding_test.go
generated
vendored
Normal file
|
@ -0,0 +1,46 @@
|
|||
package util
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type SubStruct struct {
|
||||
A string
|
||||
B int
|
||||
}
|
||||
|
||||
type TestStruct struct {
|
||||
Format string
|
||||
Version string
|
||||
AccessKeyId string
|
||||
Timestamp time.Time
|
||||
Empty string
|
||||
IntValue int `ArgName:"int-value"`
|
||||
BoolPtr *bool `ArgName:"bool-ptr"`
|
||||
IntPtr *int `ArgName:"int-ptr"`
|
||||
StringArray []string `ArgName:"str-array"`
|
||||
StructArray []SubStruct
|
||||
}
|
||||
|
||||
func TestConvertToQueryValues(t *testing.T) {
|
||||
boolValue := true
|
||||
request := TestStruct{
|
||||
Format: "JSON",
|
||||
Version: "1.0",
|
||||
Timestamp: time.Date(2015, time.Month(5), 26, 1, 2, 3, 4, time.UTC),
|
||||
IntValue: 10,
|
||||
BoolPtr: &boolValue,
|
||||
StringArray: []string{"abc", "xyz"},
|
||||
StructArray: []SubStruct{
|
||||
SubStruct{A: "a", B: 1},
|
||||
SubStruct{A: "x", B: 2},
|
||||
},
|
||||
}
|
||||
result := ConvertToQueryValues(&request).Encode()
|
||||
const expectedResult = "Format=JSON&StructArray.1.A=a&StructArray.1.B=1&StructArray.2.A=x&StructArray.2.B=2&Timestamp=2015-05-26T01%3A02%3A03Z&Version=1.0&bool-ptr=true&int-value=10&str-array=%5B%22abc%22%2C%22xyz%22%5D"
|
||||
if result != expectedResult {
|
||||
t.Error("Incorrect encoding: ", result)
|
||||
}
|
||||
|
||||
}
|
62
Godeps/_workspace/src/github.com/denverdino/aliyungo/util/iso6801.go
generated
vendored
Normal file
62
Godeps/_workspace/src/github.com/denverdino/aliyungo/util/iso6801.go
generated
vendored
Normal file
|
@ -0,0 +1,62 @@
|
|||
package util
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// GetISO8601TimeStamp gets timestamp string in ISO8601 format
|
||||
func GetISO8601TimeStamp(ts time.Time) string {
|
||||
t := ts.UTC()
|
||||
return fmt.Sprintf("%04d-%02d-%02dT%02d:%02d:%02dZ", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second())
|
||||
}
|
||||
|
||||
const formatISO8601 = "2006-01-02T15:04:05Z"
|
||||
const jsonFormatISO8601 = `"` + formatISO8601 + `"`
|
||||
|
||||
// A ISO6801Time represents a time in ISO8601 format
|
||||
type ISO6801Time time.Time
|
||||
|
||||
// New constructs a new iso8601.Time instance from an existing
|
||||
// time.Time instance. This causes the nanosecond field to be set to
|
||||
// 0, and its time zone set to a fixed zone with no offset from UTC
|
||||
// (but it is *not* UTC itself).
|
||||
func New(t time.Time) ISO6801Time {
|
||||
return ISO6801Time(time.Date(
|
||||
t.Year(),
|
||||
t.Month(),
|
||||
t.Day(),
|
||||
t.Hour(),
|
||||
t.Minute(),
|
||||
t.Second(),
|
||||
0,
|
||||
time.UTC,
|
||||
))
|
||||
}
|
||||
|
||||
// IsDefault checks if the time is default
|
||||
func (it *ISO6801Time) IsDefault() bool {
|
||||
return *it == ISO6801Time{}
|
||||
}
|
||||
|
||||
// MarshalJSON serializes the ISO6801Time into JSON string
|
||||
func (it ISO6801Time) MarshalJSON() ([]byte, error) {
|
||||
return []byte(time.Time(it).Format(jsonFormatISO8601)), nil
|
||||
}
|
||||
|
||||
// UnmarshalJSON deserializes the ISO6801Time from JSON string
|
||||
func (it *ISO6801Time) UnmarshalJSON(data []byte) error {
|
||||
if string(data) == "\"\"" {
|
||||
return nil
|
||||
}
|
||||
t, err := time.ParseInLocation(jsonFormatISO8601, string(data), time.UTC)
|
||||
if err == nil {
|
||||
*it = ISO6801Time(t)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// String returns the time in ISO6801Time format
|
||||
func (it ISO6801Time) String() string {
|
||||
return time.Time(it).String()
|
||||
}
|
50
Godeps/_workspace/src/github.com/denverdino/aliyungo/util/iso6801_test.go
generated
vendored
Normal file
50
Godeps/_workspace/src/github.com/denverdino/aliyungo/util/iso6801_test.go
generated
vendored
Normal file
|
@ -0,0 +1,50 @@
|
|||
package util
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestISO8601Time(t *testing.T) {
|
||||
now := New(time.Now().UTC())
|
||||
|
||||
data, err := json.Marshal(now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, err = time.Parse(`"`+formatISO8601+`"`, string(data))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var now2 ISO6801Time
|
||||
err = json.Unmarshal(data, &now2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if now != now2 {
|
||||
t.Fatalf("Time %s does not equal expected %s", now2, now)
|
||||
}
|
||||
|
||||
if now.String() != now2.String() {
|
||||
t.Fatalf("String format for %s does not equal expected %s", now2, now)
|
||||
}
|
||||
|
||||
type TestTimeStruct struct {
|
||||
A int
|
||||
B *ISO6801Time
|
||||
}
|
||||
var testValue TestTimeStruct
|
||||
err = json.Unmarshal([]byte("{\"A\": 1, \"B\":\"\"}"), &testValue)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Logf("%v", testValue)
|
||||
if !testValue.B.IsDefault() {
|
||||
t.Fatal("Invaid Unmarshal result for ISO6801Time from empty value")
|
||||
}
|
||||
|
||||
}
|
40
Godeps/_workspace/src/github.com/denverdino/aliyungo/util/signature.go
generated
vendored
Normal file
40
Godeps/_workspace/src/github.com/denverdino/aliyungo/util/signature.go
generated
vendored
Normal file
|
@ -0,0 +1,40 @@
|
|||
package util
|
||||
|
||||
import (
|
||||
"crypto/hmac"
|
||||
"crypto/sha1"
|
||||
"encoding/base64"
|
||||
"net/url"
|
||||
"strings"
|
||||
)
|
||||
|
||||
//CreateSignature creates signature for string following Aliyun rules
|
||||
func CreateSignature(stringToSignature, accessKeySecret string) string {
|
||||
// Crypto by HMAC-SHA1
|
||||
hmacSha1 := hmac.New(sha1.New, []byte(accessKeySecret))
|
||||
hmacSha1.Write([]byte(stringToSignature))
|
||||
sign := hmacSha1.Sum(nil)
|
||||
|
||||
// Encode to Base64
|
||||
base64Sign := base64.StdEncoding.EncodeToString(sign)
|
||||
|
||||
return base64Sign
|
||||
}
|
||||
|
||||
func percentReplace(str string) string {
|
||||
str = strings.Replace(str, "+", "%20", -1)
|
||||
str = strings.Replace(str, "*", "%2A", -1)
|
||||
str = strings.Replace(str, "%7E", "~", -1)
|
||||
|
||||
return str
|
||||
}
|
||||
|
||||
// CreateSignatureForRequest creates signature for query string values
|
||||
func CreateSignatureForRequest(method string, values *url.Values, accessKeySecret string) string {
|
||||
|
||||
canonicalizedQueryString := percentReplace(values.Encode())
|
||||
|
||||
stringToSign := method + "&%2F&" + url.QueryEscape(canonicalizedQueryString)
|
||||
|
||||
return CreateSignature(stringToSign, accessKeySecret)
|
||||
}
|
14
Godeps/_workspace/src/github.com/denverdino/aliyungo/util/signature_test.go
generated
vendored
Normal file
14
Godeps/_workspace/src/github.com/denverdino/aliyungo/util/signature_test.go
generated
vendored
Normal file
|
@ -0,0 +1,14 @@
|
|||
package util
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCreateSignature(t *testing.T) {
|
||||
|
||||
str := "GET&%2F&AccessKeyId%3Dtestid%26Action%3DDescribeRegions%26Format%3DXML%26RegionId%3Dregion1%26SignatureMethod%3DHMAC-SHA1%26SignatureNonce%3DNwDAxvLU6tFE0DVb%26SignatureVersion%3D1.0%26TimeStamp%3D2012-12-26T10%253A33%253A56Z%26Version%3D2014-05-26"
|
||||
|
||||
signature := CreateSignature(str, "testsecret")
|
||||
|
||||
t.Log(signature)
|
||||
}
|
54
Godeps/_workspace/src/github.com/denverdino/aliyungo/util/util.go
generated
vendored
Normal file
54
Godeps/_workspace/src/github.com/denverdino/aliyungo/util/util.go
generated
vendored
Normal file
|
@ -0,0 +1,54 @@
|
|||
package util
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
//CreateRandomString create random string
|
||||
func CreateRandomString() string {
|
||||
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
randInt := rand.Int63()
|
||||
randStr := strconv.FormatInt(randInt, 36)
|
||||
|
||||
return randStr
|
||||
}
|
||||
|
||||
// Encode encodes the values into ``URL encoded'' form
|
||||
// ("acl&bar=baz&foo=quux") sorted by key.
|
||||
func Encode(v url.Values) string {
|
||||
if v == nil {
|
||||
return ""
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
keys := make([]string, 0, len(v))
|
||||
for k := range v {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
vs := v[k]
|
||||
prefix := url.QueryEscape(k)
|
||||
for _, v := range vs {
|
||||
if buf.Len() > 0 {
|
||||
buf.WriteByte('&')
|
||||
}
|
||||
buf.WriteString(prefix)
|
||||
if v != "" {
|
||||
buf.WriteString("=")
|
||||
buf.WriteString(url.QueryEscape(v))
|
||||
}
|
||||
}
|
||||
}
|
||||
return buf.String()
|
||||
}
|
||||
|
||||
func GetGMTime() string {
|
||||
return time.Now().UTC().Format(http.TimeFormat)
|
||||
}
|
Loading…
Reference in a new issue