From 3d30cb38f6d29a7070626265727593e3d1e8e9f1 Mon Sep 17 00:00:00 2001 From: tgic Date: Mon, 15 Jun 2015 20:03:32 +0800 Subject: [PATCH] add endpoint support --- Godeps/Godeps.json | 4 +- .../denverdino/aliyungo/oss/client.go | 17 ++- .../denverdino/aliyungo/util/attempt.go | 4 +- .../denverdino/aliyungo/util/encoding.go | 68 ++++++----- .../denverdino/aliyungo/util/encoding_test.go | 8 +- .../denverdino/aliyungo/util/iso6801.go | 4 +- .../denverdino/aliyungo/util/iso6801_test.go | 6 +- .../denverdino/aliyungo/util/util.go | 107 ++++++++++++++++++ .../denverdino/aliyungo/util/util_test.go | 98 ++++++++++++++++ registry/storage/driver/oss/oss.go | 8 ++ registry/storage/driver/oss/oss_test.go | 20 ++-- 11 files changed, 295 insertions(+), 49 deletions(-) create mode 100644 Godeps/_workspace/src/github.com/denverdino/aliyungo/util/util_test.go mode change 100755 => 100644 registry/storage/driver/oss/oss.go mode change 100755 => 100644 registry/storage/driver/oss/oss_test.go diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index d21f52608..6c03f5446 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -46,11 +46,11 @@ }, { "ImportPath": "github.com/denverdino/aliyungo/oss", - "Rev": "17d1e888c907ffdbd875f37500f3d130ce2ee6eb" + "Rev": "56047189188d6558b6a1456d647970032343b511" }, { "ImportPath": "github.com/denverdino/aliyungo/util", - "Rev": "17d1e888c907ffdbd875f37500f3d130ce2ee6eb" + "Rev": "56047189188d6558b6a1456d647970032343b511" }, { "ImportPath": "github.com/docker/docker/pkg/tarsum", diff --git a/Godeps/_workspace/src/github.com/denverdino/aliyungo/oss/client.go b/Godeps/_workspace/src/github.com/denverdino/aliyungo/oss/client.go index 8e38ea4ae..d67c4d377 100644 --- a/Godeps/_workspace/src/github.com/denverdino/aliyungo/oss/client.go +++ b/Godeps/_workspace/src/github.com/denverdino/aliyungo/oss/client.go @@ -35,7 +35,9 @@ type Client struct { Secure bool ConnectTimeout time.Duration ReadTimeout time.Duration - debug bool + + endpoint string + debug bool } // The Bucket type encapsulates operations with an bucket. @@ -159,6 +161,12 @@ func (client *Client) locationConstraint() io.Reader { return strings.NewReader(constraint) } +// override default endpoint +func (client *Client) SetEndpoint(endpoint string) { + // TODO check endpoint + client.endpoint = endpoint +} + // PutBucket creates a new bucket. func (b *Bucket) PutBucket(perm ACL) error { headers := make(http.Header) @@ -963,7 +971,12 @@ func (client *Client) query(req *request, resp interface{}) error { // Sets baseurl on req from bucket name and the region endpoint func (client *Client) setBaseURL(req *request) error { - req.baseurl = client.Region.GetEndpoint(client.Internal, req.bucket, client.Secure) + + if client.endpoint == "" { + req.baseurl = client.Region.GetEndpoint(client.Internal, req.bucket, client.Secure) + } else { + req.baseurl = fmt.Sprintf("%s://%s", client.endpoint, getProtocol(client.Secure)) + } return nil } diff --git a/Godeps/_workspace/src/github.com/denverdino/aliyungo/util/attempt.go b/Godeps/_workspace/src/github.com/denverdino/aliyungo/util/attempt.go index e71ed19f3..2d07f03a8 100644 --- a/Godeps/_workspace/src/github.com/denverdino/aliyungo/util/attempt.go +++ b/Godeps/_workspace/src/github.com/denverdino/aliyungo/util/attempt.go @@ -4,9 +4,11 @@ import ( "time" ) +// AttemptStrategy is reused from the goamz package + // AttemptStrategy represents a strategy for waiting for an action // to complete successfully. This is an internal type used by the -// implementation of other goamz packages. +// implementation of other packages. type AttemptStrategy struct { Total time.Duration // total duration of attempt. Delay time.Duration // interval between each try in the burst. diff --git a/Godeps/_workspace/src/github.com/denverdino/aliyungo/util/encoding.go b/Godeps/_workspace/src/github.com/denverdino/aliyungo/util/encoding.go index 54a635691..56b900afa 100644 --- a/Godeps/_workspace/src/github.com/denverdino/aliyungo/util/encoding.go +++ b/Godeps/_workspace/src/github.com/denverdino/aliyungo/util/encoding.go @@ -39,49 +39,48 @@ func setQueryValues(i interface{}, values *url.Values, prefix string) { } if kind == reflect.Ptr { field = field.Elem() + kind = field.Kind() } var value string - switch field.Interface().(type) { - case int, int8, int16, int32, int64: + //switch field.Interface().(type) { + switch kind { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: i := field.Int() if i != 0 { value = strconv.FormatInt(i, 10) } - case uint, uint8, uint16, uint32, uint64: + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: i := field.Uint() if i != 0 { value = strconv.FormatUint(i, 10) } - case float32: + case reflect.Float32: value = strconv.FormatFloat(field.Float(), 'f', 4, 32) - case float64: + case reflect.Float64: value = strconv.FormatFloat(field.Float(), 'f', 4, 64) - case []byte: - value = string(field.Bytes()) - case bool: + case reflect.Bool: value = strconv.FormatBool(field.Bool()) - case string: + case reflect.String: value = field.String() - case []string: - l := field.Len() - if l > 0 { - strArray := make([]string, l) - for i := 0; i < l; i++ { - strArray[i] = field.Index(i).String() + case reflect.Slice: + switch field.Type().Elem().Kind() { + case reflect.Uint8: + value = string(field.Bytes()) + case reflect.String: + l := field.Len() + if l > 0 { + strArray := make([]string, l) + for i := 0; i < l; i++ { + strArray[i] = field.Index(i).String() + } + bytes, err := json.Marshal(strArray) + if err == nil { + value = string(bytes) + } else { + log.Printf("Failed to convert JSON: %v", err) + } } - bytes, err := json.Marshal(strArray) - if err == nil { - value = string(bytes) - } else { - log.Printf("Failed to convert JSON: %v", err) - } - } - case time.Time: - t := field.Interface().(time.Time) - value = GetISO8601TimeStamp(t) - - default: - if kind == reflect.Slice { //Array of structs + default: l := field.Len() for j := 0; j < l; j++ { prefixName := fmt.Sprintf("%s.%d.", fieldName, (j + 1)) @@ -91,7 +90,18 @@ func setQueryValues(i interface{}, values *url.Values, prefix string) { setQueryValues(ifc, values, prefixName) } } - } else { + continue + } + + default: + switch field.Interface().(type) { + case ISO6801Time: + t := field.Interface().(ISO6801Time) + value = t.String() + case time.Time: + t := field.Interface().(time.Time) + value = GetISO8601TimeStamp(t) + default: ifc := field.Interface() if ifc != nil { SetQueryValues(ifc, values) diff --git a/Godeps/_workspace/src/github.com/denverdino/aliyungo/util/encoding_test.go b/Godeps/_workspace/src/github.com/denverdino/aliyungo/util/encoding_test.go index a49eb215f..049cd86b3 100644 --- a/Godeps/_workspace/src/github.com/denverdino/aliyungo/util/encoding_test.go +++ b/Godeps/_workspace/src/github.com/denverdino/aliyungo/util/encoding_test.go @@ -5,6 +5,8 @@ import ( "time" ) +type TestString string + type SubStruct struct { A string B int @@ -21,6 +23,8 @@ type TestStruct struct { IntPtr *int `ArgName:"int-ptr"` StringArray []string `ArgName:"str-array"` StructArray []SubStruct + test TestString + tests []TestString } func TestConvertToQueryValues(t *testing.T) { @@ -36,9 +40,11 @@ func TestConvertToQueryValues(t *testing.T) { SubStruct{A: "a", B: 1}, SubStruct{A: "x", B: 2}, }, + test: TestString("test"), + tests: []TestString{TestString("test1"), TestString("test2")}, } result := ConvertToQueryValues(&request).Encode() - const expectedResult = "Format=JSON&StructArray.1.A=a&StructArray.1.B=1&StructArray.2.A=x&StructArray.2.B=2&Timestamp=2015-05-26T01%3A02%3A03Z&Version=1.0&bool-ptr=true&int-value=10&str-array=%5B%22abc%22%2C%22xyz%22%5D" + const expectedResult = "Format=JSON&StructArray.1.A=a&StructArray.1.B=1&StructArray.2.A=x&StructArray.2.B=2&Timestamp=2015-05-26T01%3A02%3A03Z&Version=1.0&bool-ptr=true&int-value=10&str-array=%5B%22abc%22%2C%22xyz%22%5D&test=test&tests=%5B%22test1%22%2C%22test2%22%5D" if result != expectedResult { t.Error("Incorrect encoding: ", result) } diff --git a/Godeps/_workspace/src/github.com/denverdino/aliyungo/util/iso6801.go b/Godeps/_workspace/src/github.com/denverdino/aliyungo/util/iso6801.go index 121d6e62f..031b61751 100644 --- a/Godeps/_workspace/src/github.com/denverdino/aliyungo/util/iso6801.go +++ b/Godeps/_workspace/src/github.com/denverdino/aliyungo/util/iso6801.go @@ -21,7 +21,7 @@ type ISO6801Time time.Time // time.Time instance. This causes the nanosecond field to be set to // 0, and its time zone set to a fixed zone with no offset from UTC // (but it is *not* UTC itself). -func New(t time.Time) ISO6801Time { +func NewISO6801Time(t time.Time) ISO6801Time { return ISO6801Time(time.Date( t.Year(), t.Month(), @@ -58,5 +58,5 @@ func (it *ISO6801Time) UnmarshalJSON(data []byte) error { // String returns the time in ISO6801Time format func (it ISO6801Time) String() string { - return time.Time(it).String() + return time.Time(it).Format(formatISO8601) } diff --git a/Godeps/_workspace/src/github.com/denverdino/aliyungo/util/iso6801_test.go b/Godeps/_workspace/src/github.com/denverdino/aliyungo/util/iso6801_test.go index 284a23c18..f2ba96a45 100644 --- a/Godeps/_workspace/src/github.com/denverdino/aliyungo/util/iso6801_test.go +++ b/Godeps/_workspace/src/github.com/denverdino/aliyungo/util/iso6801_test.go @@ -7,7 +7,7 @@ import ( ) func TestISO8601Time(t *testing.T) { - now := New(time.Now().UTC()) + now := NewISO6801Time(time.Now().UTC()) data, err := json.Marshal(now) if err != nil { @@ -26,7 +26,7 @@ func TestISO8601Time(t *testing.T) { } if now != now2 { - t.Fatalf("Time %s does not equal expected %s", now2, now) + t.Errorf("Time %s does not equal expected %s", now2, now) } if now.String() != now2.String() { @@ -46,5 +46,5 @@ func TestISO8601Time(t *testing.T) { if !testValue.B.IsDefault() { t.Fatal("Invaid Unmarshal result for ISO6801Time from empty value") } - + t.Logf("ISO6801Time String(): %s", now2.String()) } diff --git a/Godeps/_workspace/src/github.com/denverdino/aliyungo/util/util.go b/Godeps/_workspace/src/github.com/denverdino/aliyungo/util/util.go index f28266842..3e21abfcb 100644 --- a/Godeps/_workspace/src/github.com/denverdino/aliyungo/util/util.go +++ b/Godeps/_workspace/src/github.com/denverdino/aliyungo/util/util.go @@ -2,12 +2,20 @@ package util import ( "bytes" + srand "crypto/rand" + "encoding/binary" "math/rand" "net/http" "net/url" "sort" "strconv" "time" + "errors" +) + + +const ( + StatusUnKnown = "NA" ) //CreateRandomString create random string @@ -52,3 +60,102 @@ func Encode(v url.Values) string { func GetGMTime() string { return time.Now().UTC().Format(http.TimeFormat) } + +// + +func randUint32() uint32 { + return randUint32Slice(1)[0] +} + +func randUint32Slice(c int) []uint32 { + b := make([]byte, c*4) + + _, err := srand.Read(b) + + if err != nil { + // fail back to insecure rand + rand.Seed(time.Now().UnixNano()) + for i := range b { + b[i] = byte(rand.Int()) + } + } + + n := make([]uint32, c) + + for i := range n { + n[i] = binary.BigEndian.Uint32(b[i*4 : i*4+4]) + } + + return n +} + +func toByte(n uint32, st, ed byte) byte { + return byte(n%uint32(ed-st+1) + uint32(st)) +} + +func toDigit(n uint32) byte { + return toByte(n, '0', '9') +} + +func toLowerLetter(n uint32) byte { + return toByte(n, 'a', 'z') +} + +func toUpperLetter(n uint32) byte { + return toByte(n, 'A', 'Z') +} + +type convFunc func(uint32) byte + +var convFuncs = []convFunc{toDigit, toLowerLetter, toUpperLetter} + +// tools for generating a random ECS instance password +// from 8 to 30 char MUST contain digit upper, case letter and upper case letter +// http://docs.aliyun.com/#/pub/ecs/open-api/instance&createinstance +func GenerateRandomECSPassword() string { + + // [8, 30] + l := int(randUint32()%23 + 8) + + n := randUint32Slice(l) + + b := make([]byte, l) + + b[0] = toDigit(n[0]) + b[1] = toLowerLetter(n[1]) + b[2] = toUpperLetter(n[2]) + + for i := 3; i < l; i++ { + b[i] = convFuncs[n[i]%3](n[i]) + } + + s := make([]byte, l) + perm := rand.Perm(l) + for i, v := range perm { + s[v] = b[i] + } + + return string(s) + +} + + +func LoopCall(attempts AttemptStrategy,api func() (bool,interface{},error))(interface{}, error){ + + for attempt := attempts.Start(); attempt.Next(); { + needStop,status,err := api() + + if(err != nil) { + return nil, errors.New("execution failed") + } + if(needStop){ + return status,nil; + } + + if attempt.HasNext() { + continue; + } + return nil,errors.New("timeout execution ") + } + panic("unreachable") +} diff --git a/Godeps/_workspace/src/github.com/denverdino/aliyungo/util/util_test.go b/Godeps/_workspace/src/github.com/denverdino/aliyungo/util/util_test.go new file mode 100644 index 000000000..354b95f10 --- /dev/null +++ b/Godeps/_workspace/src/github.com/denverdino/aliyungo/util/util_test.go @@ -0,0 +1,98 @@ +package util + +import ( + "testing" + "errors" + "time" +) + +func TestGenerateRandomECSPassword(t *testing.T) { + for i := 0; i < 10; i++ { + s := GenerateRandomECSPassword() + + if len(s) < 8 || len(s) > 30 { + t.Errorf("Generated ECS password [%v]: bad len", s) + } + + hasDigit := false + hasLower := false + hasUpper := false + + for j := range s { + + switch { + case '0' <= s[j] && s[j] <= '9': + hasDigit = true + case 'a' <= s[j] && s[j] <= 'z': + hasLower = true + case 'A' <= s[j] && s[j] <= 'Z': + hasUpper = true + } + } + + if !hasDigit { + t.Errorf("Generated ECS password [%v]: no digit", s) + } + + if !hasLower { + t.Errorf("Generated ECS password [%v]: no lower letter ", s) + } + + if !hasUpper { + t.Errorf("Generated ECS password [%v]: no upper letter", s) + } + } +} + +func TestWaitForSignalWithTimeout(t *testing.T) { + + attempts := AttemptStrategy{ + Min: 5, + Total: 5 * time.Second, + Delay: 200 * time.Millisecond, + } + + timeoutFunc := func() (bool,interface{},error) { + return false,"-1",nil + } + + begin := time.Now() + + _, timeoutError := LoopCall(attempts, timeoutFunc); + if(timeoutError != nil) { + t.Logf("timeout func complete successful") + } else { + t.Error("Expect timeout result") + } + + end := time.Now() + duration := end.Sub(begin).Seconds() + if( duration > (float64(attempts.Min) -1)) { + t.Logf("timeout func duration is enough") + } else { + t.Error("timeout func duration is not enough") + } + + errorFunc := func() (bool, interface{}, error) { + err := errors.New("execution failed"); + return false,"-1",err + } + + _, failedError := LoopCall(attempts, errorFunc); + if(failedError != nil) { + t.Logf("error func complete successful: " + failedError.Error()) + } else { + t.Error("Expect error result") + } + + successFunc := func() (bool,interface{}, error) { + return true,nil,nil + } + + _, successError := LoopCall(attempts, successFunc); + if(successError != nil) { + t.Error("Expect success result") + } else { + t.Logf("success func complete successful") + } +} diff --git a/registry/storage/driver/oss/oss.go b/registry/storage/driver/oss/oss.go old mode 100755 new mode 100644 index 9c52d5773..91ab4b1aa --- a/registry/storage/driver/oss/oss.go +++ b/registry/storage/driver/oss/oss.go @@ -56,6 +56,7 @@ type DriverParameters struct { Secure bool ChunkSize int64 RootDirectory string + Endpoint string } func init() { @@ -175,6 +176,11 @@ func FromParameters(parameters map[string]interface{}) (*Driver, error) { rootDirectory = "" } + endpoint, ok := parameters["endpoint"] + if !ok { + endpoint = "" + } + params := DriverParameters{ AccessKeyID: fmt.Sprint(accessKey), AccessKeySecret: fmt.Sprint(secretKey), @@ -185,6 +191,7 @@ func FromParameters(parameters map[string]interface{}) (*Driver, error) { Encrypt: encryptBool, Secure: secureBool, Internal: internalBool, + Endpoint: fmt.Sprint(endpoint), } return New(params) @@ -195,6 +202,7 @@ func FromParameters(parameters map[string]interface{}) (*Driver, error) { func New(params DriverParameters) (*Driver, error) { client := oss.NewOSSClient(params.Region, params.Internal, params.AccessKeyID, params.AccessKeySecret, params.Secure) + client.SetEndpoint(params.Endpoint) bucket := client.Bucket(params.Bucket) // Validate that the given credentials have at least read permissions in the diff --git a/registry/storage/driver/oss/oss_test.go b/registry/storage/driver/oss/oss_test.go old mode 100755 new mode 100644 index ecfe36e9d..2b469f34f --- a/registry/storage/driver/oss/oss_test.go +++ b/registry/storage/driver/oss/oss_test.go @@ -27,6 +27,7 @@ func init() { internal := os.Getenv("OSS_INTERNAL") encrypt := os.Getenv("OSS_ENCRYPT") secure := os.Getenv("OSS_SECURE") + endpoint := os.Getenv("OSS_ENDPOINT") root, err := ioutil.TempDir("", "driver-") if err != nil { panic(err) @@ -59,15 +60,16 @@ func init() { } parameters := DriverParameters{ - accessKey, - secretKey, - bucket, - alioss.Region(region), - internalBool, - encryptBool, - secureBool, - minChunkSize, - rootDirectory, + AccessKeyID: accessKey, + AccessKeySecret: secretKey, + Bucket: bucket, + Region: alioss.Region(region), + Internal: internalBool, + ChunkSize: minChunkSize, + RootDirectory: rootDirectory, + Encrypt: encryptBool, + Secure: secureBool, + Endpoint: endpoint, } return New(parameters)