add endpoint support

Godeps/Godeps.json

@@ -46,11 +46,11 @@
 		},
 		{
 			"ImportPath": "github.com/denverdino/aliyungo/oss",
-			"Rev": "17d1e888c907ffdbd875f37500f3d130ce2ee6eb"
+			"Rev": "56047189188d6558b6a1456d647970032343b511"
 		},
 		{
 			"ImportPath": "github.com/denverdino/aliyungo/util",
-			"Rev": "17d1e888c907ffdbd875f37500f3d130ce2ee6eb"
+			"Rev": "56047189188d6558b6a1456d647970032343b511"
 		},
 		{
 			"ImportPath": "github.com/docker/docker/pkg/tarsum",

Godeps/_workspace/src/github.com/denverdino/aliyungo/oss/client.go

@@ -35,7 +35,9 @@ type Client struct {
 	Secure          bool
 	ConnectTimeout  time.Duration
 	ReadTimeout     time.Duration
-	debug           bool
+
+	endpoint string
+	debug    bool
 }
 
 // The Bucket type encapsulates operations with an bucket.
@@ -159,6 +161,12 @@ func (client *Client) locationConstraint() io.Reader {
 	return strings.NewReader(constraint)
 }
 
+// override default endpoint
+func (client *Client) SetEndpoint(endpoint string) {
+	// TODO check endpoint
+	client.endpoint = endpoint
+}
+
 // PutBucket creates a new bucket.
 func (b *Bucket) PutBucket(perm ACL) error {
 	headers := make(http.Header)
@@ -963,7 +971,12 @@ func (client *Client) query(req *request, resp interface{}) error {
 
 // Sets baseurl on req from bucket name and the region endpoint
 func (client *Client) setBaseURL(req *request) error {
-	req.baseurl = client.Region.GetEndpoint(client.Internal, req.bucket, client.Secure)
+
+	if client.endpoint == "" {
+		req.baseurl = client.Region.GetEndpoint(client.Internal, req.bucket, client.Secure)
+	} else {
+		req.baseurl = fmt.Sprintf("%s://%s", client.endpoint, getProtocol(client.Secure))
+	}
 
 	return nil
 }

Godeps/_workspace/src/github.com/denverdino/aliyungo/util/attempt.go

@@ -4,9 +4,11 @@ import (
 	"time"
 )
 
+// AttemptStrategy is reused from the goamz package
+
 // AttemptStrategy represents a strategy for waiting for an action
 // to complete successfully. This is an internal type used by the
-// implementation of other goamz packages.
+// implementation of other packages.
 type AttemptStrategy struct {
 	Total time.Duration // total duration of attempt.
 	Delay time.Duration // interval between each try in the burst.

Godeps/_workspace/src/github.com/denverdino/aliyungo/util/encoding.go

@@ -39,49 +39,48 @@ func setQueryValues(i interface{}, values *url.Values, prefix string) {
 		}
 		if kind == reflect.Ptr {
 			field = field.Elem()
+			kind = field.Kind()
 		}
 		var value string
-		switch field.Interface().(type) {
-		case int, int8, int16, int32, int64:
+		//switch field.Interface().(type) {
+		switch kind {
+		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
 			i := field.Int()
 			if i != 0 {
 				value = strconv.FormatInt(i, 10)
 			}
-		case uint, uint8, uint16, uint32, uint64:
+		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
 			i := field.Uint()
 			if i != 0 {
 				value = strconv.FormatUint(i, 10)
 			}
-		case float32:
+		case reflect.Float32:
 			value = strconv.FormatFloat(field.Float(), 'f', 4, 32)
-		case float64:
+		case reflect.Float64:
 			value = strconv.FormatFloat(field.Float(), 'f', 4, 64)
-		case []byte:
-			value = string(field.Bytes())
-		case bool:
+		case reflect.Bool:
 			value = strconv.FormatBool(field.Bool())
-		case string:
+		case reflect.String:
 			value = field.String()
-		case []string:
-			l := field.Len()
-			if l > 0 {
-				strArray := make([]string, l)
-				for i := 0; i < l; i++ {
-					strArray[i] = field.Index(i).String()
+		case reflect.Slice:
+			switch field.Type().Elem().Kind() {
+			case reflect.Uint8:
+				value = string(field.Bytes())
+			case reflect.String:
+				l := field.Len()
+				if l > 0 {
+					strArray := make([]string, l)
+					for i := 0; i < l; i++ {
+						strArray[i] = field.Index(i).String()
+					}
+					bytes, err := json.Marshal(strArray)
+					if err == nil {
+						value = string(bytes)
+					} else {
+						log.Printf("Failed to convert JSON: %v", err)
+					}
 				}
-				bytes, err := json.Marshal(strArray)
-				if err == nil {
-					value = string(bytes)
-				} else {
-					log.Printf("Failed to convert JSON: %v", err)
-				}
-			}
-		case time.Time:
-			t := field.Interface().(time.Time)
-			value = GetISO8601TimeStamp(t)
-
-		default:
-			if kind == reflect.Slice { //Array of structs
+			default:
 				l := field.Len()
 				for j := 0; j < l; j++ {
 					prefixName := fmt.Sprintf("%s.%d.", fieldName, (j + 1))
@@ -91,7 +90,18 @@ func setQueryValues(i interface{}, values *url.Values, prefix string) {
 						setQueryValues(ifc, values, prefixName)
 					}
 				}
-			} else {
+				continue
+			}
+
+		default:
+			switch field.Interface().(type) {
+			case ISO6801Time:
+				t := field.Interface().(ISO6801Time)
+				value = t.String()
+			case time.Time:
+				t := field.Interface().(time.Time)
+				value = GetISO8601TimeStamp(t)
+			default:
 				ifc := field.Interface()
 				if ifc != nil {
 					SetQueryValues(ifc, values)

Godeps/_workspace/src/github.com/denverdino/aliyungo/util/encoding_test.go

@@ -5,6 +5,8 @@ import (
 	"time"
 )
 
+type TestString string
+
 type SubStruct struct {
 	A string
 	B int
@@ -21,6 +23,8 @@ type TestStruct struct {
 	IntPtr      *int     `ArgName:"int-ptr"`
 	StringArray []string `ArgName:"str-array"`
 	StructArray []SubStruct
+	test        TestString
+	tests       []TestString
 }
 
 func TestConvertToQueryValues(t *testing.T) {
@@ -36,9 +40,11 @@ func TestConvertToQueryValues(t *testing.T) {
 			SubStruct{A: "a", B: 1},
 			SubStruct{A: "x", B: 2},
 		},
+		test:  TestString("test"),
+		tests: []TestString{TestString("test1"), TestString("test2")},
 	}
 	result := ConvertToQueryValues(&request).Encode()
-	const expectedResult = "Format=JSON&StructArray.1.A=a&StructArray.1.B=1&StructArray.2.A=x&StructArray.2.B=2&Timestamp=2015-05-26T01%3A02%3A03Z&Version=1.0&bool-ptr=true&int-value=10&str-array=%5B%22abc%22%2C%22xyz%22%5D"
+	const expectedResult = "Format=JSON&StructArray.1.A=a&StructArray.1.B=1&StructArray.2.A=x&StructArray.2.B=2&Timestamp=2015-05-26T01%3A02%3A03Z&Version=1.0&bool-ptr=true&int-value=10&str-array=%5B%22abc%22%2C%22xyz%22%5D&test=test&tests=%5B%22test1%22%2C%22test2%22%5D"
 	if result != expectedResult {
 		t.Error("Incorrect encoding: ", result)
 	}

Godeps/_workspace/src/github.com/denverdino/aliyungo/util/iso6801.go

@@ -21,7 +21,7 @@ type ISO6801Time time.Time
 // time.Time instance.  This causes the nanosecond field to be set to
 // 0, and its time zone set to a fixed zone with no offset from UTC
 // (but it is *not* UTC itself).
-func New(t time.Time) ISO6801Time {
+func NewISO6801Time(t time.Time) ISO6801Time {
 	return ISO6801Time(time.Date(
 		t.Year(),
 		t.Month(),
@@ -58,5 +58,5 @@ func (it *ISO6801Time) UnmarshalJSON(data []byte) error {
 
 // String returns the time in ISO6801Time format
 func (it ISO6801Time) String() string {
-	return time.Time(it).String()
+	return time.Time(it).Format(formatISO8601)
 }

Godeps/_workspace/src/github.com/denverdino/aliyungo/util/iso6801_test.go

@@ -7,7 +7,7 @@ import (
 )
 
 func TestISO8601Time(t *testing.T) {
-	now := New(time.Now().UTC())
+	now := NewISO6801Time(time.Now().UTC())
 
 	data, err := json.Marshal(now)
 	if err != nil {
@@ -26,7 +26,7 @@ func TestISO8601Time(t *testing.T) {
 	}
 
 	if now != now2 {
-		t.Fatalf("Time %s does not equal expected %s", now2, now)
+		t.Errorf("Time %s does not equal expected %s", now2, now)
 	}
 
 	if now.String() != now2.String() {
@@ -46,5 +46,5 @@ func TestISO8601Time(t *testing.T) {
 	if !testValue.B.IsDefault() {
 		t.Fatal("Invaid Unmarshal result for ISO6801Time from empty value")
 	}
-
+	t.Logf("ISO6801Time String(): %s", now2.String())
 }

Godeps/_workspace/src/github.com/denverdino/aliyungo/util/util.go

@@ -2,12 +2,20 @@ package util
 
 import (
 	"bytes"
+	srand "crypto/rand"
+	"encoding/binary"
 	"math/rand"
 	"net/http"
 	"net/url"
 	"sort"
 	"strconv"
 	"time"
+	"errors"
+)
+
+
+const (
+	StatusUnKnown     = "NA"
 )
 
 //CreateRandomString create random string
@@ -52,3 +60,102 @@ func Encode(v url.Values) string {
 func GetGMTime() string {
 	return time.Now().UTC().Format(http.TimeFormat)
 }
+
+//
+
+func randUint32() uint32 {
+	return randUint32Slice(1)[0]
+}
+
+func randUint32Slice(c int) []uint32 {
+	b := make([]byte, c*4)
+
+	_, err := srand.Read(b)
+
+	if err != nil {
+		// fail back to insecure rand
+		rand.Seed(time.Now().UnixNano())
+		for i := range b {
+			b[i] = byte(rand.Int())
+		}
+	}
+
+	n := make([]uint32, c)
+
+	for i := range n {
+		n[i] = binary.BigEndian.Uint32(b[i*4 : i*4+4])
+	}
+
+	return n
+}
+
+func toByte(n uint32, st, ed byte) byte {
+	return byte(n%uint32(ed-st+1) + uint32(st))
+}
+
+func toDigit(n uint32) byte {
+	return toByte(n, '0', '9')
+}
+
+func toLowerLetter(n uint32) byte {
+	return toByte(n, 'a', 'z')
+}
+
+func toUpperLetter(n uint32) byte {
+	return toByte(n, 'A', 'Z')
+}
+
+type convFunc func(uint32) byte
+
+var convFuncs = []convFunc{toDigit, toLowerLetter, toUpperLetter}
+
+// tools for generating a random ECS instance password
+// from 8 to 30 char MUST contain digit upper, case letter and upper case letter
+// http://docs.aliyun.com/#/pub/ecs/open-api/instance&createinstance
+func GenerateRandomECSPassword() string {
+
+	// [8, 30]
+	l := int(randUint32()%23 + 8)
+
+	n := randUint32Slice(l)
+
+	b := make([]byte, l)
+
+	b[0] = toDigit(n[0])
+	b[1] = toLowerLetter(n[1])
+	b[2] = toUpperLetter(n[2])
+
+	for i := 3; i < l; i++ {
+		b[i] = convFuncs[n[i]%3](n[i])
+	}
+
+	s := make([]byte, l)
+	perm := rand.Perm(l)
+	for i, v := range perm {
+		s[v] = b[i]
+	}
+
+	return string(s)
+
+}
+
+
+func LoopCall(attempts AttemptStrategy,api func() (bool,interface{},error))(interface{}, error){
+
+	for attempt := attempts.Start(); attempt.Next(); {
+		needStop,status,err := api()
+
+		if(err != nil) {
+			return nil, errors.New("execution failed")
+		}
+		if(needStop){
+			return status,nil;
+		}
+
+		if attempt.HasNext() {
+			continue;
+		}
+		return nil,errors.New("timeout execution ")
+	}
+	panic("unreachable")
+}

Godeps/_workspace/src/github.com/denverdino/aliyungo/util/util_test.go

@@ -0,0 +1,98 @@
+package util
+
+import (
+	"testing"
+	"errors"
+	"time"
+)
+
+func TestGenerateRandomECSPassword(t *testing.T) {
+	for i := 0; i < 10; i++ {
+		s := GenerateRandomECSPassword()
+
+		if len(s) < 8 || len(s) > 30 {
+			t.Errorf("Generated ECS password [%v]: bad len", s)
+		}
+
+		hasDigit := false
+		hasLower := false
+		hasUpper := false
+
+		for j := range s {
+
+			switch {
+			case '0' <= s[j] && s[j] <= '9':
+				hasDigit = true
+			case 'a' <= s[j] && s[j] <= 'z':
+				hasLower = true
+			case 'A' <= s[j] && s[j] <= 'Z':
+				hasUpper = true
+			}
+		}
+
+		if !hasDigit {
+			t.Errorf("Generated ECS password [%v]: no digit", s)
+		}
+
+		if !hasLower {
+			t.Errorf("Generated ECS password [%v]: no lower letter ", s)
+		}
+
+		if !hasUpper {
+			t.Errorf("Generated ECS password [%v]: no upper letter", s)
+		}
+	}
+}
+
+func TestWaitForSignalWithTimeout(t *testing.T) {
+
+	attempts := AttemptStrategy{
+		Min:   5,
+		Total: 5 * time.Second,
+		Delay: 200 * time.Millisecond,
+	}
+
+	timeoutFunc := func() (bool,interface{},error) {
+		return false,"-1",nil
+	}
+
+	begin := time.Now()
+
+	_, timeoutError := LoopCall(attempts, timeoutFunc);
+	if(timeoutError != nil) {
+		t.Logf("timeout func complete successful")
+	} else {
+		t.Error("Expect timeout result")
+	}
+
+	end := time.Now()
+	duration := end.Sub(begin).Seconds()
+	if( duration  > (float64(attempts.Min) -1)) {
+		t.Logf("timeout func duration is enough")
+	} else {
+		t.Error("timeout func duration is not enough")
+	}
+
+	errorFunc := func() (bool, interface{}, error) {
+		err := errors.New("execution failed");
+		return false,"-1",err
+	}
+
+	_, failedError := LoopCall(attempts, errorFunc);
+	if(failedError != nil) {
+		t.Logf("error func complete successful: " + failedError.Error())
+	} else {
+		t.Error("Expect error result")
+	}
+
+	successFunc := func() (bool,interface{}, error) {
+		return true,nil,nil
+	}
+
+	_, successError := LoopCall(attempts, successFunc);
+	if(successError != nil) {
+		t.Error("Expect success result")
+	} else {
+		t.Logf("success func complete successful")
+	}
+}

registry/storage/driver/oss/oss.go

@@ -56,6 +56,7 @@ type DriverParameters struct {
 	Secure          bool
 	ChunkSize       int64
 	RootDirectory   string
+	Endpoint        string
 }
 
 func init() {
@@ -175,6 +176,11 @@ func FromParameters(parameters map[string]interface{}) (*Driver, error) {
 		rootDirectory = ""
 	}
 
+	endpoint, ok := parameters["endpoint"]
+	if !ok {
+		endpoint = ""
+	}
+
 	params := DriverParameters{
 		AccessKeyID:     fmt.Sprint(accessKey),
 		AccessKeySecret: fmt.Sprint(secretKey),
@@ -185,6 +191,7 @@ func FromParameters(parameters map[string]interface{}) (*Driver, error) {
 		Encrypt:         encryptBool,
 		Secure:          secureBool,
 		Internal:        internalBool,
+		Endpoint:        fmt.Sprint(endpoint),
 	}
 
 	return New(params)
@@ -195,6 +202,7 @@ func FromParameters(parameters map[string]interface{}) (*Driver, error) {
 func New(params DriverParameters) (*Driver, error) {
 
 	client := oss.NewOSSClient(params.Region, params.Internal, params.AccessKeyID, params.AccessKeySecret, params.Secure)
+	client.SetEndpoint(params.Endpoint)
 	bucket := client.Bucket(params.Bucket)
 
 	// Validate that the given credentials have at least read permissions in the

registry/storage/driver/oss/oss_test.go

@@ -27,6 +27,7 @@ func init() {
 	internal := os.Getenv("OSS_INTERNAL")
 	encrypt := os.Getenv("OSS_ENCRYPT")
 	secure := os.Getenv("OSS_SECURE")
+	endpoint := os.Getenv("OSS_ENDPOINT")
 	root, err := ioutil.TempDir("", "driver-")
 	if err != nil {
 		panic(err)
@@ -59,15 +60,16 @@ func init() {
 		}
 
 		parameters := DriverParameters{
-			accessKey,
-			secretKey,
-			bucket,
-			alioss.Region(region),
-			internalBool,
-			encryptBool,
-			secureBool,
-			minChunkSize,
-			rootDirectory,
+			AccessKeyID:     accessKey,
+			AccessKeySecret: secretKey,
+			Bucket:          bucket,
+			Region:          alioss.Region(region),
+			Internal:        internalBool,
+			ChunkSize:       minChunkSize,
+			RootDirectory:   rootDirectory,
+			Encrypt:         encryptBool,
+			Secure:          secureBool,
+			Endpoint:        endpoint,
 		}
 
 		return New(parameters)