TrueCloudLab/distribution
13
0
Fork 3
Code Issues 3 Pull requests Projects Releases 2 Packages Wiki Activity Actions

Support BYOK for OSS storage driver

Browse source 
Change-Id: I423ad03e63bd38aded3abfcba49079ff2fbb3b74
Signed-off-by: Li Yi <denverdino@gmail.com>
This commit is contained in:
Li Yi 2018-12-25 08:30:40 +08:00
parent 40b7b5830a
commit 90bed67126
15 changed files with 1216 additions and 110 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

45
registry/storage/driver/oss/oss.go
View file

@ -54,6 +54,7 @@ type DriverParameters struct {
ChunkSize int64 ChunkSize int64
RootDirectory string RootDirectory string
Endpoint string Endpoint string
EncryptionKeyID string
} }
func init() { func init() {
@ -68,11 +69,12 @@ func (factory *ossDriverFactory) Create(parameters map[string]interface{}) (stor
} }
type driver struct { type driver struct {
Client *oss.Client Client *oss.Client
Bucket *oss.Bucket Bucket *oss.Bucket
ChunkSize int64 ChunkSize int64
Encrypt bool Encrypt bool
RootDirectory string RootDirectory string
EncryptionKeyID string
} }
type baseEmbed struct { type baseEmbed struct {
@ -132,6 +134,11 @@ func FromParameters(parameters map[string]interface{}) (*Driver, error) {
} }
} }
encryptionKeyID, ok := parameters["encryptionkeyid"]
if !ok {
encryptionKeyID = ""
}
secureBool := true secureBool := true
secure, ok := parameters["secure"] secure, ok := parameters["secure"]
if ok { if ok {
@ -185,6 +192,7 @@ func FromParameters(parameters map[string]interface{}) (*Driver, error) {
Secure: secureBool, Secure: secureBool,
Internal: internalBool, Internal: internalBool,
Endpoint: fmt.Sprint(endpoint), Endpoint: fmt.Sprint(endpoint),
EncryptionKeyID: fmt.Sprint(encryptionKeyID),
} }
return New(params) return New(params)
@ -209,11 +217,12 @@ func New(params DriverParameters) (*Driver, error) {
// if you initiated a new OSS client while another one is running on the same bucket. // if you initiated a new OSS client while another one is running on the same bucket.
d := &driver{ d := &driver{
Client: client, Client: client,
Bucket: bucket, Bucket: bucket,
ChunkSize: params.ChunkSize, ChunkSize: params.ChunkSize,
Encrypt: params.Encrypt, Encrypt: params.Encrypt,
RootDirectory: params.RootDirectory, RootDirectory: params.RootDirectory,
EncryptionKeyID: params.EncryptionKeyID,
} }
return &Driver{ return &Driver{
@ -403,7 +412,7 @@ func (d *driver) Move(ctx context.Context, sourcePath string, destPath string) e
err := d.Bucket.CopyLargeFileInParallel(d.ossPath(sourcePath), d.ossPath(destPath), err := d.Bucket.CopyLargeFileInParallel(d.ossPath(sourcePath), d.ossPath(destPath),
d.getContentType(), d.getContentType(),
getPermissions(), getPermissions(),
oss.Options{}, d.getOptions(),
maxConcurrency) maxConcurrency)
if err != nil { if err != nil {
logrus.Errorf("Failed for move from %s to %s: %v", d.ossPath(sourcePath), d.ossPath(destPath), err) logrus.Errorf("Failed for move from %s to %s: %v", d.ossPath(sourcePath), d.ossPath(destPath), err)
@ -503,7 +512,17 @@ func hasCode(err error, code string) bool {
} }
func (d *driver) getOptions() oss.Options { func (d *driver) getOptions() oss.Options {
return oss.Options{ServerSideEncryption: d.Encrypt} return oss.Options{
ServerSideEncryption: d.Encrypt,
ServerSideEncryptionKeyID: d.EncryptionKeyID,
}
}
func (d *driver) getCopyOptions() oss.CopyOptions {
return oss.CopyOptions{
ServerSideEncryption: d.Encrypt,
ServerSideEncryptionKeyID: d.EncryptionKeyID,
}
} }
func getPermissions() oss.ACL { func getPermissions() oss.ACL {
@ -580,7 +599,7 @@ func (w *writer) Write(p []byte) (int, error) {
w.readyPart = contents w.readyPart = contents
} else { } else {
// Otherwise we can use the old file as the new first part // Otherwise we can use the old file as the new first part
_, part, err := multi.PutPartCopy(1, oss.CopyOptions{}, w.driver.Bucket.Name+"/"+w.key) _, part, err := multi.PutPartCopy(1, w.driver.getCopyOptions(), w.driver.Bucket.Name+"/"+w.key)
if err != nil { if err != nil {
return 0, err return 0, err
} }

2
registry/storage/driver/oss/oss_test.go
View file

@ -31,6 +31,7 @@ func init() {
encrypt := os.Getenv("OSS_ENCRYPT") encrypt := os.Getenv("OSS_ENCRYPT")
secure := os.Getenv("OSS_SECURE") secure := os.Getenv("OSS_SECURE")
endpoint := os.Getenv("OSS_ENDPOINT") endpoint := os.Getenv("OSS_ENDPOINT")
encryptionKeyID := os.Getenv("OSS_ENCRYPTIONKEYID")
root, err := ioutil.TempDir("", "driver-") root, err := ioutil.TempDir("", "driver-")
if err != nil { if err != nil {
panic(err) panic(err)
@ -73,6 +74,7 @@ func init() {
Encrypt: encryptBool, Encrypt: encryptBool,
Secure: secureBool, Secure: secureBool,
Endpoint: endpoint, Endpoint: endpoint,
EncryptionKeyID: encryptionKeyID,
} }
return New(parameters) return New(parameters)

2
vendor.conf
View file

@ -7,7 +7,7 @@ github.com/beorn7/perks 4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9
github.com/bugsnag/bugsnag-go b1d153021fcd90ca3f080db36bec96dc690fb274 github.com/bugsnag/bugsnag-go b1d153021fcd90ca3f080db36bec96dc690fb274
github.com/bugsnag/osext 0dd3f918b21bec95ace9dc86c7e70266cfc5c702 github.com/bugsnag/osext 0dd3f918b21bec95ace9dc86c7e70266cfc5c702
github.com/bugsnag/panicwrap e2c28503fcd0675329da73bf48b33404db873782 github.com/bugsnag/panicwrap e2c28503fcd0675329da73bf48b33404db873782
github.com/denverdino/aliyungo afedced274aa9a7fcdd47ac97018f0f8db4e5de2 github.com/denverdino/aliyungo 6df11717a253d9c7d4141f9af4deaa7c580cd531
github.com/dgrijalva/jwt-go a601269ab70c205d26370c16f7c81e9017c14e04 github.com/dgrijalva/jwt-go a601269ab70c205d26370c16f7c81e9017c14e04
github.com/docker/go-metrics 399ea8c73916000c64c2c76e8da00ca82f8387ab github.com/docker/go-metrics 399ea8c73916000c64c2c76e8da00ca82f8387ab
github.com/docker/libtrust fa567046d9b14f6aa788882a950d69651d230b21 github.com/docker/libtrust fa567046d9b14f6aa788882a950d69651d230b21

130
vendor/github.com/denverdino/aliyungo/README.md generated vendored
View file

@ -1,25 +1,34 @@
# AliyunGo: Go SDK for Aliyun Services # AliyunGo: Go SDK for Aliyun Services
This is an unofficial Go SDK for Aliyun Services. You are welcome for contribution. [![Build Status](https://travis-ci.org/denverdino/aliyungo.svg?branch=master)](https://travis-ci.org/denverdino/aliyungo) [![CircleCI](https://circleci.com/gh/denverdino/aliyungo.svg?style=svg)](https://circleci.com/gh/denverdino/aliyungo) [![Go Report Card](https://goreportcard.com/badge/github.com/denverdino/aliyungo)](https://goreportcard.com/report/github.com/denverdino/aliyungo)
This is an unofficial Go SDK for Aliyun services. You are welcome for contribution.
The official SDK for Aliyun services is published. Please visit https://github.com/aliyun/alibaba-cloud-sdk-go for details
## Package Structure ## Package Structure
* ecs: [Elastic Compute Service](https://help.aliyun.com/document_detail/ecs/open-api/summary.html) * cdn: [Content Delivery Network](https://help.aliyun.com/document_detail/27101.html)
* oss: [Open Storage Service](https://help.aliyun.com/document_detail/oss/api-reference/abstract.html) * cms: [Cloud Monitor Service](https://help.aliyun.com/document_detail/28615.html)
* slb: [Server Load Balancer](https://help.aliyun.com/document_detail/slb/api-reference/brief-introduction.html) * cs: [Container Service](https://help.aliyun.com/product/25972.html)
* dns: [DNS](https://help.aliyun.com/document_detail/dns/api-reference/summary.html) * dm: [Direct Mail](https://help.aliyun.com/document_detail/29414.html)
* sls: [Logging Service](https://help.aliyun.com/document_detail/sls/api/overview.html) * dns: [DNS](https://help.aliyun.com/document_detail/dns/api-reference/summary.html)
* ram: [Resource Access Management](https://help.aliyun.com/document_detail/ram/ram-api-reference/intro/intro.html) * ecs: [Elastic Compute Service](https://help.aliyun.com/document_detail/ecs/open-api/summary.html)
* rds: [Relational Database Service](https://help.aliyun.com/document_detail/26226.html) * ess: [Auto Scaling](https://help.aliyun.com/document_detail/25857.html)
* cms: [Cloud Monitor Service](https://help.aliyun.com/document_detail/28615.html) * mns: [Message Service](https://help.aliyun.com/document_detail/27414.html)
* sts: [Security Token Service](https://help.aliyun.com/document_detail/28756.html) * mq: [Message Queue](https://help.aliyun.com/document_detail/29532.html)
* dm: [Direct Mail] * nas: [Network Attached Storage](https://help.aliyun.com/document_detail/27518.html)
(https://help.aliyun.com/document_detail/29414.html) * opensearch: [OpenSearch](https://help.aliyun.com/document_detail/29118.html)
* common: Common libary of Aliyun Go SDK * oss: [Open Storage Service](https://help.aliyun.com/document_detail/oss/api-reference/abstract.html)
* util: Utility helpers * push: [Cloud Mobile Push](https://help.aliyun.com/document_detail/30049.html)
* rds: [Relational Database Service](https://help.aliyun.com/document_detail/26226.html)
* ram: [Resource Access Management](https://help.aliyun.com/document_detail/ram/ram-api-reference/intro/intro.html)
* slb: [Server Load Balancer](https://help.aliyun.com/document_detail/slb/api-reference/brief-introduction.html)
* sls: [Logging Service](https://help.aliyun.com/document_detail/sls/api/overview.html)
* sms: [Short Message Service](https://help.aliyun.com/product/44282.html)
* sts: [Security Token Service](https://help.aliyun.com/document_detail/28756.html)
* common: Common libary of Aliyun Go SDK
* util: Utility helpers
## Quick Start ## Quick Start
@ -27,7 +36,7 @@ This is an unofficial Go SDK for Aliyun Services. You are welcome for contributi
package main package main
import ( import (
"fmt" "fmt"
"github.com/denverdino/aliyungo/ecs" "github.com/denverdino/aliyungo/ecs"
) )
@ -44,17 +53,26 @@ func main() {
## Documentation ## Documentation
* ECS: [https://godoc.org/github.com/denverdino/aliyungo/ecs](https://godoc.org/github.com/denverdino/aliyungo/ecs) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/ecs?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/ecs) * CDN: [https://godoc.org/github.com/denverdino/aliyungo/cdn](https://godoc.org/github.com/denverdino/aliyungo/cdn)[![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/cdn?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/cdn)
* OSS: [https://godoc.org/github.com/denverdino/aliyungo/oss](https://godoc.org/github.com/denverdino/aliyungo/oss) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/oss?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/oss) * CMS: [https://godoc.org/github.com/denverdino/aliyungo/cms](https://godoc.org/github.com/denverdino/aliyungo/cms) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/cms?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/cms)
* SLB: [https://godoc.org/github.com/denverdino/aliyungo/slb](https://godoc.org/github.com/denverdino/aliyungo/slb) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/slb?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/slb) * CS: [https://godoc.org/github.com/denverdino/aliyungo/cs](https://godoc.org/github.com/denverdino/aliyungo/cs) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/cs?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/cs)
* DNS: [https://godoc.org/github.com/denverdino/aliyungo/dns](https://godoc.org/github.com/denverdino/aliyungo/dns) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/dns?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/dns) * DM: [https://godoc.org/github.com/denverdino/aliyungo/dm](https://godoc.org/github.com/denverdino/aliyungo/dm) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/dm?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/dm)
* SLS: [https://godoc.org/github.com/denverdino/aliyungo/sls](https://godoc.org/github.com/denverdino/aliyungo/sls) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/sls?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/sls) * DNS: [https://godoc.org/github.com/denverdino/aliyungo/dns](https://godoc.org/github.com/denverdino/aliyungo/dns) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/dns?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/dns)
* RAM: [https://godoc.org/github.com/denverdino/aliyungo/ram](https://godoc.org/github.com/denverdino/aliyungo/ram) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/ram?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/ram) * ECS: [https://godoc.org/github.com/denverdino/aliyungo/ecs](https://godoc.org/github.com/denverdino/aliyungo/ecs) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/ecs?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/ecs)
* RDS: [https://godoc.org/github.com/denverdino/aliyungo/rds](https://godoc.org/github.com/denverdino/aliyungo/rds) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/rds?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/rds) * ESS: [https://godoc.org/github.com/denverdino/aliyungo/ess](https://godoc.org/github.com/denverdino/aliyungo/ess)[![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/ess?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/ess)
* CMS: [https://godoc.org/github.com/denverdino/aliyungo/cms](https://godoc.org/github.com/denverdino/aliyungo/cms) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/cms?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/cms) * MNS: [https://godoc.org/github.com/denverdino/aliyungo/mns](https://godoc.org/github.com/denverdino/aliyungo/mns)[![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/mns?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/mns)
* STS: [https://godoc.org/github.com/denverdino/aliyungo/sts](https://godoc.org/github.com/denverdino/aliyungo/sts) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/sts?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/sts) * MQ: [https://godoc.org/github.com/denverdino/aliyungo/mq](https://godoc.org/github.com/denverdino/aliyungo/mq) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/mq?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/mq)
* DM: [https://godoc.org/github.com/denverdino/aliyungo/dm](https://godoc.org/github.com/denverdino/aliyungo/dm) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/dm?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/dm) * NAS: [https://godoc.org/github.com/denverdino/aliyungo/nas](https://godoc.org/github.com/denverdino/aliyungo/nas) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/nas?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/nas)
* OPENSEARCH: [https://godoc.org/github.com/denverdino/aliyungo/opensearch](https://godoc.org/github.com/denverdino/aliyungo/opensearch) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/opensearch?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/opensearch)
* OSS: [https://godoc.org/github.com/denverdino/aliyungo/oss](https://godoc.org/github.com/denverdino/aliyungo/oss) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/oss?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/oss)
* PUSH: [https://godoc.org/github.com/denverdino/aliyungo/push](https://godoc.org/github.com/denverdino/aliyungo/push) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/push?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/push)
* RAM: [https://godoc.org/github.com/denverdino/aliyungo/ram](https://godoc.org/github.com/denverdino/aliyungo/ram) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/ram?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/ram)
* RDS: [https://godoc.org/github.com/denverdino/aliyungo/rds](https://godoc.org/github.com/denverdino/aliyungo/rds) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/rds?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/rds)
* SLB: [https://godoc.org/github.com/denverdino/aliyungo/slb](https://godoc.org/github.com/denverdino/aliyungo/slb) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/slb?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/slb)
* SLS: [https://godoc.org/github.com/denverdino/aliyungo/sls](https://godoc.org/github.com/denverdino/aliyungo/sls) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/sls?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/sls)
* SMS: [https://godoc.org/github.com/denverdino/aliyungo/sms](https://godoc.org/github.com/denverdino/aliyungo/sms) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/sms?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/sms)
* STS: [https://godoc.org/github.com/denverdino/aliyungo/sts](https://godoc.org/github.com/denverdino/aliyungo/sts) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/sts?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/sts)
## Build and Install ## Build and Install
go get: go get:
@ -63,10 +81,9 @@ go get:
go get github.com/denverdino/aliyungo go get github.com/denverdino/aliyungo
``` ```
## Test ECS ## Test ECS
Modify "ecs/config_test.go" Modify "ecs/config_test.go"
```sh ```sh
TestAccessKeyId = "MY_ACCESS_KEY_ID" TestAccessKeyId = "MY_ACCESS_KEY_ID"
@ -75,10 +92,10 @@ Modify "ecs/config_test.go"
TestIAmRich = false TestIAmRich = false
``` ```
* TestAccessKeyId: the Access Key Id * TestAccessKeyId: the Access Key Id
* TestAccessKeySecret: the Access Key Secret. * TestAccessKeySecret: the Access Key Secret.
* TestInstanceId: the existing instance id for testing. It will be stopped and restarted during testing. * TestInstanceId: the existing instance id for testing. It will be stopped and restarted during testing.
* TestIAmRich(Optional): If it is set to true, it will perform tests to create virtual machines and disks under your account. And you will pay the bill. :-) * TestIAmRich(Optional): If it is set to true, it will perform tests to create virtual machines and disks under your account. And you will pay the bill. :-)
Under "ecs" and run Under "ecs" and run
@ -88,7 +105,7 @@ go test
## Test OSS ## Test OSS
Modify "oss/config_test.go" Modify "oss/config_test.go"
```sh ```sh
TestAccessKeyId = "MY_ACCESS_KEY_ID" TestAccessKeyId = "MY_ACCESS_KEY_ID"
@ -97,11 +114,10 @@ Modify "oss/config_test.go"
TestBucket = "denverdino" TestBucket = "denverdino"
``` ```
* TestAccessKeyId: the Access Key Id * TestAccessKeyId: the Access Key Id
* TestAccessKeySecret: the Access Key Secret. * TestAccessKeySecret: the Access Key Secret.
* TestRegion: the region of OSS for testing * TestRegion: the region of OSS for testing
* TestBucket: the bucket name for testing * TestBucket: the bucket name for testing
Under "oss" and run Under "oss" and run
@ -112,24 +128,47 @@ go test
## Contributors ## Contributors
* Li Yi (denverdino@gmail.com) * Li Yi (denverdino@gmail.com)
* tgic (farmer1992@gmail.com) * Boshi Lian (farmer1992@gmail.com)
* Yu Zhou (oscarrr110@gmail.com) * Yu Zhou (oscarrr110@gmail.com)
* Yufei Zhang * Yufei Zhang
* linuxlikerqq * linuxlikerqq
* Changhai Yan (changhai.ych@alibaba-inc.com) * Changhai Yan
* Jizhong Jiang (jiangjizhong@gmail.com) * Jizhong Jiang (jiangjizhong@gmail.com)
* Kent Wang (pragkent@gmail.com) * Kent Wang (pragkent@gmail.com)
* ringtail (zhongwei.lzw@alibaba-inc.com) * ringtail
* aiden0z (aiden0xz@gmail.com) * aiden0z (aiden0xz@gmail.com)
* jimmycmh * jimmycmh
* menglingwei * menglingwei
* mingang.he (dustgle@gmail.com) * mingang.he (dustgle@gmail.com)
* chainone (chainone@gmail.com) * Young Chen (chainone@gmail.com)
* johnzeng * johnzeng
* spacexnice (445436286@qq.com)
* xiaoheihero
* hmgle (dustgle@gmail.com)
* jzwlqx (jiangjizhong@gmail.com)
* Linhua Tan (toolchainX@gmail.com)
* Plutonist (p@vecsight.com)
* Bin Liu
* wangyue
* demonwy
* yarous224
* yufeizyf (xazyf9111@sina.cn)
* keontang (ikeontang@gmail.com)
* Cholerae Hu (me@cholerae.com)
* Zach Bergh (berghzach@gmail.com)
* Bingshen Wang
* xiaozhu36
* Russell (yufeiwu@gmail.com)
* zhuzhih2017
* cheyang
* Hobo Chen
* Shuwei Yin
* Xujin Zheng (xujinzheng@gmail.com)
* Dino Lai (dinos80152@gmail.com)
## License ## License
This project is licensed under the Apache License, Version 2.0. See [LICENSE](https://github.com/denverdino/aliyungo/blob/master/LICENSE.txt) for the full license text.
This project is licensed under the Apache License, Version 2.0. See [LICENSE](https://github.com/denverdino/aliyungo/blob/master/LICENSE.txt) for the full license text.
## Related projects ## Related projects
@ -137,7 +176,6 @@ This project is licensed under the Apache License, Version 2.0. See [LICENSE](ht
* Aliyun OSS driver for Docker Registry V2: [Pull request](https://github.com/docker/distribution/pull/514) * Aliyun OSS driver for Docker Registry V2: [Pull request](https://github.com/docker/distribution/pull/514)
## References ## References
The GO API design of OSS refer the implementation from [https://github.com/AdRoll/goamz](https://github.com/AdRoll) The GO API design of OSS refer the implementation from [https://github.com/AdRoll/goamz](https://github.com/AdRoll)

419
vendor/github.com/denverdino/aliyungo/common/client.go generated vendored
View file

@ -3,35 +3,206 @@ package common
import ( import (
"bytes" "bytes"
"encoding/json" "encoding/json"
"errors"
"fmt"
"io/ioutil" "io/ioutil"
"log" "log"
"net/http" "net/http"
"net/url" "net/url"
"os"
"strconv"
"strings"
"time" "time"
"github.com/denverdino/aliyungo/util" "github.com/denverdino/aliyungo/util"
) )
// RemovalPolicy.N add index to array item
// RemovalPolicy=["a", "b"] => RemovalPolicy.1="a" RemovalPolicy.2="b"
type FlattenArray []string
// string contains underline which will be replaced with dot
// SystemDisk_Category => SystemDisk.Category
type UnderlineString string
// A Client represents a client of ECS services // A Client represents a client of ECS services
type Client struct { type Client struct {
AccessKeyId string //Access Key Id AccessKeyId string //Access Key Id
AccessKeySecret string //Access Key Secret AccessKeySecret string //Access Key Secret
securityToken string
debug bool debug bool
httpClient *http.Client httpClient *http.Client
endpoint string endpoint string
version string version string
serviceCode string
regionID Region
businessInfo string
userAgent string
} }
// NewClient creates a new instance of ECS client // Initialize properties of a client instance
func (client *Client) Init(endpoint, version, accessKeyId, accessKeySecret string) { func (client *Client) Init(endpoint, version, accessKeyId, accessKeySecret string) {
client.AccessKeyId = accessKeyId client.AccessKeyId = accessKeyId
client.AccessKeySecret = accessKeySecret + "&" ak := accessKeySecret
if !strings.HasSuffix(ak, "&") {
ak += "&"
}
client.AccessKeySecret = ak
client.debug = false client.debug = false
client.httpClient = &http.Client{} handshakeTimeout, err := strconv.Atoi(os.Getenv("TLSHandshakeTimeout"))
if err != nil {
handshakeTimeout = 0
}
if handshakeTimeout == 0 {
client.httpClient = &http.Client{}
} else {
t := &http.Transport{
TLSHandshakeTimeout: time.Duration(handshakeTimeout) * time.Second}
client.httpClient = &http.Client{Transport: t}
}
client.endpoint = endpoint client.endpoint = endpoint
client.version = version client.version = version
} }
// Initialize properties of a client instance including regionID
func (client *Client) NewInit(endpoint, version, accessKeyId, accessKeySecret, serviceCode string, regionID Region) {
client.Init(endpoint, version, accessKeyId, accessKeySecret)
client.serviceCode = serviceCode
client.regionID = regionID
}
// Intialize client object when all properties are ready
func (client *Client) InitClient() *Client {
client.debug = false
handshakeTimeout, err := strconv.Atoi(os.Getenv("TLSHandshakeTimeout"))
if err != nil {
handshakeTimeout = 0
}
if handshakeTimeout == 0 {
client.httpClient = &http.Client{}
} else {
t := &http.Transport{
TLSHandshakeTimeout: time.Duration(handshakeTimeout) * time.Second}
client.httpClient = &http.Client{Transport: t}
}
return client
}
func (client *Client) NewInitForAssumeRole(endpoint, version, accessKeyId, accessKeySecret, serviceCode string, regionID Region, securityToken string) {
client.NewInit(endpoint, version, accessKeyId, accessKeySecret, serviceCode, regionID)
client.securityToken = securityToken
}
//getLocationEndpoint
func (client *Client) getEndpointByLocation() string {
locationClient := NewLocationClient(client.AccessKeyId, client.AccessKeySecret, client.securityToken)
locationClient.SetDebug(true)
return locationClient.DescribeOpenAPIEndpoint(client.regionID, client.serviceCode)
}
//NewClient using location service
func (client *Client) setEndpointByLocation(region Region, serviceCode, accessKeyId, accessKeySecret, securityToken string) {
locationClient := NewLocationClient(accessKeyId, accessKeySecret, securityToken)
locationClient.SetDebug(true)
ep := locationClient.DescribeOpenAPIEndpoint(region, serviceCode)
if ep == "" {
ep = loadEndpointFromFile(region, serviceCode)
}
if ep != "" {
client.endpoint = ep
}
}
// Ensure all necessary properties are valid
func (client *Client) ensureProperties() error {
var msg string
if client.endpoint == "" {
msg = fmt.Sprintf("endpoint cannot be empty!")
} else if client.version == "" {
msg = fmt.Sprintf("version cannot be empty!")
} else if client.AccessKeyId == "" {
msg = fmt.Sprintf("AccessKeyId cannot be empty!")
} else if client.AccessKeySecret == "" {
msg = fmt.Sprintf("AccessKeySecret cannot be empty!")
}
if msg != "" {
return errors.New(msg)
}
return nil
}
// ----------------------------------------------------
// WithXXX methods
// ----------------------------------------------------
// WithEndpoint sets custom endpoint
func (client *Client) WithEndpoint(endpoint string) *Client {
client.SetEndpoint(endpoint)
return client
}
// WithVersion sets custom version
func (client *Client) WithVersion(version string) *Client {
client.SetVersion(version)
return client
}
// WithRegionID sets Region ID
func (client *Client) WithRegionID(regionID Region) *Client {
client.SetRegionID(regionID)
return client
}
//WithServiceCode sets serviceCode
func (client *Client) WithServiceCode(serviceCode string) *Client {
client.SetServiceCode(serviceCode)
return client
}
// WithAccessKeyId sets new AccessKeyId
func (client *Client) WithAccessKeyId(id string) *Client {
client.SetAccessKeyId(id)
return client
}
// WithAccessKeySecret sets new AccessKeySecret
func (client *Client) WithAccessKeySecret(secret string) *Client {
client.SetAccessKeySecret(secret)
return client
}
// WithSecurityToken sets securityToken
func (client *Client) WithSecurityToken(securityToken string) *Client {
client.SetSecurityToken(securityToken)
return client
}
// WithDebug sets debug mode to log the request/response message
func (client *Client) WithDebug(debug bool) *Client {
client.SetDebug(debug)
return client
}
// WithBusinessInfo sets business info to log the request/response message
func (client *Client) WithBusinessInfo(businessInfo string) *Client {
client.SetBusinessInfo(businessInfo)
return client
}
// WithUserAgent sets user agent to the request/response message
func (client *Client) WithUserAgent(userAgent string) *Client {
client.SetUserAgent(userAgent)
return client
}
// ----------------------------------------------------
// SetXXX methods
// ----------------------------------------------------
// SetEndpoint sets custom endpoint // SetEndpoint sets custom endpoint
func (client *Client) SetEndpoint(endpoint string) { func (client *Client) SetEndpoint(endpoint string) {
client.endpoint = endpoint client.endpoint = endpoint
@ -42,6 +213,16 @@ func (client *Client) SetVersion(version string) {
client.version = version client.version = version
} }
// SetEndpoint sets Region ID
func (client *Client) SetRegionID(regionID Region) {
client.regionID = regionID
}
//SetServiceCode sets serviceCode
func (client *Client) SetServiceCode(serviceCode string) {
client.serviceCode = serviceCode
}
// SetAccessKeyId sets new AccessKeyId // SetAccessKeyId sets new AccessKeyId
func (client *Client) SetAccessKeyId(id string) { func (client *Client) SetAccessKeyId(id string) {
client.AccessKeyId = id client.AccessKeyId = id
@ -57,11 +238,55 @@ func (client *Client) SetDebug(debug bool) {
client.debug = debug client.debug = debug
} }
// SetBusinessInfo sets business info to log the request/response message
func (client *Client) SetBusinessInfo(businessInfo string) {
if strings.HasPrefix(businessInfo, "/") {
client.businessInfo = businessInfo
} else if businessInfo != "" {
client.businessInfo = "/" + businessInfo
}
}
// SetUserAgent sets user agent to the request/response message
func (client *Client) SetUserAgent(userAgent string) {
client.userAgent = userAgent
}
//set SecurityToken
func (client *Client) SetSecurityToken(securityToken string) {
client.securityToken = securityToken
}
func (client *Client) initEndpoint() error {
// if set any value to "CUSTOMIZED_ENDPOINT" could skip location service.
// example: export CUSTOMIZED_ENDPOINT=true
if os.Getenv("CUSTOMIZED_ENDPOINT") != "" {
return nil
}
if client.serviceCode != "" && client.regionID != "" {
endpoint := client.getEndpointByLocation()
if endpoint == "" {
return GetCustomError("InvalidEndpoint", "endpoint is empty,pls check")
}
client.endpoint = endpoint
}
return nil
}
// Invoke sends the raw HTTP request for ECS services // Invoke sends the raw HTTP request for ECS services
func (client *Client) Invoke(action string, args interface{}, response interface{}) error { func (client *Client) Invoke(action string, args interface{}, response interface{}) error {
if err := client.ensureProperties(); err != nil {
return err
}
//init endpoint
if err := client.initEndpoint(); err != nil {
return err
}
request := Request{} request := Request{}
request.init(client.version, action, client.AccessKeyId) request.init(client.version, action, client.AccessKeyId, client.securityToken, client.regionID)
query := util.ConvertToQueryValues(request) query := util.ConvertToQueryValues(request)
util.SetQueryValues(args, &query) util.SetQueryValues(args, &query)
@ -74,13 +299,15 @@ func (client *Client) Invoke(action string, args interface{}, response interface
httpReq, err := http.NewRequest(ECSRequestMethod, requestURL, nil) httpReq, err := http.NewRequest(ECSRequestMethod, requestURL, nil)
// TODO move to util and add build val flag
httpReq.Header.Set("X-SDK-Client", `AliyunGO/`+Version)
if err != nil { if err != nil {
return GetClientError(err) return GetClientError(err)
} }
// TODO move to util and add build val flag
httpReq.Header.Set("X-SDK-Client", `AliyunGO/`+Version+client.businessInfo)
httpReq.Header.Set("User-Agent", httpReq.UserAgent()+" "+client.userAgent)
t0 := time.Now() t0 := time.Now()
httpResp, err := client.httpClient.Do(httpReq) httpResp, err := client.httpClient.Do(httpReq)
t1 := time.Now() t1 := time.Now()
@ -125,6 +352,174 @@ func (client *Client) Invoke(action string, args interface{}, response interface
return nil return nil
} }
// Invoke sends the raw HTTP request for ECS services
func (client *Client) InvokeByFlattenMethod(action string, args interface{}, response interface{}) error {
if err := client.ensureProperties(); err != nil {
return err
}
//init endpoint
if err := client.initEndpoint(); err != nil {
return err
}
request := Request{}
request.init(client.version, action, client.AccessKeyId, client.securityToken, client.regionID)
query := util.ConvertToQueryValues(request)
util.SetQueryValueByFlattenMethod(args, &query)
// Sign request
signature := util.CreateSignatureForRequest(ECSRequestMethod, &query, client.AccessKeySecret)
// Generate the request URL
requestURL := client.endpoint + "?" + query.Encode() + "&Signature=" + url.QueryEscape(signature)
httpReq, err := http.NewRequest(ECSRequestMethod, requestURL, nil)
if err != nil {
return GetClientError(err)
}
// TODO move to util and add build val flag
httpReq.Header.Set("X-SDK-Client", `AliyunGO/`+Version+client.businessInfo)
httpReq.Header.Set("User-Agent", httpReq.UserAgent()+" "+client.userAgent)
t0 := time.Now()
httpResp, err := client.httpClient.Do(httpReq)
t1 := time.Now()
if err != nil {
return GetClientError(err)
}
statusCode := httpResp.StatusCode
if client.debug {
log.Printf("Invoke %s %s %d (%v)", ECSRequestMethod, requestURL, statusCode, t1.Sub(t0))
}
defer httpResp.Body.Close()
body, err := ioutil.ReadAll(httpResp.Body)
if err != nil {
return GetClientError(err)
}
if client.debug {
var prettyJSON bytes.Buffer
err = json.Indent(&prettyJSON, body, "", " ")
log.Println(string(prettyJSON.Bytes()))
}
if statusCode >= 400 && statusCode <= 599 {
errorResponse := ErrorResponse{}
err = json.Unmarshal(body, &errorResponse)
ecsError := &Error{
ErrorResponse: errorResponse,
StatusCode: statusCode,
}
return ecsError
}
err = json.Unmarshal(body, response)
//log.Printf("%++v", response)
if err != nil {
return GetClientError(err)
}
return nil
}
// Invoke sends the raw HTTP request for ECS services
//改进了一下上面那个方法，可以使用各种Http方法
//2017.1.30 增加了一个path参数，用来拓展访问的地址
func (client *Client) InvokeByAnyMethod(method, action, path string, args interface{}, response interface{}) error {
if err := client.ensureProperties(); err != nil {
return err
}
//init endpoint
if err := client.initEndpoint(); err != nil {
return err
}
request := Request{}
request.init(client.version, action, client.AccessKeyId, client.securityToken, client.regionID)
data := util.ConvertToQueryValues(request)
util.SetQueryValues(args, &data)
// Sign request
signature := util.CreateSignatureForRequest(method, &data, client.AccessKeySecret)
data.Add("Signature", signature)
// Generate the request URL
var (
httpReq *http.Request
err error
)
if method == http.MethodGet {
requestURL := client.endpoint + path + "?" + data.Encode()
//fmt.Println(requestURL)
httpReq, err = http.NewRequest(method, requestURL, nil)
} else {
//fmt.Println(client.endpoint + path)
httpReq, err = http.NewRequest(method, client.endpoint+path, strings.NewReader(data.Encode()))
httpReq.Header.Set("Content-Type", "application/x-www-form-urlencoded")
}
if err != nil {
return GetClientError(err)
}
// TODO move to util and add build val flag
httpReq.Header.Set("X-SDK-Client", `AliyunGO/`+Version+client.businessInfo)
httpReq.Header.Set("User-Agent", httpReq.Header.Get("User-Agent")+" "+client.userAgent)
t0 := time.Now()
httpResp, err := client.httpClient.Do(httpReq)
t1 := time.Now()
if err != nil {
return GetClientError(err)
}
statusCode := httpResp.StatusCode
if client.debug {
log.Printf("Invoke %s %s %d (%v) %v", ECSRequestMethod, client.endpoint, statusCode, t1.Sub(t0), data.Encode())
}
defer httpResp.Body.Close()
body, err := ioutil.ReadAll(httpResp.Body)
if err != nil {
return GetClientError(err)
}
if client.debug {
var prettyJSON bytes.Buffer
err = json.Indent(&prettyJSON, body, "", " ")
log.Println(string(prettyJSON.Bytes()))
}
if statusCode >= 400 && statusCode <= 599 {
errorResponse := ErrorResponse{}
err = json.Unmarshal(body, &errorResponse)
ecsError := &Error{
ErrorResponse: errorResponse,
StatusCode: statusCode,
}
return ecsError
}
err = json.Unmarshal(body, response)
//log.Printf("%++v", response)
if err != nil {
return GetClientError(err)
}
return nil
}
// GenerateClientToken generates the Client Token with random string // GenerateClientToken generates the Client Token with random string
func (client *Client) GenerateClientToken() string { func (client *Client) GenerateClientToken() string {
return util.CreateRandomString() return util.CreateRandomString()
@ -143,3 +538,13 @@ func GetClientErrorFromString(str string) error {
func GetClientError(err error) error { func GetClientError(err error) error {
return GetClientErrorFromString(err.Error()) return GetClientErrorFromString(err.Error())
} }
func GetCustomError(code, message string) error {
return &Error{
ErrorResponse: ErrorResponse{
Code: code,
Message: message,
},
StatusCode: 400,
}
}

208
vendor/github.com/denverdino/aliyungo/common/endpoint.go generated vendored Normal file
View file

@ -0,0 +1,208 @@
package common
import (
"encoding/xml"
"fmt"
"io/ioutil"
"log"
"os"
"strings"
"time"
)
const (
// LocationDefaultEndpoint is the default API endpoint of Location services
locationDefaultEndpoint = "https://location.aliyuncs.com"
locationAPIVersion = "2015-06-12"
HTTP_PROTOCOL = "http"
HTTPS_PROTOCOL = "https"
)
var (
endpoints = make(map[Region]map[string]string)
SpecailEnpoints = map[Region]map[string]string{
APNorthEast1: {
"ecs": "https://ecs.ap-northeast-1.aliyuncs.com",
"slb": "https://slb.ap-northeast-1.aliyuncs.com",
"rds": "https://rds.ap-northeast-1.aliyuncs.com",
"vpc": "https://vpc.ap-northeast-1.aliyuncs.com",
},
APSouthEast2: {
"ecs": "https://ecs.ap-southeast-2.aliyuncs.com",
"slb": "https://slb.ap-southeast-2.aliyuncs.com",
"rds": "https://rds.ap-southeast-2.aliyuncs.com",
"vpc": "https://vpc.ap-southeast-2.aliyuncs.com",
},
APSouthEast3: {
"ecs": "https://ecs.ap-southeast-3.aliyuncs.com",
"slb": "https://slb.ap-southeast-3.aliyuncs.com",
"rds": "https://rds.ap-southeast-3.aliyuncs.com",
"vpc": "https://vpc.ap-southeast-3.aliyuncs.com",
},
MEEast1: {
"ecs": "https://ecs.me-east-1.aliyuncs.com",
"slb": "https://slb.me-east-1.aliyuncs.com",
"rds": "https://rds.me-east-1.aliyuncs.com",
"vpc": "https://vpc.me-east-1.aliyuncs.com",
},
EUCentral1: {
"ecs": "https://ecs.eu-central-1.aliyuncs.com",
"slb": "https://slb.eu-central-1.aliyuncs.com",
"rds": "https://rds.eu-central-1.aliyuncs.com",
"vpc": "https://vpc.eu-central-1.aliyuncs.com",
},
EUWest1: {
"ecs": "https://ecs.eu-west-1.aliyuncs.com",
"slb": "https://slb.eu-west-1.aliyuncs.com",
"rds": "https://rds.eu-west-1.aliyuncs.com",
"vpc": "https://vpc.eu-west-1.aliyuncs.com",
},
Zhangjiakou: {
"ecs": "https://ecs.cn-zhangjiakou.aliyuncs.com",
"slb": "https://slb.cn-zhangjiakou.aliyuncs.com",
"rds": "https://rds.cn-zhangjiakou.aliyuncs.com",
"vpc": "https://vpc.cn-zhangjiakou.aliyuncs.com",
},
Huhehaote: {
"ecs": "https://ecs.cn-huhehaote.aliyuncs.com",
"slb": "https://slb.cn-huhehaote.aliyuncs.com",
"rds": "https://rds.cn-huhehaote.aliyuncs.com",
"vpc": "https://vpc.cn-huhehaote.aliyuncs.com",
},
}
)
//init endpoints from file
func init() {
}
type LocationClient struct {
Client
}
func NewLocationClient(accessKeyId, accessKeySecret, securityToken string) *LocationClient {
endpoint := os.Getenv("LOCATION_ENDPOINT")
if endpoint == "" {
endpoint = locationDefaultEndpoint
}
client := &LocationClient{}
client.Init(endpoint, locationAPIVersion, accessKeyId, accessKeySecret)
client.securityToken = securityToken
return client
}
func NewLocationClientWithSecurityToken(accessKeyId, accessKeySecret, securityToken string) *LocationClient {
endpoint := os.Getenv("LOCATION_ENDPOINT")
if endpoint == "" {
endpoint = locationDefaultEndpoint
}
client := &LocationClient{}
client.WithEndpoint(endpoint).
WithVersion(locationAPIVersion).
WithAccessKeyId(accessKeyId).
WithAccessKeySecret(accessKeySecret).
WithSecurityToken(securityToken).
InitClient()
return client
}
func (client *LocationClient) DescribeEndpoint(args *DescribeEndpointArgs) (*DescribeEndpointResponse, error) {
response := &DescribeEndpointResponse{}
err := client.Invoke("DescribeEndpoint", args, response)
if err != nil {
return nil, err
}
return response, err
}
func (client *LocationClient) DescribeEndpoints(args *DescribeEndpointsArgs) (*DescribeEndpointsResponse, error) {
response := &DescribeEndpointsResponse{}
err := client.Invoke("DescribeEndpoints", args, response)
if err != nil {
return nil, err
}
return response, err
}
func getProductRegionEndpoint(region Region, serviceCode string) string {
if sp, ok := endpoints[region]; ok {
if endpoint, ok := sp[serviceCode]; ok {
return endpoint
}
}
return ""
}
func setProductRegionEndpoint(region Region, serviceCode string, endpoint string) {
endpoints[region] = map[string]string{
serviceCode: endpoint,
}
}
func (client *LocationClient) DescribeOpenAPIEndpoint(region Region, serviceCode string) string {
if endpoint := getProductRegionEndpoint(region, serviceCode); endpoint != "" {
return endpoint
}
defaultProtocols := HTTP_PROTOCOL
args := &DescribeEndpointsArgs{
Id: region,
ServiceCode: serviceCode,
Type: "openAPI",
}
var endpoint *DescribeEndpointsResponse
var err error
for index := 0; index < 5; index++ {
endpoint, err = client.DescribeEndpoints(args)
if err == nil && endpoint != nil && len(endpoint.Endpoints.Endpoint) > 0 {
break
}
time.Sleep(500 * time.Millisecond)
}
if err != nil || endpoint == nil || len(endpoint.Endpoints.Endpoint) <= 0 {
log.Printf("aliyungo: can not get endpoint from service, use default. endpoint=[%v], error=[%v]\n", endpoint, err)
return ""
}
for _, protocol := range endpoint.Endpoints.Endpoint[0].Protocols.Protocols {
if strings.ToLower(protocol) == HTTPS_PROTOCOL {
defaultProtocols = HTTPS_PROTOCOL
break
}
}
ep := fmt.Sprintf("%s://%s", defaultProtocols, endpoint.Endpoints.Endpoint[0].Endpoint)
setProductRegionEndpoint(region, serviceCode, ep)
return ep
}
func loadEndpointFromFile(region Region, serviceCode string) string {
data, err := ioutil.ReadFile("./endpoints.xml")
if err != nil {
return ""
}
var endpoints Endpoints
err = xml.Unmarshal(data, &endpoints)
if err != nil {
return ""
}
for _, endpoint := range endpoints.Endpoint {
if endpoint.RegionIds.RegionId == string(region) {
for _, product := range endpoint.Products.Product {
if strings.ToLower(product.ProductName) == serviceCode {
return fmt.Sprintf("%s://%s", HTTPS_PROTOCOL, product.DomainName)
}
}
}
}
return ""
}

54
vendor/github.com/denverdino/aliyungo/common/regions.go generated vendored
View file

@ -5,15 +5,51 @@ type Region string
// Constants of region definition // Constants of region definition
const ( const (
Hangzhou = Region("cn-hangzhou") Hangzhou = Region("cn-hangzhou")
Qingdao = Region("cn-qingdao") Qingdao = Region("cn-qingdao")
Beijing = Region("cn-beijing") Beijing = Region("cn-beijing")
Hongkong = Region("cn-hongkong") Hongkong = Region("cn-hongkong")
Shenzhen = Region("cn-shenzhen") Shenzhen = Region("cn-shenzhen")
USWest1 = Region("us-west-1") Shanghai = Region("cn-shanghai")
USEast1 = Region("us-east-1") Zhangjiakou = Region("cn-zhangjiakou")
Huhehaote = Region("cn-huhehaote")
APSouthEast1 = Region("ap-southeast-1") APSouthEast1 = Region("ap-southeast-1")
Shanghai = Region("cn-shanghai") APNorthEast1 = Region("ap-northeast-1")
APSouthEast2 = Region("ap-southeast-2")
APSouthEast3 = Region("ap-southeast-3")
APSouthEast5 = Region("ap-southeast-5")
APSouth1 = Region("ap-south-1")
USWest1 = Region("us-west-1")
USEast1 = Region("us-east-1")
MEEast1 = Region("me-east-1")
EUCentral1 = Region("eu-central-1")
EUWest1 = Region("eu-west-1")
ShenZhenFinance = Region("cn-shenzhen-finance-1")
ShanghaiFinance = Region("cn-shanghai-finance-1")
) )
var ValidRegions = []Region{Hangzhou, Qingdao, Beijing, Shenzhen, Hongkong, Shanghai, USWest1, USEast1, APSouthEast1} var ValidRegions = []Region{
Hangzhou, Qingdao, Beijing, Shenzhen, Hongkong, Shanghai, Zhangjiakou, Huhehaote,
USWest1, USEast1,
APNorthEast1, APSouthEast1, APSouthEast2, APSouthEast3, APSouthEast5,
APSouth1,
MEEast1,
EUCentral1, EUWest1,
ShenZhenFinance, ShanghaiFinance,
}
// IsValidRegion checks if r is an Ali supported region.
func IsValidRegion(r string) bool {
for _, v := range ValidRegions {
if r == string(v) {
return true
}
}
return false
}

6
vendor/github.com/denverdino/aliyungo/common/request.go generated vendored
View file

@ -20,7 +20,9 @@ const (
type Request struct { type Request struct {
Format string Format string
Version string Version string
RegionId Region
AccessKeyId string AccessKeyId string
SecurityToken string
Signature string Signature string
SignatureMethod string SignatureMethod string
Timestamp util.ISO6801Time Timestamp util.ISO6801Time
@ -30,7 +32,7 @@ type Request struct {
Action string Action string
} }
func (request *Request) init(version string, action string, AccessKeyId string) { func (request *Request) init(version string, action string, AccessKeyId string, securityToken string, regionId Region) {
request.Format = JSONResponseFormat request.Format = JSONResponseFormat
request.Timestamp = util.NewISO6801Time(time.Now().UTC()) request.Timestamp = util.NewISO6801Time(time.Now().UTC())
request.Version = version request.Version = version
@ -39,6 +41,8 @@ func (request *Request) init(version string, action string, AccessKeyId string)
request.SignatureNonce = util.CreateRandomString() request.SignatureNonce = util.CreateRandomString()
request.Action = action request.Action = action
request.AccessKeyId = AccessKeyId request.AccessKeyId = AccessKeyId
request.SecurityToken = securityToken
request.RegionId = regionId
} }
type Response struct { type Response struct {

92
vendor/github.com/denverdino/aliyungo/common/types.go generated vendored
View file

@ -13,3 +13,95 @@ const (
PrePaid = InstanceChargeType("PrePaid") PrePaid = InstanceChargeType("PrePaid")
PostPaid = InstanceChargeType("PostPaid") PostPaid = InstanceChargeType("PostPaid")
) )
type DescribeEndpointArgs struct {
Id Region
ServiceCode string
Type string
}
type EndpointItem struct {
Protocols struct {
Protocols []string
}
Type string
Namespace string
Id Region
SerivceCode string
Endpoint string
}
type DescribeEndpointResponse struct {
Response
EndpointItem
}
type DescribeEndpointsArgs struct {
Id Region
ServiceCode string
Type string
}
type DescribeEndpointsResponse struct {
Response
Endpoints APIEndpoints
RequestId string
Success bool
}
type APIEndpoints struct {
Endpoint []EndpointItem
}
type NetType string
const (
Internet = NetType("Internet")
Intranet = NetType("Intranet")
)
type TimeType string
const (
Hour = TimeType("Hour")
Day = TimeType("Day")
Week = TimeType("Week")
Month = TimeType("Month")
Year = TimeType("Year")
)
type NetworkType string
const (
Classic = NetworkType("Classic")
VPC = NetworkType("VPC")
)
type BusinessInfo struct {
Pack string `json:"pack,omitempty"`
ActivityId string `json:"activityId,omitempty"`
}
//xml
type Endpoints struct {
Endpoint []Endpoint `xml:"Endpoint"`
}
type Endpoint struct {
Name string `xml:"name,attr"`
RegionIds RegionIds `xml:"RegionIds"`
Products Products `xml:"Products"`
}
type RegionIds struct {
RegionId string `xml:"RegionId"`
}
type Products struct {
Product []Product `xml:"Product"`
}
type Product struct {
ProductName string `xml:"ProductName"`
DomainName string `xml:"DomainName"`
}

88
vendor/github.com/denverdino/aliyungo/oss/authenticate_callback.go generated vendored Normal file
View file

@ -0,0 +1,88 @@
package oss
import (
"crypto"
"crypto/md5"
"crypto/rsa"
"crypto/x509"
"encoding/base64"
"encoding/pem"
"errors"
"io/ioutil"
"net/http"
"regexp"
"strings"
"sync"
)
type authenticationType struct {
lock *sync.RWMutex
certificate map[string]*rsa.PublicKey
}
var (
authentication = authenticationType{lock: &sync.RWMutex{}, certificate: map[string]*rsa.PublicKey{}}
urlReg = regexp.MustCompile(`^http(|s)://gosspublic.alicdn.com/[0-9a-zA-Z]`)
)
//验证OSS向业务服务器发来的回调函数。
//该方法是并发安全的
//pubKeyUrl 回调请求头中[x-oss-pub-key-url]一项，以Base64编码
//reqUrl oss所发来请求的url，由path+query组成
//reqBody oss所发来请求的body
//authorization authorization为回调头中的签名
func AuthenticateCallBack(pubKeyUrl, reqUrl, reqBody, authorization string) error {
//获取证书url
keyURL, err := base64.URLEncoding.DecodeString(pubKeyUrl)
if err != nil {
return err
}
url := string(keyURL)
//判断证书是否来自于阿里云
if !urlReg.Match(keyURL) {
return errors.New("certificate address error")
}
//获取文件名
rs := []rune(url)
filename := string(rs[strings.LastIndex(url, "/") : len(rs)-1])
authentication.lock.RLock()
certificate := authentication.certificate[filename]
authentication.lock.RUnlock()
//内存中没有证书，下载
if certificate == nil {
authentication.lock.Lock()
res, err := http.Get(url)
if err != nil {
return err
}
defer res.Body.Close()
body, err := ioutil.ReadAll(res.Body)
if err != nil {
return err
}
block, _ := pem.Decode(body)
if block == nil {
return errors.New("certificate error")
}
pubKey, err := x509.ParsePKIXPublicKey(block.Bytes)
if err != nil {
return err
}
certificate = pubKey.(*rsa.PublicKey)
authentication.certificate[filename] = certificate
authentication.lock.Unlock()
}
//证书准备完毕，开始验证
//解析签名
signature, err := base64.StdEncoding.DecodeString(authorization)
if err != nil {
return err
}
hashed := md5.New()
hashed.Write([]byte(reqUrl + "\n" + reqBody))
if err := rsa.VerifyPKCS1v15(certificate, crypto.MD5, hashed.Sum(nil), signature); err != nil {
return err
}
//验证通过
return nil
}

43
vendor/github.com/denverdino/aliyungo/oss/client.go generated vendored
View file

@ -57,12 +57,14 @@ type Owner struct {
// Options struct // Options struct
// //
type Options struct { type Options struct {
ServerSideEncryption bool ServerSideEncryption bool
Meta map[string][]string ServerSideEncryptionKeyID string
ContentEncoding string
CacheControl string Meta map[string][]string
ContentMD5 string ContentEncoding string
ContentDisposition string CacheControl string
ContentMD5 string
ContentDisposition string
//Range string //Range string
//Expires int //Expires int
} }
@ -72,6 +74,9 @@ type CopyOptions struct {
CopySourceOptions string CopySourceOptions string
MetadataDirective string MetadataDirective string
//ContentType string //ContentType string
ServerSideEncryption bool
ServerSideEncryptionKeyID string
} }
// CopyObjectResult is the output from a Copy request // CopyObjectResult is the output from a Copy request
@ -366,7 +371,7 @@ func (b *Bucket) Exists(path string) (exists bool, err error) {
} }
if err != nil { if err != nil {
// We can treat a 403 or 404 as non existance // We can treat a 403 or 404 as non existence
if e, ok := err.(*Error); ok && (e.StatusCode == 403 || e.StatusCode == 404) { if e, ok := err.(*Error); ok && (e.StatusCode == 403 || e.StatusCode == 404) {
return false, nil return false, nil
} }
@ -430,7 +435,7 @@ func (b *Bucket) Put(path string, data []byte, contType string, perm ACL, option
func (b *Bucket) PutCopy(path string, perm ACL, options CopyOptions, source string) (*CopyObjectResult, error) { func (b *Bucket) PutCopy(path string, perm ACL, options CopyOptions, source string) (*CopyObjectResult, error) {
headers := make(http.Header) headers := make(http.Header)
headers.Set("x-oss-acl", string(perm)) headers.Set("x-oss-object-acl", string(perm))
headers.Set("x-oss-copy-source", source) headers.Set("x-oss-copy-source", source)
options.addHeaders(headers) options.addHeaders(headers)
@ -455,7 +460,7 @@ func (b *Bucket) PutReader(path string, r io.Reader, length int64, contType stri
headers := make(http.Header) headers := make(http.Header)
headers.Set("Content-Length", strconv.FormatInt(length, 10)) headers.Set("Content-Length", strconv.FormatInt(length, 10))
headers.Set("Content-Type", contType) headers.Set("Content-Type", contType)
headers.Set("x-oss-acl", string(perm)) headers.Set("x-oss-object-acl", string(perm))
options.addHeaders(headers) options.addHeaders(headers)
req := &request{ req := &request{
@ -491,7 +496,10 @@ func (b *Bucket) PutFile(path string, file *os.File, perm ACL, options Options)
// addHeaders adds o's specified fields to headers // addHeaders adds o's specified fields to headers
func (o Options) addHeaders(headers http.Header) { func (o Options) addHeaders(headers http.Header) {
if o.ServerSideEncryption { if len(o.ServerSideEncryptionKeyID) != 0 {
headers.Set("x-oss-server-side-encryption", "KMS")
headers.Set("x-oss-server-side-encryption-key-id", o.ServerSideEncryptionKeyID)
} else if o.ServerSideEncryption {
headers.Set("x-oss-server-side-encryption", "AES256") headers.Set("x-oss-server-side-encryption", "AES256")
} }
if len(o.ContentEncoding) != 0 { if len(o.ContentEncoding) != 0 {
@ -516,6 +524,13 @@ func (o Options) addHeaders(headers http.Header) {
// addHeaders adds o's specified fields to headers // addHeaders adds o's specified fields to headers
func (o CopyOptions) addHeaders(headers http.Header) { func (o CopyOptions) addHeaders(headers http.Header) {
if len(o.ServerSideEncryptionKeyID) != 0 {
headers.Set("x-oss-server-side-encryption", "KMS")
headers.Set("x-oss-server-side-encryption-key-id", o.ServerSideEncryptionKeyID)
} else if o.ServerSideEncryption {
headers.Set("x-oss-server-side-encryption", "AES256")
}
if len(o.MetadataDirective) != 0 { if len(o.MetadataDirective) != 0 {
headers.Set("x-oss-metadata-directive", o.MetadataDirective) headers.Set("x-oss-metadata-directive", o.MetadataDirective)
} }
@ -1100,7 +1115,7 @@ func (client *Client) setupHttpRequest(req *request) (*http.Request, error) {
// body will be unmarshalled on it. // body will be unmarshalled on it.
func (client *Client) doHttpRequest(c *http.Client, hreq *http.Request, resp interface{}) (*http.Response, error) { func (client *Client) doHttpRequest(c *http.Client, hreq *http.Request, resp interface{}) (*http.Response, error) {
if true { if client.debug {
log.Printf("%s %s ...\n", hreq.Method, hreq.URL.String()) log.Printf("%s %s ...\n", hreq.Method, hreq.URL.String())
} }
hresp, err := c.Do(hreq) hresp, err := c.Do(hreq)
@ -1323,9 +1338,9 @@ func (b *Bucket) CopyLargeFileInParallel(sourcePath string, destPath string, con
} }
currentLength, err := b.GetContentLength(sourcePath) currentLength, err := b.GetContentLength(sourcePath)
log.Printf("Parallel Copy large file[size: %d] from %s to %s\n",currentLength, sourcePath, destPath) log.Printf("Parallel Copy large file[size: %d] from %s to %s\n", currentLength, sourcePath, destPath)
if err != nil { if err != nil {
return err return err
} }

22
vendor/github.com/denverdino/aliyungo/oss/regions.go generated vendored
View file

@ -9,15 +9,25 @@ type Region string
// Constants of region definition // Constants of region definition
const ( const (
Hangzhou = Region("oss-cn-hangzhou") Hangzhou = Region("oss-cn-hangzhou")
Qingdao = Region("oss-cn-qingdao") Qingdao = Region("oss-cn-qingdao")
Beijing = Region("oss-cn-beijing") Beijing = Region("oss-cn-beijing")
Hongkong = Region("oss-cn-hongkong") Hongkong = Region("oss-cn-hongkong")
Shenzhen = Region("oss-cn-shenzhen") Shenzhen = Region("oss-cn-shenzhen")
Shanghai = Region("oss-cn-shanghai")
Zhangjiakou = Region("oss-cn-zhangjiakou")
Huhehaote = Region("oss-cn-huhehaote")
USWest1 = Region("oss-us-west-1") USWest1 = Region("oss-us-west-1")
USEast1 = Region("oss-us-east-1") USEast1 = Region("oss-us-east-1")
APSouthEast1 = Region("oss-ap-southeast-1") APSouthEast1 = Region("oss-ap-southeast-1")
Shanghai = Region("oss-cn-shanghai") APNorthEast1 = Region("oss-ap-northeast-1")
APSouthEast2 = Region("oss-ap-southeast-2")
MEEast1 = Region("oss-me-east-1")
EUCentral1 = Region("oss-eu-central-1")
EUWest1 = Region("oss-eu-west-1")
DefaultRegion = Hangzhou DefaultRegion = Hangzhou
) )

203
vendor/github.com/denverdino/aliyungo/util/encoding.go generated vendored
View file

@ -7,9 +7,26 @@ import (
"net/url" "net/url"
"reflect" "reflect"
"strconv" "strconv"
"strings"
"time" "time"
) )
// change instance=["a", "b"]
// to instance.1="a" instance.2="b"
func FlattenFn(fieldName string, field reflect.Value, values *url.Values) {
l := field.Len()
if l > 0 {
for i := 0; i < l; i++ {
str := field.Index(i).String()
values.Set(fieldName+"."+strconv.Itoa(i+1), str)
}
}
}
func Underline2Dot(name string) string {
return strings.Replace(name, "_", ".", -1)
}
//ConvertToQueryValues converts the struct to url.Values //ConvertToQueryValues converts the struct to url.Values
func ConvertToQueryValues(ifc interface{}) url.Values { func ConvertToQueryValues(ifc interface{}) url.Values {
values := url.Values{} values := url.Values{}
@ -22,6 +39,10 @@ func SetQueryValues(ifc interface{}, values *url.Values) {
setQueryValues(ifc, values, "") setQueryValues(ifc, values, "")
} }
func SetQueryValueByFlattenMethod(ifc interface{}, values *url.Values) {
setQueryValuesByFlattenMethod(ifc, values, "")
}
func setQueryValues(i interface{}, values *url.Values, prefix string) { func setQueryValues(i interface{}, values *url.Values, prefix string) {
// add to support url.Values // add to support url.Values
mapValues, ok := i.(url.Values) mapValues, ok := i.(url.Values)
@ -41,28 +62,32 @@ func setQueryValues(i interface{}, values *url.Values, prefix string) {
fieldName := elemType.Field(i).Name fieldName := elemType.Field(i).Name
anonymous := elemType.Field(i).Anonymous anonymous := elemType.Field(i).Anonymous
tag := elemType.Field(i).Tag.Get("query")
argName := elemType.Field(i).Tag.Get("ArgName")
field := elem.Field(i) field := elem.Field(i)
// TODO Use Tag for validation // TODO Use Tag for validation
// tag := typ.Field(i).Tag.Get("tagname") // tag := typ.Field(i).Tag.Get("tagname")
kind := field.Kind() kind := field.Kind()
isPtr := false
if (kind == reflect.Ptr || kind == reflect.Array || kind == reflect.Slice || kind == reflect.Map || kind == reflect.Chan) && field.IsNil() { if (kind == reflect.Ptr || kind == reflect.Array || kind == reflect.Slice || kind == reflect.Map || kind == reflect.Chan) && field.IsNil() {
continue continue
} }
if kind == reflect.Ptr { if kind == reflect.Ptr {
field = field.Elem() field = field.Elem()
kind = field.Kind() kind = field.Kind()
isPtr = true
} }
var value string var value string
//switch field.Interface().(type) { //switch field.Interface().(type) {
switch kind { switch kind {
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
i := field.Int() i := field.Int()
if i != 0 { if i != 0 || isPtr {
value = strconv.FormatInt(i, 10) value = strconv.FormatInt(i, 10)
} }
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
i := field.Uint() i := field.Uint()
if i != 0 { if i != 0 || isPtr {
value = strconv.FormatUint(i, 10) value = strconv.FormatUint(i, 10)
} }
case reflect.Float32: case reflect.Float32:
@ -93,15 +118,26 @@ func setQueryValues(i interface{}, values *url.Values, prefix string) {
case reflect.String: case reflect.String:
l := field.Len() l := field.Len()
if l > 0 { if l > 0 {
strArray := make([]string, l) if tag == "list" {
for i := 0; i < l; i++ { name := argName
strArray[i] = field.Index(i).String() if argName == "" {
} name = fieldName
bytes, err := json.Marshal(strArray) }
if err == nil { for i := 0; i < l; i++ {
value = string(bytes) valueName := fmt.Sprintf("%s.%d", name, (i + 1))
values.Set(valueName, field.Index(i).String())
}
} else { } else {
log.Printf("Failed to convert JSON: %v", err) strArray := make([]string, l)
for i := 0; i < l; i++ {
strArray[i] = field.Index(i).String()
}
bytes, err := json.Marshal(strArray)
if err == nil {
value = string(bytes)
} else {
log.Printf("Failed to convert JSON: %v", err)
}
} }
} }
default: default:
@ -139,8 +175,8 @@ func setQueryValues(i interface{}, values *url.Values, prefix string) {
} }
} }
if value != "" { if value != "" {
name := elemType.Field(i).Tag.Get("ArgName") name := argName
if name == "" { if argName == "" {
name = fieldName name = fieldName
} }
if prefix != "" { if prefix != "" {
@ -150,3 +186,146 @@ func setQueryValues(i interface{}, values *url.Values, prefix string) {
} }
} }
} }
func setQueryValuesByFlattenMethod(i interface{}, values *url.Values, prefix string) {
// add to support url.Values
mapValues, ok := i.(url.Values)
if ok {
for k, _ := range mapValues {
values.Set(k, mapValues.Get(k))
}
return
}
elem := reflect.ValueOf(i)
if elem.Kind() == reflect.Ptr {
elem = elem.Elem()
}
elemType := elem.Type()
for i := 0; i < elem.NumField(); i++ {
fieldName := elemType.Field(i).Name
anonymous := elemType.Field(i).Anonymous
field := elem.Field(i)
// TODO Use Tag for validation
// tag := typ.Field(i).Tag.Get("tagname")
kind := field.Kind()
isPtr := false
if (kind == reflect.Ptr || kind == reflect.Array || kind == reflect.Slice || kind == reflect.Map || kind == reflect.Chan) && field.IsNil() {
continue
}
if kind == reflect.Ptr {
field = field.Elem()
kind = field.Kind()
isPtr = true
}
var value string
//switch field.Interface().(type) {
switch kind {
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
i := field.Int()
if i != 0 || isPtr {
value = strconv.FormatInt(i, 10)
}
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
i := field.Uint()
if i != 0 || isPtr {
value = strconv.FormatUint(i, 10)
}
case reflect.Float32:
value = strconv.FormatFloat(field.Float(), 'f', 4, 32)
case reflect.Float64:
value = strconv.FormatFloat(field.Float(), 'f', 4, 64)
case reflect.Bool:
value = strconv.FormatBool(field.Bool())
case reflect.String:
value = field.String()
case reflect.Map:
ifc := field.Interface()
m := ifc.(map[string]string)
if m != nil {
j := 0
for k, v := range m {
j++
keyName := fmt.Sprintf("%s.%d.Key", fieldName, j)
values.Set(keyName, k)
valueName := fmt.Sprintf("%s.%d.Value", fieldName, j)
values.Set(valueName, v)
}
}
case reflect.Slice:
if field.Type().Name() == "FlattenArray" {
FlattenFn(fieldName, field, values)
} else {
switch field.Type().Elem().Kind() {
case reflect.Uint8:
value = string(field.Bytes())
case reflect.String:
l := field.Len()
if l > 0 {
strArray := make([]string, l)
for i := 0; i < l; i++ {
strArray[i] = field.Index(i).String()
}
bytes, err := json.Marshal(strArray)
if err == nil {
value = string(bytes)
} else {
log.Printf("Failed to convert JSON: %v", err)
}
}
default:
l := field.Len()
for j := 0; j < l; j++ {
prefixName := fmt.Sprintf("%s.%d.", fieldName, (j + 1))
ifc := field.Index(j).Interface()
//log.Printf("%s : %v", prefixName, ifc)
if ifc != nil {
setQueryValuesByFlattenMethod(ifc, values, prefixName)
}
}
continue
}
}
default:
switch field.Interface().(type) {
case ISO6801Time:
t := field.Interface().(ISO6801Time)
value = t.String()
case time.Time:
t := field.Interface().(time.Time)
value = GetISO8601TimeStamp(t)
default:
ifc := field.Interface()
if ifc != nil {
if anonymous {
SetQueryValues(ifc, values)
} else {
prefixName := fieldName + "."
setQueryValuesByFlattenMethod(ifc, values, prefixName)
}
continue
}
}
}
if value != "" {
name := elemType.Field(i).Tag.Get("ArgName")
if name == "" {
name = fieldName
}
if prefix != "" {
name = prefix + name
}
// NOTE: here we will change name to underline style when the type is UnderlineString
if field.Type().Name() == "UnderlineString" {
name = Underline2Dot(name)
}
values.Set(name, value)
}
}
}

1
vendor/github.com/denverdino/aliyungo/util/signature.go generated vendored
View file

@ -35,6 +35,5 @@ func CreateSignatureForRequest(method string, values *url.Values, accessKeySecre
canonicalizedQueryString := percentReplace(values.Encode()) canonicalizedQueryString := percentReplace(values.Encode())
stringToSign := method + "&%2F&" + url.QueryEscape(canonicalizedQueryString) stringToSign := method + "&%2F&" + url.QueryEscape(canonicalizedQueryString)
return CreateSignature(stringToSign, accessKeySecret) return CreateSignature(stringToSign, accessKeySecret)
} }

11
vendor/github.com/denverdino/aliyungo/util/util.go generated vendored
View file

@ -9,6 +9,8 @@ import (
"net/url" "net/url"
"sort" "sort"
"time" "time"
"fmt"
"encoding/json"
) )
const dictionary = "_0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" const dictionary = "_0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
@ -145,3 +147,12 @@ func GenerateRandomECSPassword() string {
return string(s) return string(s)
} }
func PrettyJson(object interface{}) string {
b,err := json.MarshalIndent(object,"", " ")
if err != nil {
fmt.Printf("ERROR: PrettyJson, %v\n %s\n",err,b)
}
return string(b)
}