From 90bed6712616f818479c40917dcd1bab0482346c Mon Sep 17 00:00:00 2001 From: Li Yi Date: Tue, 25 Dec 2018 08:30:40 +0800 Subject: [PATCH 1/2] Support BYOK for OSS storage driver Change-Id: I423ad03e63bd38aded3abfcba49079ff2fbb3b74 Signed-off-by: Li Yi --- registry/storage/driver/oss/oss.go | 45 +- registry/storage/driver/oss/oss_test.go | 2 + vendor.conf | 2 +- .../github.com/denverdino/aliyungo/README.md | 130 ++++-- .../denverdino/aliyungo/common/client.go | 419 +++++++++++++++++- .../denverdino/aliyungo/common/endpoint.go | 208 +++++++++ .../denverdino/aliyungo/common/regions.go | 54 ++- .../denverdino/aliyungo/common/request.go | 6 +- .../denverdino/aliyungo/common/types.go | 92 ++++ .../aliyungo/oss/authenticate_callback.go | 88 ++++ .../denverdino/aliyungo/oss/client.go | 43 +- .../denverdino/aliyungo/oss/regions.go | 22 +- .../denverdino/aliyungo/util/encoding.go | 203 ++++++++- .../denverdino/aliyungo/util/signature.go | 1 - .../denverdino/aliyungo/util/util.go | 11 + 15 files changed, 1216 insertions(+), 110 deletions(-) create mode 100644 vendor/github.com/denverdino/aliyungo/common/endpoint.go create mode 100644 vendor/github.com/denverdino/aliyungo/oss/authenticate_callback.go diff --git a/registry/storage/driver/oss/oss.go b/registry/storage/driver/oss/oss.go index 1dcf42b8..5cd96203 100644 --- a/registry/storage/driver/oss/oss.go +++ b/registry/storage/driver/oss/oss.go @@ -54,6 +54,7 @@ type DriverParameters struct { ChunkSize int64 RootDirectory string Endpoint string + EncryptionKeyID string } func init() { @@ -68,11 +69,12 @@ func (factory *ossDriverFactory) Create(parameters map[string]interface{}) (stor } type driver struct { - Client *oss.Client - Bucket *oss.Bucket - ChunkSize int64 - Encrypt bool - RootDirectory string + Client *oss.Client + Bucket *oss.Bucket + ChunkSize int64 + Encrypt bool + RootDirectory string + EncryptionKeyID string } type baseEmbed struct { @@ -132,6 +134,11 @@ func FromParameters(parameters map[string]interface{}) (*Driver, error) { } } + encryptionKeyID, ok := parameters["encryptionkeyid"] + if !ok { + encryptionKeyID = "" + } + secureBool := true secure, ok := parameters["secure"] if ok { @@ -185,6 +192,7 @@ func FromParameters(parameters map[string]interface{}) (*Driver, error) { Secure: secureBool, Internal: internalBool, Endpoint: fmt.Sprint(endpoint), + EncryptionKeyID: fmt.Sprint(encryptionKeyID), } return New(params) @@ -209,11 +217,12 @@ func New(params DriverParameters) (*Driver, error) { // if you initiated a new OSS client while another one is running on the same bucket. d := &driver{ - Client: client, - Bucket: bucket, - ChunkSize: params.ChunkSize, - Encrypt: params.Encrypt, - RootDirectory: params.RootDirectory, + Client: client, + Bucket: bucket, + ChunkSize: params.ChunkSize, + Encrypt: params.Encrypt, + RootDirectory: params.RootDirectory, + EncryptionKeyID: params.EncryptionKeyID, } return &Driver{ @@ -403,7 +412,7 @@ func (d *driver) Move(ctx context.Context, sourcePath string, destPath string) e err := d.Bucket.CopyLargeFileInParallel(d.ossPath(sourcePath), d.ossPath(destPath), d.getContentType(), getPermissions(), - oss.Options{}, + d.getOptions(), maxConcurrency) if err != nil { logrus.Errorf("Failed for move from %s to %s: %v", d.ossPath(sourcePath), d.ossPath(destPath), err) @@ -503,7 +512,17 @@ func hasCode(err error, code string) bool { } func (d *driver) getOptions() oss.Options { - return oss.Options{ServerSideEncryption: d.Encrypt} + return oss.Options{ + ServerSideEncryption: d.Encrypt, + ServerSideEncryptionKeyID: d.EncryptionKeyID, + } +} + +func (d *driver) getCopyOptions() oss.CopyOptions { + return oss.CopyOptions{ + ServerSideEncryption: d.Encrypt, + ServerSideEncryptionKeyID: d.EncryptionKeyID, + } } func getPermissions() oss.ACL { @@ -580,7 +599,7 @@ func (w *writer) Write(p []byte) (int, error) { w.readyPart = contents } else { // Otherwise we can use the old file as the new first part - _, part, err := multi.PutPartCopy(1, oss.CopyOptions{}, w.driver.Bucket.Name+"/"+w.key) + _, part, err := multi.PutPartCopy(1, w.driver.getCopyOptions(), w.driver.Bucket.Name+"/"+w.key) if err != nil { return 0, err } diff --git a/registry/storage/driver/oss/oss_test.go b/registry/storage/driver/oss/oss_test.go index 438d9a48..d415115a 100644 --- a/registry/storage/driver/oss/oss_test.go +++ b/registry/storage/driver/oss/oss_test.go @@ -31,6 +31,7 @@ func init() { encrypt := os.Getenv("OSS_ENCRYPT") secure := os.Getenv("OSS_SECURE") endpoint := os.Getenv("OSS_ENDPOINT") + encryptionKeyID := os.Getenv("OSS_ENCRYPTIONKEYID") root, err := ioutil.TempDir("", "driver-") if err != nil { panic(err) @@ -73,6 +74,7 @@ func init() { Encrypt: encryptBool, Secure: secureBool, Endpoint: endpoint, + EncryptionKeyID: encryptionKeyID, } return New(parameters) diff --git a/vendor.conf b/vendor.conf index a249caf2..12f71672 100644 --- a/vendor.conf +++ b/vendor.conf @@ -7,7 +7,7 @@ github.com/beorn7/perks 4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9 github.com/bugsnag/bugsnag-go b1d153021fcd90ca3f080db36bec96dc690fb274 github.com/bugsnag/osext 0dd3f918b21bec95ace9dc86c7e70266cfc5c702 github.com/bugsnag/panicwrap e2c28503fcd0675329da73bf48b33404db873782 -github.com/denverdino/aliyungo afedced274aa9a7fcdd47ac97018f0f8db4e5de2 +github.com/denverdino/aliyungo 6df11717a253d9c7d4141f9af4deaa7c580cd531 github.com/dgrijalva/jwt-go a601269ab70c205d26370c16f7c81e9017c14e04 github.com/docker/go-metrics 399ea8c73916000c64c2c76e8da00ca82f8387ab github.com/docker/libtrust fa567046d9b14f6aa788882a950d69651d230b21 diff --git a/vendor/github.com/denverdino/aliyungo/README.md b/vendor/github.com/denverdino/aliyungo/README.md index fd9da9b1..e0749c4e 100644 --- a/vendor/github.com/denverdino/aliyungo/README.md +++ b/vendor/github.com/denverdino/aliyungo/README.md @@ -1,25 +1,34 @@ # AliyunGo: Go SDK for Aliyun Services -This is an unofficial Go SDK for Aliyun Services. You are welcome for contribution. +[![Build Status](https://travis-ci.org/denverdino/aliyungo.svg?branch=master)](https://travis-ci.org/denverdino/aliyungo) [![CircleCI](https://circleci.com/gh/denverdino/aliyungo.svg?style=svg)](https://circleci.com/gh/denverdino/aliyungo) [![Go Report Card](https://goreportcard.com/badge/github.com/denverdino/aliyungo)](https://goreportcard.com/report/github.com/denverdino/aliyungo) +This is an unofficial Go SDK for Aliyun services. You are welcome for contribution. + +The official SDK for Aliyun services is published. Please visit https://github.com/aliyun/alibaba-cloud-sdk-go for details ## Package Structure -* ecs: [Elastic Compute Service](https://help.aliyun.com/document_detail/ecs/open-api/summary.html) -* oss: [Open Storage Service](https://help.aliyun.com/document_detail/oss/api-reference/abstract.html) -* slb: [Server Load Balancer](https://help.aliyun.com/document_detail/slb/api-reference/brief-introduction.html) -* dns: [DNS](https://help.aliyun.com/document_detail/dns/api-reference/summary.html) -* sls: [Logging Service](https://help.aliyun.com/document_detail/sls/api/overview.html) -* ram: [Resource Access Management](https://help.aliyun.com/document_detail/ram/ram-api-reference/intro/intro.html) -* rds: [Relational Database Service](https://help.aliyun.com/document_detail/26226.html) -* cms: [Cloud Monitor Service](https://help.aliyun.com/document_detail/28615.html) -* sts: [Security Token Service](https://help.aliyun.com/document_detail/28756.html) -* dm: [Direct Mail] -(https://help.aliyun.com/document_detail/29414.html) -* common: Common libary of Aliyun Go SDK -* util: Utility helpers - - +* cdn: [Content Delivery Network](https://help.aliyun.com/document_detail/27101.html) +* cms: [Cloud Monitor Service](https://help.aliyun.com/document_detail/28615.html) +* cs: [Container Service](https://help.aliyun.com/product/25972.html) +* dm: [Direct Mail](https://help.aliyun.com/document_detail/29414.html) +* dns: [DNS](https://help.aliyun.com/document_detail/dns/api-reference/summary.html) +* ecs: [Elastic Compute Service](https://help.aliyun.com/document_detail/ecs/open-api/summary.html) +* ess: [Auto Scaling](https://help.aliyun.com/document_detail/25857.html) +* mns: [Message Service](https://help.aliyun.com/document_detail/27414.html) +* mq: [Message Queue](https://help.aliyun.com/document_detail/29532.html) +* nas: [Network Attached Storage](https://help.aliyun.com/document_detail/27518.html) +* opensearch: [OpenSearch](https://help.aliyun.com/document_detail/29118.html) +* oss: [Open Storage Service](https://help.aliyun.com/document_detail/oss/api-reference/abstract.html) +* push: [Cloud Mobile Push](https://help.aliyun.com/document_detail/30049.html) +* rds: [Relational Database Service](https://help.aliyun.com/document_detail/26226.html) +* ram: [Resource Access Management](https://help.aliyun.com/document_detail/ram/ram-api-reference/intro/intro.html) +* slb: [Server Load Balancer](https://help.aliyun.com/document_detail/slb/api-reference/brief-introduction.html) +* sls: [Logging Service](https://help.aliyun.com/document_detail/sls/api/overview.html) +* sms: [Short Message Service](https://help.aliyun.com/product/44282.html) +* sts: [Security Token Service](https://help.aliyun.com/document_detail/28756.html) +* common: Common libary of Aliyun Go SDK +* util: Utility helpers ## Quick Start @@ -27,7 +36,7 @@ This is an unofficial Go SDK for Aliyun Services. You are welcome for contributi package main import ( - "fmt" + "fmt" "github.com/denverdino/aliyungo/ecs" ) @@ -44,17 +53,26 @@ func main() { ## Documentation - * ECS: [https://godoc.org/github.com/denverdino/aliyungo/ecs](https://godoc.org/github.com/denverdino/aliyungo/ecs) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/ecs?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/ecs) - * OSS: [https://godoc.org/github.com/denverdino/aliyungo/oss](https://godoc.org/github.com/denverdino/aliyungo/oss) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/oss?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/oss) - * SLB: [https://godoc.org/github.com/denverdino/aliyungo/slb](https://godoc.org/github.com/denverdino/aliyungo/slb) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/slb?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/slb) - * DNS: [https://godoc.org/github.com/denverdino/aliyungo/dns](https://godoc.org/github.com/denverdino/aliyungo/dns) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/dns?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/dns) - * SLS: [https://godoc.org/github.com/denverdino/aliyungo/sls](https://godoc.org/github.com/denverdino/aliyungo/sls) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/sls?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/sls) - * RAM: [https://godoc.org/github.com/denverdino/aliyungo/ram](https://godoc.org/github.com/denverdino/aliyungo/ram) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/ram?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/ram) - * RDS: [https://godoc.org/github.com/denverdino/aliyungo/rds](https://godoc.org/github.com/denverdino/aliyungo/rds) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/rds?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/rds) - * CMS: [https://godoc.org/github.com/denverdino/aliyungo/cms](https://godoc.org/github.com/denverdino/aliyungo/cms) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/cms?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/cms) - * STS: [https://godoc.org/github.com/denverdino/aliyungo/sts](https://godoc.org/github.com/denverdino/aliyungo/sts) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/sts?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/sts) - * DM: [https://godoc.org/github.com/denverdino/aliyungo/dm](https://godoc.org/github.com/denverdino/aliyungo/dm) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/dm?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/dm) - + * CDN: [https://godoc.org/github.com/denverdino/aliyungo/cdn](https://godoc.org/github.com/denverdino/aliyungo/cdn)[![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/cdn?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/cdn) + * CMS: [https://godoc.org/github.com/denverdino/aliyungo/cms](https://godoc.org/github.com/denverdino/aliyungo/cms) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/cms?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/cms) + * CS: [https://godoc.org/github.com/denverdino/aliyungo/cs](https://godoc.org/github.com/denverdino/aliyungo/cs) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/cs?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/cs) + * DM: [https://godoc.org/github.com/denverdino/aliyungo/dm](https://godoc.org/github.com/denverdino/aliyungo/dm) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/dm?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/dm) + * DNS: [https://godoc.org/github.com/denverdino/aliyungo/dns](https://godoc.org/github.com/denverdino/aliyungo/dns) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/dns?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/dns) + * ECS: [https://godoc.org/github.com/denverdino/aliyungo/ecs](https://godoc.org/github.com/denverdino/aliyungo/ecs) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/ecs?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/ecs) + * ESS: [https://godoc.org/github.com/denverdino/aliyungo/ess](https://godoc.org/github.com/denverdino/aliyungo/ess)[![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/ess?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/ess) + * MNS: [https://godoc.org/github.com/denverdino/aliyungo/mns](https://godoc.org/github.com/denverdino/aliyungo/mns)[![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/mns?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/mns) + * MQ: [https://godoc.org/github.com/denverdino/aliyungo/mq](https://godoc.org/github.com/denverdino/aliyungo/mq) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/mq?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/mq) + * NAS: [https://godoc.org/github.com/denverdino/aliyungo/nas](https://godoc.org/github.com/denverdino/aliyungo/nas) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/nas?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/nas) + * OPENSEARCH: [https://godoc.org/github.com/denverdino/aliyungo/opensearch](https://godoc.org/github.com/denverdino/aliyungo/opensearch) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/opensearch?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/opensearch) + * OSS: [https://godoc.org/github.com/denverdino/aliyungo/oss](https://godoc.org/github.com/denverdino/aliyungo/oss) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/oss?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/oss) + * PUSH: [https://godoc.org/github.com/denverdino/aliyungo/push](https://godoc.org/github.com/denverdino/aliyungo/push) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/push?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/push) + * RAM: [https://godoc.org/github.com/denverdino/aliyungo/ram](https://godoc.org/github.com/denverdino/aliyungo/ram) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/ram?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/ram) + * RDS: [https://godoc.org/github.com/denverdino/aliyungo/rds](https://godoc.org/github.com/denverdino/aliyungo/rds) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/rds?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/rds) + * SLB: [https://godoc.org/github.com/denverdino/aliyungo/slb](https://godoc.org/github.com/denverdino/aliyungo/slb) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/slb?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/slb) + * SLS: [https://godoc.org/github.com/denverdino/aliyungo/sls](https://godoc.org/github.com/denverdino/aliyungo/sls) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/sls?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/sls) + * SMS: [https://godoc.org/github.com/denverdino/aliyungo/sms](https://godoc.org/github.com/denverdino/aliyungo/sms) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/sms?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/sms) + * STS: [https://godoc.org/github.com/denverdino/aliyungo/sts](https://godoc.org/github.com/denverdino/aliyungo/sts) [![GoDoc](https://godoc.org/github.com/denverdino/aliyungo/sts?status.svg)](https://godoc.org/github.com/denverdino/aliyungo/sts) + ## Build and Install go get: @@ -63,10 +81,9 @@ go get: go get github.com/denverdino/aliyungo ``` - ## Test ECS -Modify "ecs/config_test.go" +Modify "ecs/config_test.go" ```sh TestAccessKeyId = "MY_ACCESS_KEY_ID" @@ -75,10 +92,10 @@ Modify "ecs/config_test.go" TestIAmRich = false ``` -* TestAccessKeyId: the Access Key Id -* TestAccessKeySecret: the Access Key Secret. -* TestInstanceId: the existing instance id for testing. It will be stopped and restarted during testing. -* TestIAmRich(Optional): If it is set to true, it will perform tests to create virtual machines and disks under your account. And you will pay the bill. :-) +* TestAccessKeyId: the Access Key Id +* TestAccessKeySecret: the Access Key Secret. +* TestInstanceId: the existing instance id for testing. It will be stopped and restarted during testing. +* TestIAmRich(Optional): If it is set to true, it will perform tests to create virtual machines and disks under your account. And you will pay the bill. :-) Under "ecs" and run @@ -88,7 +105,7 @@ go test ## Test OSS -Modify "oss/config_test.go" +Modify "oss/config_test.go" ```sh TestAccessKeyId = "MY_ACCESS_KEY_ID" @@ -97,11 +114,10 @@ Modify "oss/config_test.go" TestBucket = "denverdino" ``` -* TestAccessKeyId: the Access Key Id -* TestAccessKeySecret: the Access Key Secret. -* TestRegion: the region of OSS for testing -* TestBucket: the bucket name for testing - +* TestAccessKeyId: the Access Key Id +* TestAccessKeySecret: the Access Key Secret. +* TestRegion: the region of OSS for testing +* TestBucket: the bucket name for testing Under "oss" and run @@ -112,24 +128,47 @@ go test ## Contributors * Li Yi (denverdino@gmail.com) - * tgic (farmer1992@gmail.com) + * Boshi Lian (farmer1992@gmail.com) * Yu Zhou (oscarrr110@gmail.com) * Yufei Zhang * linuxlikerqq - * Changhai Yan (changhai.ych@alibaba-inc.com) + * Changhai Yan * Jizhong Jiang (jiangjizhong@gmail.com) * Kent Wang (pragkent@gmail.com) - * ringtail (zhongwei.lzw@alibaba-inc.com) + * ringtail * aiden0z (aiden0xz@gmail.com) * jimmycmh * menglingwei * mingang.he (dustgle@gmail.com) - * chainone (chainone@gmail.com) + * Young Chen (chainone@gmail.com) * johnzeng + * spacexnice (445436286@qq.com) + * xiaoheihero + * hmgle (dustgle@gmail.com) + * jzwlqx (jiangjizhong@gmail.com) + * Linhua Tan (toolchainX@gmail.com) + * Plutonist (p@vecsight.com) + * Bin Liu + * wangyue + * demonwy + * yarous224 + * yufeizyf (xazyf9111@sina.cn) + * keontang (ikeontang@gmail.com) + * Cholerae Hu (me@cholerae.com) + * Zach Bergh (berghzach@gmail.com) + * Bingshen Wang + * xiaozhu36 + * Russell (yufeiwu@gmail.com) + * zhuzhih2017 + * cheyang + * Hobo Chen + * Shuwei Yin + * Xujin Zheng (xujinzheng@gmail.com) + * Dino Lai (dinos80152@gmail.com) ## License -This project is licensed under the Apache License, Version 2.0. See [LICENSE](https://github.com/denverdino/aliyungo/blob/master/LICENSE.txt) for the full license text. +This project is licensed under the Apache License, Version 2.0. See [LICENSE](https://github.com/denverdino/aliyungo/blob/master/LICENSE.txt) for the full license text. ## Related projects @@ -137,7 +176,6 @@ This project is licensed under the Apache License, Version 2.0. See [LICENSE](ht * Aliyun OSS driver for Docker Registry V2: [Pull request](https://github.com/docker/distribution/pull/514) - ## References The GO API design of OSS refer the implementation from [https://github.com/AdRoll/goamz](https://github.com/AdRoll) diff --git a/vendor/github.com/denverdino/aliyungo/common/client.go b/vendor/github.com/denverdino/aliyungo/common/client.go index 4ed0a06f..6c00d1c1 100644 --- a/vendor/github.com/denverdino/aliyungo/common/client.go +++ b/vendor/github.com/denverdino/aliyungo/common/client.go @@ -3,35 +3,206 @@ package common import ( "bytes" "encoding/json" + "errors" + "fmt" "io/ioutil" "log" "net/http" "net/url" + "os" + "strconv" + "strings" "time" "github.com/denverdino/aliyungo/util" ) +// RemovalPolicy.N add index to array item +// RemovalPolicy=["a", "b"] => RemovalPolicy.1="a" RemovalPolicy.2="b" +type FlattenArray []string + +// string contains underline which will be replaced with dot +// SystemDisk_Category => SystemDisk.Category +type UnderlineString string + // A Client represents a client of ECS services type Client struct { AccessKeyId string //Access Key Id AccessKeySecret string //Access Key Secret + securityToken string debug bool httpClient *http.Client endpoint string version string + serviceCode string + regionID Region + businessInfo string + userAgent string } -// NewClient creates a new instance of ECS client +// Initialize properties of a client instance func (client *Client) Init(endpoint, version, accessKeyId, accessKeySecret string) { client.AccessKeyId = accessKeyId - client.AccessKeySecret = accessKeySecret + "&" + ak := accessKeySecret + if !strings.HasSuffix(ak, "&") { + ak += "&" + } + client.AccessKeySecret = ak client.debug = false - client.httpClient = &http.Client{} + handshakeTimeout, err := strconv.Atoi(os.Getenv("TLSHandshakeTimeout")) + if err != nil { + handshakeTimeout = 0 + } + if handshakeTimeout == 0 { + client.httpClient = &http.Client{} + } else { + t := &http.Transport{ + TLSHandshakeTimeout: time.Duration(handshakeTimeout) * time.Second} + client.httpClient = &http.Client{Transport: t} + } client.endpoint = endpoint client.version = version } +// Initialize properties of a client instance including regionID +func (client *Client) NewInit(endpoint, version, accessKeyId, accessKeySecret, serviceCode string, regionID Region) { + client.Init(endpoint, version, accessKeyId, accessKeySecret) + client.serviceCode = serviceCode + client.regionID = regionID +} + +// Intialize client object when all properties are ready +func (client *Client) InitClient() *Client { + client.debug = false + handshakeTimeout, err := strconv.Atoi(os.Getenv("TLSHandshakeTimeout")) + if err != nil { + handshakeTimeout = 0 + } + if handshakeTimeout == 0 { + client.httpClient = &http.Client{} + } else { + t := &http.Transport{ + TLSHandshakeTimeout: time.Duration(handshakeTimeout) * time.Second} + client.httpClient = &http.Client{Transport: t} + } + return client +} + +func (client *Client) NewInitForAssumeRole(endpoint, version, accessKeyId, accessKeySecret, serviceCode string, regionID Region, securityToken string) { + client.NewInit(endpoint, version, accessKeyId, accessKeySecret, serviceCode, regionID) + client.securityToken = securityToken +} + +//getLocationEndpoint +func (client *Client) getEndpointByLocation() string { + locationClient := NewLocationClient(client.AccessKeyId, client.AccessKeySecret, client.securityToken) + locationClient.SetDebug(true) + return locationClient.DescribeOpenAPIEndpoint(client.regionID, client.serviceCode) +} + +//NewClient using location service +func (client *Client) setEndpointByLocation(region Region, serviceCode, accessKeyId, accessKeySecret, securityToken string) { + locationClient := NewLocationClient(accessKeyId, accessKeySecret, securityToken) + locationClient.SetDebug(true) + ep := locationClient.DescribeOpenAPIEndpoint(region, serviceCode) + if ep == "" { + ep = loadEndpointFromFile(region, serviceCode) + } + + if ep != "" { + client.endpoint = ep + } +} + +// Ensure all necessary properties are valid +func (client *Client) ensureProperties() error { + var msg string + + if client.endpoint == "" { + msg = fmt.Sprintf("endpoint cannot be empty!") + } else if client.version == "" { + msg = fmt.Sprintf("version cannot be empty!") + } else if client.AccessKeyId == "" { + msg = fmt.Sprintf("AccessKeyId cannot be empty!") + } else if client.AccessKeySecret == "" { + msg = fmt.Sprintf("AccessKeySecret cannot be empty!") + } + + if msg != "" { + return errors.New(msg) + } + + return nil +} + +// ---------------------------------------------------- +// WithXXX methods +// ---------------------------------------------------- + +// WithEndpoint sets custom endpoint +func (client *Client) WithEndpoint(endpoint string) *Client { + client.SetEndpoint(endpoint) + return client +} + +// WithVersion sets custom version +func (client *Client) WithVersion(version string) *Client { + client.SetVersion(version) + return client +} + +// WithRegionID sets Region ID +func (client *Client) WithRegionID(regionID Region) *Client { + client.SetRegionID(regionID) + return client +} + +//WithServiceCode sets serviceCode +func (client *Client) WithServiceCode(serviceCode string) *Client { + client.SetServiceCode(serviceCode) + return client +} + +// WithAccessKeyId sets new AccessKeyId +func (client *Client) WithAccessKeyId(id string) *Client { + client.SetAccessKeyId(id) + return client +} + +// WithAccessKeySecret sets new AccessKeySecret +func (client *Client) WithAccessKeySecret(secret string) *Client { + client.SetAccessKeySecret(secret) + return client +} + +// WithSecurityToken sets securityToken +func (client *Client) WithSecurityToken(securityToken string) *Client { + client.SetSecurityToken(securityToken) + return client +} + +// WithDebug sets debug mode to log the request/response message +func (client *Client) WithDebug(debug bool) *Client { + client.SetDebug(debug) + return client +} + +// WithBusinessInfo sets business info to log the request/response message +func (client *Client) WithBusinessInfo(businessInfo string) *Client { + client.SetBusinessInfo(businessInfo) + return client +} + +// WithUserAgent sets user agent to the request/response message +func (client *Client) WithUserAgent(userAgent string) *Client { + client.SetUserAgent(userAgent) + return client +} + +// ---------------------------------------------------- +// SetXXX methods +// ---------------------------------------------------- + // SetEndpoint sets custom endpoint func (client *Client) SetEndpoint(endpoint string) { client.endpoint = endpoint @@ -42,6 +213,16 @@ func (client *Client) SetVersion(version string) { client.version = version } +// SetEndpoint sets Region ID +func (client *Client) SetRegionID(regionID Region) { + client.regionID = regionID +} + +//SetServiceCode sets serviceCode +func (client *Client) SetServiceCode(serviceCode string) { + client.serviceCode = serviceCode +} + // SetAccessKeyId sets new AccessKeyId func (client *Client) SetAccessKeyId(id string) { client.AccessKeyId = id @@ -57,11 +238,55 @@ func (client *Client) SetDebug(debug bool) { client.debug = debug } +// SetBusinessInfo sets business info to log the request/response message +func (client *Client) SetBusinessInfo(businessInfo string) { + if strings.HasPrefix(businessInfo, "/") { + client.businessInfo = businessInfo + } else if businessInfo != "" { + client.businessInfo = "/" + businessInfo + } +} + +// SetUserAgent sets user agent to the request/response message +func (client *Client) SetUserAgent(userAgent string) { + client.userAgent = userAgent +} + +//set SecurityToken +func (client *Client) SetSecurityToken(securityToken string) { + client.securityToken = securityToken +} + +func (client *Client) initEndpoint() error { + // if set any value to "CUSTOMIZED_ENDPOINT" could skip location service. + // example: export CUSTOMIZED_ENDPOINT=true + if os.Getenv("CUSTOMIZED_ENDPOINT") != "" { + return nil + } + + if client.serviceCode != "" && client.regionID != "" { + endpoint := client.getEndpointByLocation() + if endpoint == "" { + return GetCustomError("InvalidEndpoint", "endpoint is empty,pls check") + } + client.endpoint = endpoint + } + return nil +} + // Invoke sends the raw HTTP request for ECS services func (client *Client) Invoke(action string, args interface{}, response interface{}) error { + if err := client.ensureProperties(); err != nil { + return err + } + + //init endpoint + if err := client.initEndpoint(); err != nil { + return err + } request := Request{} - request.init(client.version, action, client.AccessKeyId) + request.init(client.version, action, client.AccessKeyId, client.securityToken, client.regionID) query := util.ConvertToQueryValues(request) util.SetQueryValues(args, &query) @@ -74,13 +299,15 @@ func (client *Client) Invoke(action string, args interface{}, response interface httpReq, err := http.NewRequest(ECSRequestMethod, requestURL, nil) - // TODO move to util and add build val flag - httpReq.Header.Set("X-SDK-Client", `AliyunGO/`+Version) - if err != nil { return GetClientError(err) } + // TODO move to util and add build val flag + httpReq.Header.Set("X-SDK-Client", `AliyunGO/`+Version+client.businessInfo) + + httpReq.Header.Set("User-Agent", httpReq.UserAgent()+" "+client.userAgent) + t0 := time.Now() httpResp, err := client.httpClient.Do(httpReq) t1 := time.Now() @@ -125,6 +352,174 @@ func (client *Client) Invoke(action string, args interface{}, response interface return nil } +// Invoke sends the raw HTTP request for ECS services +func (client *Client) InvokeByFlattenMethod(action string, args interface{}, response interface{}) error { + if err := client.ensureProperties(); err != nil { + return err + } + + //init endpoint + if err := client.initEndpoint(); err != nil { + return err + } + + request := Request{} + request.init(client.version, action, client.AccessKeyId, client.securityToken, client.regionID) + + query := util.ConvertToQueryValues(request) + + util.SetQueryValueByFlattenMethod(args, &query) + + // Sign request + signature := util.CreateSignatureForRequest(ECSRequestMethod, &query, client.AccessKeySecret) + + // Generate the request URL + requestURL := client.endpoint + "?" + query.Encode() + "&Signature=" + url.QueryEscape(signature) + + httpReq, err := http.NewRequest(ECSRequestMethod, requestURL, nil) + + if err != nil { + return GetClientError(err) + } + + // TODO move to util and add build val flag + httpReq.Header.Set("X-SDK-Client", `AliyunGO/`+Version+client.businessInfo) + + httpReq.Header.Set("User-Agent", httpReq.UserAgent()+" "+client.userAgent) + + t0 := time.Now() + httpResp, err := client.httpClient.Do(httpReq) + t1 := time.Now() + if err != nil { + return GetClientError(err) + } + statusCode := httpResp.StatusCode + + if client.debug { + log.Printf("Invoke %s %s %d (%v)", ECSRequestMethod, requestURL, statusCode, t1.Sub(t0)) + } + + defer httpResp.Body.Close() + body, err := ioutil.ReadAll(httpResp.Body) + + if err != nil { + return GetClientError(err) + } + + if client.debug { + var prettyJSON bytes.Buffer + err = json.Indent(&prettyJSON, body, "", " ") + log.Println(string(prettyJSON.Bytes())) + } + + if statusCode >= 400 && statusCode <= 599 { + errorResponse := ErrorResponse{} + err = json.Unmarshal(body, &errorResponse) + ecsError := &Error{ + ErrorResponse: errorResponse, + StatusCode: statusCode, + } + return ecsError + } + + err = json.Unmarshal(body, response) + //log.Printf("%++v", response) + if err != nil { + return GetClientError(err) + } + + return nil +} + +// Invoke sends the raw HTTP request for ECS services +//改进了一下上面那个方法，可以使用各种Http方法 +//2017.1.30 增加了一个path参数，用来拓展访问的地址 +func (client *Client) InvokeByAnyMethod(method, action, path string, args interface{}, response interface{}) error { + if err := client.ensureProperties(); err != nil { + return err + } + + //init endpoint + if err := client.initEndpoint(); err != nil { + return err + } + + request := Request{} + request.init(client.version, action, client.AccessKeyId, client.securityToken, client.regionID) + data := util.ConvertToQueryValues(request) + util.SetQueryValues(args, &data) + + // Sign request + signature := util.CreateSignatureForRequest(method, &data, client.AccessKeySecret) + + data.Add("Signature", signature) + // Generate the request URL + var ( + httpReq *http.Request + err error + ) + if method == http.MethodGet { + requestURL := client.endpoint + path + "?" + data.Encode() + //fmt.Println(requestURL) + httpReq, err = http.NewRequest(method, requestURL, nil) + } else { + //fmt.Println(client.endpoint + path) + httpReq, err = http.NewRequest(method, client.endpoint+path, strings.NewReader(data.Encode())) + httpReq.Header.Set("Content-Type", "application/x-www-form-urlencoded") + } + + if err != nil { + return GetClientError(err) + } + + // TODO move to util and add build val flag + httpReq.Header.Set("X-SDK-Client", `AliyunGO/`+Version+client.businessInfo) + httpReq.Header.Set("User-Agent", httpReq.Header.Get("User-Agent")+" "+client.userAgent) + + t0 := time.Now() + httpResp, err := client.httpClient.Do(httpReq) + t1 := time.Now() + if err != nil { + return GetClientError(err) + } + statusCode := httpResp.StatusCode + + if client.debug { + log.Printf("Invoke %s %s %d (%v) %v", ECSRequestMethod, client.endpoint, statusCode, t1.Sub(t0), data.Encode()) + } + + defer httpResp.Body.Close() + body, err := ioutil.ReadAll(httpResp.Body) + + if err != nil { + return GetClientError(err) + } + + if client.debug { + var prettyJSON bytes.Buffer + err = json.Indent(&prettyJSON, body, "", " ") + log.Println(string(prettyJSON.Bytes())) + } + + if statusCode >= 400 && statusCode <= 599 { + errorResponse := ErrorResponse{} + err = json.Unmarshal(body, &errorResponse) + ecsError := &Error{ + ErrorResponse: errorResponse, + StatusCode: statusCode, + } + return ecsError + } + + err = json.Unmarshal(body, response) + //log.Printf("%++v", response) + if err != nil { + return GetClientError(err) + } + + return nil +} + // GenerateClientToken generates the Client Token with random string func (client *Client) GenerateClientToken() string { return util.CreateRandomString() @@ -143,3 +538,13 @@ func GetClientErrorFromString(str string) error { func GetClientError(err error) error { return GetClientErrorFromString(err.Error()) } + +func GetCustomError(code, message string) error { + return &Error{ + ErrorResponse: ErrorResponse{ + Code: code, + Message: message, + }, + StatusCode: 400, + } +} diff --git a/vendor/github.com/denverdino/aliyungo/common/endpoint.go b/vendor/github.com/denverdino/aliyungo/common/endpoint.go new file mode 100644 index 00000000..757f7a78 --- /dev/null +++ b/vendor/github.com/denverdino/aliyungo/common/endpoint.go @@ -0,0 +1,208 @@ +package common + +import ( + "encoding/xml" + "fmt" + "io/ioutil" + "log" + "os" + "strings" + "time" +) + +const ( + // LocationDefaultEndpoint is the default API endpoint of Location services + locationDefaultEndpoint = "https://location.aliyuncs.com" + locationAPIVersion = "2015-06-12" + HTTP_PROTOCOL = "http" + HTTPS_PROTOCOL = "https" +) + +var ( + endpoints = make(map[Region]map[string]string) + + SpecailEnpoints = map[Region]map[string]string{ + APNorthEast1: { + "ecs": "https://ecs.ap-northeast-1.aliyuncs.com", + "slb": "https://slb.ap-northeast-1.aliyuncs.com", + "rds": "https://rds.ap-northeast-1.aliyuncs.com", + "vpc": "https://vpc.ap-northeast-1.aliyuncs.com", + }, + APSouthEast2: { + "ecs": "https://ecs.ap-southeast-2.aliyuncs.com", + "slb": "https://slb.ap-southeast-2.aliyuncs.com", + "rds": "https://rds.ap-southeast-2.aliyuncs.com", + "vpc": "https://vpc.ap-southeast-2.aliyuncs.com", + }, + APSouthEast3: { + "ecs": "https://ecs.ap-southeast-3.aliyuncs.com", + "slb": "https://slb.ap-southeast-3.aliyuncs.com", + "rds": "https://rds.ap-southeast-3.aliyuncs.com", + "vpc": "https://vpc.ap-southeast-3.aliyuncs.com", + }, + MEEast1: { + "ecs": "https://ecs.me-east-1.aliyuncs.com", + "slb": "https://slb.me-east-1.aliyuncs.com", + "rds": "https://rds.me-east-1.aliyuncs.com", + "vpc": "https://vpc.me-east-1.aliyuncs.com", + }, + EUCentral1: { + "ecs": "https://ecs.eu-central-1.aliyuncs.com", + "slb": "https://slb.eu-central-1.aliyuncs.com", + "rds": "https://rds.eu-central-1.aliyuncs.com", + "vpc": "https://vpc.eu-central-1.aliyuncs.com", + }, + EUWest1: { + "ecs": "https://ecs.eu-west-1.aliyuncs.com", + "slb": "https://slb.eu-west-1.aliyuncs.com", + "rds": "https://rds.eu-west-1.aliyuncs.com", + "vpc": "https://vpc.eu-west-1.aliyuncs.com", + }, + Zhangjiakou: { + "ecs": "https://ecs.cn-zhangjiakou.aliyuncs.com", + "slb": "https://slb.cn-zhangjiakou.aliyuncs.com", + "rds": "https://rds.cn-zhangjiakou.aliyuncs.com", + "vpc": "https://vpc.cn-zhangjiakou.aliyuncs.com", + }, + Huhehaote: { + "ecs": "https://ecs.cn-huhehaote.aliyuncs.com", + "slb": "https://slb.cn-huhehaote.aliyuncs.com", + "rds": "https://rds.cn-huhehaote.aliyuncs.com", + "vpc": "https://vpc.cn-huhehaote.aliyuncs.com", + }, + } +) + +//init endpoints from file +func init() { + +} + +type LocationClient struct { + Client +} + +func NewLocationClient(accessKeyId, accessKeySecret, securityToken string) *LocationClient { + endpoint := os.Getenv("LOCATION_ENDPOINT") + if endpoint == "" { + endpoint = locationDefaultEndpoint + } + + client := &LocationClient{} + client.Init(endpoint, locationAPIVersion, accessKeyId, accessKeySecret) + client.securityToken = securityToken + return client +} + +func NewLocationClientWithSecurityToken(accessKeyId, accessKeySecret, securityToken string) *LocationClient { + endpoint := os.Getenv("LOCATION_ENDPOINT") + if endpoint == "" { + endpoint = locationDefaultEndpoint + } + + client := &LocationClient{} + client.WithEndpoint(endpoint). + WithVersion(locationAPIVersion). + WithAccessKeyId(accessKeyId). + WithAccessKeySecret(accessKeySecret). + WithSecurityToken(securityToken). + InitClient() + return client +} + +func (client *LocationClient) DescribeEndpoint(args *DescribeEndpointArgs) (*DescribeEndpointResponse, error) { + response := &DescribeEndpointResponse{} + err := client.Invoke("DescribeEndpoint", args, response) + if err != nil { + return nil, err + } + return response, err +} + +func (client *LocationClient) DescribeEndpoints(args *DescribeEndpointsArgs) (*DescribeEndpointsResponse, error) { + response := &DescribeEndpointsResponse{} + err := client.Invoke("DescribeEndpoints", args, response) + if err != nil { + return nil, err + } + return response, err +} + +func getProductRegionEndpoint(region Region, serviceCode string) string { + if sp, ok := endpoints[region]; ok { + if endpoint, ok := sp[serviceCode]; ok { + return endpoint + } + } + + return "" +} + +func setProductRegionEndpoint(region Region, serviceCode string, endpoint string) { + endpoints[region] = map[string]string{ + serviceCode: endpoint, + } +} + +func (client *LocationClient) DescribeOpenAPIEndpoint(region Region, serviceCode string) string { + if endpoint := getProductRegionEndpoint(region, serviceCode); endpoint != "" { + return endpoint + } + defaultProtocols := HTTP_PROTOCOL + + args := &DescribeEndpointsArgs{ + Id: region, + ServiceCode: serviceCode, + Type: "openAPI", + } + + var endpoint *DescribeEndpointsResponse + var err error + for index := 0; index < 5; index++ { + endpoint, err = client.DescribeEndpoints(args) + if err == nil && endpoint != nil && len(endpoint.Endpoints.Endpoint) > 0 { + break + } + time.Sleep(500 * time.Millisecond) + } + + if err != nil || endpoint == nil || len(endpoint.Endpoints.Endpoint) <= 0 { + log.Printf("aliyungo: can not get endpoint from service, use default. endpoint=[%v], error=[%v]\n", endpoint, err) + return "" + } + + for _, protocol := range endpoint.Endpoints.Endpoint[0].Protocols.Protocols { + if strings.ToLower(protocol) == HTTPS_PROTOCOL { + defaultProtocols = HTTPS_PROTOCOL + break + } + } + + ep := fmt.Sprintf("%s://%s", defaultProtocols, endpoint.Endpoints.Endpoint[0].Endpoint) + + setProductRegionEndpoint(region, serviceCode, ep) + return ep +} + +func loadEndpointFromFile(region Region, serviceCode string) string { + data, err := ioutil.ReadFile("./endpoints.xml") + if err != nil { + return "" + } + var endpoints Endpoints + err = xml.Unmarshal(data, &endpoints) + if err != nil { + return "" + } + for _, endpoint := range endpoints.Endpoint { + if endpoint.RegionIds.RegionId == string(region) { + for _, product := range endpoint.Products.Product { + if strings.ToLower(product.ProductName) == serviceCode { + return fmt.Sprintf("%s://%s", HTTPS_PROTOCOL, product.DomainName) + } + } + } + } + + return "" +} diff --git a/vendor/github.com/denverdino/aliyungo/common/regions.go b/vendor/github.com/denverdino/aliyungo/common/regions.go index 9ceda465..e6bc728f 100644 --- a/vendor/github.com/denverdino/aliyungo/common/regions.go +++ b/vendor/github.com/denverdino/aliyungo/common/regions.go @@ -5,15 +5,51 @@ type Region string // Constants of region definition const ( - Hangzhou = Region("cn-hangzhou") - Qingdao = Region("cn-qingdao") - Beijing = Region("cn-beijing") - Hongkong = Region("cn-hongkong") - Shenzhen = Region("cn-shenzhen") - USWest1 = Region("us-west-1") - USEast1 = Region("us-east-1") + Hangzhou = Region("cn-hangzhou") + Qingdao = Region("cn-qingdao") + Beijing = Region("cn-beijing") + Hongkong = Region("cn-hongkong") + Shenzhen = Region("cn-shenzhen") + Shanghai = Region("cn-shanghai") + Zhangjiakou = Region("cn-zhangjiakou") + Huhehaote = Region("cn-huhehaote") + APSouthEast1 = Region("ap-southeast-1") - Shanghai = Region("cn-shanghai") + APNorthEast1 = Region("ap-northeast-1") + APSouthEast2 = Region("ap-southeast-2") + APSouthEast3 = Region("ap-southeast-3") + APSouthEast5 = Region("ap-southeast-5") + + APSouth1 = Region("ap-south-1") + + USWest1 = Region("us-west-1") + USEast1 = Region("us-east-1") + + MEEast1 = Region("me-east-1") + + EUCentral1 = Region("eu-central-1") + EUWest1 = Region("eu-west-1") + + ShenZhenFinance = Region("cn-shenzhen-finance-1") + ShanghaiFinance = Region("cn-shanghai-finance-1") ) -var ValidRegions = []Region{Hangzhou, Qingdao, Beijing, Shenzhen, Hongkong, Shanghai, USWest1, USEast1, APSouthEast1} +var ValidRegions = []Region{ + Hangzhou, Qingdao, Beijing, Shenzhen, Hongkong, Shanghai, Zhangjiakou, Huhehaote, + USWest1, USEast1, + APNorthEast1, APSouthEast1, APSouthEast2, APSouthEast3, APSouthEast5, + APSouth1, + MEEast1, + EUCentral1, EUWest1, + ShenZhenFinance, ShanghaiFinance, +} + +// IsValidRegion checks if r is an Ali supported region. +func IsValidRegion(r string) bool { + for _, v := range ValidRegions { + if r == string(v) { + return true + } + } + return false +} diff --git a/vendor/github.com/denverdino/aliyungo/common/request.go b/vendor/github.com/denverdino/aliyungo/common/request.go index 2a883f19..f35c2990 100644 --- a/vendor/github.com/denverdino/aliyungo/common/request.go +++ b/vendor/github.com/denverdino/aliyungo/common/request.go @@ -20,7 +20,9 @@ const ( type Request struct { Format string Version string + RegionId Region AccessKeyId string + SecurityToken string Signature string SignatureMethod string Timestamp util.ISO6801Time @@ -30,7 +32,7 @@ type Request struct { Action string } -func (request *Request) init(version string, action string, AccessKeyId string) { +func (request *Request) init(version string, action string, AccessKeyId string, securityToken string, regionId Region) { request.Format = JSONResponseFormat request.Timestamp = util.NewISO6801Time(time.Now().UTC()) request.Version = version @@ -39,6 +41,8 @@ func (request *Request) init(version string, action string, AccessKeyId string) request.SignatureNonce = util.CreateRandomString() request.Action = action request.AccessKeyId = AccessKeyId + request.SecurityToken = securityToken + request.RegionId = regionId } type Response struct { diff --git a/vendor/github.com/denverdino/aliyungo/common/types.go b/vendor/github.com/denverdino/aliyungo/common/types.go index c562aedf..cf161f11 100644 --- a/vendor/github.com/denverdino/aliyungo/common/types.go +++ b/vendor/github.com/denverdino/aliyungo/common/types.go @@ -13,3 +13,95 @@ const ( PrePaid = InstanceChargeType("PrePaid") PostPaid = InstanceChargeType("PostPaid") ) + +type DescribeEndpointArgs struct { + Id Region + ServiceCode string + Type string +} + +type EndpointItem struct { + Protocols struct { + Protocols []string + } + Type string + Namespace string + Id Region + SerivceCode string + Endpoint string +} + +type DescribeEndpointResponse struct { + Response + EndpointItem +} + +type DescribeEndpointsArgs struct { + Id Region + ServiceCode string + Type string +} + +type DescribeEndpointsResponse struct { + Response + Endpoints APIEndpoints + RequestId string + Success bool +} + +type APIEndpoints struct { + Endpoint []EndpointItem +} + +type NetType string + +const ( + Internet = NetType("Internet") + Intranet = NetType("Intranet") +) + +type TimeType string + +const ( + Hour = TimeType("Hour") + Day = TimeType("Day") + Week = TimeType("Week") + Month = TimeType("Month") + Year = TimeType("Year") +) + +type NetworkType string + +const ( + Classic = NetworkType("Classic") + VPC = NetworkType("VPC") +) + +type BusinessInfo struct { + Pack string `json:"pack,omitempty"` + ActivityId string `json:"activityId,omitempty"` +} + +//xml +type Endpoints struct { + Endpoint []Endpoint `xml:"Endpoint"` +} + +type Endpoint struct { + Name string `xml:"name,attr"` + RegionIds RegionIds `xml:"RegionIds"` + Products Products `xml:"Products"` +} + +type RegionIds struct { + RegionId string `xml:"RegionId"` +} + +type Products struct { + Product []Product `xml:"Product"` +} + +type Product struct { + ProductName string `xml:"ProductName"` + DomainName string `xml:"DomainName"` +} diff --git a/vendor/github.com/denverdino/aliyungo/oss/authenticate_callback.go b/vendor/github.com/denverdino/aliyungo/oss/authenticate_callback.go new file mode 100644 index 00000000..74b58fb0 --- /dev/null +++ b/vendor/github.com/denverdino/aliyungo/oss/authenticate_callback.go @@ -0,0 +1,88 @@ +package oss + +import ( + "crypto" + "crypto/md5" + "crypto/rsa" + "crypto/x509" + "encoding/base64" + "encoding/pem" + "errors" + "io/ioutil" + "net/http" + "regexp" + "strings" + "sync" +) + +type authenticationType struct { + lock *sync.RWMutex + certificate map[string]*rsa.PublicKey +} + +var ( + authentication = authenticationType{lock: &sync.RWMutex{}, certificate: map[string]*rsa.PublicKey{}} + urlReg = regexp.MustCompile(`^http(|s)://gosspublic.alicdn.com/[0-9a-zA-Z]`) +) + +//验证OSS向业务服务器发来的回调函数。 +//该方法是并发安全的 +//pubKeyUrl 回调请求头中[x-oss-pub-key-url]一项，以Base64编码 +//reqUrl oss所发来请求的url，由path+query组成 +//reqBody oss所发来请求的body +//authorization authorization为回调头中的签名 +func AuthenticateCallBack(pubKeyUrl, reqUrl, reqBody, authorization string) error { + //获取证书url + keyURL, err := base64.URLEncoding.DecodeString(pubKeyUrl) + if err != nil { + return err + } + url := string(keyURL) + //判断证书是否来自于阿里云 + if !urlReg.Match(keyURL) { + return errors.New("certificate address error") + } + //获取文件名 + rs := []rune(url) + filename := string(rs[strings.LastIndex(url, "/") : len(rs)-1]) + authentication.lock.RLock() + certificate := authentication.certificate[filename] + authentication.lock.RUnlock() + //内存中没有证书，下载 + if certificate == nil { + authentication.lock.Lock() + res, err := http.Get(url) + if err != nil { + return err + } + defer res.Body.Close() + body, err := ioutil.ReadAll(res.Body) + if err != nil { + return err + } + block, _ := pem.Decode(body) + if block == nil { + return errors.New("certificate error") + } + pubKey, err := x509.ParsePKIXPublicKey(block.Bytes) + if err != nil { + return err + } + certificate = pubKey.(*rsa.PublicKey) + authentication.certificate[filename] = certificate + authentication.lock.Unlock() + } + //证书准备完毕，开始验证 + //解析签名 + signature, err := base64.StdEncoding.DecodeString(authorization) + if err != nil { + return err + } + hashed := md5.New() + hashed.Write([]byte(reqUrl + "\n" + reqBody)) + if err := rsa.VerifyPKCS1v15(certificate, crypto.MD5, hashed.Sum(nil), signature); err != nil { + return err + } + //验证通过 + return nil +} diff --git a/vendor/github.com/denverdino/aliyungo/oss/client.go b/vendor/github.com/denverdino/aliyungo/oss/client.go index c5e13e51..7b5a55b8 100644 --- a/vendor/github.com/denverdino/aliyungo/oss/client.go +++ b/vendor/github.com/denverdino/aliyungo/oss/client.go @@ -57,12 +57,14 @@ type Owner struct { // Options struct // type Options struct { - ServerSideEncryption bool - Meta map[string][]string - ContentEncoding string - CacheControl string - ContentMD5 string - ContentDisposition string + ServerSideEncryption bool + ServerSideEncryptionKeyID string + + Meta map[string][]string + ContentEncoding string + CacheControl string + ContentMD5 string + ContentDisposition string //Range string //Expires int } @@ -72,6 +74,9 @@ type CopyOptions struct { CopySourceOptions string MetadataDirective string //ContentType string + + ServerSideEncryption bool + ServerSideEncryptionKeyID string } // CopyObjectResult is the output from a Copy request @@ -366,7 +371,7 @@ func (b *Bucket) Exists(path string) (exists bool, err error) { } if err != nil { - // We can treat a 403 or 404 as non existance + // We can treat a 403 or 404 as non existence if e, ok := err.(*Error); ok && (e.StatusCode == 403 || e.StatusCode == 404) { return false, nil } @@ -430,7 +435,7 @@ func (b *Bucket) Put(path string, data []byte, contType string, perm ACL, option func (b *Bucket) PutCopy(path string, perm ACL, options CopyOptions, source string) (*CopyObjectResult, error) { headers := make(http.Header) - headers.Set("x-oss-acl", string(perm)) + headers.Set("x-oss-object-acl", string(perm)) headers.Set("x-oss-copy-source", source) options.addHeaders(headers) @@ -455,7 +460,7 @@ func (b *Bucket) PutReader(path string, r io.Reader, length int64, contType stri headers := make(http.Header) headers.Set("Content-Length", strconv.FormatInt(length, 10)) headers.Set("Content-Type", contType) - headers.Set("x-oss-acl", string(perm)) + headers.Set("x-oss-object-acl", string(perm)) options.addHeaders(headers) req := &request{ @@ -491,7 +496,10 @@ func (b *Bucket) PutFile(path string, file *os.File, perm ACL, options Options) // addHeaders adds o's specified fields to headers func (o Options) addHeaders(headers http.Header) { - if o.ServerSideEncryption { + if len(o.ServerSideEncryptionKeyID) != 0 { + headers.Set("x-oss-server-side-encryption", "KMS") + headers.Set("x-oss-server-side-encryption-key-id", o.ServerSideEncryptionKeyID) + } else if o.ServerSideEncryption { headers.Set("x-oss-server-side-encryption", "AES256") } if len(o.ContentEncoding) != 0 { @@ -516,6 +524,13 @@ func (o Options) addHeaders(headers http.Header) { // addHeaders adds o's specified fields to headers func (o CopyOptions) addHeaders(headers http.Header) { + if len(o.ServerSideEncryptionKeyID) != 0 { + headers.Set("x-oss-server-side-encryption", "KMS") + headers.Set("x-oss-server-side-encryption-key-id", o.ServerSideEncryptionKeyID) + } else if o.ServerSideEncryption { + headers.Set("x-oss-server-side-encryption", "AES256") + } + if len(o.MetadataDirective) != 0 { headers.Set("x-oss-metadata-directive", o.MetadataDirective) } @@ -1100,7 +1115,7 @@ func (client *Client) setupHttpRequest(req *request) (*http.Request, error) { // body will be unmarshalled on it. func (client *Client) doHttpRequest(c *http.Client, hreq *http.Request, resp interface{}) (*http.Response, error) { - if true { + if client.debug { log.Printf("%s %s ...\n", hreq.Method, hreq.URL.String()) } hresp, err := c.Do(hreq) @@ -1323,9 +1338,9 @@ func (b *Bucket) CopyLargeFileInParallel(sourcePath string, destPath string, con } currentLength, err := b.GetContentLength(sourcePath) - - log.Printf("Parallel Copy large file[size: %d] from %s to %s\n",currentLength, sourcePath, destPath) - + + log.Printf("Parallel Copy large file[size: %d] from %s to %s\n", currentLength, sourcePath, destPath) + if err != nil { return err } diff --git a/vendor/github.com/denverdino/aliyungo/oss/regions.go b/vendor/github.com/denverdino/aliyungo/oss/regions.go index 0f250604..89bac838 100644 --- a/vendor/github.com/denverdino/aliyungo/oss/regions.go +++ b/vendor/github.com/denverdino/aliyungo/oss/regions.go @@ -9,15 +9,25 @@ type Region string // Constants of region definition const ( - Hangzhou = Region("oss-cn-hangzhou") - Qingdao = Region("oss-cn-qingdao") - Beijing = Region("oss-cn-beijing") - Hongkong = Region("oss-cn-hongkong") - Shenzhen = Region("oss-cn-shenzhen") + Hangzhou = Region("oss-cn-hangzhou") + Qingdao = Region("oss-cn-qingdao") + Beijing = Region("oss-cn-beijing") + Hongkong = Region("oss-cn-hongkong") + Shenzhen = Region("oss-cn-shenzhen") + Shanghai = Region("oss-cn-shanghai") + Zhangjiakou = Region("oss-cn-zhangjiakou") + Huhehaote = Region("oss-cn-huhehaote") + USWest1 = Region("oss-us-west-1") USEast1 = Region("oss-us-east-1") APSouthEast1 = Region("oss-ap-southeast-1") - Shanghai = Region("oss-cn-shanghai") + APNorthEast1 = Region("oss-ap-northeast-1") + APSouthEast2 = Region("oss-ap-southeast-2") + + MEEast1 = Region("oss-me-east-1") + + EUCentral1 = Region("oss-eu-central-1") + EUWest1 = Region("oss-eu-west-1") DefaultRegion = Hangzhou ) diff --git a/vendor/github.com/denverdino/aliyungo/util/encoding.go b/vendor/github.com/denverdino/aliyungo/util/encoding.go index e545e069..ec1a5b13 100644 --- a/vendor/github.com/denverdino/aliyungo/util/encoding.go +++ b/vendor/github.com/denverdino/aliyungo/util/encoding.go @@ -7,9 +7,26 @@ import ( "net/url" "reflect" "strconv" + "strings" "time" ) +// change instance=["a", "b"] +// to instance.1="a" instance.2="b" +func FlattenFn(fieldName string, field reflect.Value, values *url.Values) { + l := field.Len() + if l > 0 { + for i := 0; i < l; i++ { + str := field.Index(i).String() + values.Set(fieldName+"."+strconv.Itoa(i+1), str) + } + } +} + +func Underline2Dot(name string) string { + return strings.Replace(name, "_", ".", -1) +} + //ConvertToQueryValues converts the struct to url.Values func ConvertToQueryValues(ifc interface{}) url.Values { values := url.Values{} @@ -22,6 +39,10 @@ func SetQueryValues(ifc interface{}, values *url.Values) { setQueryValues(ifc, values, "") } +func SetQueryValueByFlattenMethod(ifc interface{}, values *url.Values) { + setQueryValuesByFlattenMethod(ifc, values, "") +} + func setQueryValues(i interface{}, values *url.Values, prefix string) { // add to support url.Values mapValues, ok := i.(url.Values) @@ -41,28 +62,32 @@ func setQueryValues(i interface{}, values *url.Values, prefix string) { fieldName := elemType.Field(i).Name anonymous := elemType.Field(i).Anonymous + tag := elemType.Field(i).Tag.Get("query") + argName := elemType.Field(i).Tag.Get("ArgName") field := elem.Field(i) // TODO Use Tag for validation // tag := typ.Field(i).Tag.Get("tagname") kind := field.Kind() + isPtr := false if (kind == reflect.Ptr || kind == reflect.Array || kind == reflect.Slice || kind == reflect.Map || kind == reflect.Chan) && field.IsNil() { continue } if kind == reflect.Ptr { field = field.Elem() kind = field.Kind() + isPtr = true } var value string //switch field.Interface().(type) { switch kind { case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: i := field.Int() - if i != 0 { + if i != 0 || isPtr { value = strconv.FormatInt(i, 10) } case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: i := field.Uint() - if i != 0 { + if i != 0 || isPtr { value = strconv.FormatUint(i, 10) } case reflect.Float32: @@ -93,15 +118,26 @@ func setQueryValues(i interface{}, values *url.Values, prefix string) { case reflect.String: l := field.Len() if l > 0 { - strArray := make([]string, l) - for i := 0; i < l; i++ { - strArray[i] = field.Index(i).String() - } - bytes, err := json.Marshal(strArray) - if err == nil { - value = string(bytes) + if tag == "list" { + name := argName + if argName == "" { + name = fieldName + } + for i := 0; i < l; i++ { + valueName := fmt.Sprintf("%s.%d", name, (i + 1)) + values.Set(valueName, field.Index(i).String()) + } } else { - log.Printf("Failed to convert JSON: %v", err) + strArray := make([]string, l) + for i := 0; i < l; i++ { + strArray[i] = field.Index(i).String() + } + bytes, err := json.Marshal(strArray) + if err == nil { + value = string(bytes) + } else { + log.Printf("Failed to convert JSON: %v", err) + } } } default: @@ -139,8 +175,8 @@ func setQueryValues(i interface{}, values *url.Values, prefix string) { } } if value != "" { - name := elemType.Field(i).Tag.Get("ArgName") - if name == "" { + name := argName + if argName == "" { name = fieldName } if prefix != "" { @@ -150,3 +186,146 @@ func setQueryValues(i interface{}, values *url.Values, prefix string) { } } } + +func setQueryValuesByFlattenMethod(i interface{}, values *url.Values, prefix string) { + // add to support url.Values + mapValues, ok := i.(url.Values) + if ok { + for k, _ := range mapValues { + values.Set(k, mapValues.Get(k)) + } + return + } + + elem := reflect.ValueOf(i) + if elem.Kind() == reflect.Ptr { + elem = elem.Elem() + } + elemType := elem.Type() + for i := 0; i < elem.NumField(); i++ { + + fieldName := elemType.Field(i).Name + anonymous := elemType.Field(i).Anonymous + field := elem.Field(i) + + // TODO Use Tag for validation + // tag := typ.Field(i).Tag.Get("tagname") + kind := field.Kind() + + isPtr := false + if (kind == reflect.Ptr || kind == reflect.Array || kind == reflect.Slice || kind == reflect.Map || kind == reflect.Chan) && field.IsNil() { + continue + } + if kind == reflect.Ptr { + field = field.Elem() + kind = field.Kind() + isPtr = true + } + + var value string + //switch field.Interface().(type) { + switch kind { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + i := field.Int() + if i != 0 || isPtr { + value = strconv.FormatInt(i, 10) + } + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + i := field.Uint() + if i != 0 || isPtr { + value = strconv.FormatUint(i, 10) + } + case reflect.Float32: + value = strconv.FormatFloat(field.Float(), 'f', 4, 32) + case reflect.Float64: + value = strconv.FormatFloat(field.Float(), 'f', 4, 64) + case reflect.Bool: + value = strconv.FormatBool(field.Bool()) + case reflect.String: + value = field.String() + case reflect.Map: + ifc := field.Interface() + m := ifc.(map[string]string) + if m != nil { + j := 0 + for k, v := range m { + j++ + keyName := fmt.Sprintf("%s.%d.Key", fieldName, j) + values.Set(keyName, k) + valueName := fmt.Sprintf("%s.%d.Value", fieldName, j) + values.Set(valueName, v) + } + } + case reflect.Slice: + if field.Type().Name() == "FlattenArray" { + FlattenFn(fieldName, field, values) + } else { + switch field.Type().Elem().Kind() { + case reflect.Uint8: + value = string(field.Bytes()) + case reflect.String: + l := field.Len() + if l > 0 { + strArray := make([]string, l) + for i := 0; i < l; i++ { + strArray[i] = field.Index(i).String() + } + bytes, err := json.Marshal(strArray) + if err == nil { + value = string(bytes) + } else { + log.Printf("Failed to convert JSON: %v", err) + } + } + default: + l := field.Len() + for j := 0; j < l; j++ { + prefixName := fmt.Sprintf("%s.%d.", fieldName, (j + 1)) + ifc := field.Index(j).Interface() + //log.Printf("%s : %v", prefixName, ifc) + if ifc != nil { + setQueryValuesByFlattenMethod(ifc, values, prefixName) + } + } + continue + } + } + + default: + switch field.Interface().(type) { + case ISO6801Time: + t := field.Interface().(ISO6801Time) + value = t.String() + case time.Time: + t := field.Interface().(time.Time) + value = GetISO8601TimeStamp(t) + default: + + ifc := field.Interface() + if ifc != nil { + if anonymous { + SetQueryValues(ifc, values) + } else { + prefixName := fieldName + "." + setQueryValuesByFlattenMethod(ifc, values, prefixName) + } + continue + } + } + } + if value != "" { + name := elemType.Field(i).Tag.Get("ArgName") + if name == "" { + name = fieldName + } + if prefix != "" { + name = prefix + name + } + // NOTE: here we will change name to underline style when the type is UnderlineString + if field.Type().Name() == "UnderlineString" { + name = Underline2Dot(name) + } + values.Set(name, value) + } + } +} diff --git a/vendor/github.com/denverdino/aliyungo/util/signature.go b/vendor/github.com/denverdino/aliyungo/util/signature.go index a00b27c1..1be6d39c 100644 --- a/vendor/github.com/denverdino/aliyungo/util/signature.go +++ b/vendor/github.com/denverdino/aliyungo/util/signature.go @@ -35,6 +35,5 @@ func CreateSignatureForRequest(method string, values *url.Values, accessKeySecre canonicalizedQueryString := percentReplace(values.Encode()) stringToSign := method + "&%2F&" + url.QueryEscape(canonicalizedQueryString) - return CreateSignature(stringToSign, accessKeySecret) } diff --git a/vendor/github.com/denverdino/aliyungo/util/util.go b/vendor/github.com/denverdino/aliyungo/util/util.go index dd68214e..ff49f2d8 100644 --- a/vendor/github.com/denverdino/aliyungo/util/util.go +++ b/vendor/github.com/denverdino/aliyungo/util/util.go @@ -9,6 +9,8 @@ import ( "net/url" "sort" "time" + "fmt" + "encoding/json" ) const dictionary = "_0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" @@ -145,3 +147,12 @@ func GenerateRandomECSPassword() string { return string(s) } + + +func PrettyJson(object interface{}) string { + b,err := json.MarshalIndent(object,"", " ") + if err != nil { + fmt.Printf("ERROR: PrettyJson, %v\n %s\n",err,b) + } + return string(b) +} \ No newline at end of file From b7839211af4fd038dbedcec1b06091ea02559a97 Mon Sep 17 00:00:00 2001 From: Li Yi Date: Tue, 25 Dec 2018 09:32:16 +0800 Subject: [PATCH 2/2] Update doc for BYOK support in OSS storage driver Change-Id: I195f29195f999e3e14ba921fd3435a6dc4788417 Signed-off-by: Li Yi --- docs/configuration.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index fe95d594..539c02a6 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -147,7 +147,8 @@ storage: endpoint: optional endpoints internal: optional internal endpoint bucket: OSS bucket - encrypt: optional data encryption setting + encrypt: optional enable server-side encryption + encryptionkeyid: optional KMS key id for encryption secure: optional ssl setting chunksize: optional size valye rootdirectory: optional root directory @@ -446,7 +447,8 @@ storage: endpoint: optional endpoints internal: optional internal endpoint bucket: OSS bucket - encrypt: optional data encryption setting + encrypt: optional enable server-side encryption + encryptionkeyid: optional KMS key id for encryption secure: optional ssl setting chunksize: optional size valye rootdirectory: optional root directory