Merge pull request #2849 from Shawnpku/master

support Alibaba Cloud CDN storage middleware
This commit is contained in:
Ryan Abrams 2019-04-16 18:43:06 -07:00 committed by GitHub
commit 3226863cbc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 385 additions and 23 deletions

View file

@ -12,6 +12,7 @@ import (
_ "github.com/docker/distribution/registry/storage/driver/filesystem"
_ "github.com/docker/distribution/registry/storage/driver/gcs"
_ "github.com/docker/distribution/registry/storage/driver/inmemory"
_ "github.com/docker/distribution/registry/storage/driver/middleware/alicdn"
_ "github.com/docker/distribution/registry/storage/driver/middleware/cloudfront"
_ "github.com/docker/distribution/registry/storage/driver/middleware/redirect"
_ "github.com/docker/distribution/registry/storage/driver/oss"

View file

@ -720,6 +720,17 @@ Value of `ipfilteredby` can be:
| `aws` | IP from AWS goes to S3 directly |
| `awsregion` | IP from certain AWS regions goes to S3 directly, use together with `awsregion`. |
### `alicdn`
`alicdn` storage middleware allows the registry to serve layers via a content delivery network provided by Alibaba Cloud. Alicdn requires the OSS storage driver.
| Parameter | Required | Description |
|--------------|----------|-------------------------------------------------------------------------|
| `baseurl` | yes | The `SCHEME://HOST` at which Alicdn is served. |
| `authtype` | yes | The URL authentication type for Alicdn, which should be `a`, `b` or `c`. See the [Authentication configuration](https://www.alibabacloud.com/help/doc-detail/85117.htm).|
| `privatekey` | yes | The URL authentication key for Alicdn. |
| `duration` | no | An integer and unit for the duration of the Alicdn session. Valid time units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, or `h`.|
### `redirect`
You can use the `redirect` storage middleware to specify a custom URL to a

View file

@ -0,0 +1,116 @@
package alicdn
import (
"context"
"fmt"
"net/url"
"strings"
"time"
dcontext "github.com/docker/distribution/context"
storagedriver "github.com/docker/distribution/registry/storage/driver"
storagemiddleware "github.com/docker/distribution/registry/storage/driver/middleware"
"github.com/denverdino/aliyungo/cdn/auth"
)
// aliCDNStorageMiddleware provides a simple implementation of layerHandler that
// constructs temporary signed AliCDN URLs from the storagedriver layer URL,
// then issues HTTP Temporary Redirects to this AliCDN content URL.
type aliCDNStorageMiddleware struct {
storagedriver.StorageDriver
baseURL string
urlSigner *auth.URLSigner
duration time.Duration
}
var _ storagedriver.StorageDriver = &aliCDNStorageMiddleware{}
// newAliCDNStorageMiddleware constructs and returns a new AliCDN
// StorageDriver implementation.
// Required options: baseurl, authtype, privatekey
// Optional options: duration
func newAliCDNStorageMiddleware(storageDriver storagedriver.StorageDriver, options map[string]interface{}) (storagedriver.StorageDriver, error) {
// parse baseurl
base, ok := options["baseurl"]
if !ok {
return nil, fmt.Errorf("no baseurl provided")
}
baseURL, ok := base.(string)
if !ok {
return nil, fmt.Errorf("baseurl must be a string")
}
if !strings.Contains(baseURL, "://") {
baseURL = "https://" + baseURL
}
if _, err := url.Parse(baseURL); err != nil {
return nil, fmt.Errorf("invalid baseurl: %v", err)
}
// parse authtype
at, ok := options["authtype"]
if !ok {
return nil, fmt.Errorf("no authtype provided")
}
authType, ok := at.(string)
if !ok {
return nil, fmt.Errorf("authtype must be a string")
}
if authType != "a" && authType != "b" && authType != "c" {
return nil, fmt.Errorf("invalid authentication type")
}
// parse privatekey
pk, ok := options["privatekey"]
if !ok {
return nil, fmt.Errorf("no privatekey provided")
}
privateKey, ok := pk.(string)
if !ok {
return nil, fmt.Errorf("privatekey must be a string")
}
urlSigner := auth.NewURLSigner(authType, privateKey)
// parse duration
duration := 20 * time.Minute
d, ok := options["duration"]
if ok {
switch d := d.(type) {
case time.Duration:
duration = d
case string:
dur, err := time.ParseDuration(d)
if err != nil {
return nil, fmt.Errorf("invalid duration: %s", err)
}
duration = dur
}
}
return &aliCDNStorageMiddleware{
StorageDriver: storageDriver,
baseURL: baseURL,
urlSigner: urlSigner,
duration: duration,
}, nil
}
// URLFor attempts to find a url which may be used to retrieve the file at the given path.
func (ac *aliCDNStorageMiddleware) URLFor(ctx context.Context, path string, options map[string]interface{}) (string, error) {
if ac.StorageDriver.Name() != "oss" {
dcontext.GetLogger(ctx).Warn("the AliCDN middleware does not support this backend storage driver")
return ac.StorageDriver.URLFor(ctx, path, options)
}
acURL, err := ac.urlSigner.Sign(ac.baseURL+path, time.Now().Add(ac.duration))
if err != nil {
return "", err
}
return acURL, nil
}
// init registers the alicdn layerHandler backend.
func init() {
storagemiddleware.Register("alicdn", storagemiddleware.InitFunc(newAliCDNStorageMiddleware))
}

View file

@ -7,7 +7,7 @@ github.com/beorn7/perks 4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9
github.com/bugsnag/bugsnag-go b1d153021fcd90ca3f080db36bec96dc690fb274
github.com/bugsnag/osext 0dd3f918b21bec95ace9dc86c7e70266cfc5c702
github.com/bugsnag/panicwrap e2c28503fcd0675329da73bf48b33404db873782
github.com/denverdino/aliyungo 6df11717a253d9c7d4141f9af4deaa7c580cd531
github.com/denverdino/aliyungo a747050bb1baf06cdd65de7cddc281a2b1c2fde5
github.com/dgrijalva/jwt-go a601269ab70c205d26370c16f7c81e9017c14e04
github.com/docker/go-metrics 399ea8c73916000c64c2c76e8da00ca82f8387ab
github.com/docker/libtrust fa567046d9b14f6aa788882a950d69651d230b21

View file

@ -0,0 +1,97 @@
package auth
import (
"crypto/rand"
"fmt"
"io"
"os"
"syscall"
"time"
)
const (
// Bits is the number of bits in a UUID
Bits = 128
// Size is the number of bytes in a UUID
Size = Bits / 8
format = "%08x%04x%04x%04x%012x"
)
var (
// Loggerf can be used to override the default logging destination. Such
// log messages in this library should be logged at warning or higher.
Loggerf = func(format string, args ...interface{}) {}
)
// UUID represents a UUID value. UUIDs can be compared and set to other values
// and accessed by byte.
type UUID [Size]byte
// GenerateUUID creates a new, version 4 uuid.
func GenerateUUID() (u UUID) {
const (
// ensures we backoff for less than 450ms total. Use the following to
// select new value, in units of 10ms:
// n*(n+1)/2 = d -> n^2 + n - 2d -> n = (sqrt(8d + 1) - 1)/2
maxretries = 9
backoff = time.Millisecond * 10
)
var (
totalBackoff time.Duration
count int
retries int
)
for {
// This should never block but the read may fail. Because of this,
// we just try to read the random number generator until we get
// something. This is a very rare condition but may happen.
b := time.Duration(retries) * backoff
time.Sleep(b)
totalBackoff += b
n, err := io.ReadFull(rand.Reader, u[count:])
if err != nil {
if retryOnError(err) && retries < maxretries {
count += n
retries++
Loggerf("error generating version 4 uuid, retrying: %v", err)
continue
}
// Any other errors represent a system problem. What did someone
// do to /dev/urandom?
panic(fmt.Errorf("error reading random number generator, retried for %v: %v", totalBackoff.String(), err))
}
break
}
u[6] = (u[6] & 0x0f) | 0x40 // set version byte
u[8] = (u[8] & 0x3f) | 0x80 // set high order byte 0b10{8,9,a,b}
return u
}
func (u UUID) String() string {
return fmt.Sprintf(format, u[:4], u[4:6], u[6:8], u[8:10], u[10:])
}
// retryOnError tries to detect whether or not retrying would be fruitful.
func retryOnError(err error) bool {
switch err := err.(type) {
case *os.PathError:
return retryOnError(err.Err) // unpack the target error
case syscall.Errno:
if err == syscall.EPERM {
// EPERM represents an entropy pool exhaustion, a condition under
// which we backoff and retry.
return true
}
}
return false
}

View file

@ -0,0 +1,80 @@
package auth
import (
"crypto/md5"
"fmt"
"net/url"
"time"
)
// An URLSigner provides URL signing utilities to sign URLs for Aliyun CDN
// resources.
// authentication document: https://help.aliyun.com/document_detail/85117.html
type URLSigner struct {
authType string
privKey string
}
// NewURLSigner returns a new signer object.
func NewURLSigner(authType string, privKey string) *URLSigner {
return &URLSigner{
authType: authType,
privKey: privKey,
}
}
// Sign returns a signed aliyuncdn url based on authentication type
func (s URLSigner) Sign(uri string, expires time.Time) (string, error) {
r, err := url.Parse(uri)
if err != nil {
return "", fmt.Errorf("unable to parse url: %s", uri)
}
switch s.authType {
case "a":
return aTypeSign(r, s.privKey, expires), nil
case "b":
return bTypeSign(r, s.privKey, expires), nil
case "c":
return cTypeSign(r, s.privKey, expires), nil
default:
return "", fmt.Errorf("invalid authentication type")
}
}
// sign by A type authentication method.
// authentication document: https://help.aliyun.com/document_detail/85113.html
func aTypeSign(r *url.URL, privateKey string, expires time.Time) string {
//rand is a random uuid without "-"
rand := GenerateUUID().String()
// not use, "0" by default
uid := "0"
secret := fmt.Sprintf("%s-%d-%s-%s-%s", r.Path, expires.Unix(), rand, uid, privateKey)
hashValue := md5.Sum([]byte(secret))
authKey := fmt.Sprintf("%d-%s-%s-%x", expires.Unix(), rand, uid, hashValue)
if r.RawQuery == "" {
return fmt.Sprintf("%s?auth_key=%s", r.String(), authKey)
}
return fmt.Sprintf("%s&auth_key=%s", r.String(), authKey)
}
// sign by B type authentication method.
// authentication document: https://help.aliyun.com/document_detail/85114.html
func bTypeSign(r *url.URL, privateKey string, expires time.Time) string {
formatExp := expires.Format("200601021504")
secret := privateKey + formatExp + r.Path
hashValue := md5.Sum([]byte(secret))
signURL := fmt.Sprintf("%s://%s/%s/%x%s?%s", r.Scheme, r.Host, formatExp, hashValue, r.Path, r.RawQuery)
return signURL
}
// sign by C type authentication method.
// authentication document: https://help.aliyun.com/document_detail/85115.html
func cTypeSign(r *url.URL, privateKey string, expires time.Time) string {
hexExp := fmt.Sprintf("%x", expires.Unix())
secret := privateKey + r.Path + hexExp
hashValue := md5.Sum([]byte(secret))
signURL := fmt.Sprintf("%s://%s/%x/%s%s?%s", r.Scheme, r.Host, hashValue, hexExp, r.Path, r.RawQuery)
return signURL
}

View file

@ -851,6 +851,17 @@ func (b *Bucket) SignedURLWithArgs(path string, expires time.Time, params url.Va
return b.SignedURLWithMethod("GET", path, expires, params, headers)
}
func (b *Bucket) SignedURLWithMethodForAssumeRole(method, path string, expires time.Time, params url.Values, headers http.Header) string {
var uv = url.Values{}
if params != nil {
uv = params
}
if len(b.Client.SecurityToken) != 0 {
uv.Set("security-token", b.Client.SecurityToken)
}
return b.SignedURLWithMethod(method, path, expires, params, headers)
}
// SignedURLWithMethod returns a signed URL that allows anyone holding the URL
// to either retrieve the object at path or make a HEAD request against it. The signature is valid until expires.
func (b *Bucket) SignedURLWithMethod(method, path string, expires time.Time, params url.Values, headers http.Header) string {
@ -1039,7 +1050,8 @@ func partiallyEscapedPath(path string) string {
func (client *Client) prepare(req *request) error {
// Copy so they can be mutated without affecting on retries.
headers := copyHeader(req.headers)
if len(client.SecurityToken) != 0 {
// security-token should be in either Params or Header, cannot be in both
if len(req.params.Get("security-token")) == 0 && len(client.SecurityToken) != 0 {
headers.Set("x-oss-security-token", client.SecurityToken)
}

View file

@ -13,26 +13,44 @@ const HeaderOSSPrefix = "x-oss-"
var ossParamsToSign = map[string]bool{
"acl": true,
"append": true,
"bucketInfo": true,
"cname": true,
"comp": true,
"cors": true,
"delete": true,
"endTime": true,
"img": true,
"lifecycle": true,
"live": true,
"location": true,
"logging": true,
"notification": true,
"objectMeta": true,
"partNumber": true,
"policy": true,
"requestPayment": true,
"torrent": true,
"uploadId": true,
"uploads": true,
"versionId": true,
"versioning": true,
"versions": true,
"response-content-type": true,
"response-content-language": true,
"response-expires": true,
"position": true,
"qos": true,
"referer": true,
"replication": true,
"replicationLocation": true,
"replicationProgress": true,
"response-cache-control": true,
"response-content-disposition": true,
"response-content-encoding": true,
"bucketInfo": true,
"response-content-language": true,
"response-content-type": true,
"response-expires": true,
"security-token": true,
"startTime": true,
"status": true,
"style": true,
"styleName": true,
"symlink": true,
"tagging": true,
"uploadId": true,
"uploads": true,
"vod": true,
"website": true,
"x-oss-process": true,
}
func (client *Client) signRequest(request *request) {
@ -62,7 +80,7 @@ func (client *Client) signRequest(request *request) {
}
if len(params) > 0 {
resource = resource + "?" + util.Encode(params)
resource = resource + "?" + util.EncodeWithoutEscape(params)
}
canonicalizedResource := resource
@ -74,7 +92,7 @@ func (client *Client) signRequest(request *request) {
//log.Println("stringToSign: ", stringToSign)
signature := util.CreateSignature(stringToSign, client.AccessKeySecret)
if query.Get("OSSAccessKeyId") != "" {
if urlSignature {
query.Set("Signature", signature)
} else {
headers.Set("Authorization", "OSS "+client.AccessKeyId+":"+signature)

View file

@ -4,13 +4,13 @@ import (
"bytes"
srand "crypto/rand"
"encoding/binary"
"encoding/json"
"fmt"
"math/rand"
"net/http"
"net/url"
"sort"
"time"
"fmt"
"encoding/json"
)
const dictionary = "_0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
@ -66,6 +66,34 @@ func Encode(v url.Values) string {
return buf.String()
}
// Like Encode, but key and value are not escaped
func EncodeWithoutEscape(v url.Values) string {
if v == nil {
return ""
}
var buf bytes.Buffer
keys := make([]string, 0, len(v))
for k := range v {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
vs := v[k]
prefix := k
for _, v := range vs {
if buf.Len() > 0 {
buf.WriteByte('&')
}
buf.WriteString(prefix)
if v != "" {
buf.WriteString("=")
buf.WriteString(v)
}
}
}
return buf.String()
}
func GetGMTime() string {
return time.Now().UTC().Format(http.TimeFormat)
}
@ -148,11 +176,10 @@ func GenerateRandomECSPassword() string {
}
func PrettyJson(object interface{}) string {
b,err := json.MarshalIndent(object,"", " ")
b, err := json.MarshalIndent(object, "", " ")
if err != nil {
fmt.Printf("ERROR: PrettyJson, %v\n %s\n",err,b)
fmt.Printf("ERROR: PrettyJson, %v\n %s\n", err, b)
}
return string(b)
}