From 5d35fa34c151571b29d42fdfb266da997ebde6f8 Mon Sep 17 00:00:00 2001 From: Richard Scothern Date: Wed, 6 Jan 2016 14:46:25 -0800 Subject: [PATCH] Change the parameters to the GCS drivers to allow CircleCI testing. Remove the requirement of file system access to run GCS unit tests. Deconstruct the input parameters to take the private key and email which can be specified on the build system via environment variables. Signed-off-by: Richard Scothern --- docs/storage/driver/gcs/gcs.go | 77 ++++++++++++++--------------- docs/storage/driver/gcs/gcs_test.go | 55 ++++++++++++++------- 2 files changed, 75 insertions(+), 57 deletions(-) diff --git a/docs/storage/driver/gcs/gcs.go b/docs/storage/driver/gcs/gcs.go index d61d88b85..765d54924 100644 --- a/docs/storage/driver/gcs/gcs.go +++ b/docs/storage/driver/gcs/gcs.go @@ -32,7 +32,7 @@ import ( "golang.org/x/net/context" "golang.org/x/oauth2" "golang.org/x/oauth2/google" - + "golang.org/x/oauth2/jwt" "google.golang.org/api/googleapi" storageapi "google.golang.org/api/storage/v1" "google.golang.org/cloud" @@ -47,10 +47,13 @@ import ( const driverName = "gcs" const dummyProjectID = "" -//DriverParameters A struct that encapsulates all of the driver parameters after all values have been set +// driverParameters is a struct that encapsulates all of the driver parameters after all values have been set type driverParameters struct { bucket string - keyfile string + config *jwt.Config + email string + privateKey []byte + client *http.Client rootDirectory string } @@ -80,25 +83,43 @@ type driver struct { // Required parameters: // - bucket func FromParameters(parameters map[string]interface{}) (storagedriver.StorageDriver, error) { - bucket, ok := parameters["bucket"] if !ok || fmt.Sprint(bucket) == "" { return nil, fmt.Errorf("No bucket parameter provided") } - keyfile, ok := parameters["keyfile"] - if !ok { - keyfile = "" - } - rootDirectory, ok := parameters["rootdirectory"] if !ok { rootDirectory = "" } + + var ts oauth2.TokenSource + jwtConf := new(jwt.Config) + if keyfile, ok := parameters["keyfile"]; ok { + jsonKey, err := ioutil.ReadFile(fmt.Sprint(keyfile)) + if err != nil { + return nil, err + } + jwtConf, err = google.JWTConfigFromJSON(jsonKey, storage.ScopeFullControl) + if err != nil { + return nil, err + } + ts = jwtConf.TokenSource(context.Background()) + } else { + var err error + ts, err = google.DefaultTokenSource(context.Background(), storage.ScopeFullControl) + if err != nil { + return nil, err + } + + } + params := driverParameters{ - fmt.Sprint(bucket), - fmt.Sprint(keyfile), - fmt.Sprint(rootDirectory), + bucket: fmt.Sprint(bucket), + rootDirectory: fmt.Sprint(rootDirectory), + email: jwtConf.Email, + privateKey: jwtConf.PrivateKey, + client: oauth2.NewClient(context.Background(), ts), } return New(params) @@ -106,8 +127,6 @@ func FromParameters(parameters map[string]interface{}) (storagedriver.StorageDri // New constructs a new driver func New(params driverParameters) (storagedriver.StorageDriver, error) { - var ts oauth2.TokenSource - var err error rootDirectory := strings.Trim(params.rootDirectory, "/") if rootDirectory != "" { rootDirectory += "/" @@ -115,33 +134,11 @@ func New(params driverParameters) (storagedriver.StorageDriver, error) { d := &driver{ bucket: params.bucket, rootDirectory: rootDirectory, + email: params.email, + privateKey: params.privateKey, + client: params.client, } - if params.keyfile == "" { - ts, err = google.DefaultTokenSource(context.Background(), storage.ScopeFullControl) - if err != nil { - return nil, err - } - } else { - jsonKey, err := ioutil.ReadFile(params.keyfile) - if err != nil { - return nil, err - } - conf, err := google.JWTConfigFromJSON( - jsonKey, - storage.ScopeFullControl, - ) - if err != nil { - return nil, err - } - ts = conf.TokenSource(context.Background()) - d.email = conf.Email - d.privateKey = conf.PrivateKey - } - client := oauth2.NewClient(context.Background(), ts) - d.client = client - if err != nil { - return nil, err - } + return &base.Base{ StorageDriver: d, }, nil diff --git a/docs/storage/driver/gcs/gcs_test.go b/docs/storage/driver/gcs/gcs_test.go index 4062b2a3e..60f3e957d 100644 --- a/docs/storage/driver/gcs/gcs_test.go +++ b/docs/storage/driver/gcs/gcs_test.go @@ -3,17 +3,18 @@ package gcs import ( - "fmt" "io/ioutil" "os" "testing" - "google.golang.org/api/googleapi" - + "fmt" ctx "github.com/docker/distribution/context" storagedriver "github.com/docker/distribution/registry/storage/driver" "github.com/docker/distribution/registry/storage/driver/testsuites" - + "golang.org/x/oauth2" + "golang.org/x/oauth2/google" + "google.golang.org/api/googleapi" + "google.golang.org/cloud/storage" "gopkg.in/check.v1" ) @@ -25,34 +26,54 @@ var skipGCS func() string func init() { bucket := os.Getenv("REGISTRY_STORAGE_GCS_BUCKET") - keyfile := os.Getenv("REGISTRY_STORAGE_GCS_KEYFILE") credentials := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") + // Skip GCS storage driver tests if environment variable parameters are not provided + skipGCS = func() string { + if bucket == "" || credentials == "" { + return "The following environment variables must be set to enable these tests: REGISTRY_STORAGE_GCS_BUCKET, REGISTRY_STORAGE_GCS_CREDS" + } + return "" + } + + if skipGCS() != "" { + return + } + root, err := ioutil.TempDir("", "driver-") if err != nil { panic(err) } defer os.Remove(root) - gcsDriverConstructor = func(rootDirectory string) (storagedriver.StorageDriver, error) { + _, err = os.Stat(credentials) + if err == nil { + jsonKey, err := ioutil.ReadFile(credentials) + if err != nil { + panic(fmt.Sprintf("Unable to read credentials from file : %s", err)) + } + credentials = string(jsonKey) + } + // Assume that the file contents are within the environment variable since it exists + // but does not contain a valid file path + jwtConfig, err := google.JWTConfigFromJSON([]byte(credentials), storage.ScopeFullControl) + if err != nil { + panic(fmt.Sprintf("Error reading JWT config : %s", err)) + } + + gcsDriverConstructor = func(rootDirectory string) (storagedriver.StorageDriver, error) { parameters := driverParameters{ - bucket, - keyfile, - rootDirectory, + bucket: bucket, + rootDirectory: root, + email: jwtConfig.Email, + privateKey: []byte(jwtConfig.PrivateKey), + client: oauth2.NewClient(ctx.Background(), jwtConfig.TokenSource(ctx.Background())), } return New(parameters) } - // Skip GCS storage driver tests if environment variable parameters are not provided - skipGCS = func() string { - if bucket == "" || (credentials == "" && keyfile == "") { - return "Must set REGISTRY_STORAGE_GCS_BUCKET and (GOOGLE_APPLICATION_CREDENTIALS or REGISTRY_STORAGE_GCS_KEYFILE) to run GCS tests" - } - return "" - } - testsuites.RegisterSuite(func() (storagedriver.StorageDriver, error) { return gcsDriverConstructor(root) }, skipGCS)