cb0d083d8d
This commit changes storagedriver.Filewriter interface by adding context.Context as an argument to its Commit func. We pass the context appropriately where need be throughout the distribution codebase to all the writers and tests. S3 driver writer unfortunately must maintain the context passed down to it from upstream so it contnues to implement io.Writer and io.Closer interfaces which do not allow accepting the context in any of their funcs. Co-authored-by: Cory Snider <corhere@gmail.com> Signed-off-by: Milos Gajdos <milosthegajdos@gmail.com>
317 lines
8.2 KiB
Go
317 lines
8.2 KiB
Go
//go:build include_gcs
|
|
// +build include_gcs
|
|
|
|
package gcs
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"testing"
|
|
|
|
"cloud.google.com/go/storage"
|
|
dcontext "github.com/distribution/distribution/v3/context"
|
|
storagedriver "github.com/distribution/distribution/v3/registry/storage/driver"
|
|
"github.com/distribution/distribution/v3/registry/storage/driver/testsuites"
|
|
"golang.org/x/oauth2"
|
|
"golang.org/x/oauth2/google"
|
|
"google.golang.org/api/googleapi"
|
|
"google.golang.org/api/option"
|
|
"gopkg.in/check.v1"
|
|
)
|
|
|
|
// Hook up gocheck into the "go test" runner.
|
|
func Test(t *testing.T) { check.TestingT(t) }
|
|
|
|
var (
|
|
gcsDriverConstructor func(rootDirectory string) (storagedriver.StorageDriver, error)
|
|
skipGCS func() string
|
|
)
|
|
|
|
func init() {
|
|
bucket := os.Getenv("REGISTRY_STORAGE_GCS_BUCKET")
|
|
credentials := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS")
|
|
|
|
// Skip GCS storage driver tests if environment variable parameters are not provided
|
|
skipGCS = func() string {
|
|
if bucket == "" || credentials == "" {
|
|
return "The following environment variables must be set to enable these tests: REGISTRY_STORAGE_GCS_BUCKET, GOOGLE_APPLICATION_CREDENTIALS"
|
|
}
|
|
return ""
|
|
}
|
|
|
|
if skipGCS() != "" {
|
|
return
|
|
}
|
|
|
|
jsonKey, err := os.ReadFile(credentials)
|
|
if err != nil {
|
|
panic(fmt.Sprintf("Error reading JSON key : %v", err))
|
|
}
|
|
|
|
root, err := os.MkdirTemp("", "driver-")
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
defer os.Remove(root)
|
|
var ts oauth2.TokenSource
|
|
var email string
|
|
var privateKey []byte
|
|
|
|
ts, err = google.DefaultTokenSource(dcontext.Background(), storage.ScopeFullControl)
|
|
if err != nil {
|
|
// Assume that the file contents are within the environment variable since it exists
|
|
// but does not contain a valid file path
|
|
jwtConfig, err := google.JWTConfigFromJSON(jsonKey, storage.ScopeFullControl)
|
|
if err != nil {
|
|
panic(fmt.Sprintf("Error reading JWT config : %s", err))
|
|
}
|
|
email = jwtConfig.Email
|
|
privateKey = jwtConfig.PrivateKey
|
|
if len(privateKey) == 0 {
|
|
panic("Error reading JWT config : missing private_key property")
|
|
}
|
|
if email == "" {
|
|
panic("Error reading JWT config : missing client_email property")
|
|
}
|
|
ts = jwtConfig.TokenSource(dcontext.Background())
|
|
}
|
|
|
|
gcs, err := storage.NewClient(dcontext.Background(), option.WithCredentialsJSON(jsonKey))
|
|
if err != nil {
|
|
panic(fmt.Sprintf("Error initializing gcs client : %v", err))
|
|
}
|
|
|
|
gcsDriverConstructor = func(rootDirectory string) (storagedriver.StorageDriver, error) {
|
|
parameters := driverParameters{
|
|
bucket: bucket,
|
|
rootDirectory: root,
|
|
email: email,
|
|
privateKey: privateKey,
|
|
client: oauth2.NewClient(dcontext.Background(), ts),
|
|
chunkSize: defaultChunkSize,
|
|
gcs: gcs,
|
|
maxConcurrency: 8,
|
|
}
|
|
|
|
return New(parameters)
|
|
}
|
|
|
|
testsuites.RegisterSuite(func() (storagedriver.StorageDriver, error) {
|
|
return gcsDriverConstructor(root)
|
|
}, skipGCS)
|
|
}
|
|
|
|
// Test Committing a FileWriter without having called Write
|
|
func TestCommitEmpty(t *testing.T) {
|
|
if skipGCS() != "" {
|
|
t.Skip(skipGCS())
|
|
}
|
|
|
|
validRoot := t.TempDir()
|
|
|
|
driver, err := gcsDriverConstructor(validRoot)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating rooted driver: %v", err)
|
|
}
|
|
|
|
filename := "/test"
|
|
ctx := dcontext.Background()
|
|
|
|
writer, err := driver.Writer(ctx, filename, false)
|
|
defer driver.Delete(ctx, filename)
|
|
if err != nil {
|
|
t.Fatalf("driver.Writer: unexpected error: %v", err)
|
|
}
|
|
err = writer.Commit(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("writer.Commit: unexpected error: %v", err)
|
|
}
|
|
err = writer.Close()
|
|
if err != nil {
|
|
t.Fatalf("writer.Close: unexpected error: %v", err)
|
|
}
|
|
if writer.Size() != 0 {
|
|
t.Fatalf("writer.Size: %d != 0", writer.Size())
|
|
}
|
|
readContents, err := driver.GetContent(ctx, filename)
|
|
if err != nil {
|
|
t.Fatalf("driver.GetContent: unexpected error: %v", err)
|
|
}
|
|
if len(readContents) != 0 {
|
|
t.Fatalf("len(driver.GetContent(..)): %d != 0", len(readContents))
|
|
}
|
|
}
|
|
|
|
// Test Committing a FileWriter after having written exactly
|
|
// defaultChunksize bytes.
|
|
func TestCommit(t *testing.T) {
|
|
if skipGCS() != "" {
|
|
t.Skip(skipGCS())
|
|
}
|
|
|
|
validRoot := t.TempDir()
|
|
|
|
driver, err := gcsDriverConstructor(validRoot)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating rooted driver: %v", err)
|
|
}
|
|
|
|
filename := "/test"
|
|
ctx := dcontext.Background()
|
|
|
|
contents := make([]byte, defaultChunkSize)
|
|
writer, err := driver.Writer(ctx, filename, false)
|
|
defer driver.Delete(ctx, filename)
|
|
if err != nil {
|
|
t.Fatalf("driver.Writer: unexpected error: %v", err)
|
|
}
|
|
_, err = writer.Write(contents)
|
|
if err != nil {
|
|
t.Fatalf("writer.Write: unexpected error: %v", err)
|
|
}
|
|
err = writer.Commit(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("writer.Commit: unexpected error: %v", err)
|
|
}
|
|
err = writer.Close()
|
|
if err != nil {
|
|
t.Fatalf("writer.Close: unexpected error: %v", err)
|
|
}
|
|
if writer.Size() != int64(len(contents)) {
|
|
t.Fatalf("writer.Size: %d != %d", writer.Size(), len(contents))
|
|
}
|
|
readContents, err := driver.GetContent(ctx, filename)
|
|
if err != nil {
|
|
t.Fatalf("driver.GetContent: unexpected error: %v", err)
|
|
}
|
|
if len(readContents) != len(contents) {
|
|
t.Fatalf("len(driver.GetContent(..)): %d != %d", len(readContents), len(contents))
|
|
}
|
|
}
|
|
|
|
func TestRetry(t *testing.T) {
|
|
if skipGCS() != "" {
|
|
t.Skip(skipGCS())
|
|
}
|
|
|
|
assertError := func(expected string, observed error) {
|
|
observedMsg := "<nil>"
|
|
if observed != nil {
|
|
observedMsg = observed.Error()
|
|
}
|
|
if observedMsg != expected {
|
|
t.Fatalf("expected %v, observed %v\n", expected, observedMsg)
|
|
}
|
|
}
|
|
|
|
err := retry(func() error {
|
|
return &googleapi.Error{
|
|
Code: 503,
|
|
Message: "google api error",
|
|
}
|
|
})
|
|
assertError("googleapi: Error 503: google api error", err)
|
|
|
|
err = retry(func() error {
|
|
return &googleapi.Error{
|
|
Code: 404,
|
|
Message: "google api error",
|
|
}
|
|
})
|
|
assertError("googleapi: Error 404: google api error", err)
|
|
|
|
err = retry(func() error {
|
|
return fmt.Errorf("error")
|
|
})
|
|
assertError("error", err)
|
|
}
|
|
|
|
func TestEmptyRootList(t *testing.T) {
|
|
if skipGCS() != "" {
|
|
t.Skip(skipGCS())
|
|
}
|
|
|
|
validRoot := t.TempDir()
|
|
|
|
rootedDriver, err := gcsDriverConstructor(validRoot)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating rooted driver: %v", err)
|
|
}
|
|
|
|
emptyRootDriver, err := gcsDriverConstructor("")
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating empty root driver: %v", err)
|
|
}
|
|
|
|
slashRootDriver, err := gcsDriverConstructor("/")
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating slash root driver: %v", err)
|
|
}
|
|
|
|
filename := "/test"
|
|
contents := []byte("contents")
|
|
ctx := dcontext.Background()
|
|
err = rootedDriver.PutContent(ctx, filename, contents)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating content: %v", err)
|
|
}
|
|
defer func() {
|
|
err := rootedDriver.Delete(ctx, filename)
|
|
if err != nil {
|
|
t.Fatalf("failed to remove %v due to %v\n", filename, err)
|
|
}
|
|
}()
|
|
keys, err := emptyRootDriver.List(ctx, "/")
|
|
if err != nil {
|
|
t.Fatalf("unexpected error listing empty root content: %v", err)
|
|
}
|
|
for _, path := range keys {
|
|
if !storagedriver.PathRegexp.MatchString(path) {
|
|
t.Fatalf("unexpected string in path: %q != %q", path, storagedriver.PathRegexp)
|
|
}
|
|
}
|
|
|
|
keys, err = slashRootDriver.List(ctx, "/")
|
|
if err != nil {
|
|
t.Fatalf("unexpected error listing slash root content: %v", err)
|
|
}
|
|
for _, path := range keys {
|
|
if !storagedriver.PathRegexp.MatchString(path) {
|
|
t.Fatalf("unexpected string in path: %q != %q", path, storagedriver.PathRegexp)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestMoveDirectory checks that moving a directory returns an error.
|
|
func TestMoveDirectory(t *testing.T) {
|
|
if skipGCS() != "" {
|
|
t.Skip(skipGCS())
|
|
}
|
|
|
|
validRoot := t.TempDir()
|
|
|
|
driver, err := gcsDriverConstructor(validRoot)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating rooted driver: %v", err)
|
|
}
|
|
|
|
ctx := dcontext.Background()
|
|
contents := []byte("contents")
|
|
// Create a regular file.
|
|
err = driver.PutContent(ctx, "/parent/dir/foo", contents)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating content: %v", err)
|
|
}
|
|
defer func() {
|
|
err := driver.Delete(ctx, "/parent")
|
|
if err != nil {
|
|
t.Fatalf("failed to remove /parent due to %v\n", err)
|
|
}
|
|
}()
|
|
|
|
err = driver.Move(ctx, "/parent/dir", "/parent/other")
|
|
if err == nil {
|
|
t.Fatalf("Moving directory /parent/dir /parent/other should have return a non-nil error\n")
|
|
}
|
|
}
|