dc5b77101d
(cherry picked from commit 483ba26165ca66bcf18a1eaadf41ebe4d3bd5f85) Signed-off-by: Collin Shoop <cshoop@digitalocean.com>
630 lines
16 KiB
Go
630 lines
16 KiB
Go
package s3
|
|
|
|
import (
|
|
"bytes"
|
|
"fmt"
|
|
"io/ioutil"
|
|
"math/rand"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gopkg.in/check.v1"
|
|
|
|
"github.com/aws/aws-sdk-go/aws"
|
|
"github.com/aws/aws-sdk-go/service/s3"
|
|
|
|
"github.com/distribution/distribution/v3/context"
|
|
storagedriver "github.com/distribution/distribution/v3/registry/storage/driver"
|
|
"github.com/distribution/distribution/v3/registry/storage/driver/testsuites"
|
|
)
|
|
|
|
// Hook up gocheck into the "go test" runner.
|
|
func Test(t *testing.T) { check.TestingT(t) }
|
|
|
|
var s3DriverConstructor func(rootDirectory, storageClass string) (*Driver, error)
|
|
var skipS3 func() string
|
|
|
|
func init() {
|
|
accessKey := os.Getenv("AWS_ACCESS_KEY")
|
|
secretKey := os.Getenv("AWS_SECRET_KEY")
|
|
bucket := os.Getenv("S3_BUCKET")
|
|
encrypt := os.Getenv("S3_ENCRYPT")
|
|
keyID := os.Getenv("S3_KEY_ID")
|
|
secure := os.Getenv("S3_SECURE")
|
|
skipVerify := os.Getenv("S3_SKIP_VERIFY")
|
|
v4Auth := os.Getenv("S3_V4_AUTH")
|
|
region := os.Getenv("AWS_REGION")
|
|
objectACL := os.Getenv("S3_OBJECT_ACL")
|
|
root, err := ioutil.TempDir("", "driver-")
|
|
regionEndpoint := os.Getenv("REGION_ENDPOINT")
|
|
sessionToken := os.Getenv("AWS_SESSION_TOKEN")
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
defer os.Remove(root)
|
|
|
|
s3DriverConstructor = func(rootDirectory, storageClass string) (*Driver, error) {
|
|
encryptBool := false
|
|
if encrypt != "" {
|
|
encryptBool, err = strconv.ParseBool(encrypt)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
secureBool := true
|
|
if secure != "" {
|
|
secureBool, err = strconv.ParseBool(secure)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
skipVerifyBool := false
|
|
if skipVerify != "" {
|
|
skipVerifyBool, err = strconv.ParseBool(skipVerify)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
v4Bool := true
|
|
if v4Auth != "" {
|
|
v4Bool, err = strconv.ParseBool(v4Auth)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
parameters := DriverParameters{
|
|
accessKey,
|
|
secretKey,
|
|
bucket,
|
|
region,
|
|
regionEndpoint,
|
|
encryptBool,
|
|
keyID,
|
|
secureBool,
|
|
skipVerifyBool,
|
|
v4Bool,
|
|
minChunkSize,
|
|
defaultMultipartCopyChunkSize,
|
|
defaultMultipartCopyMaxConcurrency,
|
|
defaultMultipartCopyThresholdSize,
|
|
rootDirectory,
|
|
storageClass,
|
|
driverName + "-test",
|
|
objectACL,
|
|
sessionToken,
|
|
}
|
|
|
|
return New(parameters)
|
|
}
|
|
|
|
// Skip S3 storage driver tests if environment variable parameters are not provided
|
|
skipS3 = func() string {
|
|
if accessKey == "" || secretKey == "" || region == "" || bucket == "" || encrypt == "" {
|
|
return "Must set AWS_ACCESS_KEY, AWS_SECRET_KEY, AWS_REGION, S3_BUCKET, and S3_ENCRYPT to run S3 tests"
|
|
}
|
|
return ""
|
|
}
|
|
|
|
testsuites.RegisterSuite(func() (storagedriver.StorageDriver, error) {
|
|
return s3DriverConstructor(root, s3.StorageClassStandard)
|
|
}, skipS3)
|
|
}
|
|
|
|
func TestEmptyRootList(t *testing.T) {
|
|
if skipS3() != "" {
|
|
t.Skip(skipS3())
|
|
}
|
|
|
|
validRoot, err := ioutil.TempDir("", "driver-")
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating temporary directory: %v", err)
|
|
}
|
|
defer os.Remove(validRoot)
|
|
|
|
rootedDriver, err := s3DriverConstructor(validRoot, s3.StorageClassStandard)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating rooted driver: %v", err)
|
|
}
|
|
|
|
emptyRootDriver, err := s3DriverConstructor("", s3.StorageClassStandard)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating empty root driver: %v", err)
|
|
}
|
|
|
|
slashRootDriver, err := s3DriverConstructor("/", s3.StorageClassStandard)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating slash root driver: %v", err)
|
|
}
|
|
|
|
filename := "/test"
|
|
contents := []byte("contents")
|
|
ctx := context.Background()
|
|
err = rootedDriver.PutContent(ctx, filename, contents)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating content: %v", err)
|
|
}
|
|
defer rootedDriver.Delete(ctx, filename)
|
|
|
|
keys, _ := emptyRootDriver.List(ctx, "/")
|
|
for _, path := range keys {
|
|
if !storagedriver.PathRegexp.MatchString(path) {
|
|
t.Fatalf("unexpected string in path: %q != %q", path, storagedriver.PathRegexp)
|
|
}
|
|
}
|
|
|
|
keys, _ = slashRootDriver.List(ctx, "/")
|
|
for _, path := range keys {
|
|
if !storagedriver.PathRegexp.MatchString(path) {
|
|
t.Fatalf("unexpected string in path: %q != %q", path, storagedriver.PathRegexp)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestWalkEmptySubDirectory assures we list an empty sub directory only once when walking
|
|
// through its parent directory.
|
|
func TestWalkEmptySubDirectory(t *testing.T) {
|
|
if skipS3() != "" {
|
|
t.Skip(skipS3())
|
|
}
|
|
|
|
drv, err := s3DriverConstructor("", s3.StorageClassStandard)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating rooted driver: %v", err)
|
|
}
|
|
|
|
// create an empty sub directory.
|
|
s3driver := drv.StorageDriver.(*driver)
|
|
if _, err := s3driver.S3.PutObject(&s3.PutObjectInput{
|
|
Bucket: aws.String(os.Getenv("S3_BUCKET")),
|
|
Key: aws.String("/testdir/emptydir/"),
|
|
}); err != nil {
|
|
t.Fatalf("error creating empty directory: %s", err)
|
|
}
|
|
|
|
bucketFiles := []string{}
|
|
s3driver.Walk(context.Background(), "/testdir", func(fileInfo storagedriver.FileInfo) error {
|
|
bucketFiles = append(bucketFiles, fileInfo.Path())
|
|
return nil
|
|
})
|
|
|
|
expected := []string{"/testdir/emptydir"}
|
|
if !reflect.DeepEqual(bucketFiles, expected) {
|
|
t.Errorf("expecting files %+v, found %+v instead", expected, bucketFiles)
|
|
}
|
|
}
|
|
|
|
func TestStorageClass(t *testing.T) {
|
|
if skipS3() != "" {
|
|
t.Skip(skipS3())
|
|
}
|
|
|
|
rootDir, err := ioutil.TempDir("", "driver-")
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating temporary directory: %v", err)
|
|
}
|
|
defer os.Remove(rootDir)
|
|
|
|
standardDriver, err := s3DriverConstructor(rootDir, s3.StorageClassStandard)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating driver with standard storage: %v", err)
|
|
}
|
|
|
|
rrDriver, err := s3DriverConstructor(rootDir, s3.StorageClassReducedRedundancy)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating driver with reduced redundancy storage: %v", err)
|
|
}
|
|
|
|
if _, err = s3DriverConstructor(rootDir, noStorageClass); err != nil {
|
|
t.Fatalf("unexpected error creating driver without storage class: %v", err)
|
|
}
|
|
|
|
standardFilename := "/test-standard"
|
|
rrFilename := "/test-rr"
|
|
contents := []byte("contents")
|
|
ctx := context.Background()
|
|
|
|
err = standardDriver.PutContent(ctx, standardFilename, contents)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating content: %v", err)
|
|
}
|
|
defer standardDriver.Delete(ctx, standardFilename)
|
|
|
|
err = rrDriver.PutContent(ctx, rrFilename, contents)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating content: %v", err)
|
|
}
|
|
defer rrDriver.Delete(ctx, rrFilename)
|
|
|
|
standardDriverUnwrapped := standardDriver.Base.StorageDriver.(*driver)
|
|
resp, err := standardDriverUnwrapped.S3.GetObject(&s3.GetObjectInput{
|
|
Bucket: aws.String(standardDriverUnwrapped.Bucket),
|
|
Key: aws.String(standardDriverUnwrapped.s3Path(standardFilename)),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("unexpected error retrieving standard storage file: %v", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
// Amazon only populates this header value for non-standard storage classes
|
|
if resp.StorageClass != nil {
|
|
t.Fatalf("unexpected storage class for standard file: %v", resp.StorageClass)
|
|
}
|
|
|
|
rrDriverUnwrapped := rrDriver.Base.StorageDriver.(*driver)
|
|
resp, err = rrDriverUnwrapped.S3.GetObject(&s3.GetObjectInput{
|
|
Bucket: aws.String(rrDriverUnwrapped.Bucket),
|
|
Key: aws.String(rrDriverUnwrapped.s3Path(rrFilename)),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("unexpected error retrieving reduced-redundancy storage file: %v", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StorageClass == nil {
|
|
t.Fatalf("unexpected storage class for reduced-redundancy file: %v", s3.StorageClassStandard)
|
|
} else if *resp.StorageClass != s3.StorageClassReducedRedundancy {
|
|
t.Fatalf("unexpected storage class for reduced-redundancy file: %v", *resp.StorageClass)
|
|
}
|
|
|
|
}
|
|
|
|
func TestPopulate(t *testing.T) {
|
|
|
|
if skipS3() != "" {
|
|
t.Skip(skipS3())
|
|
}
|
|
|
|
rootDir, err := ioutil.TempDir("", "driver-")
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating temporary directory: %v", err)
|
|
}
|
|
defer os.Remove(rootDir)
|
|
|
|
driver, err := s3DriverConstructor(rootDir, s3.StorageClassStandard)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating driver with standard storage: %v", err)
|
|
}
|
|
|
|
var objs = []string{
|
|
"/file1",
|
|
"/file1-2",
|
|
"/file1/2",
|
|
"/folder1/file1",
|
|
"/folder2/file1",
|
|
"/folder3/file1",
|
|
"/folder3/subfolder1/subfolder1/file1",
|
|
"/folder3/subfolder2/subfolder1/file1",
|
|
"/folder4/file1",
|
|
"/folder1-v2/file1",
|
|
"/folder1-v2/subfolder1/file1",
|
|
}
|
|
|
|
init := func() []string {
|
|
// init file structure matching objs above
|
|
var created []string
|
|
for _, path := range objs {
|
|
err := driver.PutContent(context.Background(), path, []byte("content "+path))
|
|
if err != nil {
|
|
fmt.Printf("unable to init file %s: %s\n", path, err)
|
|
continue
|
|
}
|
|
created = append(created, path)
|
|
}
|
|
return created
|
|
}
|
|
init()
|
|
}
|
|
|
|
func TestDelete(t *testing.T) {
|
|
if skipS3() != "" {
|
|
t.Skip(skipS3())
|
|
}
|
|
|
|
rootDir, err := ioutil.TempDir("", "driver-")
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating temporary directory: %v", err)
|
|
}
|
|
defer os.Remove(rootDir)
|
|
|
|
driver, err := s3DriverConstructor(rootDir, s3.StorageClassStandard)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating driver with standard storage: %v", err)
|
|
}
|
|
|
|
type errFn func(error) bool
|
|
type testCase struct {
|
|
name string
|
|
delete string
|
|
expected []string
|
|
// error validation function
|
|
err errFn
|
|
}
|
|
|
|
errPathNotFound := func(err error) bool {
|
|
if err == nil {
|
|
return false
|
|
}
|
|
switch err.(type) {
|
|
case storagedriver.PathNotFoundError:
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
errInvalidPath := func(err error) bool {
|
|
if err == nil {
|
|
return false
|
|
}
|
|
switch err.(type) {
|
|
case storagedriver.InvalidPathError:
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
var objs = []string{
|
|
"/file1",
|
|
"/file1-2",
|
|
"/file1/2",
|
|
"/folder1/file1",
|
|
"/folder2/file1",
|
|
"/folder3/file1",
|
|
"/folder3/subfolder1/subfolder1/file1",
|
|
"/folder3/subfolder2/subfolder1/file1",
|
|
"/folder4/file1",
|
|
"/folder1-v2/file1",
|
|
"/folder1-v2/subfolder1/file1",
|
|
}
|
|
|
|
tcs := []testCase{
|
|
{
|
|
// special case where a given path is a file and has subpaths
|
|
name: "delete file1",
|
|
delete: "/file1",
|
|
expected: []string{
|
|
"/file1",
|
|
"/file1/2",
|
|
},
|
|
},
|
|
{
|
|
name: "delete folder1",
|
|
delete: "/folder1",
|
|
expected: []string{
|
|
"/folder1/file1",
|
|
},
|
|
},
|
|
{
|
|
name: "delete folder2",
|
|
delete: "/folder2",
|
|
expected: []string{
|
|
"/folder2/file1",
|
|
},
|
|
},
|
|
{
|
|
name: "delete folder3",
|
|
delete: "/folder3",
|
|
expected: []string{
|
|
"/folder3/file1",
|
|
"/folder3/subfolder1/subfolder1/file1",
|
|
"/folder3/subfolder2/subfolder1/file1",
|
|
},
|
|
},
|
|
{
|
|
name: "delete path that doesn't exist",
|
|
delete: "/path/does/not/exist",
|
|
expected: []string{},
|
|
err: errPathNotFound,
|
|
},
|
|
{
|
|
name: "delete path invalid: trailing slash",
|
|
delete: "/path/is/invalid/",
|
|
expected: []string{},
|
|
err: errInvalidPath,
|
|
},
|
|
{
|
|
name: "delete path invalid: trailing special character",
|
|
delete: "/path/is/invalid*",
|
|
expected: []string{},
|
|
err: errInvalidPath,
|
|
},
|
|
}
|
|
|
|
// objects to skip auto-created test case
|
|
var skipCase = map[string]bool{
|
|
// special case where deleting "/file1" also deletes "/file1/2" is tested explicitly
|
|
"/file1": true,
|
|
}
|
|
// create a test case for each file
|
|
for _, path := range objs {
|
|
if skipCase[path] {
|
|
continue
|
|
}
|
|
tcs = append(tcs, testCase{
|
|
name: fmt.Sprintf("delete path:'%s'", path),
|
|
delete: path,
|
|
expected: []string{path},
|
|
})
|
|
}
|
|
|
|
init := func() []string {
|
|
// init file structure matching objs
|
|
var created []string
|
|
for _, path := range objs {
|
|
err := driver.PutContent(context.Background(), path, []byte("content "+path))
|
|
if err != nil {
|
|
fmt.Printf("unable to init file %s: %s\n", path, err)
|
|
continue
|
|
}
|
|
created = append(created, path)
|
|
}
|
|
return created
|
|
}
|
|
|
|
cleanup := func(objs []string) {
|
|
var lastErr error
|
|
for _, path := range objs {
|
|
err := driver.Delete(context.Background(), path)
|
|
if err != nil {
|
|
switch err.(type) {
|
|
case storagedriver.PathNotFoundError:
|
|
continue
|
|
}
|
|
lastErr = err
|
|
}
|
|
}
|
|
if lastErr != nil {
|
|
t.Fatalf("cleanup failed: %s", lastErr)
|
|
}
|
|
}
|
|
defer cleanup(objs)
|
|
|
|
for _, tc := range tcs {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
objs := init()
|
|
|
|
err := driver.Delete(context.Background(), tc.delete)
|
|
|
|
if tc.err != nil {
|
|
if err == nil {
|
|
t.Fatalf("expected error")
|
|
}
|
|
if !tc.err(err) {
|
|
t.Fatalf("error does not match expected: %s", err)
|
|
}
|
|
}
|
|
if tc.err == nil && err != nil {
|
|
t.Fatalf("unexpected error: %s", err)
|
|
}
|
|
|
|
var issues []string
|
|
|
|
// validate all files expected to be deleted are deleted
|
|
// and all files not marked for deletion still remain
|
|
expected := tc.expected
|
|
isExpected := func(path string) bool {
|
|
for _, epath := range expected {
|
|
if epath == path {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
for _, path := range objs {
|
|
stat, err := driver.Stat(context.Background(), path)
|
|
if err != nil {
|
|
switch err.(type) {
|
|
case storagedriver.PathNotFoundError:
|
|
if !isExpected(path) {
|
|
issues = append(issues, fmt.Sprintf("unexpected path was deleted: %s", path))
|
|
}
|
|
// path was deleted & was supposed to be
|
|
continue
|
|
}
|
|
t.Fatalf("stat: %s", err)
|
|
}
|
|
if stat.IsDir() {
|
|
// for special cases where an object path has subpaths (eg /file1)
|
|
// once /file1 is deleted it's now a directory according to stat
|
|
continue
|
|
}
|
|
if isExpected(path) {
|
|
issues = append(issues, fmt.Sprintf("expected path was not deleted: %s", path))
|
|
}
|
|
}
|
|
|
|
if len(issues) > 0 {
|
|
t.Fatalf(strings.Join(issues, "; \n\t"))
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestOverThousandBlobs(t *testing.T) {
|
|
if skipS3() != "" {
|
|
t.Skip(skipS3())
|
|
}
|
|
|
|
rootDir, err := ioutil.TempDir("", "driver-")
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating temporary directory: %v", err)
|
|
}
|
|
defer os.Remove(rootDir)
|
|
|
|
standardDriver, err := s3DriverConstructor(rootDir, s3.StorageClassStandard)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating driver with standard storage: %v", err)
|
|
}
|
|
|
|
ctx := context.Background()
|
|
for i := 0; i < 1005; i++ {
|
|
filename := "/thousandfiletest/file" + strconv.Itoa(i)
|
|
contents := []byte("contents")
|
|
err = standardDriver.PutContent(ctx, filename, contents)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating content: %v", err)
|
|
}
|
|
}
|
|
|
|
// cant actually verify deletion because read-after-delete is inconsistent, but can ensure no errors
|
|
err = standardDriver.Delete(ctx, "/thousandfiletest")
|
|
if err != nil {
|
|
t.Fatalf("unexpected error deleting thousand files: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestMoveWithMultipartCopy(t *testing.T) {
|
|
if skipS3() != "" {
|
|
t.Skip(skipS3())
|
|
}
|
|
|
|
rootDir, err := ioutil.TempDir("", "driver-")
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating temporary directory: %v", err)
|
|
}
|
|
defer os.Remove(rootDir)
|
|
|
|
d, err := s3DriverConstructor(rootDir, s3.StorageClassStandard)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating driver: %v", err)
|
|
}
|
|
|
|
ctx := context.Background()
|
|
sourcePath := "/source"
|
|
destPath := "/dest"
|
|
|
|
defer d.Delete(ctx, sourcePath)
|
|
defer d.Delete(ctx, destPath)
|
|
|
|
// An object larger than d's MultipartCopyThresholdSize will cause d.Move() to perform a multipart copy.
|
|
multipartCopyThresholdSize := d.baseEmbed.Base.StorageDriver.(*driver).MultipartCopyThresholdSize
|
|
contents := make([]byte, 2*multipartCopyThresholdSize)
|
|
rand.Read(contents)
|
|
|
|
err = d.PutContent(ctx, sourcePath, contents)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating content: %v", err)
|
|
}
|
|
|
|
err = d.Move(ctx, sourcePath, destPath)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error moving file: %v", err)
|
|
}
|
|
|
|
received, err := d.GetContent(ctx, destPath)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error getting content: %v", err)
|
|
}
|
|
if !bytes.Equal(contents, received) {
|
|
t.Fatal("content differs")
|
|
}
|
|
|
|
_, err = d.GetContent(ctx, sourcePath)
|
|
switch err.(type) {
|
|
case storagedriver.PathNotFoundError:
|
|
default:
|
|
t.Fatalf("unexpected error getting content: %v", err)
|
|
}
|
|
}
|