distribution/registry/storage/driver/s3-aws/s3_test.go
Benjamin Schanzel 8654a0ee45
Allow setting s3 forcepathstyle without regionendpoint
Currently, the `forcepathstyle` parameter for the s3 storage driver is
considered only if the `regionendpoint` parameter is set. Since setting
a region endpoint explicitly is discouraged with AWS s3, it is not clear
how to enforce path style URLs with AWS s3.
This also means, that the default value (true) only applies if a region
endpoint is configured.

This change makes sure we always forward the `forcepathstyle` parameter
to the aws-sdk if present in the config. This is a breaking change where
a `regionendpoint` is configured but no explicit `forcepathstyle` value
is set.

Signed-off-by: Benjamin Schanzel <benjamin.schanzel@bmw.de>
2024-04-08 12:45:26 +02:00

920 lines
23 KiB
Go

package s3
import (
"bytes"
"context"
"crypto/rand"
"errors"
"fmt"
"net/http"
"os"
"path"
"reflect"
"sort"
"strconv"
"strings"
"testing"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/distribution/distribution/v3/internal/dcontext"
storagedriver "github.com/distribution/distribution/v3/registry/storage/driver"
"github.com/distribution/distribution/v3/registry/storage/driver/testsuites"
)
var (
s3DriverConstructor func(rootDirectory, storageClass string) (*Driver, error)
skipCheck func(tb testing.TB)
)
func init() {
var (
accessKey = os.Getenv("AWS_ACCESS_KEY")
secretKey = os.Getenv("AWS_SECRET_KEY")
bucket = os.Getenv("S3_BUCKET")
encrypt = os.Getenv("S3_ENCRYPT")
keyID = os.Getenv("S3_KEY_ID")
secure = os.Getenv("S3_SECURE")
skipVerify = os.Getenv("S3_SKIP_VERIFY")
v4Auth = os.Getenv("S3_V4_AUTH")
region = os.Getenv("AWS_REGION")
objectACL = os.Getenv("S3_OBJECT_ACL")
regionEndpoint = os.Getenv("REGION_ENDPOINT")
forcePathStyle = os.Getenv("AWS_S3_FORCE_PATH_STYLE")
sessionToken = os.Getenv("AWS_SESSION_TOKEN")
useDualStack = os.Getenv("S3_USE_DUALSTACK")
accelerate = os.Getenv("S3_ACCELERATE")
logLevel = os.Getenv("S3_LOGLEVEL")
)
var err error
s3DriverConstructor = func(rootDirectory, storageClass string) (*Driver, error) {
encryptBool := false
if encrypt != "" {
encryptBool, err = strconv.ParseBool(encrypt)
if err != nil {
return nil, err
}
}
secureBool := true
if secure != "" {
secureBool, err = strconv.ParseBool(secure)
if err != nil {
return nil, err
}
}
skipVerifyBool := false
if skipVerify != "" {
skipVerifyBool, err = strconv.ParseBool(skipVerify)
if err != nil {
return nil, err
}
}
v4Bool := true
if v4Auth != "" {
v4Bool, err = strconv.ParseBool(v4Auth)
if err != nil {
return nil, err
}
}
forcePathStyleBool := false
if forcePathStyle != "" {
forcePathStyleBool, err = strconv.ParseBool(forcePathStyle)
if err != nil {
return nil, err
}
}
useDualStackBool := false
if useDualStack != "" {
useDualStackBool, err = strconv.ParseBool(useDualStack)
}
accelerateBool := true
if accelerate != "" {
accelerateBool, err = strconv.ParseBool(accelerate)
if err != nil {
return nil, err
}
}
parameters := DriverParameters{
accessKey,
secretKey,
bucket,
region,
regionEndpoint,
forcePathStyleBool,
encryptBool,
keyID,
secureBool,
skipVerifyBool,
v4Bool,
minChunkSize,
defaultMultipartCopyChunkSize,
defaultMultipartCopyMaxConcurrency,
defaultMultipartCopyThresholdSize,
rootDirectory,
storageClass,
driverName + "-test",
objectACL,
sessionToken,
useDualStackBool,
accelerateBool,
getS3LogLevelFromParam(logLevel),
}
return New(context.Background(), parameters)
}
// Skip S3 storage driver tests if environment variable parameters are not provided
skipCheck = func(tb testing.TB) {
tb.Helper()
if accessKey == "" || secretKey == "" || region == "" || bucket == "" || encrypt == "" {
tb.Skip("Must set AWS_ACCESS_KEY, AWS_SECRET_KEY, AWS_REGION, S3_BUCKET, and S3_ENCRYPT to run S3 tests")
}
}
}
func newDriverConstructor(tb testing.TB) testsuites.DriverConstructor {
root := tb.TempDir()
return func() (storagedriver.StorageDriver, error) {
return s3DriverConstructor(root, s3.StorageClassStandard)
}
}
func TestS3DriverSuite(t *testing.T) {
skipCheck(t)
testsuites.Driver(t, newDriverConstructor(t))
}
func BenchmarkS3DriverSuite(b *testing.B) {
skipCheck(b)
testsuites.BenchDriver(b, newDriverConstructor(b))
}
func TestEmptyRootList(t *testing.T) {
skipCheck(t)
validRoot := t.TempDir()
rootedDriver, err := s3DriverConstructor(validRoot, s3.StorageClassStandard)
if err != nil {
t.Fatalf("unexpected error creating rooted driver: %v", err)
}
emptyRootDriver, err := s3DriverConstructor("", s3.StorageClassStandard)
if err != nil {
t.Fatalf("unexpected error creating empty root driver: %v", err)
}
slashRootDriver, err := s3DriverConstructor("/", s3.StorageClassStandard)
if err != nil {
t.Fatalf("unexpected error creating slash root driver: %v", err)
}
filename := "/test"
contents := []byte("contents")
ctx := dcontext.Background()
err = rootedDriver.PutContent(ctx, filename, contents)
if err != nil {
t.Fatalf("unexpected error creating content: %v", err)
}
// nolint:errcheck
defer rootedDriver.Delete(ctx, filename)
keys, _ := emptyRootDriver.List(ctx, "/")
for _, path := range keys {
if !storagedriver.PathRegexp.MatchString(path) {
t.Fatalf("unexpected string in path: %q != %q", path, storagedriver.PathRegexp)
}
}
keys, _ = slashRootDriver.List(ctx, "/")
for _, p := range keys {
if !storagedriver.PathRegexp.MatchString(p) {
t.Fatalf("unexpected string in path: %q != %q", p, storagedriver.PathRegexp)
}
}
}
func TestClientTransport(t *testing.T) {
skipCheck(t)
testCases := []struct {
skipverify bool
}{
{true},
{false},
}
for _, tc := range testCases {
// NOTE(milosgajdos): we cannot simply reuse s3DriverConstructor
// because s3DriverConstructor is initialized in init() using the process
// env vars: we can not override S3_SKIP_VERIFY env var with t.Setenv
params := map[string]interface{}{
"region": os.Getenv("AWS_REGION"),
"bucket": os.Getenv("S3_BUCKET"),
"skipverify": tc.skipverify,
}
t.Run(fmt.Sprintf("SkipVerify %v", tc.skipverify), func(t *testing.T) {
drv, err := FromParameters(context.TODO(), params)
if err != nil {
t.Fatalf("failed to create driver: %v", err)
}
s3drv := drv.baseEmbed.Base.StorageDriver.(*driver)
if tc.skipverify {
tr, ok := s3drv.S3.Client.Config.HTTPClient.Transport.(*http.Transport)
if !ok {
t.Fatal("unexpected driver transport")
}
if !tr.TLSClientConfig.InsecureSkipVerify {
t.Errorf("unexpected TLS Config. Expected InsecureSkipVerify: %v, got %v",
tc.skipverify,
tr.TLSClientConfig.InsecureSkipVerify)
}
// make sure the proxy is always set
if tr.Proxy == nil {
t.Fatal("missing HTTP transport proxy config")
}
return
}
// if tc.skipverify is false we do not override the driver
// HTTP client transport and leave it to the AWS SDK.
if s3drv.S3.Client.Config.HTTPClient.Transport != nil {
t.Errorf("unexpected S3 driver client transport")
}
})
}
}
func TestStorageClass(t *testing.T) {
skipCheck(t)
rootDir := t.TempDir()
contents := []byte("contents")
ctx := dcontext.Background()
// We don't need to test all the storage classes, just that its selectable.
// The first 3 are common to AWS and MinIO, so use those.
for _, storageClass := range s3StorageClasses[:3] {
filename := "/test-" + storageClass
s3Driver, err := s3DriverConstructor(rootDir, storageClass)
if err != nil {
t.Fatalf("unexpected error creating driver with storage class %v: %v", storageClass, err)
}
// Can only test outposts if using s3 outposts
if storageClass == s3.StorageClassOutposts {
continue
}
err = s3Driver.PutContent(ctx, filename, contents)
if err != nil {
t.Fatalf("unexpected error creating content with storage class %v: %v", storageClass, err)
}
// nolint:errcheck
defer s3Driver.Delete(ctx, filename)
driverUnwrapped := s3Driver.Base.StorageDriver.(*driver)
resp, err := driverUnwrapped.S3.GetObject(&s3.GetObjectInput{
Bucket: aws.String(driverUnwrapped.Bucket),
Key: aws.String(driverUnwrapped.s3Path(filename)),
})
if err != nil {
t.Fatalf("unexpected error retrieving file with storage class %v: %v", storageClass, err)
}
defer resp.Body.Close()
// Amazon only populates this header value for non-standard storage classes
if storageClass == noStorageClass {
// We haven't specified a storage class so we can't confirm what it is
} else if storageClass == s3.StorageClassStandard && resp.StorageClass != nil {
t.Fatalf(
"unexpected response storage class for file with storage class %v: %v",
storageClass,
*resp.StorageClass,
)
} else if storageClass != s3.StorageClassStandard && resp.StorageClass == nil {
t.Fatalf(
"unexpected response storage class for file with storage class %v: %v",
storageClass,
s3.StorageClassStandard,
)
} else if storageClass != s3.StorageClassStandard && storageClass != *resp.StorageClass {
t.Fatalf(
"unexpected response storage class for file with storage class %v: %v",
storageClass,
*resp.StorageClass,
)
}
}
}
func TestDelete(t *testing.T) {
skipCheck(t)
rootDir := t.TempDir()
drvr, err := s3DriverConstructor(rootDir, s3.StorageClassStandard)
if err != nil {
t.Fatalf("unexpected error creating driver with standard storage: %v", err)
}
type errFn func(error) bool
type testCase struct {
name string
delete string
expected []string
// error validation function
err errFn
}
errPathNotFound := func(err error) bool {
if err == nil {
return false
}
switch err.(type) {
case storagedriver.PathNotFoundError:
return true
}
return false
}
errInvalidPath := func(err error) bool {
if err == nil {
return false
}
switch err.(type) {
case storagedriver.InvalidPathError:
return true
}
return false
}
objs := []string{
"/file1",
"/file1-2",
"/folder1/file1",
"/folder2/file1",
"/folder3/file1",
"/folder3/subfolder1/subfolder1/file1",
"/folder3/subfolder2/subfolder1/file1",
"/folder4/file1",
"/folder1-v2/file1",
"/folder1-v2/subfolder1/file1",
}
tcs := []testCase{
{
name: "delete folder1",
delete: "/folder1",
expected: []string{
"/folder1/file1",
},
},
{
name: "delete folder2",
delete: "/folder2",
expected: []string{
"/folder2/file1",
},
},
{
name: "delete folder3",
delete: "/folder3",
expected: []string{
"/folder3/file1",
"/folder3/subfolder1/subfolder1/file1",
"/folder3/subfolder2/subfolder1/file1",
},
},
{
name: "delete path that doesn't exist",
delete: "/path/does/not/exist",
expected: []string{},
err: errPathNotFound,
},
{
name: "delete path invalid: trailing slash",
delete: "/path/is/invalid/",
expected: []string{},
err: errInvalidPath,
},
{
name: "delete path invalid: trailing special character",
delete: "/path/is/invalid*",
expected: []string{},
err: errInvalidPath,
},
}
// create a test case for each file
for _, p := range objs {
tcs = append(tcs, testCase{
name: fmt.Sprintf("delete path:'%s'", p),
delete: p,
expected: []string{p},
})
}
init := func() []string {
// init file structure matching objs
var created []string
for _, p := range objs {
err := drvr.PutContent(dcontext.Background(), p, []byte("content "+p))
if err != nil {
fmt.Printf("unable to init file %s: %s\n", p, err)
continue
}
created = append(created, p)
}
return created
}
cleanup := func(objs []string) {
var lastErr error
for _, p := range objs {
err := drvr.Delete(dcontext.Background(), p)
if err != nil {
switch err.(type) {
case storagedriver.PathNotFoundError:
continue
}
lastErr = err
}
}
if lastErr != nil {
t.Fatalf("cleanup failed: %s", lastErr)
}
}
defer cleanup(objs)
for _, tc := range tcs {
t.Run(tc.name, func(t *testing.T) {
objs := init()
err := drvr.Delete(dcontext.Background(), tc.delete)
if tc.err != nil {
if err == nil {
t.Fatalf("expected error")
}
if !tc.err(err) {
t.Fatalf("error does not match expected: %s", err)
}
}
if tc.err == nil && err != nil {
t.Fatalf("unexpected error: %s", err)
}
var issues []string
// validate all files expected to be deleted are deleted
// and all files not marked for deletion still remain
expected := tc.expected
isExpected := func(path string) bool {
for _, epath := range expected {
if epath == path {
return true
}
}
return false
}
for _, path := range objs {
stat, err := drvr.Stat(dcontext.Background(), path)
if err != nil {
switch err.(type) {
case storagedriver.PathNotFoundError:
if !isExpected(path) {
issues = append(issues, fmt.Sprintf("unexpected path was deleted: %s", path))
}
// path was deleted & was supposed to be
continue
}
t.Fatalf("stat: %s", err)
}
if stat.IsDir() {
// for special cases where an object path has subpaths (eg /file1)
// once /file1 is deleted it's now a directory according to stat
continue
}
if isExpected(path) {
issues = append(issues, fmt.Sprintf("expected path was not deleted: %s", path))
}
}
if len(issues) > 0 {
t.Fatalf(strings.Join(issues, "; \n\t"))
}
})
}
}
func TestWalk(t *testing.T) {
skipCheck(t)
rootDir := t.TempDir()
drvr, err := s3DriverConstructor(rootDir, s3.StorageClassStandard)
if err != nil {
t.Fatalf("unexpected error creating driver with standard storage: %v", err)
}
fileset := []string{
"/file1",
"/folder1-suffix/file1",
"/folder1/file1",
"/folder2/file1",
"/folder3/subfolder1/subfolder1/file1",
"/folder3/subfolder2/subfolder1/file1",
"/folder4/file1",
}
// create file structure matching fileset above
created := make([]string, 0, len(fileset))
for _, p := range fileset {
err := drvr.PutContent(dcontext.Background(), p, []byte("content "+p))
if err != nil {
fmt.Printf("unable to create file %s: %s\n", p, err)
continue
}
created = append(created, p)
}
// cleanup
defer func() {
var lastErr error
for _, p := range created {
err := drvr.Delete(dcontext.Background(), p)
if err != nil {
_ = fmt.Errorf("cleanup failed for path %s: %s", p, err)
lastErr = err
}
}
if lastErr != nil {
t.Fatalf("cleanup failed: %s", err)
}
}()
noopFn := func(fileInfo storagedriver.FileInfo) error { return nil }
tcs := []struct {
name string
fn storagedriver.WalkFn
from string
options []func(*storagedriver.WalkOptions)
expected []string
err bool
}{
{
name: "walk all",
fn: noopFn,
expected: []string{
"/file1",
"/folder1-suffix",
"/folder1-suffix/file1",
"/folder1",
"/folder1/file1",
"/folder2",
"/folder2/file1",
"/folder3",
"/folder3/subfolder1",
"/folder3/subfolder1/subfolder1",
"/folder3/subfolder1/subfolder1/file1",
"/folder3/subfolder2",
"/folder3/subfolder2/subfolder1",
"/folder3/subfolder2/subfolder1/file1",
"/folder4",
"/folder4/file1",
},
},
{
name: "skip directory",
fn: func(fileInfo storagedriver.FileInfo) error {
if fileInfo.Path() == "/folder3" {
return storagedriver.ErrSkipDir
}
if strings.Contains(fileInfo.Path(), "/folder3") {
t.Fatalf("skipped dir %s and should not walk %s", "/folder3", fileInfo.Path())
}
return nil
},
expected: []string{
"/file1",
"/folder1-suffix",
"/folder1-suffix/file1",
"/folder1",
"/folder1/file1",
"/folder2",
"/folder2/file1",
"/folder3",
// folder 3 contents skipped
"/folder4",
"/folder4/file1",
},
},
{
name: "start late without from",
fn: noopFn,
options: []func(*storagedriver.WalkOptions){
storagedriver.WithStartAfterHint("/folder3/subfolder1/subfolder1/file1"),
},
expected: []string{
// start late
"/folder3",
"/folder3/subfolder2",
"/folder3/subfolder2/subfolder1",
"/folder3/subfolder2/subfolder1/file1",
"/folder4",
"/folder4/file1",
},
err: false,
},
{
name: "start late with from",
fn: noopFn,
from: "/folder3",
options: []func(*storagedriver.WalkOptions){
storagedriver.WithStartAfterHint("/folder3/subfolder1/subfolder1/file1"),
},
expected: []string{
// start late
"/folder3/subfolder2",
"/folder3/subfolder2/subfolder1",
"/folder3/subfolder2/subfolder1/file1",
},
err: false,
},
{
name: "start after from",
fn: noopFn,
from: "/folder1",
options: []func(*storagedriver.WalkOptions){
storagedriver.WithStartAfterHint("/folder2"),
},
expected: []string{},
err: false,
},
{
name: "start matches from",
fn: noopFn,
from: "/folder3",
options: []func(*storagedriver.WalkOptions){
storagedriver.WithStartAfterHint("/folder3"),
},
expected: []string{
"/folder3/subfolder1",
"/folder3/subfolder1/subfolder1",
"/folder3/subfolder1/subfolder1/file1",
"/folder3/subfolder2",
"/folder3/subfolder2/subfolder1",
"/folder3/subfolder2/subfolder1/file1",
},
err: false,
},
{
name: "start doesn't exist",
fn: noopFn,
from: "/folder3",
options: []func(*storagedriver.WalkOptions){
storagedriver.WithStartAfterHint("/folder3/notafolder/notafile"),
},
expected: []string{
"/folder3/subfolder1",
"/folder3/subfolder1/subfolder1",
"/folder3/subfolder1/subfolder1/file1",
"/folder3/subfolder2",
"/folder3/subfolder2/subfolder1",
"/folder3/subfolder2/subfolder1/file1",
},
err: false,
},
{
name: "stop early",
fn: func(fileInfo storagedriver.FileInfo) error {
if fileInfo.Path() == "/folder1/file1" {
return storagedriver.ErrFilledBuffer
}
return nil
},
expected: []string{
"/file1",
"/folder1-suffix",
"/folder1-suffix/file1",
"/folder1",
"/folder1/file1",
// stop early
},
err: false,
},
{
name: "error",
fn: func(fileInfo storagedriver.FileInfo) error {
return errors.New("foo")
},
expected: []string{
"/file1",
},
err: true,
},
{
name: "from folder",
fn: noopFn,
expected: []string{
"/folder1/file1",
},
from: "/folder1",
},
}
for _, tc := range tcs {
var walked []string
if tc.from == "" {
tc.from = "/"
}
t.Run(tc.name, func(t *testing.T) {
err := drvr.Walk(dcontext.Background(), tc.from, func(fileInfo storagedriver.FileInfo) error {
walked = append(walked, fileInfo.Path())
return tc.fn(fileInfo)
}, tc.options...)
if tc.err && err == nil {
t.Fatalf("expected err")
}
if !tc.err && err != nil {
t.Fatalf(err.Error())
}
compareWalked(t, tc.expected, walked)
})
}
}
func TestOverThousandBlobs(t *testing.T) {
skipCheck(t)
rootDir := t.TempDir()
standardDriver, err := s3DriverConstructor(rootDir, s3.StorageClassStandard)
if err != nil {
t.Fatalf("unexpected error creating driver with standard storage: %v", err)
}
ctx := dcontext.Background()
for i := 0; i < 1005; i++ {
filename := "/thousandfiletest/file" + strconv.Itoa(i)
contents := []byte("contents")
err = standardDriver.PutContent(ctx, filename, contents)
if err != nil {
t.Fatalf("unexpected error creating content: %v", err)
}
}
// cant actually verify deletion because read-after-delete is inconsistent, but can ensure no errors
err = standardDriver.Delete(ctx, "/thousandfiletest")
if err != nil {
t.Fatalf("unexpected error deleting thousand files: %v", err)
}
}
func TestMoveWithMultipartCopy(t *testing.T) {
skipCheck(t)
rootDir := t.TempDir()
d, err := s3DriverConstructor(rootDir, s3.StorageClassStandard)
if err != nil {
t.Fatalf("unexpected error creating driver: %v", err)
}
ctx := dcontext.Background()
sourcePath := "/source"
destPath := "/dest"
// nolint:errcheck
defer d.Delete(ctx, sourcePath)
// nolint:errcheck
defer d.Delete(ctx, destPath)
// An object larger than d's MultipartCopyThresholdSize will cause d.Move() to perform a multipart copy.
multipartCopyThresholdSize := d.baseEmbed.Base.StorageDriver.(*driver).MultipartCopyThresholdSize
contents := make([]byte, 2*multipartCopyThresholdSize)
if _, err := rand.Read(contents); err != nil {
t.Fatalf("unexpected error creating content: %v", err)
}
err = d.PutContent(ctx, sourcePath, contents)
if err != nil {
t.Fatalf("unexpected error writing content: %v", err)
}
err = d.Move(ctx, sourcePath, destPath)
if err != nil {
t.Fatalf("unexpected error moving file: %v", err)
}
received, err := d.GetContent(ctx, destPath)
if err != nil {
t.Fatalf("unexpected error getting content: %v", err)
}
if !bytes.Equal(contents, received) {
t.Fatal("content differs")
}
_, err = d.GetContent(ctx, sourcePath)
switch err.(type) {
case storagedriver.PathNotFoundError:
default:
t.Fatalf("unexpected error getting content: %v", err)
}
}
func TestListObjectsV2(t *testing.T) {
skipCheck(t)
rootDir := t.TempDir()
d, err := s3DriverConstructor(rootDir, s3.StorageClassStandard)
if err != nil {
t.Fatalf("unexpected error creating driver: %v", err)
}
ctx := dcontext.Background()
n := 6
prefix := "/test-list-objects-v2"
var filePaths []string
for i := 0; i < n; i++ {
filePaths = append(filePaths, fmt.Sprintf("%s/%d", prefix, i))
}
for _, p := range filePaths {
if err := d.PutContent(ctx, p, []byte(p)); err != nil {
t.Fatalf("unexpected error putting content: %v", err)
}
}
info, err := d.Stat(ctx, filePaths[0])
if err != nil {
t.Fatalf("unexpected error stating: %v", err)
}
if info.IsDir() || info.Size() != int64(len(filePaths[0])) || info.Path() != filePaths[0] {
t.Fatal("unexcepted state info")
}
subDirPath := prefix + "/sub/0"
if err := d.PutContent(ctx, subDirPath, []byte(subDirPath)); err != nil {
t.Fatalf("unexpected error putting content: %v", err)
}
subPaths := append(filePaths, path.Dir(subDirPath))
result, err := d.List(ctx, prefix)
if err != nil {
t.Fatalf("unexpected error listing: %v", err)
}
sort.Strings(subPaths)
sort.Strings(result)
if !reflect.DeepEqual(subPaths, result) {
t.Fatalf("unexpected list result")
}
var walkPaths []string
if err := d.Walk(ctx, prefix, func(fileInfo storagedriver.FileInfo) error {
walkPaths = append(walkPaths, fileInfo.Path())
if fileInfo.Path() == path.Dir(subDirPath) {
if !fileInfo.IsDir() {
t.Fatalf("unexpected walking file info")
}
} else {
if fileInfo.IsDir() || fileInfo.Size() != int64(len(fileInfo.Path())) {
t.Fatalf("unexpected walking file info")
}
}
return nil
}); err != nil {
t.Fatalf("unexpected error walking: %v", err)
}
subPaths = append(subPaths, subDirPath)
sort.Strings(walkPaths)
sort.Strings(subPaths)
if !reflect.DeepEqual(subPaths, walkPaths) {
t.Fatalf("unexpected walking paths")
}
if err := d.Delete(ctx, prefix); err != nil {
t.Fatalf("unexpected error deleting: %v", err)
}
}
func compareWalked(t *testing.T, expected, walked []string) {
if len(walked) != len(expected) {
t.Fatalf("Mismatch number of fileInfo walked %d expected %d; walked %s; expected %s;", len(walked), len(expected), walked, expected)
}
for i := range walked {
if walked[i] != expected[i] {
t.Fatalf("walked in unexpected order: expected %s; walked %s", expected, walked)
}
}
}