Merge pull request #3302 from ricardomaraschini/empty-prefixes-aws

Ignore self reference object on empty prefix
This commit is contained in:
Arko Dasgupta 2020-12-18 08:02:41 -08:00 committed by GitHub
commit 5ce1d85565
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 39 additions and 0 deletions

View file

@ -986,6 +986,11 @@ func (d *driver) doWalk(parentCtx context.Context, objectCount *int64, path, pre
} }
for _, file := range objects.Contents { for _, file := range objects.Contents {
// empty prefixes are listed as objects inside its own prefix.
// https://docs.aws.amazon.com/AmazonS3/latest/user-guide/using-folders.html
if strings.HasSuffix(*file.Key, "/") {
continue
}
walkInfos = append(walkInfos, walkInfoContainer{ walkInfos = append(walkInfos, walkInfoContainer{
FileInfoFields: storagedriver.FileInfoFields{ FileInfoFields: storagedriver.FileInfoFields{
IsDir: false, IsDir: false,

View file

@ -5,6 +5,7 @@ import (
"io/ioutil" "io/ioutil"
"math/rand" "math/rand"
"os" "os"
"reflect"
"strconv" "strconv"
"testing" "testing"
@ -164,6 +165,39 @@ func TestEmptyRootList(t *testing.T) {
} }
} }
// TestWalkEmptySubDirectory assures we list an empty sub directory only once when walking
// through its parent directory.
func TestWalkEmptySubDirectory(t *testing.T) {
if skipS3() != "" {
t.Skip(skipS3())
}
drv, err := s3DriverConstructor("", s3.StorageClassStandard)
if err != nil {
t.Fatalf("unexpected error creating rooted driver: %v", err)
}
// create an empty sub directory.
s3driver := drv.StorageDriver.(*driver)
if _, err := s3driver.S3.PutObject(&s3.PutObjectInput{
Bucket: aws.String(os.Getenv("S3_BUCKET")),
Key: aws.String("/testdir/emptydir/"),
}); err != nil {
t.Fatalf("error creating empty directory: %s", err)
}
bucketFiles := []string{}
s3driver.Walk(context.Background(), "/testdir", func(fileInfo storagedriver.FileInfo) error {
bucketFiles = append(bucketFiles, fileInfo.Path())
return nil
})
expected := []string{"/testdir/emptydir"}
if !reflect.DeepEqual(bucketFiles, expected) {
t.Errorf("expecting files %+v, found %+v instead", expected, bucketFiles)
}
}
func TestStorageClass(t *testing.T) { func TestStorageClass(t *testing.T) {
if skipS3() != "" { if skipS3() != "" {
t.Skip(skipS3()) t.Skip(skipS3())