TrueCloudLab/distribution
13
0
Fork 3
Code Issues 3 Pull requests Projects Releases 2 Packages Wiki Activity Actions

Merge pull request #3624 from milosgajdos/aws-s3-listv2

Browse source 
Update s3 ListObjects to V2 API
This commit is contained in:
Milos Gajdos 2022-04-22 13:34:13 +01:00 committed by GitHub
commit 27b5563245
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 94 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
registry/storage/driver/s3-aws/s3.go
View file

@ -703,7 +703,7 @@ func (d *driver) Writer(ctx context.Context, path string, appendParam bool) (sto
// Stat retrieves the FileInfo for the given path, including the current size
// in bytes and the creation time.
func (d *driver) Stat(ctx context.Context, path string) (storagedriver.FileInfo, error) {
resp, err := d.S3.ListObjects(&s3.ListObjectsInput{
resp, err := d.S3.ListObjectsV2(&s3.ListObjectsV2Input{
Bucket: aws.String(d.Bucket),
Prefix: aws.String(d.s3Path(path)),
MaxKeys: aws.Int64(1),
@ -748,7 +748,7 @@ func (d *driver) List(ctx context.Context, opath string) ([]string, error) {
prefix = "/"
}
resp, err := d.S3.ListObjects(&s3.ListObjectsInput{
resp, err := d.S3.ListObjectsV2(&s3.ListObjectsV2Input{
Bucket: aws.String(d.Bucket),
Prefix: aws.String(d.s3Path(path)),
Delimiter: aws.String("/"),
@ -772,12 +772,12 @@ func (d *driver) List(ctx context.Context, opath string) ([]string, error) {
}
if *resp.IsTruncated {
resp, err = d.S3.ListObjects(&s3.ListObjectsInput{
Bucket: aws.String(d.Bucket),
Prefix: aws.String(d.s3Path(path)),
Delimiter: aws.String("/"),
MaxKeys: aws.Int64(listMax),
Marker: resp.NextMarker,
resp, err = d.S3.ListObjectsV2(&s3.ListObjectsV2Input{
Bucket: aws.String(d.Bucket),
Prefix: aws.String(d.s3Path(path)),
Delimiter: aws.String("/"),
MaxKeys: aws.Int64(listMax),
ContinuationToken: resp.NextContinuationToken,
})
if err != nil {
return nil, err
@ -926,14 +926,14 @@ func (d *driver) Delete(ctx context.Context, path string) error {
// list objects under the given path as a subpath (suffix with slash "/")
s3Path := d.s3Path(path) + "/"
listObjectsInput := &s3.ListObjectsInput{
listObjectsInput := &s3.ListObjectsV2Input{
Bucket: aws.String(d.Bucket),
Prefix: aws.String(s3Path),
}
ListLoop:
for {
// list all the objects
resp, err := d.S3.ListObjects(listObjectsInput)
resp, err := d.S3.ListObjectsV2(listObjectsInput)
// resp.Contents can only be empty on the first call
// if there were no more results to return after the first call, resp.IsTruncated would have been false
@ -949,7 +949,7 @@ ListLoop:
}
// resp.Contents must have at least one element or we would have returned not found
listObjectsInput.Marker = resp.Contents[len(resp.Contents)-1].Key
listObjectsInput.StartAfter = resp.Contents[len(resp.Contents)-1].Key
// from the s3 api docs, IsTruncated "specifies whether (true) or not (false) all of the results were returned"
// if everything has been returned, break

83
registry/storage/driver/s3-aws/s3_test.go
View file

@ -7,7 +7,9 @@ import (
"io/ioutil"
"math/rand"
"os"
"path"
"reflect"
"sort"
"strconv"
"strings"
"testing"
@ -779,6 +781,87 @@ func TestMoveWithMultipartCopy(t *testing.T) {
}
}
func TestListObjectsV2(t *testing.T) {
rootDir, err := ioutil.TempDir("", "driver-")
if err != nil {
t.Fatalf("unexpected error creating temporary directory: %v", err)
}
defer os.Remove(rootDir)
d, err := s3DriverConstructor(rootDir, s3.StorageClassStandard)
if err != nil {
t.Fatalf("unexpected error creating driver: %v", err)
}
ctx := context.Background()
n := 6
prefix := "/test-list-objects-v2"
var filePaths []string
for i := 0; i < n; i++ {
filePaths = append(filePaths, fmt.Sprintf("%s/%d", prefix, i))
}
for _, path := range filePaths {
if err := d.PutContent(ctx, path, []byte(path)); err != nil {
t.Fatalf("unexpected error putting content: %v", err)
}
}
info, err := d.Stat(ctx, filePaths[0])
if err != nil {
t.Fatalf("unexpected error stating: %v", err)
}
if info.IsDir() || info.Size() != int64(len(filePaths[0])) || info.Path() != filePaths[0] {
t.Fatal("unexcepted state info")
}
subDirPath := prefix + "/sub/0"
if err := d.PutContent(ctx, subDirPath, []byte(subDirPath)); err != nil {
t.Fatalf("unexpected error putting content: %v", err)
}
subPaths := append(filePaths, path.Dir(subDirPath))
result, err := d.List(ctx, prefix)
if err != nil {
t.Fatalf("unexpected error listing: %v", err)
}
sort.Strings(subPaths)
sort.Strings(result)
if !reflect.DeepEqual(subPaths, result) {
t.Fatalf("unexpected list result")
}
var walkPaths []string
if err := d.Walk(ctx, prefix, func(fileInfo storagedriver.FileInfo) error {
walkPaths = append(walkPaths, fileInfo.Path())
if fileInfo.Path() == path.Dir(subDirPath) {
if !fileInfo.IsDir() {
t.Fatalf("unexpected walking file info")
}
} else {
if fileInfo.IsDir() || fileInfo.Size() != int64(len(fileInfo.Path())) {
t.Fatalf("unexpected walking file info")
}
}
return nil
}); err != nil {
t.Fatalf("unexpected error walking: %v", err)
}
subPaths = append(subPaths, subDirPath)
sort.Strings(walkPaths)
sort.Strings(subPaths)
if !reflect.DeepEqual(subPaths, walkPaths) {
t.Fatalf("unexpected walking paths")
}
if err := d.Delete(ctx, prefix); err != nil {
t.Fatalf("unexpected error deleting: %v", err)
}
}
func compareWalked(t *testing.T, expected, walked []string) {
if len(walked) != len(expected) {
t.Fatalf("Mismatch number of fileInfo walked %d expected %d; walked %s; expected %s;", len(walked), len(expected), walked, expected)