alexvanin/rclone
1
0
Fork 0
forked from TrueCloudLab/rclone
Code Pull requests Activity

s3: refactor f.list() to take an options struct as it had too many parameters

Browse source
This commit is contained in:
Nick Craig-Wood 2022-07-27 10:46:28 +01:00
parent 81d242473a
commit 1542a979f9
1 changed files with 66 additions and 33 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

99
backend/s3/s3.go
View file

@ -2704,7 +2704,13 @@ func (f *Fs) getMetaDataListing(ctx context.Context, wantRemote string) (info *s
return nil, nil, nil return nil, nil, nil
} }
err = f.list(ctx, bucket, bucketPath, "", false, true, f.opt.Versions, false, true, func(gotRemote string, object *s3.Object, objectVersionID *string, isDirectory bool) error { err = f.list(ctx, listOpt{
bucket: bucket,
directory: bucketPath,
recurse: true,
withVersions: f.opt.Versions,
findFile: true,
}, func(gotRemote string, object *s3.Object, objectVersionID *string, isDirectory bool) error {
if isDirectory { if isDirectory {
return nil return nil
} }
@ -3007,25 +3013,31 @@ type listFn func(remote string, object *s3.Object, versionID *string, isDirector
// listFn should return it to end the iteration with no errors. // listFn should return it to end the iteration with no errors.
var errEndList = errors.New("end list") var errEndList = errors.New("end list")
// list lists the objects into the function supplied from // list options
// the bucket and directory supplied. The remote has prefix type listOpt struct {
// removed from it and if addBucket is set then it adds the bucket string // bucket to list
// bucket to the start. directory string // directory with bucket
// prefix string // prefix to remove from listing
// Set recurse to read sub directories addBucket bool // if set, the bucket is added to the start of the remote
// recurse bool // if set, recurse to read sub directories
// if findFile is set it will look for files called (bucket, directory) withVersions bool // if set, versions are produced
func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBucket bool, recurse bool, withVersions bool, hidden bool, findFile bool, fn listFn) error { hidden bool // if set, return delete markers as objects with size == isDeleteMarker
if !findFile { findFile bool // if set, it will look for files called (bucket, directory)
if prefix != "" { }
prefix += "/"
// list lists the objects into the function supplied with the opt
// supplied.
func (f *Fs) list(ctx context.Context, opt listOpt, fn listFn) error {
if !opt.findFile {
if opt.prefix != "" {
opt.prefix += "/"
} }
if directory != "" { if opt.directory != "" {
directory += "/" opt.directory += "/"
} }
} }
delimiter := "" delimiter := ""
if !recurse { if !opt.recurse {
delimiter = "/" delimiter = "/"
} }
// URL encode the listings so we can use control characters in object names // URL encode the listings so we can use control characters in object names
@ -3045,9 +3057,9 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
// XML Syntax error is detected. // XML Syntax error is detected.
urlEncodeListings := f.opt.ListURLEncode.Value urlEncodeListings := f.opt.ListURLEncode.Value
req := s3.ListObjectsV2Input{ req := s3.ListObjectsV2Input{
Bucket: &bucket, Bucket: &opt.bucket,
Delimiter: &delimiter, Delimiter: &delimiter,
Prefix: &directory, Prefix: &opt.directory,
MaxKeys: &f.opt.ListChunk, MaxKeys: &f.opt.ListChunk,
} }
if f.opt.RequesterPays { if f.opt.RequesterPays {
@ -3055,7 +3067,7 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
} }
var listBucket bucketLister var listBucket bucketLister
switch { switch {
case withVersions: case opt.withVersions:
listBucket = f.newVersionsList(&req) listBucket = f.newVersionsList(&req)
case f.opt.ListVersion == 1: case f.opt.ListVersion == 1:
listBucket = f.newV1List(&req) listBucket = f.newV1List(&req)
@ -3068,7 +3080,7 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
var versionIDs []*string var versionIDs []*string
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
listBucket.URLEncodeListings(urlEncodeListings) listBucket.URLEncodeListings(urlEncodeListings)
resp, versionIDs, err = listBucket.List(ctx, hidden) resp, versionIDs, err = listBucket.List(ctx, opt.hidden)
if err != nil && !urlEncodeListings { if err != nil && !urlEncodeListings {
if awsErr, ok := err.(awserr.RequestFailure); ok { if awsErr, ok := err.(awserr.RequestFailure); ok {
if origErr := awsErr.OrigErr(); origErr != nil { if origErr := awsErr.OrigErr(); origErr != nil {
@ -3095,14 +3107,14 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
if reqErr, ok := err.(awserr.RequestFailure); ok { if reqErr, ok := err.(awserr.RequestFailure); ok {
// 301 if wrong region for bucket // 301 if wrong region for bucket
if reqErr.StatusCode() == http.StatusMovedPermanently { if reqErr.StatusCode() == http.StatusMovedPermanently {
fs.Errorf(f, "Can't change region for bucket %q with no bucket specified", bucket) fs.Errorf(f, "Can't change region for bucket %q with no bucket specified", opt.bucket)
return nil return nil
} }
} }
} }
return err return err
} }
if !recurse { if !opt.recurse {
for _, commonPrefix := range resp.CommonPrefixes { for _, commonPrefix := range resp.CommonPrefixes {
if commonPrefix.Prefix == nil { if commonPrefix.Prefix == nil {
fs.Logf(f, "Nil common prefix received") fs.Logf(f, "Nil common prefix received")
@ -3117,13 +3129,13 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
} }
} }
remote = f.opt.Enc.ToStandardPath(remote) remote = f.opt.Enc.ToStandardPath(remote)
if !strings.HasPrefix(remote, prefix) { if !strings.HasPrefix(remote, opt.prefix) {
fs.Logf(f, "Odd name received %q", remote) fs.Logf(f, "Odd name received %q", remote)
continue continue
} }
remote = remote[len(prefix):] remote = remote[len(opt.prefix):]
if addBucket { if opt.addBucket {
remote = path.Join(bucket, remote) remote = path.Join(opt.bucket, remote)
} }
remote = strings.TrimSuffix(remote, "/") remote = strings.TrimSuffix(remote, "/")
err = fn(remote, &s3.Object{Key: &remote}, nil, true) err = fn(remote, &s3.Object{Key: &remote}, nil, true)
@ -3145,14 +3157,14 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
} }
} }
remote = f.opt.Enc.ToStandardPath(remote) remote = f.opt.Enc.ToStandardPath(remote)
if !strings.HasPrefix(remote, prefix) { if !strings.HasPrefix(remote, opt.prefix) {
fs.Logf(f, "Odd name received %q", remote) fs.Logf(f, "Odd name received %q", remote)
continue continue
} }
remote = remote[len(prefix):] remote = remote[len(opt.prefix):]
isDirectory := remote == "" || strings.HasSuffix(remote, "/") isDirectory := remote == "" || strings.HasSuffix(remote, "/")
if addBucket { if opt.addBucket {
remote = path.Join(bucket, remote) remote = path.Join(opt.bucket, remote)
} }
// is this a directory marker? // is this a directory marker?
if isDirectory && object.Size != nil && *object.Size == 0 { if isDirectory && object.Size != nil && *object.Size == 0 {
@ -3197,7 +3209,13 @@ func (f *Fs) itemToDirEntry(ctx context.Context, remote string, object *s3.Objec
// listDir lists files and directories to out // listDir lists files and directories to out
func (f *Fs) listDir(ctx context.Context, bucket, directory, prefix string, addBucket bool) (entries fs.DirEntries, err error) { func (f *Fs) listDir(ctx context.Context, bucket, directory, prefix string, addBucket bool) (entries fs.DirEntries, err error) {
// List the objects and directories // List the objects and directories
err = f.list(ctx, bucket, directory, prefix, addBucket, false, f.opt.Versions, false, false, func(remote string, object *s3.Object, versionID *string, isDirectory bool) error { err = f.list(ctx, listOpt{
bucket: bucket,
directory: directory,
prefix: prefix,
addBucket: addBucket,
withVersions: f.opt.Versions,
}, func(remote string, object *s3.Object, versionID *string, isDirectory bool) error {
entry, err := f.itemToDirEntry(ctx, remote, object, versionID, isDirectory) entry, err := f.itemToDirEntry(ctx, remote, object, versionID, isDirectory)
if err != nil { if err != nil {
return err return err
@ -3275,7 +3293,14 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
bucket, directory := f.split(dir) bucket, directory := f.split(dir)
list := walk.NewListRHelper(callback) list := walk.NewListRHelper(callback)
listR := func(bucket, directory, prefix string, addBucket bool) error { listR := func(bucket, directory, prefix string, addBucket bool) error {
return f.list(ctx, bucket, directory, prefix, addBucket, true, f.opt.Versions, false, false, func(remote string, object *s3.Object, versionID *string, isDirectory bool) error { return f.list(ctx, listOpt{
bucket: bucket,
directory: directory,
prefix: prefix,
addBucket: addBucket,
recurse: true,
withVersions: f.opt.Versions,
}, func(remote string, object *s3.Object, versionID *string, isDirectory bool) error {
entry, err := f.itemToDirEntry(ctx, remote, object, versionID, isDirectory) entry, err := f.itemToDirEntry(ctx, remote, object, versionID, isDirectory)
if err != nil { if err != nil {
return err return err
@ -4077,7 +4102,15 @@ func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool) error {
go func() { go func() {
delErr <- operations.DeleteFiles(ctx, delChan) delErr <- operations.DeleteFiles(ctx, delChan)
}() }()
checkErr(f.list(ctx, bucket, directory, f.rootDirectory, f.rootBucket == "", true, versioned, true, false, func(remote string, object *s3.Object, versionID *string, isDirectory bool) error { checkErr(f.list(ctx, listOpt{
bucket: bucket,
directory: directory,
prefix: f.rootDirectory,
addBucket: f.rootBucket == "",
recurse: true,
withVersions: versioned,
hidden: true,
}, func(remote string, object *s3.Object, versionID *string, isDirectory bool) error {
if isDirectory { if isDirectory {
return nil return nil
} }