TrueCloudLab/distribution
13
0
Fork 3
Code Issues 2 Pull requests Projects Releases 2 Packages Wiki Activity Actions

Uses IsTruncated and NextMarker for S3 list internal pagination

Browse source
This commit is contained in:
Brian Bland 2014-11-03 16:20:38 -08:00
parent b522fbd675
commit 43716a2850
1 changed files with 11 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
storagedriver/s3/s3.go
View file

@ -161,7 +161,6 @@ func (d *S3Driver) WriteStream(path string, offset, size uint64, reader io.ReadC
} else { } else {
part, err := multi.PutPart(int(partNumber), bytes.NewReader(buf[0:bytesRead])) part, err := multi.PutPart(int(partNumber), bytes.NewReader(buf[0:bytesRead]))
if err != nil { if err != nil {
return err return err
} }
@ -192,7 +191,10 @@ func (d *S3Driver) ResumeWritePosition(path string) (uint64, error) {
} }
func (d *S3Driver) List(prefix string) ([]string, error) { func (d *S3Driver) List(prefix string) ([]string, error) {
listResponse, err := d.Bucket.List(prefix+"/", "/", "", listPartsMax) if prefix[len(prefix)-1] != '/' {
prefix = prefix + "/"
}
listResponse, err := d.Bucket.List(prefix, "/", "", listPartsMax)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -200,7 +202,7 @@ func (d *S3Driver) List(prefix string) ([]string, error) {
files := []string{} files := []string{}
directories := []string{} directories := []string{}
for len(listResponse.Contents) > 0 || len(listResponse.CommonPrefixes) > 0 { for {
for _, key := range listResponse.Contents { for _, key := range listResponse.Contents {
files = append(files, key.Key) files = append(files, key.Key)
} }
@ -209,28 +211,14 @@ func (d *S3Driver) List(prefix string) ([]string, error) {
directories = append(directories, commonPrefix[0:len(commonPrefix)-1]) directories = append(directories, commonPrefix[0:len(commonPrefix)-1])
} }
lastFile := "" if listResponse.IsTruncated {
lastDirectory := "" listResponse, err = d.Bucket.List(prefix, "/", listResponse.NextMarker, listPartsMax)
lastMarker := ""
if len(files) > 0 {
lastFile = files[len(files)-1]
}
if len(directories) > 0 {
lastDirectory = directories[len(directories)-1] + "/"
}
if lastDirectory > lastFile {
lastMarker = lastDirectory
} else {
lastMarker = lastFile
}
listResponse, err = d.Bucket.List(prefix+"/", "/", lastMarker, listPartsMax)
if err != nil { if err != nil {
return nil, err return nil, err
} }
} else {
break
}
} }
return append(files, directories...), nil return append(files, directories...), nil