fix: paginate through s3 multipart uploads
Signed-off-by: Jeremy THERIN <jtherin@scaleway.com>pull/3519/head
parent
1563384b69
commit
7736319f2e
|
@ -558,40 +558,62 @@ func (d *driver) Writer(ctx context.Context, path string, appendParam bool) (sto
|
||||||
}
|
}
|
||||||
return d.newWriter(key, *resp.UploadId, nil), nil
|
return d.newWriter(key, *resp.UploadId, nil), nil
|
||||||
}
|
}
|
||||||
resp, err := d.S3.ListMultipartUploads(&s3.ListMultipartUploadsInput{
|
|
||||||
|
listMultipartUploadsInput := &s3.ListMultipartUploadsInput{
|
||||||
Bucket: aws.String(d.Bucket),
|
Bucket: aws.String(d.Bucket),
|
||||||
Prefix: aws.String(key),
|
Prefix: aws.String(key),
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return nil, parseError(path, err)
|
|
||||||
}
|
}
|
||||||
var allParts []*s3.Part
|
for {
|
||||||
for _, multi := range resp.Uploads {
|
resp, err := d.S3.ListMultipartUploads(listMultipartUploadsInput)
|
||||||
if key != *multi.Key {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
resp, err := d.S3.ListParts(&s3.ListPartsInput{
|
|
||||||
Bucket: aws.String(d.Bucket),
|
|
||||||
Key: aws.String(key),
|
|
||||||
UploadId: multi.UploadId,
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, parseError(path, err)
|
return nil, parseError(path, err)
|
||||||
}
|
}
|
||||||
allParts = append(allParts, resp.Parts...)
|
|
||||||
for *resp.IsTruncated {
|
// resp.Uploads can only be empty on the first call
|
||||||
resp, err = d.S3.ListParts(&s3.ListPartsInput{
|
// if there were no more results to return after the first call, resp.IsTruncated would have been false
|
||||||
Bucket: aws.String(d.Bucket),
|
// and the loop would be exited without recalling ListMultipartUploads
|
||||||
Key: aws.String(key),
|
if len(resp.Uploads) == 0 {
|
||||||
UploadId: multi.UploadId,
|
break
|
||||||
PartNumberMarker: resp.NextPartNumberMarker,
|
}
|
||||||
|
|
||||||
|
var allParts []*s3.Part
|
||||||
|
for _, multi := range resp.Uploads {
|
||||||
|
if key != *multi.Key {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
partsList, err := d.S3.ListParts(&s3.ListPartsInput{
|
||||||
|
Bucket: aws.String(d.Bucket),
|
||||||
|
Key: aws.String(key),
|
||||||
|
UploadId: multi.UploadId,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, parseError(path, err)
|
return nil, parseError(path, err)
|
||||||
}
|
}
|
||||||
allParts = append(allParts, resp.Parts...)
|
allParts = append(allParts, partsList.Parts...)
|
||||||
|
for *resp.IsTruncated {
|
||||||
|
partsList, err = d.S3.ListParts(&s3.ListPartsInput{
|
||||||
|
Bucket: aws.String(d.Bucket),
|
||||||
|
Key: aws.String(key),
|
||||||
|
UploadId: multi.UploadId,
|
||||||
|
PartNumberMarker: partsList.NextPartNumberMarker,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, parseError(path, err)
|
||||||
|
}
|
||||||
|
allParts = append(allParts, partsList.Parts...)
|
||||||
|
}
|
||||||
|
return d.newWriter(key, *multi.UploadId, allParts), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// resp.NextUploadIdMarker must have at least one element or we would have returned not found
|
||||||
|
listMultipartUploadsInput.UploadIdMarker = resp.NextUploadIdMarker
|
||||||
|
|
||||||
|
// from the s3 api docs, IsTruncated "specifies whether (true) or not (false) all of the results were returned"
|
||||||
|
// if everything has been returned, break
|
||||||
|
if resp.IsTruncated == nil || !*resp.IsTruncated {
|
||||||
|
break
|
||||||
}
|
}
|
||||||
return d.newWriter(key, *multi.UploadId, allParts), nil
|
|
||||||
}
|
}
|
||||||
return nil, storagedriver.PathNotFoundError{Path: path}
|
return nil, storagedriver.PathNotFoundError{Path: path}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue