alexvanin/distribution
1
0
Fork 0
forked from TrueCloudLab/distribution
Code Pull requests Activity

storagedriver/s3: Fixed a Delete noop edgecase

Browse source 
Delete was not working when the subpath immediately followed the given path started with an ascii lower than "/" such as dash "-" and underscore "_" and requests no files to be deleted.

(cherry picked from commit 5d8fa0ce94b68cce70237805db92cdd8d40de282)
Signed-off-by: Collin Shoop <cshoop@digitalocean.com>
This commit is contained in:
Collin Shoop 2021-08-11 12:17:24 -04:00
parent 05a258e711
commit 03f9eb3a18
1 changed files with 21 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
registry/storage/driver/s3-aws/s3.go
View file

@ -806,7 +806,21 @@ func min(a, b int) int {
// We must be careful since S3 does not guarantee read after delete consistency // We must be careful since S3 does not guarantee read after delete consistency
func (d *driver) Delete(ctx context.Context, path string) error { func (d *driver) Delete(ctx context.Context, path string) error {
s3Objects := make([]*s3.ObjectIdentifier, 0, listMax) s3Objects := make([]*s3.ObjectIdentifier, 0, listMax)
s3Path := d.s3Path(path)
// manually add the given path if it's a file
stat, err := d.Stat(ctx, path)
if err != nil {
return err
}
if stat != nil && !stat.IsDir() {
path := d.s3Path(path)
s3Objects = append(s3Objects, &s3.ObjectIdentifier{
Key: &path,
})
}
// list objects under the given path as a subpath (suffix with slash "/")
s3Path := d.s3Path(path) + "/"
listObjectsInput := &s3.ListObjectsInput{ listObjectsInput := &s3.ListObjectsInput{
Bucket: aws.String(d.Bucket), Bucket: aws.String(d.Bucket),
Prefix: aws.String(s3Path), Prefix: aws.String(s3Path),
@ -820,14 +834,10 @@ ListLoop:
// if there were no more results to return after the first call, resp.IsTruncated would have been false // if there were no more results to return after the first call, resp.IsTruncated would have been false
// and the loop would be exited without recalling ListObjects // and the loop would be exited without recalling ListObjects
if err != nil || len(resp.Contents) == 0 { if err != nil || len(resp.Contents) == 0 {
return storagedriver.PathNotFoundError{Path: path} break ListLoop
} }
for _, key := range resp.Contents { for _, key := range resp.Contents {
// Stop if we encounter a key that is not a subpath (so that deleting "/a" does not delete "/ab").
if len(*key.Key) > len(s3Path) && (*key.Key)[len(s3Path)] != '/' {
break ListLoop
}
s3Objects = append(s3Objects, &s3.ObjectIdentifier{ s3Objects = append(s3Objects, &s3.ObjectIdentifier{
Key: key.Key, Key: key.Key,
}) })
@ -843,8 +853,12 @@ ListLoop:
} }
} }
// need to chunk objects into groups of 1000 per s3 restrictions
total := len(s3Objects) total := len(s3Objects)
if total == 0 {
return storagedriver.PathNotFoundError{Path: path}
}
// need to chunk objects into groups of 1000 per s3 restrictions
for i := 0; i < total; i += 1000 { for i := 0; i < total; i += 1000 {
_, err := d.S3.DeleteObjects(&s3.DeleteObjectsInput{ _, err := d.S3.DeleteObjects(&s3.DeleteObjectsInput{
Bucket: aws.String(d.Bucket), Bucket: aws.String(d.Bucket),