r.loginov/distribution
1
0
Fork 0
forked from TrueCloudLab/distribution
Code Pull requests Activity

Merge pull request #1670 from vadmeste/fix_parts_sorting

Browse source 
s3 driver: Sorting completed parts by part number for a better accordance with S3 spec
This commit is contained in:
Richard Scothern 2016-04-25 16:02:17 -07:00
commit e6ad3a227a
1 changed files with 20 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
docs/storage/driver/s3-aws/s3.go
View file

@ -18,6 +18,7 @@ import (
"io/ioutil" "io/ioutil"
"net/http" "net/http"
"reflect" "reflect"
"sort"
"strconv" "strconv"
"strings" "strings"
"time" "time"
@ -718,6 +719,12 @@ func (d *driver) newWriter(key, uploadID string, parts []*s3.Part) storagedriver
} }
} }
type completedParts []*s3.CompletedPart
func (a completedParts) Len() int { return len(a) }
func (a completedParts) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a completedParts) Less(i, j int) bool { return *a[i].PartNumber < *a[j].PartNumber }
func (w *writer) Write(p []byte) (int, error) { func (w *writer) Write(p []byte) (int, error) {
if w.closed { if w.closed {
return 0, fmt.Errorf("already closed") return 0, fmt.Errorf("already closed")
@ -730,19 +737,22 @@ func (w *writer) Write(p []byte) (int, error) {
// If the last written part is smaller than minChunkSize, we need to make a // If the last written part is smaller than minChunkSize, we need to make a
// new multipart upload :sadface: // new multipart upload :sadface:
if len(w.parts) > 0 && int(*w.parts[len(w.parts)-1].Size) < minChunkSize { if len(w.parts) > 0 && int(*w.parts[len(w.parts)-1].Size) < minChunkSize {
var completedParts []*s3.CompletedPart var completedUploadedParts completedParts
for _, part := range w.parts { for _, part := range w.parts {
completedParts = append(completedParts, &s3.CompletedPart{ completedUploadedParts = append(completedUploadedParts, &s3.CompletedPart{
ETag: part.ETag, ETag: part.ETag,
PartNumber: part.PartNumber, PartNumber: part.PartNumber,
}) })
} }
sort.Sort(completedUploadedParts)
_, err := w.driver.S3.CompleteMultipartUpload(&s3.CompleteMultipartUploadInput{ _, err := w.driver.S3.CompleteMultipartUpload(&s3.CompleteMultipartUploadInput{
Bucket: aws.String(w.driver.Bucket), Bucket: aws.String(w.driver.Bucket),
Key: aws.String(w.key), Key: aws.String(w.key),
UploadId: aws.String(w.uploadID), UploadId: aws.String(w.uploadID),
MultipartUpload: &s3.CompletedMultipartUpload{ MultipartUpload: &s3.CompletedMultipartUpload{
Parts: completedParts, Parts: completedUploadedParts,
}, },
}) })
if err != nil { if err != nil {
@ -882,19 +892,23 @@ func (w *writer) Commit() error {
return err return err
} }
w.committed = true w.committed = true
var completedParts []*s3.CompletedPart
var completedUploadedParts completedParts
for _, part := range w.parts { for _, part := range w.parts {
completedParts = append(completedParts, &s3.CompletedPart{ completedUploadedParts = append(completedUploadedParts, &s3.CompletedPart{
ETag: part.ETag, ETag: part.ETag,
PartNumber: part.PartNumber, PartNumber: part.PartNumber,
}) })
} }
sort.Sort(completedUploadedParts)
_, err = w.driver.S3.CompleteMultipartUpload(&s3.CompleteMultipartUploadInput{ _, err = w.driver.S3.CompleteMultipartUpload(&s3.CompleteMultipartUploadInput{
Bucket: aws.String(w.driver.Bucket), Bucket: aws.String(w.driver.Bucket),
Key: aws.String(w.key), Key: aws.String(w.key),
UploadId: aws.String(w.uploadID), UploadId: aws.String(w.uploadID),
MultipartUpload: &s3.CompletedMultipartUpload{ MultipartUpload: &s3.CompletedMultipartUpload{
Parts: completedParts, Parts: completedUploadedParts,
}, },
}) })
if err != nil { if err != nil {