TrueCloudLab/rclone
16
0
Fork 2
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions 1

s3: work around eventual consistency in bucket creation

Browse source 
Deleting a bucket then testing its existence can give the wrong
result.  Work around by keeping a flag as to whether we have deleted
the bucket.
This commit is contained in:
Nick Craig-Wood 2017-06-29 12:26:14 +01:00
parent 479c5a514a
commit 54561fd2bc
2 changed files with 15 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
fs/sync_test.go
View file

@ -715,6 +715,7 @@ func testServerSideMove(t *testing.T, r *Run, fremoteMove fs.Fs, withFilter bool
// Purge the original before moving
require.NoError(t, fs.Purge(r.fremote))
fstest.CheckItems(t, r.fremote)
// Move it back again, dst does not exist this time
fs.Stats.ResetCounters()

9
s3/s3.go
View file

@ -242,6 +242,7 @@ type Fs struct {
bucket string // the bucket we are working on
bucketOKMu sync.Mutex // mutex to protect bucket OK
bucketOK bool // true if we have created the bucket
bucketDeleted bool // true if we have deleted the bucket
acl string // ACL for new buckets / objects
locationConstraint string // location constraint of new buckets
sse string // the type of server-side encryption
@ -670,6 +671,8 @@ func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.
}
// Check if the bucket exists
//
// NB this can return incorrect results if called immediately after bucket deletion
func (f *Fs) dirExists() (bool, error) {
req := s3.HeadBucketInput{
Bucket: &f.bucket,
@ -693,6 +696,7 @@ func (f *Fs) Mkdir(dir string) error {
if f.bucketOK {
return nil
}
if !f.bucketDeleted {
exists, err := f.dirExists()
if err == nil {
f.bucketOK = exists
@ -700,6 +704,7 @@ func (f *Fs) Mkdir(dir string) error {
if err != nil || exists {
return err
}
}
req := s3.CreateBucketInput{
Bucket: &f.bucket,
ACL: &f.acl,
@ -709,7 +714,7 @@ func (f *Fs) Mkdir(dir string) error {
LocationConstraint: &f.locationConstraint,
}
}
_, err = f.c.CreateBucket(&req)
_, err := f.c.CreateBucket(&req)
if err, ok := err.(awserr.Error); ok {
if err.Code() == "BucketAlreadyOwnedByYou" {
err = nil
@ -717,6 +722,7 @@ func (f *Fs) Mkdir(dir string) error {
}
if err == nil {
f.bucketOK = true
f.bucketDeleted = false
}
return err
}
@ -736,6 +742,7 @@ func (f *Fs) Rmdir(dir string) error {
_, err := f.c.DeleteBucket(&req)
if err == nil {
f.bucketOK = false
f.bucketDeleted = true
}
return err
}