TrueCloudLab/rclone
16
0
Fork 2
Code Issues 1 Pull requests 1 Projects Releases Packages Wiki Activity Actions

s3: work around eventual consistency in bucket creation

Browse source 
Deleting a bucket then testing its existence can give the wrong
result.  Work around by keeping a flag as to whether we have deleted
the bucket.
This commit is contained in:
Nick Craig-Wood 2017-06-29 12:26:14 +01:00
parent 479c5a514a
commit 54561fd2bc
2 changed files with 15 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
fs/sync_test.go
View file

@ -715,6 +715,7 @@ func testServerSideMove(t *testing.T, r *Run, fremoteMove fs.Fs, withFilter bool
// Purge the original before moving // Purge the original before moving
require.NoError(t, fs.Purge(r.fremote)) require.NoError(t, fs.Purge(r.fremote))
fstest.CheckItems(t, r.fremote)
// Move it back again, dst does not exist this time // Move it back again, dst does not exist this time
fs.Stats.ResetCounters() fs.Stats.ResetCounters()

21
s3/s3.go
View file

@ -242,6 +242,7 @@ type Fs struct {
bucket string // the bucket we are working on bucket string // the bucket we are working on
bucketOKMu sync.Mutex // mutex to protect bucket OK bucketOKMu sync.Mutex // mutex to protect bucket OK
bucketOK bool // true if we have created the bucket bucketOK bool // true if we have created the bucket
bucketDeleted bool // true if we have deleted the bucket
acl string // ACL for new buckets / objects acl string // ACL for new buckets / objects
locationConstraint string // location constraint of new buckets locationConstraint string // location constraint of new buckets
sse string // the type of server-side encryption sse string // the type of server-side encryption
@ -670,6 +671,8 @@ func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.
} }
// Check if the bucket exists // Check if the bucket exists
//
// NB this can return incorrect results if called immediately after bucket deletion
func (f *Fs) dirExists() (bool, error) { func (f *Fs) dirExists() (bool, error) {
req := s3.HeadBucketInput{ req := s3.HeadBucketInput{
Bucket: &f.bucket, Bucket: &f.bucket,
@ -693,12 +696,14 @@ func (f *Fs) Mkdir(dir string) error {
if f.bucketOK { if f.bucketOK {
return nil return nil
} }
exists, err := f.dirExists() if !f.bucketDeleted {
if err == nil { exists, err := f.dirExists()
f.bucketOK = exists if err == nil {
} f.bucketOK = exists
if err != nil || exists { }
return err if err != nil || exists {
return err
}
} }
req := s3.CreateBucketInput{ req := s3.CreateBucketInput{
Bucket: &f.bucket, Bucket: &f.bucket,
@ -709,7 +714,7 @@ func (f *Fs) Mkdir(dir string) error {
LocationConstraint: &f.locationConstraint, LocationConstraint: &f.locationConstraint,
} }
} }
_, err = f.c.CreateBucket(&req) _, err := f.c.CreateBucket(&req)
if err, ok := err.(awserr.Error); ok { if err, ok := err.(awserr.Error); ok {
if err.Code() == "BucketAlreadyOwnedByYou" { if err.Code() == "BucketAlreadyOwnedByYou" {
err = nil err = nil
@ -717,6 +722,7 @@ func (f *Fs) Mkdir(dir string) error {
} }
if err == nil { if err == nil {
f.bucketOK = true f.bucketOK = true
f.bucketDeleted = false
} }
return err return err
} }
@ -736,6 +742,7 @@ func (f *Fs) Rmdir(dir string) error {
_, err := f.c.DeleteBucket(&req) _, err := f.c.DeleteBucket(&req)
if err == nil { if err == nil {
f.bucketOK = false f.bucketOK = false
f.bucketDeleted = true
} }
return err return err
} }