From f3e982d3bfea664d3cc8ef92be8f16698aebaa95 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood <nick@craig-wood.com>
Date: Thu, 1 Mar 2018 12:11:34 +0000
Subject: [PATCH] azureblob,b2,gcs,qingstor,s3,swift: Don't check for
bucket/container presense if listing was OK
In a typical rclone copy to a bucket/container based remote, before
this change we were doing a list, followed by a HEAD of the bucket to
check it existed before doing the copy. The fact the list succeeded
means the bucket exists so mark it OK at that point.
Issue #1421
---
backend/azureblob/azureblob.go | 14 ++++++++++++++
backend/b2/b2.go | 13 +++++++++++++
backend/googlecloudstorage/googlecloudstorage.go | 13 +++++++++++++
backend/qingstor/qingstor.go | 14 ++++++++++++++
backend/s3/s3.go | 14 ++++++++++++++
backend/swift/swift.go | 13 +++++++++++++
6 files changed, 81 insertions(+)
diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.go
index 2157759bb..6b4f6b8e8 100644
--- a/backend/azureblob/azureblob.go
+++ b/backend/azureblob/azureblob.go
@@ -405,6 +405,16 @@ func (f *Fs) itemToDirEntry(remote string, object *storage.Blob, isDirectory boo
return o, nil
}
+// mark the container as being OK
+func (f *Fs) markContainerOK() {
+ if f.container != "" {
+ f.containerOKMu.Lock()
+ f.containerOK = true
+ f.containerDeleted = false
+ f.containerOKMu.Unlock()
+ }
+}
+
// listDir lists a single directory
func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) {
err = f.list(dir, false, listChunkSize, func(remote string, object *storage.Blob, isDirectory bool) error {
@@ -420,6 +430,8 @@ func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) {
if err != nil {
return nil, err
}
+ // container must be present if listing succeeded
+ f.markContainerOK()
return entries, nil
}
@@ -491,6 +503,8 @@ func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) {
if err != nil {
return err
}
+ // container must be present if listing succeeded
+ f.markContainerOK()
return list.Flush()
}
diff --git a/backend/b2/b2.go b/backend/b2/b2.go
index f68bcbfe6..cf2ac601d 100644
--- a/backend/b2/b2.go
+++ b/backend/b2/b2.go
@@ -551,6 +551,15 @@ func (f *Fs) itemToDirEntry(remote string, object *api.File, isDirectory bool, l
return o, nil
}
+// mark the bucket as being OK
+func (f *Fs) markBucketOK() {
+ if f.bucket != "" {
+ f.bucketOKMu.Lock()
+ f.bucketOK = true
+ f.bucketOKMu.Unlock()
+ }
+}
+
// listDir lists a single directory
func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) {
last := ""
@@ -567,6 +576,8 @@ func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) {
if err != nil {
return nil, err
}
+ // bucket must be present if listing succeeded
+ f.markBucketOK()
return entries, nil
}
@@ -634,6 +645,8 @@ func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) {
if err != nil {
return err
}
+ // bucket must be present if listing succeeded
+ f.markBucketOK()
return list.Flush()
}
diff --git a/backend/googlecloudstorage/googlecloudstorage.go b/backend/googlecloudstorage/googlecloudstorage.go
index 07c58fe1f..e16518835 100644
--- a/backend/googlecloudstorage/googlecloudstorage.go
+++ b/backend/googlecloudstorage/googlecloudstorage.go
@@ -463,6 +463,15 @@ func (f *Fs) itemToDirEntry(remote string, object *storage.Object, isDirectory b
return o, nil
}
+// mark the bucket as being OK
+func (f *Fs) markBucketOK() {
+ if f.bucket != "" {
+ f.bucketOKMu.Lock()
+ f.bucketOK = true
+ f.bucketOKMu.Unlock()
+ }
+}
+
// listDir lists a single directory
func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) {
// List the objects
@@ -479,6 +488,8 @@ func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) {
if err != nil {
return nil, err
}
+ // bucket must be present if listing succeeded
+ f.markBucketOK()
return entries, err
}
@@ -555,6 +566,8 @@ func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) {
if err != nil {
return err
}
+ // bucket must be present if listing succeeded
+ f.markBucketOK()
return list.Flush()
}
diff --git a/backend/qingstor/qingstor.go b/backend/qingstor/qingstor.go
index 2b5213410..fa02b4ea0 100644
--- a/backend/qingstor/qingstor.go
+++ b/backend/qingstor/qingstor.go
@@ -519,6 +519,16 @@ func (f *Fs) itemToDirEntry(remote string, object *qs.KeyType, isDirectory bool)
return o, nil
}
+// mark the bucket as being OK
+func (f *Fs) markBucketOK() {
+ if f.bucket != "" {
+ f.bucketOKMu.Lock()
+ f.bucketOK = true
+ f.bucketDeleted = false
+ f.bucketOKMu.Unlock()
+ }
+}
+
// listDir lists files and directories to out
func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) {
// List the objects and directories
@@ -535,6 +545,8 @@ func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) {
if err != nil {
return nil, err
}
+ // bucket must be present if listing succeeded
+ f.markBucketOK()
return entries, nil
}
@@ -606,6 +618,8 @@ func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) {
if err != nil {
return err
}
+ // bucket must be present if listing succeeded
+ f.markBucketOK()
return list.Flush()
}
diff --git a/backend/s3/s3.go b/backend/s3/s3.go
index 0aa409172..311a38863 100644
--- a/backend/s3/s3.go
+++ b/backend/s3/s3.go
@@ -590,6 +590,16 @@ func (f *Fs) itemToDirEntry(remote string, object *s3.Object, isDirectory bool)
return o, nil
}
+// mark the bucket as being OK
+func (f *Fs) markBucketOK() {
+ if f.bucket != "" {
+ f.bucketOKMu.Lock()
+ f.bucketOK = true
+ f.bucketDeleted = false
+ f.bucketOKMu.Unlock()
+ }
+}
+
// listDir lists files and directories to out
func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) {
// List the objects and directories
@@ -606,6 +616,8 @@ func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) {
if err != nil {
return nil, err
}
+ // bucket must be present if listing succeeded
+ f.markBucketOK()
return entries, nil
}
@@ -673,6 +685,8 @@ func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) {
if err != nil {
return err
}
+ // bucket must be present if listing succeeded
+ f.markBucketOK()
return list.Flush()
}
diff --git a/backend/swift/swift.go b/backend/swift/swift.go
index c45ef4ee2..c33d4c5e5 100644
--- a/backend/swift/swift.go
+++ b/backend/swift/swift.go
@@ -407,6 +407,15 @@ func (f *Fs) list(dir string, recurse bool, fn addEntryFn) error {
return err
}
+// mark the container as being OK
+func (f *Fs) markContainerOK() {
+ if f.container != "" {
+ f.containerOKMu.Lock()
+ f.containerOK = true
+ f.containerOKMu.Unlock()
+ }
+}
+
// listDir lists a single directory
func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) {
if f.container == "" {
@@ -420,6 +429,8 @@ func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) {
if err != nil {
return nil, err
}
+ // container must be present if listing succeeded
+ f.markContainerOK()
return entries, nil
}
@@ -482,6 +493,8 @@ func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) {
if err != nil {
return err
}
+ // container must be present if listing succeeded
+ f.markContainerOK()
return list.Flush()
}