alexvanin/rclone
1
0
Fork 0
forked from TrueCloudLab/rclone
Code Pull requests Activity

acd: make upload timeouts scale by file size

Browse source 
Fixes #712
Fixes #262
This commit is contained in:
Nick Craig-Wood 2016-10-17 16:18:56 +01:00
parent eba0a3633b
commit 5b8327038a
2 changed files with 45 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

42
amazonclouddrive/amazonclouddrive.go
View file

@ -48,7 +48,9 @@ const (
var ( var (
// Flags // Flags
tempLinkThreshold = fs.SizeSuffix(9 << 30) // Download files bigger than this via the tempLink tempLinkThreshold = fs.SizeSuffix(9 << 30) // Download files bigger than this via the tempLink
uploadWaitLimit = pflag.DurationP("acd-upload-wait-limit", "", 60*time.Second, "Don't wait for completed uploads to appear if they took less than this time.")
uploadWaitTime = pflag.DurationP("acd-upload-wait-time", "", 2*60*time.Second, "Time to wait after a failed complete upload to see if it appears.") uploadWaitTime = pflag.DurationP("acd-upload-wait-time", "", 2*60*time.Second, "Time to wait after a failed complete upload to see if it appears.")
uploadWaitPerGB = pflag.DurationP("acd-upload-wait-per-gb", "", 30*time.Second, "Additional time per GB to wait after a failed complete upload to see if it appears.")
// Description of how to auth for this app // Description of how to auth for this app
acdConfig = &oauth2.Config{ acdConfig = &oauth2.Config{
Scopes: []string{"clouddrive:read_all", "clouddrive:write"}, Scopes: []string{"clouddrive:read_all", "clouddrive:write"},
@ -478,26 +480,36 @@ func (f *Fs) List(out fs.ListOpts, dir string) {
// At the end of large uploads. The speculation is that the timeout // At the end of large uploads. The speculation is that the timeout
// is waiting for the sha1 hashing to complete and the file may well // is waiting for the sha1 hashing to complete and the file may well
// be properly uploaded. // be properly uploaded.
func (f *Fs) checkUpload(resp *http.Response, in io.Reader, src fs.ObjectInfo, inInfo *acd.File, inErr error) (fixedError bool, info *acd.File, err error) { func (f *Fs) checkUpload(resp *http.Response, in io.Reader, src fs.ObjectInfo, inInfo *acd.File, inErr error, uploadTime time.Duration) (fixedError bool, info *acd.File, err error) {
// Return if no error - all is well // Return if no error - all is well
if inErr == nil { if inErr == nil {
return false, inInfo, inErr return false, inInfo, inErr
} }
// If not one of the errors we can fix return // If not one of the errors we can fix return
if resp == nil || resp.StatusCode != 408 && resp.StatusCode != 500 && resp.StatusCode != 504 { // if resp == nil || resp.StatusCode != 408 && resp.StatusCode != 500 && resp.StatusCode != 504 {
return false, inInfo, inErr // return false, inInfo, inErr
} // }
const sleepTime = 5 * time.Second // sleep between tries // check to see if we read to the end
retries := int(*uploadWaitTime / sleepTime) // number of retries
if retries <= 0 {
retries = 1
}
buf := make([]byte, 1) buf := make([]byte, 1)
n, err := in.Read(buf) n, err := in.Read(buf)
if !(n == 0 && err == io.EOF) { if !(n == 0 && err == io.EOF) {
fs.Debug(src, "Upload error detected but didn't finish upload (n=%d, err=%v): %v", n, err, inErr) fs.Debug(src, "Upload error detected but didn't finish upload: %v", inErr)
return false, inInfo, inErr return false, inInfo, inErr
} }
// Only wait for items which have been in transit for > uploadWaitLimit
if uploadTime < *uploadWaitLimit {
fs.Debug(src, "Upload error detected but not waiting since it only took %v to upload: %v", uploadTime, inErr)
return false, inInfo, inErr
}
// Time we should wait for the upload
uploadWaitPerByte := float64(*uploadWaitPerGB) / 1024 / 1024 / 1024
timeToWait := time.Duration(uploadWaitPerByte*float64(src.Size())) + *uploadWaitTime
const sleepTime = 5 * time.Second // sleep between tries
retries := int((timeToWait + sleepTime - 1) / sleepTime) // number of retries, rounded up
fs.Debug(src, "Error detected after finished upload - waiting to see if object was uploaded correctly: %v", inErr) fs.Debug(src, "Error detected after finished upload - waiting to see if object was uploaded correctly: %v", inErr)
remote := src.Remote() remote := src.Remote()
for i := 1; i <= retries; i++ { for i := 1; i <= retries; i++ {
@ -508,7 +520,7 @@ func (f *Fs) checkUpload(resp *http.Response, in io.Reader, src fs.ObjectInfo, i
fs.Debug(src, "Object returned error - waiting (%d/%d): %v", i, retries, err) fs.Debug(src, "Object returned error - waiting (%d/%d): %v", i, retries, err)
} else { } else {
if src.Size() == o.Size() { if src.Size() == o.Size() {
fs.Debug(src, "Object found with correct size - returning with no error") fs.Debug(src, "Object found with correct size %d after waiting (%d/%d) - %v - returning with no error", src.Size(), i, retries, sleepTime*time.Duration(i-1))
info = &acd.File{ info = &acd.File{
Node: o.(*Object).info, Node: o.(*Object).info,
} }
@ -518,7 +530,7 @@ func (f *Fs) checkUpload(resp *http.Response, in io.Reader, src fs.ObjectInfo, i
} }
time.Sleep(sleepTime) time.Sleep(sleepTime)
} }
fs.Debug(src, "Finished waiting for object - returning original error: %v", inErr) fs.Debug(src, "Giving up waiting for object - returning original error: %v", inErr)
return false, inInfo, inErr return false, inInfo, inErr
} }
@ -557,6 +569,7 @@ func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) {
var info *acd.File var info *acd.File
var resp *http.Response var resp *http.Response
err = f.pacer.CallNoRetry(func() (bool, error) { err = f.pacer.CallNoRetry(func() (bool, error) {
start := time.Now()
f.startUpload() f.startUpload()
if src.Size() != 0 { if src.Size() != 0 {
info, resp, err = folder.Put(in, leaf) info, resp, err = folder.Put(in, leaf)
@ -565,7 +578,7 @@ func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) {
} }
f.stopUpload() f.stopUpload()
var ok bool var ok bool
ok, info, err = f.checkUpload(resp, in, src, info, err) ok, info, err = f.checkUpload(resp, in, src, info, err, time.Since(start))
if ok { if ok {
return false, nil return false, nil
} }
@ -815,6 +828,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error {
var resp *http.Response var resp *http.Response
var err error var err error
err = o.fs.pacer.CallNoRetry(func() (bool, error) { err = o.fs.pacer.CallNoRetry(func() (bool, error) {
start := time.Now()
o.fs.startUpload() o.fs.startUpload()
if size != 0 { if size != 0 {
info, resp, err = file.OverwriteSized(in, size) info, resp, err = file.OverwriteSized(in, size)
@ -823,7 +837,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error {
} }
o.fs.stopUpload() o.fs.stopUpload()
var ok bool var ok bool
ok, info, err = o.fs.checkUpload(resp, in, src, info, err) ok, info, err = o.fs.checkUpload(resp, in, src, info, err, time.Since(start))
if ok { if ok {
return false, nil return false, nil
} }

21
docs/content/amazonclouddrive.md
View file

@ -133,13 +133,26 @@ To download files above this threshold, rclone requests a `tempLink`
which downloads the file through a temporary URL directly from the which downloads the file through a temporary URL directly from the
underlying S3 storage. underlying S3 storage.
#### --acd-upload-wait-time=TIME #### #### --acd-upload-wait-time=TIME, --acd-upload-wait-per-gb=TIME, --acd-upload-wait-limit=TIME ####
Sometimes Amazon Drive gives an error when a file has been fully Sometimes Amazon Drive gives an error when a file has been fully
uploaded but the file appears anyway after a little while. This uploaded but the file appears anyway after a little while. This
controls the time rclone waits - 2 minutes by default. You might want happens sometimes for files over 1GB in size and nearly every time for
to increase the time if you are having problems with very big files. files bigger than 10GB. These parameters control the time rclone waits
Upload with the `-v` flag for more info. for the file to appear.
If the upload took less than `--acd-upload-wait-limit` (default 60s),
then we go ahead an upload it again as that will be quicker.
We wait `--acd-upload-wait-time` (default 2m) for the file to appear,
with an additional `--acd-upload-wait-per-gb` (default 30s) per GB of
the uploaded file.
These values were determined empirically by observing lots of uploads
of big files for a range of file sizes.
Upload with the `-v` flag to see more info about what rclone is doing
in this situation.
### Limitations ### ### Limitations ###