TrueCloudLab/rclone
16
0
Fork 2
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions 1

sync: Fix --max-duration so it doesn't retry when the duration is exceeded

Browse source 
Before this change, if the --max-duration limit was reached then
rclone would retry the sync as a fatal error wasn't raised.

This checks the deadline and raises a fatal error if necessary at the
end of the sync.

Fixes #6002
This commit is contained in:
Nick Craig-Wood 2022-02-24 10:03:41 +00:00
parent 251b84ff2c
commit 21355b4208
2 changed files with 15 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
fs/sync/sync.go
View file

@ -73,6 +73,7 @@ type syncCopyMove struct {
compareCopyDest []fs.Fs // place to check for files to server side copy compareCopyDest []fs.Fs // place to check for files to server side copy
backupDir fs.Fs // place to store overwrites/deletes backupDir fs.Fs // place to store overwrites/deletes
checkFirst bool // if set run all the checkers before starting transfers checkFirst bool // if set run all the checkers before starting transfers
maxDurationEndTime time.Time // end time if --max-duration is set
} }
type trackRenamesStrategy byte type trackRenamesStrategy byte
@ -146,9 +147,9 @@ func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete
} }
// If a max session duration has been defined add a deadline to the context // If a max session duration has been defined add a deadline to the context
if ci.MaxDuration > 0 { if ci.MaxDuration > 0 {
endTime := time.Now().Add(ci.MaxDuration) s.maxDurationEndTime = time.Now().Add(ci.MaxDuration)
fs.Infof(s.fdst, "Transfer session deadline: %s", endTime.Format("2006/01/02 15:04:05")) fs.Infof(s.fdst, "Transfer session deadline: %s", s.maxDurationEndTime.Format("2006/01/02 15:04:05"))
s.ctx, s.cancel = context.WithDeadline(ctx, endTime) s.ctx, s.cancel = context.WithDeadline(ctx, s.maxDurationEndTime)
} else { } else {
s.ctx, s.cancel = context.WithCancel(ctx) s.ctx, s.cancel = context.WithCancel(ctx)
} }
@ -809,6 +810,10 @@ func (s *syncCopyMove) tryRename(src fs.Object) bool {
return true return true
} }
// errorMaxDurationReached defines error when transfer duration is reached
// Used for checking on exit and matching to correct exit code.
var errorMaxDurationReached = fserrors.FatalError(errors.New("Max transfer duration reached as set by --max-duration"))
// Syncs fsrc into fdst // Syncs fsrc into fdst
// //
// If Delete is true then it deletes any files in fdst that aren't in fsrc // If Delete is true then it deletes any files in fdst that aren't in fsrc
@ -902,6 +907,12 @@ func (s *syncCopyMove) run() error {
// Read the error out of the context if there is one // Read the error out of the context if there is one
s.processError(s.ctx.Err()) s.processError(s.ctx.Err())
// If the duration was exceeded then add a Fatal Error so we don't retry
if !s.maxDurationEndTime.IsZero() && time.Since(s.maxDurationEndTime) > 0 {
fs.Errorf(s.fdst, "%v", errorMaxDurationReached)
s.processError(errorMaxDurationReached)
}
// Print nothing to transfer message if there were no transfers and no errors // Print nothing to transfer message if there were no transfers and no errors
if s.deleteMode != fs.DeleteModeOnly && accounting.Stats(s.ctx).GetTransfers() == 0 && s.currentError() == nil { if s.deleteMode != fs.DeleteModeOnly && accounting.Stats(s.ctx).GetTransfers() == 0 && s.currentError() == nil {
fs.Infof(nil, "There was nothing to transfer") fs.Infof(nil, "There was nothing to transfer")

2
fs/sync/sync_test.go
View file

@ -1029,7 +1029,7 @@ func TestSyncWithMaxDuration(t *testing.T) {
accounting.GlobalStats().ResetCounters() accounting.GlobalStats().ResetCounters()
startTime := time.Now() startTime := time.Now()
err := Sync(ctx, r.Fremote, r.Flocal, false) err := Sync(ctx, r.Fremote, r.Flocal, false)
require.True(t, errors.Is(err, context.DeadlineExceeded)) require.True(t, errors.Is(err, errorMaxDurationReached))
elapsed := time.Since(startTime) elapsed := time.Since(startTime)
maxTransferTime := (time.Duration(len(testFiles)) * 60 * time.Second) / time.Duration(bytesPerSecond) maxTransferTime := (time.Duration(len(testFiles)) * 60 * time.Second) / time.Duration(bytesPerSecond)