forked from TrueCloudLab/rclone
Fix not transferring files that don't differ in size - fixes #911
Due to a logic error files stored on remotes which support modtime but not hashes weren't being transferred when updating with a file of the same size but different modtime. Instead the modtime of the remote file was being set to that of the local file. In practice this affected crypt with all remotes except Amazon Drive and Dropbox.
This commit is contained in:
parent
539853df36
commit
2756900749
2 changed files with 64 additions and 26 deletions
|
@ -12,7 +12,6 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
"github.com/spf13/pflag"
|
"github.com/spf13/pflag"
|
||||||
|
@ -121,34 +120,53 @@ func Equal(src, dst Object) bool {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
var srcModTime time.Time
|
// Assert: Size is equal or being ignored
|
||||||
if !Config.CheckSum {
|
|
||||||
|
// If checking checksum and not modtime
|
||||||
|
if Config.CheckSum {
|
||||||
|
// Check the hash
|
||||||
|
same, hash, _ := CheckHashes(src, dst)
|
||||||
|
if !same {
|
||||||
|
Debug(src, "%v differ", hash)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if hash == HashNone {
|
||||||
|
Debug(src, "Size of src and dst objects identical")
|
||||||
|
} else {
|
||||||
|
Debug(src, "Size and %v of src and dst objects identical", hash)
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sizes the same so check the mtime
|
||||||
if Config.ModifyWindow == ModTimeNotSupported {
|
if Config.ModifyWindow == ModTimeNotSupported {
|
||||||
Debug(src, "Sizes identical")
|
Debug(src, "Sizes identical")
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
// Size the same so check the mtime
|
srcModTime := src.ModTime()
|
||||||
srcModTime = src.ModTime()
|
|
||||||
dstModTime := dst.ModTime()
|
dstModTime := dst.ModTime()
|
||||||
dt := dstModTime.Sub(srcModTime)
|
dt := dstModTime.Sub(srcModTime)
|
||||||
ModifyWindow := Config.ModifyWindow
|
ModifyWindow := Config.ModifyWindow
|
||||||
if dt >= ModifyWindow || dt <= -ModifyWindow {
|
if dt < ModifyWindow && dt > -ModifyWindow {
|
||||||
Debug(src, "Modification times differ by %s: %v, %v", dt, srcModTime, dstModTime)
|
|
||||||
} else {
|
|
||||||
Debug(src, "Size and modification time the same (differ by %s, within tolerance %s)", dt, ModifyWindow)
|
Debug(src, "Size and modification time the same (differ by %s, within tolerance %s)", dt, ModifyWindow)
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// mtime is unreadable or different but size is the same so
|
Debug(src, "Modification times differ by %s: %v, %v", dt, srcModTime, dstModTime)
|
||||||
// check the hash
|
|
||||||
|
// Check if the hashes are the same
|
||||||
same, hash, _ := CheckHashes(src, dst)
|
same, hash, _ := CheckHashes(src, dst)
|
||||||
if !same {
|
if !same {
|
||||||
Debug(src, "Hash differ")
|
Debug(src, "%v differ", hash)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if hash == HashNone {
|
||||||
|
// if couldn't check hash, return that they differ
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
if !(Config.CheckSum || Config.NoUpdateModTime) {
|
// mod time differs but hash is the same to reset mod time if required
|
||||||
|
if !Config.NoUpdateModTime {
|
||||||
// Size and hash the same but mtime different so update the
|
// Size and hash the same but mtime different so update the
|
||||||
// mtime of the dst object here
|
// mtime of the dst object here
|
||||||
err := dst.SetModTime(srcModTime)
|
err := dst.SetModTime(srcModTime)
|
||||||
|
@ -157,14 +175,10 @@ func Equal(src, dst Object) bool {
|
||||||
return false
|
return false
|
||||||
} else if err != nil {
|
} else if err != nil {
|
||||||
Stats.Error()
|
Stats.Error()
|
||||||
ErrorLog(dst, "Failed to read set modification time: %v", err)
|
ErrorLog(dst, "Failed to set modification time: %v", err)
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if hash == HashNone {
|
|
||||||
Debug(src, "Size of src and dst objects identical")
|
|
||||||
} else {
|
} else {
|
||||||
Debug(src, "Size and %v of src and dst objects identical", hash)
|
Debug(src, "Updated modification time in destination")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
|
@ -336,6 +336,30 @@ func TestSyncAfterChangingModtimeOnlyWithNoUpdateModTime(t *testing.T) {
|
||||||
fstest.CheckItems(t, r.fremote, file2)
|
fstest.CheckItems(t, r.fremote, file2)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestSyncDoesntUpdateModtime(t *testing.T) {
|
||||||
|
if fs.Config.ModifyWindow == fs.ModTimeNotSupported {
|
||||||
|
t.Skip("Can't run this test on fs which doesn't support mod time")
|
||||||
|
}
|
||||||
|
r := NewRun(t)
|
||||||
|
defer r.Finalise()
|
||||||
|
|
||||||
|
file1 := r.WriteFile("foo", "foo", t2)
|
||||||
|
file2 := r.WriteObject("foo", "bar", t1)
|
||||||
|
|
||||||
|
fstest.CheckItems(t, r.flocal, file1)
|
||||||
|
fstest.CheckItems(t, r.fremote, file2)
|
||||||
|
|
||||||
|
fs.Stats.ResetCounters()
|
||||||
|
err := fs.Sync(r.fremote, r.flocal)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
fstest.CheckItems(t, r.flocal, file1)
|
||||||
|
fstest.CheckItems(t, r.fremote, file1)
|
||||||
|
|
||||||
|
// We should have transferred exactly one file, not set the mod time
|
||||||
|
assert.Equal(t, int64(1), fs.Stats.GetTransfers())
|
||||||
|
}
|
||||||
|
|
||||||
func TestSyncAfterAddingAFile(t *testing.T) {
|
func TestSyncAfterAddingAFile(t *testing.T) {
|
||||||
r := NewRun(t)
|
r := NewRun(t)
|
||||||
defer r.Finalise()
|
defer r.Finalise()
|
||||||
|
|
Loading…
Reference in a new issue