Add --ignore-size flag - fixes #399

This commit is contained in:
Nick Craig-Wood 2016-06-17 17:20:08 +01:00
parent 318e42e35b
commit 46135d830e
4 changed files with 73 additions and 8 deletions

View file

@ -437,6 +437,20 @@ While this isn't a generally recommended option, it can be useful
in cases where your files change due to encryption. However, it cannot
correct partial transfers in case a transfer was interrupted.
### --ignore-size ###
Normally rclone will look at modification time and size of files to
see if they are equal. If you set this flag then rclone will check
only the modification time. If `--checksum` is set then it only
checks the checksum.
It will also cause rclone to skip verifying the sizes are the same
after transfer.
This can be useful for transferring files to and from onedrive which
occasionally misreports the size of image files (see
[#399](https://github.com/ncw/rclone/issues/399) for more info).
### -I, --ignore-times ###
Using this option will cause rclone to unconditionally upload all
@ -534,9 +548,6 @@ This can be useful transferring files from dropbox which have been
modified by the desktop sync client which doesn't set checksums of
modification times in the same way as rclone.
When using this flag, rclone won't update mtimes of remote files if
they are incorrect as it would normally.
### --stats=TIME ###
Rclone will print stats at regular intervals to show its progress.

View file

@ -87,6 +87,7 @@ var (
noGzip = pflag.BoolP("no-gzip-encoding", "", false, "Don't set Accept-Encoding: gzip.")
dedupeMode = pflag.StringP("dedupe-mode", "", "interactive", "Dedupe mode interactive|skip|first|newest|oldest|rename.")
maxDepth = pflag.IntP("max-depth", "", -1, "If set limits the recursion depth to this.")
ignoreSize = pflag.BoolP("ignore-size", "", false, "Ignore size when skipping use mod-time or checksum.")
bwLimit SizeSuffix
// Key to use for password en/decryption.
@ -221,6 +222,7 @@ type ConfigInfo struct {
NoGzip bool // Disable compression
DedupeMode DeduplicateMode
MaxDepth int
IgnoreSize bool
}
// Transport returns an http.RoundTripper with the correct timeouts
@ -324,6 +326,7 @@ func LoadConfig() {
Config.UpdateOlder = *updateOlder
Config.NoGzip = *noGzip
Config.MaxDepth = *maxDepth
Config.IgnoreSize = *ignoreSize
ConfigPath = *configFile
@ -358,6 +361,10 @@ func LoadConfig() {
Config.DeleteDuring = true
}
if Config.IgnoreSize && Config.SizeOnly {
log.Fatalf(`Can't use --size-only and --ignore-size together.`)
}
// Load configuration file.
var err error
ConfigFile, err = loadConfigFile()

View file

@ -95,7 +95,8 @@ func CheckHashes(src, dst Object) (equal bool, hash HashType, err error) {
//
// If the src and dst size are different then it is considered to be
// not equal. If --size-only is in effect then this is the only check
// that is done.
// that is done. If --ignore-size is in effect then this check is
// skipped and the files are considered the same size.
//
// If the size is the same and the mtime is the same then it is
// considered to be equal. This check is skipped if using --checksum.
@ -108,9 +109,11 @@ func CheckHashes(src, dst Object) (equal bool, hash HashType, err error) {
// Otherwise the file is considered to be not equal including if there
// were errors reading info.
func Equal(src, dst Object) bool {
if src.Size() != dst.Size() {
Debug(src, "Sizes differ")
return false
if !Config.IgnoreSize {
if src.Size() != dst.Size() {
Debug(src, "Sizes differ")
return false
}
}
if Config.SizeOnly {
Debug(src, "Sizes identical")
@ -261,7 +264,7 @@ tryAgain:
}
// Verify sizes are the same after transfer
if src.Size() != dst.Size() {
if !Config.IgnoreSize && src.Size() != dst.Size() {
Stats.Error()
err = errors.Errorf("corrupted on transfer: sizes differ %d vs %d", src.Size(), dst.Size())
ErrorLog(dst, "%s", err)

View file

@ -479,6 +479,50 @@ func TestSyncSizeOnly(t *testing.T) {
fstest.CheckItems(t, r.fremote, file1)
}
// Create a file and sync it. Keep the last modified date but change
// the size. With --ignore-size we expect nothing to to be
// transferred on the second sync.
func TestSyncIgnoreSize(t *testing.T) {
r := NewRun(t)
defer r.Finalise()
fs.Config.IgnoreSize = true
defer func() { fs.Config.IgnoreSize = false }()
file1 := r.WriteFile("ignore-size", "contents", t1)
fstest.CheckItems(t, r.flocal, file1)
fs.Stats.ResetCounters()
err := fs.Sync(r.fremote, r.flocal)
if err != nil {
t.Fatalf("Initial sync failed: %v", err)
}
// We should have transferred exactly one file.
if fs.Stats.GetTransfers() != 1 {
t.Fatalf("Sync 1: want 1 transfer, got %d", fs.Stats.GetTransfers())
}
fstest.CheckItems(t, r.fremote, file1)
// Update size but not date of file
file2 := r.WriteFile("ignore-size", "longer contents but same date", t1)
fstest.CheckItems(t, r.flocal, file2)
fs.Stats.ResetCounters()
err = fs.Sync(r.fremote, r.flocal)
if err != nil {
t.Fatalf("Sync failed: %v", err)
}
// We should have transferred no files
if fs.Stats.GetTransfers() != 0 {
t.Fatalf("Sync 2: want 0 transfers, got %d", fs.Stats.GetTransfers())
}
fstest.CheckItems(t, r.flocal, file2)
fstest.CheckItems(t, r.fremote, file1)
}
func TestSyncIgnoreTimes(t *testing.T) {
r := NewRun(t)
defer r.Finalise()