diff --git a/docs/content/docs.md b/docs/content/docs.md index ee3f7e4d7..821d6f4b0 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -1107,6 +1107,26 @@ triggering follow-on actions if data was copied, or skipping if not. NB: Enabling this option turns a usually non-fatal error into a potentially fatal one - please check and adjust your scripts accordingly! +### --fix-case ### + +Normally, a sync to a case insensitive dest (such as macOS / Windows) will +not result in a matching filename if the source and dest filenames have +casing differences but are otherwise identical. For example, syncing `hello.txt` +to `HELLO.txt` will normally result in the dest filename remaining `HELLO.txt`. +If `--fix-case` is set, then `HELLO.txt` will be renamed to `hello.txt` +to match the source. + +NB: +- directory names with incorrect casing will also be fixed +- `--fix-case` will be ignored if `--immutable` is set +- using `--local-case-sensitive` instead is not advisable; +it will cause `HELLO.txt` to get deleted! +- the old dest filename must not be excluded by filters. +Be especially careful with [`--files-from`](/filtering/#files-from-read-list-of-source-file-names), +which does not respect [`--ignore-case`](/filtering/#ignore-case-make-searches-case-insensitive)! +- on remotes that do not support server-side move, `--fix-case` will require +downloading the file and re-uploading it. To avoid this, do not use `--fix-case`. + ### --fs-cache-expire-duration=TIME When using rclone via the API rclone caches created remotes for 5 diff --git a/fs/config.go b/fs/config.go index d405f55bb..5b1e483dc 100644 --- a/fs/config.go +++ b/fs/config.go @@ -81,6 +81,7 @@ type ConfigInfo struct { IgnoreSize bool IgnoreChecksum bool IgnoreCaseSync bool + FixCase bool NoTraverse bool CheckFirst bool NoCheckDest bool diff --git a/fs/config/configflags/configflags.go b/fs/config/configflags/configflags.go index 7d856cf12..232af61d5 100644 --- a/fs/config/configflags/configflags.go +++ b/fs/config/configflags/configflags.go @@ -83,6 +83,7 @@ func AddFlags(ci *fs.ConfigInfo, flagSet *pflag.FlagSet) { flags.BoolVarP(flagSet, &ci.IgnoreSize, "ignore-size", "", false, "Ignore size when skipping use modtime or checksum", "Copy") flags.BoolVarP(flagSet, &ci.IgnoreChecksum, "ignore-checksum", "", ci.IgnoreChecksum, "Skip post copy check of checksums", "Copy") flags.BoolVarP(flagSet, &ci.IgnoreCaseSync, "ignore-case-sync", "", ci.IgnoreCaseSync, "Ignore case when synchronizing", "Copy") + flags.BoolVarP(flagSet, &ci.FixCase, "fix-case", "", ci.FixCase, "Force rename of case insensitive dest to match source", "Sync") flags.BoolVarP(flagSet, &ci.NoTraverse, "no-traverse", "", ci.NoTraverse, "Don't traverse destination file system on copy", "Copy") flags.BoolVarP(flagSet, &ci.CheckFirst, "check-first", "", ci.CheckFirst, "Do all the checks before starting transfers", "Copy") flags.BoolVarP(flagSet, &ci.NoCheckDest, "no-check-dest", "", ci.NoCheckDest, "Don't check the destination, copy regardless", "Copy") diff --git a/fs/sync/sync.go b/fs/sync/sync.go index 9154171c5..39dfb3818 100644 --- a/fs/sync/sync.go +++ b/fs/sync/sync.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "path" + "path/filepath" "sort" "strings" "sync" @@ -18,6 +19,7 @@ import ( "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/fs/march" "github.com/rclone/rclone/fs/operations" + "github.com/rclone/rclone/lib/random" ) // ErrorMaxDurationReached defines error when transfer duration is reached @@ -363,6 +365,16 @@ func (s *syncCopyMove) pairChecker(in *pipe, out *pipe, fraction int, wg *sync.W needTransfer = false } } + // Fix case for case insensitive filesystems + if s.ci.FixCase && !s.ci.Immutable && src.Remote() != pair.Dst.Remote() { + if newDst, err := operations.Move(s.ctx, s.fdst, nil, src.Remote(), pair.Dst); err != nil { + fs.Errorf(pair.Dst, "Error while attempting to rename to %s: %v", src.Remote(), err) + s.processError(err) + } else { + fs.Infof(pair.Dst, "Fixed case by renaming to: %s", src.Remote()) + pair.Dst = newDst + } + } if needTransfer { // If files are treated as immutable, fail if destination exists and does not match if s.ci.Immutable && pair.Dst != nil { @@ -1136,6 +1148,30 @@ func (s *syncCopyMove) Match(ctx context.Context, dst, src fs.DirEntry) (recurse s.srcEmptyDirs[src.Remote()] = src s.srcEmptyDirsMu.Unlock() } + if s.ci.FixCase && !s.ci.Immutable && src.Remote() != dst.Remote() { + // Fix case for case insensitive filesystems + // Fix each dir before recursing into subdirs and files + oldDirFs, err := fs.NewFs(s.ctx, filepath.Join(s.fdst.Root(), dst.Remote())) + s.processError(err) + newDirPath := filepath.Join(s.fdst.Root(), filepath.Dir(dst.Remote()), filepath.Base(src.Remote())) + newDirFs, err := fs.NewFs(s.ctx, newDirPath) + s.processError(err) + // Create random name to temporarily move dir to + tmpDirName := newDirPath + "-rclone-move-" + random.String(8) + tmpDirFs, err := fs.NewFs(s.ctx, tmpDirName) + s.processError(err) + if err = MoveDir(s.ctx, tmpDirFs, oldDirFs, s.deleteEmptySrcDirs, s.copyEmptySrcDirs); err != nil { + fs.Errorf(dst, "Error while attempting to move dir to temporary location %s: %v", tmpDirName, err) + s.processError(err) + } else { + if err = MoveDir(s.ctx, newDirFs, tmpDirFs, s.deleteEmptySrcDirs, s.copyEmptySrcDirs); err != nil { + fs.Errorf(dst, "Error while attempting to rename to %s: %v", src.Remote(), err) + s.processError(err) + } else { + fs.Infof(dst, "Fixed case by renaming to: %s", src.Remote()) + } + } + } return true } diff --git a/fs/sync/sync_test.go b/fs/sync/sync_test.go index 4438477f4..34085ee6e 100644 --- a/fs/sync/sync_test.go +++ b/fs/sync/sync_test.go @@ -2197,6 +2197,39 @@ func TestSyncIgnoreCase(t *testing.T) { r.CheckRemoteItems(t, file2) } +// Test --fix-case +func TestFixCase(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + r := fstest.NewRun(t) + + // Only test if remote is case insensitive + if !r.Fremote.Features().CaseInsensitive { + t.Skip("Skipping test as local or remote are case-sensitive") + } + + ci.FixCase = true + + // Create files with different filename casing + file1a := r.WriteFile("existing", "potato", t1) + file1b := r.WriteFile("existingbutdifferent", "donut", t1) + file1c := r.WriteFile("subdira/subdirb/subdirc/hello", "donut", t1) + file1d := r.WriteFile("subdira/subdirb/subdirc/subdird/filewithoutcasedifferences", "donut", t1) + r.CheckLocalItems(t, file1a, file1b, file1c, file1d) + file2a := r.WriteObject(ctx, "EXISTING", "potato", t1) + file2b := r.WriteObject(ctx, "EXISTINGBUTDIFFERENT", "lemonade", t1) + file2c := r.WriteObject(ctx, "SUBDIRA/subdirb/SUBDIRC/HELLO", "lemonade", t1) + file2d := r.WriteObject(ctx, "SUBDIRA/subdirb/SUBDIRC/subdird/filewithoutcasedifferences", "lemonade", t1) + r.CheckRemoteItems(t, file2a, file2b, file2c, file2d) + + // Should force rename of dest file that is differently-cased + accounting.GlobalStats().ResetCounters() + err := Sync(ctx, r.Fremote, r.Flocal, false) + require.NoError(t, err) + r.CheckLocalItems(t, file1a, file1b, file1c, file1d) + r.CheckRemoteItems(t, file1a, file1b, file1c, file1d) +} + // Test that aborting on --max-transfer works func TestMaxTransfer(t *testing.T) { ctx := context.Background()