sync: --fix-case flag to rename case insensitive dest - fixes #4854
Before this change, a sync to a case insensitive dest (such as macOS / Windows) would not result in a matching filename if the source and dest had casing differences but were otherwise equal. For example, syncing `hello.txt` to `HELLO.txt` would result in the dest filename remaining `HELLO.txt`. Furthermore, `--local-case-sensitive` did not solve this, as it actually caused `HELLO.txt` to get deleted! After this change, `HELLO.txt` is renamed to `hello.txt` to match the source, only if the `--fix-case` flag is specified. (The old behavior remains the default.)
This commit is contained in:
parent
88e516adee
commit
11afc3dde0
5 changed files with 91 additions and 0 deletions
|
@ -1107,6 +1107,26 @@ triggering follow-on actions if data was copied, or skipping if not.
|
|||
NB: Enabling this option turns a usually non-fatal error into a potentially
|
||||
fatal one - please check and adjust your scripts accordingly!
|
||||
|
||||
### --fix-case ###
|
||||
|
||||
Normally, a sync to a case insensitive dest (such as macOS / Windows) will
|
||||
not result in a matching filename if the source and dest filenames have
|
||||
casing differences but are otherwise identical. For example, syncing `hello.txt`
|
||||
to `HELLO.txt` will normally result in the dest filename remaining `HELLO.txt`.
|
||||
If `--fix-case` is set, then `HELLO.txt` will be renamed to `hello.txt`
|
||||
to match the source.
|
||||
|
||||
NB:
|
||||
- directory names with incorrect casing will also be fixed
|
||||
- `--fix-case` will be ignored if `--immutable` is set
|
||||
- using `--local-case-sensitive` instead is not advisable;
|
||||
it will cause `HELLO.txt` to get deleted!
|
||||
- the old dest filename must not be excluded by filters.
|
||||
Be especially careful with [`--files-from`](/filtering/#files-from-read-list-of-source-file-names),
|
||||
which does not respect [`--ignore-case`](/filtering/#ignore-case-make-searches-case-insensitive)!
|
||||
- on remotes that do not support server-side move, `--fix-case` will require
|
||||
downloading the file and re-uploading it. To avoid this, do not use `--fix-case`.
|
||||
|
||||
### --fs-cache-expire-duration=TIME
|
||||
|
||||
When using rclone via the API rclone caches created remotes for 5
|
||||
|
|
|
@ -81,6 +81,7 @@ type ConfigInfo struct {
|
|||
IgnoreSize bool
|
||||
IgnoreChecksum bool
|
||||
IgnoreCaseSync bool
|
||||
FixCase bool
|
||||
NoTraverse bool
|
||||
CheckFirst bool
|
||||
NoCheckDest bool
|
||||
|
|
|
@ -83,6 +83,7 @@ func AddFlags(ci *fs.ConfigInfo, flagSet *pflag.FlagSet) {
|
|||
flags.BoolVarP(flagSet, &ci.IgnoreSize, "ignore-size", "", false, "Ignore size when skipping use modtime or checksum", "Copy")
|
||||
flags.BoolVarP(flagSet, &ci.IgnoreChecksum, "ignore-checksum", "", ci.IgnoreChecksum, "Skip post copy check of checksums", "Copy")
|
||||
flags.BoolVarP(flagSet, &ci.IgnoreCaseSync, "ignore-case-sync", "", ci.IgnoreCaseSync, "Ignore case when synchronizing", "Copy")
|
||||
flags.BoolVarP(flagSet, &ci.FixCase, "fix-case", "", ci.FixCase, "Force rename of case insensitive dest to match source", "Sync")
|
||||
flags.BoolVarP(flagSet, &ci.NoTraverse, "no-traverse", "", ci.NoTraverse, "Don't traverse destination file system on copy", "Copy")
|
||||
flags.BoolVarP(flagSet, &ci.CheckFirst, "check-first", "", ci.CheckFirst, "Do all the checks before starting transfers", "Copy")
|
||||
flags.BoolVarP(flagSet, &ci.NoCheckDest, "no-check-dest", "", ci.NoCheckDest, "Don't check the destination, copy regardless", "Copy")
|
||||
|
|
|
@ -6,6 +6,7 @@ import (
|
|||
"errors"
|
||||
"fmt"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
@ -18,6 +19,7 @@ import (
|
|||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/fs/march"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
)
|
||||
|
||||
// ErrorMaxDurationReached defines error when transfer duration is reached
|
||||
|
@ -363,6 +365,16 @@ func (s *syncCopyMove) pairChecker(in *pipe, out *pipe, fraction int, wg *sync.W
|
|||
needTransfer = false
|
||||
}
|
||||
}
|
||||
// Fix case for case insensitive filesystems
|
||||
if s.ci.FixCase && !s.ci.Immutable && src.Remote() != pair.Dst.Remote() {
|
||||
if newDst, err := operations.Move(s.ctx, s.fdst, nil, src.Remote(), pair.Dst); err != nil {
|
||||
fs.Errorf(pair.Dst, "Error while attempting to rename to %s: %v", src.Remote(), err)
|
||||
s.processError(err)
|
||||
} else {
|
||||
fs.Infof(pair.Dst, "Fixed case by renaming to: %s", src.Remote())
|
||||
pair.Dst = newDst
|
||||
}
|
||||
}
|
||||
if needTransfer {
|
||||
// If files are treated as immutable, fail if destination exists and does not match
|
||||
if s.ci.Immutable && pair.Dst != nil {
|
||||
|
@ -1136,6 +1148,30 @@ func (s *syncCopyMove) Match(ctx context.Context, dst, src fs.DirEntry) (recurse
|
|||
s.srcEmptyDirs[src.Remote()] = src
|
||||
s.srcEmptyDirsMu.Unlock()
|
||||
}
|
||||
if s.ci.FixCase && !s.ci.Immutable && src.Remote() != dst.Remote() {
|
||||
// Fix case for case insensitive filesystems
|
||||
// Fix each dir before recursing into subdirs and files
|
||||
oldDirFs, err := fs.NewFs(s.ctx, filepath.Join(s.fdst.Root(), dst.Remote()))
|
||||
s.processError(err)
|
||||
newDirPath := filepath.Join(s.fdst.Root(), filepath.Dir(dst.Remote()), filepath.Base(src.Remote()))
|
||||
newDirFs, err := fs.NewFs(s.ctx, newDirPath)
|
||||
s.processError(err)
|
||||
// Create random name to temporarily move dir to
|
||||
tmpDirName := newDirPath + "-rclone-move-" + random.String(8)
|
||||
tmpDirFs, err := fs.NewFs(s.ctx, tmpDirName)
|
||||
s.processError(err)
|
||||
if err = MoveDir(s.ctx, tmpDirFs, oldDirFs, s.deleteEmptySrcDirs, s.copyEmptySrcDirs); err != nil {
|
||||
fs.Errorf(dst, "Error while attempting to move dir to temporary location %s: %v", tmpDirName, err)
|
||||
s.processError(err)
|
||||
} else {
|
||||
if err = MoveDir(s.ctx, newDirFs, tmpDirFs, s.deleteEmptySrcDirs, s.copyEmptySrcDirs); err != nil {
|
||||
fs.Errorf(dst, "Error while attempting to rename to %s: %v", src.Remote(), err)
|
||||
s.processError(err)
|
||||
} else {
|
||||
fs.Infof(dst, "Fixed case by renaming to: %s", src.Remote())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
|
|
@ -2197,6 +2197,39 @@ func TestSyncIgnoreCase(t *testing.T) {
|
|||
r.CheckRemoteItems(t, file2)
|
||||
}
|
||||
|
||||
// Test --fix-case
|
||||
func TestFixCase(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ctx, ci := fs.AddConfig(ctx)
|
||||
r := fstest.NewRun(t)
|
||||
|
||||
// Only test if remote is case insensitive
|
||||
if !r.Fremote.Features().CaseInsensitive {
|
||||
t.Skip("Skipping test as local or remote are case-sensitive")
|
||||
}
|
||||
|
||||
ci.FixCase = true
|
||||
|
||||
// Create files with different filename casing
|
||||
file1a := r.WriteFile("existing", "potato", t1)
|
||||
file1b := r.WriteFile("existingbutdifferent", "donut", t1)
|
||||
file1c := r.WriteFile("subdira/subdirb/subdirc/hello", "donut", t1)
|
||||
file1d := r.WriteFile("subdira/subdirb/subdirc/subdird/filewithoutcasedifferences", "donut", t1)
|
||||
r.CheckLocalItems(t, file1a, file1b, file1c, file1d)
|
||||
file2a := r.WriteObject(ctx, "EXISTING", "potato", t1)
|
||||
file2b := r.WriteObject(ctx, "EXISTINGBUTDIFFERENT", "lemonade", t1)
|
||||
file2c := r.WriteObject(ctx, "SUBDIRA/subdirb/SUBDIRC/HELLO", "lemonade", t1)
|
||||
file2d := r.WriteObject(ctx, "SUBDIRA/subdirb/SUBDIRC/subdird/filewithoutcasedifferences", "lemonade", t1)
|
||||
r.CheckRemoteItems(t, file2a, file2b, file2c, file2d)
|
||||
|
||||
// Should force rename of dest file that is differently-cased
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
err := Sync(ctx, r.Fremote, r.Flocal, false)
|
||||
require.NoError(t, err)
|
||||
r.CheckLocalItems(t, file1a, file1b, file1c, file1d)
|
||||
r.CheckRemoteItems(t, file1a, file1b, file1c, file1d)
|
||||
}
|
||||
|
||||
// Test that aborting on --max-transfer works
|
||||
func TestMaxTransfer(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
|
Loading…
Reference in a new issue