TrueCloudLab/rclone
16
0
Fork 2
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

bisync: handle unicode and case normalization consistently - mostly-fixes #7270

Browse source 
Before this change, Bisync sometimes normalized NFD to NFC and sometimes
did not, causing errors in some scenarios (particularly for users of macOS).
It was similarly inconsistent in its handling of case-insensitivity.

There were three main places where Bisync should have normalized, but didn't:

1. When building the list of files that need to be transferred during --resync
2. When building the list of deltas during a non-resync
3. When comparing Path1 to Path2 during --check-sync

After this change, 1 and 3 are resolved, and bisync supports
--no-unicode-normalization and --ignore-case-sync in the same way as sync.
2 will be addressed in a future update.
This commit is contained in:
nielash 2023-10-08 23:16:23 -04:00
parent 11afc3dde0
commit f7f4651828
32 changed files with 441 additions and 69 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
cmd/bisync/bisync_test.go
View file

@ -98,10 +98,13 @@ var logHoppers = []string{
// subdirectories. The order inconsistency initially showed up in the
// listings and triggered reordering of log messages, but the actual
// files will in fact match.
`ERROR : - +Access test failed: Path[12] file not found in Path[12] - .*`,
`.* +.....Access test failed: Path[12] file not found in Path[12].*`,
// Test case `resync` suffered from the order of queued copies.
`(?:INFO |NOTICE): - Path2 Resync will copy to Path1 +- .*`,
// Test case `normalization` can have random order of fix-case files.
`(?:INFO |NOTICE): .*: Fixed case by renaming to: .*`,
}
// Some log lines can contain Windows path separator that must be
@ -546,6 +549,16 @@ func (b *bisyncTest) runTestStep(ctx context.Context, line string) (err error) {
case "copy-as":
b.checkArgs(args, 3, 3)
return b.copyFile(ctx, args[1], args[2], args[3])
case "copy-as-NFC":
b.checkArgs(args, 3, 3)
ci.NoUnicodeNormalization = true
ci.FixCase = true
return b.copyFile(ctx, args[1], norm.NFC.String(args[2]), norm.NFC.String(args[3]))
case "copy-as-NFD":
b.checkArgs(args, 3, 3)
ci.NoUnicodeNormalization = true
ci.FixCase = true
return b.copyFile(ctx, args[1], norm.NFD.String(args[2]), norm.NFD.String(args[3]))
case "copy-dir", "sync-dir":
b.checkArgs(args, 2, 2)
if fsrc, err = cache.Get(ctx, args[1]); err != nil {
@ -565,6 +578,9 @@ func (b *bisyncTest) runTestStep(ctx context.Context, line string) (err error) {
b.checkArgs(args, 1, 1)
return b.listSubdirs(ctx, args[1])
case "bisync":
ci.NoUnicodeNormalization = false
ci.IgnoreCaseSync = false
// ci.FixCase = true
return b.runBisync(ctx, args[1:])
case "test-func":
b.TestFn = testFunc
@ -668,6 +684,16 @@ func (b *bisyncTest) runBisync(ctx context.Context, args []string) (err error) {
fs2 = addSubdir(b.path2, val)
case "ignore-listing-checksum":
opt.IgnoreListingChecksum = true
case "no-norm":
ci.NoUnicodeNormalization = true
ci.IgnoreCaseSync = false
case "norm":
ci.NoUnicodeNormalization = false
ci.IgnoreCaseSync = true
case "fix-case":
ci.NoUnicodeNormalization = false
ci.IgnoreCaseSync = true
ci.FixCase = true
default:
return fmt.Errorf("invalid bisync option %q", arg)
}

10
cmd/bisync/deltas.go
View file

@ -381,11 +381,17 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change
}
}
// find alternate names according to normalization settings
altNames1to2 := bilib.Names{}
altNames2to1 := bilib.Names{}
b.findAltNames(ctx, b.fs1, copy2to1, b.newListing1, altNames2to1)
b.findAltNames(ctx, b.fs2, copy1to2, b.newListing2, altNames1to2)
// Do the batch operation
if copy2to1.NotEmpty() {
changes1 = true
b.indent("Path2", "Path1", "Do queued copies to")
results2to1, err = b.fastCopy(ctx, b.fs2, b.fs1, copy2to1, "copy2to1")
results2to1, err = b.fastCopy(ctx, b.fs2, b.fs1, copy2to1, "copy2to1", altNames2to1)
if err != nil {
return
}
@ -397,7 +403,7 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change
if copy1to2.NotEmpty() {
changes2 = true
b.indent("Path1", "Path2", "Do queued copies to")
results1to2, err = b.fastCopy(ctx, b.fs1, b.fs2, copy1to2, "copy1to2")
results1to2, err = b.fastCopy(ctx, b.fs1, b.fs2, copy1to2, "copy1to2", altNames1to2)
if err != nil {
return
}

68
cmd/bisync/listing.go
View file

@ -20,6 +20,7 @@ import (
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fs/operations"
"golang.org/x/exp/slices"
"golang.org/x/text/unicode/norm"
)
// ListingHeader defines first line of a listing
@ -400,6 +401,18 @@ func ConvertPrecision(Modtime time.Time, dst fs.Fs) time.Time {
return Modtime
}
// ApplyTransforms handles unicode and case normalization
func ApplyTransforms(ctx context.Context, dst fs.Fs, s string) string {
ci := fs.GetConfig(ctx)
if !ci.NoUnicodeNormalization {
s = norm.NFC.String(s)
}
if ci.IgnoreCaseSync || dst.Features().CaseInsensitive {
s = strings.ToLower(s)
}
return s
}
// modifyListing will modify the listing based on the results of the sync
func (b *bisyncRun) modifyListing(ctx context.Context, src fs.Fs, dst fs.Fs, results []Results, queues queues, is1to2 bool) (err error) {
queue := queues.copy2to1
@ -429,6 +442,10 @@ func (b *bisyncRun) modifyListing(ctx context.Context, src fs.Fs, dst fs.Fs, res
srcList.hash = src.Hashes().GetOne()
dstList.hash = dst.Hashes().GetOne()
}
dstListNew, err := b.loadListing(dstListing + "-new")
if err != nil {
return fmt.Errorf("cannot read new listing: %w", err)
}
srcWinners := newFileList()
dstWinners := newFileList()
@ -466,20 +483,47 @@ func (b *bisyncRun) modifyListing(ctx context.Context, src fs.Fs, dst fs.Fs, res
}
updateLists := func(side string, winners, list *fileList) {
// removals from side
for _, oldFile := range queue.ToList() {
if !winners.has(oldFile) && list.has(oldFile) && !errors.has(oldFile) {
list.remove(oldFile)
fs.Debugf(nil, "decision: removed from %s: %v", side, oldFile)
} else if winners.has(oldFile) {
for _, queueFile := range queue.ToList() {
if !winners.has(queueFile) && list.has(queueFile) && !errors.has(queueFile) {
// removals from side
list.remove(queueFile)
fs.Debugf(nil, "decision: removed from %s: %v", side, queueFile)
} else if winners.has(queueFile) {
// copies to side
new := winners.get(oldFile)
list.put(oldFile, new.size, new.time, new.hash, new.id, new.flags)
fs.Debugf(nil, "decision: copied to %s: %v", side, oldFile)
new := winners.get(queueFile)
// handle normalization according to settings
ci := fs.GetConfig(ctx)
if side == "dst" && (!ci.NoUnicodeNormalization || ci.IgnoreCaseSync || dst.Features().CaseInsensitive) {
// search list for existing file that matches queueFile when normalized
normalizedName := ApplyTransforms(ctx, dst, queueFile)
matchFound := false
matchedName := ""
for _, filename := range dstListNew.list {
if ApplyTransforms(ctx, dst, filename) == normalizedName {
matchFound = true
matchedName = filename // original, not normalized
break
}
}
if matchFound && matchedName != queueFile {
// use the (non-identical) existing name, unless --fix-case
if ci.FixCase {
fs.Debugf(direction, "removing %s and adding %s as --fix-case was specified", matchedName, queueFile)
list.remove(matchedName)
} else {
fs.Debugf(direction, "casing/unicode difference detected. using %s instead of %s", matchedName, queueFile)
queueFile = matchedName
}
}
}
list.put(queueFile, new.size, new.time, new.hash, new.id, new.flags)
fs.Debugf(nil, "decision: copied to %s: %v", side, queueFile)
} else {
fs.Debugf(oldFile, "file in queue but missing from %s transfers", side)
if err := filterRecheck.AddFile(oldFile); err != nil {
fs.Debugf(oldFile, "error adding file to recheck filter: %v", err)
fs.Debugf(queueFile, "file in queue but missing from %s transfers", side)
if err := filterRecheck.AddFile(queueFile); err != nil {
fs.Debugf(queueFile, "error adding file to recheck filter: %v", err)
}
}
}

4
cmd/bisync/march.go
View file

@ -183,7 +183,3 @@ func whichPath(isPath1 bool) string {
}
return s
}
// TODO:
// equality check?
// unicode stuff

21
cmd/bisync/operations.go
View file

@ -428,8 +428,11 @@ func (b *bisyncRun) resync(octx, fctx context.Context) error {
if len(copy2to1) > 0 {
b.indent("Path2", "Path1", "Resync is doing queued copies to")
resync2to1 := bilib.ToNames(copy2to1)
altNames2to1 := bilib.Names{}
b.findAltNames(octx, b.fs1, resync2to1, b.newListing1, altNames2to1)
// octx does not have extra filters!
results2to1, err = b.fastCopy(octx, b.fs2, b.fs1, bilib.ToNames(copy2to1), "resync-copy2to1")
results2to1, err = b.fastCopy(octx, b.fs2, b.fs1, resync2to1, "resync-copy2to1", altNames2to1)
if err != nil {
b.critical = true
return err
@ -516,15 +519,27 @@ func (b *bisyncRun) checkSync(listing1, listing2 string) error {
return fmt.Errorf("cannot read prior listing of Path2: %w", err)
}
transformList := func(files *fileList, fs fs.Fs) *fileList {
transformed := newFileList()
for _, file := range files.list {
f := files.get(file)
transformed.put(ApplyTransforms(context.Background(), fs, file), f.size, f.time, f.hash, f.id, f.flags)
}
return transformed
}
files1Transformed := transformList(files1, b.fs1)
files2Transformed := transformList(files2, b.fs2)
ok := true
for _, file := range files1.list {
if !files2.has(file) {
if !files2.has(file) && !files2Transformed.has(ApplyTransforms(context.Background(), b.fs1, file)) {
b.indent("ERROR", file, "Path1 file not found in Path2")
ok = false
}
}
for _, file := range files2.list {
if !files1.has(file) {
if !files1.has(file) && !files1Transformed.has(ApplyTransforms(context.Background(), b.fs2, file)) {
b.indent("ERROR", file, "Path2 file not found in Path1")
ok = false
}

31
cmd/bisync/queue.go
View file

@ -14,7 +14,6 @@ import (
"github.com/rclone/rclone/fs/filter"
"github.com/rclone/rclone/fs/operations"
"github.com/rclone/rclone/fs/sync"
"golang.org/x/text/unicode/norm"
)
// Results represents a pair of synced files, as reported by the LoggerFn
@ -130,7 +129,7 @@ func ReadResults(results io.Reader) []Results {
return slice
}
func (b *bisyncRun) fastCopy(ctx context.Context, fsrc, fdst fs.Fs, files bilib.Names, queueName string) ([]Results, error) {
func (b *bisyncRun) fastCopy(ctx context.Context, fsrc, fdst fs.Fs, files bilib.Names, queueName string, altNames bilib.Names) ([]Results, error) {
if err := b.saveQueue(files, queueName); err != nil {
return nil, err
}
@ -140,9 +139,12 @@ func (b *bisyncRun) fastCopy(ctx context.Context, fsrc, fdst fs.Fs, files bilib.
if err := filterCopy.AddFile(file); err != nil {
return nil, err
}
// macOS
if err := filterCopy.AddFile(norm.NFD.String(file)); err != nil {
return nil, err
}
if altNames.NotEmpty() {
for _, file := range altNames.ToList() {
if err := filterCopy.AddFile(file); err != nil {
return nil, err
}
}
}
@ -252,3 +254,22 @@ func (b *bisyncRun) saveQueue(files bilib.Names, jobName string) error {
queueFile := fmt.Sprintf("%s.%s.que", b.basePath, jobName)
return files.Save(queueFile)
}
func (b *bisyncRun) findAltNames(ctx context.Context, dst fs.Fs, queue bilib.Names, newListing string, altNames bilib.Names) {
ci := fs.GetConfig(ctx)
if queue.NotEmpty() && (!ci.NoUnicodeNormalization || ci.IgnoreCaseSync || b.fs1.Features().CaseInsensitive || b.fs2.Features().CaseInsensitive) {
// search list for existing file that matches queueFile when normalized
for _, queueFile := range queue.ToList() {
normalizedName := ApplyTransforms(ctx, dst, queueFile)
candidates, err := b.loadListing(newListing)
if err != nil {
fs.Errorf(candidates, "cannot read new listing: %v", err)
}
for _, filename := range candidates.list {
if ApplyTransforms(ctx, dst, filename) == normalizedName && filename != queueFile {
altNames.Add(filename) // original, not normalized
}
}
}
}
}

17
cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst vendored
View file

@ -1,10 +1,9 @@
# bisync listing v1 from test
- 6148 md5:23b446fda9938c607142c5133cf90689 - 2000-01-01T00:00:00.000000000+0000 ".DS_Store"
- 109 md5:294d25b294ff26a5243dba914ac3fbf7 - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy3.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy4.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy5.txt"
- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-02T00:00:00.000000000+0000 "file1.txt"
- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-02T00:00:00.000000000+0000 "subdir/file20.txt"
- 109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt"
- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt"
- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy3.txt"
- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy4.txt"
- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy5.txt"
- 19 - - 2001-01-02T00:00:00.000000000+0000 "file1.txt"
- 19 - - 2001-01-02T00:00:00.000000000+0000 "subdir/file20.txt"

1
cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst-new vendored
View file

@ -1,5 +1,4 @@
# bisync listing v1 from test
- 6148 md5:23b446fda9938c607142c5133cf90689 - 2000-01-01T00:00:00.000000000+0000 ".DS_Store"
- 109 md5:294d25b294ff26a5243dba914ac3fbf7 - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt"

17
cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst-old vendored
View file

@ -1,10 +1,9 @@
# bisync listing v1 from test
- 6148 md5:23b446fda9938c607142c5133cf90689 - 2000-01-01T00:00:00.000000000+0000 ".DS_Store"
- 109 md5:294d25b294ff26a5243dba914ac3fbf7 - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy3.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy4.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy5.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "subdir/file20.txt"
- 109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt"
- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt"
- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy3.txt"
- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy4.txt"
- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy5.txt"
- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.txt"
- 0 - - 2000-01-01T00:00:00.000000000+0000 "subdir/file20.txt"

17
cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst vendored
View file

@ -1,10 +1,9 @@
# bisync listing v1 from test
- 6148 md5:23b446fda9938c607142c5133cf90689 - 2000-01-01T00:00:00.000000000+0000 ".DS_Store"
- 109 md5:294d25b294ff26a5243dba914ac3fbf7 - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy3.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy4.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy5.txt"
- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-02T00:00:00.000000000+0000 "file1.txt"
- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-02T00:00:00.000000000+0000 "subdir/file20.txt"
- 109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt"
- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt"
- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy3.txt"
- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy4.txt"
- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy5.txt"
- 19 - - 2001-01-02T00:00:00.000000000+0000 "file1.txt"
- 19 - - 2001-01-02T00:00:00.000000000+0000 "subdir/file20.txt"

1
cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst-new vendored
View file

@ -1,5 +1,4 @@
# bisync listing v1 from test
- 6148 md5:23b446fda9938c607142c5133cf90689 - 2000-01-01T00:00:00.000000000+0000 ".DS_Store"
- 109 md5:294d25b294ff26a5243dba914ac3fbf7 - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt"

17
cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst-old vendored
View file

@ -1,10 +1,9 @@
# bisync listing v1 from test
- 6148 md5:23b446fda9938c607142c5133cf90689 - 2000-01-01T00:00:00.000000000+0000 ".DS_Store"
- 109 md5:294d25b294ff26a5243dba914ac3fbf7 - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy3.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy4.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy5.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "subdir/file20.txt"
- 109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt"
- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt"
- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy3.txt"
- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy4.txt"
- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy5.txt"
- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.txt"
- 0 - - 2000-01-01T00:00:00.000000000+0000 "subdir/file20.txt"

2
cmd/bisync/testdata/test_check_access_filters/golden/test.log vendored
View file

@ -142,8 +142,8 @@ INFO : Path2: 1 changes: 0 new, 0 newer, 0 older, 1 deleted
INFO : Checking access health
ERROR : Access test failed: Path1 count 3, Path2 count 4 - RCLONE_TEST
ERROR : -  Access test failed: Path1 file not found in Path2 - RCLONE_TEST
ERROR : -  Access test failed: Path2 file not found in Path1 - subdir/RCLONE_TEST
ERROR : -  Access test failed: Path2 file not found in Path1 - subdirX/subdirX1/RCLONE_TEST
ERROR : -  Access test failed: Path2 file not found in Path1 - subdir/RCLONE_TEST
ERROR : Bisync critical error: check file check failed
ERROR : Bisync aborted. Must run --resync to recover.
Bisync error: bisync aborted

3
cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.copy1to2.que vendored Normal file
View file

@ -0,0 +1,3 @@
"folder/HeLlO,wOrLd!.txt"
"folder/éééö.txt"
"測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt"

3
cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.copy2to1.que vendored Normal file
View file

@ -0,0 +1,3 @@
"file1.txt"
"folder/hello,WORLD!.txt"
"folder/éééö.txt"

5
cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path1.lst vendored Normal file
View file

@ -0,0 +1,5 @@
# bisync listing v1 from test
- 19 - - 2001-01-05T00:00:00.000000000+0000 "file1.txt"
- 19 - - 2001-01-05T00:00:00.000000000+0000 "folder/HeLlO,wOrLd!.txt"
- 19 - - 2001-01-05T00:00:00.000000000+0000 "folder/éééö.txt"
- 19 - - 2001-01-05T00:00:00.000000000+0000 "測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt"

5
cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path1.lst-new vendored Normal file
View file

@ -0,0 +1,5 @@
# bisync listing v1 from test
- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-03T00:00:00.000000000+0000 "file1.txt"
- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-05T00:00:00.000000000+0000 "folder/HeLlO,wOrLd!.txt"
- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-05T00:00:00.000000000+0000 "folder/éééö.txt"
- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-05T00:00:00.000000000+0000 "測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt"

5
cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path1.lst-old vendored Normal file
View file

@ -0,0 +1,5 @@
# bisync listing v1 from test
- 19 - - 2001-01-03T00:00:00.000000000+0000 "file1.txt"
- 19 - - 2001-01-03T00:00:00.000000000+0000 "folder/HeLlO,wOrLd!.txt"
- 19 - - 2001-01-03T00:00:00.000000000+0000 "folder/éééö.txt"
- 19 - - 2001-01-02T00:00:00.000000000+0000 "測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt"

5
cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst vendored Normal file
View file

@ -0,0 +1,5 @@
# bisync listing v1 from test
- 19 - - 2001-01-05T00:00:00.000000000+0000 "file1.txt"
- 19 - - 2001-01-05T00:00:00.000000000+0000 "folder/hello,WORLD!.txt"
- 19 - - 2001-01-05T00:00:00.000000000+0000 "folder/éééö.txt"
- 19 - - 2001-01-05T00:00:00.000000000+0000 "測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt"

5
cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst-new vendored Normal file
View file

@ -0,0 +1,5 @@
# bisync listing v1 from test
- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-05T00:00:00.000000000+0000 "file1.txt"
- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-05T00:00:00.000000000+0000 "folder/hello,WORLD!.txt"
- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-05T00:00:00.000000000+0000 "folder/éééö.txt"
- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-02T00:00:00.000000000+0000 "測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt"

5
cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst-old vendored Normal file
View file

@ -0,0 +1,5 @@
# bisync listing v1 from test
- 19 - - 2001-01-03T00:00:00.000000000+0000 "file1.txt"
- 19 - - 2001-01-03T00:00:00.000000000+0000 "folder/hello,WORLD!.txt"
- 19 - - 2001-01-03T00:00:00.000000000+0000 "folder/éééö.txt"
- 19 - - 2001-01-02T00:00:00.000000000+0000 "測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt"

2
cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.resync-copy2to1.que vendored Normal file
View file

@ -0,0 +1,2 @@
"folder/hello,WORLD!.txt"
"folder/éééö.txt"

144
cmd/bisync/testdata/test_normalization/golden/test.log vendored Normal file
View file

@ -0,0 +1,144 @@
(01) : test normalization
(02) : touch-copy 2001-01-02 {datadir/}file1.txt {path2/}
(03) : test initial bisync
(04) : bisync resync
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying unique Path2 files to Path1
INFO : Resynching Path1 to Path2
INFO : Resync updating listings
INFO : Bisync successful
(05) : copy-as-NFC {datadir/}file1.txt {path1/}測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éééö 測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éééö.txt
(06) : copy-as-NFC {datadir/}file1.txt {path1/}folder éééö.txt
(07) : copy-as-NFC {datadir/}file1.txt {path1/}folder HeLlO,wOrLd!.txt
(08) : touch-copy 2001-01-03 {datadir/}file1.txt {path2/}
(09) : copy-as-NFD {datadir/}file1.txt {path2/}folder éééö.txt
(10) : copy-as-NFD {datadir/}file1.txt {path2/}folder hello,WORLD!.txt
(11) : test bisync run with fix-case
(12) : bisync fix-case
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Building Path1 and Path2 listings
INFO : Path1 checking for diffs
INFO : - Path1 File is new - folder/HeLlO,wOrLd!.txt
INFO : - Path1 File is new - folder/éééö.txt
INFO : - Path1 File is new - "測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt"
INFO : Path1: 3 changes: 3 new, 0 newer, 0 older, 0 deleted
INFO : Path2 checking for diffs
INFO : - Path2 File is newer - file1.txt
INFO : - Path2 File is new - folder/éééö.txt
INFO : - Path2 File is new - folder/hello,WORLD!.txt
INFO : Path2: 3 changes: 2 new, 1 newer, 0 older, 0 deleted
INFO : Applying changes
INFO : - Path1 Queue copy to Path2 - {path2/}folder/HeLlO,wOrLd!.txt
INFO : - Path1 Queue copy to Path2 - {path2/}folder/éééö.txt
INFO : - Path1 Queue copy to Path2 - "{path2/}測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt"
INFO : - Path2 Queue copy to Path1 - {path1/}file1.txt
INFO : - Path2 Queue copy to Path1 - {path1/}folder/éééö.txt
INFO : - Path2 Queue copy to Path1 - {path1/}folder/hello,WORLD!.txt
INFO : - Path2 Do queued copies to - Path1
INFO : folder/HeLlO,wOrLd!.txt: Fixed case by renaming to: folder/hello,WORLD!.txt
INFO : folder/éééö.txt: Fixed case by renaming to: folder/éééö.txt
INFO : - Path1 Do queued copies to - Path2
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
(13) : purge-children {path1/}
(14) : purge-children {path2/}
(15) : touch-copy 2001-01-02 {datadir/}file1.txt {path2/}
(16) : bisync resync
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying unique Path2 files to Path1
INFO : - Path2 Resync will copy to Path1 - file1.txt
INFO : - Path2 Resync is doing queued copies to - Path1
INFO : Resynching Path1 to Path2
INFO : Resync updating listings
INFO : Bisync successful
(17) : copy-as-NFC {datadir/}file1.txt {path1/}測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éééö 測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éééö.txt
(18) : copy-as-NFC {datadir/}file1.txt {path1/}folder éééö.txt
(19) : copy-as-NFC {datadir/}file1.txt {path1/}folder HeLlO,wOrLd!.txt
(20) : touch-copy 2001-01-03 {datadir/}file1.txt {path2/}
(21) : copy-as-NFD {datadir/}file1.txt {path2/}folder éééö.txt
(22) : copy-as-NFD {datadir/}file1.txt {path2/}folder hello,WORLD!.txt
(23) : test bisync run with normalization
(24) : bisync norm force
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Building Path1 and Path2 listings
INFO : Path1 checking for diffs
INFO : - Path1 File is new - folder/HeLlO,wOrLd!.txt
INFO : - Path1 File is new - folder/éééö.txt
INFO : - Path1 File is new - "測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt"
INFO : Path1: 3 changes: 3 new, 0 newer, 0 older, 0 deleted
INFO : Path2 checking for diffs
INFO : - Path2 File is newer - file1.txt
INFO : - Path2 File is new - folder/éééö.txt
INFO : - Path2 File is new - folder/hello,WORLD!.txt
INFO : Path2: 3 changes: 2 new, 1 newer, 0 older, 0 deleted
INFO : Applying changes
INFO : - Path1 Queue copy to Path2 - {path2/}folder/HeLlO,wOrLd!.txt
INFO : - Path1 Queue copy to Path2 - {path2/}folder/éééö.txt
INFO : - Path1 Queue copy to Path2 - "{path2/}測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt"
INFO : - Path2 Queue copy to Path1 - {path1/}file1.txt
INFO : - Path2 Queue copy to Path1 - {path1/}folder/éééö.txt
INFO : - Path2 Queue copy to Path1 - {path1/}folder/hello,WORLD!.txt
INFO : - Path2 Do queued copies to - Path1
INFO : - Path1 Do queued copies to - Path2
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
(25) : test resync
(26) : bisync resync norm
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying unique Path2 files to Path1
INFO : - Path2 Resync will copy to Path1 - folder/éééö.txt
INFO : - Path2 Resync will copy to Path1 - folder/hello,WORLD!.txt
INFO : - Path2 Resync is doing queued copies to - Path1
INFO : Resynching Path1 to Path2
INFO : Resync updating listings
INFO : Bisync successful
(27) : test changed on both paths
(28) : touch-copy 2001-01-05 {datadir/}file1.txt {path2/}
(29) : copy-as-NFC {datadir/}file1.txt {path1/}測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éééö 測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éééö.txt
(30) : copy-as-NFC {datadir/}file1.txt {path1/}folder éééö.txt
(31) : copy-as-NFC {datadir/}file1.txt {path1/}folder HeLlO,wOrLd!.txt
(32) : copy-as-NFD {datadir/}file1.txt {path2/}folder éééö.txt
(33) : copy-as-NFD {datadir/}file1.txt {path2/}folder hello,WORLD!.txt
(34) : bisync norm
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Building Path1 and Path2 listings
INFO : Path1 checking for diffs
INFO : - Path1 File is newer - folder/HeLlO,wOrLd!.txt
INFO : - Path1 File is newer - folder/éééö.txt
INFO : - Path1 File is newer - "測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt"
INFO : Path1: 3 changes: 0 new, 3 newer, 0 older, 0 deleted
INFO : Path2 checking for diffs
INFO : - Path2 File is newer - file1.txt
INFO : - Path2 File is newer - folder/éééö.txt
INFO : - Path2 File is newer - folder/hello,WORLD!.txt
INFO : Path2: 3 changes: 0 new, 3 newer, 0 older, 0 deleted
INFO : Applying changes
INFO : - Path1 Queue copy to Path2 - {path2/}folder/HeLlO,wOrLd!.txt
INFO : - Path1 Queue copy to Path2 - {path2/}folder/éééö.txt
INFO : - Path1 Queue copy to Path2 - "{path2/}測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt"
INFO : - Path2 Queue copy to Path1 - {path1/}file1.txt
INFO : - Path2 Queue copy to Path1 - {path1/}folder/éééö.txt
INFO : - Path2 Queue copy to Path1 - {path1/}folder/hello,WORLD!.txt
INFO : - Path2 Do queued copies to - Path1
INFO : - Path1 Do queued copies to - Path2
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful

1
cmd/bisync/testdata/test_normalization/initial/RCLONE_TEST vendored Normal file
View file

@ -0,0 +1 @@
This file is used for testing the health of rclone accesses to the local/remote file system. Do not delete.

0
cmd/bisync/testdata/test_normalization/initial/file1.txt vendored Normal file
View file

7
cmd/bisync/testdata/test_normalization/initial/測試_Русский_ _ _ě_áñ/filename_contains_ě_.txt vendored Normal file
View file

@ -0,0 +1,7 @@
mañana << Coded as xF1, which is the unicode ID name, but not the correct byte stream coding.
filename_contains_\u011b_
filename_contains_e_
_mañana_funcionará.txt
file_enconde_mañana_funcionará << Valid byte stream unicode is read correctly

7
cmd/bisync/testdata/test_normalization/initial/測試_Русский_ _ _ě_áñ/filename_contains_ࢺ_.txt vendored Normal file
View file

@ -0,0 +1,7 @@
mañana << Coded as xF1, which is the unicode ID name, but not the correct byte stream coding.
filename_contains_\u011b_
filename_contains_e_
_mañana_funcionará.txt
file_enconde_mañana_funcionará << Valid byte stream unicode is read correctly

0
cmd/bisync/testdata/test_normalization/initial/測試_Русский_ _ _ě_áñ/測試_check file vendored Normal file
View file

1
cmd/bisync/testdata/test_normalization/modfiles/file1.txt vendored Normal file
View file

@ -0,0 +1 @@
This file is newer

5
cmd/bisync/testdata/test_normalization/modfiles/測試_filtersfile.txt vendored Normal file
View file

@ -0,0 +1,5 @@
# Test filters file
# Note that this test checks for bisync's access to and usage of a filters file, not an extensive test of rclone's filter capability
# Exclude fileZ.txt in root only. The copy in the subdir should be found and synched.
- /fileZ.txt

53
cmd/bisync/testdata/test_normalization/scenario.txt vendored Normal file
View file

@ -0,0 +1,53 @@
test normalization
# Tests support for --no-unicode-normalization and --ignore-case-sync
# note: this test is written carefully to be runnable regardless of case/unicode sensitivity
# i.e. the results should be the same on linux and macOS
# force specific modification time since file time is lost through git
touch-copy 2001-01-02 {datadir/}file1.txt {path2/}
test initial bisync
bisync resync
# copy NFC version to Path1
copy-as-NFC {datadir/}file1.txt {path1/}測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éééö 測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éééö.txt
copy-as-NFC {datadir/}file1.txt {path1/}folder éééö.txt
copy-as-NFC {datadir/}file1.txt {path1/}folder HeLlO,wOrLd!.txt
# place newer NFD version on Path2
touch-copy 2001-01-03 {datadir/}file1.txt {path2/}
copy-as-NFD {datadir/}file1.txt {path2/}folder éééö.txt
copy-as-NFD {datadir/}file1.txt {path2/}folder hello,WORLD!.txt
test bisync run with fix-case
bisync fix-case
# purge and reset
purge-children {path1/}
purge-children {path2/}
touch-copy 2001-01-02 {datadir/}file1.txt {path2/}
bisync resync
# copy NFC version to Path1
copy-as-NFC {datadir/}file1.txt {path1/}測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éééö 測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éééö.txt
copy-as-NFC {datadir/}file1.txt {path1/}folder éééö.txt
copy-as-NFC {datadir/}file1.txt {path1/}folder HeLlO,wOrLd!.txt
# place newer NFD version on Path2
touch-copy 2001-01-03 {datadir/}file1.txt {path2/}
copy-as-NFD {datadir/}file1.txt {path2/}folder éééö.txt
copy-as-NFD {datadir/}file1.txt {path2/}folder hello,WORLD!.txt
test bisync run with normalization
bisync norm force
test resync
bisync resync norm
test changed on both paths
touch-copy 2001-01-05 {datadir/}file1.txt {path2/}
copy-as-NFC {datadir/}file1.txt {path1/}測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éééö 測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éééö.txt
copy-as-NFC {datadir/}file1.txt {path1/}folder éééö.txt
copy-as-NFC {datadir/}file1.txt {path1/}folder HeLlO,wOrLd!.txt
copy-as-NFD {datadir/}file1.txt {path2/}folder éééö.txt
copy-as-NFD {datadir/}file1.txt {path2/}folder hello,WORLD!.txt
bisync norm

20
docs/content/bisync.md
View file

@ -588,13 +588,26 @@ and refuses to run again until the user runs a `--resync` (unless using `--resil
The best workaround at the moment is to set any backend-specific flags in the [config file](/commands/rclone_config/)
instead of specifying them with command flags. (You can still override them as needed for other rclone commands.)
### Case sensitivity
### Case (and unicode) sensitivity {#case-sensitivity}
Synching with **case-insensitive** filesystems, such as Windows or `Box`,
can result in file name conflicts. This will be fixed in a future release.
The near-term workaround is to make sure that files on both sides
can result in unusual behavior. As of `v1.65`, case and unicode form differences no longer cause critical errors,
however they may cause unexpected delta outcomes, due to the delta engine still being case-sensitive.
This will be fixed in a future release. The near-term workaround is to make sure that files on both sides
don't have spelling case differences (`Smile.jpg` vs. `smile.jpg`).
The same limitation applies to Unicode normalization forms.
This [particularly applies to macOS](https://github.com/rclone/rclone/issues/7270),
which prefers NFD and sometimes auto-converts filenames from the NFC form used by most other platforms.
This should no longer cause bisync to fail entirely, but may cause surprising delta results, as explained above.
See the following options (all of which are supported by bisync) to control this behavior more granularly:
- [`--fix-case`](/docs/#fix-case)
- [`--ignore-case-sync`](/docs/#ignore-case-sync)
- [`--no-unicode-normalization`](/docs/#no-unicode-normalization)
- [`--local-unicode-normalization`](/local/#local-unicode-normalization) and
[`--local-case-sensitive`](/local/#local-case-sensitive) (caution: these are normally not what you want.)
## Windows support {#windows}
Bisync has been tested on Windows 8.1, Windows 10 Pro 64-bit and on Windows
@ -1278,6 +1291,7 @@ about _Unison_ and synchronization in general.
* A few basic terminal colors are now supported, controllable with [`--color`](/docs/#color-when) (`AUTO`|`NEVER`|`ALWAYS`)
* Initial listing snapshots of Path1 and Path2 are now generated concurrently, using the same "march" infrastructure as `check` and `sync`,
for performance improvements and less [risk of error](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=4.%20Listings%20should%20alternate%20between%20paths%20to%20minimize%20errors).
* Better handling of unicode normalization and case insensitivity, support for [`--fix-case`](/docs/#fix-case), [`--ignore-case-sync`](/docs/#ignore-case-sync), [`--no-unicode-normalization`](/docs/#no-unicode-normalization)
### `v1.64`
* Fixed an [issue](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=1.%20Dry%20runs%20are%20not%20completely%20dry)