TrueCloudLab/rclone
16
0
Fork 2
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

sync: implement directory sync for mod times and metadata

Browse source 
Directory mod times are synced by default if the backend is capable
and directory metadata is synced if the --metadata flag is provided
and the backend is capable.

This updates the bisync golden tests also which were affected by
--dry-run setting of directory modtimes.

Fixes #6685
This commit is contained in:
Nick Craig-Wood 2024-02-06 16:00:34 +00:00
parent 15579c2195
commit f5f86786b2
23 changed files with 504 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
cmd/bisync/bisync_test.go
View file

@ -86,6 +86,10 @@ var logReplacements = []string{
`^.*?"SlowHashDetected":.*?$`, dropMe,
`^.*? for same-side diffs on .*?$`, dropMe,
`^.*?Downloading hashes.*?$`, dropMe,
// ignore timestamps in directory time updates
`^(INFO : .*?: Made directory with (metadata|modification time)).*$`, `$1`,
// ignore sizes in directory time updates
`^(NOTICE: .*?: Skipped set directory modification time as --dry-run is set).*$`, `$1`,
}
// Some dry-run messages differ depending on the particular remote.
@ -121,6 +125,9 @@ var logHoppers = []string{
// order of files re-checked prior to a conflict rename
`ERROR : .*: md5 differ.*`,
// Directory modification time setting can happen in any order
`INFO : .*: (Set directory modification time|Made directory with metadata).*`,
}
// Some log lines can contain Windows path separator that must be

8
cmd/bisync/testdata/test_all_changed/golden/test.log vendored
View file

@ -16,7 +16,11 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -59,6 +63,8 @@ INFO : - Path1 Queue copy to Path2 - {
INFO : - Path1 Queue copy to Path2 - {path2/}file1.txt
INFO : - Path1 Queue copy to Path2 - {path2/}subdir/file20.txt
INFO : - Path1 Do queued copies to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -133,6 +139,8 @@ INFO : - Path1 Queue copy to Path2 - {
INFO : - Path1 Queue copy to Path2 - {path2/}file1.txt
INFO : - Path1 Queue copy to Path2 - {path2/}subdir/file20.txt
INFO : - Path1 Do queued copies to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful

6
cmd/bisync/testdata/test_basic/golden/test.log vendored
View file

@ -16,7 +16,11 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -53,6 +57,8 @@ INFO : - Path1 Queue copy to Path2 - {
INFO : - Path2 Queue copy to Path1 - {path1/}file1.txt
INFO : - Path2 Do queued copies to - Path1
INFO : - Path1 Do queued copies to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful

12
cmd/bisync/testdata/test_check_access/golden/test.log vendored
View file

@ -16,7 +16,11 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -87,7 +91,11 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -180,7 +188,11 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful

24
cmd/bisync/testdata/test_check_access_filters/golden/test.log vendored
View file

@ -21,7 +21,15 @@ INFO : Using filters file {workdir/}exclude-other-filtersfile.txt
INFO : Storing filters file hash to {workdir/}exclude-other-filtersfile.txt.md5
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir/subdirB: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir/subdirB: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir/subdirB: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir/subdirB: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -136,7 +144,23 @@ INFO : Using filters file {workdir/}include-other-filtersfile.txt
INFO : Storing filters file hash to {workdir/}include-other-filtersfile.txt.md5
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdirX: Set directory modification time (using SetModTime)
INFO : subdirX/subdirX1: Set directory modification time (using SetModTime)
INFO : subdir/subdirB: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdirX: Set directory modification time (using SetModTime)
INFO : subdirX/subdirX1: Set directory modification time (using SetModTime)
INFO : subdir/subdirB: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdirX: Set directory modification time (using SetModTime)
INFO : subdir/subdirB: Set directory modification time (using SetModTime)
INFO : subdirX/subdirX1: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdirX: Set directory modification time (using SetModTime)
INFO : subdir/subdirB: Set directory modification time (using SetModTime)
INFO : subdirX/subdirX1: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful

8
cmd/bisync/testdata/test_check_filename/golden/test.log vendored
View file

@ -16,7 +16,11 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -90,7 +94,11 @@ INFO : Copying Path2 files to Path1
INFO : Checking access health
INFO : Found 2 matching ".chk_file" files on both paths
INFO : - Path2 Resync is copying files to - Path1
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful

4
cmd/bisync/testdata/test_createemptysrcdirs/golden/test.log vendored
View file

@ -147,7 +147,11 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
INFO : subdir: Made directory with metadata (mtime=2024-02-27T04:53:52.809861575-05:00)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful

12
cmd/bisync/testdata/test_extended_char_paths/golden/test.log vendored
View file

@ -84,7 +84,11 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -129,7 +133,11 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -174,7 +182,11 @@ INFO : Using filters file {workdir/}測試_filtersfile.txt
INFO : Storing filters file hash to {workdir/}測試_filtersfile.txt.md5
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful

12
cmd/bisync/testdata/test_extended_filenames/golden/test.log vendored
View file

@ -25,7 +25,11 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime)
INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime)
INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -112,7 +116,15 @@ INFO : - Path2 Queue copy to Path1 - "
INFO : - Path1 Queue delete - {path1/}subdir_with_ࢺ_/filename_contains_ě_.txt
INFO : - Path2 Queue copy to Path1 - {path1/}subdir_with_ࢺ_/filename_contains_ࢺ_p2s.txt
INFO : - Path2 Do queued copies to - Path1
INFO : subdir withâ<68>Šwhite space.txt: Made directory with metadata (mtime=2024-02-27T04:53:52.913860529-05:00)
INFO : subdir_rawchars_â<5F>™_<E284A2>_þ: Made directory with metadata (mtime=2024-02-27T04:53:52.913860529-05:00)
INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime)
INFO : subdir withâ<68>Šwhite space.txt: Set directory modification time (using SetModTime)
INFO : subdir_rawchars_â<5F>™_<E284A2>_þ: Set directory modification time (using SetModTime)
INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime)
INFO : - Path1 Do queued copies to - Path2
INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime)
INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime)
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful

6
cmd/bisync/testdata/test_filters/golden/test.log vendored
View file

@ -20,7 +20,11 @@ INFO : Using filters file {workdir/}filtersfile.flt
INFO : Storing filters file hash to {workdir/}filtersfile.flt.md5
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -54,6 +58,8 @@ INFO : Path2 checking for diffs
INFO : Applying changes
INFO : - Path1 Queue copy to Path2 - {path2/}subdir/fileZ.txt
INFO : - Path1 Do queued copies to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful

12
cmd/bisync/testdata/test_filtersfile_checks/golden/test.log vendored
View file

@ -16,7 +16,11 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -81,7 +85,11 @@ INFO : Using filters file {workdir/}filtersfile.txt
INFO : Storing filters file hash to {workdir/}filtersfile.txt.md5
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -146,7 +154,11 @@ INFO : Using filters file {workdir/}filtersfile.txt
INFO : Skipped storing filters file hash to {workdir/}filtersfile.txt.md5 as --dry-run is set
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
NOTICE: subdir: Skipped set directory modification time as --dry-run is set (size 4Ki)
NOTICE: {path1String}: Skipped set directory modification time as --dry-run is set
INFO : - Path1 Resync is copying files to - Path2
NOTICE: subdir: Skipped set directory modification time as --dry-run is set (size 4Ki)
NOTICE: {path2String}: Skipped set directory modification time as --dry-run is set
INFO : Resync updating listings
INFO : Bisync successful

10
cmd/bisync/testdata/test_ignorelistingchecksum/golden/test.log vendored
View file

@ -16,7 +16,11 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -33,7 +37,11 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -69,6 +77,8 @@ INFO : - Path1 Queue copy to Path2 - {
INFO : - Path2 Queue copy to Path1 - {path1/}file1.txt
INFO : - Path2 Do queued copies to - Path1
INFO : - Path1 Do queued copies to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful

24
cmd/bisync/testdata/test_normalization/golden/test.log vendored
View file

@ -17,7 +17,11 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -70,8 +74,12 @@ INFO : - Path1 Queue copy to Path2 - "
INFO : - Path2 Queue copy to Path1 - {path1/}file1.txt
INFO : - Path2 Do queued copies to - Path1
INFO : - Path1 Do queued copies to - Path2
INFO : 測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Made directory with metadata (mtime=2024-02-27T04:53:52.993859723-05:00)
INFO : folder: Set directory modification time (using SetModTime)
INFO : folder/hello,WORLD!.txt: Fixed case by renaming to: folder/HeLlO,wOrLd!.txt
INFO : folder/éééö.txt: Fixed case by renaming to: folder/éééö.txt
INFO : 測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Set directory modification time (using SetModTime)
INFO : folder: Set directory modification time (using SetModTime)
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -150,7 +158,11 @@ INFO : - Path2 Queue copy to Path1 - {
INFO : - Path1 Queue copy to Path2 - "{path2/}測試_Руский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éö/測試_Руский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éö.txt"
INFO : - Path2 Queue copy to Path1 - {path1/}file1.txt
INFO : - Path2 Do queued copies to - Path1
INFO : folder: Set directory modification time (using SetModTime)
INFO : folder: Set directory modification time (using SetModTime)
INFO : - Path1 Do queued copies to - Path2
INFO : 測試_Руский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Made directory with metadata (mtime=2024-02-27T04:53:53.001859642-05:00)
INFO : 測試_Руский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Set directory modification time (using SetModTime)
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -170,7 +182,15 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
INFO : folder: Set directory modification time (using SetModTime)
INFO : 測試_Руский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Set directory modification time (using SetModTime)
INFO : folder: Set directory modification time (using SetModTime)
INFO : 測試_Руский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : folder: Set directory modification time (using SetModTime)
INFO : 測試_Руский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Set directory modification time (using SetModTime)
INFO : folder: Set directory modification time (using SetModTime)
INFO : 測試_Руский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -210,6 +230,10 @@ INFO : - Path1 Queue copy to Path2 - "
INFO : - Path2 Queue copy to Path1 - {path1/}file1.txt
INFO : - Path2 Do queued copies to - Path1
INFO : - Path1 Do queued copies to - Path2
INFO : folder: Set directory modification time (using SetModTime)
INFO : 測試_Руский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Set directory modification time (using SetModTime)
INFO : folder: Set directory modification time (using SetModTime)
INFO : 測試_Руский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Set directory modification time (using SetModTime)
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful

18
cmd/bisync/testdata/test_rclone_args/golden/test.log vendored
View file

@ -15,7 +15,11 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -107,7 +111,11 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -141,6 +149,8 @@ INFO : Applying changes
INFO : - Path2 Queue copy to Path1 - {path1/}file2.txt
INFO : - Path2 Queue copy to Path1 - {path1/}subdir/file21.txt
INFO : - Path2 Do queued copies to - Path1
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -158,7 +168,11 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -200,7 +214,11 @@ INFO : - Path1 Queue copy to Path2 - {
INFO : - Path2 Queue copy to Path1 - {path1/}file2.txt
INFO : - Path2 Queue copy to Path1 - {path1/}subdir/file21.txt
INFO : - Path2 Do queued copies to - Path1
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Do queued copies to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful

2
cmd/bisync/testdata/test_resolve/golden/test.log vendored
View file

@ -283,8 +283,8 @@ INFO : Path2: 2 changes:  1 new,  1 modified, 
INFO : (Modified:  1 newer,  0 older,  1 larger,  0 smaller)
INFO : Applying changes
INFO : Checking potential conflicts...
ERROR : file2.txt: md5 differ
ERROR : file1.txt: md5 differ
ERROR : file2.txt: md5 differ
NOTICE: {path2String}: 2 differences found
NOTICE: {path2String}: 2 errors while checking
INFO : Finished checking the potential conflicts. 2 differences found

6
cmd/bisync/testdata/test_rmdirs/golden/test.log vendored
View file

@ -16,7 +16,11 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -45,6 +49,8 @@ INFO : Path2 checking for diffs
INFO : Applying changes
INFO : - Path2 Queue delete - {path2/}subdir/file20.txt
INFO : - Path1 Do queued copies to - Path2
INFO : subdir: Set directory modification time (using SetModTime)
INFO : subdir: Set directory modification time (using SetModTime)
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful

9
cmd/copy/copy.go
View file

@ -79,6 +79,15 @@ recently very efficiently like this:
rclone copy --max-age 24h --no-traverse /path/to/src remote:
Rclone will sync the modification times of files and directories if
the backend supports it. If metadata syncing is required then use the
|--metadata| flag.
Note that the modification time and metadata for the root directory
will **not** be synced. See https://github.com/rclone/rclone/issues/7652
for more info.
**Note**: Use the |-P|/|--progress| flag to view real-time transfer statistics.
**Note**: Use the |--dry-run| or the |--interactive|/|-i| flag to test without copying anything.

8
cmd/move/move.go
View file

@ -55,6 +55,14 @@ whether rclone lists the destination directory or not. Supplying this
option when moving a small number of files into a large destination
can speed transfers up greatly.
Rclone will sync the modification times of files and directories if
the backend supports it. If metadata syncing is required then use the
|--metadata| flag.
Note that the modification time and metadata for the root directory
will **not** be synced. See https://github.com/rclone/rclone/issues/7652
for more info.
**Important**: Since this can cause data loss, test first with the
|--dry-run| or the |--interactive|/|-i| flag.

8
cmd/sync/sync.go
View file

@ -170,6 +170,14 @@ the destination from the sync with a filter rule or by putting an
exclude-if-present file inside the destination directory and sync to a
destination that is inside the source directory.
Rclone will sync the modification times of files and directories if
the backend supports it. If metadata syncing is required then use the
` + "`--metadata`" + ` flag.
Note that the modification time and metadata for the root directory
will **not** be synced. See https://github.com/rclone/rclone/issues/7652
for more info.
**Note**: Use the ` + "`-P`" + `/` + "`--progress`" + ` flag to view real-time transfer statistics
**Note**: Use the ` + "`rclone dedupe`" + ` command to deal with "Duplicate object/directory found in source/destination - ignoring" errors.

19
docs/content/docs.md
View file

@ -446,18 +446,21 @@ This can be used when scripting to make aged backups efficiently, e.g.
## Metadata support {#metadata}
Metadata is data about a file which isn't the contents of the file.
Normally rclone only preserves the modification time and the content
(MIME) type where possible.
Metadata is data about a file (or directory) which isn't the contents
of the file (or directory). Normally rclone only preserves the
modification time and the content (MIME) type where possible.
Rclone supports preserving all the available metadata on files (not
directories) when using the `--metadata` or `-M` flag.
Rclone supports preserving all the available metadata on files and
directories when using the `--metadata` or `-M` flag.
Exactly what metadata is supported and what that support means depends
on the backend. Backends that support metadata have a metadata section
in their docs and are listed in the [features table](/overview/#features)
(Eg [local](/local/#metadata), [s3](/s3/#metadata))
Some backends don't support metadata, some only support metadata on
files and some support metadata on both files and directories.
Rclone only supports a one-time sync of metadata. This means that
metadata will be synced from the source object to the destination
object only when the source object has changed and needs to be
@ -1560,10 +1563,10 @@ some context for the `Metadata` which may be important.
- `SrcFsType` is the name of the source backend.
- `DstFs` is the config string for the remote that the object is being copied to
- `DstFsType` is the name of the destination backend.
- `Remote` is the path of the file relative to the root.
- `Size`, `MimeType`, `ModTime` are attributes of the file.
- `Remote` is the path of the object relative to the root.
- `Size`, `MimeType`, `ModTime` are attributes of the object.
- `IsDir` is `true` if this is a directory (not yet implemented).
- `ID` is the source `ID` of the file if known.
- `ID` is the source `ID` of the object if known.
- `Metadata` is the backend specific metadata as described in the backend docs.
```json

122
fs/sync/sync.go
View file

@ -18,6 +18,8 @@ import (
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fs/march"
"github.com/rclone/rclone/fs/operations"
"github.com/rclone/rclone/lib/errcount"
"golang.org/x/sync/errgroup"
)
// ErrorMaxDurationReached defines error when transfer duration is reached
@ -84,6 +86,20 @@ type syncCopyMove struct {
maxDurationEndTime time.Time // end time if --max-duration is set
logger operations.LoggerFn // LoggerFn used to report the results of a sync (or bisync) to an io.Writer
usingLogger bool // whether we are using logger
setDirMetadata bool // if set we set the directory metadata
setDirModTime bool // if set we set the directory modtimes
setDirModTimeAfter bool // if set we set the directory modtimes at the end of the sync
setDirModTimeMu sync.Mutex // protect setDirModTimeMu
setDirModTimes []setDirModTime // directories that need their modtime set
setDirModTimesMaxLevel int // max level of the directories to set
}
// For keeping track of delayed modtime sets
type setDirModTime struct {
dst fs.Directory
dir string
modTime time.Time
level int // the level of the directory, 0 is root
}
type trackRenamesStrategy byte
@ -136,6 +152,9 @@ func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete
modifyWindow: fs.GetModifyWindow(ctx, fsrc, fdst),
trackRenamesCh: make(chan fs.Object, ci.Checkers),
checkFirst: ci.CheckFirst,
setDirMetadata: ci.Metadata && fsrc.Features().ReadDirMetadata && fdst.Features().WriteDirMetadata,
setDirModTime: fdst.Features().WriteDirSetModTime || fdst.Features().MkdirMetadata != nil || fdst.Features().DirSetModTime != nil,
setDirModTimeAfter: fdst.Features().DirModTimeUpdatesOnWrite,
}
s.logger, s.usingLogger = operations.GetLogger(ctx)
@ -966,6 +985,11 @@ func (s *syncCopyMove) run() error {
}
}
// Update modtimes for directories if necessary
if s.setDirModTime && s.setDirModTimeAfter {
s.processError(s.setDelayedDirModTimes(s.ctx))
}
// Prune empty directories
if s.deleteMode != fs.DeleteModeOff {
if s.currentError() != nil && !s.ci.IgnoreErrors {
@ -1055,6 +1079,96 @@ func (s *syncCopyMove) DstOnly(dst fs.DirEntry) (recurse bool) {
return false
}
// copyDirMetadata copies the src directory modTime or Metadata to dst
// or f if nil. If dst is nil then it uses dir as the name of the new
// directory.
//
// It returns the destination directory if possible. Note that this may
// be nil.
func (s *syncCopyMove) copyDirMetadata(ctx context.Context, f fs.Fs, dst fs.Directory, dir string, src fs.Directory) (newDst fs.Directory) {
var err error
if s.setDirMetadata {
newDst, err = operations.CopyDirMetadata(ctx, f, dst, dir, src)
} else if s.setDirModTime {
if dst == nil {
newDst, err = operations.MkdirModTime(ctx, f, dir, src.ModTime(ctx))
} else {
newDst, err = operations.SetDirModTime(ctx, f, dst, dir, src.ModTime(ctx))
}
} else if dst == nil {
// Create the directory if it doesn't exist
err = operations.Mkdir(ctx, f, dir)
}
// If we need to set modtime after and we created a dir, then save it for later
if s.setDirModTime && s.setDirModTimeAfter && err == nil {
if newDst != nil {
dir = newDst.Remote()
}
level := strings.Count(dir, "/") + 1
// The root directory "" is at the top level
if dir == "" {
level = 0
}
s.setDirModTimeMu.Lock()
// Keep track of the maximum level inserted
if level > s.setDirModTimesMaxLevel {
s.setDirModTimesMaxLevel = level
}
s.setDirModTimes = append(s.setDirModTimes, setDirModTime{
dst: newDst,
dir: dir,
modTime: src.ModTime(ctx),
level: level,
})
s.setDirModTimeMu.Unlock()
fs.Debugf(nil, "Added delayed dir = %q, newDst=%v", dir, newDst)
}
s.processError(err)
if err != nil {
return nil
}
return newDst
}
// Set the modtimes for directories
func (s *syncCopyMove) setDelayedDirModTimes(ctx context.Context) error {
s.setDirModTimeMu.Lock()
defer s.setDirModTimeMu.Unlock()
// Timestamp all directories at the same level in parallel, deepest first
// We do this by iterating the slice multiple times to save memory
// There could be a lot of directories in this slice.
var errCount = errcount.New()
for level := s.setDirModTimesMaxLevel; level >= 0; level-- {
g, gCtx := errgroup.WithContext(ctx)
g.SetLimit(s.ci.Checkers)
for _, item := range s.setDirModTimes {
if item.level != level {
continue
}
// End early if error
if gCtx.Err() != nil {
break
}
item := item
g.Go(func() error {
_, err := operations.SetDirModTime(gCtx, s.fdst, item.dst, item.dir, item.modTime)
if err != nil {
err = fs.CountError(err)
fs.Errorf(item.dir, "Failed to timestamp directory: %v", err)
errCount.Add(err)
}
return nil // don't return errors, just count them
})
}
err := g.Wait()
if err != nil {
return err
}
}
return errCount.Err("failed to set directory modtime")
}
// SrcOnly have an object which is in the source only
func (s *syncCopyMove) SrcOnly(src fs.DirEntry) (recurse bool) {
if s.deleteMode == fs.DeleteModeOnly {
@ -1101,6 +1215,9 @@ func (s *syncCopyMove) SrcOnly(src fs.DirEntry) (recurse bool) {
s.srcEmptyDirs[src.Remote()] = src
s.logger(s.ctx, operations.MissingOnDst, src, nil, fs.ErrorIsDir)
s.srcEmptyDirsMu.Unlock()
// Create the directory and make sure the Metadata/ModTime is correct
s.copyDirMetadata(s.ctx, s.fdst, nil, x.Remote(), x)
return true
default:
panic("Bad object in DirEntries")
@ -1135,9 +1252,12 @@ func (s *syncCopyMove) Match(ctx context.Context, dst, src fs.DirEntry) (recurse
}
case fs.Directory:
// Do the same thing to the entire contents of the directory
_, ok := dst.(fs.Directory)
dstX, ok := dst.(fs.Directory)
if ok {
s.logger(s.ctx, operations.Match, src, dst, fs.ErrorIsDir)
// Create the directory and make sure the Metadata/ModTime is correct
s.copyDirMetadata(s.ctx, s.fdst, dstX, "", srcX)
// Only record matched (src & dst) empty dirs when performing move
if s.DoMove {
// Record the src directory for deletion

165
fs/sync/sync_test.go
View file

@ -7,6 +7,7 @@ import (
"context"
"errors"
"fmt"
"io"
"os"
"os/exec"
"runtime"
@ -65,15 +66,88 @@ func TestCopy(t *testing.T) {
ctx := context.Background()
r := fstest.NewRun(t)
file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
_, err := operations.SetDirModTime(ctx, r.Flocal, nil, "sub dir", t2)
if err != nil && !errors.Is(err, fs.ErrorNotImplemented) {
require.NoError(t, err)
}
r.Mkdir(ctx, r.Fremote)
ctx = predictDstFromLogger(ctx)
err := CopyDir(ctx, r.Fremote, r.Flocal, false)
err = CopyDir(ctx, r.Fremote, r.Flocal, false)
require.NoError(t, err)
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
r.CheckLocalItems(t, file1)
r.CheckRemoteItems(t, file1)
// Check that the modtimes of the directories are as expected
r.CheckDirectoryModTimes(t, "sub dir")
}
func TestCopyMetadata(t *testing.T) {
ctx := context.Background()
ctx, ci := fs.AddConfig(ctx)
ci.Metadata = true
r := fstest.NewRun(t)
features := r.Fremote.Features()
if !features.ReadMetadata && !features.WriteMetadata && !features.UserMetadata &&
!features.ReadDirMetadata && !features.WriteDirMetadata && !features.UserDirMetadata {
t.Skip("Skipping as metadata not supported")
}
const content = "hello metadata world!"
const dirPath = "metadata sub dir"
const filePath = dirPath + "/hello metadata world"
fileMetadata := fs.Metadata{
// System metadata supported by all backends
"mtime": t1.Format(time.RFC3339Nano),
// User metadata
"potato": "jersey",
}
dirMetadata := fs.Metadata{
// System metadata supported by all backends
"mtime": t2.Format(time.RFC3339Nano),
// User metadata
"potato": "king edward",
}
// Make the directory with metadata - may fall back to Mkdir
_, err := operations.MkdirMetadata(ctx, r.Flocal, dirPath, dirMetadata)
require.NoError(t, err)
// Upload the file with metadata
in := io.NopCloser(bytes.NewBufferString(content))
_, err = operations.Rcat(ctx, r.Flocal, filePath, in, t1, fileMetadata)
require.NoError(t, err)
file1 := fstest.NewItem(filePath, content, t1)
// Reset the time of the directory
_, err = operations.SetDirModTime(ctx, r.Flocal, nil, dirPath, t2)
if err != nil && !errors.Is(err, fs.ErrorNotImplemented) {
require.NoError(t, err)
}
ctx = predictDstFromLogger(ctx)
err = CopyDir(ctx, r.Fremote, r.Flocal, false)
require.NoError(t, err)
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
r.CheckLocalItems(t, file1)
r.CheckRemoteItems(t, file1)
// Check that the modtimes of the directories are as expected
r.CheckDirectoryModTimes(t, dirPath)
// Check that the metadata on the directory and file is correct
if features.ReadMetadata {
fstest.CheckEntryMetadata(ctx, t, r.Fremote, fstest.NewObject(ctx, t, r.Fremote, filePath), fileMetadata)
}
if features.ReadDirMetadata {
fstest.CheckEntryMetadata(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, dirPath), dirMetadata)
}
}
func TestCopyMissingDirectory(t *testing.T) {
@ -205,10 +279,15 @@ func TestCopyEmptyDirectories(t *testing.T) {
ctx := context.Background()
r := fstest.NewRun(t)
file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
err := operations.Mkdir(ctx, r.Flocal, "sub dir2")
_, err := operations.MkdirModTime(ctx, r.Flocal, "sub dir2", t2)
require.NoError(t, err)
r.Mkdir(ctx, r.Fremote)
// Set the modtime on "sub dir" to something specific
// Without this it fails on the CI and in VirtualBox with variances of up to 10mS
_, err = operations.SetDirModTime(ctx, r.Flocal, nil, "sub dir", t1)
require.NoError(t, err)
ctx = predictDstFromLogger(ctx)
err = CopyDir(ctx, r.Fremote, r.Flocal, true)
require.NoError(t, err)
@ -224,6 +303,9 @@ func TestCopyEmptyDirectories(t *testing.T) {
"sub dir2",
},
)
// Check that the modtimes of the directories are as expected
r.CheckDirectoryModTimes(t, "sub dir", "sub dir2")
}
// Test move empty directories
@ -231,8 +313,10 @@ func TestMoveEmptyDirectories(t *testing.T) {
ctx := context.Background()
r := fstest.NewRun(t)
file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
err := operations.Mkdir(ctx, r.Flocal, "sub dir2")
_, err := operations.MkdirModTime(ctx, r.Flocal, "sub dir2", t2)
require.NoError(t, err)
subDir := fstest.NewDirectory(ctx, t, r.Flocal, "sub dir")
subDirT := subDir.ModTime(ctx)
r.Mkdir(ctx, r.Fremote)
ctx = predictDstFromLogger(ctx)
@ -250,6 +334,14 @@ func TestMoveEmptyDirectories(t *testing.T) {
"sub dir2",
},
)
// Check that the modtimes of the directories are as expected
r.CheckDirectoryModTimes(t, "sub dir2")
// Note that "sub dir" mod time is updated when file1 is deleted from it
// So check it more manually
got := fstest.NewDirectory(ctx, t, r.Fremote, "sub dir")
gotT := got.ModTime(ctx)
fstest.AssertTimeEqualWithPrecision(t, subDir.Remote(), subDirT, gotT, fs.GetModifyWindow(ctx, r.Fremote, r.Flocal))
}
// Test sync empty directories
@ -257,8 +349,14 @@ func TestSyncEmptyDirectories(t *testing.T) {
ctx := context.Background()
r := fstest.NewRun(t)
file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
err := operations.Mkdir(ctx, r.Flocal, "sub dir2")
_, err := operations.MkdirModTime(ctx, r.Flocal, "sub dir2", t2)
require.NoError(t, err)
// Set the modtime on "sub dir" to something specific
// Without this it fails on the CI and in VirtualBox with variances of up to 10mS
_, err = operations.SetDirModTime(ctx, r.Flocal, nil, "sub dir", t1)
require.NoError(t, err)
r.Mkdir(ctx, r.Fremote)
ctx = predictDstFromLogger(ctx)
@ -276,6 +374,65 @@ func TestSyncEmptyDirectories(t *testing.T) {
"sub dir2",
},
)
// Check that the modtimes of the directories are as expected
r.CheckDirectoryModTimes(t, "sub dir", "sub dir2")
}
// Test delayed mod time setting
func TestSyncSetDelayedModTimes(t *testing.T) {
ctx := context.Background()
r := fstest.NewRun(t)
if !r.Fremote.Features().DirModTimeUpdatesOnWrite {
t.Skip("Backend doesn't have DirModTimeUpdatesOnWrite set")
}
// Create directories without timestamps
require.NoError(t, r.Flocal.Mkdir(ctx, "a1/b1/c1/d1/e1/f1"))
require.NoError(t, r.Flocal.Mkdir(ctx, "a1/b2/c1/d1/e1/f1"))
require.NoError(t, r.Flocal.Mkdir(ctx, "a1/b1/c1/d2/e1/f1"))
require.NoError(t, r.Flocal.Mkdir(ctx, "a1/b1/c1/d2/e1/f2"))
dirs := []string{
"a1",
"a1/b1",
"a1/b1/c1",
"a1/b1/c1/d1",
"a1/b1/c1/d1/e1",
"a1/b1/c1/d1/e1/f1",
"a1/b1/c1/d2",
"a1/b1/c1/d2/e1",
"a1/b1/c1/d2/e1/f1",
"a1/b1/c1/d2/e1/f2",
"a1/b2",
"a1/b2/c1",
"a1/b2/c1/d1",
"a1/b2/c1/d1/e1",
"a1/b2/c1/d1/e1/f1",
}
r.CheckLocalListing(t, []fstest.Item{}, dirs)
// Timestamp the directories in reverse order
ts := t1
for i := len(dirs) - 1; i >= 0; i-- {
dir := dirs[i]
_, err := operations.SetDirModTime(ctx, r.Flocal, nil, dir, ts)
require.NoError(t, err)
ts = ts.Add(time.Minute)
}
r.Mkdir(ctx, r.Fremote)
ctx = predictDstFromLogger(ctx)
err := Sync(ctx, r.Fremote, r.Flocal, true)
require.NoError(t, err)
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
r.CheckRemoteListing(t, []fstest.Item{}, dirs)
// Check that the modtimes of the directories are as expected
r.CheckDirectoryModTimes(t, dirs...)
}
// Test a server-side copy if possible, or the backup path if not

16
fstest/run.go
View file

@ -360,6 +360,22 @@ func (r *Run) CheckRemoteListing(t *testing.T, items []Item, expectedDirs []stri
CheckListingWithPrecision(t, r.Fremote, items, expectedDirs, r.Precision)
}
// CheckDirectoryModTimes checks that the directory names in r.Flocal has the correct modtime compared to r.Fremote
func (r *Run) CheckDirectoryModTimes(t *testing.T, names ...string) {
if r.Fremote.Features().DirSetModTime == nil && r.Fremote.Features().MkdirMetadata == nil {
fs.Debugf(r.Fremote, "Skipping modtime test as remote does not support DirSetModTime or MkdirMetadata")
return
}
ctx := context.Background()
for _, name := range names {
wantT := NewDirectory(ctx, t, r.Flocal, name).ModTime(ctx)
got := NewDirectory(ctx, t, r.Fremote, name)
gotT := got.ModTime(ctx)
fs.Debugf(r.Fremote, "Testing directory mod time of %q: wantT=%v, gotT=%v", name, wantT, gotT)
AssertTimeEqualWithPrecision(t, got.Remote(), wantT, gotT, fs.GetModifyWindow(ctx, r.Fremote, r.Flocal))
}
}
// Clean the temporary directory
func (r *Run) cleanTempDir() {
err := os.RemoveAll(r.LocalName)