TrueCloudLab/rclone
16
0
Fork 4
Code Issues 1 Pull requests Projects Releases 4 Packages Wiki Activity Actions

copy/sync/move: add --create-empty-src-dirs flag - fixes #2869

Browse source
This commit is contained in:
ishuah 2019-03-06 11:43:46 +03:00 • committed by Nick Craig-Wood
parent 2078ad68a5
commit 1b52f8d2a5
8 changed files with 149 additions and 77 deletions
Download patch file Download diff file Expand all files Collapse all files

164
fs/sync/sync_test.go
View file

@ -40,7 +40,7 @@ func TestCopyWithDryRun(t *testing.T) {
r.Mkdir(r.Fremote)
fs.Config.DryRun = true
err := CopyDir(r.Fremote, r.Flocal)
err := CopyDir(r.Fremote, r.Flocal, false)
fs.Config.DryRun = false
require.NoError(t, err)
@ -55,7 +55,7 @@ func TestCopy(t *testing.T) {
file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
r.Mkdir(r.Fremote)
err := CopyDir(r.Fremote, r.Flocal)
err := CopyDir(r.Fremote, r.Flocal, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal, file1)
@ -72,7 +72,7 @@ func TestCopyNoTraverse(t *testing.T) {
file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
err := CopyDir(r.Fremote, r.Flocal)
err := CopyDir(r.Fremote, r.Flocal, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal, file1)
@ -90,7 +90,7 @@ func TestSyncNoTraverse(t *testing.T) {
file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
accounting.Stats.ResetCounters()
err := Sync(r.Fremote, r.Flocal)
err := Sync(r.Fremote, r.Flocal, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal, file1)
@ -108,7 +108,7 @@ func TestCopyWithDepth(t *testing.T) {
fs.Config.MaxDepth = 1
defer func() { fs.Config.MaxDepth = -1 }()
err := CopyDir(r.Fremote, r.Flocal)
err := CopyDir(r.Fremote, r.Flocal, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal, file1, file2)
@ -136,7 +136,7 @@ func TestCopyWithFilesFrom(t *testing.T) {
}
defer unpatch()
err = CopyDir(r.Fremote, r.Flocal)
err = CopyDir(r.Fremote, r.Flocal, false)
require.NoError(t, err)
unpatch()
@ -153,7 +153,59 @@ func TestCopyEmptyDirectories(t *testing.T) {
require.NoError(t, err)
r.Mkdir(r.Fremote)
err = CopyDir(r.Fremote, r.Flocal)
err = CopyDir(r.Fremote, r.Flocal, true)
require.NoError(t, err)
fstest.CheckListingWithPrecision(
t,
r.Fremote,
[]fstest.Item{
file1,
},
[]string{
"sub dir",
"sub dir2",
},
fs.GetModifyWindow(r.Fremote),
)
}
// Test move empty directories
func TestMoveEmptyDirectories(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
err := operations.Mkdir(r.Flocal, "sub dir2")
require.NoError(t, err)
r.Mkdir(r.Fremote)
err = MoveDir(r.Fremote, r.Flocal, false, true)
require.NoError(t, err)
fstest.CheckListingWithPrecision(
t,
r.Fremote,
[]fstest.Item{
file1,
},
[]string{
"sub dir",
"sub dir2",
},
fs.GetModifyWindow(r.Fremote),
)
}
// Test sync empty directories
func TestSyncEmptyDirectories(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
err := operations.Mkdir(r.Flocal, "sub dir2")
require.NoError(t, err)
r.Mkdir(r.Fremote)
err = Sync(r.Fremote, r.Flocal, true)
require.NoError(t, err)
fstest.CheckListingWithPrecision(
@ -182,7 +234,7 @@ func TestServerSideCopy(t *testing.T) {
defer finaliseCopy()
t.Logf("Server side copy (if possible) %v -> %v", r.Fremote, FremoteCopy)
err = CopyDir(FremoteCopy, r.Fremote)
err = CopyDir(FremoteCopy, r.Fremote, false)
require.NoError(t, err)
fstest.CheckItems(t, FremoteCopy, file1)
@ -200,7 +252,7 @@ func TestCopyAfterDelete(t *testing.T) {
err := operations.Mkdir(r.Flocal, "")
require.NoError(t, err)
err = CopyDir(r.Fremote, r.Flocal)
err = CopyDir(r.Fremote, r.Flocal, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal)
@ -214,7 +266,7 @@ func TestCopyRedownload(t *testing.T) {
file1 := r.WriteObject("sub dir/hello world", "hello world", t1)
fstest.CheckItems(t, r.Fremote, file1)
err := CopyDir(r.Flocal, r.Fremote)
err := CopyDir(r.Flocal, r.Fremote, false)
require.NoError(t, err)
// Test with combined precision of local and remote as we copied it there and back
@ -234,7 +286,7 @@ func TestSyncBasedOnCheckSum(t *testing.T) {
fstest.CheckItems(t, r.Flocal, file1)
accounting.Stats.ResetCounters()
err := Sync(r.Fremote, r.Flocal)
err := Sync(r.Fremote, r.Flocal, false)
require.NoError(t, err)
// We should have transferred exactly one file.
@ -246,7 +298,7 @@ func TestSyncBasedOnCheckSum(t *testing.T) {
fstest.CheckItems(t, r.Flocal, file2)
accounting.Stats.ResetCounters()
err = Sync(r.Fremote, r.Flocal)
err = Sync(r.Fremote, r.Flocal, false)
require.NoError(t, err)
// We should have transferred no files
@ -268,7 +320,7 @@ func TestSyncSizeOnly(t *testing.T) {
fstest.CheckItems(t, r.Flocal, file1)
accounting.Stats.ResetCounters()
err := Sync(r.Fremote, r.Flocal)
err := Sync(r.Fremote, r.Flocal, false)
require.NoError(t, err)
// We should have transferred exactly one file.
@ -280,7 +332,7 @@ func TestSyncSizeOnly(t *testing.T) {
fstest.CheckItems(t, r.Flocal, file2)
accounting.Stats.ResetCounters()
err = Sync(r.Fremote, r.Flocal)
err = Sync(r.Fremote, r.Flocal, false)
require.NoError(t, err)
// We should have transferred no files
@ -302,7 +354,7 @@ func TestSyncIgnoreSize(t *testing.T) {
fstest.CheckItems(t, r.Flocal, file1)
accounting.Stats.ResetCounters()
err := Sync(r.Fremote, r.Flocal)
err := Sync(r.Fremote, r.Flocal, false)
require.NoError(t, err)
// We should have transferred exactly one file.
@ -314,7 +366,7 @@ func TestSyncIgnoreSize(t *testing.T) {
fstest.CheckItems(t, r.Flocal, file2)
accounting.Stats.ResetCounters()
err = Sync(r.Fremote, r.Flocal)
err = Sync(r.Fremote, r.Flocal, false)
require.NoError(t, err)
// We should have transferred no files
@ -330,7 +382,7 @@ func TestSyncIgnoreTimes(t *testing.T) {
fstest.CheckItems(t, r.Fremote, file1)
accounting.Stats.ResetCounters()
err := Sync(r.Fremote, r.Flocal)
err := Sync(r.Fremote, r.Flocal, false)
require.NoError(t, err)
// We should have transferred exactly 0 files because the
@ -341,7 +393,7 @@ func TestSyncIgnoreTimes(t *testing.T) {
defer func() { fs.Config.IgnoreTimes = false }()
accounting.Stats.ResetCounters()
err = Sync(r.Fremote, r.Flocal)
err = Sync(r.Fremote, r.Flocal, false)
require.NoError(t, err)
// We should have transferred exactly one file even though the
@ -361,7 +413,7 @@ func TestSyncIgnoreExisting(t *testing.T) {
defer func() { fs.Config.IgnoreExisting = false }()
accounting.Stats.ResetCounters()
err := Sync(r.Fremote, r.Flocal)
err := Sync(r.Fremote, r.Flocal, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal, file1)
fstest.CheckItems(t, r.Fremote, file1)
@ -369,7 +421,7 @@ func TestSyncIgnoreExisting(t *testing.T) {
// Change everything
r.WriteFile("existing", "newpotatoes", t2)
accounting.Stats.ResetCounters()
err = Sync(r.Fremote, r.Flocal)
err = Sync(r.Fremote, r.Flocal, false)
require.NoError(t, err)
// Items should not change
fstest.CheckItems(t, r.Fremote, file1)
@ -417,7 +469,7 @@ func TestSyncIgnoreErrors(t *testing.T) {
accounting.Stats.ResetCounters()
fs.CountError(nil)
assert.NoError(t, Sync(r.Fremote, r.Flocal))
assert.NoError(t, Sync(r.Fremote, r.Flocal, false))
fstest.CheckListingWithPrecision(
t,
@ -460,7 +512,7 @@ func TestSyncAfterChangingModtimeOnly(t *testing.T) {
defer func() { fs.Config.DryRun = false }()
accounting.Stats.ResetCounters()
err := Sync(r.Fremote, r.Flocal)
err := Sync(r.Fremote, r.Flocal, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal, file1)
@ -469,7 +521,7 @@ func TestSyncAfterChangingModtimeOnly(t *testing.T) {
fs.Config.DryRun = false
accounting.Stats.ResetCounters()
err = Sync(r.Fremote, r.Flocal)
err = Sync(r.Fremote, r.Flocal, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal, file1)
@ -497,7 +549,7 @@ func TestSyncAfterChangingModtimeOnlyWithNoUpdateModTime(t *testing.T) {
fstest.CheckItems(t, r.Fremote, file2)
accounting.Stats.ResetCounters()
err := Sync(r.Fremote, r.Flocal)
err := Sync(r.Fremote, r.Flocal, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal, file1)
@ -518,7 +570,7 @@ func TestSyncDoesntUpdateModtime(t *testing.T) {
fstest.CheckItems(t, r.Fremote, file2)
accounting.Stats.ResetCounters()
err := Sync(r.Fremote, r.Flocal)
err := Sync(r.Fremote, r.Flocal, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal, file1)
@ -538,7 +590,7 @@ func TestSyncAfterAddingAFile(t *testing.T) {
fstest.CheckItems(t, r.Fremote, file1)
accounting.Stats.ResetCounters()
err := Sync(r.Fremote, r.Flocal)
err := Sync(r.Fremote, r.Flocal, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal, file1, file2)
fstest.CheckItems(t, r.Fremote, file1, file2)
@ -553,7 +605,7 @@ func TestSyncAfterChangingFilesSizeOnly(t *testing.T) {
fstest.CheckItems(t, r.Flocal, file2)
accounting.Stats.ResetCounters()
err := Sync(r.Fremote, r.Flocal)
err := Sync(r.Fremote, r.Flocal, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal, file2)
fstest.CheckItems(t, r.Fremote, file2)
@ -576,7 +628,7 @@ func TestSyncAfterChangingContentsOnly(t *testing.T) {
fstest.CheckItems(t, r.Flocal, file2)
accounting.Stats.ResetCounters()
err := Sync(r.Fremote, r.Flocal)
err := Sync(r.Fremote, r.Flocal, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal, file2)
fstest.CheckItems(t, r.Fremote, file2)
@ -592,7 +644,7 @@ func TestSyncAfterRemovingAFileAndAddingAFileDryRun(t *testing.T) {
fs.Config.DryRun = true
accounting.Stats.ResetCounters()
err := Sync(r.Fremote, r.Flocal)
err := Sync(r.Fremote, r.Flocal, false)
fs.Config.DryRun = false
require.NoError(t, err)
@ -611,7 +663,7 @@ func TestSyncAfterRemovingAFileAndAddingAFile(t *testing.T) {
fstest.CheckItems(t, r.Flocal, file1, file3)
accounting.Stats.ResetCounters()
err := Sync(r.Fremote, r.Flocal)
err := Sync(r.Fremote, r.Flocal, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal, file1, file3)
fstest.CheckItems(t, r.Fremote, file1, file3)
@ -657,7 +709,7 @@ func TestSyncAfterRemovingAFileAndAddingAFileSubDir(t *testing.T) {
)
accounting.Stats.ResetCounters()
err := Sync(r.Fremote, r.Flocal)
err := Sync(r.Fremote, r.Flocal, false)
require.NoError(t, err)
fstest.CheckListingWithPrecision(
@ -727,7 +779,7 @@ func TestSyncAfterRemovingAFileAndAddingAFileSubDirWithErrors(t *testing.T) {
accounting.Stats.ResetCounters()
fs.CountError(nil)
err := Sync(r.Fremote, r.Flocal)
err := Sync(r.Fremote, r.Flocal, false)
assert.Equal(t, fs.ErrorNotDeleting, err)
fstest.CheckListingWithPrecision(
@ -804,7 +856,7 @@ func TestCopyDeleteBefore(t *testing.T) {
fstest.CheckItems(t, r.Flocal, file2)
accounting.Stats.ResetCounters()
err := CopyDir(r.Fremote, r.Flocal)
err := CopyDir(r.Fremote, r.Flocal, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Fremote, file1, file2)
@ -827,14 +879,14 @@ func TestSyncWithExclude(t *testing.T) {
}()
accounting.Stats.ResetCounters()
err := Sync(r.Fremote, r.Flocal)
err := Sync(r.Fremote, r.Flocal, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Fremote, file2, file1)
// Now sync the other way round and check enormous doesn't get
// deleted as it is excluded from the sync
accounting.Stats.ResetCounters()
err = Sync(r.Flocal, r.Fremote)
err = Sync(r.Flocal, r.Fremote, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal, file2, file1, file3)
}
@ -857,14 +909,14 @@ func TestSyncWithExcludeAndDeleteExcluded(t *testing.T) {
}()
accounting.Stats.ResetCounters()
err := Sync(r.Fremote, r.Flocal)
err := Sync(r.Fremote, r.Flocal, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Fremote, file2)
// Check sync the other way round to make sure enormous gets
// deleted even though it is excluded
accounting.Stats.ResetCounters()
err = Sync(r.Flocal, r.Fremote)
err = Sync(r.Flocal, r.Fremote, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal, file2)
}
@ -899,7 +951,7 @@ func TestSyncWithUpdateOlder(t *testing.T) {
}()
accounting.Stats.ResetCounters()
err := Sync(r.Fremote, r.Flocal)
err := Sync(r.Fremote, r.Flocal, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Fremote, oneO, twoF, threeO, fourF, fiveF)
}
@ -923,7 +975,7 @@ func TestSyncWithTrackRenames(t *testing.T) {
f2 := r.WriteFile("yam", "Yam Content", t2)
accounting.Stats.ResetCounters()
require.NoError(t, Sync(r.Fremote, r.Flocal))
require.NoError(t, Sync(r.Fremote, r.Flocal, false))
fstest.CheckItems(t, r.Fremote, f1, f2)
fstest.CheckItems(t, r.Flocal, f1, f2)
@ -932,7 +984,7 @@ func TestSyncWithTrackRenames(t *testing.T) {
f2 = r.RenameFile(f2, "yaml")
accounting.Stats.ResetCounters()
require.NoError(t, Sync(r.Fremote, r.Flocal))
require.NoError(t, Sync(r.Fremote, r.Flocal, false))
fstest.CheckItems(t, r.Fremote, f1, f2)
@ -969,7 +1021,7 @@ func testServerSideMove(t *testing.T, r *fstest.Run, withFilter, testDeleteEmpty
// Do server side move
accounting.Stats.ResetCounters()
err = MoveDir(FremoteMove, r.Fremote, testDeleteEmptyDirs)
err = MoveDir(FremoteMove, r.Fremote, testDeleteEmptyDirs, false)
require.NoError(t, err)
if withFilter {
@ -996,7 +1048,7 @@ func testServerSideMove(t *testing.T, r *fstest.Run, withFilter, testDeleteEmpty
// Move it back to a new empty remote, dst does not exist this time
accounting.Stats.ResetCounters()
err = MoveDir(FremoteMove2, FremoteMove, testDeleteEmptyDirs)
err = MoveDir(FremoteMove2, FremoteMove, testDeleteEmptyDirs, false)
require.NoError(t, err)
if withFilter {
@ -1021,7 +1073,7 @@ func TestMoveWithDeleteEmptySrcDirs(t *testing.T) {
r.Mkdir(r.Fremote)
// run move with --delete-empty-src-dirs
err := MoveDir(r.Fremote, r.Flocal, true)
err := MoveDir(r.Fremote, r.Flocal, true, false)
require.NoError(t, err)
fstest.CheckListingWithPrecision(
@ -1041,7 +1093,7 @@ func TestMoveWithoutDeleteEmptySrcDirs(t *testing.T) {
file2 := r.WriteFile("nested/sub dir/file", "nested", t1)
r.Mkdir(r.Fremote)
err := MoveDir(r.Fremote, r.Flocal, false)
err := MoveDir(r.Fremote, r.Flocal, false, false)
require.NoError(t, err)
fstest.CheckListingWithPrecision(
@ -1102,7 +1154,7 @@ func TestServerSideMoveOverlap(t *testing.T) {
fstest.CheckItems(t, r.Fremote, file1)
// Subdir move with no filters should return ErrorCantMoveOverlapping
err = MoveDir(FremoteMove, r.Fremote, false)
err = MoveDir(FremoteMove, r.Fremote, false, false)
assert.EqualError(t, err, fs.ErrorOverlapping.Error())
// Now try with a filter which should also fail with ErrorCantMoveOverlapping
@ -1110,7 +1162,7 @@ func TestServerSideMoveOverlap(t *testing.T) {
defer func() {
filter.Active.Opt.MinSize = -1
}()
err = MoveDir(FremoteMove, r.Fremote, false)
err = MoveDir(FremoteMove, r.Fremote, false, false)
assert.EqualError(t, err, fs.ErrorOverlapping.Error())
}
@ -1129,10 +1181,10 @@ func TestSyncOverlap(t *testing.T) {
assert.Equal(t, fs.ErrorOverlapping.Error(), err.Error())
}
checkErr(Sync(FremoteSync, r.Fremote))
checkErr(Sync(r.Fremote, FremoteSync))
checkErr(Sync(r.Fremote, r.Fremote))
checkErr(Sync(FremoteSync, FremoteSync))
checkErr(Sync(FremoteSync, r.Fremote, false))
checkErr(Sync(r.Fremote, FremoteSync, false))
checkErr(Sync(r.Fremote, r.Fremote, false))
checkErr(Sync(FremoteSync, FremoteSync, false))
}
// Test with BackupDir set
@ -1167,7 +1219,7 @@ func testSyncBackupDir(t *testing.T, suffix string) {
require.NoError(t, err)
accounting.Stats.ResetCounters()
err = Sync(fdst, r.Flocal)
err = Sync(fdst, r.Flocal, false)
require.NoError(t, err)
// one should be moved to the backup dir and the new one installed
@ -1188,7 +1240,7 @@ func testSyncBackupDir(t *testing.T, suffix string) {
// This should delete three and overwrite one again, checking
// the files got overwritten correctly in backup-dir
accounting.Stats.ResetCounters()
err = Sync(fdst, r.Flocal)
err = Sync(fdst, r.Flocal, false)
require.NoError(t, err)
// one should be moved to the backup dir and the new one installed
@ -1225,7 +1277,7 @@ func TestSyncUTFNorm(t *testing.T) {
fstest.CheckItems(t, r.Fremote, file2)
accounting.Stats.ResetCounters()
err := Sync(r.Fremote, r.Flocal)
err := Sync(r.Fremote, r.Flocal, false)
require.NoError(t, err)
// We should have transferred exactly one file, but kept the
@ -1251,7 +1303,7 @@ func TestSyncImmutable(t *testing.T) {
// Should succeed
accounting.Stats.ResetCounters()
err := Sync(r.Fremote, r.Flocal)
err := Sync(r.Fremote, r.Flocal, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal, file1)
fstest.CheckItems(t, r.Fremote, file1)
@ -1263,7 +1315,7 @@ func TestSyncImmutable(t *testing.T) {
// Should fail with ErrorImmutableModified and not modify local or remote files
accounting.Stats.ResetCounters()
err = Sync(r.Fremote, r.Flocal)
err = Sync(r.Fremote, r.Flocal, false)
assert.EqualError(t, err, fs.ErrorImmutableModified.Error())
fstest.CheckItems(t, r.Flocal, file2)
fstest.CheckItems(t, r.Fremote, file1)
@ -1299,6 +1351,6 @@ func TestAbort(t *testing.T) {
accounting.Stats.ResetCounters()
err := Sync(r.Fremote, r.Flocal)
err := Sync(r.Fremote, r.Flocal, false)
assert.Equal(t, accounting.ErrorMaxTransferLimitReached, err)
}