TrueCloudLab/rclone
16
0
Fork 2
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions 1

Fix deletion of some excluded files without --delete-excluded #205

Browse source 
This only happened if the destination file was present but the source
file was missing.
This commit is contained in:
Nick Craig-Wood 2015-11-12 11:46:04 +00:00
parent 7beb50caa7
commit 80ffbade22
2 changed files with 33 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
fs/operations.go
View file

@ -381,12 +381,18 @@ func DeleteFiles(toBeDeleted ObjectsChan) {
}
// Read a map of Object.Remote to Object for the given Fs
func readFilesMap(fs Fs) map[string]Object {
func readFilesMap(fs Fs, obeyInclude bool) map[string]Object {
files := make(map[string]Object)
for o := range fs.List() {
remote := o.Remote()
if _, ok := files[remote]; !ok {
// Make sure we don't delete excluded files if not required
if !obeyInclude || Config.Filter.DeleteExcluded || Config.Filter.Include(remote, o.Size()) {
files[remote] = o
} else {
Debug(o, "Excluded from sync (and deletion)")
}
} else {
Log(o, "Duplicate file detected")
}
@ -420,7 +426,7 @@ func syncCopyMove(fdst, fsrc Fs, Delete bool, DoMove bool) error {
// Read the destination files first
// FIXME could do this in parallel and make it use less memory
delFiles := readFilesMap(fdst)
delFiles := readFilesMap(fdst, true)
// Read source files checking them off against dest files
toBeChecked := make(ObjectPairChan, Config.Transfers)
@ -445,14 +451,10 @@ func syncCopyMove(fdst, fsrc Fs, Delete bool, DoMove bool) error {
go func() {
for src := range fsrc.List() {
remote := src.Remote()
dst, dstFound := delFiles[remote]
if !Config.Filter.Include(remote, src.Size()) {
Debug(src, "Excluding from sync")
if dstFound && !Config.Filter.DeleteExcluded {
delete(delFiles, remote)
}
} else {
if dstFound {
if dst, dstFound := delFiles[remote]; dstFound {
delete(delFiles, remote)
toBeChecked <- ObjectPair{src, dst}
} else {
@ -539,11 +541,11 @@ func Check(fdst, fsrc Fs) error {
// Read the destination files first
// FIXME could do this in parallel and make it use less memory
dstFiles := readFilesMap(fdst)
dstFiles := readFilesMap(fdst, false)
// Read the source files checking them against dstFiles
// FIXME could do this in parallel and make it use less memory
srcFiles := readFilesMap(fsrc)
srcFiles := readFilesMap(fsrc, false)
// Move all the common files into commonFiles and delete then
// from srcFiles and dstFiles

26
fs/operations_test.go
View file

@ -411,7 +411,7 @@ func TestSyncAfterRemovingAFileAndAddingAFile(t *testing.T) {
// Test with exclude
func TestSyncWithExclude(t *testing.T) {
WriteFile("enormous", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 100 bytes
fs.Config.Filter.MaxSize = 80
fs.Config.Filter.MaxSize = 40
defer func() {
fs.Config.Filter.MaxSize = 0
}()
@ -424,6 +424,17 @@ func TestSyncWithExclude(t *testing.T) {
{Path: "potato2", Size: 60, ModTime: t1, Md5sum: "d6548b156ea68a4e003e786df99eee76"},
}
fstest.CheckListingWithPrecision(t, fremote, items, fs.Config.ModifyWindow)
// Now sync the other way round and check enormous doesn't get
// deleted as it is excluded from the sync
items = append(items, fstest.Item{
Path: "enormous", Size: 100, ModTime: t1, Md5sum: "8adc5937e635f6c9af646f0b23560fae",
})
err = fs.Sync(flocal, fremote)
if err != nil {
t.Fatalf("Sync failed: %v", err)
}
fstest.CheckListingWithPrecision(t, flocal, items, fs.Config.ModifyWindow)
}
// Test with exclude and delete excluded
@ -444,12 +455,17 @@ func TestSyncWithExcludeAndDeleleteExcluded(t *testing.T) {
}
fstest.CheckListingWithPrecision(t, fremote, items, fs.Config.ModifyWindow)
// Tidy up
reset()
err = os.Remove(localName + "/enormous")
// Check sync the other way round to make sure enormous gets
// deleted even though it is excluded
err = fs.Sync(flocal, fremote)
if err != nil {
t.Fatalf("Remove failed: %v", err)
t.Fatalf("Sync failed: %v", err)
}
fstest.CheckListingWithPrecision(t, flocal, items, fs.Config.ModifyWindow)
// Tidy up - put potato2 back!
reset()
WriteFile("potato2", "------------------------------------------------------------", t1)
err = fs.Sync(fremote, flocal)
if err != nil {
t.Fatalf("Sync failed: %v", err)