TrueCloudLab/rclone
16
0
Fork 2
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

sync,copy: Fix --fast-list --create-empty-src-dirs and --exclude

Browse source 
Before this change, if --fast-list was in use while doing a sync or
copy with --create-empty-src-dirs and --exclude excluded all the files
from the directory (but not the directory), then the directory would
not be created.

This is also visible with `rclone tree` which uses the same tree
building approach as `rclone sync --fast-list` where the directories
would go missing from the tree view.

This was caused by not adding the parents of excluded files to the
directory tree.

See: https://forum.rclone.org/t/create-empty-src-dirs-issue-with-b2/30856
This commit is contained in:
Nick Craig-Wood 2022-05-20 14:54:04 +01:00
parent 28e43fe7af
commit da404dc0f2
2 changed files with 151 additions and 49 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
fs/walk/walk.go
View file

@ -471,23 +471,39 @@ func walkRDirTree(ctx context.Context, f fs.Fs, startPath string, includeAll boo
defer mu.Unlock()
for _, entry := range entries {
slashes := strings.Count(entry.Remote(), "/")
excluded := true
switch x := entry.(type) {
case fs.Object:
// Make sure we don't delete excluded files if not required
if includeAll || fi.IncludeObject(ctx, x) {
if maxLevel < 0 || slashes <= maxLevel-1 {
dirs.Add(x)
} else {
// Make sure we include any parent directories of excluded objects
dirPath := x.Remote()
for ; slashes > maxLevel-1; slashes-- {
dirPath = parentDir(dirPath)
}
dirs.CheckParent(startPath, dirPath)
excluded = false
}
} else {
fs.Debugf(x, "Excluded from sync (and deletion)")
}
// Make sure we include any parent directories of excluded objects
if excluded {
dirPath := parentDir(x.Remote())
slashes--
if maxLevel >= 0 {
for ; slashes > maxLevel-1; slashes-- {
dirPath = parentDir(dirPath)
}
}
inc, err := includeDirectory(dirPath)
if err != nil {
return err
}
if inc || includeAll {
// If the directory doesn't exist already, create it
_, obj := dirs.Find(dirPath)
if obj == nil {
dirs.AddDir(fs.NewDir(dirPath, time.Now()))
}
}
}
// Check if we need to prune a directory later.
if !includeAll && len(fi.Opt.ExcludeFile) > 0 {
basename := path.Base(x.Remote())

170
fs/walk/walk_test.go
View file

@ -448,17 +448,32 @@ func TestWalkRDirTree(t *testing.T) {
err error
root string
level int
exclude string
}{
{fs.DirEntries{}, "/\n", nil, "", -1},
{fs.DirEntries{mockobject.Object("a")}, `/
{
entries: fs.DirEntries{},
want: "/\n",
level: -1,
},
{
entries: fs.DirEntries{mockobject.Object("a")},
want: `/
a
`, nil, "", -1},
{fs.DirEntries{mockobject.Object("a/b")}, `/
`,
level: -1,
},
{
entries: fs.DirEntries{mockobject.Object("a/b")},
want: `/
a/
a/
b
`, nil, "", -1},
{fs.DirEntries{mockobject.Object("a/b/c/d")}, `/
`,
level: -1,
},
{
entries: fs.DirEntries{mockobject.Object("a/b/c/d")},
want: `/
a/
a/
b/
@ -466,19 +481,27 @@ a/b/
c/
a/b/c/
d
`, nil, "", -1},
{fs.DirEntries{mockobject.Object("a")}, "", errorBoom, "", -1},
{fs.DirEntries{
mockobject.Object("0/1/2/3"),
mockobject.Object("4/5/6/7"),
mockobject.Object("8/9/a/b"),
mockobject.Object("c/d/e/f"),
mockobject.Object("g/h/i/j"),
mockobject.Object("k/l/m/n"),
mockobject.Object("o/p/q/r"),
mockobject.Object("s/t/u/v"),
mockobject.Object("w/x/y/z"),
}, `/
`,
level: -1,
},
{
entries: fs.DirEntries{mockobject.Object("a")},
err: errorBoom,
level: -1,
},
{
entries: fs.DirEntries{
mockobject.Object("0/1/2/3"),
mockobject.Object("4/5/6/7"),
mockobject.Object("8/9/a/b"),
mockobject.Object("c/d/e/f"),
mockobject.Object("g/h/i/j"),
mockobject.Object("k/l/m/n"),
mockobject.Object("o/p/q/r"),
mockobject.Object("s/t/u/v"),
mockobject.Object("w/x/y/z"),
},
want: `/
0/
4/
8/
@ -542,12 +565,16 @@ w/x/
y/
w/x/y/
z
`, nil, "", -1},
{fs.DirEntries{
mockobject.Object("a/b/c/d/e/f1"),
mockobject.Object("a/b/c/d/e/f2"),
mockobject.Object("a/b/c/d/e/f3"),
}, `a/b/c/
`,
level: -1,
},
{
entries: fs.DirEntries{
mockobject.Object("a/b/c/d/e/f1"),
mockobject.Object("a/b/c/d/e/f2"),
mockobject.Object("a/b/c/d/e/f3"),
},
want: `a/b/c/
d/
a/b/c/d/
e/
@ -555,32 +582,91 @@ a/b/c/d/e/
f1
f2
f3
`, nil, "a/b/c", -1},
{fs.DirEntries{
mockobject.Object("A"),
mockobject.Object("a/B"),
mockobject.Object("a/b/C"),
mockobject.Object("a/b/c/D"),
mockobject.Object("a/b/c/d/E"),
}, `/
`,
root: "a/b/c",
level: -1,
},
{
entries: fs.DirEntries{
mockobject.Object("A"),
mockobject.Object("a/B"),
mockobject.Object("a/b/C"),
mockobject.Object("a/b/c/D"),
mockobject.Object("a/b/c/d/E"),
},
want: `/
A
a/
a/
B
b/
`, nil, "", 2},
{fs.DirEntries{
mockobject.Object("a/b/c"),
mockobject.Object("a/b/c/d/e"),
}, `/
a/b/
`,
level: 2,
},
{
entries: fs.DirEntries{
mockobject.Object("a/b/c"),
mockobject.Object("a/b/c/d/e"),
},
want: `/
a/
a/
b/
`, nil, "", 2},
a/b/
`,
level: 2,
},
{
entries: fs.DirEntries{
mockobject.Object("a/.bzEmpty"),
mockobject.Object("a/b1/.bzEmpty"),
mockobject.Object("a/b2/.bzEmpty"),
},
want: `/
a/
a/
.bzEmpty
b1/
b2/
a/b1/
.bzEmpty
a/b2/
.bzEmpty
`,
level: -1,
exclude: ""},
{
entries: fs.DirEntries{
mockobject.Object("a/.bzEmpty"),
mockobject.Object("a/b1/.bzEmpty"),
mockobject.Object("a/b2/.bzEmpty"),
},
want: `/
a/
a/
b1/
b2/
a/b1/
a/b2/
`,
level: -1,
exclude: ".bzEmpty",
},
} {
r, err := walkRDirTree(context.Background(), nil, test.root, true, test.level, makeListRCallback(test.entries, test.err))
assert.Equal(t, test.err, err, fmt.Sprintf("%+v", test))
assert.Equal(t, test.want, r.String(), fmt.Sprintf("%+v", test))
ctx := context.Background()
if test.exclude != "" {
fi, err := filter.NewFilter(nil)
require.NoError(t, err)
require.NoError(t, fi.Add(false, test.exclude))
// Change the active filter
ctx = filter.ReplaceConfig(ctx, fi)
}
r, err := walkRDirTree(ctx, nil, test.root, test.exclude == "", test.level, makeListRCallback(test.entries, test.err))
what := fmt.Sprintf("%+v", test)
assert.Equal(t, test.err, err, what)
assert.Equal(t, test.want, r.String(), what)
}
}