sync,copy: Fix --fast-list --create-empty-src-dirs and --exclude

Before this change, if --fast-list was in use while doing a sync or
copy with --create-empty-src-dirs and --exclude excluded all the files
from the directory (but not the directory), then the directory would
not be created.

This is also visible with `rclone tree` which uses the same tree
building approach as `rclone sync --fast-list` where the directories
would go missing from the tree view.

This was caused by not adding the parents of excluded files to the
directory tree.

See: https://forum.rclone.org/t/create-empty-src-dirs-issue-with-b2/30856

fs/walk/walk.go

@@ -471,23 +471,39 @@ func walkRDirTree(ctx context.Context, f fs.Fs, startPath string, includeAll boo
 		defer mu.Unlock()
 		for _, entry := range entries {
 			slashes := strings.Count(entry.Remote(), "/")
+			excluded := true
 			switch x := entry.(type) {
 			case fs.Object:
 				// Make sure we don't delete excluded files if not required
 				if includeAll || fi.IncludeObject(ctx, x) {
 					if maxLevel < 0 || slashes <= maxLevel-1 {
 						dirs.Add(x)
-					} else {
+						excluded = false
-						// Make sure we include any parent directories of excluded objects
-						dirPath := x.Remote()
-						for ; slashes > maxLevel-1; slashes-- {
-							dirPath = parentDir(dirPath)
-						}
-						dirs.CheckParent(startPath, dirPath)
 					}
 				} else {
 					fs.Debugf(x, "Excluded from sync (and deletion)")
 				}
+				// Make sure we include any parent directories of excluded objects
+				if excluded {
+					dirPath := parentDir(x.Remote())
+					slashes--
+					if maxLevel >= 0 {
+						for ; slashes > maxLevel-1; slashes-- {
+							dirPath = parentDir(dirPath)
+						}
+					}
+					inc, err := includeDirectory(dirPath)
+					if err != nil {
+						return err
+					}
+					if inc || includeAll {
+						// If the directory doesn't exist already, create it
+						_, obj := dirs.Find(dirPath)
+						if obj == nil {
+							dirs.AddDir(fs.NewDir(dirPath, time.Now()))
+						}
+					}
+				}
 				// Check if we need to prune a directory later.
 				if !includeAll && len(fi.Opt.ExcludeFile) > 0 {
 					basename := path.Base(x.Remote())

fs/walk/walk_test.go

@@ -448,17 +448,32 @@ func TestWalkRDirTree(t *testing.T) {
 		err     error
 		root    string
 		level   int
+		exclude string
 	}{
-		{fs.DirEntries{}, "/\n", nil, "", -1},
+		{
-		{fs.DirEntries{mockobject.Object("a")}, `/
+			entries: fs.DirEntries{},
+			want:    "/\n",
+			level:   -1,
+		},
+		{
+			entries: fs.DirEntries{mockobject.Object("a")},
+			want: `/
   a
-`, nil, "", -1},
+`,
-		{fs.DirEntries{mockobject.Object("a/b")}, `/
+			level: -1,
+		},
+		{
+			entries: fs.DirEntries{mockobject.Object("a/b")},
+			want: `/
   a/
 a/
   b
-`, nil, "", -1},
+`,
-		{fs.DirEntries{mockobject.Object("a/b/c/d")}, `/
+			level: -1,
+		},
+		{
+			entries: fs.DirEntries{mockobject.Object("a/b/c/d")},
+			want: `/
   a/
 a/
   b/
@@ -466,9 +481,16 @@ a/b/
   c/
 a/b/c/
   d
-`, nil, "", -1},
+`,
-		{fs.DirEntries{mockobject.Object("a")}, "", errorBoom, "", -1},
+			level: -1,
-		{fs.DirEntries{
+		},
+		{
+			entries: fs.DirEntries{mockobject.Object("a")},
+			err:     errorBoom,
+			level:   -1,
+		},
+		{
+			entries: fs.DirEntries{
 				mockobject.Object("0/1/2/3"),
 				mockobject.Object("4/5/6/7"),
 				mockobject.Object("8/9/a/b"),
@@ -478,7 +500,8 @@ a/b/c/
 				mockobject.Object("o/p/q/r"),
 				mockobject.Object("s/t/u/v"),
 				mockobject.Object("w/x/y/z"),
-		}, `/
+			},
+			want: `/
   0/
   4/
   8/
@@ -542,12 +565,16 @@ w/x/
   y/
 w/x/y/
   z
-`, nil, "", -1},
+`,
-		{fs.DirEntries{
+			level: -1,
+		},
+		{
+			entries: fs.DirEntries{
 				mockobject.Object("a/b/c/d/e/f1"),
 				mockobject.Object("a/b/c/d/e/f2"),
 				mockobject.Object("a/b/c/d/e/f3"),
-		}, `a/b/c/
+			},
+			want: `a/b/c/
   d/
 a/b/c/d/
   e/
@@ -555,32 +582,91 @@ a/b/c/d/e/
   f1
   f2
   f3
-`, nil, "a/b/c", -1},
+`,
-		{fs.DirEntries{
+			root:  "a/b/c",
+			level: -1,
+		},
+		{
+			entries: fs.DirEntries{
 				mockobject.Object("A"),
 				mockobject.Object("a/B"),
 				mockobject.Object("a/b/C"),
 				mockobject.Object("a/b/c/D"),
 				mockobject.Object("a/b/c/d/E"),
-		}, `/
+			},
+			want: `/
   A
   a/
 a/
   B
   b/
-`, nil, "", 2},
+a/b/
-		{fs.DirEntries{
+`,
+			level: 2,
+		},
+		{
+			entries: fs.DirEntries{
 				mockobject.Object("a/b/c"),
 				mockobject.Object("a/b/c/d/e"),
-		}, `/
+			},
+			want: `/
   a/
 a/
   b/
-`, nil, "", 2},
+a/b/
+`,
+			level: 2,
+		},
+		{
+			entries: fs.DirEntries{
+				mockobject.Object("a/.bzEmpty"),
+				mockobject.Object("a/b1/.bzEmpty"),
+				mockobject.Object("a/b2/.bzEmpty"),
+			},
+			want: `/
+  a/
+a/
+  .bzEmpty
+  b1/
+  b2/
+a/b1/
+  .bzEmpty
+a/b2/
+  .bzEmpty
+`,
+			level:   -1,
+			exclude: ""},
+		{
+			entries: fs.DirEntries{
+				mockobject.Object("a/.bzEmpty"),
+				mockobject.Object("a/b1/.bzEmpty"),
+				mockobject.Object("a/b2/.bzEmpty"),
+			},
+			want: `/
+  a/
+a/
+  b1/
+  b2/
+a/b1/
+a/b2/
+`,
+			level:   -1,
+			exclude: ".bzEmpty",
+		},
 	} {
-		r, err := walkRDirTree(context.Background(), nil, test.root, true, test.level, makeListRCallback(test.entries, test.err))
+		ctx := context.Background()
-		assert.Equal(t, test.err, err, fmt.Sprintf("%+v", test))
+		if test.exclude != "" {
-		assert.Equal(t, test.want, r.String(), fmt.Sprintf("%+v", test))
+			fi, err := filter.NewFilter(nil)
+			require.NoError(t, err)
+			require.NoError(t, fi.Add(false, test.exclude))
+			// Change the active filter
+			ctx = filter.ReplaceConfig(ctx, fi)
+
+		}
+		r, err := walkRDirTree(ctx, nil, test.root, test.exclude == "", test.level, makeListRCallback(test.entries, test.err))
+		what := fmt.Sprintf("%+v", test)
+		assert.Equal(t, test.err, err, what)
+		assert.Equal(t, test.want, r.String(), what)
 	}
 }