Make --files-from only read the objects specified and don't scan directories

Before this change using --files-from would scan all the directories
that the files could possibly be in causing rclone to do more work
that was necessary.

After this change, rclone constructs an in memory tree using the
--fast-list mechanism but from all of the files in the --files-from
list and without scanning any directories.

Any objects that are not found in the --files-from list are ignored
silently.

This mechanism is used for sync/copy/move (march) and all of the
listing commands ls/lsf/md5sum/etc (walk).
This commit is contained in:
Nick Craig-Wood 2018-10-19 17:41:14 +01:00
parent 9959c5f17f
commit c5ac96e9e7
7 changed files with 179 additions and 1 deletions

View file

@ -79,6 +79,35 @@ func TestCopyWithDepth(t *testing.T) {
fstest.CheckItems(t, r.Fremote, file2)
}
// Test copy with files from
func TestCopyWithFilesFrom(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
file1 := r.WriteFile("potato2", "hello world", t1)
file2 := r.WriteFile("hello world2", "hello world2", t2)
// Set the --files-from equivalent
f, err := filter.NewFilter(nil)
require.NoError(t, err)
require.NoError(t, f.AddFile("potato2"))
require.NoError(t, f.AddFile("notfound"))
// Monkey patch the active filter
oldFilter := filter.Active
filter.Active = f
unpatch := func() {
filter.Active = oldFilter
}
defer unpatch()
err = CopyDir(r.Fremote, r.Flocal)
require.NoError(t, err)
unpatch()
fstest.CheckItems(t, r.Flocal, file1, file2)
fstest.CheckItems(t, r.Fremote, file1)
}
// Test copy empty directories
func TestCopyEmptyDirectories(t *testing.T) {
r := fstest.NewRun(t)