alexvanin/rclone
1
0
Fork 0
forked from TrueCloudLab/rclone
Code Pull requests Activity

dedupe: Use walk.ListR for listing commands.

Browse source 
This dramatically increases the speed (7x in my tests) of the de-dupe
as google drive supports ListR directly and dedupe did not work with
`--fast-list`.

Fixes #2902
This commit is contained in:
Nick Craig-Wood 2019-01-21 16:59:23 +00:00
parent e5f4210b09
commit 4376019062
2 changed files with 3 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
fs/operations/dedupe.go
View file

@ -192,10 +192,7 @@ var _ pflag.Value = (*DeduplicateMode)(nil)
// dedupeFindDuplicateDirs scans f for duplicate directories // dedupeFindDuplicateDirs scans f for duplicate directories
func dedupeFindDuplicateDirs(f fs.Fs) ([][]fs.Directory, error) { func dedupeFindDuplicateDirs(f fs.Fs) ([][]fs.Directory, error) {
duplicateDirs := [][]fs.Directory{} duplicateDirs := [][]fs.Directory{}
err := walk.Walk(f, "", true, fs.Config.MaxDepth, func(dirPath string, entries fs.DirEntries, err error) error { err := walk.ListR(f, "", true, fs.Config.MaxDepth, walk.ListDirs, func(entries fs.DirEntries) error {
if err != nil {
return err
}
dirs := map[string][]fs.Directory{} dirs := map[string][]fs.Directory{}
entries.ForDir(func(d fs.Directory) { entries.ForDir(func(d fs.Directory) {
dirs[d.Remote()] = append(dirs[d.Remote()], d) dirs[d.Remote()] = append(dirs[d.Remote()], d)
@ -268,10 +265,7 @@ func Deduplicate(f fs.Fs, mode DeduplicateMode) error {
// Now find duplicate files // Now find duplicate files
files := map[string][]fs.Object{} files := map[string][]fs.Object{}
err := walk.Walk(f, "", true, fs.Config.MaxDepth, func(dirPath string, entries fs.DirEntries, err error) error { err := walk.ListR(f, "", true, fs.Config.MaxDepth, walk.ListObjects, func(entries fs.DirEntries) error {
if err != nil {
return err
}
entries.ForObject(func(o fs.Object) { entries.ForObject(func(o fs.Object) {
remote := o.Remote() remote := o.Remote()
files[remote] = append(files[remote], o) files[remote] = append(files[remote], o)

5
fs/operations/dedupe_test.go
View file

@ -161,10 +161,7 @@ func TestDeduplicateRename(t *testing.T) {
err := operations.Deduplicate(r.Fremote, operations.DeduplicateRename) err := operations.Deduplicate(r.Fremote, operations.DeduplicateRename)
require.NoError(t, err) require.NoError(t, err)
require.NoError(t, walk.Walk(r.Fremote, "", true, -1, func(dirPath string, entries fs.DirEntries, err error) error { require.NoError(t, walk.ListR(r.Fremote, "", true, -1, walk.ListObjects, func(entries fs.DirEntries) error {
if err != nil {
return err
}
entries.ForObject(func(o fs.Object) { entries.ForObject(func(o fs.Object) {
remote := o.Remote() remote := o.Remote()
if remote != "one-1.txt" && if remote != "one-1.txt" &&