diff --git a/fs/operations/operations.go b/fs/operations/operations.go index 72bcf4526..a5bdb7337 100644 --- a/fs/operations/operations.go +++ b/fs/operations/operations.go @@ -26,6 +26,7 @@ import ( "github.com/ncw/rclone/fs/walk" "github.com/ncw/rclone/lib/readers" "github.com/pkg/errors" + "golang.org/x/sync/errgroup" ) // CheckHashes checks the two files to see if they have common @@ -1585,3 +1586,81 @@ func (l *ListFormat) Format(entry fs.DirEntry) (result string) { } return result } + +// DirMove renames srcRemote to dstRemote +// +// It does this by loading the directory tree into memory (using ListR +// if available) and doing renames in parallel. +func DirMove(f fs.Fs, srcRemote, dstRemote string) (err error) { + // Use DirMove if possible + if doDirMove := f.Features().DirMove; doDirMove != nil { + return doDirMove(f, srcRemote, dstRemote) + } + + // Load the directory tree into memory + tree, err := walk.NewDirTree(f, srcRemote, true, -1) + if err != nil { + return errors.Wrap(err, "RenameDir tree walk") + } + + // Get the directories in sorted order + dirs := tree.Dirs() + + // Make the destination directories - must be done in order not in parallel + for _, dir := range dirs { + dstPath := dstRemote + dir[len(srcRemote):] + err := f.Mkdir(dstPath) + if err != nil { + return errors.Wrap(err, "RenameDir mkdir") + } + } + + // Rename the files in parallel + type rename struct { + o fs.Object + newPath string + } + renames := make(chan rename, fs.Config.Transfers) + g, ctx := errgroup.WithContext(context.Background()) + for i := 0; i < fs.Config.Transfers; i++ { + g.Go(func() error { + for job := range renames { + dstOverwritten, _ := f.NewObject(job.newPath) + _, err := Move(f, dstOverwritten, job.newPath, job.o) + if err != nil { + return err + } + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + } + return nil + }) + } + for dir, entries := range tree { + dstPath := dstRemote + dir[len(srcRemote):] + for _, entry := range entries { + if o, ok := entry.(fs.Object); ok { + renames <- rename{o, path.Join(dstPath, path.Base(o.Remote()))} + } + } + } + close(renames) + err = g.Wait() + if err != nil { + return errors.Wrap(err, "RenameDir renames") + } + + // Remove the source directories in reverse order + for i := len(dirs) - 1; i >= 0; i-- { + err := f.Rmdir(dirs[i]) + if err != nil { + return errors.Wrap(err, "RenameDir rmdir") + } + } + + return nil +} diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go index 44f71e99d..c219187b6 100644 --- a/fs/operations/operations_test.go +++ b/fs/operations/operations_test.go @@ -956,3 +956,90 @@ func TestListFormat(t *testing.T) { assert.Equal(t, fmt.Sprintf("%d", items[1].Size())+"|subdir/|"+items[1].ModTime().Local().Format("2006-01-02 15:04:05"), list.Format(items[1])) } + +func TestDirMove(t *testing.T) { + r := fstest.NewRun(t) + defer r.Finalise() + + r.Mkdir(r.Fremote) + + // Make some files and dirs + r.ForceMkdir(r.Fremote) + files := []fstest.Item{ + r.WriteObject("A1/one", "one", t1), + r.WriteObject("A1/two", "two", t2), + r.WriteObject("A1/B1/three", "three", t3), + r.WriteObject("A1/B1/C1/four", "four", t1), + r.WriteObject("A1/B1/C2/five", "five", t2), + } + require.NoError(t, operations.Mkdir(r.Fremote, "A1/B2")) + require.NoError(t, operations.Mkdir(r.Fremote, "A1/B1/C3")) + + fstest.CheckListingWithPrecision( + t, + r.Fremote, + files, + []string{ + "A1", + "A1/B1", + "A1/B2", + "A1/B1/C1", + "A1/B1/C2", + "A1/B1/C3", + }, + fs.GetModifyWindow(r.Fremote), + ) + + require.NoError(t, operations.DirMove(r.Fremote, "A1", "A2")) + + for i := range files { + files[i].Path = strings.Replace(files[i].Path, "A1/", "A2/", -1) + files[i].WinPath = "" + } + + fstest.CheckListingWithPrecision( + t, + r.Fremote, + files, + []string{ + "A2", + "A2/B1", + "A2/B2", + "A2/B1/C1", + "A2/B1/C2", + "A2/B1/C3", + }, + fs.GetModifyWindow(r.Fremote), + ) + + // Disable DirMove + features := r.Fremote.Features() + oldDirMove := features.DirMove + features.DirMove = nil + defer func() { + features.DirMove = oldDirMove + }() + + require.NoError(t, operations.DirMove(r.Fremote, "A2", "A3")) + + for i := range files { + files[i].Path = strings.Replace(files[i].Path, "A2/", "A3/", -1) + files[i].WinPath = "" + } + + fstest.CheckListingWithPrecision( + t, + r.Fremote, + files, + []string{ + "A3", + "A3/B1", + "A3/B2", + "A3/B1/C1", + "A3/B1/C2", + "A3/B1/C3", + }, + fs.GetModifyWindow(r.Fremote), + ) + +}