From c265f451f278b2d04f856af4dc2f4d524616e910 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Sun, 23 Oct 2016 17:34:17 +0100 Subject: [PATCH] Implement moveto and copyto commands for choosing a destination name on copy/move Fixes #227 Fixes #476 --- cmd/all/all.go | 2 + cmd/cmd.go | 92 +++++++++++++++++++++++++++++++++++++------ cmd/cmd_test.go | 28 +++++++++++++ cmd/copy/copy.go | 4 +- cmd/copyto/copyto.go | 53 +++++++++++++++++++++++++ cmd/move/move.go | 10 ++--- cmd/moveto/moveto.go | 57 +++++++++++++++++++++++++++ fs/operations.go | 56 +++++++++++++++++++++++++- fs/operations_test.go | 45 +++++++++++++++++++++ fs/sync.go | 11 +++--- 10 files changed, 332 insertions(+), 26 deletions(-) create mode 100644 cmd/cmd_test.go create mode 100644 cmd/copyto/copyto.go create mode 100644 cmd/moveto/moveto.go diff --git a/cmd/all/all.go b/cmd/all/all.go index 25bd07327..2cf1160ed 100644 --- a/cmd/all/all.go +++ b/cmd/all/all.go @@ -10,6 +10,7 @@ import ( _ "github.com/ncw/rclone/cmd/cleanup" _ "github.com/ncw/rclone/cmd/config" _ "github.com/ncw/rclone/cmd/copy" + _ "github.com/ncw/rclone/cmd/copyto" _ "github.com/ncw/rclone/cmd/dedupe" _ "github.com/ncw/rclone/cmd/delete" _ "github.com/ncw/rclone/cmd/genautocomplete" @@ -23,6 +24,7 @@ import ( _ "github.com/ncw/rclone/cmd/mkdir" _ "github.com/ncw/rclone/cmd/mount" _ "github.com/ncw/rclone/cmd/move" + _ "github.com/ncw/rclone/cmd/moveto" _ "github.com/ncw/rclone/cmd/purge" _ "github.com/ncw/rclone/cmd/rmdir" _ "github.com/ncw/rclone/cmd/rmdirs" diff --git a/cmd/cmd.go b/cmd/cmd.go index 4c628d01d..098a8383b 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -14,6 +14,7 @@ import ( "regexp" "runtime" "runtime/pprof" + "strings" "time" "github.com/spf13/cobra" @@ -94,31 +95,50 @@ func ShowVersion() { fmt.Printf("rclone %s\n", fs.Version) } -// newFsSrc creates a src Fs from a name +// newFsFile creates a dst Fs from a name but may point to a file. // -// This can point to a file -func newFsSrc(remote string) fs.Fs { +// It returns a string with the file name if points to a file +func newFsFile(remote string) (fs.Fs, string) { fsInfo, configName, fsPath, err := fs.ParseRemote(remote) if err != nil { fs.Stats.Error() log.Fatalf("Failed to create file system for %q: %v", remote, err) } f, err := fsInfo.NewFs(configName, fsPath) - if err == fs.ErrorIsFile { + switch err { + case fs.ErrorIsFile: + return f, path.Base(fsPath) + case nil: + return f, "" + default: + fs.Stats.Error() + log.Fatalf("Failed to create file system for %q: %v", remote, err) + } + return nil, "" +} + +// newFsSrc creates a src Fs from a name +// +// It returns a string with the file name if limiting to one file +// +// This can point to a file +func newFsSrc(remote string) (fs.Fs, string) { + f, fileName := newFsFile(remote) + if fileName != "" { if !fs.Config.Filter.InActive() { fs.Stats.Error() log.Fatalf("Can't limit to single files when using filters: %v", remote) } // Limit transfers to this file - err = fs.Config.Filter.AddFile(path.Base(fsPath)) + err := fs.Config.Filter.AddFile(fileName) + if err != nil { + fs.Stats.Error() + log.Fatalf("Failed to limit to single file %q: %v", remote, err) + } // Set --no-traverse as only one file fs.Config.NoTraverse = true } - if err != nil { - fs.Stats.Error() - log.Fatalf("Failed to create file system for %q: %v", remote, err) - } - return f + return f, fileName } // newFsDst creates a dst Fs from a name @@ -135,14 +155,62 @@ func newFsDst(remote string) fs.Fs { // NewFsSrcDst creates a new src and dst fs from the arguments func NewFsSrcDst(args []string) (fs.Fs, fs.Fs) { - fsrc, fdst := newFsSrc(args[0]), newFsDst(args[1]) + fsrc, _ := newFsSrc(args[0]) + fdst := newFsDst(args[1]) fs.CalculateModifyWindow(fdst, fsrc) return fsrc, fdst } +// RemoteSplit splits a remote into a parent and a leaf +// +// if it returns parent as an empty string then it wasn't possible +func RemoteSplit(remote string) (parent string, leaf string) { + // Split remote on : + i := strings.Index(remote, ":") + remoteName := "" + remotePath := remote + if i >= 0 { + remoteName = remote[:i+1] + remotePath = remote[i+1:] + } + if remotePath == "" { + return "", "" + } + // Construct new remote name without last segment + parent, leaf = path.Split(remotePath) + if leaf == "" { + return "", "" + } + if parent != "/" { + parent = strings.TrimSuffix(parent, "/") + } + parent = remoteName + parent + if parent == "" { + parent = "." + } + return parent, leaf +} + +// NewFsSrcDstFiles creates a new src and dst fs from the arguments +// If src is a file then srcFileName and dstFileName will be non-empty +func NewFsSrcDstFiles(args []string) (fsrc fs.Fs, srcFileName string, fdst fs.Fs, dstFileName string) { + fsrc, srcFileName = newFsSrc(args[0]) + // If copying a file... + dstRemote := args[1] + if srcFileName != "" { + dstRemote, dstFileName = RemoteSplit(dstRemote) + if dstRemote == "" { + log.Fatalf("Can't find parent directory for %q", args[1]) + } + } + fdst = newFsDst(dstRemote) + fs.CalculateModifyWindow(fdst, fsrc) + return +} + // NewFsSrc creates a new src fs from the arguments func NewFsSrc(args []string) fs.Fs { - fsrc := newFsSrc(args[0]) + fsrc, _ := newFsSrc(args[0]) fs.CalculateModifyWindow(fsrc) return fsrc } diff --git a/cmd/cmd_test.go b/cmd/cmd_test.go new file mode 100644 index 000000000..d8056ee8f --- /dev/null +++ b/cmd/cmd_test.go @@ -0,0 +1,28 @@ +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestRemoteSplit(t *testing.T) { + + for _, test := range []struct { + remote, wantParent, wantLeaf string + }{ + {"", "", ""}, + {"remote:", "", ""}, + {"remote:potato", "remote:", "potato"}, + {"remote:potato/sausage", "remote:potato", "sausage"}, + {"/", "", ""}, + {"/root", "/", "root"}, + {"/a/b", "/a", "b"}, + {"root", ".", "root"}, + {"a/b", "a", "b"}, + } { + gotParent, gotLeaf := RemoteSplit(test.remote) + assert.Equal(t, test.wantParent, gotParent, test.remote) + assert.Equal(t, test.wantLeaf, gotLeaf, test.remote) + } +} diff --git a/cmd/copy/copy.go b/cmd/copy/copy.go index 58eaaeade..80f2b8e11 100644 --- a/cmd/copy/copy.go +++ b/cmd/copy/copy.go @@ -45,12 +45,12 @@ Not to destpath/sourcepath/one.txt destpath/sourcepath/two.txt -If you are familiar with ` + "`" + `rsync` + "`" + `, rclone always works as if you had +If you are familiar with ` + "`rsync`" + `, rclone always works as if you had written a trailing / - meaning "copy the contents of this directory". This applies to all commands and whether you are talking about the source or destination. -See the ` + "`" + `--no-traverse` + "`" + ` option for controlling whether rclone lists +See the ` + "`--no-traverse`" + ` option for controlling whether rclone lists the destination directory or not. `, Run: func(command *cobra.Command, args []string) { diff --git a/cmd/copyto/copyto.go b/cmd/copyto/copyto.go new file mode 100644 index 000000000..d0ddbf698 --- /dev/null +++ b/cmd/copyto/copyto.go @@ -0,0 +1,53 @@ +package copyto + +import ( + "github.com/ncw/rclone/cmd" + "github.com/ncw/rclone/fs" + "github.com/spf13/cobra" +) + +func init() { + cmd.Root.AddCommand(commandDefintion) +} + +var commandDefintion = &cobra.Command{ + Use: "copyto source:path dest:path", + Short: `Copy files from source to dest, skipping already copied`, + Long: ` +If source:path is a file or directory then it copies it to a file or +directory named dest:path. + +This can be used to upload single files to other than their current +name. If the source is a directory then it acts exactly like the copy +command. + +So + + rclone copyto src dst + +where src and dst are rclone paths, either remote:path or +/path/to/local or C:\windows\path\if\on\windows. + +This will: + + if src is file + copy it to dst, overwriting an existing file if it exists + if src is directory + copy it to dst, overwriting existing files if they exist + see copy command for full details + +This doesn't transfer unchanged files, testing by size and +modification time or MD5SUM. It doesn't delete files from the +destination. +`, + Run: func(command *cobra.Command, args []string) { + cmd.CheckArgs(2, 2, command, args) + fsrc, srcFileName, fdst, dstFileName := cmd.NewFsSrcDstFiles(args) + cmd.Run(true, command, func() error { + if srcFileName == "" { + return fs.CopyDir(fdst, fsrc) + } + return fs.CopyFile(fdst, fsrc, dstFileName, srcFileName) + }) + }, +} diff --git a/cmd/move/move.go b/cmd/move/move.go index fee132dd8..e32c048bb 100644 --- a/cmd/move/move.go +++ b/cmd/move/move.go @@ -19,14 +19,14 @@ directory. Rclone will error if the source and destination overlap and the remote does not support a server side directory move operation. If no filters are in use and if possible this will server side move -` + "`" + `source:path` + "`" + ` into ` + "`" + `dest:path` + "`" + `. After this ` + "`" + `source:path` + "`" + ` will no +` + "`source:path`" + ` into ` + "`dest:path`" + `. After this ` + "`source:path`" + ` will no longer longer exist. -Otherwise for each file in ` + "`" + `source:path` + "`" + ` selected by the filters (if -any) this will move it into ` + "`" + `dest:path` + "`" + `. If possible a server side +Otherwise for each file in ` + "`source:path`" + ` selected by the filters (if +any) this will move it into ` + "`dest:path`" + `. If possible a server side move will be used, otherwise it will copy it (server side if possible) -into ` + "`" + `dest:path` + "`" + ` then delete the original (if no errors on copy) in -` + "`" + `source:path` + "`" + `. +into ` + "`dest:path`" + ` then delete the original (if no errors on copy) in +` + "`source:path`" + `. **Important**: Since this can cause data loss, test first with the --dry-run flag. diff --git a/cmd/moveto/moveto.go b/cmd/moveto/moveto.go new file mode 100644 index 000000000..0078cb611 --- /dev/null +++ b/cmd/moveto/moveto.go @@ -0,0 +1,57 @@ +package moveto + +import ( + "github.com/ncw/rclone/cmd" + "github.com/ncw/rclone/fs" + "github.com/spf13/cobra" +) + +func init() { + cmd.Root.AddCommand(commandDefintion) +} + +var commandDefintion = &cobra.Command{ + Use: "moveto source:path dest:path", + Short: `Move file or directory from source to dest.`, + Long: ` +If source:path is a file or directory then it moves it to a file or +directory named dest:path. + +This can be used to rename files or upload single files to other than +their existing name. If the source is a directory then it acts exacty +like the move command. + +So + + rclone moveto src dst + +where src and dst are rclone paths, either remote:path or +/path/to/local or C:\windows\path\if\on\windows. + +This will: + + if src is file + move it to dst, overwriting an existing file if it exists + if src is directory + move it to dst, overwriting existing files if they exist + see move command for full details + +This doesn't transfer unchanged files, testing by size and +modification time or MD5SUM. src will be deleted on successful +transfer. + +**Important**: Since this can cause data loss, test first with the +--dry-run flag. +`, + Run: func(command *cobra.Command, args []string) { + cmd.CheckArgs(2, 2, command, args) + fsrc, srcFileName, fdst, dstFileName := cmd.NewFsSrcDstFiles(args) + + cmd.Run(true, command, func() error { + if srcFileName == "" { + return fs.MoveDir(fdst, fsrc) + } + return fs.MoveFile(fdst, fsrc, dstFileName, srcFileName) + }) + }, +} diff --git a/fs/operations.go b/fs/operations.go index c2c414798..850635e84 100644 --- a/fs/operations.go +++ b/fs/operations.go @@ -222,6 +222,17 @@ func removeFailedCopy(dst Object) bool { return true } +// Wrapper to override the remote for an object +type overrideRemoteObject struct { + Object + remote string +} + +// Remote returns the overriden remote name +func (o *overrideRemoteObject) Remote() string { + return o.remote +} + // Copy src object to dst or f if nil. If dst is nil then it uses // remote as the name of the new object. func Copy(f Fs, dst Object, remote string, src Object) (err error) { @@ -260,12 +271,13 @@ func Copy(f Fs, dst Object, remote string, src Object) (err error) { in := NewAccount(in0, src) // account the transfer + wrappedSrc := &overrideRemoteObject{Object: src, remote: remote} if doUpdate { actionTaken = "Copied (replaced existing)" - err = dst.Update(in, src) + err = dst.Update(in, wrappedSrc) } else { actionTaken = "Copied (new)" - dst, err = f.Put(in, src) + dst, err = f.Put(in, wrappedSrc) } closeErr := in.Close() if err == nil { @@ -1186,3 +1198,43 @@ func Rmdirs(f Fs) error { } return nil } + +// moveOrCopyFile moves or copies a single file possibly to a new name +func moveOrCopyFile(fdst Fs, fsrc Fs, dstFileName string, srcFileName string, cp bool) (err error) { + // Choose operations + Op := Move + if cp { + Op = Copy + } + + // Find src object + srcObj, err := fsrc.NewObject(srcFileName) + if err != nil { + return err + } + + // Find dst object if it exists + dstObj, err := fdst.NewObject(dstFileName) + if err == ErrorObjectNotFound { + dstObj = nil + } else if err != nil { + return err + } + + if NeedTransfer(dstObj, srcObj) { + return Op(fdst, dstObj, dstFileName, srcObj) + } else if !cp { + return DeleteFile(srcObj) + } + return nil +} + +// MoveFile moves a single file possibly to a new name +func MoveFile(fdst Fs, fsrc Fs, dstFileName string, srcFileName string) (err error) { + return moveOrCopyFile(fdst, fsrc, dstFileName, srcFileName, false) +} + +// CopyFile moves a single file possibly to a new name +func CopyFile(fdst Fs, fsrc Fs, dstFileName string, srcFileName string) (err error) { + return moveOrCopyFile(fdst, fsrc, dstFileName, srcFileName, true) +} diff --git a/fs/operations_test.go b/fs/operations_test.go index 089a5d35a..503dc1674 100644 --- a/fs/operations_test.go +++ b/fs/operations_test.go @@ -697,3 +697,48 @@ func TestRmdirs(t *testing.T) { ) } + +func TestMoveFile(t *testing.T) { + r := NewRun(t) + defer r.Finalise() + + file1 := r.WriteFile("file1", "file1 contents", t1) + fstest.CheckItems(t, r.flocal, file1) + + file2 := file1 + file2.Path = "sub/file2" + + err := fs.MoveFile(r.fremote, r.flocal, file2.Path, file1.Path) + require.NoError(t, err) + fstest.CheckItems(t, r.flocal) + fstest.CheckItems(t, r.fremote, file2) + + r.WriteFile("file1", "file1 contents", t1) + fstest.CheckItems(t, r.flocal, file1) + + err = fs.MoveFile(r.fremote, r.flocal, file2.Path, file1.Path) + require.NoError(t, err) + fstest.CheckItems(t, r.flocal) + fstest.CheckItems(t, r.fremote, file2) +} + +func TestCopyFile(t *testing.T) { + r := NewRun(t) + defer r.Finalise() + + file1 := r.WriteFile("file1", "file1 contents", t1) + fstest.CheckItems(t, r.flocal, file1) + + file2 := file1 + file2.Path = "sub/file2" + + err := fs.CopyFile(r.fremote, r.flocal, file2.Path, file1.Path) + require.NoError(t, err) + fstest.CheckItems(t, r.flocal, file1) + fstest.CheckItems(t, r.fremote, file2) + + err = fs.CopyFile(r.fremote, r.flocal, file2.Path, file1.Path) + require.NoError(t, err) + fstest.CheckItems(t, r.flocal, file1) + fstest.CheckItems(t, r.fremote, file2) +} diff --git a/fs/sync.go b/fs/sync.go index 896191e00..6e9138951 100644 --- a/fs/sync.go +++ b/fs/sync.go @@ -120,11 +120,12 @@ func (s *syncCopyMove) readDstFiles() { s.dstFilesResult <- err } -// Check to see if src needs to be copied to dst and if so puts it in out +// NeedTransfer checks to see if src needs to be copied to dst using +// the current config. // -// Returns a flag which indicates whether the file needs to be transferred or not. -func (s *syncCopyMove) checkOne(pair ObjectPair) bool { - src, dst := pair.src, pair.dst +// Returns a flag which indicates whether the file needs to be +// transferred or not. +func NeedTransfer(dst, src Object) bool { if dst == nil { Debug(src, "Couldn't find file - need to transfer") return true @@ -213,7 +214,7 @@ func (s *syncCopyMove) pairChecker(in ObjectPairChan, out ObjectPairChan, wg *sy Stats.Checking(src.Remote()) // Check to see if can store this if src.Storable() { - if s.checkOne(pair) { + if NeedTransfer(pair.dst, pair.src) { out <- pair } else { // If moving need to delete the files we don't need to copy