From 31a1cc46b7d06e6e0d7c03af77ff0ae4bcb072f8 Mon Sep 17 00:00:00 2001 From: Denis Date: Sun, 19 Apr 2020 14:40:17 +0300 Subject: [PATCH] copyurl: add no-clobber flag - fixes #3950 --- cmd/copyurl/copyurl.go | 7 ++++++- docs/content/commands/rclone_copyurl.md | 4 ++++ fs/operations/operations.go | 9 ++++++++- fs/operations/operations_test.go | 14 +++++++++----- fs/operations/rc.go | 3 ++- fs/operations/rc_test.go | 14 ++++++++++++++ 6 files changed, 43 insertions(+), 8 deletions(-) diff --git a/cmd/copyurl/copyurl.go b/cmd/copyurl/copyurl.go index 7c4aedd5c..738b4a861 100644 --- a/cmd/copyurl/copyurl.go +++ b/cmd/copyurl/copyurl.go @@ -15,12 +15,14 @@ import ( var ( autoFilename = false stdout = false + noClobber = false ) func init() { cmd.Root.AddCommand(commandDefinition) cmdFlags := commandDefinition.Flags() flags.BoolVarP(cmdFlags, &autoFilename, "auto-filename", "a", autoFilename, "Get the file name from the URL and use it for destination file path") + flags.BoolVarP(cmdFlags, &noClobber, "no-clobber", "", noClobber, "Prevent overwriting file with same name") flags.BoolVarP(cmdFlags, &stdout, "stdout", "", stdout, "Write the output to stdout rather than a file") } @@ -35,6 +37,9 @@ Setting --auto-filename will cause the file name to be retreived from the from URL (after any redirections) and used in the destination path. +Setting --no-clobber will prevent overwriting file on the +destination if there is one with the same name. + Setting --stdout or making the output file name "-" will cause the output to be written to standard output. `, @@ -59,7 +64,7 @@ output to be written to standard output. if stdout { err = operations.CopyURLToWriter(context.Background(), args[0], os.Stdout) } else { - _, err = operations.CopyURL(context.Background(), fsdst, dstFileName, args[0], autoFilename) + _, err = operations.CopyURL(context.Background(), fsdst, dstFileName, args[0], autoFilename, noClobber) } return err }) diff --git a/docs/content/commands/rclone_copyurl.md b/docs/content/commands/rclone_copyurl.md index f2e3958c1..d42fbbad0 100644 --- a/docs/content/commands/rclone_copyurl.md +++ b/docs/content/commands/rclone_copyurl.md @@ -19,6 +19,9 @@ Setting --auto-filename will cause the file name to be retreived from the from URL (after any redirections) and used in the destination path. +Setting --no-clobber will prevent overwriting file on the +destination if there is one with the same name. + Setting --stdout or making the output file name "-" will cause the output to be written to standard output. @@ -31,6 +34,7 @@ rclone copyurl https://example.com dest:path [flags] ``` -a, --auto-filename Get the file name from the URL and use it for destination file path + --no-clobber Prevent file overwriting on destination -h, --help help for copyurl --stdout Write the output to stdout rather than a file ``` diff --git a/fs/operations/operations.go b/fs/operations/operations.go index 51d4f6b38..129ea7ca4 100644 --- a/fs/operations/operations.go +++ b/fs/operations/operations.go @@ -1718,8 +1718,15 @@ func copyURLFn(ctx context.Context, dstFileName string, url string, dstFileNameF } // CopyURL copies the data from the url to (fdst, dstFileName) -func CopyURL(ctx context.Context, fdst fs.Fs, dstFileName string, url string, dstFileNameFromURL bool) (dst fs.Object, err error) { +func CopyURL(ctx context.Context, fdst fs.Fs, dstFileName string, url string, dstFileNameFromURL bool, noClobber bool) (dst fs.Object, err error) { + err = copyURLFn(ctx, dstFileName, url, dstFileNameFromURL, func(ctx context.Context, dstFileName string, in io.ReadCloser, size int64, modTime time.Time) (err error) { + if noClobber { + _, err = fdst.NewObject(ctx, dstFileName) + if err == nil { + return errors.New("CopyURL failed: file already exist") + } + } dst, err = RcatSize(ctx, fdst, dstFileName, in, size, modTime) return err }) diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go index 20d00e4af..384c51a41 100644 --- a/fs/operations/operations_test.go +++ b/fs/operations/operations_test.go @@ -663,27 +663,31 @@ func TestCopyURL(t *testing.T) { ts := httptest.NewServer(handler) defer ts.Close() - o, err := operations.CopyURL(context.Background(), r.Fremote, "file1", ts.URL, false) + o, err := operations.CopyURL(context.Background(), r.Fremote, "file1", ts.URL, false, false) require.NoError(t, err) assert.Equal(t, int64(len(contents)), o.Size()) fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1}, nil, fs.ModTimeNotSupported) + // Check file clobbering + o, err = operations.CopyURL(context.Background(), r.Fremote, "file1", ts.URL, false, true) + require.Error(t, err) + // Check auto file naming status = 0 urlFileName := "filename.txt" - o, err = operations.CopyURL(context.Background(), r.Fremote, "", ts.URL+"/"+urlFileName, true) + o, err = operations.CopyURL(context.Background(), r.Fremote, "", ts.URL+"/"+urlFileName, true, false) require.NoError(t, err) assert.Equal(t, int64(len(contents)), o.Size()) assert.Equal(t, urlFileName, o.Remote()) // Check auto file naming when url without file name - o, err = operations.CopyURL(context.Background(), r.Fremote, "file1", ts.URL, true) + o, err = operations.CopyURL(context.Background(), r.Fremote, "file1", ts.URL, true, false) require.Error(t, err) // Check an error is returned for a 404 status = http.StatusNotFound - o, err = operations.CopyURL(context.Background(), r.Fremote, "file1", ts.URL, false) + o, err = operations.CopyURL(context.Background(), r.Fremote, "file1", ts.URL, false, false) require.Error(t, err) assert.Contains(t, err.Error(), "Not Found") assert.Nil(t, o) @@ -699,7 +703,7 @@ func TestCopyURL(t *testing.T) { tss := httptest.NewTLSServer(handler) defer tss.Close() - o, err = operations.CopyURL(context.Background(), r.Fremote, "file2", tss.URL, false) + o, err = operations.CopyURL(context.Background(), r.Fremote, "file2", tss.URL, false, false) require.NoError(t, err) assert.Equal(t, int64(len(contents)), o.Size()) fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1, file2, fstest.NewItem(urlFileName, contents, t1)}, nil, fs.ModTimeNotSupported) diff --git a/fs/operations/rc.go b/fs/operations/rc.go index fd2ceb81e..3c29c3c61 100644 --- a/fs/operations/rc.go +++ b/fs/operations/rc.go @@ -215,8 +215,9 @@ func rcSingleCommand(ctx context.Context, in rc.Params, name string, noRemote bo return nil, err } autoFilename, _ := in.GetBool("autoFilename") + noClobber, _ := in.GetBool("noClobber") - _, err = CopyURL(ctx, f, remote, url, autoFilename) + _, err = CopyURL(ctx, f, remote, url, autoFilename, noClobber) return nil, err case "cleanup": return nil, CleanUp(ctx, f) diff --git a/fs/operations/rc_test.go b/fs/operations/rc_test.go index 20f0cb289..c888509a5 100644 --- a/fs/operations/rc_test.go +++ b/fs/operations/rc_test.go @@ -109,17 +109,30 @@ func TestRcCopyurl(t *testing.T) { "remote": "file1", "url": ts.URL, "autoFilename": false, + "noClobber": false, } out, err := call.Fn(context.Background(), in) require.NoError(t, err) assert.Equal(t, rc.Params(nil), out) + in = rc.Params{ + "fs": r.FremoteName, + "remote": "file1", + "url": ts.URL, + "autoFilename": false, + "noClobber": true, + } + out, err = call.Fn(context.Background(), in) + require.Error(t, err) + assert.Equal(t, rc.Params(nil), out) + urlFileName := "filename.txt" in = rc.Params{ "fs": r.FremoteName, "remote": "", "url": ts.URL + "/" + urlFileName, "autoFilename": true, + "noClobber": false, } out, err = call.Fn(context.Background(), in) require.NoError(t, err) @@ -130,6 +143,7 @@ func TestRcCopyurl(t *testing.T) { "remote": "", "url": ts.URL, "autoFilename": true, + "noClobber": false, } out, err = call.Fn(context.Background(), in) require.Error(t, err)