copyurl: add no-clobber flag - fixes #3950

cmd/copyurl/copyurl.go

@@ -15,12 +15,14 @@ import (
 var (
 	autoFilename = false
 	stdout       = false
+	noClobber    = false
 )
 
 func init() {
 	cmd.Root.AddCommand(commandDefinition)
 	cmdFlags := commandDefinition.Flags()
 	flags.BoolVarP(cmdFlags, &autoFilename, "auto-filename", "a", autoFilename, "Get the file name from the URL and use it for destination file path")
+	flags.BoolVarP(cmdFlags, &noClobber, "no-clobber", "", noClobber, "Prevent overwriting file with same name")
 	flags.BoolVarP(cmdFlags, &stdout, "stdout", "", stdout, "Write the output to stdout rather than a file")
 }
 
@@ -35,6 +37,9 @@ Setting --auto-filename will cause the file name to be retreived from
 the from URL (after any redirections) and used in the destination
 path.
 
+Setting --no-clobber will prevent overwriting file on the 
+destination if there is one with the same name.
+
 Setting --stdout or making the output file name "-" will cause the
 output to be written to standard output.
 `,
@@ -59,7 +64,7 @@ output to be written to standard output.
 			if stdout {
 				err = operations.CopyURLToWriter(context.Background(), args[0], os.Stdout)
 			} else {
-				_, err = operations.CopyURL(context.Background(), fsdst, dstFileName, args[0], autoFilename)
+				_, err = operations.CopyURL(context.Background(), fsdst, dstFileName, args[0], autoFilename, noClobber)
 			}
 			return err
 		})

docs/content/commands/rclone_copyurl.md

@@ -19,6 +19,9 @@ Setting --auto-filename will cause the file name to be retreived from
 the from URL (after any redirections) and used in the destination
 path.
 
+Setting --no-clobber will prevent overwriting file on the 
+destination if there is one with the same name.
+
 Setting --stdout or making the output file name "-" will cause the
 output to be written to standard output.
 
@@ -31,6 +34,7 @@ rclone copyurl https://example.com dest:path [flags]
 
 ```
   -a, --auto-filename   Get the file name from the URL and use it for destination file path
+      --no-clobber      Prevent file overwriting on destination
   -h, --help            help for copyurl
       --stdout          Write the output to stdout rather than a file
 ```

fs/operations/operations.go

@@ -1718,8 +1718,15 @@ func copyURLFn(ctx context.Context, dstFileName string, url string, dstFileNameF
 }
 
 // CopyURL copies the data from the url to (fdst, dstFileName)
-func CopyURL(ctx context.Context, fdst fs.Fs, dstFileName string, url string, dstFileNameFromURL bool) (dst fs.Object, err error) {
+func CopyURL(ctx context.Context, fdst fs.Fs, dstFileName string, url string, dstFileNameFromURL bool, noClobber bool) (dst fs.Object, err error) {
+
 	err = copyURLFn(ctx, dstFileName, url, dstFileNameFromURL, func(ctx context.Context, dstFileName string, in io.ReadCloser, size int64, modTime time.Time) (err error) {
+		if noClobber {
+			_, err = fdst.NewObject(ctx, dstFileName)
+			if err == nil {
+				return errors.New("CopyURL failed: file already exist")
+			}
+		}
 		dst, err = RcatSize(ctx, fdst, dstFileName, in, size, modTime)
 		return err
 	})

fs/operations/operations_test.go

@@ -663,27 +663,31 @@ func TestCopyURL(t *testing.T) {
 	ts := httptest.NewServer(handler)
 	defer ts.Close()
 
-	o, err := operations.CopyURL(context.Background(), r.Fremote, "file1", ts.URL, false)
+	o, err := operations.CopyURL(context.Background(), r.Fremote, "file1", ts.URL, false, false)
 	require.NoError(t, err)
 	assert.Equal(t, int64(len(contents)), o.Size())
 
 	fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1}, nil, fs.ModTimeNotSupported)
 
+	// Check file clobbering
+	o, err = operations.CopyURL(context.Background(), r.Fremote, "file1", ts.URL, false, true)
+	require.Error(t, err)
+
 	// Check auto file naming
 	status = 0
 	urlFileName := "filename.txt"
-	o, err = operations.CopyURL(context.Background(), r.Fremote, "", ts.URL+"/"+urlFileName, true)
+	o, err = operations.CopyURL(context.Background(), r.Fremote, "", ts.URL+"/"+urlFileName, true, false)
 	require.NoError(t, err)
 	assert.Equal(t, int64(len(contents)), o.Size())
 	assert.Equal(t, urlFileName, o.Remote())
 
 	// Check auto file naming when url without file name
-	o, err = operations.CopyURL(context.Background(), r.Fremote, "file1", ts.URL, true)
+	o, err = operations.CopyURL(context.Background(), r.Fremote, "file1", ts.URL, true, false)
 	require.Error(t, err)
 
 	// Check an error is returned for a 404
 	status = http.StatusNotFound
-	o, err = operations.CopyURL(context.Background(), r.Fremote, "file1", ts.URL, false)
+	o, err = operations.CopyURL(context.Background(), r.Fremote, "file1", ts.URL, false, false)
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "Not Found")
 	assert.Nil(t, o)
@@ -699,7 +703,7 @@ func TestCopyURL(t *testing.T) {
 	tss := httptest.NewTLSServer(handler)
 	defer tss.Close()
 
-	o, err = operations.CopyURL(context.Background(), r.Fremote, "file2", tss.URL, false)
+	o, err = operations.CopyURL(context.Background(), r.Fremote, "file2", tss.URL, false, false)
 	require.NoError(t, err)
 	assert.Equal(t, int64(len(contents)), o.Size())
 	fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1, file2, fstest.NewItem(urlFileName, contents, t1)}, nil, fs.ModTimeNotSupported)

fs/operations/rc.go

@@ -215,8 +215,9 @@ func rcSingleCommand(ctx context.Context, in rc.Params, name string, noRemote bo
 			return nil, err
 		}
 		autoFilename, _ := in.GetBool("autoFilename")
+		noClobber, _ := in.GetBool("noClobber")
 
-		_, err = CopyURL(ctx, f, remote, url, autoFilename)
+		_, err = CopyURL(ctx, f, remote, url, autoFilename, noClobber)
 		return nil, err
 	case "cleanup":
 		return nil, CleanUp(ctx, f)

fs/operations/rc_test.go

@@ -109,17 +109,30 @@ func TestRcCopyurl(t *testing.T) {
 		"remote":       "file1",
 		"url":          ts.URL,
 		"autoFilename": false,
+		"noClobber":    false,
 	}
 	out, err := call.Fn(context.Background(), in)
 	require.NoError(t, err)
 	assert.Equal(t, rc.Params(nil), out)
 
+	in = rc.Params{
+		"fs":           r.FremoteName,
+		"remote":       "file1",
+		"url":          ts.URL,
+		"autoFilename": false,
+		"noClobber":    true,
+	}
+	out, err = call.Fn(context.Background(), in)
+	require.Error(t, err)
+	assert.Equal(t, rc.Params(nil), out)
+
 	urlFileName := "filename.txt"
 	in = rc.Params{
 		"fs":           r.FremoteName,
 		"remote":       "",
 		"url":          ts.URL + "/" + urlFileName,
 		"autoFilename": true,
+		"noClobber":    false,
 	}
 	out, err = call.Fn(context.Background(), in)
 	require.NoError(t, err)
@@ -130,6 +143,7 @@ func TestRcCopyurl(t *testing.T) {
 		"remote":       "",
 		"url":          ts.URL,
 		"autoFilename": true,
+		"noClobber":    false,
 	}
 	out, err = call.Fn(context.Background(), in)
 	require.Error(t, err)