diff --git a/cmd/copyurl/copyurl.go b/cmd/copyurl/copyurl.go index 54b16e7e9..343dc8679 100644 --- a/cmd/copyurl/copyurl.go +++ b/cmd/copyurl/copyurl.go @@ -4,12 +4,18 @@ import ( "context" "github.com/rclone/rclone/cmd" + "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/operations" "github.com/spf13/cobra" ) +var ( + autoFilename = false +) + func init() { cmd.Root.AddCommand(commandDefintion) + commandDefintion.Flags().BoolVarP(&autoFilename, "auto-filename", "a", autoFilename, "Get the file name from the url and use it for destination file path") } var commandDefintion = &cobra.Command{ @@ -18,13 +24,22 @@ var commandDefintion = &cobra.Command{ Long: ` Download urls content and copy it to destination without saving it in tmp storage. + +Setting --auto-filename flag will cause retrieving file name from url and using it in destination path. `, Run: func(command *cobra.Command, args []string) { cmd.CheckArgs(2, 2, command, args) - fsdst, dstFileName := cmd.NewFsDstFile(args[1:]) + + var dstFileName string + var fsdst fs.Fs + if autoFilename { + fsdst = cmd.NewFsDir(args[1:]) + } else { + fsdst, dstFileName = cmd.NewFsDstFile(args[1:]) + } cmd.Run(true, true, command, func() error { - _, err := operations.CopyURL(context.Background(), fsdst, dstFileName, args[0]) + _, err := operations.CopyURL(context.Background(), fsdst, dstFileName, args[0], autoFilename) return err }) }, diff --git a/docs/content/commands/rclone_copyurl.md b/docs/content/commands/rclone_copyurl.md index 5f96c95c4..e549dee65 100644 --- a/docs/content/commands/rclone_copyurl.md +++ b/docs/content/commands/rclone_copyurl.md @@ -22,7 +22,8 @@ rclone copyurl https://example.com dest:path [flags] ### Options ``` - -h, --help help for copyurl + -a, --auto-filename Get the file name from the url and use it for destination file path + -h, --help help for copyurl ``` See the [global flags page](/flags/) for global options not listed here. diff --git a/fs/operations/operations.go b/fs/operations/operations.go index e0b7c5ca0..65fd1d8ff 100644 --- a/fs/operations/operations.go +++ b/fs/operations/operations.go @@ -1586,7 +1586,7 @@ func RcatSize(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadClo } // CopyURL copies the data from the url to (fdst, dstFileName) -func CopyURL(ctx context.Context, fdst fs.Fs, dstFileName string, url string) (dst fs.Object, err error) { +func CopyURL(ctx context.Context, fdst fs.Fs, dstFileName string, url string, dstFileNameFromURL bool) (dst fs.Object, err error) { client := fshttp.NewClient(fs.Config) resp, err := client.Get(url) if err != nil { @@ -1596,6 +1596,14 @@ func CopyURL(ctx context.Context, fdst fs.Fs, dstFileName string, url string) (d if resp.StatusCode < 200 || resp.StatusCode >= 300 { return nil, errors.Errorf("CopyURL failed: %s", resp.Status) } + + if dstFileNameFromURL { + dstFileName = path.Base(resp.Request.URL.Path) + if dstFileName == "." || dstFileName == "/" { + return nil, errors.Errorf("CopyURL failed: file name wasn't found in url") + } + } + return RcatSize(ctx, fdst, dstFileName, resp.Body, resp.ContentLength, time.Now()) } diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go index 61fa00f98..526690820 100644 --- a/fs/operations/operations_test.go +++ b/fs/operations/operations_test.go @@ -706,15 +706,27 @@ func TestCopyURL(t *testing.T) { ts := httptest.NewServer(handler) defer ts.Close() - o, err := operations.CopyURL(context.Background(), r.Fremote, "file1", ts.URL) + o, err := operations.CopyURL(context.Background(), r.Fremote, "file1", ts.URL, false) require.NoError(t, err) assert.Equal(t, int64(len(contents)), o.Size()) fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1}, nil, fs.ModTimeNotSupported) + // Check auto file naming + status = 0 + urlFileName := "filename.txt" + o, err = operations.CopyURL(context.Background(), r.Fremote, "", ts.URL+"/"+urlFileName, true) + require.NoError(t, err) + assert.Equal(t, int64(len(contents)), o.Size()) + assert.Equal(t, urlFileName, o.Remote()) + + // Check auto file naming when url without file name + o, err = operations.CopyURL(context.Background(), r.Fremote, "file1", ts.URL, true) + require.Error(t, err) + // Check an error is returned for a 404 status = http.StatusNotFound - o, err = operations.CopyURL(context.Background(), r.Fremote, "file1", ts.URL) + o, err = operations.CopyURL(context.Background(), r.Fremote, "file1", ts.URL, false) require.Error(t, err) assert.Contains(t, err.Error(), "Not Found") assert.Nil(t, o) @@ -730,10 +742,10 @@ func TestCopyURL(t *testing.T) { tss := httptest.NewTLSServer(handler) defer tss.Close() - o, err = operations.CopyURL(context.Background(), r.Fremote, "file2", tss.URL) + o, err = operations.CopyURL(context.Background(), r.Fremote, "file2", tss.URL, false) require.NoError(t, err) assert.Equal(t, int64(len(contents)), o.Size()) - fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1, file2}, nil, fs.ModTimeNotSupported) + fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1, file2, fstest.NewItem(urlFileName, contents, t1)}, nil, fs.ModTimeNotSupported) } func TestMoveFile(t *testing.T) { diff --git a/fs/operations/rc.go b/fs/operations/rc.go index 0ba58799b..fd2ceb81e 100644 --- a/fs/operations/rc.go +++ b/fs/operations/rc.go @@ -149,7 +149,7 @@ func init() { {name: "rmdirs", title: "Remove all the empty directories in the path", help: "- leaveRoot - boolean, set to true not to delete the root\n"}, {name: "delete", title: "Remove files in the path", noRemote: true}, {name: "deletefile", title: "Remove the single file pointed to"}, - {name: "copyurl", title: "Copy the URL to the object", help: "- url - string, URL to read from\n"}, + {name: "copyurl", title: "Copy the URL to the object", help: "- url - string, URL to read from\n - autoFilename - boolean, set to true to retrieve destination file name from url"}, {name: "cleanup", title: "Remove trashed files in the remote or path", noRemote: true}, } { op := op @@ -214,7 +214,9 @@ func rcSingleCommand(ctx context.Context, in rc.Params, name string, noRemote bo if err != nil { return nil, err } - _, err = CopyURL(ctx, f, remote, url) + autoFilename, _ := in.GetBool("autoFilename") + + _, err = CopyURL(ctx, f, remote, url, autoFilename) return nil, err case "cleanup": return nil, CleanUp(ctx, f) diff --git a/fs/operations/rc_test.go b/fs/operations/rc_test.go index 3282fe71b..20f0cb289 100644 --- a/fs/operations/rc_test.go +++ b/fs/operations/rc_test.go @@ -105,15 +105,37 @@ func TestRcCopyurl(t *testing.T) { defer ts.Close() in := rc.Params{ - "fs": r.FremoteName, - "remote": "file1", - "url": ts.URL, + "fs": r.FremoteName, + "remote": "file1", + "url": ts.URL, + "autoFilename": false, } out, err := call.Fn(context.Background(), in) require.NoError(t, err) assert.Equal(t, rc.Params(nil), out) - fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1}, nil, fs.ModTimeNotSupported) + urlFileName := "filename.txt" + in = rc.Params{ + "fs": r.FremoteName, + "remote": "", + "url": ts.URL + "/" + urlFileName, + "autoFilename": true, + } + out, err = call.Fn(context.Background(), in) + require.NoError(t, err) + assert.Equal(t, rc.Params(nil), out) + + in = rc.Params{ + "fs": r.FremoteName, + "remote": "", + "url": ts.URL, + "autoFilename": true, + } + out, err = call.Fn(context.Background(), in) + require.Error(t, err) + assert.Equal(t, rc.Params(nil), out) + + fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1, fstest.NewItem(urlFileName, contents, t1)}, nil, fs.ModTimeNotSupported) } // operations/delete: Remove files in the path