copyurl: factor code into operations and write test

cmd/copyurl/copyurl.go

@@ -1,9 +1,6 @@
 package copyurl
 
 import (
-	"net/http"
-	"time"
-
 	"github.com/ncw/rclone/cmd"
 	"github.com/ncw/rclone/fs/operations"
 	"github.com/spf13/cobra"
@@ -25,14 +22,7 @@ without saving it in tmp storage.
 		fsdst, dstFileName := cmd.NewFsDstFile(args[1:])
 
 		cmd.Run(true, true, command, func() error {
-			resp, err := http.Get(args[0])
+			_, err := operations.CopyURL(fsdst, dstFileName, args[0])
-			if err != nil {
-
-				return err
-			}
-
-			_, err = operations.RcatSize(fsdst, dstFileName, resp.Body, resp.ContentLength, time.Now())
-
 			return err
 		})
 	},

fs/operations/operations.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"net/http"
 	"path"
 	"sort"
 	"strconv"
@@ -1359,6 +1360,16 @@ func RcatSize(fdst fs.Fs, dstFileName string, in io.ReadCloser, size int64, modT
 	return obj, nil
 }
 
+// CopyURL copies the data from the url to (fdst, dstFileName)
+func CopyURL(fdst fs.Fs, dstFileName string, url string) (dst fs.Object, err error) {
+	resp, err := http.Get(url)
+	if err != nil {
+		return nil, err
+	}
+	defer fs.CheckClose(resp.Body, &err)
+	return RcatSize(fdst, dstFileName, resp.Body, resp.ContentLength, time.Now())
+}
+
 // moveOrCopyFile moves or copies a single file possibly to a new name
 func moveOrCopyFile(fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName string, cp bool) (err error) {
 	dstFilePath := path.Join(fdst.Root(), dstFileName)

fs/operations/operations_test.go

@@ -25,6 +25,8 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
 	"regexp"
 	"strings"
 	"testing"
@@ -615,6 +617,28 @@ func TestRcatSize(t *testing.T) {
 	fstest.CheckItems(t, r.Fremote, file1, file2)
 }
 
+func TestCopyURL(t *testing.T) {
+	r := fstest.NewRun(t)
+	defer r.Finalise()
+
+	contents := "file1 contents\n"
+	file1 := r.WriteFile("file1", contents, t1)
+	r.Mkdir(r.Fremote)
+	fstest.CheckItems(t, r.Fremote)
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, err := w.Write([]byte(contents))
+		assert.NoError(t, err)
+	}))
+	defer ts.Close()
+
+	o, err := operations.CopyURL(r.Fremote, "file1", ts.URL)
+	require.NoError(t, err)
+	assert.Equal(t, int64(len(contents)), o.Size())
+
+	fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1}, nil, fs.ModTimeNotSupported)
+}
+
 func TestMoveFile(t *testing.T) {
 	r := fstest.NewRun(t)
 	defer r.Finalise()