diff --git a/cmd/all/all.go b/cmd/all/all.go index 3d0e42922..da0bd6009 100644 --- a/cmd/all/all.go +++ b/cmd/all/all.go @@ -14,6 +14,7 @@ import ( _ "github.com/ncw/rclone/cmd/config" _ "github.com/ncw/rclone/cmd/copy" _ "github.com/ncw/rclone/cmd/copyto" + _ "github.com/ncw/rclone/cmd/copyurl" _ "github.com/ncw/rclone/cmd/cryptcheck" _ "github.com/ncw/rclone/cmd/cryptdecode" _ "github.com/ncw/rclone/cmd/dbhashsum" diff --git a/cmd/copyurl/copyurl.go b/cmd/copyurl/copyurl.go new file mode 100644 index 000000000..0d5764925 --- /dev/null +++ b/cmd/copyurl/copyurl.go @@ -0,0 +1,39 @@ +package copyurl + +import ( + "net/http" + "time" + + "github.com/ncw/rclone/cmd" + "github.com/ncw/rclone/fs/operations" + "github.com/spf13/cobra" +) + +func init() { + cmd.Root.AddCommand(commandDefintion) +} + +var commandDefintion = &cobra.Command{ + Use: "copyurl https://example.com dest:path", + Short: `Copy url content to dest.`, + Long: ` +Download urls content and copy it to destination +without saving it in tmp storage. +`, + Run: func(command *cobra.Command, args []string) { + cmd.CheckArgs(2, 2, command, args) + fsdst, dstFileName := cmd.NewFsDstFile(args[1:]) + + cmd.Run(true, true, command, func() error { + resp, err := http.Get(args[0]) + if err != nil { + + return err + } + + _, err = operations.RcatSize(fsdst, dstFileName, resp.Body, resp.ContentLength, time.Now()) + + return err + }) + }, +} diff --git a/cmd/serve/restic/restic.go b/cmd/serve/restic/restic.go index c4d451dde..68c2ba5a3 100644 --- a/cmd/serve/restic/restic.go +++ b/cmd/serve/restic/restic.go @@ -6,13 +6,13 @@ import ( "errors" "fmt" "io" - "io/ioutil" "net/http" "os" "path" "regexp" "strconv" "strings" + "time" "github.com/ncw/rclone/cmd" @@ -21,7 +21,6 @@ import ( "github.com/ncw/rclone/fs" "github.com/ncw/rclone/fs/accounting" "github.com/ncw/rclone/fs/fserrors" - "github.com/ncw/rclone/fs/object" "github.com/ncw/rclone/fs/operations" "github.com/ncw/rclone/fs/walk" "github.com/spf13/cobra" @@ -326,46 +325,18 @@ func (s *server) postObject(w http.ResponseWriter, r *http.Request, remote strin if err == nil { fs.Errorf(remote, "Post request: file already exists, refusing to overwrite in append-only mode") http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden) + return } } - // fs.Debugf(s.f, "content length = %d", r.ContentLength) - if r.ContentLength >= 0 { - // Size known use Put - accounting.Stats.Transferring(remote) - body := ioutil.NopCloser(r.Body) // we let the server close the body - in := accounting.NewAccountSizeName(body, r.ContentLength, remote) // account the transfer (no buffering) - var err error - defer func() { - closeErr := in.Close() - if closeErr != nil { - fs.Errorf(remote, "Post request: close failed: %v", closeErr) - if err == nil { - err = closeErr - } - } - ok := err == nil - accounting.Stats.DoneTransferring(remote, err == nil) - if !ok { - accounting.Stats.Error(err) - } - }() - info := object.NewStaticObjectInfo(remote, time.Now(), r.ContentLength, true, nil, s.f) - _, err = s.f.Put(in, info) - if err != nil { - fs.Errorf(remote, "Post request put error: %v", err) - http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) - return - } - } else { - // Size unknown use Rcat - _, err := operations.Rcat(s.f, remote, r.Body, time.Now()) - if err != nil { - fs.Errorf(remote, "Post request rcat error: %v", err) - http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) - return - } + _, err := operations.RcatSize(s.f, remote, r.Body, r.ContentLength, time.Now()) + if err != nil { + accounting.Stats.Error(err) + fs.Errorf(remote, "Post request rcat error: %v", err) + http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) + + return } } diff --git a/fs/operations/operations.go b/fs/operations/operations.go index bec56ffe1..a3ace8b02 100644 --- a/fs/operations/operations.go +++ b/fs/operations/operations.go @@ -1315,6 +1315,45 @@ func NeedTransfer(dst, src fs.Object) bool { return true } +// RcatSize reads data from the Reader until EOF and uploads it to a file on remote. +// Pass in size >=0 if known, <0 if not known +func RcatSize(fdst fs.Fs, dstFileName string, in io.ReadCloser, size int64, modTime time.Time) (dst fs.Object, err error) { + var obj fs.Object + + if size >= 0 { + // Size known use Put + accounting.Stats.Transferring(dstFileName) + body := ioutil.NopCloser(in) // we let the server close the body + in := accounting.NewAccountSizeName(body, size, dstFileName) // account the transfer (no buffering) + var err error + defer func() { + closeErr := in.Close() + if closeErr != nil { + accounting.Stats.Error(closeErr) + fs.Errorf(dstFileName, "Post request: close failed: %v", closeErr) + } + accounting.Stats.DoneTransferring(dstFileName, err == nil) + }() + info := object.NewStaticObjectInfo(dstFileName, modTime, size, true, nil, fdst) + obj, err = fdst.Put(in, info) + if err != nil { + fs.Errorf(dstFileName, "Post request put error: %v", err) + + return nil, err + } + } else { + // Size unknown use Rcat + obj, err = Rcat(fdst, dstFileName, in, modTime) + if err != nil { + fs.Errorf(dstFileName, "Post request rcat error: %v", err) + + return nil, err + } + } + + return obj, nil +} + // moveOrCopyFile moves or copies a single file possibly to a new name func moveOrCopyFile(fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName string, cp bool) (err error) { dstFilePath := path.Join(fdst.Root(), dstFileName) diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go index 4e1ff4cb4..a4870d79e 100644 --- a/fs/operations/operations_test.go +++ b/fs/operations/operations_test.go @@ -468,6 +468,32 @@ func TestRmdirsLeaveRoot(t *testing.T) { ) } +func TestRcatSize(t *testing.T) { + r := fstest.NewRun(t) + defer r.Finalise() + + const body = "------------------------------------------------------------" + file1 := r.WriteFile("potato1", body, t1) + file2 := r.WriteFile("potato2", body, t2) + // Test with known length + bodyReader := ioutil.NopCloser(strings.NewReader(body)) + obj, err := operations.RcatSize(r.Fremote, file1.Path, bodyReader, int64(len(body)), file1.ModTime) + require.NoError(t, err) + assert.Equal(t, int64(len(body)), obj.Size()) + assert.Equal(t, file1.Path, obj.Remote()) + + // Test with unknown length + bodyReader = ioutil.NopCloser(strings.NewReader(body)) // reset Reader + ioutil.NopCloser(strings.NewReader(body)) + obj, err = operations.RcatSize(r.Fremote, file2.Path, bodyReader, -1, file2.ModTime) + require.NoError(t, err) + assert.Equal(t, int64(len(body)), obj.Size()) + assert.Equal(t, file2.Path, obj.Remote()) + + // Check files exist + fstest.CheckItems(t, r.Fremote, file1, file2) +} + func TestMoveFile(t *testing.T) { r := fstest.NewRun(t) defer r.Finalise()