From 964c3e07329c7b79086510cb69f52934b6117a25 Mon Sep 17 00:00:00 2001 From: Nazar Mishturak Date: Tue, 13 Apr 2021 14:25:47 +0300 Subject: [PATCH] rcat: add --size flag for more efficient uploads of known size - fixes #4403 This allows preallocating space at remote end with RcatSize. --- cmd/rcat/rcat.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/cmd/rcat/rcat.go b/cmd/rcat/rcat.go index 205254562..ef969b170 100644 --- a/cmd/rcat/rcat.go +++ b/cmd/rcat/rcat.go @@ -7,12 +7,19 @@ import ( "time" "github.com/rclone/rclone/cmd" + "github.com/rclone/rclone/fs/config/flags" "github.com/rclone/rclone/fs/operations" "github.com/spf13/cobra" ) +var ( + size = int64(-1) +) + func init() { cmd.Root.AddCommand(commandDefinition) + cmdFlags := commandDefinition.Flags() + flags.Int64VarP(cmdFlags, &size, "size", "", size, "File size hint to preallocate") } var commandDefinition = &cobra.Command{ @@ -37,6 +44,13 @@ must fit into RAM. The cutoff needs to be small enough to adhere the limits of your remote, please see there. Generally speaking, setting this cutoff too high will decrease your performance. +Use the |--size| flag to preallocate the file in advance at the remote end +and actually stream it, even if remote backend doesn't support streaming. + +|--size| should be the exact size of the input stream in bytes. If the +size of the stream is different in length to the |--size| passed in +then the transfer will likely fail. + Note that the upload can also not be retried because the data is not kept around until the upload succeeds. If you need to transfer a lot of data, you're better off caching locally and then @@ -51,7 +65,7 @@ a lot of data, you're better off caching locally and then fdst, dstFileName := cmd.NewFsDstFile(args) cmd.Run(false, false, command, func() error { - _, err := operations.Rcat(context.Background(), fdst, dstFileName, os.Stdin, time.Now()) + _, err := operations.RcatSize(context.Background(), fdst, dstFileName, os.Stdin, size, time.Now()) return err }) },