forked from TrueCloudLab/rclone
rcat: add --size flag for more efficient uploads of known size - fixes #4403
This allows preallocating space at remote end with RcatSize.
This commit is contained in:
parent
a46a3c0811
commit
964c3e0732
1 changed files with 15 additions and 1 deletions
|
@ -7,12 +7,19 @@ import (
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/rclone/rclone/cmd"
|
"github.com/rclone/rclone/cmd"
|
||||||
|
"github.com/rclone/rclone/fs/config/flags"
|
||||||
"github.com/rclone/rclone/fs/operations"
|
"github.com/rclone/rclone/fs/operations"
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
size = int64(-1)
|
||||||
|
)
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
cmd.Root.AddCommand(commandDefinition)
|
cmd.Root.AddCommand(commandDefinition)
|
||||||
|
cmdFlags := commandDefinition.Flags()
|
||||||
|
flags.Int64VarP(cmdFlags, &size, "size", "", size, "File size hint to preallocate")
|
||||||
}
|
}
|
||||||
|
|
||||||
var commandDefinition = &cobra.Command{
|
var commandDefinition = &cobra.Command{
|
||||||
|
@ -37,6 +44,13 @@ must fit into RAM. The cutoff needs to be small enough to adhere
|
||||||
the limits of your remote, please see there. Generally speaking,
|
the limits of your remote, please see there. Generally speaking,
|
||||||
setting this cutoff too high will decrease your performance.
|
setting this cutoff too high will decrease your performance.
|
||||||
|
|
||||||
|
Use the |--size| flag to preallocate the file in advance at the remote end
|
||||||
|
and actually stream it, even if remote backend doesn't support streaming.
|
||||||
|
|
||||||
|
|--size| should be the exact size of the input stream in bytes. If the
|
||||||
|
size of the stream is different in length to the |--size| passed in
|
||||||
|
then the transfer will likely fail.
|
||||||
|
|
||||||
Note that the upload can also not be retried because the data is
|
Note that the upload can also not be retried because the data is
|
||||||
not kept around until the upload succeeds. If you need to transfer
|
not kept around until the upload succeeds. If you need to transfer
|
||||||
a lot of data, you're better off caching locally and then
|
a lot of data, you're better off caching locally and then
|
||||||
|
@ -51,7 +65,7 @@ a lot of data, you're better off caching locally and then
|
||||||
|
|
||||||
fdst, dstFileName := cmd.NewFsDstFile(args)
|
fdst, dstFileName := cmd.NewFsDstFile(args)
|
||||||
cmd.Run(false, false, command, func() error {
|
cmd.Run(false, false, command, func() error {
|
||||||
_, err := operations.Rcat(context.Background(), fdst, dstFileName, os.Stdin, time.Now())
|
_, err := operations.RcatSize(context.Background(), fdst, dstFileName, os.Stdin, size, time.Now())
|
||||||
return err
|
return err
|
||||||
})
|
})
|
||||||
},
|
},
|
||||||
|
|
Loading…
Reference in a new issue