TrueCloudLab/rclone
16
0
Fork 2
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

sftp: add --sftp-chunk-size to control packets sizes for high latency links

Browse source 
See: https://forum.rclone.org/t/increasing-sftp-transfer-speed/29928
This commit is contained in:
Nick Craig-Wood 2022-03-28 12:47:22 +01:00
parent 1651429041
commit 95e0934755
1 changed files with 47 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

72
backend/sftp/sftp.go
View file

@ -272,6 +272,26 @@ given, rclone will empty the connection pool.
Set to 0 to keep connections indefinitely. Set to 0 to keep connections indefinitely.
`, `,
Advanced: true, Advanced: true,
}, {
Name: "chunk_size",
Help: `Upload and download chunk size.
This controls the maximum packet size used in the SFTP protocol. The
RFC limits this to 32768 bytes (32k), however a lot of servers
support larger sizes and setting it larger will increase transfer
speed dramatically on high latency links.
Only use a setting higher than 32k if you always connect to the same
server or after sufficiently broad testing.
For example using the value of 252k with OpenSSH works well with its
maximum packet size of 256k.
If you get the error "failed to send packet header: EOF" when copying
a large file, try lowering this number.
`,
Default: 32 * fs.Kibi,
Advanced: true,
}}, }},
} }
fs.Register(fsi) fs.Register(fsi)
@ -279,31 +299,32 @@ Set to 0 to keep connections indefinitely.
// Options defines the configuration for this backend // Options defines the configuration for this backend
type Options struct { type Options struct {
Host string `config:"host"` Host string `config:"host"`
User string `config:"user"` User string `config:"user"`
Port string `config:"port"` Port string `config:"port"`
Pass string `config:"pass"` Pass string `config:"pass"`
KeyPem string `config:"key_pem"` KeyPem string `config:"key_pem"`
KeyFile string `config:"key_file"` KeyFile string `config:"key_file"`
KeyFilePass string `config:"key_file_pass"` KeyFilePass string `config:"key_file_pass"`
PubKeyFile string `config:"pubkey_file"` PubKeyFile string `config:"pubkey_file"`
KnownHostsFile string `config:"known_hosts_file"` KnownHostsFile string `config:"known_hosts_file"`
KeyUseAgent bool `config:"key_use_agent"` KeyUseAgent bool `config:"key_use_agent"`
UseInsecureCipher bool `config:"use_insecure_cipher"` UseInsecureCipher bool `config:"use_insecure_cipher"`
DisableHashCheck bool `config:"disable_hashcheck"` DisableHashCheck bool `config:"disable_hashcheck"`
AskPassword bool `config:"ask_password"` AskPassword bool `config:"ask_password"`
PathOverride string `config:"path_override"` PathOverride string `config:"path_override"`
SetModTime bool `config:"set_modtime"` SetModTime bool `config:"set_modtime"`
ShellType string `config:"shell_type"` ShellType string `config:"shell_type"`
Md5sumCommand string `config:"md5sum_command"` Md5sumCommand string `config:"md5sum_command"`
Sha1sumCommand string `config:"sha1sum_command"` Sha1sumCommand string `config:"sha1sum_command"`
SkipLinks bool `config:"skip_links"` SkipLinks bool `config:"skip_links"`
Subsystem string `config:"subsystem"` Subsystem string `config:"subsystem"`
ServerCommand string `config:"server_command"` ServerCommand string `config:"server_command"`
UseFstat bool `config:"use_fstat"` UseFstat bool `config:"use_fstat"`
DisableConcurrentReads bool `config:"disable_concurrent_reads"` DisableConcurrentReads bool `config:"disable_concurrent_reads"`
DisableConcurrentWrites bool `config:"disable_concurrent_writes"` DisableConcurrentWrites bool `config:"disable_concurrent_writes"`
IdleTimeout fs.Duration `config:"idle_timeout"` IdleTimeout fs.Duration `config:"idle_timeout"`
ChunkSize fs.SizeSuffix `config:"chunk_size"`
} }
// Fs stores the interface to the remote SFTP files // Fs stores the interface to the remote SFTP files
@ -481,6 +502,7 @@ func (f *Fs) newSftpClient(conn *ssh.Client, opts ...sftp.ClientOption) (*sftp.C
sftp.UseFstat(f.opt.UseFstat), sftp.UseFstat(f.opt.UseFstat),
sftp.UseConcurrentReads(!f.opt.DisableConcurrentReads), sftp.UseConcurrentReads(!f.opt.DisableConcurrentReads),
sftp.UseConcurrentWrites(!f.opt.DisableConcurrentWrites), sftp.UseConcurrentWrites(!f.opt.DisableConcurrentWrites),
sftp.MaxPacketUnchecked(int(f.opt.ChunkSize)),
) )
return sftp.NewClientPipe(pr, pw, opts...) return sftp.NewClientPipe(pr, pw, opts...)
} }