alexvanin/rclone
1
0
Fork 0
forked from TrueCloudLab/rclone
Code Pull requests Activity

ftp,sftp: Make --tpslimit apply - fixes #4906

Browse source
This commit is contained in:
Nick Craig-Wood 2021-01-06 12:19:23 +00:00
parent 50344e7792
commit e775328523
3 changed files with 16 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
backend/ftp/ftp.go
View file

@ -15,6 +15,7 @@ import (
"github.com/jlaffaye/ftp" "github.com/jlaffaye/ftp"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/accounting"
"github.com/rclone/rclone/fs/config" "github.com/rclone/rclone/fs/config"
"github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct" "github.com/rclone/rclone/fs/config/configstruct"
@ -258,6 +259,7 @@ func (f *Fs) getFtpConnection(ctx context.Context) (c *ftp.ServerConn, err error
if f.opt.Concurrency > 0 { if f.opt.Concurrency > 0 {
f.tokens.Get() f.tokens.Get()
} }
accounting.LimitTPS(ctx)
f.poolMu.Lock() f.poolMu.Lock()
if len(f.pool) > 0 { if len(f.pool) > 0 {
c = f.pool[0] c = f.pool[0]

2
backend/sftp/sftp.go
View file

@ -21,6 +21,7 @@ import (
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/pkg/sftp" "github.com/pkg/sftp"
"github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/accounting"
"github.com/rclone/rclone/fs/config" "github.com/rclone/rclone/fs/config"
"github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct" "github.com/rclone/rclone/fs/config/configstruct"
@ -349,6 +350,7 @@ func (f *Fs) newSftpClient(conn *ssh.Client, opts ...sftp.ClientOption) (*sftp.C
// Get an SFTP connection from the pool, or open a new one // Get an SFTP connection from the pool, or open a new one
func (f *Fs) getSftpConnection(ctx context.Context) (c *conn, err error) { func (f *Fs) getSftpConnection(ctx context.Context) (c *conn, err error) {
accounting.LimitTPS(ctx)
f.poolMu.Lock() f.poolMu.Lock()
for len(f.pool) > 0 { for len(f.pool) > 0 {
c = f.pool[0] c = f.pool[0]

15
docs/content/docs.md
View file

@ -1297,10 +1297,15 @@ facility is `DAEMON`.
### --tpslimit float ### ### --tpslimit float ###
Limit HTTP transactions per second to this. Default is 0 which is used Limit transactions per second to this number. Default is 0 which is
to mean unlimited transactions per second. used to mean unlimited transactions per second.
For example to limit rclone to 10 HTTP transactions per second use A transaction is roughly defined as an API call; its exact meaning
will depend on the backend. For HTTP based backends it is an HTTP
PUT/GET/POST/etc and its response. For FTP/SFTP it is a round trip
transaction over TCP.
For example to limit rclone to 10 transactions per second use
`--tpslimit 10`, or to 1 transaction every 2 seconds use `--tpslimit `--tpslimit 10`, or to 1 transaction every 2 seconds use `--tpslimit
0.5`. 0.5`.
@ -1311,6 +1316,10 @@ banned or rate limited).
This can be very useful for `rclone mount` to control the behaviour of This can be very useful for `rclone mount` to control the behaviour of
applications using it. applications using it.
This limit applies to all HTTP based backends and to the FTP and SFTP
backends. It does not apply to the local backend or the Tardigrade
backend.
See also `--tpslimit-burst`. See also `--tpslimit-burst`.
### --tpslimit-burst int ### ### --tpslimit-burst int ###