sftp: fix "file not found" errors for read once servers - fixes #5077
It introduces a new flag --sftp-disable-concurrent-reads to stop the problematic behaviour in the SFTP library for read-once servers. This upgrades the sftp library to v1.13.0 which has the fix.
This commit is contained in:
Nick Craig-Wood
parent
83368998be
commit
713f8f357d
3 changed files with 55 additions and 26 deletions
|
|
@ -204,6 +204,25 @@ Fstat instead of Stat which is called on an already open file handle.
|
||||||
It has been found that this helps with IBM Sterling SFTP servers which have
|
It has been found that this helps with IBM Sterling SFTP servers which have
|
||||||
"extractability" level set to 1 which means only 1 file can be opened at
|
"extractability" level set to 1 which means only 1 file can be opened at
|
||||||
any given time.
|
any given time.
|
||||||
|
`,
|
||||||
|
Advanced: true,
|
||||||
|
}, {
|
||||||
|
Name: "disable_concurrent_reads",
|
||||||
|
Default: false,
|
||||||
|
Help: `If set don't use concurrent reads
|
||||||
|
|
||||||
|
Normally concurrent reads are safe to use and not using them will
|
||||||
|
degrade performance, so this option is disabled by default.
|
||||||
|
|
||||||
|
Some servers limit the amount number of times a file can be
|
||||||
|
downloaded. Using concurrent reads can trigger this limit, so if you
|
||||||
|
have a server which returns
|
||||||
|
|
||||||
|
Failed to copy: file does not exist
|
||||||
|
|
||||||
|
Then you may need to enable this flag.
|
||||||
|
|
||||||
|
If concurrent reads are disabled, the use_fstat option is ignored.
|
||||||
`,
|
`,
|
||||||
Advanced: true,
|
Advanced: true,
|
||||||
}, {
|
}, {
|
||||||
|
|
@ -224,28 +243,29 @@ Set to 0 to keep connections indefinitely.
|
||||||
|
|
||||||
// Options defines the configuration for this backend
|
// Options defines the configuration for this backend
|
||||||
type Options struct {
|
type Options struct {
|
||||||
Host string `config:"host"`
|
Host string `config:"host"`
|
||||||
User string `config:"user"`
|
User string `config:"user"`
|
||||||
Port string `config:"port"`
|
Port string `config:"port"`
|
||||||
Pass string `config:"pass"`
|
Pass string `config:"pass"`
|
||||||
KeyPem string `config:"key_pem"`
|
KeyPem string `config:"key_pem"`
|
||||||
KeyFile string `config:"key_file"`
|
KeyFile string `config:"key_file"`
|
||||||
KeyFilePass string `config:"key_file_pass"`
|
KeyFilePass string `config:"key_file_pass"`
|
||||||
PubKeyFile string `config:"pubkey_file"`
|
PubKeyFile string `config:"pubkey_file"`
|
||||||
KnownHostsFile string `config:"known_hosts_file"`
|
KnownHostsFile string `config:"known_hosts_file"`
|
||||||
KeyUseAgent bool `config:"key_use_agent"`
|
KeyUseAgent bool `config:"key_use_agent"`
|
||||||
UseInsecureCipher bool `config:"use_insecure_cipher"`
|
UseInsecureCipher bool `config:"use_insecure_cipher"`
|
||||||
DisableHashCheck bool `config:"disable_hashcheck"`
|
DisableHashCheck bool `config:"disable_hashcheck"`
|
||||||
AskPassword bool `config:"ask_password"`
|
AskPassword bool `config:"ask_password"`
|
||||||
PathOverride string `config:"path_override"`
|
PathOverride string `config:"path_override"`
|
||||||
SetModTime bool `config:"set_modtime"`
|
SetModTime bool `config:"set_modtime"`
|
||||||
Md5sumCommand string `config:"md5sum_command"`
|
Md5sumCommand string `config:"md5sum_command"`
|
||||||
Sha1sumCommand string `config:"sha1sum_command"`
|
Sha1sumCommand string `config:"sha1sum_command"`
|
||||||
SkipLinks bool `config:"skip_links"`
|
SkipLinks bool `config:"skip_links"`
|
||||||
Subsystem string `config:"subsystem"`
|
Subsystem string `config:"subsystem"`
|
||||||
ServerCommand string `config:"server_command"`
|
ServerCommand string `config:"server_command"`
|
||||||
UseFstat bool `config:"use_fstat"`
|
UseFstat bool `config:"use_fstat"`
|
||||||
IdleTimeout fs.Duration `config:"idle_timeout"`
|
DisableConcurrentReads bool `config:"disable_concurrent_reads"`
|
||||||
|
IdleTimeout fs.Duration `config:"idle_timeout"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fs stores the interface to the remote SFTP files
|
// Fs stores the interface to the remote SFTP files
|
||||||
|
|
@ -373,7 +393,10 @@ func (f *Fs) newSftpClient(conn *ssh.Client, opts ...sftp.ClientOption) (*sftp.C
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
opts = opts[:len(opts):len(opts)] // make sure we don't overwrite the callers opts
|
opts = opts[:len(opts):len(opts)] // make sure we don't overwrite the callers opts
|
||||||
opts = append(opts, sftp.UseFstat(f.opt.UseFstat))
|
opts = append(opts,
|
||||||
|
sftp.UseFstat(f.opt.UseFstat),
|
||||||
|
sftp.UseConcurrentReads(!f.opt.DisableConcurrentReads),
|
||||||
|
)
|
||||||
|
|
||||||
return sftp.NewClientPipe(pr, pw, opts...)
|
return sftp.NewClientPipe(pr, pw, opts...)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
6
go.mod
6
go.mod
|
|
@ -45,7 +45,7 @@ require (
|
||||||
github.com/nsf/termbox-go v0.0.0-20210114135735-d04385b850e8
|
github.com/nsf/termbox-go v0.0.0-20210114135735-d04385b850e8
|
||||||
github.com/patrickmn/go-cache v2.1.0+incompatible
|
github.com/patrickmn/go-cache v2.1.0+incompatible
|
||||||
github.com/pkg/errors v0.9.1
|
github.com/pkg/errors v0.9.1
|
||||||
github.com/pkg/sftp v1.12.0
|
github.com/pkg/sftp v1.13.0
|
||||||
github.com/prometheus/client_golang v1.9.0
|
github.com/prometheus/client_golang v1.9.0
|
||||||
github.com/prometheus/procfs v0.3.0 // indirect
|
github.com/prometheus/procfs v0.3.0 // indirect
|
||||||
github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8
|
github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8
|
||||||
|
|
@ -66,11 +66,11 @@ require (
|
||||||
go.opencensus.io v0.22.6 // indirect
|
go.opencensus.io v0.22.6 // indirect
|
||||||
go.uber.org/zap v1.16.0 // indirect
|
go.uber.org/zap v1.16.0 // indirect
|
||||||
goftp.io/server v0.4.1
|
goftp.io/server v0.4.1
|
||||||
golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad
|
golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83
|
||||||
golang.org/x/net v0.0.0-20210119194325-5f4716e94777
|
golang.org/x/net v0.0.0-20210119194325-5f4716e94777
|
||||||
golang.org/x/oauth2 v0.0.0-20210201163806-010130855d6c
|
golang.org/x/oauth2 v0.0.0-20210201163806-010130855d6c
|
||||||
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a
|
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a
|
||||||
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c
|
golang.org/x/sys v0.0.0-20210313110737-8e9fff1a3a18
|
||||||
golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf // indirect
|
golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf // indirect
|
||||||
golang.org/x/text v0.3.5
|
golang.org/x/text v0.3.5
|
||||||
golang.org/x/time v0.0.0-20201208040808-7e3f01d25324
|
golang.org/x/time v0.0.0-20201208040808-7e3f01d25324
|
||||||
|
|
|
||||||
6
go.sum
6
go.sum
|
|
@ -524,6 +524,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
|
||||||
github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
|
github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
|
||||||
github.com/pkg/sftp v1.12.0 h1:/f3b24xrDhkhddlaobPe2JgBqfdt+gC/NYl0QY9IOuI=
|
github.com/pkg/sftp v1.12.0 h1:/f3b24xrDhkhddlaobPe2JgBqfdt+gC/NYl0QY9IOuI=
|
||||||
github.com/pkg/sftp v1.12.0/go.mod h1:fUqqXB5vEgVCZ131L+9say31RAri6aF6KDViawhxKK8=
|
github.com/pkg/sftp v1.12.0/go.mod h1:fUqqXB5vEgVCZ131L+9say31RAri6aF6KDViawhxKK8=
|
||||||
|
github.com/pkg/sftp v1.13.0 h1:Riw6pgOKK41foc1I1Uu03CjvbLZDXeGpInycM4shXoI=
|
||||||
|
github.com/pkg/sftp v1.13.0/go.mod h1:41g+FIPlQUTDCveupEmEA65IoiQFrtgCeDopC4ajGIM=
|
||||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
|
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
|
||||||
|
|
@ -736,6 +738,8 @@ golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9 h1:umElSU9WZirRdgu2yFHY0a
|
||||||
golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||||
golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY=
|
golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY=
|
||||||
golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
|
golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
|
||||||
|
golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g=
|
||||||
|
golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
|
||||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||||
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||||
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
||||||
|
|
@ -902,6 +906,8 @@ golang.org/x/sys v0.0.0-20210112091331-59c308dcf3cc/go.mod h1:h1NjWce9XRLGQEsW7w
|
||||||
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk=
|
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk=
|
||||||
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20210313110737-8e9fff1a3a18 h1:jxr7/dEo+rR29uEBoLSWJ1tRHCFAMwFbGUU9nRqzpds=
|
||||||
|
golang.org/x/sys v0.0.0-20210313110737-8e9fff1a3a18/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
|
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
|
||||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
|
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
|
||||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue