2021-09-09 12:25:25 +00:00
//go:build !plan9
2019-04-15 19:03:33 +00:00
// +build !plan9
2017-01-31 20:34:11 +00:00
2022-08-28 11:21:57 +00:00
// Package sftp provides a filesystem interface using github.com/pkg/sftp
2016-11-12 23:36:08 +00:00
package sftp
import (
2018-04-19 08:45:46 +00:00
"bytes"
2018-04-06 18:13:27 +00:00
"context"
2021-11-04 10:12:57 +00:00
"errors"
2018-03-14 23:17:09 +00:00
"fmt"
2016-11-12 23:36:08 +00:00
"io"
2023-03-17 11:44:19 +00:00
iofs "io/fs"
2016-11-12 23:36:08 +00:00
"os"
"path"
2017-08-07 13:50:31 +00:00
"regexp"
2019-04-25 09:51:15 +00:00
"strconv"
2017-08-07 13:50:31 +00:00
"strings"
2017-08-07 16:19:37 +00:00
"sync"
2021-04-05 13:18:49 +00:00
"sync/atomic"
2016-11-12 23:36:08 +00:00
"time"
"github.com/pkg/sftp"
2019-07-28 17:47:38 +00:00
"github.com/rclone/rclone/fs"
2021-01-06 12:19:23 +00:00
"github.com/rclone/rclone/fs/accounting"
2019-07-28 17:47:38 +00:00
"github.com/rclone/rclone/fs/config"
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/config/obscure"
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/lib/env"
2019-11-07 13:57:42 +00:00
"github.com/rclone/rclone/lib/pacer"
2019-07-28 17:47:38 +00:00
"github.com/rclone/rclone/lib/readers"
2018-12-04 10:11:57 +00:00
sshagent "github.com/xanzy/ssh-agent"
2017-05-24 14:39:17 +00:00
"golang.org/x/crypto/ssh"
2020-10-04 01:03:19 +00:00
"golang.org/x/crypto/ssh/knownhosts"
2017-08-07 17:01:31 +00:00
)
const (
2021-10-27 15:01:54 +00:00
defaultShellType = "unix"
shellTypeNotSupported = "none"
2019-06-26 15:50:31 +00:00
hashCommandNotSupported = "none"
2019-11-07 13:57:42 +00:00
minSleep = 100 * time . Millisecond
maxSleep = 2 * time . Second
2021-11-15 12:24:31 +00:00
decayConstant = 2 // bigger for slower decay, exponential
keepAliveInterval = time . Minute // send keepalives every this long while running commands
2016-11-12 23:36:08 +00:00
)
2018-01-07 12:57:46 +00:00
var (
2021-10-27 15:01:54 +00:00
currentUser = env . CurrentUser ( )
posixWinAbsPathRegex = regexp . MustCompile ( ` ^/[a-zA-Z]\:($|/) ` ) // E.g. "/C:" or anything starting with "/C:/"
unixShellEscapeRegex = regexp . MustCompile ( "[^A-Za-z0-9_.,:/\\@\u0080-\uFFFFFFFF\n-]" )
2018-01-07 12:57:46 +00:00
)
2016-11-12 23:36:08 +00:00
func init ( ) {
fsi := & fs . RegInfo {
Name : "sftp" ,
2022-06-19 17:29:02 +00:00
Description : "SSH/SFTP" ,
2016-11-12 23:36:08 +00:00
NewFs : NewFs ,
Options : [ ] fs . Option { {
2023-07-06 16:55:53 +00:00
Name : "host" ,
Help : "SSH host to connect to.\n\nE.g. \"example.com\"." ,
Required : true ,
Sensitive : true ,
2016-11-12 23:36:08 +00:00
} , {
2023-07-06 16:55:53 +00:00
Name : "user" ,
Help : "SSH username." ,
Default : currentUser ,
Sensitive : true ,
2016-11-12 23:36:08 +00:00
} , {
2021-09-11 16:29:57 +00:00
Name : "port" ,
Help : "SSH port number." ,
Default : 22 ,
2016-11-12 23:36:08 +00:00
} , {
Name : "pass" ,
2017-06-23 15:25:35 +00:00
Help : "SSH password, leave blank to use ssh-agent." ,
2016-11-12 23:36:08 +00:00
IsPassword : true ,
2020-05-19 10:55:38 +00:00
} , {
2023-07-06 16:55:53 +00:00
Name : "key_pem" ,
Help : "Raw PEM-encoded private key.\n\nIf specified, will override key_file parameter." ,
Sensitive : true ,
2017-06-23 15:25:35 +00:00
} , {
2018-05-14 17:06:57 +00:00
Name : "key_file" ,
2021-08-16 09:30:01 +00:00
Help : "Path to PEM-encoded private key file.\n\nLeave blank or set key-use-agent to use ssh-agent." + env . ShellExpandHelp ,
2019-01-03 11:24:31 +00:00
} , {
Name : "key_file_pass" ,
Help : ` The passphrase to decrypt the PEM - encoded private key file .
Only PEM encrypted key files ( old OpenSSH format ) are supported . Encrypted keys
in the new OpenSSH format can ' t be used . ` ,
IsPassword : true ,
2023-07-06 16:55:53 +00:00
Sensitive : true ,
2020-09-24 18:51:35 +00:00
} , {
Name : "pubkey_file" ,
Help : ` Optional path to public key file .
Set this if you have a signed certificate you want to use for authentication . ` + env . ShellExpandHelp ,
2020-10-04 01:03:19 +00:00
} , {
Name : "known_hosts_file" ,
Help : ` Optional path to known_hosts file .
Set this value to enable server host key validation . ` + env . ShellExpandHelp ,
Advanced : true ,
Examples : [ ] fs . OptionExample { {
Value : "~/.ssh/known_hosts" ,
2021-08-16 09:30:01 +00:00
Help : "Use OpenSSH's known_hosts file." ,
2020-10-04 01:03:19 +00:00
} } ,
2019-01-03 11:25:13 +00:00
} , {
Name : "key_use_agent" ,
Help : ` When set forces the usage of the ssh - agent .
When key - file is also set , the ".pub" file of the specified key - file is read and only the associated key is
requested from the ssh - agent . This allows to avoid ` + " ` Too many authentication failures for * username * ` " + ` errors
when the ssh - agent contains many keys . ` ,
Default : false ,
2017-12-08 12:22:09 +00:00
} , {
2019-10-16 21:22:45 +00:00
Name : "use_insecure_cipher" ,
2022-05-16 16:11:45 +00:00
Help : ` Enable the use of insecure ciphers and key exchange methods .
2019-10-16 21:22:45 +00:00
2020-05-25 06:05:53 +00:00
This enables the use of the following insecure ciphers and key exchange methods :
2019-10-16 21:22:45 +00:00
- aes128 - cbc
- aes192 - cbc
- aes256 - cbc
- 3 des - cbc
- diffie - hellman - group - exchange - sha256
- diffie - hellman - group - exchange - sha1
2022-11-03 17:11:28 +00:00
Those algorithms are insecure and may allow plaintext data to be recovered by an attacker .
This must be false if you use either ciphers or key_exchange advanced options .
` ,
2018-05-14 17:06:57 +00:00
Default : false ,
2017-12-08 12:22:09 +00:00
Examples : [ ] fs . OptionExample {
{
Value : "false" ,
Help : "Use default Cipher list." ,
} , {
Value : "true" ,
2019-07-10 12:23:02 +00:00
Help : "Enables the use of the aes128-cbc cipher and diffie-hellman-group-exchange-sha256, diffie-hellman-group-exchange-sha1 key exchange." ,
2017-12-08 12:22:09 +00:00
} ,
} ,
2018-01-05 09:01:35 +00:00
} , {
2018-05-14 17:06:57 +00:00
Name : "disable_hashcheck" ,
Default : false ,
2021-08-16 09:30:01 +00:00
Help : "Disable the execution of SSH commands to determine if remote file hashing is available.\n\nLeave blank or set to false to enable hashing (recommended), set to true to disable hashing." ,
2018-05-14 17:06:57 +00:00
} , {
2019-09-15 17:23:19 +00:00
Name : "ask_password" ,
Default : false ,
Help : ` Allow asking for SFTP password when needed .
If this is set and no password is supplied then rclone will :
- ask for a password
- not contact the ssh agent
` ,
2018-05-14 17:06:57 +00:00
Advanced : true ,
} , {
2018-10-01 17:36:15 +00:00
Name : "path_override" ,
Default : "" ,
2021-10-27 15:01:54 +00:00
Help : ` Override path used by SSH shell commands .
2018-10-01 17:36:15 +00:00
This allows checksum calculation when SFTP and SSH paths are
different . This issue affects among others Synology NAS boxes .
2021-10-27 15:01:54 +00:00
E . g . if shared folders can be found in directories representing volumes :
2018-10-01 17:36:15 +00:00
2021-11-11 18:20:15 +00:00
rclone sync / home / local / directory remote : / directory -- sftp - path - override / volume2 / directory
2018-10-01 17:36:15 +00:00
2021-10-27 15:01:54 +00:00
E . g . if home directory can be found in a shared folder called "home" :
2018-10-01 17:36:15 +00:00
2021-11-11 18:20:15 +00:00
rclone sync / home / local / directory remote : / home / directory -- sftp - path - override / volume1 / homes / USER / directory ` ,
2018-05-14 17:06:57 +00:00
Advanced : true ,
} , {
Name : "set_modtime" ,
Default : true ,
Help : "Set the modified time on the remote if set." ,
Advanced : true ,
2021-10-27 15:01:54 +00:00
} , {
Name : "shell_type" ,
Default : "" ,
Help : "The type of SSH shell on remote server, if any.\n\nLeave blank for autodetect." ,
Advanced : true ,
Examples : [ ] fs . OptionExample {
{
Value : shellTypeNotSupported ,
Help : "No shell access" ,
} , {
Value : "unix" ,
Help : "Unix shell" ,
} , {
Value : "powershell" ,
Help : "PowerShell" ,
} , {
Value : "cmd" ,
Help : "Windows Command Prompt" ,
} ,
} ,
2019-06-26 15:50:31 +00:00
} , {
Name : "md5sum_command" ,
Default : "" ,
2021-08-16 09:30:01 +00:00
Help : "The command used to read md5 hashes.\n\nLeave blank for autodetect." ,
2019-06-26 15:50:31 +00:00
Advanced : true ,
} , {
Name : "sha1sum_command" ,
Default : "" ,
2021-08-16 09:30:01 +00:00
Help : "The command used to read sha1 hashes.\n\nLeave blank for autodetect." ,
2019-06-26 15:50:31 +00:00
Advanced : true ,
2019-11-14 22:00:30 +00:00
} , {
Name : "skip_links" ,
Default : false ,
Help : "Set to skip any symlinks and any other non regular files." ,
Advanced : true ,
2020-08-13 17:23:54 +00:00
} , {
Name : "subsystem" ,
Default : "sftp" ,
Help : "Specifies the SSH2 subsystem on the remote host." ,
Advanced : true ,
} , {
Name : "server_command" ,
Default : "" ,
Help : ` Specifies the path or command to run a sftp server on the remote host .
The subsystem option is ignored when server_command is defined . ` ,
Advanced : true ,
2021-01-22 12:17:16 +00:00
} , {
Name : "use_fstat" ,
Default : false ,
2021-08-16 09:30:01 +00:00
Help : ` If set use fstat instead of stat .
2021-01-22 12:17:16 +00:00
Some servers limit the amount of open files and calling Stat after opening
the file will throw an error from the server . Setting this flag will call
Fstat instead of Stat which is called on an already open file handle .
It has been found that this helps with IBM Sterling SFTP servers which have
"extractability" level set to 1 which means only 1 file can be opened at
any given time .
2021-02-28 11:36:56 +00:00
` ,
Advanced : true ,
} , {
Name : "disable_concurrent_reads" ,
Default : false ,
2021-08-16 09:30:01 +00:00
Help : ` If set don ' t use concurrent reads .
2021-02-28 11:36:56 +00:00
Normally concurrent reads are safe to use and not using them will
degrade performance , so this option is disabled by default .
Some servers limit the amount number of times a file can be
downloaded . Using concurrent reads can trigger this limit , so if you
have a server which returns
Failed to copy : file does not exist
Then you may need to enable this flag .
If concurrent reads are disabled , the use_fstat option is ignored .
2021-04-05 09:32:20 +00:00
` ,
Advanced : true ,
} , {
Name : "disable_concurrent_writes" ,
Default : false ,
2021-08-16 09:30:01 +00:00
Help : ` If set don ' t use concurrent writes .
2021-04-05 09:32:20 +00:00
Normally rclone uses concurrent writes to upload files . This improves
the performance greatly , especially for distant servers .
This option disables concurrent writes should that be necessary .
2021-02-12 18:41:37 +00:00
` ,
Advanced : true ,
} , {
Name : "idle_timeout" ,
Default : fs . Duration ( 60 * time . Second ) ,
2021-08-16 09:30:01 +00:00
Help : ` Max time before closing idle connections .
2021-02-12 18:41:37 +00:00
If no connections have been returned to the connection pool in the time
given , rclone will empty the connection pool .
Set to 0 to keep connections indefinitely .
2021-01-22 12:17:16 +00:00
` ,
Advanced : true ,
2022-03-28 11:47:22 +00:00
} , {
Name : "chunk_size" ,
Help : ` Upload and download chunk size .
2022-08-30 17:51:00 +00:00
This controls the maximum size of payload in SFTP protocol packets .
The RFC limits this to 32768 bytes ( 32 k ) , which is the default . However ,
a lot of servers support larger sizes , typically limited to a maximum
total package size of 256 k , and setting it larger will increase transfer
speed dramatically on high latency links . This includes OpenSSH , and ,
for example , using the value of 255 k works well , leaving plenty of room
for overhead while still being within a total packet size of 256 k .
Make sure to test thoroughly before using a value higher than 32 k ,
and only use it if you always connect to the same server or after
sufficiently broad testing . If you get errors such as
"failed to send packet payload: EOF" , lots of "connection lost" ,
or "corrupted on transfer" , when copying a larger file , try lowering
the value . The server run by [ rclone serve sftp ] ( / commands / rclone_serve_sftp )
sends packets with standard 32 k maximum payload so you must not
set a different chunk_size when downloading files , but it accepts
packets up to the 256 k total size , so for uploads the chunk_size
can be set as for the OpenSSH example above .
2022-03-28 11:47:22 +00:00
` ,
Default : 32 * fs . Kibi ,
Advanced : true ,
2022-03-28 11:57:34 +00:00
} , {
Name : "concurrency" ,
Help : ` The maximum number of outstanding requests for one file
This controls the maximum number of outstanding requests for one file .
Increasing it will increase throughput on high latency links at the
cost of using more memory .
` ,
Default : 64 ,
Advanced : true ,
2022-04-11 09:41:17 +00:00
} , {
Name : "set_env" ,
Default : fs . SpaceSepList { } ,
Help : ` Environment variables to pass to sftp and commands
Set environment variables in the form :
VAR = value
to be passed to the sftp client and to any commands run ( eg md5sum ) .
Pass multiple variables space separated , eg
VAR1 = value VAR2 = value
2023-03-25 07:58:44 +00:00
and pass variables with spaces in quotes , eg
2022-04-11 09:41:17 +00:00
"VAR3=value with space" "VAR4=value with space" VAR5 = nospacehere
2022-11-03 17:11:28 +00:00
` ,
Advanced : true ,
} , {
Name : "ciphers" ,
Default : fs . SpaceSepList { } ,
Help : ` Space separated list of ciphers to be used for session encryption , ordered by preference .
At least one must match with server configuration . This can be checked for example using ssh - Q cipher .
This must not be set if use_insecure_cipher is true .
Example :
aes128 - ctr aes192 - ctr aes256 - ctr aes128 - gcm @ openssh . com aes256 - gcm @ openssh . com
` ,
Advanced : true ,
} , {
Name : "key_exchange" ,
Default : fs . SpaceSepList { } ,
Help : ` Space separated list of key exchange algorithms , ordered by preference .
At least one must match with server configuration . This can be checked for example using ssh - Q kex .
This must not be set if use_insecure_cipher is true .
Example :
sntrup761x25519 - sha512 @ openssh . com curve25519 - sha256 curve25519 - sha256 @ libssh . org ecdh - sha2 - nistp256
` ,
Advanced : true ,
} , {
Name : "macs" ,
Default : fs . SpaceSepList { } ,
Help : ` Space separated list of MACs ( message authentication code ) algorithms , ordered by preference .
At least one must match with server configuration . This can be checked for example using ssh - Q mac .
Example :
umac - 64 - etm @ openssh . com umac - 128 - etm @ openssh . com hmac - sha2 - 256 - etm @ openssh . com
2023-03-30 17:00:54 +00:00
` ,
Advanced : true ,
} , {
Name : "host_key_algorithms" ,
Default : fs . SpaceSepList { } ,
Help : ` Space separated list of host key algorithms , ordered by preference .
At least one must match with server configuration . This can be checked for example using ssh - Q HostKeyAlgorithms .
Note : This can affect the outcome of key negotiation with the server even if server host key validation is not enabled .
Example :
ssh - ed25519 ssh - rsa ssh - dss
2022-04-11 09:41:17 +00:00
` ,
Advanced : true ,
2016-11-12 23:36:08 +00:00
} } ,
}
fs . Register ( fsi )
}
2018-05-14 17:06:57 +00:00
// Options defines the configuration for this backend
type Options struct {
2022-04-11 09:41:17 +00:00
Host string ` config:"host" `
User string ` config:"user" `
Port string ` config:"port" `
Pass string ` config:"pass" `
KeyPem string ` config:"key_pem" `
KeyFile string ` config:"key_file" `
KeyFilePass string ` config:"key_file_pass" `
PubKeyFile string ` config:"pubkey_file" `
KnownHostsFile string ` config:"known_hosts_file" `
KeyUseAgent bool ` config:"key_use_agent" `
UseInsecureCipher bool ` config:"use_insecure_cipher" `
DisableHashCheck bool ` config:"disable_hashcheck" `
AskPassword bool ` config:"ask_password" `
PathOverride string ` config:"path_override" `
SetModTime bool ` config:"set_modtime" `
ShellType string ` config:"shell_type" `
Md5sumCommand string ` config:"md5sum_command" `
Sha1sumCommand string ` config:"sha1sum_command" `
SkipLinks bool ` config:"skip_links" `
Subsystem string ` config:"subsystem" `
ServerCommand string ` config:"server_command" `
UseFstat bool ` config:"use_fstat" `
DisableConcurrentReads bool ` config:"disable_concurrent_reads" `
DisableConcurrentWrites bool ` config:"disable_concurrent_writes" `
IdleTimeout fs . Duration ` config:"idle_timeout" `
ChunkSize fs . SizeSuffix ` config:"chunk_size" `
Concurrency int ` config:"concurrency" `
SetEnv fs . SpaceSepList ` config:"set_env" `
2022-11-03 17:11:28 +00:00
Ciphers fs . SpaceSepList ` config:"ciphers" `
KeyExchange fs . SpaceSepList ` config:"key_exchange" `
MACs fs . SpaceSepList ` config:"macs" `
2023-03-30 17:00:54 +00:00
HostKeyAlgorithms fs . SpaceSepList ` config:"host_key_algorithms" `
2018-05-14 17:06:57 +00:00
}
2016-11-12 23:36:08 +00:00
// Fs stores the interface to the remote SFTP files
type Fs struct {
2018-05-14 17:06:57 +00:00
name string
root string
2020-06-29 14:49:19 +00:00
absRoot string
2021-10-27 15:01:54 +00:00
shellRoot string
shellType string
2019-06-26 15:50:31 +00:00
opt Options // parsed options
2020-11-05 11:33:32 +00:00
ci * fs . ConfigInfo // global config
2019-06-26 15:50:31 +00:00
m configmap . Mapper // config
features * fs . Features // optional features
2018-05-14 17:06:57 +00:00
config * ssh . ClientConfig
url string
mkdirLock * stringLock
cachedHashes * hash . Set
poolMu sync . Mutex
pool [ ] * conn
2021-02-12 18:41:37 +00:00
drain * time . Timer // used to drain the pool when we stop using the connections
pacer * fs . Pacer // pacer for operations
2020-10-08 12:27:39 +00:00
savedpswd string
2021-10-06 10:50:35 +00:00
sessions int32 // count in use sessions
2016-11-12 23:36:08 +00:00
}
// Object is a remote SFTP file that has been stat'd (so it exists, but is not necessarily open for reading)
type Object struct {
2017-06-30 09:24:06 +00:00
fs * Fs
remote string
size int64 // size of the object
modTime time . Time // modification time of the object
mode os . FileMode // mode bits from the file
2017-08-06 10:49:52 +00:00
md5sum * string // Cached MD5 checksum
sha1sum * string // Cached SHA1 checksum
2016-11-12 23:36:08 +00:00
}
2019-05-10 06:51:01 +00:00
// dial starts a client connection to the given SSH server. It is a
2017-07-23 15:10:23 +00:00
// convenience function that connects to the given network address,
// initiates the SSH handshake, and then sets up a Client.
2020-11-05 11:33:32 +00:00
func ( f * Fs ) dial ( ctx context . Context , network , addr string , sshConfig * ssh . ClientConfig ) ( * ssh . Client , error ) {
2020-11-13 15:24:43 +00:00
dialer := fshttp . NewDialer ( ctx )
2017-07-23 15:10:23 +00:00
conn , err := dialer . Dial ( network , addr )
if err != nil {
return nil , err
}
2018-01-12 16:30:54 +00:00
c , chans , reqs , err := ssh . NewClientConn ( conn , addr , sshConfig )
2017-07-23 15:10:23 +00:00
if err != nil {
return nil , err
}
2019-05-10 06:51:01 +00:00
fs . Debugf ( f , "New connection %s->%s to %q" , c . LocalAddr ( ) , c . RemoteAddr ( ) , c . ServerVersion ( ) )
2017-07-23 15:10:23 +00:00
return ssh . NewClient ( c , chans , reqs ) , nil
}
2017-08-07 16:19:37 +00:00
// conn encapsulates an ssh client and corresponding sftp client
type conn struct {
sshClient * ssh . Client
sftpClient * sftp . Client
err chan error
}
// Wait for connection to close
func ( c * conn ) wait ( ) {
c . err <- c . sshClient . Conn . Wait ( )
}
2021-11-15 12:24:31 +00:00
// Send a keepalive over the ssh connection
func ( c * conn ) sendKeepAlive ( ) {
_ , _ , err := c . sshClient . SendRequest ( "keepalive@openssh.com" , true , nil )
if err != nil {
fs . Debugf ( nil , "Failed to send keep alive: %v" , err )
}
}
// Send keepalives every interval over the ssh connection until done is closed
func ( c * conn ) sendKeepAlives ( interval time . Duration ) ( done chan struct { } ) {
done = make ( chan struct { } )
go func ( ) {
t := time . NewTicker ( interval )
defer t . Stop ( )
for {
select {
case <- t . C :
c . sendKeepAlive ( )
case <- done :
return
}
}
} ( )
return done
}
2017-08-07 16:19:37 +00:00
// Closes the connection
func ( c * conn ) close ( ) error {
sftpErr := c . sftpClient . Close ( )
sshErr := c . sshClient . Close ( )
if sftpErr != nil {
return sftpErr
}
return sshErr
}
// Returns an error if closed
func ( c * conn ) closed ( ) error {
select {
case err := <- c . err :
return err
default :
}
return nil
}
2021-10-06 10:50:35 +00:00
// Show that we are using an ssh session
2021-04-05 13:18:49 +00:00
//
2021-10-06 10:50:35 +00:00
// Call removeSession() when done
func ( f * Fs ) addSession ( ) {
atomic . AddInt32 ( & f . sessions , 1 )
2021-04-05 13:18:49 +00:00
}
2021-10-06 10:50:35 +00:00
// Show the ssh session is no longer in use
func ( f * Fs ) removeSession ( ) {
atomic . AddInt32 ( & f . sessions , - 1 )
2021-04-05 13:18:49 +00:00
}
2021-10-06 10:50:35 +00:00
// getSessions shows whether there are any sessions in use
func ( f * Fs ) getSessions ( ) int32 {
return atomic . LoadInt32 ( & f . sessions )
2021-04-05 13:18:49 +00:00
}
2017-08-07 16:19:37 +00:00
// Open a new connection to the SFTP server.
2020-11-05 11:33:32 +00:00
func ( f * Fs ) sftpConnection ( ctx context . Context ) ( c * conn , err error ) {
2017-08-07 17:01:31 +00:00
// Rate limit rate of new connections
2017-08-07 16:19:37 +00:00
c = & conn {
err : make ( chan error , 1 ) ,
}
2020-11-05 11:33:32 +00:00
c . sshClient , err = f . dial ( ctx , "tcp" , f . opt . Host + ":" + f . opt . Port , f . config )
2017-08-07 16:19:37 +00:00
if err != nil {
2021-11-04 10:12:57 +00:00
return nil , fmt . Errorf ( "couldn't connect SSH: %w" , err )
2017-08-07 16:19:37 +00:00
}
2020-08-13 17:23:54 +00:00
c . sftpClient , err = f . newSftpClient ( c . sshClient )
2017-08-07 16:19:37 +00:00
if err != nil {
_ = c . sshClient . Close ( )
2021-11-04 10:12:57 +00:00
return nil , fmt . Errorf ( "couldn't initialise SFTP: %w" , err )
2017-08-07 16:19:37 +00:00
}
go c . wait ( )
return c , nil
}
2022-04-11 09:41:17 +00:00
// Set any environment variables on the ssh.Session
func ( f * Fs ) setEnv ( s * ssh . Session ) error {
for _ , env := range f . opt . SetEnv {
equal := strings . IndexRune ( env , '=' )
if equal < 0 {
return fmt . Errorf ( "no = found in env var %q" , env )
}
// fs.Debugf(f, "Setting env %q = %q", env[:equal], env[equal+1:])
err := s . Setenv ( env [ : equal ] , env [ equal + 1 : ] )
if err != nil {
2022-06-08 20:54:39 +00:00
return fmt . Errorf ( "failed to set env var %q: %w" , env [ : equal ] , err )
2022-04-11 09:41:17 +00:00
}
}
return nil
}
2020-08-13 17:23:54 +00:00
// Creates a new SFTP client on conn, using the specified subsystem
// or sftp server, and zero or more option functions
func ( f * Fs ) newSftpClient ( conn * ssh . Client , opts ... sftp . ClientOption ) ( * sftp . Client , error ) {
s , err := conn . NewSession ( )
if err != nil {
return nil , err
}
2022-04-11 09:41:17 +00:00
err = f . setEnv ( s )
if err != nil {
return nil , err
}
2020-08-13 17:23:54 +00:00
pw , err := s . StdinPipe ( )
if err != nil {
return nil , err
}
pr , err := s . StdoutPipe ( )
if err != nil {
return nil , err
}
if f . opt . ServerCommand != "" {
if err := s . Start ( f . opt . ServerCommand ) ; err != nil {
return nil , err
}
} else {
if err := s . RequestSubsystem ( f . opt . Subsystem ) ; err != nil {
return nil , err
}
}
2021-01-22 12:17:16 +00:00
opts = opts [ : len ( opts ) : len ( opts ) ] // make sure we don't overwrite the callers opts
2021-02-28 11:36:56 +00:00
opts = append ( opts ,
sftp . UseFstat ( f . opt . UseFstat ) ,
2021-04-05 09:32:20 +00:00
sftp . UseConcurrentReads ( ! f . opt . DisableConcurrentReads ) ,
sftp . UseConcurrentWrites ( ! f . opt . DisableConcurrentWrites ) ,
2022-03-28 11:47:22 +00:00
sftp . MaxPacketUnchecked ( int ( f . opt . ChunkSize ) ) ,
2022-03-28 11:57:34 +00:00
sftp . MaxConcurrentRequestsPerFile ( f . opt . Concurrency ) ,
2021-02-28 11:36:56 +00:00
)
2020-08-13 17:23:54 +00:00
return sftp . NewClientPipe ( pr , pw , opts ... )
}
2017-08-07 16:19:37 +00:00
// Get an SFTP connection from the pool, or open a new one
2020-11-05 11:33:32 +00:00
func ( f * Fs ) getSftpConnection ( ctx context . Context ) ( c * conn , err error ) {
2021-01-06 12:19:23 +00:00
accounting . LimitTPS ( ctx )
2017-08-07 16:19:37 +00:00
f . poolMu . Lock ( )
for len ( f . pool ) > 0 {
c = f . pool [ 0 ]
f . pool = f . pool [ 1 : ]
err := c . closed ( )
if err == nil {
break
}
fs . Errorf ( f , "Discarding closed SSH connection: %v" , err )
c = nil
}
f . poolMu . Unlock ( )
if c != nil {
return c , nil
}
2019-11-07 13:57:42 +00:00
err = f . pacer . Call ( func ( ) ( bool , error ) {
2020-11-05 11:33:32 +00:00
c , err = f . sftpConnection ( ctx )
2019-11-07 13:57:42 +00:00
if err != nil {
return true , err
}
return false , nil
} )
return c , err
2017-08-07 16:19:37 +00:00
}
// Return an SFTP connection to the pool
//
// It nils the pointed to connection out so it can't be reused
//
// if err is not nil then it checks the connection is alive using a
// Getwd request
func ( f * Fs ) putSftpConnection ( pc * * conn , err error ) {
c := * pc
* pc = nil
if err != nil {
// work out if this is an expected error
isRegularError := false
2021-11-04 10:12:57 +00:00
var statusErr * sftp . StatusError
var pathErr * os . PathError
switch {
case errors . Is ( err , os . ErrNotExist ) :
isRegularError = true
case errors . As ( err , & statusErr ) :
isRegularError = true
case errors . As ( err , & pathErr ) :
2017-08-07 16:19:37 +00:00
isRegularError = true
}
// If not a regular SFTP error code then check the connection
if ! isRegularError {
_ , nopErr := c . sftpClient . Getwd ( )
if nopErr != nil {
fs . Debugf ( f , "Connection failed, closing: %v" , nopErr )
_ = c . close ( )
return
}
fs . Debugf ( f , "Connection OK after error: %v" , err )
}
}
f . poolMu . Lock ( )
f . pool = append ( f . pool , c )
2021-02-12 18:41:37 +00:00
if f . opt . IdleTimeout > 0 {
f . drain . Reset ( time . Duration ( f . opt . IdleTimeout ) ) // nudge on the pool emptying timer
}
2017-08-07 16:19:37 +00:00
f . poolMu . Unlock ( )
}
2020-11-27 17:25:57 +00:00
// Drain the pool of any connections
func ( f * Fs ) drainPool ( ctx context . Context ) ( err error ) {
f . poolMu . Lock ( )
defer f . poolMu . Unlock ( )
2021-10-06 10:50:35 +00:00
if sessions := f . getSessions ( ) ; sessions != 0 {
fs . Debugf ( f , "Not closing %d unused connections as %d sessions active" , len ( f . pool ) , sessions )
2021-04-05 13:18:49 +00:00
if f . opt . IdleTimeout > 0 {
f . drain . Reset ( time . Duration ( f . opt . IdleTimeout ) ) // nudge on the pool emptying timer
}
return nil
}
2021-02-12 18:41:37 +00:00
if f . opt . IdleTimeout > 0 {
f . drain . Stop ( )
}
if len ( f . pool ) != 0 {
2021-10-27 15:01:54 +00:00
fs . Debugf ( f , "Closing %d unused connections" , len ( f . pool ) )
2021-02-12 18:41:37 +00:00
}
2020-11-27 17:25:57 +00:00
for i , c := range f . pool {
if cErr := c . closed ( ) ; cErr == nil {
cErr = c . close ( )
if cErr != nil {
err = cErr
}
}
f . pool [ i ] = nil
}
f . pool = nil
return err
}
2016-11-12 23:36:08 +00:00
// NewFs creates a new Fs object from the name and root. It connects to
// the host specified in the config file.
2020-11-05 15:18:51 +00:00
func NewFs ( ctx context . Context , name , root string , m configmap . Mapper ) ( fs . Fs , error ) {
2020-10-08 12:27:39 +00:00
// This will hold the Fs object. We need to create it here
// so we can refer to it in the SSH callback, but it's populated
// in NewFsWithConnection
2020-11-05 11:33:32 +00:00
f := & Fs {
ci : fs . GetConfig ( ctx ) ,
}
2018-05-14 17:06:57 +00:00
// Parse config into Options struct
opt := new ( Options )
err := configstruct . Set ( m , opt )
if err != nil {
return nil , err
}
if opt . User == "" {
opt . User = currentUser
}
if opt . Port == "" {
opt . Port = "22"
2016-11-12 23:36:08 +00:00
}
2020-10-04 01:03:19 +00:00
2018-01-12 16:30:54 +00:00
sshConfig := & ssh . ClientConfig {
2018-05-14 17:06:57 +00:00
User : opt . User ,
2017-05-15 13:00:07 +00:00
Auth : [ ] ssh . AuthMethod { } ,
HostKeyCallback : ssh . InsecureIgnoreHostKey ( ) ,
2020-11-05 11:33:32 +00:00
Timeout : f . ci . ConnectTimeout ,
ClientVersion : "SSH-2.0-" + f . ci . UserAgent ,
2016-11-12 23:36:08 +00:00
}
2017-06-23 15:25:35 +00:00
2023-03-30 17:00:54 +00:00
if len ( opt . HostKeyAlgorithms ) != 0 {
sshConfig . HostKeyAlgorithms = [ ] string ( opt . HostKeyAlgorithms )
}
2020-10-04 01:03:19 +00:00
if opt . KnownHostsFile != "" {
2021-05-11 17:58:26 +00:00
hostcallback , err := knownhosts . New ( env . ShellExpand ( opt . KnownHostsFile ) )
2020-10-04 01:03:19 +00:00
if err != nil {
2021-11-04 10:12:57 +00:00
return nil , fmt . Errorf ( "couldn't parse known_hosts_file: %w" , err )
2020-10-04 01:03:19 +00:00
}
sshConfig . HostKeyCallback = hostcallback
}
2022-11-03 17:11:28 +00:00
if opt . UseInsecureCipher && ( opt . Ciphers != nil || opt . KeyExchange != nil ) {
return nil , fmt . Errorf ( "use_insecure_cipher must be false if ciphers or key_exchange are set in advanced configuration" )
}
sshConfig . Config . SetDefaults ( )
2018-05-14 17:06:57 +00:00
if opt . UseInsecureCipher {
2019-10-16 21:22:45 +00:00
sshConfig . Config . Ciphers = append ( sshConfig . Config . Ciphers , "aes128-cbc" , "aes192-cbc" , "aes256-cbc" , "3des-cbc" )
2019-07-10 12:23:02 +00:00
sshConfig . Config . KeyExchanges = append ( sshConfig . Config . KeyExchanges , "diffie-hellman-group-exchange-sha1" , "diffie-hellman-group-exchange-sha256" )
2022-11-03 17:11:28 +00:00
} else {
if opt . Ciphers != nil {
sshConfig . Config . Ciphers = opt . Ciphers
}
if opt . KeyExchange != nil {
sshConfig . Config . KeyExchanges = opt . KeyExchange
}
}
if opt . MACs != nil {
sshConfig . Config . MACs = opt . MACs
2017-12-08 12:22:09 +00:00
}
2019-05-10 21:07:36 +00:00
keyFile := env . ShellExpand ( opt . KeyFile )
2020-09-24 18:51:35 +00:00
pubkeyFile := env . ShellExpand ( opt . PubKeyFile )
2020-05-19 10:55:38 +00:00
//keyPem := env.ShellExpand(opt.KeyPem)
2020-06-11 11:05:30 +00:00
// Add ssh agent-auth if no password or file or key PEM specified
if ( opt . Pass == "" && keyFile == "" && ! opt . AskPassword && opt . KeyPem == "" ) || opt . KeyUseAgent {
2017-04-10 13:50:06 +00:00
sshAgentClient , _ , err := sshagent . New ( )
2017-01-31 20:34:11 +00:00
if err != nil {
2021-11-04 10:12:57 +00:00
return nil , fmt . Errorf ( "couldn't connect to ssh-agent: %w" , err )
2017-01-31 20:34:11 +00:00
}
signers , err := sshAgentClient . Signers ( )
if err != nil {
2021-11-04 10:12:57 +00:00
return nil , fmt . Errorf ( "couldn't read ssh agent signers: %w" , err )
2017-01-31 20:34:11 +00:00
}
2019-01-03 12:42:13 +00:00
if keyFile != "" {
2023-03-17 11:44:19 +00:00
// If `opt.KeyUseAgent` is false, then it's expected that `opt.KeyFile` contains the private key
// and `${opt.KeyFile}.pub` contains the public key.
//
// If `opt.KeyUseAgent` is true, then it's expected that `opt.KeyFile` contains the public key.
// This is how it works with openssh; the `IdentityFile` in openssh config points to the public key.
// It's not necessary to specify the public key explicitly when using ssh-agent, since openssh and rclone
// will try all the keys they find in the ssh-agent until they find one that works. But just like
// `IdentityFile` is used in openssh config to limit the search to one specific key, so does
// `opt.KeyFile` in rclone config limit the search to that specific key.
//
// However, previous versions of rclone would always expect to find the public key in
// `${opt.KeyFile}.pub` even if `opt.KeyUseAgent` was true. So for the sake of backward compatibility
// we still first attempt to read the public key from `${opt.KeyFile}.pub`. But if it fails with
// an `fs.ErrNotExist` then we also try to read the public key from `opt.KeyFile`.
2022-08-20 14:38:02 +00:00
pubBytes , err := os . ReadFile ( keyFile + ".pub" )
2019-01-03 11:25:13 +00:00
if err != nil {
2023-03-17 11:44:19 +00:00
if errors . Is ( err , iofs . ErrNotExist ) && opt . KeyUseAgent {
pubBytes , err = os . ReadFile ( keyFile )
if err != nil {
return nil , fmt . Errorf ( "failed to read public key file: %w" , err )
}
} else {
return nil , fmt . Errorf ( "failed to read public key file: %w" , err )
}
2019-01-03 11:25:13 +00:00
}
2023-03-17 11:44:19 +00:00
2019-01-03 11:25:13 +00:00
pub , _ , _ , _ , err := ssh . ParseAuthorizedKey ( pubBytes )
if err != nil {
2021-11-04 10:12:57 +00:00
return nil , fmt . Errorf ( "failed to parse public key file: %w" , err )
2019-01-03 11:25:13 +00:00
}
pubM := pub . Marshal ( )
found := false
for _ , s := range signers {
if bytes . Equal ( pubM , s . PublicKey ( ) . Marshal ( ) ) {
sshConfig . Auth = append ( sshConfig . Auth , ssh . PublicKeys ( s ) )
found = true
break
}
}
if ! found {
return nil , errors . New ( "private key not found in the ssh-agent" )
}
} else {
sshConfig . Auth = append ( sshConfig . Auth , ssh . PublicKeys ( signers ... ) )
}
2017-06-23 15:25:35 +00:00
}
2023-03-17 11:44:19 +00:00
// Load key file as a private key, if specified. This is only needed when not using an ssh agent.
if ( keyFile != "" && ! opt . KeyUseAgent ) || opt . KeyPem != "" {
2020-05-19 10:55:38 +00:00
var key [ ] byte
if opt . KeyPem == "" {
2022-08-20 14:38:02 +00:00
key , err = os . ReadFile ( keyFile )
2020-05-19 10:55:38 +00:00
if err != nil {
2021-11-04 10:12:57 +00:00
return nil , fmt . Errorf ( "failed to read private key file: %w" , err )
2020-05-19 10:55:38 +00:00
}
} else {
// wrap in quotes because the config is a coming as a literal without them.
opt . KeyPem , err = strconv . Unquote ( "\"" + opt . KeyPem + "\"" )
if err != nil {
2021-11-04 10:12:57 +00:00
return nil , fmt . Errorf ( "pem key not formatted properly: %w" , err )
2020-05-19 10:55:38 +00:00
}
key = [ ] byte ( opt . KeyPem )
2017-06-23 15:25:35 +00:00
}
2019-01-03 11:24:31 +00:00
clearpass := ""
if opt . KeyFilePass != "" {
clearpass , err = obscure . Reveal ( opt . KeyFilePass )
if err != nil {
return nil , err
}
}
2020-01-13 11:05:16 +00:00
var signer ssh . Signer
if clearpass == "" {
signer , err = ssh . ParsePrivateKey ( key )
} else {
signer , err = ssh . ParsePrivateKeyWithPassphrase ( key , [ ] byte ( clearpass ) )
}
2017-06-23 15:25:35 +00:00
if err != nil {
2021-11-04 10:12:57 +00:00
return nil , fmt . Errorf ( "failed to parse private key file: %w" , err )
2017-06-23 15:25:35 +00:00
}
2020-09-24 18:51:35 +00:00
// If a public key has been specified then use that
if pubkeyFile != "" {
2022-08-20 14:38:02 +00:00
certfile , err := os . ReadFile ( pubkeyFile )
2020-09-24 18:51:35 +00:00
if err != nil {
2021-11-04 10:12:57 +00:00
return nil , fmt . Errorf ( "unable to read cert file: %w" , err )
2020-09-24 18:51:35 +00:00
}
pk , _ , _ , _ , err := ssh . ParseAuthorizedKey ( certfile )
if err != nil {
2021-11-04 10:12:57 +00:00
return nil , fmt . Errorf ( "unable to parse cert file: %w" , err )
2020-09-24 18:51:35 +00:00
}
// And the signer for this, which includes the private key signer
// This is what we'll pass to the ssh client.
// Normally the ssh client will use the public key built
// into the private key, but we need to tell it to use the user
// specified public key cert. This signer is specific to the
// cert and will include the private key signer. Now ssh
// knows everything it needs.
cert , ok := pk . ( * ssh . Certificate )
if ! ok {
return nil , errors . New ( "public key file is not a certificate file: " + pubkeyFile )
}
pubsigner , err := ssh . NewCertSigner ( cert , signer )
if err != nil {
2021-11-04 10:12:57 +00:00
return nil , fmt . Errorf ( "error generating cert signer: %w" , err )
2020-09-24 18:51:35 +00:00
}
sshConfig . Auth = append ( sshConfig . Auth , ssh . PublicKeys ( pubsigner ) )
} else {
sshConfig . Auth = append ( sshConfig . Auth , ssh . PublicKeys ( signer ) )
}
2017-06-23 15:25:35 +00:00
}
// Auth from password if specified
2018-05-14 17:06:57 +00:00
if opt . Pass != "" {
clearpass , err := obscure . Reveal ( opt . Pass )
2016-11-12 23:36:08 +00:00
if err != nil {
return nil , err
}
2020-12-29 17:15:16 +00:00
sshConfig . Auth = append ( sshConfig . Auth ,
ssh . Password ( clearpass ) ,
ssh . KeyboardInteractive ( func ( user , instruction string , questions [ ] string , echos [ ] bool ) ( [ ] string , error ) {
return f . keyboardInteractiveReponse ( user , instruction , questions , echos , clearpass )
} ) ,
)
2016-11-12 23:36:08 +00:00
}
2017-06-23 15:25:35 +00:00
2020-10-06 22:56:28 +00:00
// Config for password if none was defined and we're allowed to
// We don't ask now; we ask if the ssh connection succeeds
2018-05-14 17:06:57 +00:00
if opt . Pass == "" && opt . AskPassword {
2020-12-29 17:15:16 +00:00
sshConfig . Auth = append ( sshConfig . Auth ,
ssh . PasswordCallback ( f . getPass ) ,
ssh . KeyboardInteractive ( func ( user , instruction string , questions [ ] string , echos [ ] bool ) ( [ ] string , error ) {
pass , _ := f . getPass ( )
return f . keyboardInteractiveReponse ( user , instruction , questions , echos , pass )
} ) ,
)
2018-03-14 23:17:09 +00:00
}
2020-10-08 12:27:39 +00:00
return NewFsWithConnection ( ctx , f , name , root , m , opt , sshConfig )
2019-02-19 16:40:15 +00:00
}
2020-12-29 17:15:16 +00:00
// Do the keyboard interactive challenge
//
// Just send the password back for all questions
func ( f * Fs ) keyboardInteractiveReponse ( user , instruction string , questions [ ] string , echos [ ] bool , pass string ) ( [ ] string , error ) {
2021-10-27 15:01:54 +00:00
fs . Debugf ( f , "Keyboard interactive auth requested" )
2020-12-29 17:15:16 +00:00
answers := make ( [ ] string , len ( questions ) )
for i := range answers {
answers [ i ] = pass
}
return answers , nil
}
2020-10-06 22:56:28 +00:00
// If we're in password mode and ssh connection succeeds then this
2020-10-08 12:27:39 +00:00
// callback is called. First time around we ask the user, and then
// save it so on reconnection we give back the previous string.
// This removes the ability to let the user correct a mistaken entry,
// but means that reconnects are transparent.
// We'll re-use config.Pass for this, 'cos we know it's not been
// specified.
func ( f * Fs ) getPass ( ) ( string , error ) {
for f . savedpswd == "" {
_ , _ = fmt . Fprint ( os . Stderr , "Enter SFTP password: " )
f . savedpswd = config . ReadPassword ( )
}
return f . savedpswd , nil
2020-10-06 22:56:28 +00:00
}
2020-05-20 10:39:20 +00:00
// NewFsWithConnection creates a new Fs object from the name and root and an ssh.ClientConfig. It connects to
2019-02-19 16:40:15 +00:00
// the host specified in the ssh.ClientConfig
2020-10-08 12:27:39 +00:00
func NewFsWithConnection ( ctx context . Context , f * Fs , name string , root string , m configmap . Mapper , opt * Options , sshConfig * ssh . ClientConfig ) ( fs . Fs , error ) {
// Populate the Filesystem Object
f . name = name
f . root = root
f . absRoot = root
2021-10-27 15:01:54 +00:00
f . shellRoot = root
2020-10-08 12:27:39 +00:00
f . opt = * opt
f . m = m
f . config = sshConfig
f . url = "sftp://" + opt . User + "@" + opt . Host + ":" + opt . Port + "/" + root
f . mkdirLock = newStringLock ( )
2020-11-05 11:33:32 +00:00
f . pacer = fs . NewPacer ( ctx , pacer . NewDefault ( pacer . MinSleep ( minSleep ) , pacer . MaxSleep ( maxSleep ) , pacer . DecayConstant ( decayConstant ) ) )
2020-10-08 12:27:39 +00:00
f . savedpswd = ""
2021-02-12 18:41:37 +00:00
// set the pool drainer timer going
if f . opt . IdleTimeout > 0 {
2021-10-27 15:01:54 +00:00
f . drain = time . AfterFunc ( time . Duration ( f . opt . IdleTimeout ) , func ( ) { _ = f . drainPool ( ctx ) } )
2021-02-12 18:41:37 +00:00
}
2020-10-08 12:27:39 +00:00
2017-08-09 14:27:43 +00:00
f . features = ( & fs . Features {
CanHaveEmptyDirectories : true ,
2020-06-19 09:28:34 +00:00
SlowHash : true ,
2023-04-04 15:19:16 +00:00
PartialUploads : true ,
2020-11-05 16:00:40 +00:00
} ) . Fill ( ctx , f )
2017-08-07 16:19:37 +00:00
// Make a connection and pool it to return errors early
2020-11-05 11:33:32 +00:00
c , err := f . getSftpConnection ( ctx )
2017-08-07 16:19:37 +00:00
if err != nil {
2021-11-04 10:12:57 +00:00
return nil , fmt . Errorf ( "NewFs: %w" , err )
2017-08-07 16:19:37 +00:00
}
2021-10-27 15:01:54 +00:00
// Check remote shell type, try to auto-detect if not configured and save to config for later
if f . opt . ShellType != "" {
f . shellType = f . opt . ShellType
fs . Debugf ( f , "Shell type %q from config" , f . shellType )
} else {
session , err := c . sshClient . NewSession ( )
if err != nil {
f . shellType = shellTypeNotSupported
fs . Debugf ( f , "Failed to get shell session for shell type detection command: %v" , err )
} else {
var stdout , stderr bytes . Buffer
session . Stdout = & stdout
session . Stderr = & stderr
shellCmd := "echo ${ShellId}%ComSpec%"
fs . Debugf ( f , "Running shell type detection remote command: %s" , shellCmd )
err = session . Run ( shellCmd )
_ = session . Close ( )
2022-11-08 17:52:39 +00:00
f . shellType = defaultShellType
2021-10-27 15:01:54 +00:00
if err != nil {
fs . Debugf ( f , "Remote command failed: %v (stdout=%v) (stderr=%v)" , err , bytes . TrimSpace ( stdout . Bytes ( ) ) , bytes . TrimSpace ( stderr . Bytes ( ) ) )
} else {
outBytes := stdout . Bytes ( )
fs . Debugf ( f , "Remote command result: %s" , outBytes )
outString := string ( bytes . TrimSpace ( stdout . Bytes ( ) ) )
2022-11-08 17:52:39 +00:00
if outString != "" {
if strings . HasPrefix ( outString , "Microsoft.PowerShell" ) { // PowerShell: "Microsoft.PowerShell%ComSpec%"
f . shellType = "powershell"
} else if ! strings . HasSuffix ( outString , "%ComSpec%" ) { // Command Prompt: "${ShellId}C:\WINDOWS\system32\cmd.exe"
// Additional positive test, to avoid misdetection on unpredicted Unix shell variants
s := strings . ToLower ( outString )
if strings . Contains ( s , ".exe" ) || strings . Contains ( s , ".com" ) {
f . shellType = "cmd"
}
} // POSIX-based Unix shell: "%ComSpec%"
} // fish Unix shell: ""
2021-10-27 15:01:54 +00:00
}
}
// Save permanently in config to avoid the extra work next time
fs . Debugf ( f , "Shell type %q detected (set option shell_type to override)" , f . shellType )
f . m . Set ( "shell_type" , f . shellType )
}
// Ensure we have absolute path to root
// It appears that WS FTP doesn't like relative paths,
// and the openssh sftp tool also uses absolute paths.
if ! path . IsAbs ( f . root ) {
2022-07-19 07:15:50 +00:00
// Trying RealPath first, to perform proper server-side canonicalize.
// It may fail (SSH_FX_FAILURE reported on WS FTP) and will then resort
// to simple path join with current directory from Getwd (which can work
// on WS FTP, even though it is also based on RealPath).
absRoot , err := c . sftpClient . RealPath ( f . root )
2021-10-27 15:01:54 +00:00
if err != nil {
2022-07-19 07:15:50 +00:00
fs . Debugf ( f , "Failed to resolve path using RealPath: %v" , err )
cwd , err := c . sftpClient . Getwd ( )
if err != nil {
fs . Debugf ( f , "Failed to to read current directory - using relative paths: %v" , err )
} else {
f . absRoot = path . Join ( cwd , f . root )
fs . Debugf ( f , "Relative path joined with current directory to get absolute path %q" , f . absRoot )
}
2021-10-27 15:01:54 +00:00
} else {
2022-07-19 07:15:50 +00:00
f . absRoot = absRoot
2021-10-27 15:01:54 +00:00
fs . Debugf ( f , "Relative path resolved to %q" , f . absRoot )
}
2020-06-29 14:49:19 +00:00
}
2021-10-27 15:01:54 +00:00
f . putSftpConnection ( & c , err )
2023-05-10 14:06:13 +00:00
if root != "" && ! strings . HasSuffix ( root , "/" ) {
2021-10-27 15:01:54 +00:00
// Check to see if the root is actually an existing file,
// and if so change the filesystem root to its parent directory.
2020-06-29 14:49:19 +00:00
oldAbsRoot := f . absRoot
2017-01-31 20:34:11 +00:00
remote := path . Base ( root )
f . root = path . Dir ( root )
2020-06-29 14:49:19 +00:00
f . absRoot = path . Dir ( f . absRoot )
2017-01-31 20:34:11 +00:00
if f . root == "." {
f . root = ""
}
2021-10-27 15:01:54 +00:00
_ , err = f . NewObject ( ctx , remote )
2017-01-31 20:34:11 +00:00
if err != nil {
2021-10-27 15:01:54 +00:00
if err != fs . ErrorObjectNotFound && err != fs . ErrorIsDir {
return nil , err
2017-01-31 20:34:11 +00:00
}
2021-10-27 15:01:54 +00:00
// File doesn't exist so keep the old f
f . root = root
f . absRoot = oldAbsRoot
err = nil
} else {
// File exists so change fs to point to the parent and return it with an error
err = fs . ErrorIsFile
2016-11-12 23:36:08 +00:00
}
2021-10-27 15:01:54 +00:00
} else {
err = nil
2016-11-12 23:36:08 +00:00
}
2021-10-27 15:01:54 +00:00
fs . Debugf ( f , "Using root directory %q" , f . absRoot )
return f , err
2016-11-12 23:36:08 +00:00
}
// Name returns the configured name of the file system
func ( f * Fs ) Name ( ) string {
return f . name
}
// Root returns the root for the filesystem
func ( f * Fs ) Root ( ) string {
return f . root
}
// String returns the URL for the filesystem
func ( f * Fs ) String ( ) string {
return f . url
}
// Features returns the optional features of this Fs
func ( f * Fs ) Features ( ) * fs . Features {
return f . features
}
// Precision is the remote sftp file system's modtime precision, which we have no way of knowing. We estimate at 1s
func ( f * Fs ) Precision ( ) time . Duration {
return time . Second
}
// NewObject creates a new remote sftp file object
2019-06-17 08:34:30 +00:00
func ( f * Fs ) NewObject ( ctx context . Context , remote string ) ( fs . Object , error ) {
2017-01-31 20:34:11 +00:00
o := & Object {
2016-11-12 23:36:08 +00:00
fs : f ,
remote : remote ,
}
2020-11-05 11:33:32 +00:00
err := o . stat ( ctx )
2017-01-31 20:34:11 +00:00
if err != nil {
return nil , err
}
return o , nil
2016-11-12 23:36:08 +00:00
}
2017-01-31 20:34:11 +00:00
// dirExists returns true,nil if the directory exists, false, nil if
// it doesn't or false, err
2020-11-05 11:33:32 +00:00
func ( f * Fs ) dirExists ( ctx context . Context , dir string ) ( bool , error ) {
2017-01-31 20:34:11 +00:00
if dir == "" {
dir = "."
}
2020-11-05 11:33:32 +00:00
c , err := f . getSftpConnection ( ctx )
2017-08-07 16:19:37 +00:00
if err != nil {
2021-11-04 10:12:57 +00:00
return false , fmt . Errorf ( "dirExists: %w" , err )
2017-08-07 16:19:37 +00:00
}
info , err := c . sftpClient . Stat ( dir )
f . putSftpConnection ( & c , err )
2017-01-31 20:34:11 +00:00
if err != nil {
if os . IsNotExist ( err ) {
return false , nil
}
2021-11-04 10:12:57 +00:00
return false , fmt . Errorf ( "dirExists stat failed: %w" , err )
2017-01-31 20:34:11 +00:00
}
if ! info . IsDir ( ) {
return false , fs . ErrorIsFile
}
return true , nil
}
2017-06-11 21:43:31 +00:00
// List the objects and directories in dir into entries. The
// entries can be returned in any order but should be for a
// complete directory.
//
// dir should be "" to list the root, and should not have
// trailing slashes.
//
// This should return ErrDirNotFound if the directory isn't
// found.
2019-06-17 08:34:30 +00:00
func ( f * Fs ) List ( ctx context . Context , dir string ) ( entries fs . DirEntries , err error ) {
2020-06-29 14:49:19 +00:00
root := path . Join ( f . absRoot , dir )
2017-06-11 21:43:31 +00:00
sftpDir := root
2017-01-31 20:34:11 +00:00
if sftpDir == "" {
sftpDir = "."
}
2020-11-05 11:33:32 +00:00
c , err := f . getSftpConnection ( ctx )
2017-08-07 16:19:37 +00:00
if err != nil {
2021-11-04 10:12:57 +00:00
return nil , fmt . Errorf ( "List: %w" , err )
2017-08-07 16:19:37 +00:00
}
infos , err := c . sftpClient . ReadDir ( sftpDir )
f . putSftpConnection ( & c , err )
2017-01-31 20:34:11 +00:00
if err != nil {
2023-05-10 10:34:20 +00:00
if errors . Is ( err , os . ErrNotExist ) {
return nil , fs . ErrorDirNotFound
}
2021-11-04 10:12:57 +00:00
return nil , fmt . Errorf ( "error listing %q: %w" , dir , err )
2017-01-31 20:34:11 +00:00
}
for _ , info := range infos {
remote := path . Join ( dir , info . Name ( ) )
2018-03-16 15:36:47 +00:00
// If file is a symlink (not a regular file is the best cross platform test we can do), do a stat to
// pick up the size and type of the destination, instead of the size and type of the symlink.
2019-11-14 22:00:30 +00:00
if ! info . Mode ( ) . IsRegular ( ) && ! info . IsDir ( ) {
if f . opt . SkipLinks {
// skip non regular file if SkipLinks is set
continue
}
2019-01-28 17:10:00 +00:00
oldInfo := info
2020-11-05 11:33:32 +00:00
info , err = f . stat ( ctx , remote )
2018-03-16 15:36:47 +00:00
if err != nil {
2019-01-28 17:10:00 +00:00
if ! os . IsNotExist ( err ) {
2019-11-14 22:00:30 +00:00
fs . Errorf ( remote , "stat of non-regular file failed: %v" , err )
2019-01-28 17:10:00 +00:00
}
info = oldInfo
2018-03-16 15:36:47 +00:00
}
}
2017-01-31 20:34:11 +00:00
if info . IsDir ( ) {
2017-06-30 12:37:29 +00:00
d := fs . NewDir ( remote , info . ModTime ( ) )
2017-06-11 21:43:31 +00:00
entries = append ( entries , d )
2017-01-31 20:34:11 +00:00
} else {
2017-06-11 21:43:31 +00:00
o := & Object {
2017-01-31 20:34:11 +00:00
fs : f ,
remote : remote ,
}
2017-06-30 09:24:06 +00:00
o . setMetadata ( info )
2017-06-11 21:43:31 +00:00
entries = append ( entries , o )
2016-11-12 23:36:08 +00:00
}
}
2017-06-11 21:43:31 +00:00
return entries , nil
2016-11-12 23:36:08 +00:00
}
2019-06-17 08:34:30 +00:00
// Put data from <in> into a new remote sftp file object described by <src.Remote()> and <src.ModTime(ctx)>
func ( f * Fs ) Put ( ctx context . Context , in io . Reader , src fs . ObjectInfo , options ... fs . OpenOption ) ( fs . Object , error ) {
2020-11-05 11:33:32 +00:00
err := f . mkParentDir ( ctx , src . Remote ( ) )
2016-11-12 23:36:08 +00:00
if err != nil {
2021-11-04 10:12:57 +00:00
return nil , fmt . Errorf ( "Put mkParentDir failed: %w" , err )
2016-11-12 23:36:08 +00:00
}
2017-01-31 20:34:11 +00:00
// Temporary object under construction
o := & Object {
fs : f ,
remote : src . Remote ( ) ,
2016-11-12 23:36:08 +00:00
}
2019-06-17 08:34:30 +00:00
err = o . Update ( ctx , in , src , options ... )
2016-11-12 23:36:08 +00:00
if err != nil {
return nil , err
}
return o , nil
}
2017-08-03 19:42:35 +00:00
// PutStream uploads to the remote path with the modTime given of indeterminate size
2019-06-17 08:34:30 +00:00
func ( f * Fs ) PutStream ( ctx context . Context , in io . Reader , src fs . ObjectInfo , options ... fs . OpenOption ) ( fs . Object , error ) {
return f . Put ( ctx , in , src , options ... )
2017-08-03 19:42:35 +00:00
}
2017-01-31 20:34:11 +00:00
// mkParentDir makes the parent of remote if necessary and any
// directories above that
2020-11-05 11:33:32 +00:00
func ( f * Fs ) mkParentDir ( ctx context . Context , remote string ) error {
2017-01-31 20:34:11 +00:00
parent := path . Dir ( remote )
2020-11-05 11:33:32 +00:00
return f . mkdir ( ctx , path . Join ( f . absRoot , parent ) )
2017-01-31 20:34:11 +00:00
}
// mkdir makes the directory and parents using native paths
2020-11-05 11:33:32 +00:00
func ( f * Fs ) mkdir ( ctx context . Context , dirPath string ) error {
2017-05-24 14:39:17 +00:00
f . mkdirLock . Lock ( dirPath )
defer f . mkdirLock . Unlock ( dirPath )
if dirPath == "." || dirPath == "/" {
2017-01-31 20:34:11 +00:00
return nil
}
2020-11-05 11:33:32 +00:00
ok , err := f . dirExists ( ctx , dirPath )
2017-01-31 20:34:11 +00:00
if err != nil {
2021-11-04 10:12:57 +00:00
return fmt . Errorf ( "mkdir dirExists failed: %w" , err )
2017-01-31 20:34:11 +00:00
}
if ok {
return nil
}
2017-05-24 14:39:17 +00:00
parent := path . Dir ( dirPath )
2020-11-05 11:33:32 +00:00
err = f . mkdir ( ctx , parent )
2017-01-31 20:34:11 +00:00
if err != nil {
return err
}
2020-11-05 11:33:32 +00:00
c , err := f . getSftpConnection ( ctx )
2017-08-07 16:19:37 +00:00
if err != nil {
2021-11-04 10:12:57 +00:00
return fmt . Errorf ( "mkdir: %w" , err )
2017-08-07 16:19:37 +00:00
}
err = c . sftpClient . Mkdir ( dirPath )
f . putSftpConnection ( & c , err )
2017-01-31 20:34:11 +00:00
if err != nil {
2022-09-14 15:45:35 +00:00
if os . IsExist ( err ) {
fs . Debugf ( f , "directory %q exists after Mkdir is attempted" , dirPath )
return nil
}
2021-11-04 10:12:57 +00:00
return fmt . Errorf ( "mkdir %q failed: %w" , dirPath , err )
2016-11-12 23:36:08 +00:00
}
2017-01-31 20:34:11 +00:00
return nil
2016-11-12 23:36:08 +00:00
}
// Mkdir makes the root directory of the Fs object
2019-06-17 08:34:30 +00:00
func ( f * Fs ) Mkdir ( ctx context . Context , dir string ) error {
2020-06-29 14:49:19 +00:00
root := path . Join ( f . absRoot , dir )
2020-11-05 11:33:32 +00:00
return f . mkdir ( ctx , root )
2016-11-12 23:36:08 +00:00
}
// Rmdir removes the root directory of the Fs object
2019-06-17 08:34:30 +00:00
func ( f * Fs ) Rmdir ( ctx context . Context , dir string ) error {
2018-11-30 17:37:55 +00:00
// Check to see if directory is empty as some servers will
// delete recursively with RemoveDirectory
2019-06-17 08:34:30 +00:00
entries , err := f . List ( ctx , dir )
2018-11-30 17:37:55 +00:00
if err != nil {
2021-11-04 10:12:57 +00:00
return fmt . Errorf ( "Rmdir: %w" , err )
2018-11-30 17:37:55 +00:00
}
if len ( entries ) != 0 {
return fs . ErrorDirectoryNotEmpty
}
// Remove the directory
2020-06-29 14:49:19 +00:00
root := path . Join ( f . absRoot , dir )
2020-11-05 11:33:32 +00:00
c , err := f . getSftpConnection ( ctx )
2017-08-07 16:19:37 +00:00
if err != nil {
2021-11-04 10:12:57 +00:00
return fmt . Errorf ( "Rmdir: %w" , err )
2017-08-07 16:19:37 +00:00
}
2018-11-29 21:34:37 +00:00
err = c . sftpClient . RemoveDirectory ( root )
2017-08-07 16:19:37 +00:00
f . putSftpConnection ( & c , err )
return err
2016-11-12 23:36:08 +00:00
}
// Move renames a remote sftp file object
2019-06-17 08:34:30 +00:00
func ( f * Fs ) Move ( ctx context . Context , src fs . Object , remote string ) ( fs . Object , error ) {
2017-01-31 20:34:11 +00:00
srcObj , ok := src . ( * Object )
if ! ok {
2017-02-09 11:01:20 +00:00
fs . Debugf ( src , "Can't move - not same remote type" )
2017-01-31 20:34:11 +00:00
return nil , fs . ErrorCantMove
}
2020-11-05 11:33:32 +00:00
err := f . mkParentDir ( ctx , remote )
2016-11-12 23:36:08 +00:00
if err != nil {
2021-11-04 10:12:57 +00:00
return nil , fmt . Errorf ( "Move mkParentDir failed: %w" , err )
2017-01-31 20:34:11 +00:00
}
2020-11-05 11:33:32 +00:00
c , err := f . getSftpConnection ( ctx )
2017-08-07 16:19:37 +00:00
if err != nil {
2021-11-04 10:12:57 +00:00
return nil , fmt . Errorf ( "Move: %w" , err )
2017-08-07 16:19:37 +00:00
}
2023-05-09 14:29:55 +00:00
srcPath , dstPath := srcObj . path ( ) , path . Join ( f . absRoot , remote )
if _ , ok := c . sftpClient . HasExtension ( "posix-rename@openssh.com" ) ; ok {
err = c . sftpClient . PosixRename ( srcPath , dstPath )
} else {
// If haven't got PosixRename then remove source first before renaming
err = c . sftpClient . Remove ( dstPath )
if err != nil && ! errors . Is ( err , iofs . ErrNotExist ) {
fs . Errorf ( f , "Move: Failed to remove existing file %q: %v" , dstPath , err )
}
err = c . sftpClient . Rename ( srcPath , dstPath )
}
2017-08-07 16:19:37 +00:00
f . putSftpConnection ( & c , err )
2017-01-31 20:34:11 +00:00
if err != nil {
2021-11-04 10:12:57 +00:00
return nil , fmt . Errorf ( "Move Rename failed: %w" , err )
2016-11-12 23:36:08 +00:00
}
2019-06-17 08:34:30 +00:00
dstObj , err := f . NewObject ( ctx , remote )
2017-01-31 20:34:11 +00:00
if err != nil {
2021-11-04 10:12:57 +00:00
return nil , fmt . Errorf ( "Move NewObject failed: %w" , err )
2017-01-31 20:34:11 +00:00
}
return dstObj , nil
}
2017-02-05 21:20:56 +00:00
// DirMove moves src, srcRemote to this remote at dstRemote
2020-10-13 21:43:40 +00:00
// using server-side move operations.
2017-01-31 20:34:11 +00:00
//
// Will only be called if src.Fs().Name() == f.Name()
//
// If it isn't possible then return fs.ErrorCantDirMove
//
// If destination exists then return fs.ErrorDirExists
2019-06-17 08:34:30 +00:00
func ( f * Fs ) DirMove ( ctx context . Context , src fs . Fs , srcRemote , dstRemote string ) error {
2017-01-31 20:34:11 +00:00
srcFs , ok := src . ( * Fs )
if ! ok {
2017-02-09 11:01:20 +00:00
fs . Debugf ( srcFs , "Can't move directory - not same remote type" )
2017-01-31 20:34:11 +00:00
return fs . ErrorCantDirMove
}
2020-06-29 14:49:19 +00:00
srcPath := path . Join ( srcFs . absRoot , srcRemote )
dstPath := path . Join ( f . absRoot , dstRemote )
2017-01-31 20:34:11 +00:00
// Check if destination exists
2020-11-05 11:33:32 +00:00
ok , err := f . dirExists ( ctx , dstPath )
2017-01-31 20:34:11 +00:00
if err != nil {
2021-11-04 10:12:57 +00:00
return fmt . Errorf ( "DirMove dirExists dst failed: %w" , err )
2017-01-31 20:34:11 +00:00
}
if ok {
return fs . ErrorDirExists
}
// Make sure the parent directory exists
2020-11-05 11:33:32 +00:00
err = f . mkdir ( ctx , path . Dir ( dstPath ) )
2017-01-31 20:34:11 +00:00
if err != nil {
2021-11-04 10:12:57 +00:00
return fmt . Errorf ( "DirMove mkParentDir dst failed: %w" , err )
2017-01-31 20:34:11 +00:00
}
// Do the move
2020-11-05 11:33:32 +00:00
c , err := f . getSftpConnection ( ctx )
2017-08-07 16:19:37 +00:00
if err != nil {
2021-11-04 10:12:57 +00:00
return fmt . Errorf ( "DirMove: %w" , err )
2017-08-07 16:19:37 +00:00
}
err = c . sftpClient . Rename (
2017-02-05 21:20:56 +00:00
srcPath ,
dstPath ,
2017-01-31 20:34:11 +00:00
)
2017-08-07 16:19:37 +00:00
f . putSftpConnection ( & c , err )
2017-01-31 20:34:11 +00:00
if err != nil {
2021-11-04 10:12:57 +00:00
return fmt . Errorf ( "DirMove Rename(%q,%q) failed: %w" , srcPath , dstPath , err )
2017-01-31 20:34:11 +00:00
}
return nil
2016-11-12 23:36:08 +00:00
}
2019-06-26 15:50:31 +00:00
// run runds cmd on the remote end returning standard output
2020-11-05 11:33:32 +00:00
func ( f * Fs ) run ( ctx context . Context , cmd string ) ( [ ] byte , error ) {
2021-10-06 10:50:35 +00:00
f . addSession ( ) // Show session in use
defer f . removeSession ( )
2020-11-05 11:33:32 +00:00
c , err := f . getSftpConnection ( ctx )
2019-06-26 15:50:31 +00:00
if err != nil {
2021-11-04 10:12:57 +00:00
return nil , fmt . Errorf ( "run: get SFTP connection: %w" , err )
2017-08-06 10:49:52 +00:00
}
2019-06-26 15:50:31 +00:00
defer f . putSftpConnection ( & c , err )
2017-08-06 10:49:52 +00:00
2021-11-15 12:24:31 +00:00
// Send keepalives while the connection is open
defer close ( c . sendKeepAlives ( keepAliveInterval ) )
2019-06-26 15:50:31 +00:00
session , err := c . sshClient . NewSession ( )
if err != nil {
2021-11-04 10:12:57 +00:00
return nil , fmt . Errorf ( "run: get SFTP session: %w" , err )
2018-01-05 09:01:35 +00:00
}
2022-04-11 09:41:17 +00:00
err = f . setEnv ( session )
if err != nil {
return nil , err
}
2019-06-26 15:50:31 +00:00
defer func ( ) {
_ = session . Close ( )
} ( )
2018-01-05 09:01:35 +00:00
2019-06-26 15:50:31 +00:00
var stdout , stderr bytes . Buffer
session . Stdout = & stdout
session . Stderr = & stderr
2021-11-15 12:21:43 +00:00
fs . Debugf ( f , "Running remote command: %s" , cmd )
2019-06-26 15:50:31 +00:00
err = session . Run ( cmd )
2017-08-07 16:19:37 +00:00
if err != nil {
2021-11-15 12:21:43 +00:00
return nil , fmt . Errorf ( "failed to run %q: %s: %w" , cmd , bytes . TrimSpace ( stderr . Bytes ( ) ) , err )
2019-06-26 15:50:31 +00:00
}
2021-11-15 12:21:43 +00:00
fs . Debugf ( f , "Remote command result: %s" , bytes . TrimSpace ( stdout . Bytes ( ) ) )
2019-06-26 15:50:31 +00:00
return stdout . Bytes ( ) , nil
}
// Hashes returns the supported hash types of the filesystem
func ( f * Fs ) Hashes ( ) hash . Set {
2020-11-05 11:33:32 +00:00
ctx := context . TODO ( )
2019-06-26 15:50:31 +00:00
if f . cachedHashes != nil {
return * f . cachedHashes
}
2017-08-06 10:49:52 +00:00
2021-10-27 15:01:54 +00:00
hashSet := hash . NewHashSet ( )
f . cachedHashes = & hashSet
if f . opt . DisableHashCheck || f . shellType == shellTypeNotSupported {
return hashSet
}
2019-06-26 12:33:36 +00:00
// look for a hash command which works
2021-10-27 15:01:54 +00:00
checkHash := func ( hashType hash . Type , commands [ ] struct { hashFile , hashEmpty string } , expected string , hashCommand * string , changed * bool ) bool {
2019-06-26 15:50:31 +00:00
if * hashCommand == hashCommandNotSupported {
return false
}
if * hashCommand != "" {
return true
}
2021-10-27 15:01:54 +00:00
fs . Debugf ( f , "Checking default %v hash commands" , hashType )
2019-06-26 15:50:31 +00:00
* changed = true
2019-06-26 12:33:36 +00:00
for _ , command := range commands {
2021-10-27 15:01:54 +00:00
output , err := f . run ( ctx , command . hashEmpty )
2019-06-26 12:33:36 +00:00
if err != nil {
2021-10-27 15:01:54 +00:00
fs . Debugf ( f , "Hash command skipped: %v" , err )
2019-06-26 12:33:36 +00:00
continue
}
output = bytes . TrimSpace ( output )
2019-06-26 15:50:31 +00:00
if parseHash ( output ) == expected {
2021-10-27 15:01:54 +00:00
* hashCommand = command . hashFile
fs . Debugf ( f , "Hash command accepted" )
2019-06-26 12:33:36 +00:00
return true
}
2021-10-27 15:01:54 +00:00
fs . Debugf ( f , "Hash command skipped: Wrong output" )
2019-06-26 12:33:36 +00:00
}
2019-06-26 15:50:31 +00:00
* hashCommand = hashCommandNotSupported
2019-06-26 12:33:36 +00:00
return false
2017-08-06 10:49:52 +00:00
}
2019-06-26 15:50:31 +00:00
changed := false
2021-10-27 15:01:54 +00:00
md5Commands := [ ] struct {
hashFile , hashEmpty string
} {
{ "md5sum" , "md5sum" } ,
{ "md5 -r" , "md5 -r" } ,
{ "rclone md5sum" , "rclone md5sum" } ,
}
sha1Commands := [ ] struct {
hashFile , hashEmpty string
} {
{ "sha1sum" , "sha1sum" } ,
{ "sha1 -r" , "sha1 -r" } ,
{ "rclone sha1sum" , "rclone sha1sum" } ,
}
if f . shellType == "powershell" {
md5Commands = append ( md5Commands , struct {
hashFile , hashEmpty string
} {
"&{param($Path);Get-FileHash -Algorithm MD5 -LiteralPath $Path -ErrorAction Stop|Select-Object -First 1 -ExpandProperty Hash|ForEach-Object{\"$($_.ToLower()) ${Path}\"}}" ,
"Get-FileHash -Algorithm MD5 -InputStream ([System.IO.MemoryStream]::new()) -ErrorAction Stop|Select-Object -First 1 -ExpandProperty Hash|ForEach-Object{$_.ToLower()}" ,
} )
sha1Commands = append ( sha1Commands , struct {
hashFile , hashEmpty string
} {
"&{param($Path);Get-FileHash -Algorithm SHA1 -LiteralPath $Path -ErrorAction Stop|Select-Object -First 1 -ExpandProperty Hash|ForEach-Object{\"$($_.ToLower()) ${Path}\"}}" ,
"Get-FileHash -Algorithm SHA1 -InputStream ([System.IO.MemoryStream]::new()) -ErrorAction Stop|Select-Object -First 1 -ExpandProperty Hash|ForEach-Object{$_.ToLower()}" ,
} )
}
md5Works := checkHash ( hash . MD5 , md5Commands , "d41d8cd98f00b204e9800998ecf8427e" , & f . opt . Md5sumCommand , & changed )
sha1Works := checkHash ( hash . SHA1 , sha1Commands , "da39a3ee5e6b4b0d3255bfef95601890afd80709" , & f . opt . Sha1sumCommand , & changed )
2019-06-26 15:50:31 +00:00
if changed {
2021-10-27 15:01:54 +00:00
// Save permanently in config to avoid the extra work next time
fs . Debugf ( f , "Setting hash command for %v to %q (set sha1sum_command to override)" , hash . MD5 , f . opt . Md5sumCommand )
2019-06-26 15:50:31 +00:00
f . m . Set ( "md5sum_command" , f . opt . Md5sumCommand )
2021-10-27 15:01:54 +00:00
fs . Debugf ( f , "Setting hash command for %v to %q (set md5sum_command to override)" , hash . SHA1 , f . opt . Sha1sumCommand )
2019-06-26 15:50:31 +00:00
f . m . Set ( "sha1sum_command" , f . opt . Sha1sumCommand )
}
2017-08-06 10:49:52 +00:00
if sha1Works {
2021-10-27 15:01:54 +00:00
hashSet . Add ( hash . SHA1 )
2017-08-06 10:49:52 +00:00
}
if md5Works {
2021-10-27 15:01:54 +00:00
hashSet . Add ( hash . MD5 )
2017-08-06 10:49:52 +00:00
}
2021-10-27 15:01:54 +00:00
return hashSet
2016-11-12 23:36:08 +00:00
}
2019-04-25 09:51:15 +00:00
// About gets usage stats
2019-06-26 10:24:48 +00:00
func ( f * Fs ) About ( ctx context . Context ) ( * fs . Usage , error ) {
2021-10-31 13:04:36 +00:00
// If server implements the vendor-specific VFS statistics extension prefer that
// (OpenSSH implements it on using syscall.Statfs on Linux and API function GetDiskFreeSpace on Windows)
c , err := f . getSftpConnection ( ctx )
if err != nil {
return nil , err
}
var vfsStats * sftp . StatVFS
if _ , found := c . sftpClient . HasExtension ( "statvfs@openssh.com" ) ; found {
fs . Debugf ( f , "Server has VFS statistics extension" )
aboutPath := f . absRoot
if aboutPath == "" {
aboutPath = "/"
}
fs . Debugf ( f , "About path %q" , aboutPath )
vfsStats , err = c . sftpClient . StatVFS ( aboutPath )
}
f . putSftpConnection ( & c , err ) // Return to pool asap, if running shell command below it will be re-used
if vfsStats != nil {
total := vfsStats . TotalSpace ( )
free := vfsStats . FreeSpace ( )
used := total - free
return & fs . Usage {
Total : fs . NewUsageValue ( int64 ( total ) ) ,
Used : fs . NewUsageValue ( int64 ( used ) ) ,
Free : fs . NewUsageValue ( int64 ( free ) ) ,
} , nil
} else if err != nil {
if errors . Is ( err , os . ErrNotExist ) {
return nil , err
}
fs . Debugf ( f , "Failed to retrieve VFS statistics, trying shell command instead: %v" , err )
} else {
fs . Debugf ( f , "Server does not have the VFS statistics extension, trying shell command instead" )
}
// Fall back to shell command method if possible
2021-10-27 15:01:54 +00:00
if f . shellType == shellTypeNotSupported || f . shellType == "cmd" {
fs . Debugf ( f , "About shell command is not available for shell type %q (set option shell_type to override)" , f . shellType )
return nil , fmt . Errorf ( "not supported with shell type %q" , f . shellType )
2019-04-25 09:51:15 +00:00
}
2021-10-27 15:01:54 +00:00
aboutShellPath := f . remoteShellPath ( "" )
if aboutShellPath == "" {
aboutShellPath = "/"
}
fs . Debugf ( f , "About path %q" , aboutShellPath )
aboutShellPathArg , err := f . quoteOrEscapeShellPath ( aboutShellPath )
if err != nil {
return nil , err
}
// PowerShell
if f . shellType == "powershell" {
shellCmd := "Get-Item " + aboutShellPathArg + " -ErrorAction Stop|Select-Object -First 1 -ExpandProperty PSDrive|ForEach-Object{\"$($_.Used) $($_.Free)\"}"
fs . Debugf ( f , "About using shell command for shell type %q" , f . shellType )
stdout , err := f . run ( ctx , shellCmd )
if err != nil {
fs . Debugf ( f , "About shell command for shell type %q failed (set option shell_type to override): %v" , f . shellType , err )
return nil , fmt . Errorf ( "powershell command failed: %w" , err )
}
split := strings . Fields ( string ( stdout ) )
usage := & fs . Usage { }
if len ( split ) == 2 {
usedValue , usedErr := strconv . ParseInt ( split [ 0 ] , 10 , 64 )
if usedErr == nil {
usage . Used = fs . NewUsageValue ( usedValue )
}
freeValue , freeErr := strconv . ParseInt ( split [ 1 ] , 10 , 64 )
if freeErr == nil {
usage . Free = fs . NewUsageValue ( freeValue )
if usedErr == nil {
usage . Total = fs . NewUsageValue ( usedValue + freeValue )
}
}
}
return usage , nil
2019-04-25 09:51:15 +00:00
}
2021-10-27 15:01:54 +00:00
// Unix/default shell
shellCmd := "df -k " + aboutShellPathArg
fs . Debugf ( f , "About using shell command for shell type %q" , f . shellType )
stdout , err := f . run ( ctx , shellCmd )
2019-04-25 09:51:15 +00:00
if err != nil {
2021-10-27 15:01:54 +00:00
fs . Debugf ( f , "About shell command for shell type %q failed (set option shell_type to override): %v" , f . shellType , err )
2022-01-14 21:18:32 +00:00
return nil , fmt . Errorf ( "your remote may not have the required df utility: %w" , err )
2019-04-25 09:51:15 +00:00
}
2019-06-26 15:50:31 +00:00
usageTotal , usageUsed , usageAvail := parseUsage ( stdout )
2019-05-09 13:29:52 +00:00
usage := & fs . Usage { }
if usageTotal >= 0 {
usage . Total = fs . NewUsageValue ( usageTotal )
2019-04-25 09:51:15 +00:00
}
2019-05-09 13:29:52 +00:00
if usageUsed >= 0 {
usage . Used = fs . NewUsageValue ( usageUsed )
}
if usageAvail >= 0 {
usage . Free = fs . NewUsageValue ( usageAvail )
2019-04-25 09:51:15 +00:00
}
return usage , nil
}
2020-11-27 17:25:57 +00:00
// Shutdown the backend, closing any background tasks and any
// cached connections.
func ( f * Fs ) Shutdown ( ctx context . Context ) error {
return f . drainPool ( ctx )
}
2016-11-12 23:36:08 +00:00
// Fs is the filesystem this remote sftp file object is located within
func ( o * Object ) Fs ( ) fs . Info {
return o . fs
}
// String returns the URL to the remote SFTP file
func ( o * Object ) String ( ) string {
if o == nil {
return "<nil>"
}
2017-01-31 20:34:11 +00:00
return o . remote
2016-11-12 23:36:08 +00:00
}
// Remote the name of the remote SFTP file, relative to the fs root
func ( o * Object ) Remote ( ) string {
return o . remote
}
2017-08-07 13:50:31 +00:00
// Hash returns the selected checksum of the file
// If no checksum is available it returns ""
2019-06-17 08:34:30 +00:00
func ( o * Object ) Hash ( ctx context . Context , r hash . Type ) ( string , error ) {
2019-06-26 12:33:36 +00:00
if o . fs . opt . DisableHashCheck {
return "" , nil
}
2019-06-26 15:50:31 +00:00
_ = o . fs . Hashes ( )
2019-06-26 12:33:36 +00:00
2018-04-19 08:45:46 +00:00
var hashCmd string
if r == hash . MD5 {
if o . md5sum != nil {
return * o . md5sum , nil
}
2019-06-26 15:50:31 +00:00
hashCmd = o . fs . opt . Md5sumCommand
2018-04-19 08:45:46 +00:00
} else if r == hash . SHA1 {
if o . sha1sum != nil {
return * o . sha1sum , nil
}
2019-06-26 15:50:31 +00:00
hashCmd = o . fs . opt . Sha1sumCommand
2018-04-19 08:45:46 +00:00
} else {
return "" , hash . ErrUnsupported
2017-08-06 10:49:52 +00:00
}
2019-06-26 15:50:31 +00:00
if hashCmd == "" || hashCmd == hashCommandNotSupported {
2019-06-26 12:33:36 +00:00
return "" , hash . ErrUnsupported
2018-10-22 10:01:41 +00:00
}
2021-10-27 15:01:54 +00:00
shellPathArg , err := o . fs . quoteOrEscapeShellPath ( o . shellPath ( ) )
if err != nil {
return "" , fmt . Errorf ( "failed to calculate %v hash: %w" , r , err )
2018-04-30 16:05:10 +00:00
}
2021-10-27 15:01:54 +00:00
outBytes , err := o . fs . run ( ctx , hashCmd + " " + shellPathArg )
2017-08-06 10:49:52 +00:00
if err != nil {
2021-11-15 12:21:43 +00:00
return "" , fmt . Errorf ( "failed to calculate %v hash: %w" , r , err )
2017-08-06 10:49:52 +00:00
}
2021-10-27 15:01:54 +00:00
hashString := parseHash ( outBytes )
fs . Debugf ( o , "Parsed hash: %s" , hashString )
2018-01-18 20:27:52 +00:00
if r == hash . MD5 {
2021-10-27 15:01:54 +00:00
o . md5sum = & hashString
2018-01-18 20:27:52 +00:00
} else if r == hash . SHA1 {
2021-10-27 15:01:54 +00:00
o . sha1sum = & hashString
2017-08-06 10:49:52 +00:00
}
2021-10-27 15:01:54 +00:00
return hashString , nil
}
// quoteOrEscapeShellPath makes path a valid string argument in configured shell
// and also ensures it cannot cause unintended behavior.
func quoteOrEscapeShellPath ( shellType string , shellPath string ) ( string , error ) {
// PowerShell
if shellType == "powershell" {
return "'" + strings . ReplaceAll ( shellPath , "'" , "''" ) + "'" , nil
}
// Windows Command Prompt
if shellType == "cmd" {
if strings . Contains ( shellPath , "\"" ) {
return "" , fmt . Errorf ( "path is not valid in shell type %s: %s" , shellType , shellPath )
}
return "\"" + shellPath + "\"" , nil
}
// Unix shell
safe := unixShellEscapeRegex . ReplaceAllString ( shellPath , ` \$0 ` )
return strings . ReplaceAll ( safe , "\n" , "'\n'" ) , nil
}
// quoteOrEscapeShellPath makes path a valid string argument in configured shell
func ( f * Fs ) quoteOrEscapeShellPath ( shellPath string ) ( string , error ) {
return quoteOrEscapeShellPath ( f . shellType , shellPath )
2017-08-06 10:49:52 +00:00
}
2021-10-27 15:01:54 +00:00
// remotePath returns the native SFTP path of the file or directory at the remote given
func ( f * Fs ) remotePath ( remote string ) string {
return path . Join ( f . absRoot , remote )
}
2017-08-06 10:49:52 +00:00
2021-10-27 15:01:54 +00:00
// remoteShellPath returns the SSH shell path of the file or directory at the remote given
func ( f * Fs ) remoteShellPath ( remote string ) string {
if f . opt . PathOverride != "" {
shellPath := path . Join ( f . opt . PathOverride , remote )
fs . Debugf ( f , "Shell path redirected to %q with option path_override" , shellPath )
return shellPath
}
shellPath := path . Join ( f . absRoot , remote )
if f . shellType == "powershell" || f . shellType == "cmd" {
// If remote shell is powershell or cmd, then server is probably Windows.
// The sftp package converts everything to POSIX paths: Forward slashes, and
// absolute paths starts with a slash. An absolute path on a Windows server will
// then look like this "/C:/Windows/System32". We must remove the "/" prefix
// to make this a valid path for shell commands. In case of PowerShell there is a
// possibility that it is a Unix server, with PowerShell Core shell, but assuming
// root folders with names such as "C:" are rare, we just take this risk,
// and option path_override can always be used to work around corner cases.
if posixWinAbsPathRegex . MatchString ( shellPath ) {
shellPath = strings . TrimPrefix ( shellPath , "/" )
fs . Debugf ( f , "Shell path adjusted to %q (set option path_override to override)" , shellPath )
return shellPath
}
}
fs . Debugf ( f , "Shell path %q" , shellPath )
return shellPath
2017-08-06 10:49:52 +00:00
}
// Converts a byte array from the SSH session returned by
// an invocation of md5sum/sha1sum to a hash string
// as expected by the rest of this application
func parseHash ( bytes [ ] byte ) string {
2019-07-26 11:19:47 +00:00
// For strings with backslash *sum writes a leading \
// https://unix.stackexchange.com/q/313733/94054
2020-09-22 01:15:09 +00:00
return strings . ToLower ( strings . Split ( strings . TrimLeft ( string ( bytes ) , "\\" ) , " " ) [ 0 ] ) // Split at hash / filename separator / all convert to lowercase
2016-11-12 23:36:08 +00:00
}
2019-04-25 09:51:15 +00:00
// Parses the byte array output from the SSH session
// returned by an invocation of df into
2020-05-20 10:39:20 +00:00
// the disk size, used space, and available space on the disk, in that order.
2019-04-25 09:51:15 +00:00
// Only works when `df` has output info on only one disk
2019-05-09 13:29:52 +00:00
func parseUsage ( bytes [ ] byte ) ( spaceTotal int64 , spaceUsed int64 , spaceAvail int64 ) {
spaceTotal , spaceUsed , spaceAvail = - 1 , - 1 , - 1
2019-04-25 09:51:15 +00:00
lines := strings . Split ( string ( bytes ) , "\n" )
if len ( lines ) < 2 {
2019-05-09 13:29:52 +00:00
return
2019-04-25 09:51:15 +00:00
}
split := strings . Fields ( lines [ 1 ] )
if len ( split ) < 6 {
2019-05-09 13:29:52 +00:00
return
2019-04-25 09:51:15 +00:00
}
spaceTotal , err := strconv . ParseInt ( split [ 1 ] , 10 , 64 )
if err != nil {
2019-05-09 13:29:52 +00:00
spaceTotal = - 1
2019-04-25 09:51:15 +00:00
}
2019-05-09 13:29:52 +00:00
spaceUsed , err = strconv . ParseInt ( split [ 2 ] , 10 , 64 )
2019-04-25 09:51:15 +00:00
if err != nil {
2019-05-09 13:29:52 +00:00
spaceUsed = - 1
2019-04-25 09:51:15 +00:00
}
2019-05-09 13:29:52 +00:00
spaceAvail , err = strconv . ParseInt ( split [ 3 ] , 10 , 64 )
2019-04-25 09:51:15 +00:00
if err != nil {
2019-05-09 13:29:52 +00:00
spaceAvail = - 1
2019-04-25 09:51:15 +00:00
}
return spaceTotal * 1024 , spaceUsed * 1024 , spaceAvail * 1024
}
2016-11-12 23:36:08 +00:00
// Size returns the size in bytes of the remote sftp file
func ( o * Object ) Size ( ) int64 {
2017-06-30 09:24:06 +00:00
return o . size
2016-11-12 23:36:08 +00:00
}
// ModTime returns the modification time of the remote sftp file
2019-06-17 08:34:30 +00:00
func ( o * Object ) ModTime ( ctx context . Context ) time . Time {
2017-06-30 09:24:06 +00:00
return o . modTime
2016-11-12 23:36:08 +00:00
}
2021-10-27 15:01:54 +00:00
// path returns the native SFTP path of the object
2017-01-31 20:34:11 +00:00
func ( o * Object ) path ( ) string {
2021-10-27 15:01:54 +00:00
return o . fs . remotePath ( o . remote )
}
// shellPath returns the SSH shell path of the object
func ( o * Object ) shellPath ( ) string {
return o . fs . remoteShellPath ( o . remote )
2017-01-31 20:34:11 +00:00
}
2017-06-30 09:24:06 +00:00
// setMetadata updates the info in the object from the stat result passed in
func ( o * Object ) setMetadata ( info os . FileInfo ) {
o . modTime = info . ModTime ( )
o . size = info . Size ( )
o . mode = info . Mode ( )
}
2018-03-16 15:36:47 +00:00
// statRemote stats the file or directory at the remote given
2020-11-05 11:33:32 +00:00
func ( f * Fs ) stat ( ctx context . Context , remote string ) ( info os . FileInfo , err error ) {
2022-12-14 14:37:07 +00:00
absPath := remote
if ! strings . HasPrefix ( remote , "/" ) {
absPath = path . Join ( f . absRoot , remote )
}
2020-11-05 11:33:32 +00:00
c , err := f . getSftpConnection ( ctx )
2017-08-07 16:19:37 +00:00
if err != nil {
2021-11-04 10:12:57 +00:00
return nil , fmt . Errorf ( "stat: %w" , err )
2017-08-07 16:19:37 +00:00
}
2018-03-16 15:36:47 +00:00
info , err = c . sftpClient . Stat ( absPath )
f . putSftpConnection ( & c , err )
return info , err
}
// stat updates the info in the Object
2020-11-05 11:33:32 +00:00
func ( o * Object ) stat ( ctx context . Context ) error {
info , err := o . fs . stat ( ctx , o . remote )
2017-01-31 20:34:11 +00:00
if err != nil {
if os . IsNotExist ( err ) {
return fs . ErrorObjectNotFound
}
2021-11-04 10:12:57 +00:00
return fmt . Errorf ( "stat failed: %w" , err )
2017-01-31 20:34:11 +00:00
}
if info . IsDir ( ) {
2021-09-06 12:54:08 +00:00
return fs . ErrorIsDir
2017-01-31 20:34:11 +00:00
}
2017-06-30 09:24:06 +00:00
o . setMetadata ( info )
2017-01-31 20:34:11 +00:00
return nil
}
2016-11-12 23:36:08 +00:00
// SetModTime sets the modification and access time to the specified time
2017-01-31 20:34:11 +00:00
//
// it also updates the info field
2019-06-17 08:34:30 +00:00
func ( o * Object ) SetModTime ( ctx context . Context , modTime time . Time ) error {
2021-03-18 08:00:08 +00:00
if ! o . fs . opt . SetModTime {
return nil
2016-11-12 23:36:08 +00:00
}
2021-03-18 08:00:08 +00:00
c , err := o . fs . getSftpConnection ( ctx )
if err != nil {
2021-11-04 10:12:57 +00:00
return fmt . Errorf ( "SetModTime: %w" , err )
2021-03-18 08:00:08 +00:00
}
err = c . sftpClient . Chtimes ( o . path ( ) , modTime , modTime )
o . fs . putSftpConnection ( & c , err )
if err != nil {
2021-11-04 10:12:57 +00:00
return fmt . Errorf ( "SetModTime failed: %w" , err )
2021-03-18 08:00:08 +00:00
}
err = o . stat ( ctx )
2017-01-31 20:34:11 +00:00
if err != nil {
2021-11-04 10:12:57 +00:00
return fmt . Errorf ( "SetModTime stat failed: %w" , err )
2017-01-31 20:34:11 +00:00
}
return nil
2016-11-12 23:36:08 +00:00
}
2020-10-13 22:07:12 +00:00
// Storable returns whether the remote sftp file is a regular file (not a directory, symbolic link, block device, character device, named pipe, etc.)
2016-11-12 23:36:08 +00:00
func ( o * Object ) Storable ( ) bool {
2017-06-30 09:24:06 +00:00
return o . mode . IsRegular ( )
2016-11-12 23:36:08 +00:00
}
2018-05-24 14:03:57 +00:00
// objectReader represents a file open for reading on the SFTP server
type objectReader struct {
2021-04-05 13:18:49 +00:00
f * Fs
2018-05-24 14:03:57 +00:00
sftpFile * sftp . File
pipeReader * io . PipeReader
done chan struct { }
}
2021-04-05 13:18:49 +00:00
func ( f * Fs ) newObjectReader ( sftpFile * sftp . File ) * objectReader {
2018-05-24 14:03:57 +00:00
pipeReader , pipeWriter := io . Pipe ( )
file := & objectReader {
2021-04-05 13:18:49 +00:00
f : f ,
2018-05-24 14:03:57 +00:00
sftpFile : sftpFile ,
pipeReader : pipeReader ,
done : make ( chan struct { } ) ,
}
2021-04-05 13:18:49 +00:00
// Show connection in use
2021-10-06 10:50:35 +00:00
f . addSession ( )
2018-05-24 14:03:57 +00:00
go func ( ) {
// Use sftpFile.WriteTo to pump data so that it gets a
// chance to build the window up.
_ , err := sftpFile . WriteTo ( pipeWriter )
// Close the pipeWriter so the pipeReader fails with
// the same error or EOF if err == nil
_ = pipeWriter . CloseWithError ( err )
// signal that we've finished
close ( file . done )
} ( )
return file
}
2016-11-12 23:36:08 +00:00
// Read from a remote sftp file object reader
2018-05-24 14:03:57 +00:00
func ( file * objectReader ) Read ( p [ ] byte ) ( n int , err error ) {
n , err = file . pipeReader . Read ( p )
2016-11-12 23:36:08 +00:00
return n , err
}
// Close a reader of a remote sftp file
2018-05-24 14:03:57 +00:00
func ( file * objectReader ) Close ( ) ( err error ) {
// Close the sftpFile - this will likely cause the WriteTo to error
2016-11-12 23:36:08 +00:00
err = file . sftpFile . Close ( )
2018-05-24 14:03:57 +00:00
// Close the pipeReader so writes to the pipeWriter fail
_ = file . pipeReader . Close ( )
// Wait for the background process to finish
<- file . done
2021-04-05 13:18:49 +00:00
// Show connection no longer in use
2021-10-06 10:50:35 +00:00
file . f . removeSession ( )
2016-11-12 23:36:08 +00:00
return err
}
// Open a remote sftp file object for reading. Seek is supported
2019-06-17 08:34:30 +00:00
func ( o * Object ) Open ( ctx context . Context , options ... fs . OpenOption ) ( in io . ReadCloser , err error ) {
2018-01-27 10:07:17 +00:00
var offset , limit int64 = 0 , - 1
2016-11-12 23:36:08 +00:00
for _ , option := range options {
switch x := option . ( type ) {
case * fs . SeekOption :
2018-01-27 10:07:17 +00:00
offset = x . Offset
2018-01-23 20:21:19 +00:00
case * fs . RangeOption :
offset , limit = x . Decode ( o . Size ( ) )
2016-11-12 23:36:08 +00:00
default :
if option . Mandatory ( ) {
2017-02-09 11:01:20 +00:00
fs . Logf ( o , "Unsupported mandatory option: %v" , option )
2016-11-12 23:36:08 +00:00
}
}
}
2020-11-05 11:33:32 +00:00
c , err := o . fs . getSftpConnection ( ctx )
2017-08-07 16:19:37 +00:00
if err != nil {
2021-11-04 10:12:57 +00:00
return nil , fmt . Errorf ( "Open: %w" , err )
2017-08-07 16:19:37 +00:00
}
sftpFile , err := c . sftpClient . Open ( o . path ( ) )
o . fs . putSftpConnection ( & c , err )
2016-11-12 23:36:08 +00:00
if err != nil {
2021-11-04 10:12:57 +00:00
return nil , fmt . Errorf ( "Open failed: %w" , err )
2016-11-12 23:36:08 +00:00
}
if offset > 0 {
2018-04-06 18:53:06 +00:00
off , err := sftpFile . Seek ( offset , io . SeekStart )
2016-11-12 23:36:08 +00:00
if err != nil || off != offset {
2021-11-04 10:12:57 +00:00
return nil , fmt . Errorf ( "Open Seek failed: %w" , err )
2016-11-12 23:36:08 +00:00
}
}
2021-04-05 13:18:49 +00:00
in = readers . NewLimitedReadCloser ( o . fs . newObjectReader ( sftpFile ) , limit )
2016-11-12 23:36:08 +00:00
return in , nil
}
2021-04-05 09:32:20 +00:00
type sizeReader struct {
io . Reader
size int64
}
// Size returns the expected size of the stream
//
// It is used in sftpFile.ReadFrom as a hint to work out the
// concurrency needed
func ( sr * sizeReader ) Size ( ) int64 {
return sr . size
}
2016-11-12 23:36:08 +00:00
// Update a remote sftp file using the data <in> and ModTime from <src>
2019-06-17 08:34:30 +00:00
func ( o * Object ) Update ( ctx context . Context , in io . Reader , src fs . ObjectInfo , options ... fs . OpenOption ) error {
2021-10-06 10:50:35 +00:00
o . fs . addSession ( ) // Show session in use
defer o . fs . removeSession ( )
2017-08-07 16:36:59 +00:00
// Clear the hash cache since we are about to update the object
o . md5sum = nil
o . sha1sum = nil
2020-11-05 11:33:32 +00:00
c , err := o . fs . getSftpConnection ( ctx )
2017-08-07 16:19:37 +00:00
if err != nil {
2021-11-04 10:12:57 +00:00
return fmt . Errorf ( "Update: %w" , err )
2017-08-07 16:19:37 +00:00
}
2019-12-02 17:28:50 +00:00
file , err := c . sftpClient . OpenFile ( o . path ( ) , os . O_WRONLY | os . O_CREATE | os . O_TRUNC )
2017-08-07 16:19:37 +00:00
o . fs . putSftpConnection ( & c , err )
2017-01-31 20:34:11 +00:00
if err != nil {
2021-11-04 10:12:57 +00:00
return fmt . Errorf ( "Update Create failed: %w" , err )
2017-01-31 20:34:11 +00:00
}
// remove the file if upload failed
remove := func ( ) {
2020-11-05 11:33:32 +00:00
c , removeErr := o . fs . getSftpConnection ( ctx )
2017-08-07 16:19:37 +00:00
if removeErr != nil {
fs . Debugf ( src , "Failed to open new SSH connection for delete: %v" , removeErr )
return
}
removeErr = c . sftpClient . Remove ( o . path ( ) )
o . fs . putSftpConnection ( & c , removeErr )
2017-01-31 20:34:11 +00:00
if removeErr != nil {
2017-02-09 11:01:20 +00:00
fs . Debugf ( src , "Failed to remove: %v" , removeErr )
2017-01-31 20:34:11 +00:00
} else {
2017-02-09 11:01:20 +00:00
fs . Debugf ( src , "Removed after failed upload: %v" , err )
2016-11-12 23:36:08 +00:00
}
}
2021-04-05 09:32:20 +00:00
_ , err = file . ReadFrom ( & sizeReader { Reader : in , size : src . Size ( ) } )
2017-01-31 20:34:11 +00:00
if err != nil {
remove ( )
2021-11-04 10:12:57 +00:00
return fmt . Errorf ( "Update ReadFrom failed: %w" , err )
2017-01-31 20:34:11 +00:00
}
err = file . Close ( )
if err != nil {
remove ( )
2021-11-04 10:12:57 +00:00
return fmt . Errorf ( "Update Close failed: %w" , err )
2017-01-31 20:34:11 +00:00
}
2021-03-18 08:00:08 +00:00
// Set the mod time - this stats the object if o.fs.opt.SetModTime == true
2019-06-17 08:34:30 +00:00
err = o . SetModTime ( ctx , src . ModTime ( ctx ) )
2017-01-31 20:34:11 +00:00
if err != nil {
2021-11-04 10:12:57 +00:00
return fmt . Errorf ( "Update SetModTime failed: %w" , err )
2017-01-31 20:34:11 +00:00
}
2021-03-18 08:00:08 +00:00
// Stat the file after the upload to read its stats back if o.fs.opt.SetModTime == false
if ! o . fs . opt . SetModTime {
err = o . stat ( ctx )
if err == fs . ErrorObjectNotFound {
// In the specific case of o.fs.opt.SetModTime == false
// if the object wasn't found then don't return an error
fs . Debugf ( o , "Not found after upload with set_modtime=false so returning best guess" )
o . modTime = src . ModTime ( ctx )
o . size = src . Size ( )
o . mode = os . FileMode ( 0666 ) // regular file
} else if err != nil {
2021-11-04 10:12:57 +00:00
return fmt . Errorf ( "Update stat failed: %w" , err )
2021-03-18 08:00:08 +00:00
}
}
2017-01-31 20:34:11 +00:00
return nil
2016-11-12 23:36:08 +00:00
}
// Remove a remote sftp file object
2019-06-17 08:34:30 +00:00
func ( o * Object ) Remove ( ctx context . Context ) error {
2020-11-05 11:33:32 +00:00
c , err := o . fs . getSftpConnection ( ctx )
2017-08-07 16:19:37 +00:00
if err != nil {
2021-11-04 10:12:57 +00:00
return fmt . Errorf ( "Remove: %w" , err )
2017-08-07 16:19:37 +00:00
}
err = c . sftpClient . Remove ( o . path ( ) )
o . fs . putSftpConnection ( & c , err )
return err
2016-11-12 23:36:08 +00:00
}
// Check the interfaces are satisfied
var (
2017-08-03 19:42:35 +00:00
_ fs . Fs = & Fs { }
_ fs . PutStreamer = & Fs { }
_ fs . Mover = & Fs { }
_ fs . DirMover = & Fs { }
2019-06-26 10:24:48 +00:00
_ fs . Abouter = & Fs { }
2020-11-27 17:25:57 +00:00
_ fs . Shutdowner = & Fs { }
2017-08-03 19:42:35 +00:00
_ fs . Object = & Object { }
2016-11-12 23:36:08 +00:00
)