Basic SFTP support, Issue #521
This commit is contained in:
Jack Schmidt
Nick Craig-Wood
parent
27e18b6efa
commit
b9a8315696
2 changed files with 422 additions and 0 deletions
|
|
@ -12,6 +12,7 @@ import (
|
|||
_ "github.com/ncw/rclone/local"
|
||||
_ "github.com/ncw/rclone/onedrive"
|
||||
_ "github.com/ncw/rclone/s3"
|
||||
_ "github.com/ncw/rclone/sftp"
|
||||
_ "github.com/ncw/rclone/swift"
|
||||
_ "github.com/ncw/rclone/yandex"
|
||||
)
|
||||
|
|
|
|||
421
sftp/sftp.go
Normal file
421
sftp/sftp.go
Normal file
|
|
@ -0,0 +1,421 @@
|
|||
// Package sftp provides a filesystem interface using github.com/pkg/sftp
|
||||
package sftp
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"golang.org/x/crypto/ssh"
|
||||
"golang.org/x/crypto/ssh/agent"
|
||||
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/pkg/sftp"
|
||||
)
|
||||
|
||||
func init() {
|
||||
fsi := &fs.RegInfo{
|
||||
Name: "sftp",
|
||||
Description: "SSH/SFTP Connection",
|
||||
NewFs: NewFs,
|
||||
Options: []fs.Option{{
|
||||
Name: "host",
|
||||
Help: "SSH host to connect to",
|
||||
Optional: false,
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "example.com",
|
||||
Help: "Connect to example.com",
|
||||
}},
|
||||
}, {
|
||||
Name: "user",
|
||||
Help: "SSH username, leave blank for current username, " + os.Getenv("USER"),
|
||||
Optional: true,
|
||||
}, {
|
||||
Name: "port",
|
||||
Help: "SSH port",
|
||||
Optional: true,
|
||||
}, {
|
||||
Name: "pass",
|
||||
Help: "SSH password, leave blank to use ssh-agent",
|
||||
Optional: true,
|
||||
IsPassword: true,
|
||||
}},
|
||||
}
|
||||
fs.Register(fsi)
|
||||
}
|
||||
|
||||
type stringer interface {
|
||||
String() string
|
||||
}
|
||||
|
||||
func debug(o stringer, msg string) {
|
||||
fs.Debug(o, msg)
|
||||
}
|
||||
|
||||
// Fs stores the interface to the remote SFTP files
|
||||
type Fs struct {
|
||||
name string
|
||||
root string
|
||||
features *fs.Features // optional features
|
||||
url string
|
||||
sshClient *ssh.Client
|
||||
sftpClient *sftp.Client
|
||||
}
|
||||
|
||||
// Object is a remote SFTP file that has been stat'd (so it exists, but is not necessarily open for reading)
|
||||
type Object struct {
|
||||
fs *Fs
|
||||
remote string
|
||||
info os.FileInfo
|
||||
}
|
||||
|
||||
// ObjectReader holds the sftp.File interface to a remote SFTP file opened for reading
|
||||
type ObjectReader struct {
|
||||
object *Object
|
||||
sftpFile *sftp.File
|
||||
}
|
||||
|
||||
// NewFs creates a new Fs object from the name and root. It connects to
|
||||
// the host specified in the config file.
|
||||
func NewFs(name, root string) (fs.Fs, error) {
|
||||
user := fs.ConfigFileGet(name, "user")
|
||||
host := fs.ConfigFileGet(name, "host")
|
||||
port := fs.ConfigFileGet(name, "port")
|
||||
pass := fs.ConfigFileGet(name, "pass")
|
||||
if root == "" {
|
||||
root = "."
|
||||
}
|
||||
if user == "" {
|
||||
user = os.Getenv("USER")
|
||||
}
|
||||
if port == "" {
|
||||
port = "22"
|
||||
}
|
||||
config := &ssh.ClientConfig{
|
||||
User: user,
|
||||
Auth: []ssh.AuthMethod{},
|
||||
}
|
||||
if pass == "" {
|
||||
if sshAgent, err := net.Dial("unix", os.Getenv("SSH_AUTH_SOCK")); err == nil {
|
||||
sshAgentClient := agent.NewClient(sshAgent)
|
||||
signers, _ := sshAgentClient.Signers()
|
||||
for i, signer := range signers {
|
||||
if 2*i < len(signers) {
|
||||
signers[i] = signers[len(signers)-i-1]
|
||||
signers[len(signers)-i-1] = signer
|
||||
}
|
||||
}
|
||||
config.Auth = append(config.Auth, ssh.PublicKeys(signers...))
|
||||
}
|
||||
} else {
|
||||
clearpass, err := fs.Reveal(pass)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
config.Auth = append(config.Auth, ssh.Password(clearpass))
|
||||
}
|
||||
if sshClient, err := ssh.Dial("tcp", host+":"+port, config); err != nil {
|
||||
return nil, err
|
||||
} else if sftpClient, err := sftp.NewClient(sshClient); err != nil {
|
||||
_ = sshClient.Close()
|
||||
return nil, err
|
||||
} else {
|
||||
f := &Fs{
|
||||
name: name,
|
||||
root: root,
|
||||
sshClient: sshClient,
|
||||
sftpClient: sftpClient,
|
||||
url: "sftp://" + user + "@" + host + ":" + port + "/" + root,
|
||||
}
|
||||
f.features = (&fs.Features{}).Fill(f)
|
||||
return f, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Name returns the configured name of the file system
|
||||
func (f *Fs) Name() string {
|
||||
return f.name
|
||||
}
|
||||
|
||||
// Root returns the root for the filesystem
|
||||
func (f *Fs) Root() string {
|
||||
return f.root
|
||||
}
|
||||
|
||||
// String returns the URL for the filesystem
|
||||
func (f *Fs) String() string {
|
||||
return f.url
|
||||
}
|
||||
|
||||
// Features returns the optional features of this Fs
|
||||
func (f *Fs) Features() *fs.Features {
|
||||
return f.features
|
||||
}
|
||||
|
||||
// Precision is the remote sftp file system's modtime precision, which we have no way of knowing. We estimate at 1s
|
||||
func (f *Fs) Precision() time.Duration {
|
||||
return time.Second
|
||||
}
|
||||
|
||||
// NewObject creates a new remote sftp file object
|
||||
func (f *Fs) NewObject(remote string) (fs.Object, error) {
|
||||
debug(f, "New '"+remote+"'")
|
||||
info, err := f.sftpClient.Stat(f.sftpClient.Join(f.root, remote))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
object := &Object{
|
||||
fs: f,
|
||||
remote: remote,
|
||||
info: info,
|
||||
}
|
||||
return object, nil
|
||||
}
|
||||
|
||||
func (f *Fs) list(out fs.ListOpts, dirs string, name string, info os.FileInfo, level int, done *sync.WaitGroup) {
|
||||
debug(f, "list '"+f.sftpClient.Join(dirs, name)+"'")
|
||||
defer done.Done()
|
||||
if info.IsDir() {
|
||||
if out.IncludeDirectory(info.Name()) {
|
||||
dir := &fs.Dir{
|
||||
Name: info.Name(),
|
||||
When: info.ModTime(),
|
||||
Bytes: -1,
|
||||
Count: -1,
|
||||
}
|
||||
if level >= out.Level() {
|
||||
return
|
||||
}
|
||||
if infos, err := f.sftpClient.ReadDir(f.sftpClient.Join(f.root, dirs, name)); err == nil {
|
||||
dir.Count = int64(len(infos))
|
||||
out.AddDir(dir)
|
||||
done.Add(len(infos))
|
||||
for _, newInfo := range infos {
|
||||
go f.list(out, f.sftpClient.Join(dirs, name), newInfo.Name(), newInfo, level+1, done)
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
file := &Object{
|
||||
fs: f,
|
||||
remote: f.sftpClient.Join(dirs, info.Name()),
|
||||
info: info,
|
||||
}
|
||||
out.Add(file)
|
||||
}
|
||||
}
|
||||
|
||||
// List the files and directories starting at <dir>
|
||||
func (f *Fs) List(out fs.ListOpts, dir string) {
|
||||
debug(f, "List '"+dir+"'")
|
||||
if dir == "" {
|
||||
dir = "."
|
||||
}
|
||||
var done sync.WaitGroup
|
||||
if info, _ := f.sftpClient.Stat(f.sftpClient.Join(f.root, dir)); info != nil {
|
||||
done.Add(1)
|
||||
f.list(out, "", ".", info, 0, &done)
|
||||
}
|
||||
debug(f, "List--waiting")
|
||||
done.Wait()
|
||||
out.Finished()
|
||||
}
|
||||
|
||||
// Put data from <in> into a new remote sftp file object described by <src.Remote()> and <src.ModTime()>
|
||||
func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) {
|
||||
debug(f, "Put '"+src.Remote()+"'")
|
||||
_ = f.mkdir(f.sftpClient.Join(f.root, filepath.Dir(src.Remote())))
|
||||
file, err := f.sftpClient.Create(f.sftpClient.Join(f.root, src.Remote()))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_, err = file.ReadFrom(in)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
o, err := f.NewObject(src.Remote())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
err = o.SetModTime(src.ModTime())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return o, nil
|
||||
}
|
||||
|
||||
func (f *Fs) mkdir(path string) error {
|
||||
debug(f, "mkdir '"+path+"'")
|
||||
parent := filepath.Dir(path)
|
||||
if parent != "." && parent != "/" {
|
||||
_ = f.mkdir(parent)
|
||||
}
|
||||
return f.sftpClient.Mkdir(path)
|
||||
}
|
||||
|
||||
// Mkdir makes the root directory of the Fs object
|
||||
func (f *Fs) Mkdir(dir string) error {
|
||||
root := path.Join(f.root, dir)
|
||||
debug(f, "Mkdir '"+root+"'")
|
||||
o, _ := f.NewObject("")
|
||||
if o == nil {
|
||||
return f.mkdir(root)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Rmdir removes the root directory of the Fs object
|
||||
func (f *Fs) Rmdir(dir string) error {
|
||||
root := path.Join(f.root, dir)
|
||||
debug(f, "Rmdir '"+root+"'")
|
||||
return f.sftpClient.Remove(f.root)
|
||||
}
|
||||
|
||||
// Move renames a remote sftp file object
|
||||
func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
|
||||
debug(f, "Move '"+src.Remote()+"' to '"+remote+"'")
|
||||
err := f.sftpClient.Rename(
|
||||
f.sftpClient.Join(f.root, src.Remote()),
|
||||
f.sftpClient.Join(f.root, remote))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dstObj, err := f.NewObject(remote)
|
||||
return dstObj, err
|
||||
}
|
||||
|
||||
// Hashes returns fs.HashNone to indicate remote hashing is unavailable
|
||||
func (f *Fs) Hashes() fs.HashSet {
|
||||
return fs.HashSet(fs.HashNone)
|
||||
}
|
||||
|
||||
// Fs is the filesystem this remote sftp file object is located within
|
||||
func (o *Object) Fs() fs.Info {
|
||||
return o.fs
|
||||
}
|
||||
|
||||
// String returns the URL to the remote SFTP file
|
||||
func (o *Object) String() string {
|
||||
if o == nil {
|
||||
return "<nil>"
|
||||
}
|
||||
return o.fs.url + "/" + o.remote
|
||||
}
|
||||
|
||||
// Remote the name of the remote SFTP file, relative to the fs root
|
||||
func (o *Object) Remote() string {
|
||||
return o.remote
|
||||
}
|
||||
|
||||
// Hash returns "" since SFTP (in Go or OpenSSH) doesn't support remote calculation of hashes
|
||||
func (o *Object) Hash(r fs.HashType) (string, error) {
|
||||
debug(o.fs, "Hash '"+o.remote+"'")
|
||||
return "", fs.ErrHashUnsupported
|
||||
}
|
||||
|
||||
// Size returns the size in bytes of the remote sftp file
|
||||
func (o *Object) Size() int64 {
|
||||
debug(o.fs, "Size '"+o.remote+"'")
|
||||
return o.info.Size()
|
||||
}
|
||||
|
||||
// ModTime returns the modification time of the remote sftp file
|
||||
func (o *Object) ModTime() time.Time {
|
||||
debug(o.fs, "ModTime '"+o.remote+"'")
|
||||
return o.info.ModTime()
|
||||
}
|
||||
|
||||
// SetModTime sets the modification and access time to the specified time
|
||||
func (o *Object) SetModTime(modTime time.Time) error {
|
||||
debug(o.fs, "SetModTime '"+o.remote+"'")
|
||||
err := o.fs.sftpClient.Chtimes(o.fs.sftpClient.Join(o.fs.root, o.remote), modTime, modTime)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
o.info, err = o.fs.sftpClient.Stat(o.fs.sftpClient.Join(o.fs.root, o.remote))
|
||||
return err
|
||||
}
|
||||
|
||||
// Storable returns whether the remote sftp file is a regular file (not a directory, symbolic link, block device, character device, named pipe, etc)
|
||||
func (o *Object) Storable() bool {
|
||||
debug(o.fs, "Storable '"+o.remote+"'?")
|
||||
return o.info.Mode().IsRegular()
|
||||
}
|
||||
|
||||
// Read from a remote sftp file object reader
|
||||
func (file *ObjectReader) Read(p []byte) (n int, err error) {
|
||||
debug(file.object.fs, "Read '"+file.object.remote+"'")
|
||||
n, err = file.sftpFile.Read(p)
|
||||
return n, err
|
||||
}
|
||||
|
||||
// Close a reader of a remote sftp file
|
||||
func (file *ObjectReader) Close() (err error) {
|
||||
debug(file.object.fs, "Close '"+file.object.remote+"'")
|
||||
err = file.sftpFile.Close()
|
||||
return err
|
||||
}
|
||||
|
||||
// Open a remote sftp file object for reading. Seek is supported
|
||||
func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
|
||||
debug(o.fs, "Open '"+o.remote+"'")
|
||||
var offset int64
|
||||
offset = 0
|
||||
for _, option := range options {
|
||||
switch x := option.(type) {
|
||||
case *fs.SeekOption:
|
||||
offset = x.Offset
|
||||
default:
|
||||
if option.Mandatory() {
|
||||
fs.Log(o, "Unsupported mandatory option: %v", option)
|
||||
}
|
||||
}
|
||||
}
|
||||
sftpFile, err := o.fs.sftpClient.Open(o.fs.sftpClient.Join(o.fs.root, o.remote))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if offset > 0 {
|
||||
off, err := sftpFile.Seek(offset, 0)
|
||||
if err != nil || off != offset {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
in = &ObjectReader{
|
||||
object: o,
|
||||
sftpFile: sftpFile,
|
||||
}
|
||||
return in, nil
|
||||
}
|
||||
|
||||
// Update a remote sftp file using the data <in> and ModTime from <src>
|
||||
func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error {
|
||||
debug(o.fs, "Update '"+o.remote+"'")
|
||||
file, err := o.fs.sftpClient.Create(o.fs.sftpClient.Join(o.fs.root, o.remote))
|
||||
if err == nil {
|
||||
_, err = file.ReadFrom(in)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = o.SetModTime(src.ModTime())
|
||||
return err
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Remove a remote sftp file object
|
||||
func (o *Object) Remove() error {
|
||||
debug(o.fs, "Remove '"+o.remote+"'")
|
||||
return o.fs.sftpClient.Remove(o.fs.sftpClient.Join(o.fs.root, o.remote))
|
||||
}
|
||||
|
||||
// Check the interfaces are satisfied
|
||||
var (
|
||||
_ fs.Fs = &Fs{}
|
||||
_ fs.Mover = &Fs{}
|
||||
_ fs.Object = &Object{}
|
||||
)
|
||||
Loading…
Reference in a new issue