serve sftp: serve an rclone remote over SFTP

cmd/serve/serve.go

@@ -3,12 +3,12 @@ package serve
 import (
 	"errors"
 
-	"github.com/ncw/rclone/cmd/serve/dlna"
-
 	"github.com/ncw/rclone/cmd"
+	"github.com/ncw/rclone/cmd/serve/dlna"
 	"github.com/ncw/rclone/cmd/serve/ftp"
 	"github.com/ncw/rclone/cmd/serve/http"
 	"github.com/ncw/rclone/cmd/serve/restic"
+	"github.com/ncw/rclone/cmd/serve/sftp"
 	"github.com/ncw/rclone/cmd/serve/webdav"
 	"github.com/spf13/cobra"
 )
@@ -27,6 +27,9 @@ func init() {
 	if ftp.Command != nil {
 		Command.AddCommand(ftp.Command)
 	}
+	if sftp.Command != nil {
+		Command.AddCommand(sftp.Command)
+	}
 	cmd.Root.AddCommand(Command)
 }
 

cmd/serve/sftp/connection.go

@@ -0,0 +1,254 @@
+// +build !plan9
+
+package sftp
+
+import (
+	"fmt"
+	"io"
+	"net"
+	"regexp"
+	"strings"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/hash"
+	"github.com/ncw/rclone/vfs"
+	"github.com/pkg/errors"
+	"github.com/pkg/sftp"
+	"golang.org/x/crypto/ssh"
+)
+
+func describeConn(c interface {
+	RemoteAddr() net.Addr
+	LocalAddr() net.Addr
+}) string {
+	return fmt.Sprintf("serve sftp %s->%s", c.RemoteAddr(), c.LocalAddr())
+}
+
+// Return the exit status of the command
+type exitStatus struct {
+	RC uint32
+}
+
+// The incoming exec command
+type execCommand struct {
+	Command string
+}
+
+var shellUnEscapeRegex = regexp.MustCompile(`\\(.)`)
+
+// Unescape a string that was escaped by rclone
+func shellUnEscape(str string) string {
+	str = strings.Replace(str, "'\n'", "\n", -1)
+	str = shellUnEscapeRegex.ReplaceAllString(str, `$1`)
+	return str
+}
+
+// Info about the current connection
+type conn struct {
+	vfs      *vfs.VFS
+	f        fs.Fs
+	handlers sftp.Handlers
+	what     string
+}
+
+// execCommand implements an extrememly limited number of commands to
+// interoperate with the rclone sftp backend
+func (c *conn) execCommand(out io.Writer, command string) (err error) {
+	binary, args := command, ""
+	space := strings.Index(command, " ")
+	if space >= 0 {
+		binary = command[:space]
+		args = strings.TrimLeft(command[space+1:], " ")
+	}
+	args = shellUnEscape(args)
+	fs.Debugf(c.what, "exec command: binary = %q, args = %q", binary, args)
+	switch binary {
+	case "df":
+		about := c.f.Features().About
+		if about == nil {
+			return errors.New("df not supported")
+		}
+		usage, err := about()
+		if err != nil {
+			return errors.Wrap(err, "About failed")
+		}
+		total, used, free := int64(-1), int64(-1), int64(-1)
+		if usage.Total != nil {
+			total = *usage.Total / 1024
+		}
+		if usage.Used != nil {
+			used = *usage.Used / 1024
+		}
+		if usage.Free != nil {
+			free = *usage.Free / 1024
+		}
+		perc := int64(0)
+		if total > 0 && used >= 0 {
+			perc = (100 * used) / total
+		}
+		_, err = fmt.Fprintf(out, `		Filesystem                   1K-blocks      Used Available Use%% Mounted on
+/dev/root %d %d  %d  %d%% /
+`, total, used, free, perc)
+		if err != nil {
+			return errors.Wrap(err, "send output failed")
+		}
+	case "md5sum", "sha1sum":
+		ht := hash.MD5
+		if binary == "sha1sum" {
+			ht = hash.SHA1
+		}
+		node, err := c.vfs.Stat(args)
+		if err != nil {
+			return errors.Wrapf(err, "hash failed finding file %q", args)
+		}
+		if node.IsDir() {
+			return errors.New("can't hash directory")
+		}
+		o, ok := node.DirEntry().(fs.ObjectInfo)
+		if !ok {
+			return errors.New("unexpected non file")
+		}
+		hash, err := o.Hash(ht)
+		if err != nil {
+			return errors.Wrap(err, "hash failed")
+		}
+		_, err = fmt.Fprintf(out, "%s  %s\n", hash, args)
+		if err != nil {
+			return errors.Wrap(err, "send output failed")
+		}
+	case "echo":
+		// special cases for rclone command detection
+		switch args {
+		case "'abc' | md5sum":
+			if c.f.Hashes().Contains(hash.MD5) {
+				_, err = fmt.Fprintf(out, "0bee89b07a248e27c83fc3d5951213c1  -\n")
+				if err != nil {
+					return errors.Wrap(err, "send output failed")
+				}
+			} else {
+				return errors.New("md5 hash not supported")
+			}
+		case "'abc' | sha1sum":
+			if c.f.Hashes().Contains(hash.SHA1) {
+				_, err = fmt.Fprintf(out, "03cfd743661f07975fa2f1220c5194cbaff48451  -\n")
+				if err != nil {
+					return errors.Wrap(err, "send output failed")
+				}
+			} else {
+				return errors.New("sha1 hash not supported")
+			}
+		default:
+			_, err = fmt.Fprintf(out, "%s\n", args)
+			if err != nil {
+				return errors.Wrap(err, "send output failed")
+			}
+		}
+	default:
+		return errors.Errorf("%q not implemented\n", command)
+	}
+	return nil
+}
+
+// handle a new incoming channel request
+func (c *conn) handleChannel(newChannel ssh.NewChannel) {
+	fs.Debugf(c.what, "Incoming channel: %s\n", newChannel.ChannelType())
+	if newChannel.ChannelType() != "session" {
+		err := newChannel.Reject(ssh.UnknownChannelType, "unknown channel type")
+		fs.Debugf(c.what, "Unknown channel type: %s\n", newChannel.ChannelType())
+		if err != nil {
+			fs.Errorf(c.what, "Failed to reject unknown channel: %v", err)
+		}
+		return
+	}
+	channel, requests, err := newChannel.Accept()
+	if err != nil {
+		fs.Errorf(c.what, "could not accept channel: %v", err)
+		return
+	}
+	defer func() {
+		err := channel.Close()
+		if err != nil {
+			fs.Debugf(c.what, "Failed to close channel: %v", err)
+		}
+	}()
+	fs.Debugf(c.what, "Channel accepted\n")
+
+	isSFTP := make(chan bool, 1)
+	var command execCommand
+
+	// Handle out-of-band requests
+	go func(in <-chan *ssh.Request) {
+		for req := range in {
+			fs.Debugf(c.what, "Request: %v\n", req.Type)
+			ok := false
+			var subSystemIsSFTP bool
+			var reply []byte
+			switch req.Type {
+			case "subsystem":
+				fs.Debugf(c.what, "Subsystem: %s\n", req.Payload[4:])
+				if string(req.Payload[4:]) == "sftp" {
+					ok = true
+					subSystemIsSFTP = true
+				}
+			case "exec":
+				err := ssh.Unmarshal(req.Payload, &command)
+				if err != nil {
+					fs.Errorf(c.what, "ignoring bad exec command: %v", err)
+				} else {
+					ok = true
+					subSystemIsSFTP = false
+				}
+			}
+			fs.Debugf(c.what, " - accepted: %v\n", ok)
+			err = req.Reply(ok, reply)
+			if err != nil {
+				fs.Errorf(c.what, "Failed to Reply to request: %v", err)
+				return
+			}
+			if ok {
+				// Wake up main routine after we have responded
+				isSFTP <- subSystemIsSFTP
+			}
+		}
+	}(requests)
+
+	// Wait for either subsystem "sftp" or "exec" request
+	if <-isSFTP {
+		fs.Debugf(c.what, "Starting SFTP server")
+		server := sftp.NewRequestServer(channel, c.handlers)
+		defer func() {
+			err := server.Close()
+			if err != nil {
+				fs.Debugf(c.what, "Failed to close server: %v", err)
+			}
+		}()
+		err = server.Serve()
+		if err == io.EOF || err == nil {
+			fs.Debugf(c.what, "exited session")
+		} else {
+			fs.Errorf(c.what, "completed with error: %v", err)
+		}
+	} else {
+		var rc = uint32(0)
+		err := c.execCommand(channel, command.Command)
+		if err != nil {
+			rc = 1
+			_, errPrint := fmt.Fprintf(channel.Stderr(), "%v\n", err)
+			if errPrint != nil {
+				fs.Errorf(c.what, "Failed to write to stderr: %v", errPrint)
+			}
+			fs.Debugf(c.what, "command %q failed with error: %v", command.Command, err)
+		}
+		_, err = channel.SendRequest("exit-status", false, ssh.Marshal(exitStatus{RC: rc}))
+		if err != nil {
+			fs.Errorf(c.what, "Failed to send exit status: %v", err)
+		}
+	}
+}
+
+// Service the incoming Channel channel in go routine
+func (c *conn) handleChannels(chans <-chan ssh.NewChannel) {
+	for newChannel := range chans {
+		go c.handleChannel(newChannel)
+	}
+}

cmd/serve/sftp/connection_test.go

@@ -0,0 +1,25 @@
+// +build !plan9
+
+package sftp
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestShellEscape(t *testing.T) {
+	for i, test := range []struct {
+		unescaped, escaped string
+	}{
+		{"", ""},
+		{"/this/is/harmless", "/this/is/harmless"},
+		{"$(rm -rf /)", "\\$\\(rm\\ -rf\\ /\\)"},
+		{"/test/\n", "/test/'\n'"},
+		{":\"'", ":\\\"\\'"},
+	} {
+		got := shellUnEscape(test.escaped)
+		assert.Equal(t, test.unescaped, got, fmt.Sprintf("Test %d unescaped = %q", i, test.unescaped))
+	}
+}

cmd/serve/sftp/handler.go

@@ -0,0 +1,154 @@
+// +build !plan9
+
+package sftp
+
+import (
+	"io"
+	"os"
+	"syscall"
+	"time"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/vfs"
+	"github.com/pkg/sftp"
+)
+
+// vfsHandler converts the VFS to be served by SFTP
+type vfsHandler struct {
+	*vfs.VFS
+}
+
+// vfsHandler returns a Handlers object with the test handlers.
+func newVFSHandler(vfs *vfs.VFS) (sftp.Handlers, error) {
+	v := vfsHandler{VFS: vfs}
+	return sftp.Handlers{
+		FileGet:  v,
+		FilePut:  v,
+		FileCmd:  v,
+		FileList: v,
+	}, nil
+}
+
+func (v vfsHandler) Fileread(r *sftp.Request) (io.ReaderAt, error) {
+	file, err := v.OpenFile(r.Filepath, os.O_RDONLY, 0777)
+	if err != nil {
+		return nil, err
+	}
+	return file, nil
+}
+
+func (v vfsHandler) Filewrite(r *sftp.Request) (io.WriterAt, error) {
+	file, err := v.OpenFile(r.Filepath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0777)
+	if err != nil {
+		return nil, err
+	}
+	return file, nil
+}
+
+func (v vfsHandler) Filecmd(r *sftp.Request) error {
+	switch r.Method {
+	case "Setstat":
+		node, err := v.Stat(r.Filepath)
+		if err != nil {
+			return err
+		}
+		attr := r.Attributes()
+		if attr.Mtime != 0 {
+			modTime := time.Unix(int64(attr.Mtime), 0)
+			err := node.SetModTime(modTime)
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	case "Rename":
+		err := v.Rename(r.Filepath, r.Target)
+		if err != nil {
+			return err
+		}
+	case "Rmdir", "Remove":
+		node, err := v.Stat(r.Filepath)
+		if err != nil {
+			return err
+		}
+		err = node.Remove()
+		if err != nil {
+			return err
+		}
+	case "Mkdir":
+		dir, leaf, err := v.StatParent(r.Filepath)
+		if err != nil {
+			return err
+		}
+		_, err = dir.Mkdir(leaf)
+		if err != nil {
+			return err
+		}
+	case "Symlink":
+		// FIXME
+		// _, err := v.fetch(r.Filepath)
+		// if err != nil {
+		// 	return err
+		// }
+		// link := newMemFile(r.Target, false)
+		// link.symlink = r.Filepath
+		// v.files[r.Target] = link
+	}
+	return nil
+}
+
+type listerat []os.FileInfo
+
+// Modeled after strings.Reader's ReadAt() implementation
+func (f listerat) ListAt(ls []os.FileInfo, offset int64) (int, error) {
+	var n int
+	if offset >= int64(len(f)) {
+		return 0, io.EOF
+	}
+	n = copy(ls, f[offset:])
+	if n < len(ls) {
+		return n, io.EOF
+	}
+	return n, nil
+}
+
+func (v vfsHandler) Filelist(r *sftp.Request) (l sftp.ListerAt, err error) {
+	var node vfs.Node
+	var handle vfs.Handle
+	switch r.Method {
+	case "List":
+		node, err = v.Stat(r.Filepath)
+		if err != nil {
+			return nil, err
+		}
+		if !node.IsDir() {
+			return nil, syscall.ENOTDIR
+		}
+		handle, err = node.Open(os.O_RDONLY)
+		if err != nil {
+			return nil, err
+		}
+		defer fs.CheckClose(handle, &err)
+		fis, err := handle.Readdir(-1)
+		if err != nil {
+			return nil, err
+		}
+		return listerat(fis), nil
+	case "Stat":
+		node, err = v.Stat(r.Filepath)
+		if err != nil {
+			return nil, err
+		}
+		return listerat([]os.FileInfo{node}), nil
+	case "Readlink":
+		// FIXME
+		// if file.symlink != "" {
+		// 	file, err = v.fetch(file.symlink)
+		// 	if err != nil {
+		// 		return nil, err
+		// 	}
+		// }
+		// return listerat([]os.FileInfo{file}), nil
+	}
+	return nil, nil
+}

cmd/serve/sftp/server.go

@@ -0,0 +1,282 @@
+// +build !plan9
+
+package sftp
+
+import (
+	"bytes"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/subtle"
+	"crypto/x509"
+	"encoding/pem"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/config"
+	"github.com/ncw/rclone/lib/env"
+	"github.com/ncw/rclone/vfs"
+	"github.com/ncw/rclone/vfs/vfsflags"
+	"github.com/pkg/errors"
+	"github.com/pkg/sftp"
+	"golang.org/x/crypto/ssh"
+)
+
+// server contains everything to run the server
+type server struct {
+	f        fs.Fs
+	opt      Options
+	vfs      *vfs.VFS
+	config   *ssh.ServerConfig
+	handlers sftp.Handlers
+	listener net.Listener
+	waitChan chan struct{} // for waiting on the listener to close
+}
+
+func newServer(f fs.Fs, opt *Options) *server {
+	s := &server{
+		f:        f,
+		vfs:      vfs.New(f, &vfsflags.Opt),
+		opt:      *opt,
+		waitChan: make(chan struct{}),
+	}
+	return s
+}
+
+func (s *server) acceptConnections() {
+	for {
+		nConn, err := s.listener.Accept()
+		if err != nil {
+			if strings.Contains(err.Error(), "use of closed network connection") {
+				return
+			}
+			fs.Errorf(nil, "Failed to accept incoming connection: %v", err)
+			continue
+		}
+		what := describeConn(nConn)
+
+		// Before use, a handshake must be performed on the incoming net.Conn.
+		sshConn, chans, reqs, err := ssh.NewServerConn(nConn, s.config)
+		if err != nil {
+			fs.Errorf(what, "SSH login failed: %v", err)
+			continue
+		}
+
+		fs.Infof(what, "SSH login from %s using %s", sshConn.User(), sshConn.ClientVersion())
+
+		// Discard all global out-of-band Requests
+		go ssh.DiscardRequests(reqs)
+
+		c := &conn{
+			vfs:      s.vfs,
+			f:        s.f,
+			handlers: s.handlers,
+			what:     what,
+		}
+
+		// Accept all channels
+		go c.handleChannels(chans)
+	}
+}
+
+// Based on example server code from golang.org/x/crypto/ssh and server_standalone
+func (s *server) serve() (err error) {
+	var authorizedKeysMap map[string]struct{}
+
+	// Load the authorized keys
+	if s.opt.AuthorizedKeys != "" {
+		authKeysFile := env.ShellExpand(s.opt.AuthorizedKeys)
+		authorizedKeysMap, err = loadAuthorizedKeys(authKeysFile)
+		// If user set the flag away from the default then report an error
+		if err != nil && s.opt.AuthorizedKeys != DefaultOpt.AuthorizedKeys {
+			return err
+		}
+		fs.Logf(nil, "Loaded %d authorized keys from %q", len(authorizedKeysMap), authKeysFile)
+	}
+
+	if !s.opt.NoAuth && len(authorizedKeysMap) == 0 && s.opt.User == "" && s.opt.Pass == "" {
+		return errors.New("no authorization found, use --user/--pass or --authorized-keys or --no-auth")
+	}
+
+	// An SSH server is represented by a ServerConfig, which holds
+	// certificate details and handles authentication of ServerConns.
+	s.config = &ssh.ServerConfig{
+		ServerVersion: "SSH-2.0-" + fs.Config.UserAgent,
+		PasswordCallback: func(c ssh.ConnMetadata, pass []byte) (*ssh.Permissions, error) {
+			fs.Debugf(describeConn(c), "Password login attempt for %s", c.User())
+			if s.opt.User != "" && s.opt.Pass != "" {
+				userOK := subtle.ConstantTimeCompare([]byte(c.User()), []byte(s.opt.User))
+				passOK := subtle.ConstantTimeCompare(pass, []byte(s.opt.Pass))
+				if (userOK & passOK) == 1 {
+					return nil, nil
+				}
+			}
+			return nil, fmt.Errorf("password rejected for %q", c.User())
+		},
+		PublicKeyCallback: func(c ssh.ConnMetadata, pubKey ssh.PublicKey) (*ssh.Permissions, error) {
+			fs.Debugf(describeConn(c), "Public key login attempt for %s", c.User())
+			if _, ok := authorizedKeysMap[string(pubKey.Marshal())]; ok {
+				return &ssh.Permissions{
+					// Record the public key used for authentication.
+					Extensions: map[string]string{
+						"pubkey-fp": ssh.FingerprintSHA256(pubKey),
+					},
+				}, nil
+			}
+			return nil, fmt.Errorf("unknown public key for %q", c.User())
+		},
+		AuthLogCallback: func(conn ssh.ConnMetadata, method string, err error) {
+			status := "OK"
+			if err != nil {
+				status = err.Error()
+			}
+			fs.Debugf(describeConn(conn), "ssh auth %q from %q: %s", method, conn.ClientVersion(), status)
+		},
+		NoClientAuth: s.opt.NoAuth,
+	}
+
+	// Load the private key, from the cache if not explicitly configured
+	keyPath := s.opt.Key
+	cachePath := filepath.Join(config.CacheDir, "serve-sftp")
+	if keyPath == "" {
+		keyPath = filepath.Join(cachePath, "id_rsa")
+	}
+	private, err := loadPrivateKey(keyPath)
+	if err != nil && s.opt.Key == "" {
+		fs.Debugf(nil, "Failed to load %q: %v", keyPath, err)
+		// If loading a cached key failed, make the keys and retry
+		err = os.MkdirAll(cachePath, 0700)
+		if err != nil {
+			return errors.Wrap(err, "failed to create cache path")
+		}
+		const bits = 2048
+		fs.Logf(nil, "Generating %d bit key pair at %q", bits, keyPath)
+		err = makeSSHKeyPair(bits, keyPath+".pub", keyPath)
+		if err != nil {
+			return errors.Wrap(err, "failed to create SSH key pair")
+		}
+		// reload the new keys
+		private, err = loadPrivateKey(keyPath)
+	}
+	if err != nil {
+		return err
+	}
+	fs.Debugf(nil, "Loaded private key from %q", keyPath)
+
+	s.config.AddHostKey(private)
+
+	// Once a ServerConfig has been configured, connections can be
+	// accepted.
+	s.listener, err = net.Listen("tcp", s.opt.ListenAddr)
+	if err != nil {
+		log.Fatal("failed to listen for connection", err)
+	}
+	fs.Logf(nil, "SFTP server listening on %v\n", s.listener.Addr())
+
+	s.handlers, err = newVFSHandler(s.vfs)
+	if err != nil {
+		return errors.Wrap(err, "serve sftp: failed to create fs")
+	}
+
+	go s.acceptConnections()
+
+	return nil
+}
+
+// Addr returns the address the server is listening on
+func (s *server) Addr() string {
+	return s.listener.Addr().String()
+}
+
+// Serve runs the sftp server in the background.
+//
+// Use s.Close() and s.Wait() to shutdown server
+func (s *server) Serve() error {
+	err := s.serve()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// Wait blocks while the listener is open.
+func (s *server) Wait() {
+	<-s.waitChan
+}
+
+// Close shuts the running server down
+func (s *server) Close() {
+	err := s.listener.Close()
+	if err != nil {
+		fs.Errorf(nil, "Error on closing SFTP server: %v", err)
+		return
+	}
+	close(s.waitChan)
+}
+
+func loadPrivateKey(keyPath string) (ssh.Signer, error) {
+	privateBytes, err := ioutil.ReadFile(keyPath)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to load private key")
+	}
+	private, err := ssh.ParsePrivateKey(privateBytes)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to parse private key")
+	}
+	return private, nil
+}
+
+// Public key authentication is done by comparing
+// the public key of a received connection
+// with the entries in the authorized_keys file.
+func loadAuthorizedKeys(authorizedKeysPath string) (authorizedKeysMap map[string]struct{}, err error) {
+	authorizedKeysBytes, err := ioutil.ReadFile(authorizedKeysPath)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to load authorized keys")
+	}
+	authorizedKeysMap = make(map[string]struct{})
+	for len(authorizedKeysBytes) > 0 {
+		pubKey, _, _, rest, err := ssh.ParseAuthorizedKey(authorizedKeysBytes)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to parse authorized keys")
+		}
+		authorizedKeysMap[string(pubKey.Marshal())] = struct{}{}
+		authorizedKeysBytes = bytes.TrimSpace(rest)
+	}
+	return authorizedKeysMap, nil
+}
+
+// makeSSHKeyPair make a pair of public and private keys for SSH access.
+// Public key is encoded in the format for inclusion in an OpenSSH authorized_keys file.
+// Private Key generated is PEM encoded
+//
+// Originally from: https://stackoverflow.com/a/34347463/164234
+func makeSSHKeyPair(bits int, pubKeyPath, privateKeyPath string) (err error) {
+	privateKey, err := rsa.GenerateKey(rand.Reader, bits)
+	if err != nil {
+		return err
+	}
+
+	// generate and write private key as PEM
+	privateKeyFile, err := os.OpenFile(privateKeyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+	if err != nil {
+		return err
+	}
+	defer fs.CheckClose(privateKeyFile, &err)
+	privateKeyPEM := &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(privateKey)}
+	if err := pem.Encode(privateKeyFile, privateKeyPEM); err != nil {
+		return err
+	}
+
+	// generate and write public key
+	pub, err := ssh.NewPublicKey(&privateKey.PublicKey)
+	if err != nil {
+		return err
+	}
+	return ioutil.WriteFile(pubKeyPath, ssh.MarshalAuthorizedKey(pub), 0644)
+}

cmd/serve/sftp/sftp.go

@@ -0,0 +1,101 @@
+// Package sftp implements an SFTP server to serve an rclone VFS
+
+// +build !plan9
+
+package sftp
+
+import (
+	"github.com/ncw/rclone/cmd"
+	"github.com/ncw/rclone/fs/config/flags"
+	"github.com/ncw/rclone/fs/rc"
+	"github.com/ncw/rclone/vfs"
+	"github.com/ncw/rclone/vfs/vfsflags"
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+)
+
+// Options contains options for the http Server
+type Options struct {
+	ListenAddr     string // Port to listen on
+	Key            string // Path to private key
+	AuthorizedKeys string // Path to authorized keys file
+	User           string // single username
+	Pass           string // password for user
+	NoAuth         bool   // allow no authentication on connections
+}
+
+// DefaultOpt is the default values used for Options
+var DefaultOpt = Options{
+	ListenAddr:     "localhost:2022",
+	AuthorizedKeys: "~/.ssh/authorized_keys",
+}
+
+// Opt is options set by command line flags
+var Opt = DefaultOpt
+
+// AddFlags adds flags for the sftp
+func AddFlags(flagSet *pflag.FlagSet, Opt *Options) {
+	rc.AddOption("sftp", &Opt)
+	flags.StringVarP(flagSet, &Opt.ListenAddr, "addr", "", Opt.ListenAddr, "IPaddress:Port or :Port to bind server to.")
+	flags.StringVarP(flagSet, &Opt.Key, "key", "", Opt.Key, "SSH private key file (leave blank to auto generate)")
+	flags.StringVarP(flagSet, &Opt.AuthorizedKeys, "authorized-keys", "", Opt.AuthorizedKeys, "Authorized keys file")
+	flags.StringVarP(flagSet, &Opt.User, "user", "", Opt.User, "User name for authentication.")
+	flags.StringVarP(flagSet, &Opt.Pass, "pass", "", Opt.Pass, "Password for authentication.")
+	flags.BoolVarP(flagSet, &Opt.NoAuth, "no-auth", "", Opt.NoAuth, "Allow connections with no authentication if set.")
+}
+
+func init() {
+	vfsflags.AddFlags(Command.Flags())
+	AddFlags(Command.Flags(), &Opt)
+}
+
+// Command definition for cobra
+var Command = &cobra.Command{
+	Use:   "sftp remote:path",
+	Short: `Serve the remote over SFTP.`,
+	Long: `rclone serve sftp implements an SFTP server to serve the remote
+over SFTP.  This can be used with an SFTP client or you can make a
+remote of type sftp to use with it.
+
+You can use the filter flags (eg --include, --exclude) to control what
+is served.
+
+The server will log errors.  Use -v to see access logs.
+
+--bwlimit will be respected for file transfers.  Use --stats to
+control the stats printing.
+
+You must provide some means of authentication, either with --user/--pass,
+an authorized keys file (specify location with --authorized-keys - the
+default is the same as ssh) or set the --no-auth flag for no
+authentication when logging in.
+
+Note that this also implements a small number of shell commands so
+that it can provide md5sum/sha1sum/df information for the rclone sftp
+backend.  This means that is can support SHA1SUMs, MD5SUMs and the
+about command when paired with the rclone sftp backend.
+
+If you don't supply a --key then rclone will generate one and cache it
+for later use.
+
+By default the server binds to localhost:2022 - if you want it to be
+reachable externally then supply "--addr :2022" for example.
+
+Note that the default of "--vfs-cache-mode off" is fine for the rclone
+sftp backend, but it may not be with other SFTP clients.
+
+` + vfs.Help,
+	Run: func(command *cobra.Command, args []string) {
+		cmd.CheckArgs(1, 1, command, args)
+		f := cmd.NewFsSrc(args)
+		cmd.Run(false, true, command, func() error {
+			s := newServer(f, &Opt)
+			err := s.Serve()
+			if err != nil {
+				return err
+			}
+			s.Wait()
+			return nil
+		})
+	},
+}

cmd/serve/sftp/sftp_test.go

@@ -0,0 +1,94 @@
+// Serve sftp tests set up a server and run the integration tests
+// for the sftp remote against it.
+//
+// We skip tests on platforms with troublesome character mappings
+
+//+build !windows,!darwin,!plan9
+
+package sftp
+
+import (
+	"os"
+	"os/exec"
+	"strings"
+	"testing"
+
+	_ "github.com/ncw/rclone/backend/local"
+	"github.com/ncw/rclone/fs/config/obscure"
+	"github.com/ncw/rclone/fstest"
+	"github.com/pkg/sftp"
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	testBindAddress = "localhost:0"
+	testUser        = "testuser"
+	testPass        = "testpass"
+)
+
+// check interfaces
+var (
+	_ sftp.FileReader = vfsHandler{}
+	_ sftp.FileWriter = vfsHandler{}
+	_ sftp.FileCmder  = vfsHandler{}
+	_ sftp.FileLister = vfsHandler{}
+)
+
+// TestSftp runs the sftp server then runs the unit tests for the
+// sftp remote against it.
+func TestSftp(t *testing.T) {
+	fstest.Initialise()
+
+	fremote, _, clean, err := fstest.RandomRemote(*fstest.RemoteName, *fstest.SubDir)
+	assert.NoError(t, err)
+	defer clean()
+
+	err = fremote.Mkdir("")
+	assert.NoError(t, err)
+
+	opt := DefaultOpt
+	opt.ListenAddr = testBindAddress
+	opt.User = testUser
+	opt.Pass = testPass
+
+	// Start the server
+	w := newServer(fremote, &opt)
+	assert.NoError(t, w.serve())
+	defer func() {
+		w.Close()
+		w.Wait()
+	}()
+
+	// Change directory to run the tests
+	err = os.Chdir("../../../backend/sftp")
+	assert.NoError(t, err, "failed to cd to sftp backend")
+
+	// Run the sftp tests with an on the fly remote
+	args := []string{"test"}
+	if testing.Verbose() {
+		args = append(args, "-v")
+	}
+	if *fstest.Verbose {
+		args = append(args, "-verbose")
+	}
+	args = append(args, "-remote", "sftptest:")
+	cmd := exec.Command("go", args...)
+	addr := w.Addr()
+	colon := strings.LastIndex(addr, ":")
+	if colon < 0 {
+		panic("need a : in the address: " + addr)
+	}
+	host, port := addr[:colon], addr[colon+1:]
+	cmd.Env = append(os.Environ(),
+		"RCLONE_CONFIG_SFTPTEST_TYPE=sftp",
+		"RCLONE_CONFIG_SFTPTEST_HOST="+host,
+		"RCLONE_CONFIG_SFTPTEST_PORT="+port,
+		"RCLONE_CONFIG_SFTPTEST_USER="+testUser,
+		"RCLONE_CONFIG_SFTPTEST_PASS="+obscure.MustObscure(testPass),
+	)
+	out, err := cmd.CombinedOutput()
+	if len(out) != 0 {
+		t.Logf("\n----------\n%s----------\n", string(out))
+	}
+	assert.NoError(t, err, "Running sftp integration tests")
+}

cmd/serve/sftp/sftp_unsupported.go

@@ -0,0 +1,11 @@
+// Build for sftp for unsupported platforms to stop go complaining
+// about "no buildable Go source files "
+
+// +build plan9
+
+package sftp
+
+import "github.com/spf13/cobra"
+
+// Command definition is nil to show not implemented
+var Command *cobra.Command = nil