diff --git a/cmd/serve/serve.go b/cmd/serve/serve.go index 4ab583188..79b2e1a59 100644 --- a/cmd/serve/serve.go +++ b/cmd/serve/serve.go @@ -3,12 +3,12 @@ package serve import ( "errors" - "github.com/ncw/rclone/cmd/serve/dlna" - "github.com/ncw/rclone/cmd" + "github.com/ncw/rclone/cmd/serve/dlna" "github.com/ncw/rclone/cmd/serve/ftp" "github.com/ncw/rclone/cmd/serve/http" "github.com/ncw/rclone/cmd/serve/restic" + "github.com/ncw/rclone/cmd/serve/sftp" "github.com/ncw/rclone/cmd/serve/webdav" "github.com/spf13/cobra" ) @@ -27,6 +27,9 @@ func init() { if ftp.Command != nil { Command.AddCommand(ftp.Command) } + if sftp.Command != nil { + Command.AddCommand(sftp.Command) + } cmd.Root.AddCommand(Command) } diff --git a/cmd/serve/sftp/connection.go b/cmd/serve/sftp/connection.go new file mode 100644 index 000000000..497e2a1fa --- /dev/null +++ b/cmd/serve/sftp/connection.go @@ -0,0 +1,254 @@ +// +build !plan9 + +package sftp + +import ( + "fmt" + "io" + "net" + "regexp" + "strings" + + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/hash" + "github.com/ncw/rclone/vfs" + "github.com/pkg/errors" + "github.com/pkg/sftp" + "golang.org/x/crypto/ssh" +) + +func describeConn(c interface { + RemoteAddr() net.Addr + LocalAddr() net.Addr +}) string { + return fmt.Sprintf("serve sftp %s->%s", c.RemoteAddr(), c.LocalAddr()) +} + +// Return the exit status of the command +type exitStatus struct { + RC uint32 +} + +// The incoming exec command +type execCommand struct { + Command string +} + +var shellUnEscapeRegex = regexp.MustCompile(`\\(.)`) + +// Unescape a string that was escaped by rclone +func shellUnEscape(str string) string { + str = strings.Replace(str, "'\n'", "\n", -1) + str = shellUnEscapeRegex.ReplaceAllString(str, `$1`) + return str +} + +// Info about the current connection +type conn struct { + vfs *vfs.VFS + f fs.Fs + handlers sftp.Handlers + what string +} + +// execCommand implements an extrememly limited number of commands to +// interoperate with the rclone sftp backend +func (c *conn) execCommand(out io.Writer, command string) (err error) { + binary, args := command, "" + space := strings.Index(command, " ") + if space >= 0 { + binary = command[:space] + args = strings.TrimLeft(command[space+1:], " ") + } + args = shellUnEscape(args) + fs.Debugf(c.what, "exec command: binary = %q, args = %q", binary, args) + switch binary { + case "df": + about := c.f.Features().About + if about == nil { + return errors.New("df not supported") + } + usage, err := about() + if err != nil { + return errors.Wrap(err, "About failed") + } + total, used, free := int64(-1), int64(-1), int64(-1) + if usage.Total != nil { + total = *usage.Total / 1024 + } + if usage.Used != nil { + used = *usage.Used / 1024 + } + if usage.Free != nil { + free = *usage.Free / 1024 + } + perc := int64(0) + if total > 0 && used >= 0 { + perc = (100 * used) / total + } + _, err = fmt.Fprintf(out, ` Filesystem 1K-blocks Used Available Use%% Mounted on +/dev/root %d %d %d %d%% / +`, total, used, free, perc) + if err != nil { + return errors.Wrap(err, "send output failed") + } + case "md5sum", "sha1sum": + ht := hash.MD5 + if binary == "sha1sum" { + ht = hash.SHA1 + } + node, err := c.vfs.Stat(args) + if err != nil { + return errors.Wrapf(err, "hash failed finding file %q", args) + } + if node.IsDir() { + return errors.New("can't hash directory") + } + o, ok := node.DirEntry().(fs.ObjectInfo) + if !ok { + return errors.New("unexpected non file") + } + hash, err := o.Hash(ht) + if err != nil { + return errors.Wrap(err, "hash failed") + } + _, err = fmt.Fprintf(out, "%s %s\n", hash, args) + if err != nil { + return errors.Wrap(err, "send output failed") + } + case "echo": + // special cases for rclone command detection + switch args { + case "'abc' | md5sum": + if c.f.Hashes().Contains(hash.MD5) { + _, err = fmt.Fprintf(out, "0bee89b07a248e27c83fc3d5951213c1 -\n") + if err != nil { + return errors.Wrap(err, "send output failed") + } + } else { + return errors.New("md5 hash not supported") + } + case "'abc' | sha1sum": + if c.f.Hashes().Contains(hash.SHA1) { + _, err = fmt.Fprintf(out, "03cfd743661f07975fa2f1220c5194cbaff48451 -\n") + if err != nil { + return errors.Wrap(err, "send output failed") + } + } else { + return errors.New("sha1 hash not supported") + } + default: + _, err = fmt.Fprintf(out, "%s\n", args) + if err != nil { + return errors.Wrap(err, "send output failed") + } + } + default: + return errors.Errorf("%q not implemented\n", command) + } + return nil +} + +// handle a new incoming channel request +func (c *conn) handleChannel(newChannel ssh.NewChannel) { + fs.Debugf(c.what, "Incoming channel: %s\n", newChannel.ChannelType()) + if newChannel.ChannelType() != "session" { + err := newChannel.Reject(ssh.UnknownChannelType, "unknown channel type") + fs.Debugf(c.what, "Unknown channel type: %s\n", newChannel.ChannelType()) + if err != nil { + fs.Errorf(c.what, "Failed to reject unknown channel: %v", err) + } + return + } + channel, requests, err := newChannel.Accept() + if err != nil { + fs.Errorf(c.what, "could not accept channel: %v", err) + return + } + defer func() { + err := channel.Close() + if err != nil { + fs.Debugf(c.what, "Failed to close channel: %v", err) + } + }() + fs.Debugf(c.what, "Channel accepted\n") + + isSFTP := make(chan bool, 1) + var command execCommand + + // Handle out-of-band requests + go func(in <-chan *ssh.Request) { + for req := range in { + fs.Debugf(c.what, "Request: %v\n", req.Type) + ok := false + var subSystemIsSFTP bool + var reply []byte + switch req.Type { + case "subsystem": + fs.Debugf(c.what, "Subsystem: %s\n", req.Payload[4:]) + if string(req.Payload[4:]) == "sftp" { + ok = true + subSystemIsSFTP = true + } + case "exec": + err := ssh.Unmarshal(req.Payload, &command) + if err != nil { + fs.Errorf(c.what, "ignoring bad exec command: %v", err) + } else { + ok = true + subSystemIsSFTP = false + } + } + fs.Debugf(c.what, " - accepted: %v\n", ok) + err = req.Reply(ok, reply) + if err != nil { + fs.Errorf(c.what, "Failed to Reply to request: %v", err) + return + } + if ok { + // Wake up main routine after we have responded + isSFTP <- subSystemIsSFTP + } + } + }(requests) + + // Wait for either subsystem "sftp" or "exec" request + if <-isSFTP { + fs.Debugf(c.what, "Starting SFTP server") + server := sftp.NewRequestServer(channel, c.handlers) + defer func() { + err := server.Close() + if err != nil { + fs.Debugf(c.what, "Failed to close server: %v", err) + } + }() + err = server.Serve() + if err == io.EOF || err == nil { + fs.Debugf(c.what, "exited session") + } else { + fs.Errorf(c.what, "completed with error: %v", err) + } + } else { + var rc = uint32(0) + err := c.execCommand(channel, command.Command) + if err != nil { + rc = 1 + _, errPrint := fmt.Fprintf(channel.Stderr(), "%v\n", err) + if errPrint != nil { + fs.Errorf(c.what, "Failed to write to stderr: %v", errPrint) + } + fs.Debugf(c.what, "command %q failed with error: %v", command.Command, err) + } + _, err = channel.SendRequest("exit-status", false, ssh.Marshal(exitStatus{RC: rc})) + if err != nil { + fs.Errorf(c.what, "Failed to send exit status: %v", err) + } + } +} + +// Service the incoming Channel channel in go routine +func (c *conn) handleChannels(chans <-chan ssh.NewChannel) { + for newChannel := range chans { + go c.handleChannel(newChannel) + } +} diff --git a/cmd/serve/sftp/connection_test.go b/cmd/serve/sftp/connection_test.go new file mode 100644 index 000000000..062affe4b --- /dev/null +++ b/cmd/serve/sftp/connection_test.go @@ -0,0 +1,25 @@ +// +build !plan9 + +package sftp + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestShellEscape(t *testing.T) { + for i, test := range []struct { + unescaped, escaped string + }{ + {"", ""}, + {"/this/is/harmless", "/this/is/harmless"}, + {"$(rm -rf /)", "\\$\\(rm\\ -rf\\ /\\)"}, + {"/test/\n", "/test/'\n'"}, + {":\"'", ":\\\"\\'"}, + } { + got := shellUnEscape(test.escaped) + assert.Equal(t, test.unescaped, got, fmt.Sprintf("Test %d unescaped = %q", i, test.unescaped)) + } +} diff --git a/cmd/serve/sftp/handler.go b/cmd/serve/sftp/handler.go new file mode 100644 index 000000000..30d6720fe --- /dev/null +++ b/cmd/serve/sftp/handler.go @@ -0,0 +1,154 @@ +// +build !plan9 + +package sftp + +import ( + "io" + "os" + "syscall" + "time" + + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/vfs" + "github.com/pkg/sftp" +) + +// vfsHandler converts the VFS to be served by SFTP +type vfsHandler struct { + *vfs.VFS +} + +// vfsHandler returns a Handlers object with the test handlers. +func newVFSHandler(vfs *vfs.VFS) (sftp.Handlers, error) { + v := vfsHandler{VFS: vfs} + return sftp.Handlers{ + FileGet: v, + FilePut: v, + FileCmd: v, + FileList: v, + }, nil +} + +func (v vfsHandler) Fileread(r *sftp.Request) (io.ReaderAt, error) { + file, err := v.OpenFile(r.Filepath, os.O_RDONLY, 0777) + if err != nil { + return nil, err + } + return file, nil +} + +func (v vfsHandler) Filewrite(r *sftp.Request) (io.WriterAt, error) { + file, err := v.OpenFile(r.Filepath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0777) + if err != nil { + return nil, err + } + return file, nil +} + +func (v vfsHandler) Filecmd(r *sftp.Request) error { + switch r.Method { + case "Setstat": + node, err := v.Stat(r.Filepath) + if err != nil { + return err + } + attr := r.Attributes() + if attr.Mtime != 0 { + modTime := time.Unix(int64(attr.Mtime), 0) + err := node.SetModTime(modTime) + if err != nil { + return err + } + } + return nil + case "Rename": + err := v.Rename(r.Filepath, r.Target) + if err != nil { + return err + } + case "Rmdir", "Remove": + node, err := v.Stat(r.Filepath) + if err != nil { + return err + } + err = node.Remove() + if err != nil { + return err + } + case "Mkdir": + dir, leaf, err := v.StatParent(r.Filepath) + if err != nil { + return err + } + _, err = dir.Mkdir(leaf) + if err != nil { + return err + } + case "Symlink": + // FIXME + // _, err := v.fetch(r.Filepath) + // if err != nil { + // return err + // } + // link := newMemFile(r.Target, false) + // link.symlink = r.Filepath + // v.files[r.Target] = link + } + return nil +} + +type listerat []os.FileInfo + +// Modeled after strings.Reader's ReadAt() implementation +func (f listerat) ListAt(ls []os.FileInfo, offset int64) (int, error) { + var n int + if offset >= int64(len(f)) { + return 0, io.EOF + } + n = copy(ls, f[offset:]) + if n < len(ls) { + return n, io.EOF + } + return n, nil +} + +func (v vfsHandler) Filelist(r *sftp.Request) (l sftp.ListerAt, err error) { + var node vfs.Node + var handle vfs.Handle + switch r.Method { + case "List": + node, err = v.Stat(r.Filepath) + if err != nil { + return nil, err + } + if !node.IsDir() { + return nil, syscall.ENOTDIR + } + handle, err = node.Open(os.O_RDONLY) + if err != nil { + return nil, err + } + defer fs.CheckClose(handle, &err) + fis, err := handle.Readdir(-1) + if err != nil { + return nil, err + } + return listerat(fis), nil + case "Stat": + node, err = v.Stat(r.Filepath) + if err != nil { + return nil, err + } + return listerat([]os.FileInfo{node}), nil + case "Readlink": + // FIXME + // if file.symlink != "" { + // file, err = v.fetch(file.symlink) + // if err != nil { + // return nil, err + // } + // } + // return listerat([]os.FileInfo{file}), nil + } + return nil, nil +} diff --git a/cmd/serve/sftp/server.go b/cmd/serve/sftp/server.go new file mode 100644 index 000000000..7407489de --- /dev/null +++ b/cmd/serve/sftp/server.go @@ -0,0 +1,282 @@ +// +build !plan9 + +package sftp + +import ( + "bytes" + "crypto/rand" + "crypto/rsa" + "crypto/subtle" + "crypto/x509" + "encoding/pem" + "fmt" + "io/ioutil" + "log" + "net" + "os" + "path/filepath" + "strings" + + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/config" + "github.com/ncw/rclone/lib/env" + "github.com/ncw/rclone/vfs" + "github.com/ncw/rclone/vfs/vfsflags" + "github.com/pkg/errors" + "github.com/pkg/sftp" + "golang.org/x/crypto/ssh" +) + +// server contains everything to run the server +type server struct { + f fs.Fs + opt Options + vfs *vfs.VFS + config *ssh.ServerConfig + handlers sftp.Handlers + listener net.Listener + waitChan chan struct{} // for waiting on the listener to close +} + +func newServer(f fs.Fs, opt *Options) *server { + s := &server{ + f: f, + vfs: vfs.New(f, &vfsflags.Opt), + opt: *opt, + waitChan: make(chan struct{}), + } + return s +} + +func (s *server) acceptConnections() { + for { + nConn, err := s.listener.Accept() + if err != nil { + if strings.Contains(err.Error(), "use of closed network connection") { + return + } + fs.Errorf(nil, "Failed to accept incoming connection: %v", err) + continue + } + what := describeConn(nConn) + + // Before use, a handshake must be performed on the incoming net.Conn. + sshConn, chans, reqs, err := ssh.NewServerConn(nConn, s.config) + if err != nil { + fs.Errorf(what, "SSH login failed: %v", err) + continue + } + + fs.Infof(what, "SSH login from %s using %s", sshConn.User(), sshConn.ClientVersion()) + + // Discard all global out-of-band Requests + go ssh.DiscardRequests(reqs) + + c := &conn{ + vfs: s.vfs, + f: s.f, + handlers: s.handlers, + what: what, + } + + // Accept all channels + go c.handleChannels(chans) + } +} + +// Based on example server code from golang.org/x/crypto/ssh and server_standalone +func (s *server) serve() (err error) { + var authorizedKeysMap map[string]struct{} + + // Load the authorized keys + if s.opt.AuthorizedKeys != "" { + authKeysFile := env.ShellExpand(s.opt.AuthorizedKeys) + authorizedKeysMap, err = loadAuthorizedKeys(authKeysFile) + // If user set the flag away from the default then report an error + if err != nil && s.opt.AuthorizedKeys != DefaultOpt.AuthorizedKeys { + return err + } + fs.Logf(nil, "Loaded %d authorized keys from %q", len(authorizedKeysMap), authKeysFile) + } + + if !s.opt.NoAuth && len(authorizedKeysMap) == 0 && s.opt.User == "" && s.opt.Pass == "" { + return errors.New("no authorization found, use --user/--pass or --authorized-keys or --no-auth") + } + + // An SSH server is represented by a ServerConfig, which holds + // certificate details and handles authentication of ServerConns. + s.config = &ssh.ServerConfig{ + ServerVersion: "SSH-2.0-" + fs.Config.UserAgent, + PasswordCallback: func(c ssh.ConnMetadata, pass []byte) (*ssh.Permissions, error) { + fs.Debugf(describeConn(c), "Password login attempt for %s", c.User()) + if s.opt.User != "" && s.opt.Pass != "" { + userOK := subtle.ConstantTimeCompare([]byte(c.User()), []byte(s.opt.User)) + passOK := subtle.ConstantTimeCompare(pass, []byte(s.opt.Pass)) + if (userOK & passOK) == 1 { + return nil, nil + } + } + return nil, fmt.Errorf("password rejected for %q", c.User()) + }, + PublicKeyCallback: func(c ssh.ConnMetadata, pubKey ssh.PublicKey) (*ssh.Permissions, error) { + fs.Debugf(describeConn(c), "Public key login attempt for %s", c.User()) + if _, ok := authorizedKeysMap[string(pubKey.Marshal())]; ok { + return &ssh.Permissions{ + // Record the public key used for authentication. + Extensions: map[string]string{ + "pubkey-fp": ssh.FingerprintSHA256(pubKey), + }, + }, nil + } + return nil, fmt.Errorf("unknown public key for %q", c.User()) + }, + AuthLogCallback: func(conn ssh.ConnMetadata, method string, err error) { + status := "OK" + if err != nil { + status = err.Error() + } + fs.Debugf(describeConn(conn), "ssh auth %q from %q: %s", method, conn.ClientVersion(), status) + }, + NoClientAuth: s.opt.NoAuth, + } + + // Load the private key, from the cache if not explicitly configured + keyPath := s.opt.Key + cachePath := filepath.Join(config.CacheDir, "serve-sftp") + if keyPath == "" { + keyPath = filepath.Join(cachePath, "id_rsa") + } + private, err := loadPrivateKey(keyPath) + if err != nil && s.opt.Key == "" { + fs.Debugf(nil, "Failed to load %q: %v", keyPath, err) + // If loading a cached key failed, make the keys and retry + err = os.MkdirAll(cachePath, 0700) + if err != nil { + return errors.Wrap(err, "failed to create cache path") + } + const bits = 2048 + fs.Logf(nil, "Generating %d bit key pair at %q", bits, keyPath) + err = makeSSHKeyPair(bits, keyPath+".pub", keyPath) + if err != nil { + return errors.Wrap(err, "failed to create SSH key pair") + } + // reload the new keys + private, err = loadPrivateKey(keyPath) + } + if err != nil { + return err + } + fs.Debugf(nil, "Loaded private key from %q", keyPath) + + s.config.AddHostKey(private) + + // Once a ServerConfig has been configured, connections can be + // accepted. + s.listener, err = net.Listen("tcp", s.opt.ListenAddr) + if err != nil { + log.Fatal("failed to listen for connection", err) + } + fs.Logf(nil, "SFTP server listening on %v\n", s.listener.Addr()) + + s.handlers, err = newVFSHandler(s.vfs) + if err != nil { + return errors.Wrap(err, "serve sftp: failed to create fs") + } + + go s.acceptConnections() + + return nil +} + +// Addr returns the address the server is listening on +func (s *server) Addr() string { + return s.listener.Addr().String() +} + +// Serve runs the sftp server in the background. +// +// Use s.Close() and s.Wait() to shutdown server +func (s *server) Serve() error { + err := s.serve() + if err != nil { + return err + } + return nil +} + +// Wait blocks while the listener is open. +func (s *server) Wait() { + <-s.waitChan +} + +// Close shuts the running server down +func (s *server) Close() { + err := s.listener.Close() + if err != nil { + fs.Errorf(nil, "Error on closing SFTP server: %v", err) + return + } + close(s.waitChan) +} + +func loadPrivateKey(keyPath string) (ssh.Signer, error) { + privateBytes, err := ioutil.ReadFile(keyPath) + if err != nil { + return nil, errors.Wrap(err, "failed to load private key") + } + private, err := ssh.ParsePrivateKey(privateBytes) + if err != nil { + return nil, errors.Wrap(err, "failed to parse private key") + } + return private, nil +} + +// Public key authentication is done by comparing +// the public key of a received connection +// with the entries in the authorized_keys file. +func loadAuthorizedKeys(authorizedKeysPath string) (authorizedKeysMap map[string]struct{}, err error) { + authorizedKeysBytes, err := ioutil.ReadFile(authorizedKeysPath) + if err != nil { + return nil, errors.Wrap(err, "failed to load authorized keys") + } + authorizedKeysMap = make(map[string]struct{}) + for len(authorizedKeysBytes) > 0 { + pubKey, _, _, rest, err := ssh.ParseAuthorizedKey(authorizedKeysBytes) + if err != nil { + return nil, errors.Wrap(err, "failed to parse authorized keys") + } + authorizedKeysMap[string(pubKey.Marshal())] = struct{}{} + authorizedKeysBytes = bytes.TrimSpace(rest) + } + return authorizedKeysMap, nil +} + +// makeSSHKeyPair make a pair of public and private keys for SSH access. +// Public key is encoded in the format for inclusion in an OpenSSH authorized_keys file. +// Private Key generated is PEM encoded +// +// Originally from: https://stackoverflow.com/a/34347463/164234 +func makeSSHKeyPair(bits int, pubKeyPath, privateKeyPath string) (err error) { + privateKey, err := rsa.GenerateKey(rand.Reader, bits) + if err != nil { + return err + } + + // generate and write private key as PEM + privateKeyFile, err := os.OpenFile(privateKeyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) + if err != nil { + return err + } + defer fs.CheckClose(privateKeyFile, &err) + privateKeyPEM := &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(privateKey)} + if err := pem.Encode(privateKeyFile, privateKeyPEM); err != nil { + return err + } + + // generate and write public key + pub, err := ssh.NewPublicKey(&privateKey.PublicKey) + if err != nil { + return err + } + return ioutil.WriteFile(pubKeyPath, ssh.MarshalAuthorizedKey(pub), 0644) +} diff --git a/cmd/serve/sftp/sftp.go b/cmd/serve/sftp/sftp.go new file mode 100644 index 000000000..c594faafb --- /dev/null +++ b/cmd/serve/sftp/sftp.go @@ -0,0 +1,101 @@ +// Package sftp implements an SFTP server to serve an rclone VFS + +// +build !plan9 + +package sftp + +import ( + "github.com/ncw/rclone/cmd" + "github.com/ncw/rclone/fs/config/flags" + "github.com/ncw/rclone/fs/rc" + "github.com/ncw/rclone/vfs" + "github.com/ncw/rclone/vfs/vfsflags" + "github.com/spf13/cobra" + "github.com/spf13/pflag" +) + +// Options contains options for the http Server +type Options struct { + ListenAddr string // Port to listen on + Key string // Path to private key + AuthorizedKeys string // Path to authorized keys file + User string // single username + Pass string // password for user + NoAuth bool // allow no authentication on connections +} + +// DefaultOpt is the default values used for Options +var DefaultOpt = Options{ + ListenAddr: "localhost:2022", + AuthorizedKeys: "~/.ssh/authorized_keys", +} + +// Opt is options set by command line flags +var Opt = DefaultOpt + +// AddFlags adds flags for the sftp +func AddFlags(flagSet *pflag.FlagSet, Opt *Options) { + rc.AddOption("sftp", &Opt) + flags.StringVarP(flagSet, &Opt.ListenAddr, "addr", "", Opt.ListenAddr, "IPaddress:Port or :Port to bind server to.") + flags.StringVarP(flagSet, &Opt.Key, "key", "", Opt.Key, "SSH private key file (leave blank to auto generate)") + flags.StringVarP(flagSet, &Opt.AuthorizedKeys, "authorized-keys", "", Opt.AuthorizedKeys, "Authorized keys file") + flags.StringVarP(flagSet, &Opt.User, "user", "", Opt.User, "User name for authentication.") + flags.StringVarP(flagSet, &Opt.Pass, "pass", "", Opt.Pass, "Password for authentication.") + flags.BoolVarP(flagSet, &Opt.NoAuth, "no-auth", "", Opt.NoAuth, "Allow connections with no authentication if set.") +} + +func init() { + vfsflags.AddFlags(Command.Flags()) + AddFlags(Command.Flags(), &Opt) +} + +// Command definition for cobra +var Command = &cobra.Command{ + Use: "sftp remote:path", + Short: `Serve the remote over SFTP.`, + Long: `rclone serve sftp implements an SFTP server to serve the remote +over SFTP. This can be used with an SFTP client or you can make a +remote of type sftp to use with it. + +You can use the filter flags (eg --include, --exclude) to control what +is served. + +The server will log errors. Use -v to see access logs. + +--bwlimit will be respected for file transfers. Use --stats to +control the stats printing. + +You must provide some means of authentication, either with --user/--pass, +an authorized keys file (specify location with --authorized-keys - the +default is the same as ssh) or set the --no-auth flag for no +authentication when logging in. + +Note that this also implements a small number of shell commands so +that it can provide md5sum/sha1sum/df information for the rclone sftp +backend. This means that is can support SHA1SUMs, MD5SUMs and the +about command when paired with the rclone sftp backend. + +If you don't supply a --key then rclone will generate one and cache it +for later use. + +By default the server binds to localhost:2022 - if you want it to be +reachable externally then supply "--addr :2022" for example. + +Note that the default of "--vfs-cache-mode off" is fine for the rclone +sftp backend, but it may not be with other SFTP clients. + +` + vfs.Help, + Run: func(command *cobra.Command, args []string) { + cmd.CheckArgs(1, 1, command, args) + f := cmd.NewFsSrc(args) + cmd.Run(false, true, command, func() error { + s := newServer(f, &Opt) + err := s.Serve() + if err != nil { + return err + } + s.Wait() + return nil + }) + }, +} diff --git a/cmd/serve/sftp/sftp_test.go b/cmd/serve/sftp/sftp_test.go new file mode 100644 index 000000000..2ee9b9f83 --- /dev/null +++ b/cmd/serve/sftp/sftp_test.go @@ -0,0 +1,94 @@ +// Serve sftp tests set up a server and run the integration tests +// for the sftp remote against it. +// +// We skip tests on platforms with troublesome character mappings + +//+build !windows,!darwin,!plan9 + +package sftp + +import ( + "os" + "os/exec" + "strings" + "testing" + + _ "github.com/ncw/rclone/backend/local" + "github.com/ncw/rclone/fs/config/obscure" + "github.com/ncw/rclone/fstest" + "github.com/pkg/sftp" + "github.com/stretchr/testify/assert" +) + +const ( + testBindAddress = "localhost:0" + testUser = "testuser" + testPass = "testpass" +) + +// check interfaces +var ( + _ sftp.FileReader = vfsHandler{} + _ sftp.FileWriter = vfsHandler{} + _ sftp.FileCmder = vfsHandler{} + _ sftp.FileLister = vfsHandler{} +) + +// TestSftp runs the sftp server then runs the unit tests for the +// sftp remote against it. +func TestSftp(t *testing.T) { + fstest.Initialise() + + fremote, _, clean, err := fstest.RandomRemote(*fstest.RemoteName, *fstest.SubDir) + assert.NoError(t, err) + defer clean() + + err = fremote.Mkdir("") + assert.NoError(t, err) + + opt := DefaultOpt + opt.ListenAddr = testBindAddress + opt.User = testUser + opt.Pass = testPass + + // Start the server + w := newServer(fremote, &opt) + assert.NoError(t, w.serve()) + defer func() { + w.Close() + w.Wait() + }() + + // Change directory to run the tests + err = os.Chdir("../../../backend/sftp") + assert.NoError(t, err, "failed to cd to sftp backend") + + // Run the sftp tests with an on the fly remote + args := []string{"test"} + if testing.Verbose() { + args = append(args, "-v") + } + if *fstest.Verbose { + args = append(args, "-verbose") + } + args = append(args, "-remote", "sftptest:") + cmd := exec.Command("go", args...) + addr := w.Addr() + colon := strings.LastIndex(addr, ":") + if colon < 0 { + panic("need a : in the address: " + addr) + } + host, port := addr[:colon], addr[colon+1:] + cmd.Env = append(os.Environ(), + "RCLONE_CONFIG_SFTPTEST_TYPE=sftp", + "RCLONE_CONFIG_SFTPTEST_HOST="+host, + "RCLONE_CONFIG_SFTPTEST_PORT="+port, + "RCLONE_CONFIG_SFTPTEST_USER="+testUser, + "RCLONE_CONFIG_SFTPTEST_PASS="+obscure.MustObscure(testPass), + ) + out, err := cmd.CombinedOutput() + if len(out) != 0 { + t.Logf("\n----------\n%s----------\n", string(out)) + } + assert.NoError(t, err, "Running sftp integration tests") +} diff --git a/cmd/serve/sftp/sftp_unsupported.go b/cmd/serve/sftp/sftp_unsupported.go new file mode 100644 index 000000000..f0126b027 --- /dev/null +++ b/cmd/serve/sftp/sftp_unsupported.go @@ -0,0 +1,11 @@ +// Build for sftp for unsupported platforms to stop go complaining +// about "no buildable Go source files " + +// +build plan9 + +package sftp + +import "github.com/spf13/cobra" + +// Command definition is nil to show not implemented +var Command *cobra.Command = nil