diff --git a/fs/all/all.go b/fs/all/all.go
index 06016df2b..986339e3a 100644
--- a/fs/all/all.go
+++ b/fs/all/all.go
@@ -12,6 +12,7 @@ import (
 	_ "github.com/ncw/rclone/local"
 	_ "github.com/ncw/rclone/onedrive"
 	_ "github.com/ncw/rclone/s3"
+	_ "github.com/ncw/rclone/sftp"
 	_ "github.com/ncw/rclone/swift"
 	_ "github.com/ncw/rclone/yandex"
 )
diff --git a/sftp/sftp.go b/sftp/sftp.go
new file mode 100644
index 000000000..bb97c5758
--- /dev/null
+++ b/sftp/sftp.go
@@ -0,0 +1,421 @@
+// Package sftp provides a filesystem interface using github.com/pkg/sftp
+package sftp
+
+import (
+	"io"
+	"net"
+	"os"
+	"path"
+	"path/filepath"
+	"sync"
+	"time"
+
+	"golang.org/x/crypto/ssh"
+	"golang.org/x/crypto/ssh/agent"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/pkg/sftp"
+)
+
+func init() {
+	fsi := &fs.RegInfo{
+		Name:        "sftp",
+		Description: "SSH/SFTP Connection",
+		NewFs:       NewFs,
+		Options: []fs.Option{{
+			Name:     "host",
+			Help:     "SSH host to connect to",
+			Optional: false,
+			Examples: []fs.OptionExample{{
+				Value: "example.com",
+				Help:  "Connect to example.com",
+			}},
+		}, {
+			Name:     "user",
+			Help:     "SSH username, leave blank for current username, " + os.Getenv("USER"),
+			Optional: true,
+		}, {
+			Name:     "port",
+			Help:     "SSH port",
+			Optional: true,
+		}, {
+			Name:       "pass",
+			Help:       "SSH password, leave blank to use ssh-agent",
+			Optional:   true,
+			IsPassword: true,
+		}},
+	}
+	fs.Register(fsi)
+}
+
+type stringer interface {
+	String() string
+}
+
+func debug(o stringer, msg string) {
+	fs.Debug(o, msg)
+}
+
+// Fs stores the interface to the remote SFTP files
+type Fs struct {
+	name       string
+	root       string
+	features   *fs.Features // optional features
+	url        string
+	sshClient  *ssh.Client
+	sftpClient *sftp.Client
+}
+
+// Object is a remote SFTP file that has been stat'd (so it exists, but is not necessarily open for reading)
+type Object struct {
+	fs     *Fs
+	remote string
+	info   os.FileInfo
+}
+
+// ObjectReader holds the sftp.File interface to a remote SFTP file opened for reading
+type ObjectReader struct {
+	object   *Object
+	sftpFile *sftp.File
+}
+
+// NewFs creates a new Fs object from the name and root. It connects to
+// the host specified in the config file.
+func NewFs(name, root string) (fs.Fs, error) {
+	user := fs.ConfigFileGet(name, "user")
+	host := fs.ConfigFileGet(name, "host")
+	port := fs.ConfigFileGet(name, "port")
+	pass := fs.ConfigFileGet(name, "pass")
+	if root == "" {
+		root = "."
+	}
+	if user == "" {
+		user = os.Getenv("USER")
+	}
+	if port == "" {
+		port = "22"
+	}
+	config := &ssh.ClientConfig{
+		User: user,
+		Auth: []ssh.AuthMethod{},
+	}
+	if pass == "" {
+		if sshAgent, err := net.Dial("unix", os.Getenv("SSH_AUTH_SOCK")); err == nil {
+			sshAgentClient := agent.NewClient(sshAgent)
+			signers, _ := sshAgentClient.Signers()
+			for i, signer := range signers {
+				if 2*i < len(signers) {
+					signers[i] = signers[len(signers)-i-1]
+					signers[len(signers)-i-1] = signer
+				}
+			}
+			config.Auth = append(config.Auth, ssh.PublicKeys(signers...))
+		}
+	} else {
+		clearpass, err := fs.Reveal(pass)
+		if err != nil {
+			return nil, err
+		}
+		config.Auth = append(config.Auth, ssh.Password(clearpass))
+	}
+	if sshClient, err := ssh.Dial("tcp", host+":"+port, config); err != nil {
+		return nil, err
+	} else if sftpClient, err := sftp.NewClient(sshClient); err != nil {
+		_ = sshClient.Close()
+		return nil, err
+	} else {
+		f := &Fs{
+			name:       name,
+			root:       root,
+			sshClient:  sshClient,
+			sftpClient: sftpClient,
+			url:        "sftp://" + user + "@" + host + ":" + port + "/" + root,
+		}
+		f.features = (&fs.Features{}).Fill(f)
+		return f, nil
+	}
+}
+
+// Name returns the configured name of the file system
+func (f *Fs) Name() string {
+	return f.name
+}
+
+// Root returns the root for the filesystem
+func (f *Fs) Root() string {
+	return f.root
+}
+
+// String returns the URL for the filesystem
+func (f *Fs) String() string {
+	return f.url
+}
+
+// Features returns the optional features of this Fs
+func (f *Fs) Features() *fs.Features {
+	return f.features
+}
+
+// Precision is the remote sftp file system's modtime precision, which we have no way of knowing. We estimate at 1s
+func (f *Fs) Precision() time.Duration {
+	return time.Second
+}
+
+// NewObject creates a new remote sftp file object
+func (f *Fs) NewObject(remote string) (fs.Object, error) {
+	debug(f, "New '"+remote+"'")
+	info, err := f.sftpClient.Stat(f.sftpClient.Join(f.root, remote))
+	if err != nil {
+		return nil, err
+	}
+	object := &Object{
+		fs:     f,
+		remote: remote,
+		info:   info,
+	}
+	return object, nil
+}
+
+func (f *Fs) list(out fs.ListOpts, dirs string, name string, info os.FileInfo, level int, done *sync.WaitGroup) {
+	debug(f, "list '"+f.sftpClient.Join(dirs, name)+"'")
+	defer done.Done()
+	if info.IsDir() {
+		if out.IncludeDirectory(info.Name()) {
+			dir := &fs.Dir{
+				Name:  info.Name(),
+				When:  info.ModTime(),
+				Bytes: -1,
+				Count: -1,
+			}
+			if level >= out.Level() {
+				return
+			}
+			if infos, err := f.sftpClient.ReadDir(f.sftpClient.Join(f.root, dirs, name)); err == nil {
+				dir.Count = int64(len(infos))
+				out.AddDir(dir)
+				done.Add(len(infos))
+				for _, newInfo := range infos {
+					go f.list(out, f.sftpClient.Join(dirs, name), newInfo.Name(), newInfo, level+1, done)
+				}
+			}
+		}
+	} else {
+		file := &Object{
+			fs:     f,
+			remote: f.sftpClient.Join(dirs, info.Name()),
+			info:   info,
+		}
+		out.Add(file)
+	}
+}
+
+// List the files and directories starting at <dir>
+func (f *Fs) List(out fs.ListOpts, dir string) {
+	debug(f, "List '"+dir+"'")
+	if dir == "" {
+		dir = "."
+	}
+	var done sync.WaitGroup
+	if info, _ := f.sftpClient.Stat(f.sftpClient.Join(f.root, dir)); info != nil {
+		done.Add(1)
+		f.list(out, "", ".", info, 0, &done)
+	}
+	debug(f, "List--waiting")
+	done.Wait()
+	out.Finished()
+}
+
+// Put data from <in> into a new remote sftp file object described by <src.Remote()> and <src.ModTime()>
+func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) {
+	debug(f, "Put '"+src.Remote()+"'")
+	_ = f.mkdir(f.sftpClient.Join(f.root, filepath.Dir(src.Remote())))
+	file, err := f.sftpClient.Create(f.sftpClient.Join(f.root, src.Remote()))
+	if err != nil {
+		return nil, err
+	}
+	_, err = file.ReadFrom(in)
+	if err != nil {
+		return nil, err
+	}
+	o, err := f.NewObject(src.Remote())
+	if err != nil {
+		return nil, err
+	}
+	err = o.SetModTime(src.ModTime())
+	if err != nil {
+		return nil, err
+	}
+	return o, nil
+}
+
+func (f *Fs) mkdir(path string) error {
+	debug(f, "mkdir '"+path+"'")
+	parent := filepath.Dir(path)
+	if parent != "." && parent != "/" {
+		_ = f.mkdir(parent)
+	}
+	return f.sftpClient.Mkdir(path)
+}
+
+// Mkdir makes the root directory of the Fs object
+func (f *Fs) Mkdir(dir string) error {
+	root := path.Join(f.root, dir)
+	debug(f, "Mkdir '"+root+"'")
+	o, _ := f.NewObject("")
+	if o == nil {
+		return f.mkdir(root)
+	}
+	return nil
+}
+
+// Rmdir removes the root directory of the Fs object
+func (f *Fs) Rmdir(dir string) error {
+	root := path.Join(f.root, dir)
+	debug(f, "Rmdir '"+root+"'")
+	return f.sftpClient.Remove(f.root)
+}
+
+// Move renames a remote sftp file object
+func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
+	debug(f, "Move '"+src.Remote()+"' to '"+remote+"'")
+	err := f.sftpClient.Rename(
+		f.sftpClient.Join(f.root, src.Remote()),
+		f.sftpClient.Join(f.root, remote))
+	if err != nil {
+		return nil, err
+	}
+	dstObj, err := f.NewObject(remote)
+	return dstObj, err
+}
+
+// Hashes returns fs.HashNone to indicate remote hashing is unavailable
+func (f *Fs) Hashes() fs.HashSet {
+	return fs.HashSet(fs.HashNone)
+}
+
+// Fs is the filesystem this remote sftp file object is located within
+func (o *Object) Fs() fs.Info {
+	return o.fs
+}
+
+// String returns the URL to the remote SFTP file
+func (o *Object) String() string {
+	if o == nil {
+		return "<nil>"
+	}
+	return o.fs.url + "/" + o.remote
+}
+
+// Remote the name of the remote SFTP file, relative to the fs root
+func (o *Object) Remote() string {
+	return o.remote
+}
+
+// Hash returns "" since SFTP (in Go or OpenSSH) doesn't support remote calculation of hashes
+func (o *Object) Hash(r fs.HashType) (string, error) {
+	debug(o.fs, "Hash '"+o.remote+"'")
+	return "", fs.ErrHashUnsupported
+}
+
+// Size returns the size in bytes of the remote sftp file
+func (o *Object) Size() int64 {
+	debug(o.fs, "Size '"+o.remote+"'")
+	return o.info.Size()
+}
+
+// ModTime returns the modification time of the remote sftp file
+func (o *Object) ModTime() time.Time {
+	debug(o.fs, "ModTime '"+o.remote+"'")
+	return o.info.ModTime()
+}
+
+// SetModTime sets the modification and access time to the specified time
+func (o *Object) SetModTime(modTime time.Time) error {
+	debug(o.fs, "SetModTime '"+o.remote+"'")
+	err := o.fs.sftpClient.Chtimes(o.fs.sftpClient.Join(o.fs.root, o.remote), modTime, modTime)
+	if err != nil {
+		return err
+	}
+	o.info, err = o.fs.sftpClient.Stat(o.fs.sftpClient.Join(o.fs.root, o.remote))
+	return err
+}
+
+// Storable returns whether the remote sftp file is a regular file (not a directory, symbolic link, block device, character device, named pipe, etc)
+func (o *Object) Storable() bool {
+	debug(o.fs, "Storable '"+o.remote+"'?")
+	return o.info.Mode().IsRegular()
+}
+
+// Read from a remote sftp file object reader
+func (file *ObjectReader) Read(p []byte) (n int, err error) {
+	debug(file.object.fs, "Read '"+file.object.remote+"'")
+	n, err = file.sftpFile.Read(p)
+	return n, err
+}
+
+// Close a reader of a remote sftp file
+func (file *ObjectReader) Close() (err error) {
+	debug(file.object.fs, "Close '"+file.object.remote+"'")
+	err = file.sftpFile.Close()
+	return err
+}
+
+// Open a remote sftp file object for reading. Seek is supported
+func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
+	debug(o.fs, "Open '"+o.remote+"'")
+	var offset int64
+	offset = 0
+	for _, option := range options {
+		switch x := option.(type) {
+		case *fs.SeekOption:
+			offset = x.Offset
+		default:
+			if option.Mandatory() {
+				fs.Log(o, "Unsupported mandatory option: %v", option)
+			}
+		}
+	}
+	sftpFile, err := o.fs.sftpClient.Open(o.fs.sftpClient.Join(o.fs.root, o.remote))
+	if err != nil {
+		return nil, err
+	}
+	if offset > 0 {
+		off, err := sftpFile.Seek(offset, 0)
+		if err != nil || off != offset {
+			return nil, err
+		}
+	}
+	in = &ObjectReader{
+		object:   o,
+		sftpFile: sftpFile,
+	}
+	return in, nil
+}
+
+// Update a remote sftp file using the data <in> and ModTime from <src>
+func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error {
+	debug(o.fs, "Update '"+o.remote+"'")
+	file, err := o.fs.sftpClient.Create(o.fs.sftpClient.Join(o.fs.root, o.remote))
+	if err == nil {
+		_, err = file.ReadFrom(in)
+		if err != nil {
+			return err
+		}
+		err = o.SetModTime(src.ModTime())
+		return err
+	}
+	return err
+}
+
+// Remove a remote sftp file object
+func (o *Object) Remove() error {
+	debug(o.fs, "Remove '"+o.remote+"'")
+	return o.fs.sftpClient.Remove(o.fs.sftpClient.Join(o.fs.root, o.remote))
+}
+
+// Check the interfaces are satisfied
+var (
+	_ fs.Fs     = &Fs{}
+	_ fs.Mover  = &Fs{}
+	_ fs.Object = &Object{}
+)