ftp: fix remaining issues to make tests work

* fix root
  * factor ftpConnection
  * fix path munging
  * fix recursive dir loops after update
  * use fs.Trace and comment out debugs
  * re-arrange and supplement docs

README.md

@@ -24,8 +24,8 @@ Rclone is a command line program to sync files and directories to and from
   * Backblaze B2
   * Yandex Disk
   * SFTP
-  * The local filesystem
   * FTP
+  * The local filesystem
 
 Features
 

docs/content/about.md

@@ -24,8 +24,8 @@ Rclone is a command line program to sync files and directories to and from
   * Backblaze B2
   * Yandex Disk
   * SFTP
-  * The local filesystem
   * FTP
+  * The local filesystem
 
 Features
 

docs/content/ftp.md

@@ -7,29 +7,117 @@ date: "2017-01-01"
 <i class="fa fa-file"></i> FTP
 ------------------------------
 
-FTP support is provided via
+FTP is the File Transfer Protocl. FTP support is provided using the
 [github.com/jlaffaye/ftp](https://godoc.org/github.com/jlaffaye/ftp)
 package.
 
-### Configuration ###
+Here is an example of making an FTP configuration.  First run
 
-An Ftp backend only needs an Url and and username and password. With
+    rclone config
+
+This will guide you through an interactive setup process. An FTP
+backend only needs an URL and and username and password. With
 anonymous FTP server you will need to use `anonymous` as username and
 your email address as password.
 
-Example:
 ```
+No remotes found - make a new one
+n) New remote
+r) Rename remote
+c) Copy remote
+s) Set configuration password
+q) Quit config
+n/r/c/s/q> n
+name> remote
+Type of storage to configure.
+Choose a number from below, or type in your own value
+ 1 / Amazon Drive
+   \ "amazon cloud drive"
+ 2 / Amazon S3 (also Dreamhost, Ceph, Minio)
+   \ "s3"
+ 3 / Backblaze B2
+   \ "b2"
+ 4 / Dropbox
+   \ "dropbox"
+ 5 / Encrypt/Decrypt a remote
+   \ "crypt"
+ 6 / FTP interface
+   \ "ftp"
+ 7 / Google Cloud Storage (this is not Google Drive)
+   \ "google cloud storage"
+ 8 / Google Drive
+   \ "drive"
+ 9 / Hubic
+   \ "hubic"
+10 / Local Disk
+   \ "local"
+11 / Microsoft OneDrive
+   \ "onedrive"
+12 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH)
+   \ "swift"
+13 / SSH/SFTP Connection
+   \ "sftp"
+14 / Yandex Disk
+   \ "yandex"
+Storage> ftp
+Username
+username> anonymous
+Password
+y) Yes type in my own password
+g) Generate random password
+y/g> y
+Enter the password:
+password:
+Confirm the password:
+password:
+FTP URL
+url> ftp://ftp.mirrorservice.org/
+Remote config
+--------------------
 [remote]
-type = Ftp
 username = anonymous
-password = john.snow@example.org
+password = *** ENCRYPTED ***
-url = ftp://ftp.kernel.org/pub
+url = ftp://ftp.mirrorservice.org/
+--------------------
+y) Yes this is OK
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
 ```
 
-### Unsupported features ###
+This remote is called `remote` and can now be used like this
 
-FTP backends does not support:
+See all directories in the home directory
 
-* Any hash mechanism
+    rclone lsd remote:
-* Modified Time
+
-* remote copy/move
+Make a new directory
+
+    rclone mkdir remote:path/to/directory
+
+List the contents of a directory
+
+    rclone ls remote:path/to/directory
+
+Sync `/home/local/directory` to the remote directory, deleting any
+excess files in the directory.
+
+    rclone sync /home/local/directory remote:directory
+
+### Modified time ###
+
+FTP does not support modified times.  Any times you see on the server
+will be time of upload.
+
+### Checksums ###
+
+FTP does not support any checksums.
+
+### Limitations ###
+
+Note that since FTP isn't HTTP based the following flags don't work
+with it: `--dump-headers`, `--dump-bodies`, `--dump-auth`
+
+Note that `--timeout` and `--contimeout` aren't supported.
+
+FTP could support server side move but doesn't yet.

docs/layouts/chrome/navbar.html

@@ -61,6 +61,7 @@
                     <li><a href="/local/"><i class="fa fa-file"></i> Local</a></li>
                     <li><a href="/yandex/"><i class="fa fa-space-shuttle"></i> Yandex Disk</a></li>
                     <li><a href="/sftp/"><i class="fa fa-server"></i> SFTP</a></li>
+                    <li><a href="/ftp/"><i class="fa fa-file"></i> FTP</a></li>
                     <li><a href="/crypt/"><i class="fa fa-lock"></i> Crypt (encrypts the above)</a></li>
                   </ul>
                 </li>

ftp/ftp.go

@@ -1,8 +1,13 @@
 // Package ftp interfaces with FTP servers
 package ftp
 
+// FIXME Mover and DirMover are possible using f.c.Rename
+// FIXME Should have a pool of connections rather than a global lock
+
 import (
 	"io"
+	"io/ioutil"
+	"net/textproto"
 	"net/url"
 	"path"
 	"strings"
@@ -49,6 +54,9 @@ type Fs struct {
 	c        *ftp.ServerConn // the connection to the FTP server
 	url      *url.URL
 	mu       sync.Mutex
+	user     string
+	pass     string
+	dialAddr string
 }
 
 // Object describes an FTP file
@@ -89,76 +97,124 @@ func (f *Fs) Features() *fs.Features {
 }
 
 // Open a new connection to the FTP server.
-func ftpConnection(name, root string) (*ftp.ServerConn, *url.URL, error) {
+func (f *Fs) ftpConnection() (*ftp.ServerConn, error) {
+	globalMux.Lock()
+	defer globalMux.Unlock()
+	fs.Debugf(f, "Connecting to FTP server")
+	c, err := ftp.DialTimeout(f.dialAddr, 30*time.Second)
+	if err != nil {
+		fs.Errorf(nil, "Error while Dialing %s: %s", f.dialAddr, err)
+		return nil, err
+	}
+	err = c.Login(f.user, f.pass)
+	if err != nil {
+		fs.Errorf(nil, "Error while Logging in into %s: %s", f.dialAddr, err)
+		return nil, err
+	}
+	return c, nil
+}
+
+// NewFs contstructs an Fs from the path, container:path
+func NewFs(name, root string) (ff fs.Fs, err error) {
+	// defer fs.Trace(nil, "name=%q, root=%q", name, root)("fs=%v, err=%v", &ff, &err)
 	URL := fs.ConfigFileGet(name, "url")
 	user := fs.ConfigFileGet(name, "username")
 	pass := fs.ConfigFileGet(name, "password")
-	pass, err := fs.Reveal(pass)
+	pass, err = fs.Reveal(pass)
 	if err != nil {
-		return nil, nil, errors.Wrap(err, "failed to decrypt password")
+		return nil, errors.Wrap(err, "NewFS decrypt password")
 	}
 	u, err := url.Parse(URL)
 	if err != nil {
-		return nil, nil, errors.Wrap(err, "open ftp connection url parse")
+		return nil, errors.Wrap(err, "NewFS URL parse")
 	}
-	u.Path = path.Join(u.Path, root)
+	urlPath := strings.Trim(u.Path, "/")
-	fs.Debugf(nil, "New ftp Connection with name %s and url %s (path %s)", name, u.String(), u.Path)
+	fullPath := root
-	globalMux.Lock()
+	if urlPath != "" && !strings.HasPrefix("/", root) {
-	defer globalMux.Unlock()
+		fullPath = path.Join(u.Path, root)
+	}
+	root = fullPath
 	dialAddr := u.Hostname()
 	if u.Port() != "" {
 		dialAddr += ":" + u.Port()
 	} else {
 		dialAddr += ":21"
 	}
-	c, err := ftp.DialTimeout(dialAddr, 30*time.Second)
+	f := &Fs{
-	if err != nil {
+		name:     name,
-		fs.Errorf(nil, "Error while Dialing %s: %s", dialAddr, err)
+		root:     root,
-		return nil, u, err
+		url:      u,
+		user:     user,
+		pass:     pass,
+		dialAddr: dialAddr,
 	}
-	err = c.Login(user, pass)
+	f.features = (&fs.Features{}).Fill(f)
-	if err != nil {
+	f.c, err = f.ftpConnection()
-		fs.Errorf(nil, "Error while Logging in into %s: %s", dialAddr, err)
-		return nil, u, err
-	}
-	return c, u, nil
-}
-
-// NewFs contstructs an Fs from the path, container:path
-func NewFs(name, root string) (fs.Fs, error) {
-	fs.Debugf(nil, "ENTER function 'NewFs' with name %s and root %s", name, root)
-	defer fs.Debugf(nil, "EXIT function 'NewFs'")
-	c, u, err := ftpConnection(name, root)
 	if err != nil {
 		return nil, err
 	}
-	f := &Fs{
+	if root != "" {
-		name: name,
+		// Check to see if the root actually an existing file
-		root: u.Path,
+		remote := path.Base(root)
-		c:    c,
+		f.root = path.Dir(root)
-		url:  u,
+		if f.root == "." {
-		mu:   sync.Mutex{},
+			f.root = ""
+		}
+		_, err := f.NewObject(remote)
+		if err != nil {
+			if err == fs.ErrorObjectNotFound || errors.Cause(err) == fs.ErrorNotAFile {
+				// File doesn't exist so return old f
+				f.root = root
+				return f, nil
+			}
+			return nil, err
+		}
+		// return an error with an fs which points to the parent
+		return f, fs.ErrorIsFile
 	}
-	f.features = (&fs.Features{}).Fill(f)
 	return f, err
 }
 
+// translateErrorFile turns FTP errors into rclone errors if possible for a file
+func translateErrorFile(err error) error {
+	switch errX := err.(type) {
+	case *textproto.Error:
+		switch errX.Code {
+		case ftp.StatusFileUnavailable:
+			err = fs.ErrorObjectNotFound
+		}
+	}
+	return err
+}
+
+// translateErrorDir turns FTP errors into rclone errors if possible for a directory
+func translateErrorDir(err error) error {
+	switch errX := err.(type) {
+	case *textproto.Error:
+		switch errX.Code {
+		case ftp.StatusFileUnavailable:
+			err = fs.ErrorDirNotFound
+		}
+	}
+	return err
+}
+
 // NewObject finds the Object at remote.  If it can't be found
 // it returns the error fs.ErrorObjectNotFound.
-func (f *Fs) NewObject(remote string) (fs.Object, error) {
+func (f *Fs) NewObject(remote string) (o fs.Object, err error) {
-	fs.Debugf(f, "ENTER function 'NewObject' called with remote %s", remote)
+	// defer fs.Trace(remote, "")("o=%v, err=%v", &o, &err)
-	defer fs.Debugf(f, "EXIT function 'NewObject'")
+	fullPath := path.Join(f.root, remote)
-	dir := path.Dir(remote)
+	dir := path.Dir(fullPath)
-	base := path.Base(remote)
+	base := path.Base(fullPath)
 
 	f.mu.Lock()
 	files, err := f.c.List(dir)
 	f.mu.Unlock()
 	if err != nil {
-		return nil, err
+		return nil, translateErrorFile(err)
 	}
-	for i := range files {
+	for i, file := range files {
-		if files[i].Name == base {
+		if file.Type != ftp.EntryTypeFolder && file.Name == base {
 			o := &Object{
 				fs:     f,
 				remote: remote,
@@ -177,13 +233,12 @@ func (f *Fs) NewObject(remote string) (fs.Object, error) {
 }
 
 func (f *Fs) list(out fs.ListOpts, dir string, curlevel int) {
-	fs.Debugf(f, "ENTER function 'list'")
+	// defer fs.Trace(dir, "curlevel=%d", curlevel)("")
-	defer fs.Debugf(f, "EXIT function 'list'")
 	f.mu.Lock()
 	files, err := f.c.List(path.Join(f.root, dir))
 	f.mu.Unlock()
 	if err != nil {
-		out.SetError(err)
+		out.SetError(translateErrorDir(err))
 		return
 	}
 	for i := range files {
@@ -191,6 +246,9 @@ func (f *Fs) list(out fs.ListOpts, dir string, curlevel int) {
 		newremote := path.Join(dir, object.Name)
 		switch object.Type {
 		case ftp.EntryTypeFolder:
+			if object.Name == "." || object.Name == ".." {
+				continue
+			}
 			if out.IncludeDirectory(newremote) {
 				d := &fs.Dir{
 					Name:  newremote,
@@ -234,8 +292,7 @@ func (f *Fs) list(out fs.ListOpts, dir string, curlevel int) {
 // Fses must support recursion levels of fs.MaxLevel and 1.
 // They may return ErrorLevelNotSupported otherwise.
 func (f *Fs) List(out fs.ListOpts, dir string) {
-	fs.Debugf(f, "ENTER function 'List' on directory '%s/%s'", f.root, dir)
+	// defer fs.Trace(dir, "")("")
-	defer fs.Debugf(f, "EXIT function 'List' for directory '%s/%s'", f.root, dir)
 	f.list(out, dir, 1)
 	out.Finished()
 }
@@ -256,7 +313,7 @@ func (f *Fs) Precision() time.Duration {
 // will return the object and the error, otherwise will return
 // nil and the error
 func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) {
-	fs.Debugf(f, "Trying to put file %s", src.Remote())
+	// fs.Debugf(f, "Trying to put file %s", src.Remote())
 	o := &Object{
 		fs:     f,
 		remote: src.Remote(),
@@ -266,9 +323,8 @@ func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) {
 }
 
 // getInfo reads the FileInfo for a path
-func (f *Fs) getInfo(remote string) (*FileInfo, error) {
+func (f *Fs) getInfo(remote string) (fi *FileInfo, err error) {
-	fs.Debugf(f, "ENTER function 'getInfo' on file %s", remote)
+	// defer fs.Trace(remote, "")("fi=%v, err=%v", &fi, &err)
-	defer fs.Debugf(f, "EXIT function 'getInfo'")
 	dir := path.Dir(remote)
 	base := path.Base(remote)
 
@@ -276,7 +332,7 @@ func (f *Fs) getInfo(remote string) (*FileInfo, error) {
 	files, err := f.c.List(dir)
 	f.mu.Unlock()
 	if err != nil {
-		return nil, err
+		return nil, translateErrorFile(err)
 	}
 
 	for i := range files {
@@ -295,32 +351,32 @@ func (f *Fs) getInfo(remote string) (*FileInfo, error) {
 
 func (f *Fs) mkdir(abspath string) error {
 	_, err := f.getInfo(abspath)
-	if err != nil {
+	if err == fs.ErrorObjectNotFound {
-		fs.Debugf(f, "Trying to create directory %s", abspath)
+		// fs.Debugf(f, "Trying to create directory %s", abspath)
 		f.mu.Lock()
-		err := f.c.MakeDir(abspath)
+		err = f.c.MakeDir(abspath)
 		f.mu.Unlock()
-		if err != nil {
-			return err
-		}
 	}
 	return err
 }
 
 // Mkdir creates the directory if it doesn't exist
-func (f *Fs) Mkdir(dir string) error {
+func (f *Fs) Mkdir(dir string) (err error) {
+	// defer fs.Trace(dir, "")("err=%v", &err)
 	// This actually works as mkdir -p
-	fs.Debugf(f, "ENTER function 'Mkdir' on '%s/%s'", f.root, dir)
-	defer fs.Debugf(f, "EXIT function 'Mkdir' on '%s/%s'", f.root, dir)
 	abspath := path.Join(f.root, dir)
 	tokens := strings.Split(abspath, "/")
 	curdir := ""
 	for i := range tokens {
-		curdir += "/" + tokens[i]
+		curdir += tokens[i]
+		if curdir == "" {
+			continue
+		}
 		err := f.mkdir(curdir)
 		if err != nil {
 			return err
 		}
+		curdir += "/"
 	}
 	return nil
 }
@@ -334,11 +390,11 @@ func (f *Fs) Rmdir(dir string) error {
 	files, err := f.c.List(path.Join(f.root, dir))
 	f.mu.Unlock()
 	if err != nil {
-		return errors.Wrap(err, "rmdir")
+		return translateErrorDir(err)
 	}
-	for i := range files {
+	for _, file := range files {
-		if files[i].Type == ftp.EntryTypeFolder {
+		if file.Type == ftp.EntryTypeFolder && file.Name != "." && file.Name != ".." {
-			err = f.Rmdir(path.Join(dir, files[i].Name))
+			err = f.Rmdir(path.Join(dir, file.Name))
 			if err != nil {
 				return errors.Wrap(err, "rmdir")
 			}
@@ -412,11 +468,21 @@ func (f *ftpReadCloser) Close() error {
 }
 
 // Open an object for read
-func (o *Object) Open(options ...fs.OpenOption) (io.ReadCloser, error) {
+func (o *Object) Open(options ...fs.OpenOption) (rc io.ReadCloser, err error) {
+	// defer fs.Trace(o, "")("rc=%v, err=%v", &rc, &err)
 	path := path.Join(o.fs.root, o.remote)
-	fs.Debugf(o.fs, "ENTER function 'Open' on file '%s' in root '%s'", o.remote, o.fs.root)
+	var offset int64
-	defer fs.Debugf(o.fs, "EXIT function 'Open' %s", path)
+	for _, option := range options {
-	c, _, err := ftpConnection(o.fs.name, o.fs.root)
+		switch x := option.(type) {
+		case *fs.SeekOption:
+			offset = x.Offset
+		default:
+			if option.Mandatory() {
+				fs.Logf(o, "Unsupported mandatory option: %v", option)
+			}
+		}
+	}
+	c, err := o.fs.ftpConnection()
 	if err != nil {
 		return nil, errors.Wrap(err, "open")
 	}
@@ -424,13 +490,22 @@ func (o *Object) Open(options ...fs.OpenOption) (io.ReadCloser, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "open")
 	}
-	return &ftpReadCloser{ReadCloser: fd, c: c}, nil
+	rc = &ftpReadCloser{ReadCloser: fd, c: c}
+	if offset != 0 {
+		_, err = io.CopyN(ioutil.Discard, fd, offset)
+		if err != nil {
+			_ = rc.Close()
+			return nil, errors.Wrap(err, "open skipping bytes")
+		}
+	}
+	return rc, nil
 }
 
 // makeAllDir creates the parent directories for the object
 func (o *Object) makeAllDir() error {
-	tokens := strings.Split(path.Dir(o.remote), "/")
+	dir, _ := path.Split(o.remote)
-	dir := ""
+	tokens := strings.Split(dir, "/")
+	dir = ""
 	for i := range tokens {
 		dir += tokens[i] + "/"
 		err := o.fs.Mkdir(dir)
@@ -450,29 +525,38 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error {
 	// Create all upper directory first...
 	err := o.makeAllDir()
 	if err != nil {
-		return errors.Wrap(err, "update")
+		return errors.Wrap(err, "update mkdir")
 	}
 	path := path.Join(o.fs.root, o.remote)
-	c, _, err := ftpConnection(o.fs.name, o.fs.root)
+	c, err := o.fs.ftpConnection()
 	if err != nil {
-		return errors.Wrap(err, "update")
+		return errors.Wrap(err, "update connect")
+	}
+	// remove the file if upload failed
+	remove := func() {
+		removeErr := o.Remove()
+		if removeErr != nil {
+			fs.Debugf(o, "Failed to remove: %v", removeErr)
+		} else {
+			fs.Debugf(o, "Removed after failed upload: %v", err)
+		}
 	}
 	err = c.Stor(path, in)
 	if err != nil {
-		return errors.Wrap(err, "update")
+		remove()
+		return errors.Wrap(err, "update stor")
 	}
 	o.info, err = o.fs.getInfo(path)
 	if err != nil {
-		return errors.Wrap(err, "update")
+		return errors.Wrap(err, "update getinfo")
 	}
 	return nil
 }
 
 // Remove an object
-func (o *Object) Remove() error {
+func (o *Object) Remove() (err error) {
+	// defer fs.Trace(o, "")("err=%v", &err)
 	path := path.Join(o.fs.root, o.remote)
-	fs.Debugf(o, "ENTER function 'Remove' for obejct at %s", path)
-	defer fs.Debugf(o, "EXIT function 'Remove' for obejct at %s", path)
 	// Check if it's a directory or a file
 	info, err := o.fs.getInfo(path)
 	if err != nil {