dropbox: Update dropbox to use the v2 API #349

This is feature complete with the old version but now supports modification times.

docs/content/dropbox.md

@@ -93,18 +93,14 @@ To copy a local directory to a dropbox directory called backup
 
 ### Modified time and MD5SUMs ###
 
-Dropbox doesn't provide the ability to set modification times in the
+Dropbox supports modified times, but the only way to set a
-V1 public API, so rclone can't support modified time with Dropbox.
+modification time is to re-upload the file.
 
-This may change in the future - see these issues for details:
+This means that if you uploaded your data with an older version of
-
+rclone which didn't support the v2 API and modified times, rclone will
-  * [Dropbox V2 API](https://github.com/ncw/rclone/issues/349)
+decide to upload all your old data to fix the modification times.  If
-  * [Allow syncs for remotes that can't set modtime on existing objects](https://github.com/ncw/rclone/issues/348)
+you don't want this to happen use `--size-only` or `--checksum` flag
-
+to stop it.
-Dropbox doesn't return any sort of checksum (MD5 or SHA1).
-
-Together that means that syncs to dropbox will effectively have the
-`--size-only` flag set.
 
 ### Specific options ###
 

docs/content/overview.md

@@ -20,7 +20,7 @@ Here is an overview of the major features of each cloud storage system.
 | Google Drive           | MD5     | Yes     | No               | Yes             | R/W       |
 | Amazon S3              | MD5     | Yes     | No               | No              | R/W       |
 | Openstack Swift        | MD5     | Yes     | No               | No              | R/W       |
-| Dropbox                | -       | No      | Yes              | No              | R         |
+| Dropbox                | -       | Yes     | Yes              | No              | -         |
 | Google Cloud Storage   | MD5     | Yes     | No               | No              | R/W       |
 | Amazon Drive           | MD5     | No      | Yes              | No              | R         |
 | Microsoft OneDrive     | SHA1    | Yes     | Yes              | No              | R         |

dropbox/dropbox.go

@@ -1,38 +1,61 @@
 // Package dropbox provides an interface to Dropbox object storage
 package dropbox
 
-/*
+// FIXME put low level retries in
-Limitations of dropbox
+// FIXME add dropbox style hashes
+// FIXME dropbox for business would be quite easy to add
 
-File system is case insensitive
+/*
+FIXME is case folding of PathDisplay going to cause a problem?
+
+From the docs
+
+path_display String. The cased path to be used for display purposes
+only. In rare instances the casing will not correctly match the user's
+filesystem, but this behavior will match the path provided in the Core
+API v1, and at least the last path component will have the correct
+casing. Changes to only the casing of paths won't be returned by
+list_folder/continue. This field will be null if the file or folder is
+not mounted. This field is optional.
 */
 
 import (
 	"crypto/md5"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
-	"net/http"
 	"path"
 	"regexp"
 	"strings"
 	"time"
 
+	"golang.org/x/oauth2"
+
+	"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox"
+	"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/files"
 	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/oauthutil"
 	"github.com/pkg/errors"
-	"github.com/stacktic/dropbox"
 )
 
 // Constants
 const (
-	rcloneAppKey             = "5jcck7diasz0rqy"
+	rcloneClientID              = "5jcck7diasz0rqy"
-	rcloneEncryptedAppSecret = "fRS5vVLr2v6FbyXYnIgjwBuUAt0osq_QZTXAEcmZ7g"
+	rcloneEncryptedClientSecret = "fRS5vVLr2v6FbyXYnIgjwBuUAt0osq_QZTXAEcmZ7g"
-	metadataLimit            = dropbox.MetadataLimitDefault // max items to fetch at once
 )
 
 var (
+	// Description of how to auth for this app
+	dropboxConfig = &oauth2.Config{
+		Scopes: []string{},
+		Endpoint: oauth2.Endpoint{
+			AuthURL:  "https://www.dropbox.com/1/oauth2/authorize",
+			TokenURL: "https://api.dropboxapi.com/1/oauth2/token",
+		}, // FIXME replace with this once vendored dropbox.OAuthEndpoint(""),
+		ClientID:     rcloneClientID,
+		ClientSecret: fs.MustReveal(rcloneEncryptedClientSecret),
+		RedirectURL:  oauthutil.RedirectLocalhostURL,
+	}
 	// A regexp matching path names for files Dropbox ignores
 	// See https://www.dropbox.com/en/help/145 - Ignored files
 	ignoredFiles = regexp.MustCompile(`(?i)(^|/)(desktop\.ini|thumbs\.db|\.ds_store|icon\r|\.dropbox|\.dropbox.attr)$`)
@@ -48,7 +71,12 @@ func init() {
 		Name:        "dropbox",
 		Description: "Dropbox",
 		NewFs:       NewFs,
-		Config:      configHelper,
+		Config: func(name string) {
+			err := oauthutil.Config("dropbox", name, dropboxConfig)
+			if err != nil {
+				log.Fatalf("Failed to configure token: %v", err)
+			}
+		},
 		Options: []fs.Option{{
 			Name: "app_key",
 			Help: "Dropbox App Key - leave blank normally.",
@@ -60,45 +88,12 @@ func init() {
 	fs.VarP(&uploadChunkSize, "dropbox-chunk-size", "", fmt.Sprintf("Upload chunk size. Max %v.", maxUploadChunkSize))
 }
 
-// Configuration helper - called after the user has put in the defaults
-func configHelper(name string) {
-	// See if already have a token
-	token := fs.ConfigFileGet(name, "token")
-	if token != "" {
-		fmt.Printf("Already have a dropbox token - refresh?\n")
-		if !fs.Confirm() {
-			return
-		}
-	}
-
-	// Get a dropbox
-	db, err := newDropbox(name)
-	if err != nil {
-		log.Fatalf("Failed to create dropbox client: %v", err)
-	}
-
-	// This method will ask the user to visit an URL and paste the generated code.
-	if err := db.Auth(); err != nil {
-		log.Fatalf("Failed to authorize: %v", err)
-	}
-
-	// Get the token
-	token = db.AccessToken()
-
-	// Stuff it in the config file if it has changed
-	old := fs.ConfigFileGet(name, "token")
-	if token != old {
-		fs.ConfigFileSet(name, "token", token)
-		fs.SaveConfig()
-	}
-}
-
 // Fs represents a remote dropbox server
 type Fs struct {
 	name           string       // name of this remote
 	root           string       // the path we are working on
 	features       *fs.Features // optional features
-	db             *dropbox.Dropbox // the connection to the dropbox server
+	srv            files.Client // the connection to the dropbox server
 	slashRoot      string       // root with "/" prefix, lowercase
 	slashRootSlash string       // root with "/" prefix and postfix, lowercase
 }
@@ -110,7 +105,6 @@ type Object struct {
 	bytes       int64     // size of the object
 	modTime     time.Time // time it was last modified
 	hasMetadata bool      // metadata is valid
-	mimeType    string    // content type according to the server
 }
 
 // ------------------------------------------------------------
@@ -135,51 +129,45 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// Makes a new dropbox from the config
-func newDropbox(name string) (*dropbox.Dropbox, error) {
-	db := dropbox.NewDropbox()
-
-	appKey := fs.ConfigFileGet(name, "app_key")
-	if appKey == "" {
-		appKey = rcloneAppKey
-	}
-	appSecret := fs.ConfigFileGet(name, "app_secret")
-	if appSecret == "" {
-		appSecret = fs.MustReveal(rcloneEncryptedAppSecret)
-	}
-
-	err := db.SetAppInfo(appKey, appSecret)
-	return db, err
-}
-
 // NewFs contstructs an Fs from the path, container:path
 func NewFs(name, root string) (fs.Fs, error) {
 	if uploadChunkSize > maxUploadChunkSize {
 		return nil, errors.Errorf("chunk size too big, must be < %v", maxUploadChunkSize)
 	}
-	db, err := newDropbox(name)
+
+	// Convert the old token if it exists.  The old token was just
+	// just a string, the new one is a JSON blob
+	oldToken := strings.TrimSpace(fs.ConfigFileGet(name, fs.ConfigToken))
+	if oldToken != "" && oldToken[0] != '{' {
+		fs.Infof(name, "Converting token to new format")
+		newToken := fmt.Sprintf(`{"access_token":"%s","token_type":"bearer","expiry":"0001-01-01T00:00:00Z"}`, oldToken)
+		err := fs.ConfigSetValueAndSave(name, fs.ConfigToken, newToken)
 		if err != nil {
-		return nil, err
+			return nil, errors.Wrap(err, "NewFS convert token")
 		}
+	}
+
+	oAuthClient, _, err := oauthutil.NewClient(name, dropboxConfig)
+	if err != nil {
+		log.Fatalf("Failed to configure dropbox: %v", err)
+	}
+
+	config := dropbox.Config{
+		Verbose: false,       // enables verbose logging in the SDK
+		Client:  oAuthClient, // maybe???
+	}
+	srv := files.New(config)
+
 	f := &Fs{
 		name: name,
-		db:   db,
+		srv:  srv,
 	}
 	f.features = (&fs.Features{CaseInsensitive: true, ReadMimeType: true}).Fill(f)
 	f.setRoot(root)
 
-	// Read the token from the config file
-	token := fs.ConfigFileGet(name, "token")
-
-	// Set our custom context which enables our custom transport for timeouts etc
-	db.SetContext(oauthutil.Context())
-
-	// Authorize the client
-	db.SetAccessToken(token)
-
 	// See if the root is actually an object
-	entry, err := f.db.Metadata(f.slashRoot, false, false, "", "", metadataLimit)
+	_, err = f.getFileMetadata(f.slashRoot)
-	if err == nil && !entry.IsDir {
+	if err == nil {
 		newRoot := path.Dir(f.root)
 		if newRoot == "." {
 			newRoot = ""
@@ -188,7 +176,6 @@ func NewFs(name, root string) (fs.Fs, error) {
 		// return an error with an fs which points to the parent
 		return f, fs.ErrorIsFile
 	}
-
 	return f, nil
 }
 
@@ -204,10 +191,58 @@ func (f *Fs) setRoot(root string) {
 	}
 }
 
+// getMetadata gets the metadata for a file or directory
+func (f *Fs) getMetadata(objPath string) (entry files.IsMetadata, notFound bool, err error) {
+	entry, err = f.srv.GetMetadata(&files.GetMetadataArg{Path: objPath})
+	if err != nil {
+		switch e := err.(type) {
+		case files.GetMetadataAPIError:
+			switch e.EndpointError.Path.Tag {
+			case files.LookupErrorNotFound:
+				notFound = true
+				err = nil
+			}
+		}
+	}
+	return
+}
+
+// getFileMetadata gets the metadata for a file
+func (f *Fs) getFileMetadata(filePath string) (fileInfo *files.FileMetadata, err error) {
+	entry, notFound, err := f.getMetadata(filePath)
+	if err != nil {
+		return nil, err
+	}
+	if notFound {
+		return nil, fs.ErrorObjectNotFound
+	}
+	fileInfo, ok := entry.(*files.FileMetadata)
+	if !ok {
+		return nil, fs.ErrorNotAFile
+	}
+	return fileInfo, nil
+}
+
+// getDirMetadata gets the metadata for a directory
+func (f *Fs) getDirMetadata(dirPath string) (dirInfo *files.FolderMetadata, err error) {
+	entry, notFound, err := f.getMetadata(dirPath)
+	if err != nil {
+		return nil, err
+	}
+	if notFound {
+		return nil, fs.ErrorDirNotFound
+	}
+	dirInfo, ok := entry.(*files.FolderMetadata)
+	if !ok {
+		return nil, fs.ErrorIsFile
+	}
+	return dirInfo, nil
+}
+
 // Return an Object from a path
 //
 // If it can't be found it returns the error fs.ErrorObjectNotFound.
-func (f *Fs) newObjectWithInfo(remote string, info *dropbox.Entry) (fs.Object, error) {
+func (f *Fs) newObjectWithInfo(remote string, info *files.FileMetadata) (fs.Object, error) {
 	o := &Object{
 		fs:     f,
 		remote: remote,
@@ -254,56 +289,67 @@ func (f *Fs) stripRoot(path string) (string, error) {
 }
 
 // Walk the root returning a channel of Objects
-func (f *Fs) list(out fs.ListOpts, dir string) {
+func (f *Fs) list(out fs.ListOpts, dir string, recursive bool) {
-	// Track path component case, it could be different for entries coming from DropBox API
-	// See https://www.dropboxforum.com/hc/communities/public/questions/201665409-Wrong-character-case-of-folder-name-when-calling-listFolder-using-Sync-API?locale=en-us
-	// and https://github.com/ncw/rclone/issues/53
-	nameTree := newNameTree()
-	cursor := ""
 	root := f.slashRoot
 	if dir != "" {
 		root += "/" + dir
-		// We assume that dir is entered in the correct case
-		// here which is likely since it probably came from a
-		// directory listing
-		nameTree.PutCaseCorrectPath(strings.Trim(root, "/"))
 	}
+
+	started := false
+	var res *files.ListFolderResult
+	var err error
 	for {
-		deltaPage, err := f.db.Delta(cursor, root)
+		if !started {
+			arg := files.ListFolderArg{
+				Path:      root,
+				Recursive: recursive,
+			}
+			if root == "/" {
+				arg.Path = "" // Specify root folder as empty string
+			}
+			res, err = f.srv.ListFolder(&arg)
 			if err != nil {
-			out.SetError(errors.Wrap(err, "couldn't list"))
+				switch e := err.(type) {
+				case files.ListFolderAPIError:
+					switch e.EndpointError.Path.Tag {
+					case files.LookupErrorNotFound:
+						err = fs.ErrorDirNotFound
+					}
+				}
+				out.SetError(err)
 				return
 			}
-		if deltaPage.Reset && cursor != "" {
+			started = false
-			err = errors.New("unexpected reset during listing")
-			out.SetError(err)
-			break
-		}
-		fs.Debugf(f, "%d delta entries received", len(deltaPage.Entries))
-		for i := range deltaPage.Entries {
-			deltaEntry := &deltaPage.Entries[i]
-			entry := deltaEntry.Entry
-			if entry == nil {
-				// This notifies of a deleted object
 		} else {
-				if len(entry.Path) <= 1 || entry.Path[0] != '/' {
+			arg := files.ListFolderContinueArg{
-					fs.Debugf(f, "dropbox API inconsistency: a path should always start with a slash and be at least 2 characters: %s", entry.Path)
+				Cursor: res.Cursor,
+			}
+			res, err = f.srv.ListFolderContinue(&arg)
+			if err != nil {
+				out.SetError(errors.Wrap(err, "list continue"))
+				return
+			}
+		}
+		for _, entry := range res.Entries {
+			var fileInfo *files.FileMetadata
+			var folderInfo *files.FolderMetadata
+			var metadata *files.Metadata
+			switch info := entry.(type) {
+			case *files.FolderMetadata:
+				folderInfo = info
+				metadata = &info.Metadata
+			case *files.FileMetadata:
+				fileInfo = info
+				metadata = &info.Metadata
+			default:
+				fs.Errorf(f, "Unknown type %T", entry)
 				continue
 			}
 
-				lastSlashIndex := strings.LastIndex(entry.Path, "/")
+			entryPath := metadata.PathDisplay // FIXME  PathLower
 
-				var parentPath string
+			if folderInfo != nil {
-				if lastSlashIndex == 0 {
+				name, err := f.stripRoot(entryPath + "/")
-					parentPath = ""
-				} else {
-					parentPath = entry.Path[1:lastSlashIndex]
-				}
-				lastComponent := entry.Path[lastSlashIndex+1:]
-
-				if entry.IsDir {
-					nameTree.PutCaseCorrectDirectoryName(parentPath, lastComponent)
-					name, err := f.stripRoot(entry.Path + "/")
 				if err != nil {
 					out.SetError(err)
 					return
@@ -312,23 +358,22 @@ func (f *Fs) list(out fs.ListOpts, dir string) {
 				if name != "" && name != dir {
 					dir := &fs.Dir{
 						Name: name,
-							When:  time.Time(entry.ClientMtime),
+						When: time.Now(),
-							Bytes: entry.Bytes,
+						//When:  folderInfo.ClientMtime,
-							Count: -1,
+						//Bytes: folderInfo.Bytes,
+						//Count: -1,
 					}
 					if out.AddDir(dir) {
 						return
 					}
 				}
-				} else {
+			} else if fileInfo != nil {
-					parentPathCorrectCase := nameTree.GetPathWithCorrectCase(parentPath)
+				path, err := f.stripRoot(entryPath)
-					if parentPathCorrectCase != nil {
-						path, err := f.stripRoot(*parentPathCorrectCase + "/" + lastComponent)
 				if err != nil {
 					out.SetError(err)
 					return
 				}
-						o, err := f.newObjectWithInfo(path, entry)
+				o, err := f.newObjectWithInfo(path, fileInfo)
 				if err != nil {
 					out.SetError(err)
 					return
@@ -336,74 +381,11 @@ func (f *Fs) list(out fs.ListOpts, dir string) {
 				if out.Add(o) {
 					return
 				}
-					} else {
-						nameTree.PutFile(parentPath, lastComponent, entry)
 			}
 		}
-			}
+		if !res.HasMore {
-		}
-		if !deltaPage.HasMore {
 			break
 		}
-		cursor = deltaPage.Cursor.Cursor
-
-	}
-
-	walkFunc := func(caseCorrectFilePath string, entry *dropbox.Entry) error {
-		path, err := f.stripRoot("/" + caseCorrectFilePath)
-		if err != nil {
-			return err
-		}
-		o, err := f.newObjectWithInfo(path, entry)
-		if err != nil {
-			return err
-		}
-		if out.Add(o) {
-			return fs.ErrorListAborted
-		}
-		return nil
-	}
-	err := nameTree.WalkFiles(f.root, walkFunc)
-	if err != nil {
-		out.SetError(err)
-	}
-}
-
-// listOneLevel walks the path one level deep
-func (f *Fs) listOneLevel(out fs.ListOpts, dir string) {
-	root := f.root
-	if dir != "" {
-		root += "/" + dir
-	}
-	dirEntry, err := f.db.Metadata(root, true, false, "", "", metadataLimit)
-	if err != nil {
-		out.SetError(errors.Wrap(err, "couldn't list single level"))
-		return
-	}
-	for i := range dirEntry.Contents {
-		entry := &dirEntry.Contents[i]
-		// Normalise the path to the dir passed in
-		remote := path.Join(dir, path.Base(entry.Path))
-		if entry.IsDir {
-			dir := &fs.Dir{
-				Name:  remote,
-				When:  time.Time(entry.ClientMtime),
-				Bytes: entry.Bytes,
-				Count: -1,
-			}
-			if out.AddDir(dir) {
-				return
-			}
-		} else {
-			o, err := f.newObjectWithInfo(remote, entry)
-			if err != nil {
-				out.SetError(err)
-				return
-			}
-			if out.Add(o) {
-				return
-			}
-		}
 	}
 }
 
@@ -413,9 +395,9 @@ func (f *Fs) List(out fs.ListOpts, dir string) {
 	level := out.Level()
 	switch level {
 	case 1:
-		f.listOneLevel(out, dir)
+		f.list(out, dir, false)
 	case fs.MaxLevel:
-		f.list(out, dir)
+		f.list(out, dir, true)
 	default:
 		out.SetError(fs.ErrorLevelNotSupported)
 	}
@@ -453,14 +435,25 @@ func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) {
 // Mkdir creates the container if it doesn't exist
 func (f *Fs) Mkdir(dir string) error {
 	root := path.Join(f.slashRoot, dir)
-	entry, err := f.db.Metadata(root, false, false, "", "", metadataLimit)
+
-	if err == nil {
+	// can't create or run metadata on root
-		if entry.IsDir {
+	if root == "/" {
 		return nil
 	}
-		return errors.Errorf("%q already exists as file", f.root)
+
+	// check directory doesn't exist
+	_, err := f.getDirMetadata(root)
+	if err == nil {
+		return nil // directory exists already
+	} else if err != fs.ErrorDirNotFound {
+		return err // some other error
 	}
-	_, err = f.db.CreateFolder(root)
+
+	// create it
+	arg2 := files.CreateFolderArg{
+		Path: root,
+	}
+	_, err = f.srv.CreateFolder(&arg2)
 	return err
 }
 
@@ -469,20 +462,42 @@ func (f *Fs) Mkdir(dir string) error {
 // Returns an error if it isn't empty
 func (f *Fs) Rmdir(dir string) error {
 	root := path.Join(f.slashRoot, dir)
-	entry, err := f.db.Metadata(root, true, false, "", "", 16)
+
-	if err != nil {
+	// can't remove root
-		return err
+	if root == "/" {
+		return errors.New("can't remove root directory")
 	}
-	if len(entry.Contents) != 0 {
+
+	// check directory exists
+	_, err := f.getDirMetadata(root)
+	if err != nil {
+		return errors.Wrap(err, "Rmdir")
+	}
+
+	// check directory empty
+	arg := files.ListFolderArg{
+		Path:      root,
+		Recursive: false,
+	}
+	if root == "/" {
+		arg.Path = "" // Specify root folder as empty string
+	}
+	res, err := f.srv.ListFolder(&arg)
+	if err != nil {
+		return errors.Wrap(err, "Rmdir")
+	}
+	if len(res.Entries) != 0 {
 		return errors.New("directory not empty")
 	}
-	_, err = f.db.Delete(root)
+
+	// remove it
+	_, err = f.srv.Delete(&files.DeleteArg{Path: root})
 	return err
 }
 
 // Precision returns the precision
 func (f *Fs) Precision() time.Duration {
-	return fs.ModTimeNotSupported
+	return time.Second
 }
 
 // Copy src to this remote using server side copy operations.
@@ -507,16 +522,25 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
 		remote: remote,
 	}
 
-	srcPath := srcObj.remotePath()
+	// Copy
-	dstPath := dstObj.remotePath()
+	arg := files.RelocationArg{}
-	entry, err := f.db.Copy(srcPath, dstPath, false)
+	arg.FromPath = srcObj.remotePath()
+	arg.ToPath = dstObj.remotePath()
+	entry, err := f.srv.Copy(&arg)
 	if err != nil {
 		return nil, errors.Wrap(err, "copy failed")
 	}
-	err = dstObj.setMetadataFromEntry(entry)
+
+	// Set the metadata
+	fileInfo, ok := entry.(*files.FileMetadata)
+	if !ok {
+		return nil, fs.ErrorNotAFile
+	}
+	err = dstObj.setMetadataFromEntry(fileInfo)
 	if err != nil {
 		return nil, errors.Wrap(err, "copy failed")
 	}
+
 	return dstObj, nil
 }
 
@@ -527,7 +551,7 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
 // result of List()
 func (f *Fs) Purge() error {
 	// Let dropbox delete the filesystem tree
-	_, err := f.db.Delete(f.slashRoot)
+	_, err := f.srv.Delete(&files.DeleteArg{Path: f.slashRoot})
 	return err
 }
 
@@ -553,13 +577,21 @@ func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
 		remote: remote,
 	}
 
-	srcPath := srcObj.remotePath()
+	// Do the move
-	dstPath := dstObj.remotePath()
+	arg := files.RelocationArg{}
-	entry, err := f.db.Move(srcPath, dstPath)
+	arg.FromPath = srcObj.remotePath()
+	arg.ToPath = dstObj.remotePath()
+	entry, err := f.srv.Move(&arg)
 	if err != nil {
 		return nil, errors.Wrap(err, "move failed")
 	}
-	err = dstObj.setMetadataFromEntry(entry)
+
+	// Set the metadata
+	fileInfo, ok := entry.(*files.FileMetadata)
+	if !ok {
+		return nil, fs.ErrorNotAFile
+	}
+	err = dstObj.setMetadataFromEntry(fileInfo)
 	if err != nil {
 		return nil, errors.Wrap(err, "move failed")
 	}
@@ -584,19 +616,25 @@ func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
 	dstPath := path.Join(f.slashRoot, dstRemote)
 
 	// Check if destination exists
-	entry, err := f.db.Metadata(f.slashRoot, false, false, "", "", metadataLimit)
+	_, err := f.getDirMetadata(f.slashRoot)
-	if err == nil && !entry.IsDeleted {
+	if err == nil {
 		return fs.ErrorDirExists
+	} else if err != fs.ErrorDirNotFound {
+		return err
 	}
 
 	// Make sure the parent directory exists
 	// ...apparently not necessary
 
 	// Do the move
-	_, err = f.db.Move(srcPath, dstPath)
+	arg := files.RelocationArg{}
+	arg.FromPath = srcPath
+	arg.ToPath = dstPath
+	_, err = f.srv.Move(&arg)
 	if err != nil {
 		return errors.Wrap(err, "MoveDir failed")
 	}
+
 	return nil
 }
 
@@ -635,32 +673,19 @@ func (o *Object) Size() int64 {
 	return o.bytes
 }
 
-// setMetadataFromEntry sets the fs data from a dropbox.Entry
+// setMetadataFromEntry sets the fs data from a files.FileMetadata
 //
 // This isn't a complete set of metadata and has an inacurate date
-func (o *Object) setMetadataFromEntry(info *dropbox.Entry) error {
+func (o *Object) setMetadataFromEntry(info *files.FileMetadata) error {
-	if info.IsDir {
+	o.bytes = int64(info.Size)
-		return errors.Wrapf(fs.ErrorNotAFile, "%q", o.remote)
+	o.modTime = info.ClientModified
-	}
-	o.bytes = info.Bytes
-	o.modTime = time.Time(info.ClientMtime)
-	o.mimeType = info.MimeType
 	o.hasMetadata = true
 	return nil
 }
 
-// Reads the entry from dropbox
+// Reads the entry for a file from dropbox
-func (o *Object) readEntry() (*dropbox.Entry, error) {
+func (o *Object) readEntry() (*files.FileMetadata, error) {
-	entry, err := o.fs.db.Metadata(o.remotePath(), false, false, "", "", metadataLimit)
+	return o.fs.getFileMetadata(o.remotePath())
-	if err != nil {
-		if dropboxErr, ok := err.(*dropbox.Error); ok {
-			if dropboxErr.StatusCode == http.StatusNotFound {
-				return nil, fs.ErrorObjectNotFound
-			}
-		}
-		return nil, err
-	}
-	return entry, nil
 }
 
 // Read entry if not set and set metadata from it
@@ -721,7 +746,9 @@ func (o *Object) ModTime() time.Time {
 //
 // Commits the datastore
 func (o *Object) SetModTime(modTime time.Time) error {
-	// FIXME not implemented
+	// Dropbox doesn't have a way of doing this so returning this
+	// error will cause the file to be re-uploaded to set the
+	// time.
 	return fs.ErrorCantSetModTime
 }
 
@@ -732,30 +759,69 @@ func (o *Object) Storable() bool {
 
 // Open an object for read
 func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
-	// FIXME should send a patch for dropbox module which allow setting headers
+	headers := fs.OpenOptionHeaders(options)
-	var offset int64
+	arg := files.DownloadArg{Path: o.remotePath(), ExtraHeaders: headers}
-	for _, option := range options {
+	_, in, err = o.fs.srv.Download(&arg)
-		switch x := option.(type) {
-		case *fs.SeekOption:
-			offset = x.Offset
-		default:
-			if option.Mandatory() {
-				fs.Logf(o, "Unsupported mandatory option: %v", option)
-			}
-		}
-	}
 
-	in, _, err = o.fs.db.Download(o.remotePath(), "", offset)
+	switch e := err.(type) {
-	if dropboxErr, ok := err.(*dropbox.Error); ok {
+	case files.DownloadAPIError:
-		// Dropbox return 461 for copyright violation so don't
+		// Don't attempt to retry copyright violation errors
-		// attempt to retry this error
+		if e.EndpointError.Path.Tag == files.LookupErrorRestrictedContent {
-		if dropboxErr.StatusCode == 461 {
 			return nil, fs.NoRetryError(err)
 		}
 	}
+
 	return
 }
 
+// uploadChunked uploads the object in parts
+//
+// Call only if size is >= uploadChunkSize
+//
+// FIXME rework for retries
+func (o *Object) uploadChunked(in io.Reader, commitInfo *files.CommitInfo, size int64) (entry *files.FileMetadata, err error) {
+	chunkSize := int64(uploadChunkSize)
+	chunks := int(size/chunkSize) + 1
+
+	// write the first whole chunk
+	fs.Debugf(o, "Uploading chunk 1/%d", chunks)
+	res, err := o.fs.srv.UploadSessionStart(&files.UploadSessionStartArg{}, &io.LimitedReader{R: in, N: chunkSize})
+	if err != nil {
+		return nil, err
+	}
+
+	cursor := files.UploadSessionCursor{
+		SessionId: res.SessionId,
+		Offset:    uint64(chunkSize),
+	}
+	appendArg := files.UploadSessionAppendArg{
+		Cursor: &cursor,
+		Close:  false,
+	}
+
+	// write more whole chunks (if any)
+	for i := 2; i < chunks; i++ {
+		fs.Debugf(o, "Uploading chunk %d/%d", i, chunks)
+		err = o.fs.srv.UploadSessionAppendV2(&appendArg, &io.LimitedReader{R: in, N: chunkSize})
+		if err != nil {
+			return nil, err
+		}
+		cursor.Offset += uint64(chunkSize)
+	}
+
+	// write the remains
+	args := &files.UploadSessionFinishArg{
+		Cursor: &cursor,
+		Commit: commitInfo,
+	}
+	fs.Debugf(o, "Uploading chunk %d/%d", chunks, chunks)
+	entry, err = o.fs.srv.UploadSessionFinish(args, in)
+	if err != nil {
+		return nil, err
+	}
+	return entry, nil
+}
+
 // Update the already existing object
 //
 // Copy the reader into the object updating modTime and size
@@ -767,7 +833,19 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error {
 		fs.Logf(o, "File name disallowed - not uploading")
 		return nil
 	}
-	entry, err := o.fs.db.UploadByChunk(ioutil.NopCloser(in), int(uploadChunkSize), remote, true, "")
+	commitInfo := files.NewCommitInfo(o.remotePath())
+	commitInfo.Mode.Tag = "overwrite"
+	// The Dropbox API only accepts timestamps in UTC with second precision.
+	commitInfo.ClientModified = src.ModTime().UTC().Round(time.Second)
+
+	size := src.Size()
+	var err error
+	var entry *files.FileMetadata
+	if size > int64(uploadChunkSize) {
+		entry, err = o.uploadChunked(in, commitInfo, size)
+	} else {
+		entry, err = o.fs.srv.Upload(commitInfo, in)
+	}
 	if err != nil {
 		return errors.Wrap(err, "upload failed")
 	}
@@ -776,20 +854,10 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error {
 
 // Remove an object
 func (o *Object) Remove() error {
-	_, err := o.fs.db.Delete(o.remotePath())
+	_, err := o.fs.srv.Delete(&files.DeleteArg{Path: o.remotePath()})
 	return err
 }
 
-// MimeType of an Object if known, "" otherwise
-func (o *Object) MimeType() string {
-	err := o.readMetaData()
-	if err != nil {
-		fs.Logf(o, "Failed to read metadata: %v", err)
-		return ""
-	}
-	return o.mimeType
-}
-
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs       = (*Fs)(nil)
@@ -798,5 +866,4 @@ var (
 	_ fs.Mover    = (*Fs)(nil)
 	_ fs.DirMover = (*Fs)(nil)
 	_ fs.Object   = (*Object)(nil)
-	_ fs.MimeTyper = (*Object)(nil)
 )

dropbox/nametree.go

@@ -1,211 +0,0 @@
-package dropbox
-
-import (
-	"bytes"
-	"fmt"
-	"strings"
-
-	"github.com/ncw/rclone/fs"
-	"github.com/stacktic/dropbox"
-)
-
-// FIXME Get rid of Stats.Error() counting and return errors
-
-type nameTreeNode struct {
-	// Map from lowercase directory name to tree node
-	Directories map[string]*nameTreeNode
-
-	// Map from file name (case sensitive) to dropbox entry
-	Files map[string]*dropbox.Entry
-
-	// Empty string if exact case is unknown or root node
-	CaseCorrectName string
-}
-
-// ------------------------------------------------------------
-
-func newNameTreeNode(caseCorrectName string) *nameTreeNode {
-	return &nameTreeNode{
-		CaseCorrectName: caseCorrectName,
-		Directories:     make(map[string]*nameTreeNode),
-		Files:           make(map[string]*dropbox.Entry),
-	}
-}
-
-func newNameTree() *nameTreeNode {
-	return newNameTreeNode("")
-}
-
-func (tree *nameTreeNode) String() string {
-	if len(tree.CaseCorrectName) == 0 {
-		return "nameTreeNode/<root>"
-	}
-	return fmt.Sprintf("nameTreeNode/%q", tree.CaseCorrectName)
-}
-
-func (tree *nameTreeNode) getTreeNode(path string) *nameTreeNode {
-	if len(path) == 0 {
-		// no lookup required, just return root
-		return tree
-	}
-	current := tree
-	for _, component := range strings.Split(path, "/") {
-		if len(component) == 0 {
-			fs.Stats.Error()
-			fs.Errorf(tree, "getTreeNode: path component is empty (full path %q)", path)
-			return nil
-		}
-
-		lowercase := strings.ToLower(component)
-
-		lookup := current.Directories[lowercase]
-		if lookup == nil {
-			lookup = newNameTreeNode("")
-			current.Directories[lowercase] = lookup
-		}
-
-		current = lookup
-	}
-
-	return current
-}
-
-// PutCaseCorrectPath puts a known good path into the nameTree
-func (tree *nameTreeNode) PutCaseCorrectPath(caseCorrectPath string) {
-	if len(caseCorrectPath) == 0 {
-		return
-	}
-	current := tree
-	for _, component := range strings.Split(caseCorrectPath, "/") {
-		if len(component) == 0 {
-			fs.Stats.Error()
-			fs.Errorf(tree, "PutCaseCorrectPath: path component is empty (full path %q)", caseCorrectPath)
-			return
-		}
-		lowercase := strings.ToLower(component)
-		lookup := current.Directories[lowercase]
-		if lookup == nil {
-			lookup = newNameTreeNode(component)
-			current.Directories[lowercase] = lookup
-		}
-		current = lookup
-	}
-	return
-}
-
-func (tree *nameTreeNode) PutCaseCorrectDirectoryName(parentPath string, caseCorrectDirectoryName string) {
-	if len(caseCorrectDirectoryName) == 0 {
-		fs.Stats.Error()
-		fs.Errorf(tree, "PutCaseCorrectDirectoryName: empty caseCorrectDirectoryName is not allowed (parentPath: %q)", parentPath)
-		return
-	}
-
-	node := tree.getTreeNode(parentPath)
-	if node == nil {
-		return
-	}
-
-	lowerCaseDirectoryName := strings.ToLower(caseCorrectDirectoryName)
-	directory := node.Directories[lowerCaseDirectoryName]
-	if directory == nil {
-		directory = newNameTreeNode(caseCorrectDirectoryName)
-		node.Directories[lowerCaseDirectoryName] = directory
-	} else {
-		if len(directory.CaseCorrectName) > 0 {
-			fs.Stats.Error()
-			fs.Errorf(tree, "PutCaseCorrectDirectoryName: directory %q is already exists under parent path %q", caseCorrectDirectoryName, parentPath)
-			return
-		}
-
-		directory.CaseCorrectName = caseCorrectDirectoryName
-	}
-}
-
-func (tree *nameTreeNode) PutFile(parentPath string, caseCorrectFileName string, dropboxEntry *dropbox.Entry) {
-	node := tree.getTreeNode(parentPath)
-	if node == nil {
-		return
-	}
-
-	if node.Files[caseCorrectFileName] != nil {
-		fs.Stats.Error()
-		fs.Errorf(tree, "PutFile: file %q is already exists at %q", caseCorrectFileName, parentPath)
-		return
-	}
-
-	node.Files[caseCorrectFileName] = dropboxEntry
-}
-
-func (tree *nameTreeNode) GetPathWithCorrectCase(path string) *string {
-	if path == "" {
-		empty := ""
-		return &empty
-	}
-
-	var result bytes.Buffer
-
-	current := tree
-	for _, component := range strings.Split(path, "/") {
-		if component == "" {
-			fs.Stats.Error()
-			fs.Errorf(tree, "GetPathWithCorrectCase: path component is empty (full path %q)", path)
-			return nil
-		}
-
-		lowercase := strings.ToLower(component)
-
-		current = current.Directories[lowercase]
-		if current == nil || current.CaseCorrectName == "" {
-			return nil
-		}
-
-		_, _ = result.WriteString("/")
-		_, _ = result.WriteString(current.CaseCorrectName)
-	}
-
-	resultString := result.String()
-	return &resultString
-}
-
-type nameTreeFileWalkFunc func(caseCorrectFilePath string, entry *dropbox.Entry) error
-
-func (tree *nameTreeNode) walkFilesRec(currentPath string, walkFunc nameTreeFileWalkFunc) error {
-	var prefix string
-	if currentPath == "" {
-		prefix = ""
-	} else {
-		prefix = currentPath + "/"
-	}
-
-	for name, entry := range tree.Files {
-		err := walkFunc(prefix+name, entry)
-		if err != nil {
-			return err
-		}
-	}
-
-	for lowerCaseName, directory := range tree.Directories {
-		caseCorrectName := directory.CaseCorrectName
-		if caseCorrectName == "" {
-			fs.Stats.Error()
-			fs.Errorf(tree, "WalkFiles: exact name of the directory %q is unknown (parent path: %q)", lowerCaseName, currentPath)
-			continue
-		}
-
-		err := directory.walkFilesRec(prefix+caseCorrectName, walkFunc)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (tree *nameTreeNode) WalkFiles(rootPath string, walkFunc nameTreeFileWalkFunc) error {
-	node := tree.getTreeNode(rootPath)
-	if node == nil {
-		return nil
-	}
-
-	return node.walkFilesRec(rootPath, walkFunc)
-}

dropbox/nametree_test.go

@@ -1,140 +0,0 @@
-package dropbox
-
-import (
-	"testing"
-
-	"github.com/ncw/rclone/fs"
-	dropboxapi "github.com/stacktic/dropbox"
-	"github.com/stretchr/testify/assert"
-)
-
-func TestPutCaseCorrectDirectoryName(t *testing.T) {
-	errors := fs.Stats.GetErrors()
-
-	tree := newNameTree()
-	tree.PutCaseCorrectDirectoryName("a/b", "C")
-
-	assert.Equal(t, "", tree.CaseCorrectName, "Root CaseCorrectName should be empty")
-
-	a := tree.Directories["a"]
-	assert.Equal(t, "", a.CaseCorrectName, "CaseCorrectName at 'a' should be empty")
-
-	b := a.Directories["b"]
-	assert.Equal(t, "", b.CaseCorrectName, "CaseCorrectName at 'a/b' should be empty")
-
-	c := b.Directories["c"]
-	assert.Equal(t, "C", c.CaseCorrectName, "CaseCorrectName at 'a/b/c' should be 'C'")
-
-	assert.Equal(t, errors, fs.Stats.GetErrors(), "No errors should be reported")
-}
-
-func TestPutCaseCorrectPath(t *testing.T) {
-	errors := fs.Stats.GetErrors()
-
-	tree := newNameTree()
-	tree.PutCaseCorrectPath("A/b/C")
-
-	assert.Equal(t, "", tree.CaseCorrectName, "Root CaseCorrectName should be empty")
-
-	a := tree.Directories["a"]
-	assert.Equal(t, "A", a.CaseCorrectName, "CaseCorrectName at 'a' should be 'A'")
-
-	b := a.Directories["b"]
-	assert.Equal(t, "b", b.CaseCorrectName, "CaseCorrectName at 'a/b' should be 'b'")
-
-	c := b.Directories["c"]
-	assert.Equal(t, "C", c.CaseCorrectName, "CaseCorrectName at 'a/b/c' should be 'C'")
-
-	assert.Equal(t, errors, fs.Stats.GetErrors(), "No errors should be reported")
-}
-
-func TestPutCaseCorrectDirectoryNameEmptyComponent(t *testing.T) {
-	errors := fs.Stats.GetErrors()
-
-	tree := newNameTree()
-	tree.PutCaseCorrectDirectoryName("/a", "C")
-	tree.PutCaseCorrectDirectoryName("b/", "C")
-	tree.PutCaseCorrectDirectoryName("a//b", "C")
-
-	assert.True(t, fs.Stats.GetErrors() == errors+3, "3 errors should be reported")
-}
-
-func TestPutCaseCorrectDirectoryNameEmptyParent(t *testing.T) {
-	errors := fs.Stats.GetErrors()
-
-	tree := newNameTree()
-	tree.PutCaseCorrectDirectoryName("", "C")
-
-	c := tree.Directories["c"]
-	assert.True(t, c.CaseCorrectName == "C", "CaseCorrectName at 'c' should be 'C'")
-
-	assert.True(t, fs.Stats.GetErrors() == errors, "No errors should be reported")
-}
-
-func TestGetPathWithCorrectCase(t *testing.T) {
-	errors := fs.Stats.GetErrors()
-
-	tree := newNameTree()
-	tree.PutCaseCorrectDirectoryName("a", "C")
-	assert.True(t, tree.GetPathWithCorrectCase("a/c") == nil, "Path for 'a' should not be available")
-
-	tree.PutCaseCorrectDirectoryName("", "A")
-	assert.True(t, *tree.GetPathWithCorrectCase("a/c") == "/A/C", "Path for 'a/c' should be '/A/C'")
-
-	assert.True(t, fs.Stats.GetErrors() == errors, "No errors should be reported")
-}
-
-func TestPutAndWalk(t *testing.T) {
-	errors := fs.Stats.GetErrors()
-
-	tree := newNameTree()
-	tree.PutFile("a", "F", &dropboxapi.Entry{Path: "xxx"})
-	tree.PutCaseCorrectDirectoryName("", "A")
-
-	numCalled := 0
-	walkFunc := func(caseCorrectFilePath string, entry *dropboxapi.Entry) error {
-		assert.True(t, caseCorrectFilePath == "A/F", "caseCorrectFilePath should be A/F, not "+caseCorrectFilePath)
-		assert.True(t, entry.Path == "xxx", "entry.Path should be xxx")
-		numCalled++
-		return nil
-	}
-	err := tree.WalkFiles("", walkFunc)
-	assert.True(t, err == nil, "No error should be returned")
-	assert.True(t, numCalled == 1, "walk func should be called only once")
-	assert.True(t, fs.Stats.GetErrors() == errors, "No errors should be reported")
-}
-
-func TestPutAndWalkWithPrefix(t *testing.T) {
-	errors := fs.Stats.GetErrors()
-
-	tree := newNameTree()
-	tree.PutFile("a", "F", &dropboxapi.Entry{Path: "xxx"})
-	tree.PutCaseCorrectDirectoryName("", "A")
-
-	numCalled := 0
-	walkFunc := func(caseCorrectFilePath string, entry *dropboxapi.Entry) error {
-		assert.True(t, caseCorrectFilePath == "A/F", "caseCorrectFilePath should be A/F, not "+caseCorrectFilePath)
-		assert.True(t, entry.Path == "xxx", "entry.Path should be xxx")
-		numCalled++
-		return nil
-	}
-	err := tree.WalkFiles("A", walkFunc)
-	assert.True(t, err == nil, "No error should be returned")
-	assert.True(t, numCalled == 1, "walk func should be called only once")
-	assert.True(t, fs.Stats.GetErrors() == errors, "No errors should be reported")
-}
-
-func TestPutAndWalkIncompleteTree(t *testing.T) {
-	errors := fs.Stats.GetErrors()
-
-	tree := newNameTree()
-	tree.PutFile("a", "F", &dropboxapi.Entry{Path: "xxx"})
-
-	walkFunc := func(caseCorrectFilePath string, entry *dropboxapi.Entry) error {
-		t.Fatal("Should not be called")
-		return nil
-	}
-	err := tree.WalkFiles("", walkFunc)
-	assert.True(t, err == nil, "No error should be returned")
-	assert.True(t, fs.Stats.GetErrors() == errors+1, "One error should be reported")
-}