dropbox: record names coming from dropbox API, fixes #53 case insensitivity causes duplicated files

dropbox/dropbox.go

@@ -113,8 +113,8 @@ func configHelper(name string) {
 type FsDropbox struct {
 	db               *dropbox.Dropbox // the connection to the dropbox server
 	root             string           // the path we are working on
-	slashRoot        string           // root with "/" prefix
+	slashRoot        string           // root with "/" prefix, lowercase
-	slashRootSlash   string           // root with "/" prefix and postix
+	slashRootSlash   string           // root with "/" prefix and postfix, lowercase
 	datastoreManager *dropbox.DatastoreManager
 	datastore        *dropbox.Datastore
 	table            *dropbox.Table
@@ -196,9 +196,11 @@ func NewFs(name, root string) (fs.Fs, error) {
 // Sets root in f
 func (f *FsDropbox) setRoot(root string) {
 	f.root = strings.Trim(root, "/")
-	f.slashRoot = "/" + f.root
+	lowerCaseRoot := strings.ToLower(f.root)
+
+	f.slashRoot = "/" + lowerCaseRoot
 	f.slashRootSlash = f.slashRoot
-	if f.root != "" {
+	if lowerCaseRoot != "" {
 		f.slashRootSlash += "/"
 	}
 }
@@ -254,17 +256,26 @@ func (f *FsDropbox) NewFsObject(remote string) fs.Object {
 	return f.newFsObjectWithInfo(remote, nil)
 }
 
-// Strips the root off entry and returns it
+// Strips the root off path and returns it
-func (f *FsDropbox) stripRoot(entry *dropbox.Entry) string {
+func (f *FsDropbox) stripRoot(path string) *string {
-	path := entry.Path
+	lowercase := strings.ToLower(path)
-	if strings.HasPrefix(path, f.slashRootSlash) {
+
-		path = path[len(f.slashRootSlash):]
+	if !strings.HasPrefix(lowercase, f.slashRootSlash) {
+		fs.Stats.Error()
+		fs.Log(f, "Path '%s' is not under root '%s'", path, f.slashRootSlash)
+		return nil
 	}
-	return path
+
+	stripped := path[len(f.slashRootSlash):]
+	return &stripped
 }
 
 // Walk the root returning a channel of FsObjects
 func (f *FsDropbox) list(out fs.ObjectsChan) {
+	// Track path component case, it could be different for entries coming from DropBox API
+	// See https://www.dropboxforum.com/hc/communities/public/questions/201665409-Wrong-character-case-of-folder-name-when-calling-listFolder-using-Sync-API?locale=en-us
+	// and https://github.com/ncw/rclone/issues/53
+	nameTree := NewNameTree()
 	cursor := ""
 	for {
 		deltaPage, err := f.db.Delta(cursor, f.slashRoot)
@@ -291,13 +302,38 @@ func (f *FsDropbox) list(out fs.ObjectsChan) {
 						fs.Debug(f, "Failed to delete metadata for %q", deltaEntry.Path)
 						// Don't accumulate Error here
 					}
-
 				} else {
-					if entry.IsDir {
+					if len(entry.Path) <= 1 || entry.Path[0] != '/' {
-						// ignore directories
+						fs.Stats.Error()
+						fs.Log(f, "dropbox API inconsistency: a path should always start with a slash and be at least 2 characters: %s", entry.Path)
+						continue
+					}
+
+					lastSlashIndex := strings.LastIndex(entry.Path, "/")
+
+					var parentPath string
+					if lastSlashIndex == 0 {
+						parentPath = ""
 					} else {
-						path := f.stripRoot(entry)
+						parentPath = entry.Path[1:lastSlashIndex]
-						out <- f.newFsObjectWithInfo(path, entry)
+					}
+					lastComponent := entry.Path[lastSlashIndex+1:]
+
+					if entry.IsDir {
+						nameTree.PutCaseCorrectDirectoryName(parentPath, lastComponent)
+					} else {
+						parentPathCorrectCase := nameTree.GetPathWithCorrectCase(parentPath)
+						if parentPathCorrectCase != nil {
+							path := f.stripRoot(*parentPathCorrectCase + "/" + lastComponent)
+							if path == nil {
+								// an error occurred and logged by stripRoot
+								continue
+							}
+
+							out <- f.newFsObjectWithInfo(*path, entry)
+						} else {
+							nameTree.PutFile(parentPath, lastComponent, entry)
+						}
 					}
 				}
 			}
@@ -307,6 +343,17 @@ func (f *FsDropbox) list(out fs.ObjectsChan) {
 			cursor = deltaPage.Cursor.Cursor
 		}
 	}
+
+	walkFunc := func(caseCorrectFilePath string, entry *dropbox.Entry) {
+		path := f.stripRoot("/" + caseCorrectFilePath)
+		if path == nil {
+			// an error occurred and logged by stripRoot
+			return
+		}
+
+		out <- f.newFsObjectWithInfo(*path, entry)
+	}
+	nameTree.WalkFiles(f.root, walkFunc)
 }
 
 // Walk the path returning a channel of FsObjects
@@ -332,8 +379,14 @@ func (f *FsDropbox) ListDir() fs.DirChan {
 			for i := range entry.Contents {
 				entry := &entry.Contents[i]
 				if entry.IsDir {
+					name := f.stripRoot(entry.Path)
+					if name == nil {
+						// an error occurred and logged by stripRoot
+						continue
+					}
+
 					out <- &fs.Dir{
-						Name:  f.stripRoot(entry),
+						Name:  *name,
 						When:  time.Time(entry.ClientMtime),
 						Bytes: int64(entry.Bytes),
 						Count: -1,

dropbox/nametree.go

@@ -0,0 +1,179 @@
+package dropbox
+
+import (
+	"bytes"
+	"fmt"
+	"github.com/ncw/rclone/fs"
+	"github.com/stacktic/dropbox"
+	"strings"
+)
+
+type NameTreeNode struct {
+	// Map from lowercase directory name to tree node
+	Directories map[string]*NameTreeNode
+
+	// Map from file name (case sensitive) to dropbox entry
+	Files map[string]*dropbox.Entry
+
+	// Empty string if exact case is unknown or root node
+	CaseCorrectName string
+}
+
+// ------------------------------------------------------------
+
+func newNameTreeNode(caseCorrectName string) *NameTreeNode {
+	return &NameTreeNode{
+		CaseCorrectName: caseCorrectName,
+		Directories:     make(map[string]*NameTreeNode),
+		Files:           make(map[string]*dropbox.Entry),
+	}
+}
+
+func NewNameTree() *NameTreeNode {
+	return newNameTreeNode("")
+}
+
+func (tree *NameTreeNode) String() string {
+	if len(tree.CaseCorrectName) == 0 {
+		return "NameTreeNode/<root>"
+	} else {
+		return fmt.Sprintf("NameTreeNode/%q", tree.CaseCorrectName)
+	}
+}
+
+func (tree *NameTreeNode) getTreeNode(path string) *NameTreeNode {
+	if len(path) == 0 {
+		// no lookup required, just return root
+		return tree
+	}
+
+	current := tree
+	for _, component := range strings.Split(path, "/") {
+		if len(component) == 0 {
+			fs.Stats.Error()
+			fs.Log(tree, "getTreeNode: path component is empty (full path %q)", path)
+			return nil
+		}
+
+		lowercase := strings.ToLower(component)
+
+		lookup := current.Directories[lowercase]
+		if lookup == nil {
+			lookup = newNameTreeNode("")
+			current.Directories[lowercase] = lookup
+		}
+
+		current = lookup
+	}
+
+	return current
+}
+
+func (tree *NameTreeNode) PutCaseCorrectDirectoryName(parentPath string, caseCorrectDirectoryName string) {
+	if len(caseCorrectDirectoryName) == 0 {
+		fs.Stats.Error()
+		fs.Log(tree, "PutCaseCorrectDirectoryName: empty caseCorrectDirectoryName is not allowed (parentPath: %q)", parentPath)
+		return
+	}
+
+	node := tree.getTreeNode(parentPath)
+	if node == nil {
+		return
+	}
+
+	lowerCaseDirectoryName := strings.ToLower(caseCorrectDirectoryName)
+	directory := node.Directories[lowerCaseDirectoryName]
+	if directory == nil {
+		directory = newNameTreeNode(caseCorrectDirectoryName)
+		node.Directories[lowerCaseDirectoryName] = directory
+	} else {
+		if len(directory.CaseCorrectName) > 0 {
+			fs.Stats.Error()
+			fs.Log(tree, "PutCaseCorrectDirectoryName: directory %q is already exists under parent path %q", caseCorrectDirectoryName, parentPath)
+			return
+		}
+
+		directory.CaseCorrectName = caseCorrectDirectoryName
+	}
+}
+
+func (tree *NameTreeNode) PutFile(parentPath string, caseCorrectFileName string, dropboxEntry *dropbox.Entry) {
+	node := tree.getTreeNode(parentPath)
+	if node == nil {
+		return
+	}
+
+	if node.Files[caseCorrectFileName] != nil {
+		fs.Stats.Error()
+		fs.Log(tree, "PutFile: file %q is already exists at %q", caseCorrectFileName, parentPath)
+		return
+	}
+
+	node.Files[caseCorrectFileName] = dropboxEntry
+}
+
+func (tree *NameTreeNode) GetPathWithCorrectCase(path string) *string {
+	if path == "" {
+		empty := ""
+		return &empty
+	}
+
+	var result bytes.Buffer
+
+	current := tree
+	for _, component := range strings.Split(path, "/") {
+		if component == "" {
+			fs.Stats.Error()
+			fs.Log(tree, "GetPathWithCorrectCase: path component is empty (full path %q)", path)
+			return nil
+		}
+
+		lowercase := strings.ToLower(component)
+
+		current = current.Directories[lowercase]
+		if current == nil || current.CaseCorrectName == "" {
+			return nil
+		}
+
+		result.WriteString("/")
+		result.WriteString(current.CaseCorrectName)
+	}
+
+	resultString := result.String()
+	return &resultString
+}
+
+type NameTreeFileWalkFunc func(caseCorrectFilePath string, entry *dropbox.Entry)
+
+func (tree *NameTreeNode) walkFilesRec(currentPath string, walkFunc NameTreeFileWalkFunc) {
+	var prefix string
+	if currentPath == "" {
+		prefix = ""
+	} else {
+		prefix = currentPath + "/"
+	}
+
+	for name, entry := range tree.Files {
+		walkFunc(prefix+name, entry)
+	}
+
+	for lowerCaseName, directory := range tree.Directories {
+		caseCorrectName := directory.CaseCorrectName
+		if caseCorrectName == "" {
+			fs.Stats.Error()
+			fs.Log(tree, "WalkFiles: exact name of the directory %q is unknown (parent path: %q)", lowerCaseName, currentPath)
+			continue
+		}
+
+		directory.walkFilesRec(prefix+caseCorrectName, walkFunc)
+	}
+}
+
+func (tree *NameTreeNode) WalkFiles(rootPath string, walkFunc NameTreeFileWalkFunc) {
+	node := tree.getTreeNode(rootPath)
+	if node == nil {
+		return
+	}
+
+	node.walkFilesRec(rootPath, walkFunc)
+}

dropbox/nametree_test.go

@@ -0,0 +1,124 @@
+package dropbox_test
+
+import (
+	"github.com/ncw/rclone/dropbox"
+	"github.com/ncw/rclone/fs"
+	dropboxapi "github.com/stacktic/dropbox"
+	"testing"
+)
+
+func assert(t *testing.T, shouldBeTrue bool, failMessage string) {
+	if !shouldBeTrue {
+		t.Fatal(failMessage)
+	}
+}
+
+func TestPutCaseCorrectDirectoryName(t *testing.T) {
+	errors := fs.Stats.GetErrors()
+
+	tree := dropbox.NewNameTree()
+	tree.PutCaseCorrectDirectoryName("a/b", "C")
+
+	assert(t, tree.CaseCorrectName == "", "Root CaseCorrectName should be empty")
+
+	a := tree.Directories["a"]
+	assert(t, a.CaseCorrectName == "", "CaseCorrectName at 'a' should be empty")
+
+	b := a.Directories["b"]
+	assert(t, b.CaseCorrectName == "", "CaseCorrectName at 'a/b' should be empty")
+
+	c := b.Directories["c"]
+	assert(t, c.CaseCorrectName == "C", "CaseCorrectName at 'a/b/c' should be 'C'")
+
+	assert(t, fs.Stats.GetErrors() == errors, "No errors should be reported")
+}
+
+func TestPutCaseCorrectDirectoryNameEmptyComponent(t *testing.T) {
+	errors := fs.Stats.GetErrors()
+
+	tree := dropbox.NewNameTree()
+	tree.PutCaseCorrectDirectoryName("/a", "C")
+	tree.PutCaseCorrectDirectoryName("b/", "C")
+	tree.PutCaseCorrectDirectoryName("a//b", "C")
+
+	assert(t, fs.Stats.GetErrors() == errors+3, "3 errors should be reported")
+}
+
+func TestPutCaseCorrectDirectoryNameEmptyParent(t *testing.T) {
+	errors := fs.Stats.GetErrors()
+
+	tree := dropbox.NewNameTree()
+	tree.PutCaseCorrectDirectoryName("", "C")
+
+	c := tree.Directories["c"]
+	assert(t, c.CaseCorrectName == "C", "CaseCorrectName at 'c' should be 'C'")
+
+	assert(t, fs.Stats.GetErrors() == errors, "No errors should be reported")
+}
+
+func TestGetPathWithCorrectCase(t *testing.T) {
+	errors := fs.Stats.GetErrors()
+
+	tree := dropbox.NewNameTree()
+	tree.PutCaseCorrectDirectoryName("a", "C")
+	assert(t, tree.GetPathWithCorrectCase("a/c") == nil, "Path for 'a' should not be available")
+
+	tree.PutCaseCorrectDirectoryName("", "A")
+	assert(t, *tree.GetPathWithCorrectCase("a/c") == "/A/C", "Path for 'a/c' should be '/A/C'")
+
+	assert(t, fs.Stats.GetErrors() == errors, "No errors should be reported")
+}
+
+func TestPutAndWalk(t *testing.T) {
+	errors := fs.Stats.GetErrors()
+
+	tree := dropbox.NewNameTree()
+	tree.PutFile("a", "F", &dropboxapi.Entry{Path: "xxx"})
+	tree.PutCaseCorrectDirectoryName("", "A")
+
+	numCalled := 0
+	walkFunc := func(caseCorrectFilePath string, entry *dropboxapi.Entry) {
+		assert(t, caseCorrectFilePath == "A/F", "caseCorrectFilePath should be A/F, not "+caseCorrectFilePath)
+		assert(t, entry.Path == "xxx", "entry.Path should be xxx")
+		numCalled++
+	}
+	tree.WalkFiles("", walkFunc)
+
+	assert(t, numCalled == 1, "walk func should be called only once")
+
+	assert(t, fs.Stats.GetErrors() == errors, "No errors should be reported")
+}
+
+func TestPutAndWalkWithPrefix(t *testing.T) {
+	errors := fs.Stats.GetErrors()
+
+	tree := dropbox.NewNameTree()
+	tree.PutFile("a", "F", &dropboxapi.Entry{Path: "xxx"})
+	tree.PutCaseCorrectDirectoryName("", "A")
+
+	numCalled := 0
+	walkFunc := func(caseCorrectFilePath string, entry *dropboxapi.Entry) {
+		assert(t, caseCorrectFilePath == "A/F", "caseCorrectFilePath should be A/F, not "+caseCorrectFilePath)
+		assert(t, entry.Path == "xxx", "entry.Path should be xxx")
+		numCalled++
+	}
+	tree.WalkFiles("A", walkFunc)
+
+	assert(t, numCalled == 1, "walk func should be called only once")
+
+	assert(t, fs.Stats.GetErrors() == errors, "No errors should be reported")
+}
+
+func TestPutAndWalkIncompleteTree(t *testing.T) {
+	errors := fs.Stats.GetErrors()
+
+	tree := dropbox.NewNameTree()
+	tree.PutFile("a", "F", &dropboxapi.Entry{Path: "xxx"})
+
+	walkFunc := func(caseCorrectFilePath string, entry *dropboxapi.Entry) {
+		t.Fatal("Should not be called")
+	}
+	tree.WalkFiles("", walkFunc)
+
+	assert(t, fs.Stats.GetErrors() == errors+1, "One error should be reported")
+}