drive: handle gdocs when filtering file names in list

Fixes #2399

backend/drive/drive.go

@@ -359,6 +359,15 @@ func parseDrivePath(path string) (root string, err error) {
 // Should return true to finish processing
 type listFn func(*drive.File) bool
 
+func containsString(slice []string, s string) bool {
+	for _, e := range slice {
+		if e == s {
+			return true
+		}
+	}
+	return false
+}
+
 // Lists the directory required calling the user function on each item found
 //
 // If the user fn ever returns true then it early exits with found = true
@@ -382,14 +391,25 @@ func (f *Fs) list(dirID string, title string, directoriesOnly bool, filesOnly bo
 	if dirID != "" && !(f.opt.SharedWithMe && dirID == f.rootFolderID) {
 		query = append(query, fmt.Sprintf("'%s' in parents", dirID))
 	}
+	stem := ""
 	if title != "" {
 		// Escaping the backslash isn't documented but seems to work
 		searchTitle := strings.Replace(title, `\`, `\\`, -1)
 		searchTitle = strings.Replace(searchTitle, `'`, `\'`, -1)
 		// Convert ／ to / for search
 		searchTitle = strings.Replace(searchTitle, "／", "/", -1)
+
+		handleGdocs := !directoriesOnly && !f.opt.SkipGdocs
+		// if the search title contains an extension and the extension is in the export extensions add a search
+		// for the filename without the extension.
+		// assume that export extensions don't contain escape sequences and only have one part (not .tar.gz)
+		if ext := path.Ext(searchTitle); handleGdocs && len(ext) > 0 && containsString(f.extensions, ext[1:]) {
+			stem = title[:len(title)-len(ext)]
+			query = append(query, fmt.Sprintf("(name='%s' or name='%s')", searchTitle, searchTitle[:len(title)-len(ext)]))
+		} else {
 			query = append(query, fmt.Sprintf("name='%s'", searchTitle))
 		}
+	}
 	if directoriesOnly {
 		query = append(query, fmt.Sprintf("mimeType='%s'", driveFolderType))
 	}
@@ -438,9 +458,16 @@ OUTER:
 			item.Name = strings.Replace(item.Name, "/", "／", -1)
 			// Check the case of items is correct since
 			// the `=` operator is case insensitive.
+
 			if title != "" && title != item.Name {
+				if stem == "" || stem != item.Name {
 					continue
 				}
+				_, exportName, _, _ := f.findExportFormat(item)
+				if exportName == "" || exportName != title {
+					continue
+				}
+			}
 			if fn(item) {
 				found = true
 				break OUTER
@@ -665,13 +692,11 @@ func NewFs(name, path string, m configmap.Mapper) (fs.Fs, error) {
 			// No root so return old f
 			return f, nil
 		}
-		entries, err := tempF.List("")
+		_, err := tempF.NewObject(remote)
 		if err != nil {
 			// unable to list folder so return old f
 			return f, nil
 		}
-		for _, e := range entries {
-			if _, isObject := e.(fs.Object); isObject && e.Remote() == remote {
 		// XXX: update the old f here instead of returning tempF, since
 		// `features` were already filled with functions having *f as a receiver.
 		// See https://github.com/ncw/rclone/issues/2182
@@ -679,10 +704,6 @@ func NewFs(name, path string, m configmap.Mapper) (fs.Fs, error) {
 		f.root = tempF.root
 		return f, fs.ErrorIsFile
 	}
-		}
-		// File doesn't exist so return old f
-		return f, nil
-	}
 	// fmt.Printf("Root id %s", f.dirCache.RootID())
 	return f, nil
 }
@@ -720,6 +741,13 @@ func (f *Fs) FindLeaf(pathID, leaf string) (pathIDOut string, found bool, err er
 			pathIDOut = item.Id
 			return true
 		}
+		if !f.opt.SkipGdocs {
+			_, exportName, _, _ := f.findExportFormat(item)
+			if exportName == leaf {
+				pathIDOut = item.Id
+				return true
+			}
+		}
 		return false
 	})
 	return pathIDOut, found, err
@@ -783,18 +811,21 @@ func (f *Fs) exportFormats() map[string][]string {
 //
 // Look through the extensions and find the first format that can be
 // converted.  If none found then return "", ""
-func (f *Fs) findExportFormat(filepath string, exportMimeTypes []string) (extension, mimeType string) {
+func (f *Fs) findExportFormat(item *drive.File) (extension, filename, mimeType string, isDocument bool) {
+	exportMimeTypes, isDocument := f.exportFormats()[item.MimeType]
+	if isDocument {
 		for _, extension := range f.extensions {
 			mimeType := extensionToMimeType[extension]
 			for _, emt := range exportMimeTypes {
 				if emt == mimeType {
-				return extension, mimeType
+					return extension, fmt.Sprintf("%s.%s", item.Name, extension), mimeType, true
+				}
 			}
 		}
 	}
 
 	// else return empty
-	return "", ""
+	return "", "", "", isDocument
 }
 
 // List the objects and directories in dir into entries.  The
@@ -839,40 +870,23 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 		case f.opt.SkipGdocs:
 			fs.Debugf(remote, "Skipping google document type %q", item.MimeType)
 		default:
-			exportMimeTypes, isDocument := f.exportFormats()[item.MimeType]
+			// If item MimeType is in the ExportFormats then it is a google doc
+			extension, _, exportMimeType, isDocument := f.findExportFormat(item)
 			if !isDocument {
 				fs.Debugf(remote, "Ignoring unknown document type %q", item.MimeType)
 				break
 			}
-			// If item has export links then it is a google doc
-			extension, exportMimeType := f.findExportFormat(remote, exportMimeTypes)
 			if extension == "" {
 				fs.Debugf(remote, "No export formats found for %q", item.MimeType)
 				break
 			}
-			o, err := f.newObjectWithInfo(remote+"."+extension, item)
+			obj, err := f.newObjectWithInfo(remote+"."+extension, item)
 			if err != nil {
 				iErr = err
 				return true
 			}
-			obj := o.(*Object)
+			obj.(*Object).setGdocsMetaData(item, extension, exportMimeType)
-			obj.url = fmt.Sprintf("%sfiles/%s/export?mimeType=%s", f.svc.BasePath, item.Id, url.QueryEscape(exportMimeType))
+			entries = append(entries, obj)
-			if f.opt.AlternateExport {
-				switch item.MimeType {
-				case "application/vnd.google-apps.drawing":
-					obj.url = fmt.Sprintf("https://docs.google.com/drawings/d/%s/export/%s", item.Id, extension)
-				case "application/vnd.google-apps.document":
-					obj.url = fmt.Sprintf("https://docs.google.com/document/d/%s/export?format=%s", item.Id, extension)
-				case "application/vnd.google-apps.spreadsheet":
-					obj.url = fmt.Sprintf("https://docs.google.com/spreadsheets/d/%s/export?format=%s", item.Id, extension)
-				case "application/vnd.google-apps.presentation":
-					obj.url = fmt.Sprintf("https://docs.google.com/presentation/d/%s/export/%s", item.Id, extension)
-				}
-			}
-			obj.isDocument = true
-			obj.mimeType = exportMimeType
-			obj.bytes = -1
-			entries = append(entries, o)
 		}
 		return false
 	})
@@ -1549,6 +1563,26 @@ func (o *Object) setMetaData(info *drive.File) {
 	o.mimeType = info.MimeType
 }
 
+// setGdocsMetaData only sets the gdocs related fields
+func (o *Object) setGdocsMetaData(info *drive.File, extension, exportMimeType string) {
+	o.url = fmt.Sprintf("%sfiles/%s/export?mimeType=%s", o.fs.svc.BasePath, info.Id, url.QueryEscape(exportMimeType))
+	if o.fs.opt.AlternateExport {
+		switch info.MimeType {
+		case "application/vnd.google-apps.drawing":
+			o.url = fmt.Sprintf("https://docs.google.com/drawings/d/%s/export/%s", info.Id, extension)
+		case "application/vnd.google-apps.document":
+			o.url = fmt.Sprintf("https://docs.google.com/document/d/%s/export?format=%s", info.Id, extension)
+		case "application/vnd.google-apps.spreadsheet":
+			o.url = fmt.Sprintf("https://docs.google.com/spreadsheets/d/%s/export?format=%s", info.Id, extension)
+		case "application/vnd.google-apps.presentation":
+			o.url = fmt.Sprintf("https://docs.google.com/presentation/d/%s/export/%s", info.Id, extension)
+		}
+	}
+	o.isDocument = true
+	o.mimeType = exportMimeType
+	o.bytes = -1
+}
+
 // readMetaData gets the info if it hasn't already been fetched
 func (o *Object) readMetaData() (err error) {
 	if o.id != "" {
@@ -1568,6 +1602,14 @@ func (o *Object) readMetaData() (err error) {
 			o.setMetaData(item)
 			return true
 		}
+		if !o.fs.opt.SkipGdocs {
+			extension, exportName, exportMimeType, _ := o.fs.findExportFormat(item)
+			if exportName == leaf {
+				o.setMetaData(item)
+				o.setGdocsMetaData(item, extension, exportMimeType)
+				return true
+			}
+		}
 		return false
 	})
 	if err != nil {

backend/drive/drive_internal_test.go

@@ -53,11 +53,10 @@ const exampleExportFormats = `{
 	]
 }`
 
-var exportFormats map[string][]string
-
 // Load the example export formats into exportFormats for testing
 func TestInternalLoadExampleExportFormats(t *testing.T) {
-	assert.NoError(t, json.Unmarshal([]byte(exampleExportFormats), &exportFormats))
+	exportFormatsOnce.Do(func() {})
+	assert.NoError(t, json.Unmarshal([]byte(exampleExportFormats), &_exportFormats))
 }
 
 func TestInternalParseExtensions(t *testing.T) {
@@ -90,8 +89,10 @@ func TestInternalParseExtensions(t *testing.T) {
 }
 
 func TestInternalFindExportFormat(t *testing.T) {
-	item := new(drive.File)
+	item := &drive.File{
-	item.MimeType = "application/vnd.google-apps.document"
+		Name:     "file",
+		MimeType: "application/vnd.google-apps.document",
+	}
 	for _, test := range []struct {
 		extensions    []string
 		wantExtension string
@@ -105,8 +106,14 @@ func TestInternalFindExportFormat(t *testing.T) {
 	} {
 		f := new(Fs)
 		f.extensions = test.extensions
-		gotExtension, gotMimeType := f.findExportFormat("file", exportFormats[item.MimeType])
+		gotExtension, gotFilename, gotMimeType, gotIsDocument := f.findExportFormat(item)
 		assert.Equal(t, test.wantExtension, gotExtension)
+		if test.wantExtension != "" {
+			assert.Equal(t, item.Name+"."+gotExtension, gotFilename)
+		} else {
+			assert.Equal(t, "", gotFilename)
+		}
 		assert.Equal(t, test.wantMimeType, gotMimeType)
+		assert.Equal(t, true, gotIsDocument)
 	}
 }