Make Google Drive Directory reads concurrent for increased speed.

If you use the -v flag then this will show the progress of directory
listing.  Fixes #143.
This commit is contained in:
klauspost 2015-09-17 14:31:10 +02:00 committed by Nick Craig-Wood
parent bc5b63ffef
commit 6fbd9cf24b

View file

@ -13,6 +13,7 @@ import (
"log" "log"
"net/http" "net/http"
"strings" "strings"
"sync"
"time" "time"
"golang.org/x/oauth2" "golang.org/x/oauth2"
@ -370,15 +371,24 @@ func (f *FsDrive) CreateDir(pathId, leaf string) (newId string, err error) {
func (f *FsDrive) listDirRecursive(dirId string, path string, out fs.ObjectsChan) error { func (f *FsDrive) listDirRecursive(dirId string, path string, out fs.ObjectsChan) error {
var subError error var subError error
// Make the API request // Make the API request
var wg sync.WaitGroup
_, err := f.listAll(dirId, "", false, false, func(item *drive.File) bool { _, err := f.listAll(dirId, "", false, false, func(item *drive.File) bool {
// Recurse on directories // Recurse on directories
// FIXME should do this in parallel
// use a wg to sync then collect error
if item.MimeType == driveFolderType { if item.MimeType == driveFolderType {
subError = f.listDirRecursive(item.Id, path+item.Title+"/", out) wg.Add(1)
if subError != nil { folder := path + item.Title + "/"
return true fs.Debug(f, "Reading %s", folder)
}
go func() {
defer wg.Done()
err := f.listDirRecursive(item.Id, folder, out)
if err != nil {
subError = err
fs.ErrorLog(f, "Error reading %s:%s", folder, err)
}
}()
return false
} else { } else {
// If item has no MD5 sum it isn't stored on drive, so ignore it // If item has no MD5 sum it isn't stored on drive, so ignore it
if item.Md5Checksum != "" { if item.Md5Checksum != "" {
@ -389,6 +399,8 @@ func (f *FsDrive) listDirRecursive(dirId string, path string, out fs.ObjectsChan
} }
return false return false
}) })
wg.Wait()
fs.Debug(f, "Finished reading %s", path)
if err != nil { if err != nil {
return err return err
} }