http: HEAD directory entries in parallel to speedup #3523

This commit is contained in:
Nick Craig-Wood 2019-09-09 21:03:20 +01:00
parent 7b29ed8ec1
commit 7982aaf151

View file

@ -13,6 +13,7 @@ import (
"path" "path"
"strconv" "strconv"
"strings" "strings"
"sync"
"time" "time"
"github.com/pkg/errors" "github.com/pkg/errors"
@ -415,30 +416,49 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
if err != nil { if err != nil {
return nil, errors.Wrapf(err, "error listing %q", dir) return nil, errors.Wrapf(err, "error listing %q", dir)
} }
for _, name := range names { var (
isDir := name[len(name)-1] == '/' entriesMu sync.Mutex // to protect entries
name = strings.TrimRight(name, "/") wg sync.WaitGroup
remote := path.Join(dir, name) in = make(chan string, fs.Config.Checkers)
if isDir { )
dir := fs.NewDir(remote, timeUnset) add := func(entry fs.DirEntry) {
entries = append(entries, dir) entriesMu.Lock()
} else { entries = append(entries, entry)
entriesMu.Unlock()
}
for i := 0; i < fs.Config.Checkers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for remote := range in {
file := &Object{ file := &Object{
fs: f, fs: f,
remote: remote, remote: remote,
} }
switch err = file.stat(ctx); err { switch err = file.stat(ctx); err {
case nil: case nil:
entries = append(entries, file) add(file)
case fs.ErrorNotAFile: case fs.ErrorNotAFile:
// ...found a directory not a file // ...found a directory not a file
dir := fs.NewDir(remote, timeUnset) add(fs.NewDir(remote, timeUnset))
entries = append(entries, dir)
default: default:
fs.Debugf(remote, "skipping because of error: %v", err) fs.Debugf(remote, "skipping because of error: %v", err)
} }
} }
}()
} }
for _, name := range names {
isDir := name[len(name)-1] == '/'
name = strings.TrimRight(name, "/")
remote := path.Join(dir, name)
if isDir {
add(fs.NewDir(remote, timeUnset))
} else {
in <- remote
}
}
close(in)
wg.Wait()
return entries, nil return entries, nil
} }