http: HEAD directory entries in parallel to speedup #3523
This commit is contained in:
parent
7b29ed8ec1
commit
7982aaf151
1 changed files with 36 additions and 16 deletions
|
@ -13,6 +13,7 @@ import (
|
||||||
"path"
|
"path"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
|
@ -415,30 +416,49 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.Wrapf(err, "error listing %q", dir)
|
return nil, errors.Wrapf(err, "error listing %q", dir)
|
||||||
}
|
}
|
||||||
|
var (
|
||||||
|
entriesMu sync.Mutex // to protect entries
|
||||||
|
wg sync.WaitGroup
|
||||||
|
in = make(chan string, fs.Config.Checkers)
|
||||||
|
)
|
||||||
|
add := func(entry fs.DirEntry) {
|
||||||
|
entriesMu.Lock()
|
||||||
|
entries = append(entries, entry)
|
||||||
|
entriesMu.Unlock()
|
||||||
|
}
|
||||||
|
for i := 0; i < fs.Config.Checkers; i++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
for remote := range in {
|
||||||
|
file := &Object{
|
||||||
|
fs: f,
|
||||||
|
remote: remote,
|
||||||
|
}
|
||||||
|
switch err = file.stat(ctx); err {
|
||||||
|
case nil:
|
||||||
|
add(file)
|
||||||
|
case fs.ErrorNotAFile:
|
||||||
|
// ...found a directory not a file
|
||||||
|
add(fs.NewDir(remote, timeUnset))
|
||||||
|
default:
|
||||||
|
fs.Debugf(remote, "skipping because of error: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
for _, name := range names {
|
for _, name := range names {
|
||||||
isDir := name[len(name)-1] == '/'
|
isDir := name[len(name)-1] == '/'
|
||||||
name = strings.TrimRight(name, "/")
|
name = strings.TrimRight(name, "/")
|
||||||
remote := path.Join(dir, name)
|
remote := path.Join(dir, name)
|
||||||
if isDir {
|
if isDir {
|
||||||
dir := fs.NewDir(remote, timeUnset)
|
add(fs.NewDir(remote, timeUnset))
|
||||||
entries = append(entries, dir)
|
|
||||||
} else {
|
} else {
|
||||||
file := &Object{
|
in <- remote
|
||||||
fs: f,
|
|
||||||
remote: remote,
|
|
||||||
}
|
|
||||||
switch err = file.stat(ctx); err {
|
|
||||||
case nil:
|
|
||||||
entries = append(entries, file)
|
|
||||||
case fs.ErrorNotAFile:
|
|
||||||
// ...found a directory not a file
|
|
||||||
dir := fs.NewDir(remote, timeUnset)
|
|
||||||
entries = append(entries, dir)
|
|
||||||
default:
|
|
||||||
fs.Debugf(remote, "skipping because of error: %v", err)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
close(in)
|
||||||
|
wg.Wait()
|
||||||
return entries, nil
|
return entries, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue