KurlesHS/rclone
1
0
Fork 0
forked from TrueCloudLab/rclone
Code Pull requests Activity

Limit concurrent directory listings.

Browse source 
Never run more than "fs.Config.Checkers" directory listings at once.
This commit is contained in:
Klaus Post 2015-09-13 16:05:13 +02:00 committed by Nick Craig-Wood
parent a1a780e847
commit dd48e62b7e
1 changed files with 20 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
amazonclouddrive/amazonclouddrive.go
View file

@ -82,11 +82,12 @@ func init() {
// FsAcd represents a remote acd server // FsAcd represents a remote acd server
type FsAcd struct { type FsAcd struct {
name string // name of this remote name string // name of this remote
c *acd.Client // the connection to the acd server c *acd.Client // the connection to the acd server
root string // the path we are working on root string // the path we are working on
dirCache *dircache.DirCache // Map of directory path to directory id dirCache *dircache.DirCache // Map of directory path to directory id
pacer *pacer.Pacer // pacer for API calls pacer *pacer.Pacer // pacer for API calls
connTokens chan struct{} // Connection tokens for directory listings
} }
// FsObjectAcd describes a acd object // FsObjectAcd describes a acd object
@ -161,10 +162,16 @@ func NewFs(name, root string) (fs.Fs, error) {
c := acd.NewClient(oAuthClient) c := acd.NewClient(oAuthClient)
c.UserAgent = fs.UserAgent c.UserAgent = fs.UserAgent
f := &FsAcd{ f := &FsAcd{
name: name, name: name,
root: root, root: root,
c: c, c: c,
pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant), pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
connTokens: make(chan struct{}, fs.Config.Checkers),
}
// Insert connection tokens.
for i := 0; i < fs.Config.Checkers; i++ {
f.connTokens <- struct{}{}
} }
// Update endpoints // Update endpoints
@ -317,10 +324,14 @@ func (f *FsAcd) listAll(dirId string, title string, directoriesOnly bool, filesO
OUTER: OUTER:
for { for {
var resp *http.Response var resp *http.Response
// Get a token
_ = <-f.connTokens
err = f.pacer.Call(func() (bool, error) { err = f.pacer.Call(func() (bool, error) {
nodes, resp, err = f.c.Nodes.GetNodes(&opts) nodes, resp, err = f.c.Nodes.GetNodes(&opts)
return shouldRetry(resp, err) return shouldRetry(resp, err)
}) })
// Reinsert token
f.connTokens <- struct{}{}
if err != nil { if err != nil {
fs.Stats.Error() fs.Stats.Error()
fs.ErrorLog(f, "Couldn't list files: %v", err) fs.ErrorLog(f, "Couldn't list files: %v", err)