TrueCloudLab/rclone
16
0
Fork 2
Code Issues 1 Pull requests 1 Projects Releases Packages Wiki Activity Actions

drive: add --drive-pacer-burst config to control bursting of the rate limiter

Browse source
This commit is contained in:
Nick Craig-Wood 2019-01-20 08:36:22 +00:00
parent f8dbf8292a
commit 0855608bc1
2 changed files with 9 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
backend/drive/drive.go
View file

@ -55,6 +55,7 @@ const (
timeFormatIn = time.RFC3339 timeFormatIn = time.RFC3339
timeFormatOut = "2006-01-02T15:04:05.000000000Z07:00" timeFormatOut = "2006-01-02T15:04:05.000000000Z07:00"
defaultMinSleep = fs.Duration(100 * time.Millisecond) defaultMinSleep = fs.Duration(100 * time.Millisecond)
defaultBurst = 100
defaultExportExtensions = "docx,xlsx,pptx,svg" defaultExportExtensions = "docx,xlsx,pptx,svg"
scopePrefix = "https://www.googleapis.com/auth/" scopePrefix = "https://www.googleapis.com/auth/"
defaultScope = "drive" defaultScope = "drive"
@ -361,6 +362,11 @@ will download it anyway.`,
Default: defaultMinSleep, Default: defaultMinSleep,
Help: "Minimum time to sleep between API calls.", Help: "Minimum time to sleep between API calls.",
Advanced: true, Advanced: true,
}, {
Name: "pacer_burst",
Default: defaultBurst,
Help: "Number of API calls to allow without sleeping.",
Advanced: true,
}}, }},
}) })
@ -404,6 +410,7 @@ type Options struct {
KeepRevisionForever bool `config:"keep_revision_forever"` KeepRevisionForever bool `config:"keep_revision_forever"`
V2DownloadMinSize fs.SizeSuffix `config:"v2_download_min_size"` V2DownloadMinSize fs.SizeSuffix `config:"v2_download_min_size"`
PacerMinSleep fs.Duration `config:"pacer_min_sleep"` PacerMinSleep fs.Duration `config:"pacer_min_sleep"`
PacerBurst int `config:"pacer_burst"`
} }
// Fs represents a remote drive server // Fs represents a remote drive server
@ -781,7 +788,7 @@ func configTeamDrive(opt *Options, m configmap.Mapper, name string) error {
// newPacer makes a pacer configured for drive // newPacer makes a pacer configured for drive
func newPacer(opt *Options) *pacer.Pacer { func newPacer(opt *Options) *pacer.Pacer {
return pacer.New().SetMinSleep(time.Duration(opt.PacerMinSleep)).SetPacer(pacer.GoogleDrivePacer) return pacer.New().SetMinSleep(time.Duration(opt.PacerMinSleep)).SetBurst(opt.PacerBurst).SetPacer(pacer.GoogleDrivePacer)
} }
func getServiceAccountClient(opt *Options, credentialsData []byte) (*http.Client, error) { func getServiceAccountClient(opt *Options, credentialsData []byte) (*http.Client, error) {

2
lib/pacer/pacer.go
View file

@ -234,7 +234,7 @@ func (p *Pacer) beginCall() {
// fs.Debugf(f, "New sleep for %v at %v", t, time.Now()) // fs.Debugf(f, "New sleep for %v at %v", t, time.Now())
// Sleep the minimum time with the rate limiter // Sleep the minimum time with the rate limiter
if minSleep > 0 && sleepTime >= minSleep { if minSleep > 0 && sleepTime >= minSleep {
p.limiter.Wait(context.Background()) _ = p.limiter.Wait(context.Background())
sleepTime -= minSleep sleepTime -= minSleep
} }
// Then sleep the remaining time // Then sleep the remaining time