Make exponential backoff work exactly as per google specification - fixes #583

drive/drive.go

@@ -35,9 +35,6 @@ const (
 	timeFormatIn                = time.RFC3339
 	timeFormatOut               = "2006-01-02T15:04:05.000000000Z07:00"
 	minSleep                    = 10 * time.Millisecond
-	maxSleep                    = 2000 * time.Millisecond
-	decayConstant               = 0 // bigger for slower decay, exponential
-	attackConstant              = 0 // bigger for slower attack, exponential
 	defaultExtensions           = "docx,xlsx,pptx,svg"
 )
 
@@ -295,7 +292,7 @@ func NewFs(name, path string) (fs.Fs, error) {
 	f := &Fs{
 		name:  name,
 		root:  root,
-		pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant).SetAttackConstant(attackConstant),
+		pacer: pacer.New().SetMinSleep(minSleep).SetPacer(pacer.GoogleDrivePacer),
 	}
 
 	// Create a new authorized Drive client.

pacer/pacer.go

@@ -48,6 +48,16 @@ const (
 	//
 	// See https://developer.amazon.com/public/apis/experience/cloud-drive/content/restful-api-best-practices
 	AmazonCloudDrivePacer
+
+	// GoogleDrivePacer is a specialised pacer for Google Drive
+	//
+	// It implements a truncated exponential backoff strategy with
+	// randomization.  Normally operations are paced at the
+	// interval set with SetMinSleep.  On errors the sleep timer
+	// is set to (2 ^ n) + random_number_milliseconds seconds
+	//
+	// See https://developers.google.com/drive/v2/web/handle-errors#exponential-backoff
+	GoogleDrivePacer
 )
 
 // Paced is a function which is called by the Call and CallNoRetry
@@ -172,6 +182,8 @@ func (p *Pacer) SetPacer(t Type) *Pacer {
 	switch t {
 	case AmazonCloudDrivePacer:
 		p.calculatePace = p.acdPacer
+	case GoogleDrivePacer:
+		p.calculatePace = p.drivePacer
 	default:
 		p.calculatePace = p.defaultPacer
 	}
@@ -265,7 +277,34 @@ func (p *Pacer) acdPacer(retry bool) {
 		if p.sleepTime < p.minSleep {
 			p.sleepTime = p.minSleep
 		}
-		fs.Debug("pacer", "Rate limited, sleeping for %v (%d consecutive low level retries)", p.sleepTime, consecutiveRetries)
+		fs.Debug("pacer", "Rate limited, sleeping for %v (%d consecutive low level retries)", p.sleepTime, p.consecutiveRetries)
+	}
+}
+
+// drivePacer implements a truncated exponential backoff strategy with
+// randomization for Google Drive
+//
+// See the description for GoogleDrivePacer
+//
+// This should calculate a new sleepTime.  It takes a boolean as to
+// whether the operation should be retried or not.
+//
+// Call with p.mu held
+func (p *Pacer) drivePacer(retry bool) {
+	consecutiveRetries := p.consecutiveRetries
+	if consecutiveRetries == 0 {
+		if p.sleepTime != p.minSleep {
+			p.sleepTime = p.minSleep
+			fs.Debug("pacer", "Resetting sleep to minimum %v on success", p.sleepTime)
+		}
+	} else {
+		if consecutiveRetries > 5 {
+			consecutiveRetries = 5
+		}
+		// consecutiveRetries starts at 1 so go from 1,2,3,4,5,5 => 1,2,4,8,16,16
+		// maxSleep is 2**(consecutiveRetries-1) seconds + random milliseconds
+		p.sleepTime = time.Second<<uint(consecutiveRetries-1) + time.Duration(rand.Int63n(int64(time.Second)))
+		fs.Debug("pacer", "Rate limited, sleeping for %v (%d consecutive low level retries)", p.sleepTime, p.consecutiveRetries)
 	}
 }
 

pacer/pacer_test.go

@@ -167,6 +167,10 @@ func TestSetPacer(t *testing.T) {
 	if fmt.Sprintf("%p", p.calculatePace) != fmt.Sprintf("%p", p.acdPacer) {
 		t.Errorf("calculatePace is not acdPacer")
 	}
+	p.SetPacer(GoogleDrivePacer)
+	if fmt.Sprintf("%p", p.calculatePace) != fmt.Sprintf("%p", p.drivePacer) {
+		t.Errorf("calculatePace is not drivePacer")
+	}
 	p.SetPacer(DefaultPacer)
 	if fmt.Sprintf("%p", p.calculatePace) != fmt.Sprintf("%p", p.defaultPacer) {
 		t.Errorf("calculatePace is not defaultPacer")
@@ -299,6 +303,42 @@ func TestAmazonCloudDrivePacer(t *testing.T) {
 	}
 }
 
+func TestGoogleDrivePacer(t *testing.T) {
+	p := New().SetMinSleep(time.Millisecond).SetPacer(GoogleDrivePacer).SetMaxSleep(time.Second).SetDecayConstant(2)
+	// Do lots of times because of the random number!
+	for _, test := range []struct {
+		in                 time.Duration
+		consecutiveRetries int
+		retry              bool
+		want               time.Duration
+	}{
+		{time.Millisecond, 0, true, time.Millisecond},
+		{10 * time.Millisecond, 0, true, time.Millisecond},
+		{1 * time.Second, 1, true, 1*time.Second + 500*time.Millisecond},
+		{1 * time.Second, 2, true, 2*time.Second + 500*time.Millisecond},
+		{1 * time.Second, 3, true, 4*time.Second + 500*time.Millisecond},
+		{1 * time.Second, 4, true, 8*time.Second + 500*time.Millisecond},
+		{1 * time.Second, 5, true, 16*time.Second + 500*time.Millisecond},
+		{1 * time.Second, 6, true, 16*time.Second + 500*time.Millisecond},
+		{1 * time.Second, 7, true, 16*time.Second + 500*time.Millisecond},
+	} {
+		const n = 1000
+		var sum time.Duration
+		// measure average time over n cycles
+		for i := 0; i < n; i++ {
+			p.sleepTime = test.in
+			p.consecutiveRetries = test.consecutiveRetries
+			p.drivePacer(test.retry)
+			sum += p.sleepTime
+		}
+		got := sum / n
+		//t.Logf("%+v: got = %v", test, got)
+		if got < (test.want*9)/10 || got > (test.want*11)/10 {
+			t.Fatalf("%+v: bad sleep want %v+/-10%% got %v", test, test.want, got)
+		}
+	}
+}
+
 func TestEndCall(t *testing.T) {
 	p := New().SetMaxConnections(5)
 	emptyTokens(p)