Merge pull request #1353 from restic/backend-retry

Retry failed backend requests

Gopkg.lock

@@ -25,6 +25,12 @@
   revision = "f6be1abbb5abd0517522f850dd785990d373da7e"
   version = "v8.4.0"
 
+[[projects]]
+  name = "github.com/cenkalti/backoff"
+  packages = ["."]
+  revision = "61153c768f31ee5f130071d08fc82b85208528de"
+  version = "v1.1.0"
+
 [[projects]]
   name = "github.com/cpuguy83/go-md2man"
   packages = ["md2man"]
@@ -208,6 +214,6 @@
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "1cd40a4f4ee9009890482c6076d6725360d8b4475a8106cc5150d825989ba163"
+  inputs-digest = "ea711bd1a9bfc8902b973a4de3a840f42536b9091fd8558980f44d6ca1622227"
   solver-name = "gps-cdcl"
   solver-version = 1

cmd/restic/global.go

@@ -9,6 +9,7 @@ import (
 	"runtime"
 	"strings"
 	"syscall"
+	"time"
 
 	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/backend/azure"
@@ -328,6 +329,10 @@ func OpenRepository(opts GlobalOptions) (*repository.Repository, error) {
 		be = limiter.LimitBackend(be, limiter.NewStaticLimiter(opts.LimitUploadKb, opts.LimitDownloadKb))
 	}
 
+	be = backend.NewRetryBackend(be, 10, func(msg string, err error, d time.Duration) {
+		Warnf("%v returned error, retrying after %v: %v\n", msg, d, err)
+	})
+
 	s := repository.New(be)
 
 	opts.password, err = ReadPassword(opts, "enter password for repository: ")

cmd/restic/lock.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/restic/restic/internal/debug"
+	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
 )
@@ -35,7 +36,7 @@ func lockRepository(repo *repository.Repository, exclusive bool) (*restic.Lock,
 
 	lock, err := lockFn(context.TODO(), repo)
 	if err != nil {
-		return nil, err
+		return nil, errors.Fatalf("unable to create lock in backend: %v", err)
 	}
 	debug.Log("create lock %p (exclusive %v)", lock, exclusive)
 

internal/backend/azure/azure_test.go

@@ -76,11 +76,7 @@ func newAzureTestSuite(t testing.TB) *test.Suite {
 				return err
 			}
 
-			if err := be.(restic.Deleter).Delete(context.TODO()); err != nil {
-				return err
-			}
-
-			return nil
+			return be.Delete(context.TODO())
 		},
 	}
 }

internal/backend/b2/b2_test.go

@@ -62,11 +62,7 @@ func newB2TestSuite(t testing.TB) *test.Suite {
 				return err
 			}
 
-			if err := be.(restic.Deleter).Delete(context.TODO()); err != nil {
-				return err
-			}
-
-			return nil
+			return be.Delete(context.TODO())
 		},
 	}
 }

internal/backend/backend_error.go

@@ -0,0 +1,73 @@
+package backend
+
+import (
+	"context"
+	"io"
+	"math/rand"
+	"sync"
+
+	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/restic"
+)
+
+// ErrorBackend is used to induce errors into various function calls and test
+// the retry functions.
+type ErrorBackend struct {
+	FailSave float32
+	FailLoad float32
+	FailStat float32
+	restic.Backend
+
+	r *rand.Rand
+	m sync.Mutex
+}
+
+// statically ensure that ErrorBackend implements restic.Backend.
+var _ restic.Backend = &ErrorBackend{}
+
+// NewErrorBackend wraps be with a backend that returns errors according to
+// given probabilities.
+func NewErrorBackend(be restic.Backend, seed int64) *ErrorBackend {
+	return &ErrorBackend{
+		Backend: be,
+		r:       rand.New(rand.NewSource(seed)),
+	}
+}
+
+func (be *ErrorBackend) fail(p float32) bool {
+	be.m.Lock()
+	v := be.r.Float32()
+	be.m.Unlock()
+
+	return v < p
+}
+
+// Save stores the data in the backend under the given handle.
+func (be *ErrorBackend) Save(ctx context.Context, h restic.Handle, rd io.Reader) error {
+	if be.fail(be.FailSave) {
+		return errors.Errorf("Save(%v) random error induced", h)
+	}
+
+	return be.Backend.Save(ctx, h, rd)
+}
+
+// Load returns a reader that yields the contents of the file at h at the
+// given offset. If length is larger than zero, only a portion of the file
+// is returned. rd must be closed after use. If an error is returned, the
+// ReadCloser must be nil.
+func (be *ErrorBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	if be.fail(be.FailLoad) {
+		return nil, errors.Errorf("Load(%v, %v, %v) random error induced", h, length, offset)
+	}
+
+	return be.Backend.Load(ctx, h, length, offset)
+}
+
+// Stat returns information about the File identified by h.
+func (be *ErrorBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
+	if be.fail(be.FailLoad) {
+		return restic.FileInfo{}, errors.Errorf("Stat(%v) random error induced", h)
+	}
+
+	return be.Stat(ctx, h)
+}

internal/backend/backend_retry.go

@@ -0,0 +1,77 @@
+package backend
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/cenkalti/backoff"
+	"github.com/restic/restic/internal/restic"
+)
+
+// RetryBackend retries operations on the backend in case of an error with a
+// backoff.
+type RetryBackend struct {
+	restic.Backend
+	MaxTries int
+	Report   func(string, error, time.Duration)
+}
+
+// statically ensure that RetryBackend implements restic.Backend.
+var _ restic.Backend = &RetryBackend{}
+
+// NewRetryBackend wraps be with a backend that retries operations after a
+// backoff. report is called with a description and the error, if one occurred.
+func NewRetryBackend(be restic.Backend, maxTries int, report func(string, error, time.Duration)) *RetryBackend {
+	return &RetryBackend{
+		Backend:  be,
+		MaxTries: maxTries,
+		Report:   report,
+	}
+}
+
+func (be *RetryBackend) retry(msg string, f func() error) error {
+	return backoff.RetryNotify(f,
+		backoff.WithMaxTries(backoff.NewExponentialBackOff(), uint64(be.MaxTries)),
+		func(err error, d time.Duration) {
+			if be.Report != nil {
+				be.Report(msg, err, d)
+			}
+		},
+	)
+}
+
+// Save stores the data in the backend under the given handle.
+func (be *RetryBackend) Save(ctx context.Context, h restic.Handle, rd io.Reader) error {
+	return be.retry(fmt.Sprintf("Save(%v)", h), func() error {
+		return be.Backend.Save(ctx, h, rd)
+	})
+}
+
+// Load returns a reader that yields the contents of the file at h at the
+// given offset. If length is larger than zero, only a portion of the file
+// is returned. rd must be closed after use. If an error is returned, the
+// ReadCloser must be nil.
+func (be *RetryBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64) (rd io.ReadCloser, err error) {
+	err = be.retry(fmt.Sprintf("Load(%v, %v, %v)", h, length, offset),
+		func() error {
+			var innerError error
+			rd, innerError = be.Backend.Load(ctx, h, length, offset)
+
+			return innerError
+		})
+	return rd, err
+}
+
+// Stat returns information about the File identified by h.
+func (be *RetryBackend) Stat(ctx context.Context, h restic.Handle) (fi restic.FileInfo, err error) {
+	err = be.retry(fmt.Sprintf("Stat(%v)", h),
+		func() error {
+			var innerError error
+			fi, innerError = be.Backend.Stat(ctx, h)
+
+			return innerError
+		})
+	return fi, err
+}

internal/backend/gs/gs_test.go

@@ -69,11 +69,7 @@ func newGSTestSuite(t testing.TB) *test.Suite {
 				return err
 			}
 
-			if err := be.(restic.Deleter).Delete(context.TODO()); err != nil {
-				return err
-			}
-
-			return nil
+			return be.Delete(context.TODO())
 		},
 	}
 }

internal/backend/local/local.go

@@ -275,7 +275,7 @@ func (b *Local) List(ctx context.Context, t restic.FileType) <-chan string {
 }
 
 // Delete removes the repository and all files.
-func (b *Local) Delete() error {
+func (b *Local) Delete(ctx context.Context) error {
 	debug.Log("Delete()")
 	return fs.RemoveAll(b.Path)
 }

internal/backend/rest/rest.go

@@ -349,3 +349,38 @@ func (b *restBackend) Close() error {
 	// same function.
 	return nil
 }
+
+// Remove keys for a specified backend type.
+func (b *restBackend) removeKeys(ctx context.Context, t restic.FileType) error {
+	for key := range b.List(ctx, restic.DataFile) {
+		err := b.Remove(ctx, restic.Handle{Type: restic.DataFile, Name: key})
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// Delete removes all data in the backend.
+func (b *restBackend) Delete(ctx context.Context) error {
+	alltypes := []restic.FileType{
+		restic.DataFile,
+		restic.KeyFile,
+		restic.LockFile,
+		restic.SnapshotFile,
+		restic.IndexFile}
+
+	for _, t := range alltypes {
+		err := b.removeKeys(ctx, t)
+		if err != nil {
+			return nil
+		}
+	}
+
+	err := b.Remove(ctx, restic.Handle{Type: restic.ConfigFile})
+	if err != nil && b.IsNotExist(err) {
+		return nil
+	}
+	return err
+}

internal/backend/s3/s3_test.go

@@ -280,11 +280,7 @@ func newS3TestSuite(t testing.TB) *test.Suite {
 				return err
 			}
 
-			if err := be.(restic.Deleter).Delete(context.TODO()); err != nil {
-				return err
-			}
-
-			return nil
+			return be.Delete(context.TODO())
 		},
 	}
 }

internal/backend/sftp/sftp.go

@@ -499,3 +499,39 @@ func (r *SFTP) Close() error {
 	<-r.result
 	return nil
 }
+
+func (r *SFTP) deleteRecursive(name string) error {
+	entries, err := r.c.ReadDir(name)
+	if err != nil {
+		return errors.Wrap(err, "ReadDir")
+	}
+
+	for _, fi := range entries {
+		itemName := r.Join(name, fi.Name())
+		if fi.IsDir() {
+			err := r.deleteRecursive(itemName)
+			if err != nil {
+				return errors.Wrap(err, "ReadDir")
+			}
+
+			err = r.c.RemoveDirectory(itemName)
+			if err != nil {
+				return errors.Wrap(err, "RemoveDirectory")
+			}
+
+			continue
+		}
+
+		err := r.c.Remove(itemName)
+		if err != nil {
+			return errors.Wrap(err, "ReadDir")
+		}
+	}
+
+	return nil
+}
+
+// Delete removes all data in the backend.
+func (r *SFTP) Delete(context.Context) error {
+	return r.deleteRecursive(r.p)
+}

internal/backend/swift/swift_test.go

@@ -93,11 +93,7 @@ func newSwiftTestSuite(t testing.TB) *test.Suite {
 				return err
 			}
 
-			if err := be.(restic.Deleter).Delete(context.TODO()); err != nil {
-				return err
-			}
-
-			return nil
+			return be.Delete(context.TODO())
 		},
 	}
 }

internal/backend/test/tests.go

@@ -675,29 +675,26 @@ func (s *Suite) TestBackend(t *testing.T) {
 			t.Fatalf("lists aren't equal, want:\n  %v\n  got:\n%v\n", IDs, list)
 		}
 
-		// remove content if requested
-		if test.TestCleanupTempDirs {
-			var handles []restic.Handle
-			for _, ts := range testStrings {
-				id, err := restic.ParseID(ts.id)
-				test.OK(t, err)
+		var handles []restic.Handle
+		for _, ts := range testStrings {
+			id, err := restic.ParseID(ts.id)
+			test.OK(t, err)
 
-				h := restic.Handle{Type: tpe, Name: id.String()}
+			h := restic.Handle{Type: tpe, Name: id.String()}
 
-				found, err := b.Test(context.TODO(), h)
-				test.OK(t, err)
-				test.Assert(t, found, fmt.Sprintf("id %q not found", id))
+			found, err := b.Test(context.TODO(), h)
+			test.OK(t, err)
+			test.Assert(t, found, fmt.Sprintf("id %q not found", id))
 
-				handles = append(handles, h)
-			}
-
-			test.OK(t, s.delayedRemove(t, b, handles...))
+			handles = append(handles, h)
 		}
+
+		test.OK(t, s.delayedRemove(t, b, handles...))
 	}
 }
 
-// TestDelete tests the Delete function.
-func (s *Suite) TestDelete(t *testing.T) {
+// TestZZZDelete tests the Delete function. The name ensures that this test is executed last.
+func (s *Suite) TestZZZDelete(t *testing.T) {
 	if !test.TestCleanupTempDirs {
 		t.Skipf("not removing backend, TestCleanupTempDirs is false")
 	}
@@ -705,12 +702,7 @@ func (s *Suite) TestDelete(t *testing.T) {
 	b := s.open(t)
 	defer s.close(t, b)
 
-	be, ok := b.(restic.Deleter)
-	if !ok {
-		return
-	}
-
-	err := be.Delete(context.TODO())
+	err := b.Delete(context.TODO())
 	if err != nil {
 		t.Fatalf("error deleting backend: %+v", err)
 	}

internal/repository/repository.go

@@ -591,11 +591,7 @@ func (r *Repository) ListPack(ctx context.Context, id restic.ID) ([]restic.Blob,
 // Delete calls backend.Delete() if implemented, and returns an error
 // otherwise.
 func (r *Repository) Delete(ctx context.Context) error {
-	if b, ok := r.be.(restic.Deleter); ok {
-		return b.Delete(ctx)
-	}
-
-	return errors.New("Delete() called for backend that does not implement this method")
+	return r.be.Delete(ctx)
 }
 
 // Close closes the repository by closing the backend.

internal/restic/backend.go

@@ -40,6 +40,9 @@ type Backend interface {
 	// IsNotExist returns true if the error was caused by a non-existing file
 	// in the backend.
 	IsNotExist(err error) bool
+
+	// Delete removes all data in the backend.
+	Delete(ctx context.Context) error
 }
 
 // FileInfo is returned by Stat() and contains information about a file in the

internal/restic/repository.go

@@ -44,11 +44,6 @@ type Repository interface {
 	SaveTree(context.Context, *Tree) (ID, error)
 }
 
-// Deleter removes all data stored in a backend/repo.
-type Deleter interface {
-	Delete(context.Context) error
-}
-
 // Lister allows listing files in a backend.
 type Lister interface {
 	List(context.Context, FileType) <-chan string

vendor/github.com/cenkalti/backoff/.gitignore

@@ -0,0 +1,22 @@
+# Compiled Object files, Static and Dynamic libs (Shared Objects)
+*.o
+*.a
+*.so
+
+# Folders
+_obj
+_test
+
+# Architecture specific extensions/prefixes
+*.[568vq]
+[568vq].out
+
+*.cgo1.go
+*.cgo2.c
+_cgo_defun.c
+_cgo_gotypes.go
+_cgo_export.*
+
+_testmain.go
+
+*.exe

vendor/github.com/cenkalti/backoff/.travis.yml

@@ -0,0 +1,9 @@
+language: go
+go:
+  - 1.3.3
+  - tip
+before_install:
+  - go get github.com/mattn/goveralls
+  - go get golang.org/x/tools/cmd/cover
+script:
+  - $HOME/gopath/bin/goveralls -service=travis-ci

vendor/github.com/cenkalti/backoff/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Cenk Altı
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

vendor/github.com/cenkalti/backoff/README.md

@@ -0,0 +1,30 @@
+# Exponential Backoff [![GoDoc][godoc image]][godoc] [![Build Status][travis image]][travis] [![Coverage Status][coveralls image]][coveralls]
+
+This is a Go port of the exponential backoff algorithm from [Google's HTTP Client Library for Java][google-http-java-client].
+
+[Exponential backoff][exponential backoff wiki]
+is an algorithm that uses feedback to multiplicatively decrease the rate of some process,
+in order to gradually find an acceptable rate.
+The retries exponentially increase and stop increasing when a certain threshold is met.
+
+## Usage
+
+See https://godoc.org/github.com/cenkalti/backoff#pkg-examples
+
+## Contributing
+
+* I would like to keep this library as small as possible.
+* Please don't send a PR without opening an issue and discussing it first.
+* If proposed change is not a common use case, I will probably not accept it.
+
+[godoc]: https://godoc.org/github.com/cenkalti/backoff
+[godoc image]: https://godoc.org/github.com/cenkalti/backoff?status.png
+[travis]: https://travis-ci.org/cenkalti/backoff
+[travis image]: https://travis-ci.org/cenkalti/backoff.png?branch=master
+[coveralls]: https://coveralls.io/github/cenkalti/backoff?branch=master
+[coveralls image]: https://coveralls.io/repos/github/cenkalti/backoff/badge.svg?branch=master
+
+[google-http-java-client]: https://github.com/google/google-http-java-client
+[exponential backoff wiki]: http://en.wikipedia.org/wiki/Exponential_backoff
+
+[advanced example]: https://godoc.org/github.com/cenkalti/backoff#example_

vendor/github.com/cenkalti/backoff/backoff.go

@@ -0,0 +1,66 @@
+// Package backoff implements backoff algorithms for retrying operations.
+//
+// Use Retry function for retrying operations that may fail.
+// If Retry does not meet your needs,
+// copy/paste the function into your project and modify as you wish.
+//
+// There is also Ticker type similar to time.Ticker.
+// You can use it if you need to work with channels.
+//
+// See Examples section below for usage examples.
+package backoff
+
+import "time"
+
+// BackOff is a backoff policy for retrying an operation.
+type BackOff interface {
+	// NextBackOff returns the duration to wait before retrying the operation,
+	// or backoff.Stop to indicate that no more retries should be made.
+	//
+	// Example usage:
+	//
+	// 	duration := backoff.NextBackOff();
+	// 	if (duration == backoff.Stop) {
+	// 		// Do not retry operation.
+	// 	} else {
+	// 		// Sleep for duration and retry operation.
+	// 	}
+	//
+	NextBackOff() time.Duration
+
+	// Reset to initial state.
+	Reset()
+}
+
+// Stop indicates that no more retries should be made for use in NextBackOff().
+const Stop time.Duration = -1
+
+// ZeroBackOff is a fixed backoff policy whose backoff time is always zero,
+// meaning that the operation is retried immediately without waiting, indefinitely.
+type ZeroBackOff struct{}
+
+func (b *ZeroBackOff) Reset() {}
+
+func (b *ZeroBackOff) NextBackOff() time.Duration { return 0 }
+
+// StopBackOff is a fixed backoff policy that always returns backoff.Stop for
+// NextBackOff(), meaning that the operation should never be retried.
+type StopBackOff struct{}
+
+func (b *StopBackOff) Reset() {}
+
+func (b *StopBackOff) NextBackOff() time.Duration { return Stop }
+
+// ConstantBackOff is a backoff policy that always returns the same backoff delay.
+// This is in contrast to an exponential backoff policy,
+// which returns a delay that grows longer as you call NextBackOff() over and over again.
+type ConstantBackOff struct {
+	Interval time.Duration
+}
+
+func (b *ConstantBackOff) Reset()                     {}
+func (b *ConstantBackOff) NextBackOff() time.Duration { return b.Interval }
+
+func NewConstantBackOff(d time.Duration) *ConstantBackOff {
+	return &ConstantBackOff{Interval: d}
+}

vendor/github.com/cenkalti/backoff/backoff_test.go

@@ -0,0 +1,27 @@
+package backoff
+
+import (
+	"testing"
+	"time"
+)
+
+func TestNextBackOffMillis(t *testing.T) {
+	subtestNextBackOff(t, 0, new(ZeroBackOff))
+	subtestNextBackOff(t, Stop, new(StopBackOff))
+}
+
+func subtestNextBackOff(t *testing.T, expectedValue time.Duration, backOffPolicy BackOff) {
+	for i := 0; i < 10; i++ {
+		next := backOffPolicy.NextBackOff()
+		if next != expectedValue {
+			t.Errorf("got: %d expected: %d", next, expectedValue)
+		}
+	}
+}
+
+func TestConstantBackOff(t *testing.T) {
+	backoff := NewConstantBackOff(time.Second)
+	if backoff.NextBackOff() != time.Second {
+		t.Error("invalid interval")
+	}
+}

vendor/github.com/cenkalti/backoff/context.go

@@ -0,0 +1,60 @@
+package backoff
+
+import (
+	"time"
+
+	"golang.org/x/net/context"
+)
+
+// BackOffContext is a backoff policy that stops retrying after the context
+// is canceled.
+type BackOffContext interface {
+	BackOff
+	Context() context.Context
+}
+
+type backOffContext struct {
+	BackOff
+	ctx context.Context
+}
+
+// WithContext returns a BackOffContext with context ctx
+//
+// ctx must not be nil
+func WithContext(b BackOff, ctx context.Context) BackOffContext {
+	if ctx == nil {
+		panic("nil context")
+	}
+
+	if b, ok := b.(*backOffContext); ok {
+		return &backOffContext{
+			BackOff: b.BackOff,
+			ctx:     ctx,
+		}
+	}
+
+	return &backOffContext{
+		BackOff: b,
+		ctx:     ctx,
+	}
+}
+
+func ensureContext(b BackOff) BackOffContext {
+	if cb, ok := b.(BackOffContext); ok {
+		return cb
+	}
+	return WithContext(b, context.Background())
+}
+
+func (b *backOffContext) Context() context.Context {
+	return b.ctx
+}
+
+func (b *backOffContext) NextBackOff() time.Duration {
+	select {
+	case <-b.Context().Done():
+		return Stop
+	default:
+		return b.BackOff.NextBackOff()
+	}
+}

vendor/github.com/cenkalti/backoff/context_test.go

@@ -0,0 +1,26 @@
+package backoff
+
+import (
+	"testing"
+	"time"
+
+	"golang.org/x/net/context"
+)
+
+func TestContext(t *testing.T) {
+	b := NewConstantBackOff(time.Millisecond)
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	cb := WithContext(b, ctx)
+
+	if cb.Context() != ctx {
+		t.Error("invalid context")
+	}
+
+	cancel()
+
+	if cb.NextBackOff() != Stop {
+		t.Error("invalid next back off")
+	}
+}

vendor/github.com/cenkalti/backoff/example_test.go

@@ -0,0 +1,73 @@
+package backoff
+
+import (
+	"log"
+
+	"golang.org/x/net/context"
+)
+
+func ExampleRetry() {
+	// An operation that may fail.
+	operation := func() error {
+		return nil // or an error
+	}
+
+	err := Retry(operation, NewExponentialBackOff())
+	if err != nil {
+		// Handle error.
+		return
+	}
+
+	// Operation is successful.
+}
+
+func ExampleRetryContext() {
+	// A context
+	ctx := context.Background()
+
+	// An operation that may fail.
+	operation := func() error {
+		return nil // or an error
+	}
+
+	b := WithContext(NewExponentialBackOff(), ctx)
+
+	err := Retry(operation, b)
+	if err != nil {
+		// Handle error.
+		return
+	}
+
+	// Operation is successful.
+}
+
+func ExampleTicker() {
+	// An operation that may fail.
+	operation := func() error {
+		return nil // or an error
+	}
+
+	ticker := NewTicker(NewExponentialBackOff())
+
+	var err error
+
+	// Ticks will continue to arrive when the previous operation is still running,
+	// so operations that take a while to fail could run in quick succession.
+	for _ = range ticker.C {
+		if err = operation(); err != nil {
+			log.Println(err, "will retry...")
+			continue
+		}
+
+		ticker.Stop()
+		break
+	}
+
+	if err != nil {
+		// Operation has failed.
+		return
+	}
+
+	// Operation is successful.
+	return
+}

vendor/github.com/cenkalti/backoff/exponential.go

@@ -0,0 +1,156 @@
+package backoff
+
+import (
+	"math/rand"
+	"time"
+)
+
+/*
+ExponentialBackOff is a backoff implementation that increases the backoff
+period for each retry attempt using a randomization function that grows exponentially.
+
+NextBackOff() is calculated using the following formula:
+
+ randomized interval =
+     RetryInterval * (random value in range [1 - RandomizationFactor, 1 + RandomizationFactor])
+
+In other words NextBackOff() will range between the randomization factor
+percentage below and above the retry interval.
+
+For example, given the following parameters:
+
+ RetryInterval = 2
+ RandomizationFactor = 0.5
+ Multiplier = 2
+
+the actual backoff period used in the next retry attempt will range between 1 and 3 seconds,
+multiplied by the exponential, that is, between 2 and 6 seconds.
+
+Note: MaxInterval caps the RetryInterval and not the randomized interval.
+
+If the time elapsed since an ExponentialBackOff instance is created goes past the
+MaxElapsedTime, then the method NextBackOff() starts returning backoff.Stop.
+
+The elapsed time can be reset by calling Reset().
+
+Example: Given the following default arguments, for 10 tries the sequence will be,
+and assuming we go over the MaxElapsedTime on the 10th try:
+
+ Request #  RetryInterval (seconds)  Randomized Interval (seconds)
+
+  1          0.5                     [0.25,   0.75]
+  2          0.75                    [0.375,  1.125]
+  3          1.125                   [0.562,  1.687]
+  4          1.687                   [0.8435, 2.53]
+  5          2.53                    [1.265,  3.795]
+  6          3.795                   [1.897,  5.692]
+  7          5.692                   [2.846,  8.538]
+  8          8.538                   [4.269, 12.807]
+  9         12.807                   [6.403, 19.210]
+ 10         19.210                   backoff.Stop
+
+Note: Implementation is not thread-safe.
+*/
+type ExponentialBackOff struct {
+	InitialInterval     time.Duration
+	RandomizationFactor float64
+	Multiplier          float64
+	MaxInterval         time.Duration
+	// After MaxElapsedTime the ExponentialBackOff stops.
+	// It never stops if MaxElapsedTime == 0.
+	MaxElapsedTime time.Duration
+	Clock          Clock
+
+	currentInterval time.Duration
+	startTime       time.Time
+	random          *rand.Rand
+}
+
+// Clock is an interface that returns current time for BackOff.
+type Clock interface {
+	Now() time.Time
+}
+
+// Default values for ExponentialBackOff.
+const (
+	DefaultInitialInterval     = 500 * time.Millisecond
+	DefaultRandomizationFactor = 0.5
+	DefaultMultiplier          = 1.5
+	DefaultMaxInterval         = 60 * time.Second
+	DefaultMaxElapsedTime      = 15 * time.Minute
+)
+
+// NewExponentialBackOff creates an instance of ExponentialBackOff using default values.
+func NewExponentialBackOff() *ExponentialBackOff {
+	b := &ExponentialBackOff{
+		InitialInterval:     DefaultInitialInterval,
+		RandomizationFactor: DefaultRandomizationFactor,
+		Multiplier:          DefaultMultiplier,
+		MaxInterval:         DefaultMaxInterval,
+		MaxElapsedTime:      DefaultMaxElapsedTime,
+		Clock:               SystemClock,
+		random:              rand.New(rand.NewSource(time.Now().UnixNano())),
+	}
+	b.Reset()
+	return b
+}
+
+type systemClock struct{}
+
+func (t systemClock) Now() time.Time {
+	return time.Now()
+}
+
+// SystemClock implements Clock interface that uses time.Now().
+var SystemClock = systemClock{}
+
+// Reset the interval back to the initial retry interval and restarts the timer.
+func (b *ExponentialBackOff) Reset() {
+	b.currentInterval = b.InitialInterval
+	b.startTime = b.Clock.Now()
+}
+
+// NextBackOff calculates the next backoff interval using the formula:
+// 	Randomized interval = RetryInterval +/- (RandomizationFactor * RetryInterval)
+func (b *ExponentialBackOff) NextBackOff() time.Duration {
+	// Make sure we have not gone over the maximum elapsed time.
+	if b.MaxElapsedTime != 0 && b.GetElapsedTime() > b.MaxElapsedTime {
+		return Stop
+	}
+	defer b.incrementCurrentInterval()
+	if b.random == nil {
+		b.random = rand.New(rand.NewSource(time.Now().UnixNano()))
+	}
+	return getRandomValueFromInterval(b.RandomizationFactor, b.random.Float64(), b.currentInterval)
+}
+
+// GetElapsedTime returns the elapsed time since an ExponentialBackOff instance
+// is created and is reset when Reset() is called.
+//
+// The elapsed time is computed using time.Now().UnixNano().
+func (b *ExponentialBackOff) GetElapsedTime() time.Duration {
+	return b.Clock.Now().Sub(b.startTime)
+}
+
+// Increments the current interval by multiplying it with the multiplier.
+func (b *ExponentialBackOff) incrementCurrentInterval() {
+	// Check for overflow, if overflow is detected set the current interval to the max interval.
+	if float64(b.currentInterval) >= float64(b.MaxInterval)/b.Multiplier {
+		b.currentInterval = b.MaxInterval
+	} else {
+		b.currentInterval = time.Duration(float64(b.currentInterval) * b.Multiplier)
+	}
+}
+
+// Returns a random value from the following interval:
+// 	[randomizationFactor * currentInterval, randomizationFactor * currentInterval].
+func getRandomValueFromInterval(randomizationFactor, random float64, currentInterval time.Duration) time.Duration {
+	var delta = randomizationFactor * float64(currentInterval)
+	var minInterval = float64(currentInterval) - delta
+	var maxInterval = float64(currentInterval) + delta
+
+	// Get a random value from the range [minInterval, maxInterval].
+	// The formula used below has a +1 because if the minInterval is 1 and the maxInterval is 3 then
+	// we want a 33% chance for selecting either 1, 2 or 3.
+	return time.Duration(minInterval + (random * (maxInterval - minInterval + 1)))
+}

vendor/github.com/cenkalti/backoff/exponential_test.go

@@ -0,0 +1,108 @@
+package backoff
+
+import (
+	"math"
+	"testing"
+	"time"
+)
+
+func TestBackOff(t *testing.T) {
+	var (
+		testInitialInterval     = 500 * time.Millisecond
+		testRandomizationFactor = 0.1
+		testMultiplier          = 2.0
+		testMaxInterval         = 5 * time.Second
+		testMaxElapsedTime      = 15 * time.Minute
+	)
+
+	exp := NewExponentialBackOff()
+	exp.InitialInterval = testInitialInterval
+	exp.RandomizationFactor = testRandomizationFactor
+	exp.Multiplier = testMultiplier
+	exp.MaxInterval = testMaxInterval
+	exp.MaxElapsedTime = testMaxElapsedTime
+	exp.Reset()
+
+	var expectedResults = []time.Duration{500, 1000, 2000, 4000, 5000, 5000, 5000, 5000, 5000, 5000}
+	for i, d := range expectedResults {
+		expectedResults[i] = d * time.Millisecond
+	}
+
+	for _, expected := range expectedResults {
+		assertEquals(t, expected, exp.currentInterval)
+		// Assert that the next backoff falls in the expected range.
+		var minInterval = expected - time.Duration(testRandomizationFactor*float64(expected))
+		var maxInterval = expected + time.Duration(testRandomizationFactor*float64(expected))
+		var actualInterval = exp.NextBackOff()
+		if !(minInterval <= actualInterval && actualInterval <= maxInterval) {
+			t.Error("error")
+		}
+	}
+}
+
+func TestGetRandomizedInterval(t *testing.T) {
+	// 33% chance of being 1.
+	assertEquals(t, 1, getRandomValueFromInterval(0.5, 0, 2))
+	assertEquals(t, 1, getRandomValueFromInterval(0.5, 0.33, 2))
+	// 33% chance of being 2.
+	assertEquals(t, 2, getRandomValueFromInterval(0.5, 0.34, 2))
+	assertEquals(t, 2, getRandomValueFromInterval(0.5, 0.66, 2))
+	// 33% chance of being 3.
+	assertEquals(t, 3, getRandomValueFromInterval(0.5, 0.67, 2))
+	assertEquals(t, 3, getRandomValueFromInterval(0.5, 0.99, 2))
+}
+
+type TestClock struct {
+	i     time.Duration
+	start time.Time
+}
+
+func (c *TestClock) Now() time.Time {
+	t := c.start.Add(c.i)
+	c.i += time.Second
+	return t
+}
+
+func TestGetElapsedTime(t *testing.T) {
+	var exp = NewExponentialBackOff()
+	exp.Clock = &TestClock{}
+	exp.Reset()
+
+	var elapsedTime = exp.GetElapsedTime()
+	if elapsedTime != time.Second {
+		t.Errorf("elapsedTime=%d", elapsedTime)
+	}
+}
+
+func TestMaxElapsedTime(t *testing.T) {
+	var exp = NewExponentialBackOff()
+	exp.Clock = &TestClock{start: time.Time{}.Add(10000 * time.Second)}
+	// Change the currentElapsedTime to be 0 ensuring that the elapsed time will be greater
+	// than the max elapsed time.
+	exp.startTime = time.Time{}
+	assertEquals(t, Stop, exp.NextBackOff())
+}
+
+func TestBackOffOverflow(t *testing.T) {
+	var (
+		testInitialInterval time.Duration = math.MaxInt64 / 2
+		testMaxInterval     time.Duration = math.MaxInt64
+		testMultiplier                    = 2.1
+	)
+
+	exp := NewExponentialBackOff()
+	exp.InitialInterval = testInitialInterval
+	exp.Multiplier = testMultiplier
+	exp.MaxInterval = testMaxInterval
+	exp.Reset()
+
+	exp.NextBackOff()
+	// Assert that when an overflow is possible the current varerval   time.Duration    is set to the max varerval   time.Duration   .
+	assertEquals(t, testMaxInterval, exp.currentInterval)
+}
+
+func assertEquals(t *testing.T, expected, value time.Duration) {
+	if expected != value {
+		t.Errorf("got: %d, expected: %d", value, expected)
+	}
+}

vendor/github.com/cenkalti/backoff/retry.go

@@ -0,0 +1,78 @@
+package backoff
+
+import "time"
+
+// An Operation is executing by Retry() or RetryNotify().
+// The operation will be retried using a backoff policy if it returns an error.
+type Operation func() error
+
+// Notify is a notify-on-error function. It receives an operation error and
+// backoff delay if the operation failed (with an error).
+//
+// NOTE that if the backoff policy stated to stop retrying,
+// the notify function isn't called.
+type Notify func(error, time.Duration)
+
+// Retry the operation o until it does not return error or BackOff stops.
+// o is guaranteed to be run at least once.
+// It is the caller's responsibility to reset b after Retry returns.
+//
+// If o returns a *PermanentError, the operation is not retried, and the
+// wrapped error is returned.
+//
+// Retry sleeps the goroutine for the duration returned by BackOff after a
+// failed operation returns.
+func Retry(o Operation, b BackOff) error { return RetryNotify(o, b, nil) }
+
+// RetryNotify calls notify function with the error and wait duration
+// for each failed attempt before sleep.
+func RetryNotify(operation Operation, b BackOff, notify Notify) error {
+	var err error
+	var next time.Duration
+
+	cb := ensureContext(b)
+
+	b.Reset()
+	for {
+		if err = operation(); err == nil {
+			return nil
+		}
+
+		if permanent, ok := err.(*PermanentError); ok {
+			return permanent.Err
+		}
+
+		if next = b.NextBackOff(); next == Stop {
+			return err
+		}
+
+		if notify != nil {
+			notify(err, next)
+		}
+
+		t := time.NewTimer(next)
+
+		select {
+		case <-cb.Context().Done():
+			t.Stop()
+			return err
+		case <-t.C:
+		}
+	}
+}
+
+// PermanentError signals that the operation should not be retried.
+type PermanentError struct {
+	Err error
+}
+
+func (e *PermanentError) Error() string {
+	return e.Err.Error()
+}
+
+// Permanent wraps the given err in a *PermanentError.
+func Permanent(err error) *PermanentError {
+	return &PermanentError{
+		Err: err,
+	}
+}

vendor/github.com/cenkalti/backoff/retry_test.go

@@ -0,0 +1,99 @@
+package backoff
+
+import (
+	"errors"
+	"fmt"
+	"log"
+	"testing"
+	"time"
+
+	"golang.org/x/net/context"
+)
+
+func TestRetry(t *testing.T) {
+	const successOn = 3
+	var i = 0
+
+	// This function is successful on "successOn" calls.
+	f := func() error {
+		i++
+		log.Printf("function is called %d. time\n", i)
+
+		if i == successOn {
+			log.Println("OK")
+			return nil
+		}
+
+		log.Println("error")
+		return errors.New("error")
+	}
+
+	err := Retry(f, NewExponentialBackOff())
+	if err != nil {
+		t.Errorf("unexpected error: %s", err.Error())
+	}
+	if i != successOn {
+		t.Errorf("invalid number of retries: %d", i)
+	}
+}
+
+func TestRetryContext(t *testing.T) {
+	var cancelOn = 3
+	var i = 0
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	// This function cancels context on "cancelOn" calls.
+	f := func() error {
+		i++
+		log.Printf("function is called %d. time\n", i)
+
+		// cancelling the context in the operation function is not a typical
+		// use-case, however it allows to get predictable test results.
+		if i == cancelOn {
+			cancel()
+		}
+
+		log.Println("error")
+		return fmt.Errorf("error (%d)", i)
+	}
+
+	err := Retry(f, WithContext(NewConstantBackOff(time.Millisecond), ctx))
+	if err == nil {
+		t.Errorf("error is unexpectedly nil")
+	}
+	if err.Error() != "error (3)" {
+		t.Errorf("unexpected error: %s", err.Error())
+	}
+	if i != cancelOn {
+		t.Errorf("invalid number of retries: %d", i)
+	}
+}
+
+func TestRetryPermenent(t *testing.T) {
+	const permanentOn = 3
+	var i = 0
+
+	// This function fails permanently after permanentOn tries
+	f := func() error {
+		i++
+		log.Printf("function is called %d. time\n", i)
+
+		if i == permanentOn {
+			log.Println("permanent error")
+			return Permanent(errors.New("permanent error"))
+		}
+
+		log.Println("error")
+		return errors.New("error")
+	}
+
+	err := Retry(f, NewExponentialBackOff())
+	if err == nil || err.Error() != "permanent error" {
+		t.Errorf("unexpected error: %s", err)
+	}
+	if i != permanentOn {
+		t.Errorf("invalid number of retries: %d", i)
+	}
+}

vendor/github.com/cenkalti/backoff/ticker.go

@@ -0,0 +1,81 @@
+package backoff
+
+import (
+	"runtime"
+	"sync"
+	"time"
+)
+
+// Ticker holds a channel that delivers `ticks' of a clock at times reported by a BackOff.
+//
+// Ticks will continue to arrive when the previous operation is still running,
+// so operations that take a while to fail could run in quick succession.
+type Ticker struct {
+	C        <-chan time.Time
+	c        chan time.Time
+	b        BackOffContext
+	stop     chan struct{}
+	stopOnce sync.Once
+}
+
+// NewTicker returns a new Ticker containing a channel that will send the time at times
+// specified by the BackOff argument. Ticker is guaranteed to tick at least once.
+// The channel is closed when Stop method is called or BackOff stops.
+func NewTicker(b BackOff) *Ticker {
+	c := make(chan time.Time)
+	t := &Ticker{
+		C:    c,
+		c:    c,
+		b:    ensureContext(b),
+		stop: make(chan struct{}),
+	}
+	go t.run()
+	runtime.SetFinalizer(t, (*Ticker).Stop)
+	return t
+}
+
+// Stop turns off a ticker. After Stop, no more ticks will be sent.
+func (t *Ticker) Stop() {
+	t.stopOnce.Do(func() { close(t.stop) })
+}
+
+func (t *Ticker) run() {
+	c := t.c
+	defer close(c)
+	t.b.Reset()
+
+	// Ticker is guaranteed to tick at least once.
+	afterC := t.send(time.Now())
+
+	for {
+		if afterC == nil {
+			return
+		}
+
+		select {
+		case tick := <-afterC:
+			afterC = t.send(tick)
+		case <-t.stop:
+			t.c = nil // Prevent future ticks from being sent to the channel.
+			return
+		case <-t.b.Context().Done():
+			return
+		}
+	}
+}
+
+func (t *Ticker) send(tick time.Time) <-chan time.Time {
+	select {
+	case t.c <- tick:
+	case <-t.stop:
+		return nil
+	}
+
+	next := t.b.NextBackOff()
+	if next == Stop {
+		t.Stop()
+		return nil
+	}
+
+	return time.After(next)
+}

vendor/github.com/cenkalti/backoff/ticker_test.go

@@ -0,0 +1,94 @@
+package backoff
+
+import (
+	"errors"
+	"fmt"
+	"log"
+	"testing"
+	"time"
+
+	"golang.org/x/net/context"
+)
+
+func TestTicker(t *testing.T) {
+	const successOn = 3
+	var i = 0
+
+	// This function is successful on "successOn" calls.
+	f := func() error {
+		i++
+		log.Printf("function is called %d. time\n", i)
+
+		if i == successOn {
+			log.Println("OK")
+			return nil
+		}
+
+		log.Println("error")
+		return errors.New("error")
+	}
+
+	b := NewExponentialBackOff()
+	ticker := NewTicker(b)
+
+	var err error
+	for _ = range ticker.C {
+		if err = f(); err != nil {
+			t.Log(err)
+			continue
+		}
+
+		break
+	}
+	if err != nil {
+		t.Errorf("unexpected error: %s", err.Error())
+	}
+	if i != successOn {
+		t.Errorf("invalid number of retries: %d", i)
+	}
+}
+
+func TestTickerContext(t *testing.T) {
+	const cancelOn = 3
+	var i = 0
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	// This function cancels context on "cancelOn" calls.
+	f := func() error {
+		i++
+		log.Printf("function is called %d. time\n", i)
+
+		// cancelling the context in the operation function is not a typical
+		// use-case, however it allows to get predictable test results.
+		if i == cancelOn {
+			cancel()
+		}
+
+		log.Println("error")
+		return fmt.Errorf("error (%d)", i)
+	}
+
+	b := WithContext(NewConstantBackOff(time.Millisecond), ctx)
+	ticker := NewTicker(b)
+
+	var err error
+	for _ = range ticker.C {
+		if err = f(); err != nil {
+			t.Log(err)
+			continue
+		}
+
+		break
+	}
+	if err == nil {
+		t.Errorf("error is unexpectedly nil")
+	}
+	if err.Error() != "error (3)" {
+		t.Errorf("unexpected error: %s", err.Error())
+	}
+	if i != cancelOn {
+		t.Errorf("invalid number of retries: %d", i)
+	}
+}

vendor/github.com/cenkalti/backoff/tries.go

@@ -0,0 +1,35 @@
+package backoff
+
+import "time"
+
+/*
+WithMaxTries creates a wrapper around another BackOff, which will
+return Stop if NextBackOff() has been called too many times since
+the last time Reset() was called
+
+Note: Implementation is not thread-safe.
+*/
+func WithMaxTries(b BackOff, max uint64) BackOff {
+	return &backOffTries{delegate: b, maxTries: max}
+}
+
+type backOffTries struct {
+	delegate BackOff
+	maxTries uint64
+	numTries uint64
+}
+
+func (b *backOffTries) NextBackOff() time.Duration {
+	if b.maxTries > 0 {
+		if b.maxTries <= b.numTries {
+			return Stop
+		}
+		b.numTries++
+	}
+	return b.delegate.NextBackOff()
+}
+
+func (b *backOffTries) Reset() {
+	b.numTries = 0
+	b.delegate.Reset()
+}

vendor/github.com/cenkalti/backoff/tries_test.go

@@ -0,0 +1,55 @@
+package backoff
+
+import (
+	"math/rand"
+	"testing"
+	"time"
+)
+
+func TestMaxTriesHappy(t *testing.T) {
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+	max := 17 + r.Intn(13)
+	bo := WithMaxTries(&ZeroBackOff{}, uint64(max))
+
+	// Load up the tries count, but reset should clear the record
+	for ix := 0; ix < max/2; ix++ {
+		bo.NextBackOff()
+	}
+	bo.Reset()
+
+	// Now fill the tries count all the way up
+	for ix := 0; ix < max; ix++ {
+		d := bo.NextBackOff()
+		if d == Stop {
+			t.Errorf("returned Stop on try %d", ix)
+		}
+	}
+
+	// We have now called the BackOff max number of times, we expect
+	// the next result to be Stop, even if we try it multiple times
+	for ix := 0; ix < 7; ix++ {
+		d := bo.NextBackOff()
+		if d != Stop {
+			t.Error("invalid next back off")
+		}
+	}
+
+	// Reset makes it all work again
+	bo.Reset()
+	d := bo.NextBackOff()
+	if d == Stop {
+		t.Error("returned Stop after reset")
+	}
+
+}
+
+func TestMaxTriesZero(t *testing.T) {
+	// It might not make sense, but its okay to send a zero
+	bo := WithMaxTries(&ZeroBackOff{}, uint64(0))
+	for ix := 0; ix < 11; ix++ {
+		d := bo.NextBackOff()
+		if d == Stop {
+			t.Errorf("returned Stop on try %d", ix)
+		}
+	}
+}