Merge pull request #1362 from restic/fix-backend-retry

Fix Save() backend retry
This commit is contained in:
Alexander Neumann 2017-10-18 07:45:56 +02:00 committed by GitHub
commit 7507a658ac
4 changed files with 150 additions and 30 deletions

View file

@ -3,6 +3,7 @@ package backend
import (
"context"
"io"
"io/ioutil"
"math/rand"
"sync"
@ -14,6 +15,7 @@ import (
// the retry functions.
type ErrorBackend struct {
FailSave float32
FailSaveRead float32
FailLoad float32
FailStat float32
restic.Backend
@ -48,6 +50,15 @@ func (be *ErrorBackend) Save(ctx context.Context, h restic.Handle, rd io.Reader)
return errors.Errorf("Save(%v) random error induced", h)
}
if be.fail(be.FailSaveRead) {
_, err := io.CopyN(ioutil.Discard, rd, be.r.Int63n(1000))
if err != nil {
return err
}
return errors.Errorf("Save(%v) random error with partial read induced", h)
}
return be.Backend.Save(ctx, h, rd)
}

View file

@ -7,6 +7,7 @@ import (
"time"
"github.com/cenkalti/backoff"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
)
@ -44,7 +45,26 @@ func (be *RetryBackend) retry(msg string, f func() error) error {
// Save stores the data in the backend under the given handle.
func (be *RetryBackend) Save(ctx context.Context, h restic.Handle, rd io.Reader) error {
seeker, ok := rd.(io.Seeker)
if !ok {
return errors.Errorf("reader %T is not a seeker", rd)
}
pos, err := seeker.Seek(0, io.SeekCurrent)
if err != nil {
return errors.Wrap(err, "Seek")
}
if pos != 0 {
return errors.Errorf("reader is not at the beginning (pos %v)", pos)
}
return be.retry(fmt.Sprintf("Save(%v)", h), func() error {
_, err := seeker.Seek(0, io.SeekStart)
if err != nil {
return err
}
return be.Backend.Save(ctx, h, rd)
})
}

View file

@ -0,0 +1,90 @@
package backend
import (
"bytes"
"context"
"io"
"io/ioutil"
"testing"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/mock"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/test"
)
func TestBackendRetrySeeker(t *testing.T) {
be := &mock.Backend{
SaveFn: func(ctx context.Context, h restic.Handle, rd io.Reader) error {
return nil
},
}
retryBackend := RetryBackend{
Backend: be,
}
data := test.Random(24, 23*14123)
type wrapReader struct {
io.Reader
}
var rd io.Reader
rd = wrapReader{bytes.NewReader(data)}
err := retryBackend.Save(context.TODO(), restic.Handle{}, rd)
if err == nil {
t.Fatal("did not get expected error for retry backend with non-seeker reader")
}
rd = bytes.NewReader(data)
_, err = io.CopyN(ioutil.Discard, rd, 5)
if err != nil {
t.Fatal(err)
}
err = retryBackend.Save(context.TODO(), restic.Handle{}, rd)
if err == nil {
t.Fatal("did not get expected error for partial reader")
}
}
func TestBackendSaveRetry(t *testing.T) {
buf := bytes.NewBuffer(nil)
errcount := 0
be := &mock.Backend{
SaveFn: func(ctx context.Context, h restic.Handle, rd io.Reader) error {
if errcount == 0 {
errcount++
_, err := io.CopyN(ioutil.Discard, rd, 120)
if err != nil {
return err
}
return errors.New("injected error")
}
_, err := io.Copy(buf, rd)
return err
},
}
retryBackend := RetryBackend{
Backend: be,
}
data := test.Random(23, 5*1024*1024+11241)
err := retryBackend.Save(context.TODO(), restic.Handle{}, bytes.NewReader(data))
if err != nil {
t.Fatal(err)
}
if len(data) != buf.Len() {
t.Errorf("wrong number of bytes written: want %d, got %d", len(data), buf.Len())
}
if !bytes.Equal(data, buf.Bytes()) {
t.Errorf("wrong data written to backend")
}
}

View file

@ -5,6 +5,7 @@ import (
"io"
"sync"
"github.com/pkg/errors"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/restic"
)
@ -43,52 +44,50 @@ func (b *Backend) Remove(ctx context.Context, h restic.Handle) error {
return b.Cache.Remove(h)
}
type teeReader struct {
rd io.Reader
wr io.Writer
err error
}
func (t *teeReader) Read(p []byte) (n int, err error) {
n, err = t.rd.Read(p)
if t.err == nil && n > 0 {
_, t.err = t.wr.Write(p[:n])
}
return n, err
}
var autoCacheTypes = map[restic.FileType]struct{}{
restic.IndexFile: struct{}{},
restic.SnapshotFile: struct{}{},
}
// Save stores a new file is the backend and the cache.
// Save stores a new file in the backend and the cache.
func (b *Backend) Save(ctx context.Context, h restic.Handle, rd io.Reader) (err error) {
if _, ok := autoCacheTypes[h.Type]; !ok {
return b.Backend.Save(ctx, h, rd)
}
debug.Log("Save(%v): auto-store in the cache", h)
wr, err := b.Cache.SaveWriter(h)
if err != nil {
debug.Log("unable to save %v to cache: %v", h, err)
return b.Backend.Save(ctx, h, rd)
seeker, ok := rd.(io.Seeker)
if !ok {
return errors.New("reader is not a seeker")
}
tr := &teeReader{rd: rd, wr: wr}
err = b.Backend.Save(ctx, h, tr)
pos, err := seeker.Seek(0, io.SeekCurrent)
if err != nil {
return errors.Wrapf(err, "Seek")
}
if pos != 0 {
return errors.Errorf("reader is not rewind (pos %d)", pos)
}
err = b.Backend.Save(ctx, h, rd)
if err != nil {
wr.Close()
b.Cache.Remove(h)
return err
}
err = wr.Close()
_, err = seeker.Seek(pos, io.SeekStart)
if err != nil {
debug.Log("cache writer returned error: %v", err)
_ = b.Cache.Remove(h)
return errors.Wrapf(err, "Seek")
}
err = b.Cache.Save(h, rd)
if err != nil {
debug.Log("unable to save %v to cache: %v", h, err)
_ = b.Cache.Remove(h)
return nil
}
return nil
}