KurlesHS/restic
1
0
Fork 0
forked from TrueCloudLab/restic
Code Pull requests Activity

Add []byte to repo.LoadAndDecrypt and utils.LoadAll

Browse source 
This commit changes the signatures for repository.LoadAndDecrypt and
utils.LoadAll to allow passing in a []byte as the buffer to use. This
buffer is enlarged as needed, and returned back to the caller for
further use.

In later commits, this allows reducing allocations by reusing a buffer
for multiple calls, e.g. in a worker function.
This commit is contained in:
Alexander Neumann 2019-03-24 21:59:14 +01:00
parent e046428c94
commit d51e9d1b98
10 changed files with 112 additions and 71 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
cmd/restic/cmd_cat.go
View file

@ -74,7 +74,7 @@ func runCat(gopts GlobalOptions, args []string) error {
fmt.Println(string(buf))
return nil
case "index":
buf, err := repo.LoadAndDecrypt(gopts.ctx, restic.IndexFile, id)
buf, err := repo.LoadAndDecrypt(gopts.ctx, nil, restic.IndexFile, id)
if err != nil {
return err
}
@ -99,7 +99,7 @@ func runCat(gopts GlobalOptions, args []string) error {
return nil
case "key":
h := restic.Handle{Type: restic.KeyFile, Name: id.String()}
buf, err := backend.LoadAll(gopts.ctx, repo.Backend(), h)
buf, err := backend.LoadAll(gopts.ctx, nil, repo.Backend(), h)
if err != nil {
return err
}
@ -150,7 +150,7 @@ func runCat(gopts GlobalOptions, args []string) error {
switch tpe {
case "pack":
h := restic.Handle{Type: restic.DataFile, Name: id.String()}
buf, err := backend.LoadAll(gopts.ctx, repo.Backend(), h)
buf, err := backend.LoadAll(gopts.ctx, nil, repo.Backend(), h)
if err != nil {
return err
}

10
internal/backend/test/tests.go
View file

@ -79,7 +79,7 @@ func (s *Suite) TestConfig(t *testing.T) {
var testString = "Config"
// create config and read it back
_, err := backend.LoadAll(context.TODO(), b, restic.Handle{Type: restic.ConfigFile})
_, err := backend.LoadAll(context.TODO(), nil, b, restic.Handle{Type: restic.ConfigFile})
if err == nil {
t.Fatalf("did not get expected error for non-existing config")
}
@ -93,7 +93,7 @@ func (s *Suite) TestConfig(t *testing.T) {
// same config
for _, name := range []string{"", "foo", "bar", "0000000000000000000000000000000000000000000000000000000000000000"} {
h := restic.Handle{Type: restic.ConfigFile, Name: name}
buf, err := backend.LoadAll(context.TODO(), b, h)
buf, err := backend.LoadAll(context.TODO(), nil, b, h)
if err != nil {
t.Fatalf("unable to read config with name %q: %+v", name, err)
}
@ -491,7 +491,7 @@ func (s *Suite) TestSave(t *testing.T) {
err := b.Save(context.TODO(), h, restic.NewByteReader(data))
test.OK(t, err)
buf, err := backend.LoadAll(context.TODO(), b, h)
buf, err := backend.LoadAll(context.TODO(), nil, b, h)
test.OK(t, err)
if len(buf) != len(data) {
t.Fatalf("number of bytes does not match, want %v, got %v", len(data), len(buf))
@ -584,7 +584,7 @@ func (s *Suite) TestSaveFilenames(t *testing.T) {
continue
}
buf, err := backend.LoadAll(context.TODO(), b, h)
buf, err := backend.LoadAll(context.TODO(), nil, b, h)
if err != nil {
t.Errorf("test %d failed: Load() returned %+v", i, err)
continue
@ -734,7 +734,7 @@ func (s *Suite) TestBackend(t *testing.T) {
// test Load()
h := restic.Handle{Type: tpe, Name: ts.id}
buf, err := backend.LoadAll(context.TODO(), b, h)
buf, err := backend.LoadAll(context.TODO(), nil, b, h)
test.OK(t, err)
test.Equals(t, ts.data, string(buf))

27
internal/backend/utils.go
View file

@ -1,20 +1,33 @@
package backend
import (
"bytes"
"context"
"io"
"io/ioutil"
"github.com/restic/restic/internal/restic"
)
// LoadAll reads all data stored in the backend for the handle.
func LoadAll(ctx context.Context, be restic.Backend, h restic.Handle) (buf []byte, err error) {
err = be.Load(ctx, h, 0, 0, func(rd io.Reader) (ierr error) {
buf, ierr = ioutil.ReadAll(rd)
return ierr
// LoadAll reads all data stored in the backend for the handle into the given
// buffer, which is truncated. If the buffer is not large enough or nil, a new
// one is allocated.
func LoadAll(ctx context.Context, buf []byte, be restic.Backend, h restic.Handle) ([]byte, error) {
err := be.Load(ctx, h, 0, 0, func(rd io.Reader) error {
// make sure this is idempotent, in case an error occurs this function may be called multiple times!
wr := bytes.NewBuffer(buf[:0])
_, cerr := io.Copy(wr, rd)
if cerr != nil {
return cerr
}
buf = wr.Bytes()
return nil
})
return buf, err
if err != nil {
return nil, err
}
return buf, nil
}
// LimitedReadCloser wraps io.LimitedReader and exposes the Close() method.

92
internal/backend/utils_test.go
View file

@ -19,6 +19,7 @@ const MiB = 1 << 20
func TestLoadAll(t *testing.T) {
b := mem.New()
var buf []byte
for i := 0; i < 20; i++ {
data := rtest.Random(23+i, rand.Intn(MiB)+500*KiB)
@ -28,7 +29,7 @@ func TestLoadAll(t *testing.T) {
err := b.Save(context.TODO(), h, restic.NewByteReader(data))
rtest.OK(t, err)
buf, err := backend.LoadAll(context.TODO(), b, restic.Handle{Type: restic.DataFile, Name: id.String()})
buf, err := backend.LoadAll(context.TODO(), buf, b, restic.Handle{Type: restic.DataFile, Name: id.String()})
rtest.OK(t, err)
if len(buf) != len(data) {
@ -43,55 +44,66 @@ func TestLoadAll(t *testing.T) {
}
}
func TestLoadSmallBuffer(t *testing.T) {
b := mem.New()
for i := 0; i < 20; i++ {
data := rtest.Random(23+i, rand.Intn(MiB)+500*KiB)
id := restic.Hash(data)
func save(t testing.TB, be restic.Backend, buf []byte) restic.Handle {
id := restic.Hash(buf)
h := restic.Handle{Name: id.String(), Type: restic.DataFile}
err := b.Save(context.TODO(), h, restic.NewByteReader(data))
rtest.OK(t, err)
buf, err := backend.LoadAll(context.TODO(), b, restic.Handle{Type: restic.DataFile, Name: id.String()})
rtest.OK(t, err)
if len(buf) != len(data) {
t.Errorf("length of returned buffer does not match, want %d, got %d", len(data), len(buf))
continue
}
if !bytes.Equal(buf, data) {
t.Errorf("wrong data returned")
continue
}
err := be.Save(context.TODO(), h, restic.NewByteReader(buf))
if err != nil {
t.Fatal(err)
}
return h
}
func TestLoadLargeBuffer(t *testing.T) {
func TestLoadAllAppend(t *testing.T) {
b := mem.New()
for i := 0; i < 20; i++ {
data := rtest.Random(23+i, rand.Intn(MiB)+500*KiB)
h1 := save(t, b, []byte("foobar test string"))
randomData := rtest.Random(23, rand.Intn(MiB)+500*KiB)
h2 := save(t, b, randomData)
id := restic.Hash(data)
h := restic.Handle{Name: id.String(), Type: restic.DataFile}
err := b.Save(context.TODO(), h, restic.NewByteReader(data))
rtest.OK(t, err)
buf, err := backend.LoadAll(context.TODO(), b, restic.Handle{Type: restic.DataFile, Name: id.String()})
rtest.OK(t, err)
if len(buf) != len(data) {
t.Errorf("length of returned buffer does not match, want %d, got %d", len(data), len(buf))
continue
var tests = []struct {
handle restic.Handle
buf []byte
want []byte
}{
{
handle: h1,
buf: nil,
want: []byte("foobar test string"),
},
{
handle: h1,
buf: []byte("xxx"),
want: []byte("foobar test string"),
},
{
handle: h2,
buf: nil,
want: randomData,
},
{
handle: h2,
buf: make([]byte, 0, 200),
want: randomData,
},
{
handle: h2,
buf: []byte("foobarbaz"),
want: randomData,
},
}
if !bytes.Equal(buf, data) {
t.Errorf("wrong data returned")
continue
for _, test := range tests {
t.Run("", func(t *testing.T) {
buf, err := backend.LoadAll(context.TODO(), test.buf, b, test.handle)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(buf, test.want) {
t.Errorf("wrong data returned, want %q, got %q", test.want, buf)
}
})
}
}

4
internal/cache/backend_test.go vendored
View file

@ -17,7 +17,7 @@ import (
)
func loadAndCompare(t testing.TB, be restic.Backend, h restic.Handle, data []byte) {
buf, err := backend.LoadAll(context.TODO(), be, h)
buf, err := backend.LoadAll(context.TODO(), nil, be, h)
if err != nil {
t.Fatal(err)
}
@ -147,7 +147,7 @@ func TestErrorBackend(t *testing.T) {
loadTest := func(wg *sync.WaitGroup, be restic.Backend) {
defer wg.Done()
buf, err := backend.LoadAll(context.TODO(), be, h)
buf, err := backend.LoadAll(context.TODO(), nil, be, h)
if err == testErr {
return
}

2
internal/repository/index.go
View file

@ -552,7 +552,7 @@ func DecodeOldIndex(buf []byte) (idx *Index, err error) {
func LoadIndexWithDecoder(ctx context.Context, repo restic.Repository, id restic.ID, fn func([]byte) (*Index, error)) (idx *Index, err error) {
debug.Log("Loading index %v", id)
buf, err := repo.LoadAndDecrypt(ctx, restic.IndexFile, id)
buf, err := repo.LoadAndDecrypt(ctx, nil, restic.IndexFile, id)
if err != nil {
return nil, err
}

2
internal/repository/key.go
View file

@ -184,7 +184,7 @@ func SearchKey(ctx context.Context, s *Repository, password string, maxKeys int,
// LoadKey loads a key from the backend.
func LoadKey(ctx context.Context, s *Repository, name string) (k *Key, err error) {
h := restic.Handle{Type: restic.KeyFile, Name: name}
data, err := backend.LoadAll(ctx, s.be, h)
data, err := backend.LoadAll(ctx, nil, s.be, h)
if err != nil {
return nil, err
}

27
internal/repository/repository.go
View file

@ -9,7 +9,6 @@ import (
"io"
"os"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/cache"
"github.com/restic/restic/internal/crypto"
"github.com/restic/restic/internal/debug"
@ -67,15 +66,29 @@ func (r *Repository) PrefixLength(t restic.FileType) (int, error) {
return restic.PrefixLength(r.be, t)
}
// LoadAndDecrypt loads and decrypts data identified by t and id from the
// backend.
func (r *Repository) LoadAndDecrypt(ctx context.Context, t restic.FileType, id restic.ID) (buf []byte, err error) {
// LoadAndDecrypt loads and decrypts the file with the given type and ID, using
// the supplied buffer (which must be empty). If the buffer is nil, a new
// buffer will be allocated and returned.
func (r *Repository) LoadAndDecrypt(ctx context.Context, buf []byte, t restic.FileType, id restic.ID) ([]byte, error) {
if len(buf) != 0 {
panic("buf is not empty")
}
debug.Log("load %v with id %v", t, id)
h := restic.Handle{Type: t, Name: id.String()}
buf, err = backend.LoadAll(ctx, r.be, h)
err := r.be.Load(ctx, h, 0, 0, func(rd io.Reader) error {
// make sure this call is idempotent, in case an error occurs
wr := bytes.NewBuffer(buf[:0])
_, cerr := io.Copy(wr, rd)
if cerr != nil {
return cerr
}
buf = wr.Bytes()
return nil
})
if err != nil {
debug.Log("error loading %v: %v", h, err)
return nil, err
}
@ -188,7 +201,7 @@ func (r *Repository) loadBlob(ctx context.Context, id restic.ID, t restic.BlobTy
// LoadJSONUnpacked decrypts the data and afterwards calls json.Unmarshal on
// the item.
func (r *Repository) LoadJSONUnpacked(ctx context.Context, t restic.FileType, id restic.ID, item interface{}) (err error) {
buf, err := r.LoadAndDecrypt(ctx, t, id)
buf, err := r.LoadAndDecrypt(ctx, nil, t, id)
if err != nil {
return err
}

2
internal/repository/repository_test.go
View file

@ -244,7 +244,7 @@ func BenchmarkLoadAndDecrypt(b *testing.B) {
b.SetBytes(int64(length))
for i := 0; i < b.N; i++ {
data, err := repo.LoadAndDecrypt(context.TODO(), restic.DataFile, storageID)
data, err := repo.LoadAndDecrypt(context.TODO(), nil, restic.DataFile, storageID)
rtest.OK(b, err)
if len(data) != length {
b.Errorf("wanted %d bytes, got %d", length, len(data))

7
internal/restic/repository.go
View file

@ -39,8 +39,11 @@ type Repository interface {
SaveUnpacked(context.Context, FileType, []byte) (ID, error)
SaveJSONUnpacked(context.Context, FileType, interface{}) (ID, error)
LoadJSONUnpacked(context.Context, FileType, ID, interface{}) error
LoadAndDecrypt(context.Context, FileType, ID) ([]byte, error)
LoadJSONUnpacked(ctx context.Context, t FileType, id ID, dest interface{}) error
// LoadAndDecrypt loads and decrypts the file with the given type and ID,
// using the supplied buffer (which must be empty). If the buffer is nil, a
// new buffer will be allocated and returned.
LoadAndDecrypt(ctx context.Context, buf []byte, t FileType, id ID) (data []byte, err error)
LoadBlob(context.Context, BlobType, ID, []byte) (int, error)
SaveBlob(context.Context, BlobType, []byte, ID) (ID, error)