KurlesHS/restic
1
0
Fork 0
forked from TrueCloudLab/restic
Code Pull requests Activity

Merge pull request #731 from restic/improve-memory-usage

Browse source 
Improve memory usage
This commit is contained in:
Alexander Neumann 2017-01-20 15:56:31 +01:00
commit 164ba823e5
15 changed files with 304 additions and 156 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
src/cmds/restic/cmd_prune.go
View file

@ -103,11 +103,13 @@ func runPrune(gopts GlobalOptions) error {
return err
}
blobs := 0
for _, pack := range idx.Packs {
stats.bytes += pack.Size
blobs += len(pack.Entries)
}
Verbosef("repository contains %v packs (%v blobs) with %v bytes\n",
len(idx.Packs), len(idx.Blobs), formatBytes(uint64(stats.bytes)))
len(idx.Packs), blobs, formatBytes(uint64(stats.bytes)))
blobCount := make(map[restic.BlobHandle]int)
duplicateBlobs := 0
@ -164,14 +166,17 @@ func runPrune(gopts GlobalOptions) error {
// find packs that need a rewrite
rewritePacks := restic.NewIDSet()
for h, blob := range idx.Blobs {
if !usedBlobs.Has(h) {
rewritePacks.Merge(blob.Packs)
continue
}
for _, pack := range idx.Packs {
for _, blob := range pack.Entries {
h := restic.BlobHandle{ID: blob.ID, Type: blob.Type}
if !usedBlobs.Has(h) {
rewritePacks.Insert(pack.ID)
continue
}
if blobCount[h] > 1 {
rewritePacks.Merge(blob.Packs)
if blobCount[h] > 1 {
rewritePacks.Insert(pack.ID)
}
}
}

2
src/restic/archiver/archive_reader_test.go
View file

@ -44,7 +44,7 @@ func checkSavedFile(t *testing.T, repo restic.Repository, treeID restic.ID, name
t.Fatal(err)
}
buf := make([]byte, int(size))
buf := restic.NewBlobBuffer(int(size))
n := loadBlob(t, repo, id, buf)
if n != len(buf) {
t.Errorf("wrong number of bytes read, want %d, got %d", len(buf), n)

21
src/restic/buffer.go Normal file
View file

@ -0,0 +1,21 @@
package restic
import "restic/crypto"
// NewBlobBuffer returns a buffer that is large enough to hold a blob of size
// plaintext bytes, including the crypto overhead.
func NewBlobBuffer(size int) []byte {
return make([]byte, size, size+crypto.Extension)
}
// PlaintextLength returns the plaintext length of a blob with ciphertextSize
// bytes.
func PlaintextLength(ciphertextSize int) int {
return ciphertextSize - crypto.Extension
}
// CiphertextLength returns the encrypted length of a blob with plaintextSize
// bytes.
func CiphertextLength(plaintextSize int) int {
return plaintextSize + crypto.Extension
}

81
src/restic/index/index.go
View file

@ -14,27 +14,20 @@ import (
// Pack contains information about the contents of a pack.
type Pack struct {
ID restic.ID
Size int64
Entries []restic.Blob
}
// Blob contains information about a blob.
type Blob struct {
Size int64
Packs restic.IDSet
}
// Index contains information about blobs and packs stored in a repo.
type Index struct {
Packs map[restic.ID]Pack
Blobs map[restic.BlobHandle]Blob
IndexIDs restic.IDSet
}
func newIndex() *Index {
return &Index{
Packs: make(map[restic.ID]Pack),
Blobs: make(map[restic.BlobHandle]Blob),
IndexIDs: restic.NewIDSet(),
}
}
@ -69,9 +62,6 @@ func New(repo restic.Repository, p *restic.Progress) (*Index, error) {
if err != nil {
return nil, err
}
p := Pack{Entries: j.Entries(), Size: j.Size()}
idx.Packs[packID] = p
}
return idx, nil
@ -181,18 +171,6 @@ func (idx *Index) AddPack(id restic.ID, size int64, entries []restic.Blob) error
idx.Packs[id] = Pack{Size: size, Entries: entries}
for _, entry := range entries {
h := restic.BlobHandle{ID: entry.ID, Type: entry.Type}
if _, ok := idx.Blobs[h]; !ok {
idx.Blobs[h] = Blob{
Size: int64(entry.Length),
Packs: restic.NewIDSet(),
}
}
idx.Blobs[h].Packs.Insert(id)
}
return nil
}
@ -202,15 +180,6 @@ func (idx *Index) RemovePack(id restic.ID) error {
return errors.Errorf("pack %v not found in the index", id.Str())
}
for _, blob := range idx.Packs[id].Entries {
h := restic.BlobHandle{ID: blob.ID, Type: blob.Type}
idx.Blobs[h].Packs.Delete(id)
if len(idx.Blobs[h].Packs) == 0 {
delete(idx.Blobs, h)
}
}
delete(idx.Packs, id)
return nil
@ -239,14 +208,11 @@ func (idx *Index) DuplicateBlobs() (dups restic.BlobSet) {
func (idx *Index) PacksForBlobs(blobs restic.BlobSet) (packs restic.IDSet) {
packs = restic.NewIDSet()
for h := range blobs {
blob, ok := idx.Blobs[h]
if !ok {
continue
}
for id := range blob.Packs {
packs.Insert(id)
for id, p := range idx.Packs {
for _, entry := range p.Entries {
if blobs.Has(restic.BlobHandle{ID: entry.ID, Type: entry.Type}) {
packs.Insert(id)
}
}
}
@ -264,31 +230,20 @@ type Location struct {
var ErrBlobNotFound = errors.New("blob not found in index")
// FindBlob returns a list of packs and positions the blob can be found in.
func (idx *Index) FindBlob(h restic.BlobHandle) ([]Location, error) {
blob, ok := idx.Blobs[h]
if !ok {
return nil, ErrBlobNotFound
func (idx *Index) FindBlob(h restic.BlobHandle) (result []Location, err error) {
for id, p := range idx.Packs {
for _, entry := range p.Entries {
if entry.ID.Equal(h.ID) && entry.Type == h.Type {
result = append(result, Location{
PackID: id,
Blob: entry,
})
}
}
}
result := make([]Location, 0, len(blob.Packs))
for packID := range blob.Packs {
pack, ok := idx.Packs[packID]
if !ok {
return nil, errors.Errorf("pack %v not found in index", packID.Str())
}
for _, entry := range pack.Entries {
if entry.Type != h.Type {
continue
}
if !entry.ID.Equal(h.ID) {
continue
}
loc := Location{PackID: packID, Blob: entry}
result = append(result, loc)
}
if len(result) == 0 {
return nil, ErrBlobNotFound
}
return result, nil

80
src/restic/index/index_test.go
View file

@ -3,7 +3,9 @@ package index
import (
"math/rand"
"restic"
"restic/checker"
"restic/repository"
"restic/test"
"testing"
"time"
)
@ -135,6 +137,40 @@ func BenchmarkIndexNew(b *testing.B) {
if idx == nil {
b.Fatalf("New() returned nil index")
}
b.Logf("idx %v packs", len(idx.Packs))
}
}
func BenchmarkIndexSave(b *testing.B) {
repo, cleanup := repository.TestRepository(b)
defer cleanup()
idx, err := New(repo, nil)
test.OK(b, err)
for i := 0; i < 8000; i++ {
entries := make([]restic.Blob, 0, 200)
for j := 0; j < len(entries); j++ {
entries = append(entries, restic.Blob{
ID: restic.NewRandomID(),
Length: 1000,
Offset: 5,
Type: restic.DataBlob,
})
}
idx.AddPack(restic.NewRandomID(), 10000, entries)
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
id, err := idx.Save(repo, nil)
if err != nil {
b.Fatalf("New() returned error %v", err)
}
b.Logf("saved as %v", id.Str())
}
}
@ -151,7 +187,7 @@ func TestIndexDuplicateBlobs(t *testing.T) {
if len(dups) == 0 {
t.Errorf("no duplicate blobs found")
}
t.Logf("%d packs, %d unique blobs", len(idx.Packs), len(idx.Blobs))
t.Logf("%d packs, %d duplicate blobs", len(idx.Packs), len(dups))
packs := idx.PacksForBlobs(dups)
if len(packs) == 0 {
@ -169,7 +205,7 @@ func loadIndex(t testing.TB, repo restic.Repository) *Index {
return idx
}
func TestIndexSave(t *testing.T) {
func TestSave(t *testing.T) {
repo, cleanup := createFilledRepo(t, 3, 0)
defer cleanup()
@ -219,6 +255,41 @@ func TestIndexSave(t *testing.T) {
}
}
func TestIndexSave(t *testing.T) {
repo, cleanup := createFilledRepo(t, 3, 0)
defer cleanup()
idx := loadIndex(t, repo)
id, err := idx.Save(repo, idx.IndexIDs.List())
if err != nil {
t.Fatalf("unable to save new index: %v", err)
}
t.Logf("new index saved as %v", id.Str())
for id := range idx.IndexIDs {
t.Logf("remove index %v", id.Str())
err = repo.Backend().Remove(restic.IndexFile, id.String())
if err != nil {
t.Errorf("error removing index %v: %v", id, err)
}
}
idx2 := loadIndex(t, repo)
t.Logf("load new index with %d packs", len(idx2.Packs))
checker := checker.New(repo)
hints, errs := checker.LoadIndex()
for _, h := range hints {
t.Logf("hint: %v\n", h)
}
for _, err := range errs {
t.Errorf("checker found error: %v", err)
}
}
func TestIndexAddRemovePack(t *testing.T) {
repo, cleanup := createFilledRepo(t, 3, 0)
defer cleanup()
@ -249,12 +320,7 @@ func TestIndexAddRemovePack(t *testing.T) {
if err == nil {
t.Errorf("removed blob %v found in index", h)
}
if _, ok := idx.Blobs[h]; ok {
t.Errorf("removed blob %v found in index.Blobs", h)
}
}
}
// example index serialization from doc/Design.md

2
src/restic/node.go
View file

@ -208,7 +208,7 @@ func (node Node) createFileAt(path string, repo Repository) error {
buf = buf[:cap(buf)]
if uint(len(buf)) < size {
buf = make([]byte, size)
buf = NewBlobBuffer(int(size))
}
n, err := repo.LoadBlob(DataBlob, id, buf)

10
src/restic/pack/pack.go
View file

@ -85,15 +85,15 @@ func (p *Packer) Finalize() (uint, error) {
return 0, errors.Wrap(err, "Write")
}
hdrBytes := bytesHeader + crypto.Extension
if uint(n) != hdrBytes {
hdrBytes := restic.CiphertextLength(int(bytesHeader))
if n != hdrBytes {
return 0, errors.New("wrong number of bytes written")
}
bytesWritten += hdrBytes
bytesWritten += uint(hdrBytes)
// write length
err = binary.Write(p.wr, binary.LittleEndian, uint32(uint(len(p.blobs))*entrySize+crypto.Extension))
err = binary.Write(p.wr, binary.LittleEndian, uint32(restic.CiphertextLength(len(p.blobs)*int(entrySize))))
if err != nil {
return 0, errors.Wrap(err, "binary.Write")
}
@ -233,6 +233,8 @@ func List(k *crypto.Key, rd io.ReaderAt, size int64) (entries []restic.Blob, err
hdrRd := bytes.NewReader(buf)
entries = make([]restic.Blob, 0, uint(n)/entrySize)
pos := uint(0)
for {
e := headerEntry{}

7
src/restic/pack/pack_test.go
View file

@ -54,10 +54,9 @@ func verifyBlobs(t testing.TB, bufs []Buf, k *crypto.Key, rd io.ReaderAt, packSi
}
// header length
written += binary.Size(uint32(0))
// header
written += len(bufs) * (binary.Size(restic.BlobType(0)) + binary.Size(uint32(0)) + len(restic.ID{}))
// header crypto
written += crypto.Extension
// header + header crypto
headerSize := len(bufs) * (binary.Size(restic.BlobType(0)) + binary.Size(uint32(0)) + len(restic.ID{}))
written += restic.CiphertextLength(headerSize)
// check length
Equals(t, uint(written), packSize)

26
src/restic/repository/index.go
View file

@ -1,7 +1,6 @@
package repository
import (
"bytes"
"encoding/json"
"io"
"restic"
@ -10,7 +9,6 @@ import (
"restic/errors"
"restic/crypto"
"restic/debug"
)
@ -177,15 +175,15 @@ func (idx *Index) Has(id restic.ID, tpe restic.BlobType) bool {
return false
}
// LookupSize returns the length of the cleartext content behind the
// given id
func (idx *Index) LookupSize(id restic.ID, tpe restic.BlobType) (cleartextLength uint, err error) {
// LookupSize returns the length of the plaintext content of the blob with the
// given id.
func (idx *Index) LookupSize(id restic.ID, tpe restic.BlobType) (plaintextLength uint, err error) {
blobs, err := idx.Lookup(id, tpe)
if err != nil {
return 0, err
}
return blobs[0].Length - crypto.Extension, nil
return uint(restic.PlaintextLength(int(blobs[0].Length))), nil
}
// Supersedes returns the list of indexes this index supersedes, if any.
@ -452,12 +450,11 @@ func isErrOldIndex(err error) bool {
var ErrOldIndexFormat = errors.New("index has old format")
// DecodeIndex loads and unserializes an index from rd.
func DecodeIndex(rd io.Reader) (idx *Index, err error) {
func DecodeIndex(buf []byte) (idx *Index, err error) {
debug.Log("Start decoding index")
idxJSON := jsonIndex{}
idxJSON := &jsonIndex{}
dec := json.NewDecoder(rd)
err = dec.Decode(&idxJSON)
err = json.Unmarshal(buf, idxJSON)
if err != nil {
debug.Log("Error %v", err)
@ -491,12 +488,11 @@ func DecodeIndex(rd io.Reader) (idx *Index, err error) {
}
// DecodeOldIndex loads and unserializes an index in the old format from rd.
func DecodeOldIndex(rd io.Reader) (idx *Index, err error) {
func DecodeOldIndex(buf []byte) (idx *Index, err error) {
debug.Log("Start decoding old index")
list := []*packJSON{}
dec := json.NewDecoder(rd)
err = dec.Decode(&list)
err = json.Unmarshal(buf, &list)
if err != nil {
debug.Log("Error %#v", err)
return nil, errors.Wrap(err, "Decode")
@ -523,7 +519,7 @@ func DecodeOldIndex(rd io.Reader) (idx *Index, err error) {
}
// LoadIndexWithDecoder loads the index and decodes it with fn.
func LoadIndexWithDecoder(repo restic.Repository, id restic.ID, fn func(io.Reader) (*Index, error)) (idx *Index, err error) {
func LoadIndexWithDecoder(repo restic.Repository, id restic.ID, fn func([]byte) (*Index, error)) (idx *Index, err error) {
debug.Log("Loading index %v", id.Str())
buf, err := repo.LoadAndDecrypt(restic.IndexFile, id)
@ -531,7 +527,7 @@ func LoadIndexWithDecoder(repo restic.Repository, id restic.ID, fn func(io.Reade
return nil, err
}
idx, err = fn(bytes.NewReader(buf))
idx, err = fn(buf)
if err != nil {
debug.Log("error while decoding index %v: %v", id, err)
return nil, err

17
src/restic/repository/index_test.go
View file

@ -54,7 +54,7 @@ func TestIndexSerialize(t *testing.T) {
err := idx.Encode(wr)
OK(t, err)
idx2, err := repository.DecodeIndex(wr)
idx2, err := repository.DecodeIndex(wr.Bytes())
OK(t, err)
Assert(t, idx2 != nil,
"nil returned for decoded index")
@ -136,7 +136,7 @@ func TestIndexSerialize(t *testing.T) {
Assert(t, id2.Equal(id),
"wrong ID returned: want %v, got %v", id, id2)
idx3, err := repository.DecodeIndex(wr3)
idx3, err := repository.DecodeIndex(wr3.Bytes())
OK(t, err)
Assert(t, idx3 != nil,
"nil returned for decoded index")
@ -288,7 +288,7 @@ var exampleLookupTest = struct {
func TestIndexUnserialize(t *testing.T) {
oldIdx := restic.IDs{restic.TestParseID("ed54ae36197f4745ebc4b54d10e0f623eaaaedd03013eb7ae90df881b7781452")}
idx, err := repository.DecodeIndex(bytes.NewReader(docExample))
idx, err := repository.DecodeIndex(docExample)
OK(t, err)
for _, test := range exampleTests {
@ -326,8 +326,17 @@ func TestIndexUnserialize(t *testing.T) {
}
}
func BenchmarkDecodeIndex(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, err := repository.DecodeIndex(docExample)
OK(b, err)
}
}
func TestIndexUnserializeOld(t *testing.T) {
idx, err := repository.DecodeOldIndex(bytes.NewReader(docOldExample))
idx, err := repository.DecodeOldIndex(docOldExample)
OK(t, err)
for _, test := range exampleTests {

8
src/restic/repository/master_index.go
View file

@ -30,8 +30,7 @@ func (mi *MasterIndex) Lookup(id restic.ID, tpe restic.BlobType) (blobs []restic
for _, idx := range mi.idx {
blobs, err = idx.Lookup(id, tpe)
if err == nil {
debug.Log("MasterIndex.Lookup",
"found id %v: %v", id.Str(), blobs)
debug.Log("found id %v: %v", id.Str(), blobs)
return
}
}
@ -46,9 +45,8 @@ func (mi *MasterIndex) LookupSize(id restic.ID, tpe restic.BlobType) (uint, erro
defer mi.idxMutex.RUnlock()
for _, idx := range mi.idx {
length, err := idx.LookupSize(id, tpe)
if err == nil {
return length, nil
if idx.Has(id, tpe) {
return idx.LookupSize(id, tpe)
}
}

36
src/restic/repository/repository.go
View file

@ -64,15 +64,13 @@ func (r *Repository) LoadAndDecrypt(t restic.FileType, id restic.ID) ([]byte, er
return nil, errors.New("invalid data returned")
}
plain := make([]byte, len(buf))
// decrypt
n, err := r.decryptTo(plain, buf)
n, err := r.decryptTo(buf, buf)
if err != nil {
return nil, err
}
return plain[:n], nil
return buf[:n], nil
}
// loadBlob tries to load and decrypt content identified by t and id from a
@ -81,17 +79,6 @@ func (r *Repository) LoadAndDecrypt(t restic.FileType, id restic.ID) ([]byte, er
func (r *Repository) loadBlob(id restic.ID, t restic.BlobType, plaintextBuf []byte) (int, error) {
debug.Log("load %v with id %v (buf %p, len %d)", t, id.Str(), plaintextBuf, len(plaintextBuf))
// lookup plaintext size of blob
size, err := r.idx.LookupSize(id, t)
if err != nil {
return 0, err
}
// make sure the plaintext buffer is large enough, extend otherwise
if len(plaintextBuf) < int(size) {
return 0, errors.Errorf("buffer is too small: %d < %d", len(plaintextBuf), size)
}
// lookup packs
blobs, err := r.idx.Lookup(id, t)
if err != nil {
@ -109,8 +96,8 @@ func (r *Repository) loadBlob(id restic.ID, t restic.BlobType, plaintextBuf []by
// load blob from pack
h := restic.Handle{Type: restic.DataFile, Name: blob.PackID.String()}
ciphertextBuf := make([]byte, blob.Length)
n, err := r.be.Load(h, ciphertextBuf, int64(blob.Offset))
plaintextBuf = plaintextBuf[:cap(plaintextBuf)]
n, err := r.be.Load(h, plaintextBuf, int64(blob.Offset))
if err != nil {
debug.Log("error loading blob %v: %v", blob, err)
lastError = err
@ -125,7 +112,7 @@ func (r *Repository) loadBlob(id restic.ID, t restic.BlobType, plaintextBuf []by
}
// decrypt
n, err = r.decryptTo(plaintextBuf, ciphertextBuf)
n, err = r.decryptTo(plaintextBuf, plaintextBuf)
if err != nil {
lastError = errors.Errorf("decrypting blob %v failed: %v", id, err)
continue
@ -224,7 +211,7 @@ func (r *Repository) SaveJSONUnpacked(t restic.FileType, item interface{}) (rest
// SaveUnpacked encrypts data and stores it in the backend. Returned is the
// storage hash.
func (r *Repository) SaveUnpacked(t restic.FileType, p []byte) (id restic.ID, err error) {
ciphertext := make([]byte, len(p)+crypto.Extension)
ciphertext := restic.NewBlobBuffer(len(p))
ciphertext, err = r.Encrypt(ciphertext, p)
if err != nil {
return restic.ID{}, err
@ -528,7 +515,9 @@ func (r *Repository) Close() error {
return r.be.Close()
}
// LoadBlob loads a blob of type t from the repository to the buffer.
// LoadBlob loads a blob of type t from the repository to the buffer. buf must
// be large enough to hold the encrypted blob, since it is used as scratch
// space.
func (r *Repository) LoadBlob(t restic.BlobType, id restic.ID, buf []byte) (int, error) {
debug.Log("load blob %v into buf %p", id.Str(), buf)
size, err := r.idx.LookupSize(id, t)
@ -536,8 +525,9 @@ func (r *Repository) LoadBlob(t restic.BlobType, id restic.ID, buf []byte) (int,
return 0, err
}
if len(buf) < int(size) {
return 0, errors.Errorf("buffer is too small for data blob (%d < %d)", len(buf), size)
buf = buf[:cap(buf)]
if len(buf) < restic.CiphertextLength(int(size)) {
return 0, errors.Errorf("buffer is too small for data blob (%d < %d)", len(buf), restic.CiphertextLength(int(size)))
}
n, err := r.loadBlob(id, t, buf)
@ -571,7 +561,7 @@ func (r *Repository) LoadTree(id restic.ID) (*restic.Tree, error) {
}
debug.Log("size is %d, create buffer", size)
buf := make([]byte, size)
buf := restic.NewBlobBuffer(int(size))
n, err := r.loadBlob(id, restic.TreeBlob, buf)
if err != nil {

113
src/restic/repository/repository_test.go
View file

@ -2,12 +2,12 @@ package repository_test
import (
"bytes"
"crypto/rand"
"crypto/sha256"
"io"
mrand "math/rand"
"math/rand"
"path/filepath"
"testing"
"time"
"restic"
"restic/archiver"
@ -17,13 +17,15 @@ import (
var testSizes = []int{5, 23, 2<<18 + 23, 1 << 20}
var rnd = rand.New(rand.NewSource(time.Now().UnixNano()))
func TestSave(t *testing.T) {
repo, cleanup := repository.TestRepository(t)
defer cleanup()
for _, size := range testSizes {
data := make([]byte, size)
_, err := io.ReadFull(rand.Reader, data)
_, err := io.ReadFull(rnd, data)
OK(t, err)
id := restic.Hash(data)
@ -38,7 +40,7 @@ func TestSave(t *testing.T) {
// OK(t, repo.SaveIndex())
// read back
buf := make([]byte, size)
buf := restic.NewBlobBuffer(size)
n, err := repo.LoadBlob(restic.DataBlob, id, buf)
OK(t, err)
Equals(t, len(buf), n)
@ -59,7 +61,7 @@ func TestSaveFrom(t *testing.T) {
for _, size := range testSizes {
data := make([]byte, size)
_, err := io.ReadFull(rand.Reader, data)
_, err := io.ReadFull(rnd, data)
OK(t, err)
id := restic.Hash(data)
@ -72,7 +74,7 @@ func TestSaveFrom(t *testing.T) {
OK(t, repo.Flush())
// read back
buf := make([]byte, size)
buf := restic.NewBlobBuffer(size)
n, err := repo.LoadBlob(restic.DataBlob, id, buf)
OK(t, err)
Equals(t, len(buf), n)
@ -94,7 +96,7 @@ func BenchmarkSaveAndEncrypt(t *testing.B) {
size := 4 << 20 // 4MiB
data := make([]byte, size)
_, err := io.ReadFull(rand.Reader, data)
_, err := io.ReadFull(rnd, data)
OK(t, err)
id := restic.ID(sha256.Sum256(data))
@ -145,6 +147,68 @@ func BenchmarkLoadTree(t *testing.B) {
}
}
func BenchmarkLoadBlob(b *testing.B) {
repo, cleanup := repository.TestRepository(b)
defer cleanup()
length := 1000000
buf := restic.NewBlobBuffer(length)
_, err := io.ReadFull(rnd, buf)
OK(b, err)
id, err := repo.SaveBlob(restic.DataBlob, buf, restic.ID{})
OK(b, err)
OK(b, repo.Flush())
b.ResetTimer()
b.SetBytes(int64(length))
for i := 0; i < b.N; i++ {
n, err := repo.LoadBlob(restic.DataBlob, id, buf)
OK(b, err)
if n != length {
b.Errorf("wanted %d bytes, got %d", length, n)
}
id2 := restic.Hash(buf[:n])
if !id.Equal(id2) {
b.Errorf("wrong data returned, wanted %v, got %v", id.Str(), id2.Str())
}
}
}
func BenchmarkLoadAndDecrypt(b *testing.B) {
repo, cleanup := repository.TestRepository(b)
defer cleanup()
length := 1000000
buf := restic.NewBlobBuffer(length)
_, err := io.ReadFull(rnd, buf)
OK(b, err)
dataID := restic.Hash(buf)
storageID, err := repo.SaveUnpacked(restic.DataFile, buf)
OK(b, err)
// OK(b, repo.Flush())
b.ResetTimer()
b.SetBytes(int64(length))
for i := 0; i < b.N; i++ {
data, err := repo.LoadAndDecrypt(restic.DataFile, storageID)
OK(b, err)
if len(data) != length {
b.Errorf("wanted %d bytes, got %d", length, len(data))
}
id2 := restic.Hash(data)
if !dataID.Equal(id2) {
b.Errorf("wrong data returned, wanted %v, got %v", storageID.Str(), id2.Str())
}
}
}
func TestLoadJSONUnpacked(t *testing.T) {
repo, cleanup := repository.TestRepository(t)
defer cleanup()
@ -182,25 +246,48 @@ func TestRepositoryLoadIndex(t *testing.T) {
}
func BenchmarkLoadIndex(b *testing.B) {
repodir, cleanup := Env(b, repoFixture)
repository.TestUseLowSecurityKDFParameters(b)
repo, cleanup := repository.TestRepository(b)
defer cleanup()
repo := repository.TestOpenLocal(b, repodir)
idx := repository.NewIndex()
for i := 0; i < 5000; i++ {
idx.Store(restic.PackedBlob{
Blob: restic.Blob{
Type: restic.DataBlob,
Length: 1234,
ID: restic.NewRandomID(),
Offset: 1235,
},
PackID: restic.NewRandomID(),
})
}
id, err := repository.SaveIndex(repo, idx)
OK(b, err)
b.Logf("index saved as %v (%v entries)", id.Str(), idx.Count(restic.DataBlob))
fi, err := repo.Backend().Stat(restic.Handle{Type: restic.IndexFile, Name: id.String()})
OK(b, err)
b.Logf("filesize is %v", fi.Size)
b.ResetTimer()
for i := 0; i < b.N; i++ {
repo.SetIndex(repository.NewMasterIndex())
OK(b, repo.LoadIndex())
_, err := repository.LoadIndex(repo, id)
OK(b, err)
}
}
// saveRandomDataBlobs generates random data blobs and saves them to the repository.
func saveRandomDataBlobs(t testing.TB, repo restic.Repository, num int, sizeMax int) {
for i := 0; i < num; i++ {
size := mrand.Int() % sizeMax
size := rand.Int() % sizeMax
buf := make([]byte, size)
_, err := io.ReadFull(rand.Reader, buf)
_, err := io.ReadFull(rnd, buf)
OK(t, err)
_, err = repo.SaveBlob(restic.DataBlob, buf, restic.ID{})

25
src/restic/testing.go
View file

@ -23,16 +23,26 @@ type fakeFileSystem struct {
repo Repository
knownBlobs IDSet
duplication float32
buf []byte
chunker *chunker.Chunker
}
// saveFile reads from rd and saves the blobs in the repository. The list of
// IDs is returned.
func (fs fakeFileSystem) saveFile(rd io.Reader) (blobs IDs) {
blobs = IDs{}
ch := chunker.New(rd, fs.repo.Config().ChunkerPolynomial)
func (fs *fakeFileSystem) saveFile(rd io.Reader) (blobs IDs) {
if fs.buf == nil {
fs.buf = make([]byte, chunker.MaxSize)
}
if fs.chunker == nil {
fs.chunker = chunker.New(rd, fs.repo.Config().ChunkerPolynomial)
} else {
fs.chunker.Reset(rd, fs.repo.Config().ChunkerPolynomial)
}
blobs = IDs{}
for {
chunk, err := ch.Next(getBuf())
chunk, err := fs.chunker.Next(fs.buf)
if errors.Cause(err) == io.EOF {
break
}
@ -50,7 +60,6 @@ func (fs fakeFileSystem) saveFile(rd io.Reader) (blobs IDs) {
fs.knownBlobs.Insert(id)
}
freeBuf(chunk.Data)
blobs = append(blobs, id)
}
@ -64,7 +73,7 @@ const (
maxNodes = 32
)
func (fs fakeFileSystem) treeIsKnown(tree *Tree) (bool, []byte, ID) {
func (fs *fakeFileSystem) treeIsKnown(tree *Tree) (bool, []byte, ID) {
data, err := json.Marshal(tree)
if err != nil {
fs.t.Fatalf("json.Marshal(tree) returned error: %v", err)
@ -76,7 +85,7 @@ func (fs fakeFileSystem) treeIsKnown(tree *Tree) (bool, []byte, ID) {
return fs.blobIsKnown(id, TreeBlob), data, id
}
func (fs fakeFileSystem) blobIsKnown(id ID, t BlobType) bool {
func (fs *fakeFileSystem) blobIsKnown(id ID, t BlobType) bool {
if rand.Float32() < fs.duplication {
return false
}
@ -94,7 +103,7 @@ func (fs fakeFileSystem) blobIsKnown(id ID, t BlobType) bool {
}
// saveTree saves a tree of fake files in the repo and returns the ID.
func (fs fakeFileSystem) saveTree(seed int64, depth int) ID {
func (fs *fakeFileSystem) saveTree(seed int64, depth int) ID {
rnd := rand.NewSource(seed)
numNodes := int(rnd.Int63() % maxNodes)

11
src/restic/testing_test.go
View file

@ -47,3 +47,14 @@ func TestCreateSnapshot(t *testing.T) {
checker.TestCheckRepo(t, repo)
}
func BenchmarkTestCreateSnapshot(t *testing.B) {
repo, cleanup := repository.TestRepository(t)
defer cleanup()
t.ResetTimer()
for i := 0; i < t.N; i++ {
restic.TestCreateSnapshot(t, repo, testSnapshotTime.Add(time.Duration(i)*time.Second), testDepth, 0)
}
}