2016-03-05 14:58:39 +00:00
|
|
|
package repository
|
|
|
|
|
|
|
|
import (
|
2017-06-05 21:56:59 +00:00
|
|
|
"context"
|
2016-03-05 14:58:39 +00:00
|
|
|
"io"
|
|
|
|
"math/rand"
|
2016-03-06 11:26:25 +00:00
|
|
|
"os"
|
2016-03-05 14:58:39 +00:00
|
|
|
"testing"
|
2017-07-23 12:21:03 +00:00
|
|
|
|
|
|
|
"github.com/restic/restic/internal/backend/mem"
|
|
|
|
"github.com/restic/restic/internal/crypto"
|
|
|
|
"github.com/restic/restic/internal/fs"
|
|
|
|
"github.com/restic/restic/internal/mock"
|
2017-07-24 15:42:25 +00:00
|
|
|
"github.com/restic/restic/internal/restic"
|
2016-03-05 14:58:39 +00:00
|
|
|
)
|
|
|
|
|
2016-08-31 18:29:54 +00:00
|
|
|
func randomID(rd io.Reader) restic.ID {
|
|
|
|
id := restic.ID{}
|
2016-03-05 14:58:39 +00:00
|
|
|
_, err := io.ReadFull(rd, id[:])
|
|
|
|
if err != nil {
|
|
|
|
panic(err)
|
|
|
|
}
|
|
|
|
return id
|
|
|
|
}
|
|
|
|
|
2016-03-06 11:26:25 +00:00
|
|
|
const maxBlobSize = 1 << 20
|
|
|
|
|
2018-03-03 13:20:54 +00:00
|
|
|
func saveFile(t testing.TB, be Saver, length int, f *os.File, id restic.ID) {
|
2017-01-22 16:53:00 +00:00
|
|
|
h := restic.Handle{Type: restic.DataFile, Name: id.String()}
|
|
|
|
t.Logf("save file %v", h)
|
2016-03-06 11:26:25 +00:00
|
|
|
|
2018-03-03 13:20:54 +00:00
|
|
|
rd, err := restic.NewFileReader(f)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
err = be.Save(context.TODO(), h, rd)
|
|
|
|
if err != nil {
|
2017-01-23 17:49:12 +00:00
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
2017-01-25 16:51:34 +00:00
|
|
|
if err := f.Close(); err != nil {
|
2016-03-06 11:26:25 +00:00
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
2017-05-10 17:48:22 +00:00
|
|
|
if err := fs.RemoveIfExists(f.Name()); err != nil {
|
2016-03-06 11:26:25 +00:00
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-03-05 17:50:55 +00:00
|
|
|
func fillPacks(t testing.TB, rnd *rand.Rand, be Saver, pm *packerManager, buf []byte) (bytes int) {
|
2016-03-05 14:58:39 +00:00
|
|
|
for i := 0; i < 100; i++ {
|
2020-03-05 17:50:55 +00:00
|
|
|
l := rnd.Intn(1 << 20)
|
|
|
|
seed := rnd.Int63()
|
2016-03-05 14:58:39 +00:00
|
|
|
|
2017-07-16 18:16:02 +00:00
|
|
|
packer, err := pm.findPacker()
|
2016-03-05 14:58:39 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
2020-03-05 17:50:55 +00:00
|
|
|
rd := rand.New(rand.NewSource(seed))
|
2016-03-05 14:58:39 +00:00
|
|
|
id := randomID(rd)
|
2016-03-06 11:26:25 +00:00
|
|
|
buf = buf[:l]
|
|
|
|
_, err = io.ReadFull(rd, buf)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
2016-03-05 14:58:39 +00:00
|
|
|
|
2016-08-31 21:07:50 +00:00
|
|
|
n, err := packer.Add(restic.DataBlob, id, buf)
|
2018-10-19 01:43:09 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
2016-03-06 11:26:25 +00:00
|
|
|
if n != l {
|
2016-03-05 14:58:39 +00:00
|
|
|
t.Errorf("Add() returned invalid number of bytes: want %v, got %v", n, l)
|
|
|
|
}
|
|
|
|
bytes += l
|
|
|
|
|
2017-07-16 18:02:59 +00:00
|
|
|
if packer.Size() < minPackSize {
|
2016-03-05 14:58:39 +00:00
|
|
|
pm.insertPacker(packer)
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2017-01-22 16:53:00 +00:00
|
|
|
_, err = packer.Finalize()
|
2016-03-05 14:58:39 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
2017-01-22 16:53:00 +00:00
|
|
|
packID := restic.IDFromHash(packer.hw.Sum(nil))
|
2018-03-03 13:20:54 +00:00
|
|
|
saveFile(t, be, int(packer.Size()), packer.tmpfile, packID)
|
2016-03-05 14:58:39 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return bytes
|
|
|
|
}
|
|
|
|
|
2020-03-05 17:50:55 +00:00
|
|
|
func flushRemainingPacks(t testing.TB, be Saver, pm *packerManager) (bytes int) {
|
2016-03-05 14:58:39 +00:00
|
|
|
if pm.countPacker() > 0 {
|
2017-01-22 16:53:00 +00:00
|
|
|
for _, packer := range pm.packers {
|
2016-03-06 11:26:25 +00:00
|
|
|
n, err := packer.Finalize()
|
2016-03-05 14:58:39 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
2016-03-06 11:26:25 +00:00
|
|
|
bytes += int(n)
|
2016-03-05 14:58:39 +00:00
|
|
|
|
2017-01-22 16:53:00 +00:00
|
|
|
packID := restic.IDFromHash(packer.hw.Sum(nil))
|
2018-03-03 13:20:54 +00:00
|
|
|
saveFile(t, be, int(packer.Size()), packer.tmpfile, packID)
|
2016-03-05 14:58:39 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return bytes
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestPackerManager(t *testing.T) {
|
2020-03-05 17:50:55 +00:00
|
|
|
rnd := rand.New(rand.NewSource(23))
|
2016-03-05 14:58:39 +00:00
|
|
|
|
|
|
|
be := mem.New()
|
2016-03-06 13:20:48 +00:00
|
|
|
pm := newPackerManager(be, crypto.NewRandomKey())
|
2016-03-05 14:58:39 +00:00
|
|
|
|
2016-03-06 11:26:25 +00:00
|
|
|
blobBuf := make([]byte, maxBlobSize)
|
|
|
|
|
|
|
|
bytes := fillPacks(t, rnd, be, pm, blobBuf)
|
2020-03-05 17:50:55 +00:00
|
|
|
bytes += flushRemainingPacks(t, be, pm)
|
2016-03-05 14:58:39 +00:00
|
|
|
|
|
|
|
t.Logf("saved %d bytes", bytes)
|
|
|
|
}
|
|
|
|
|
|
|
|
func BenchmarkPackerManager(t *testing.B) {
|
2020-03-05 17:50:55 +00:00
|
|
|
rnd := rand.New(rand.NewSource(23))
|
2016-03-05 14:58:39 +00:00
|
|
|
|
2017-01-22 11:32:20 +00:00
|
|
|
be := &mock.Backend{
|
2018-03-03 13:20:54 +00:00
|
|
|
SaveFn: func(context.Context, restic.Handle, restic.RewindReader) error { return nil },
|
2017-01-22 11:32:20 +00:00
|
|
|
}
|
2016-03-06 11:26:25 +00:00
|
|
|
blobBuf := make([]byte, maxBlobSize)
|
2016-03-05 14:58:39 +00:00
|
|
|
|
|
|
|
t.ResetTimer()
|
|
|
|
|
|
|
|
for i := 0; i < t.N; i++ {
|
2017-01-22 16:53:00 +00:00
|
|
|
bytes := 0
|
|
|
|
pm := newPackerManager(be, crypto.NewRandomKey())
|
2016-03-06 11:26:25 +00:00
|
|
|
bytes += fillPacks(t, rnd, be, pm, blobBuf)
|
2020-03-05 17:50:55 +00:00
|
|
|
bytes += flushRemainingPacks(t, be, pm)
|
2017-01-22 16:53:00 +00:00
|
|
|
t.Logf("saved %d bytes", bytes)
|
2016-03-05 14:58:39 +00:00
|
|
|
}
|
|
|
|
}
|