2015-05-09 21:52:03 +00:00
|
|
|
package repository_test
|
2015-02-16 18:32:36 +00:00
|
|
|
|
|
|
|
import (
|
|
|
|
"bytes"
|
2017-06-05 21:56:59 +00:00
|
|
|
"context"
|
2015-02-16 18:32:36 +00:00
|
|
|
"crypto/sha256"
|
2020-11-07 17:53:59 +00:00
|
|
|
"fmt"
|
2015-02-16 19:00:23 +00:00
|
|
|
"io"
|
2017-01-13 11:20:37 +00:00
|
|
|
"math/rand"
|
2020-11-07 17:53:59 +00:00
|
|
|
"os"
|
2015-07-04 14:52:17 +00:00
|
|
|
"path/filepath"
|
2024-05-09 13:26:41 +00:00
|
|
|
"sync"
|
2015-02-16 18:32:36 +00:00
|
|
|
"testing"
|
2017-01-13 11:20:37 +00:00
|
|
|
"time"
|
2015-02-16 18:32:36 +00:00
|
|
|
|
2023-10-01 09:40:12 +00:00
|
|
|
"github.com/restic/restic/internal/backend"
|
2021-09-19 18:02:38 +00:00
|
|
|
"github.com/restic/restic/internal/backend/local"
|
2024-05-09 19:22:19 +00:00
|
|
|
"github.com/restic/restic/internal/backend/mem"
|
|
|
|
"github.com/restic/restic/internal/cache"
|
2022-03-21 19:38:53 +00:00
|
|
|
"github.com/restic/restic/internal/crypto"
|
2024-05-09 16:59:29 +00:00
|
|
|
"github.com/restic/restic/internal/errors"
|
2022-06-12 12:43:43 +00:00
|
|
|
"github.com/restic/restic/internal/index"
|
2017-07-23 12:21:03 +00:00
|
|
|
"github.com/restic/restic/internal/repository"
|
2017-07-24 15:42:25 +00:00
|
|
|
"github.com/restic/restic/internal/restic"
|
2024-05-09 19:22:19 +00:00
|
|
|
"github.com/restic/restic/internal/test"
|
2017-10-02 13:06:39 +00:00
|
|
|
rtest "github.com/restic/restic/internal/test"
|
2021-08-07 20:52:05 +00:00
|
|
|
"golang.org/x/sync/errgroup"
|
2015-02-16 18:32:36 +00:00
|
|
|
)
|
|
|
|
|
2015-02-16 19:00:23 +00:00
|
|
|
var testSizes = []int{5, 23, 2<<18 + 23, 1 << 20}
|
|
|
|
|
2017-01-13 11:20:37 +00:00
|
|
|
var rnd = rand.New(rand.NewSource(time.Now().UnixNano()))
|
|
|
|
|
2015-04-26 15:44:38 +00:00
|
|
|
func TestSave(t *testing.T) {
|
2024-01-20 20:50:23 +00:00
|
|
|
repository.TestAllVersions(t, testSavePassID)
|
|
|
|
repository.TestAllVersions(t, testSaveCalculateID)
|
2022-04-29 21:16:16 +00:00
|
|
|
}
|
|
|
|
|
2024-01-20 20:50:23 +00:00
|
|
|
func testSavePassID(t *testing.T, version uint) {
|
|
|
|
testSave(t, version, false)
|
2015-04-26 15:44:38 +00:00
|
|
|
}
|
|
|
|
|
2024-01-20 20:50:23 +00:00
|
|
|
func testSaveCalculateID(t *testing.T, version uint) {
|
|
|
|
testSave(t, version, true)
|
2022-04-29 21:16:16 +00:00
|
|
|
}
|
|
|
|
|
2024-01-20 20:50:23 +00:00
|
|
|
func testSave(t *testing.T, version uint, calculateID bool) {
|
2022-12-11 09:41:22 +00:00
|
|
|
repo := repository.TestRepositoryWithVersion(t, version)
|
2015-02-16 19:00:23 +00:00
|
|
|
|
|
|
|
for _, size := range testSizes {
|
|
|
|
data := make([]byte, size)
|
2017-01-13 11:20:37 +00:00
|
|
|
_, err := io.ReadFull(rnd, data)
|
2017-10-02 13:06:39 +00:00
|
|
|
rtest.OK(t, err)
|
2015-02-16 19:00:23 +00:00
|
|
|
|
2016-08-31 18:29:54 +00:00
|
|
|
id := restic.Hash(data)
|
2015-02-16 19:00:23 +00:00
|
|
|
|
2021-08-07 20:52:05 +00:00
|
|
|
var wg errgroup.Group
|
|
|
|
repo.StartPackUploader(context.TODO(), &wg)
|
|
|
|
|
2015-02-16 19:00:23 +00:00
|
|
|
// save
|
2024-01-20 20:50:23 +00:00
|
|
|
inputID := restic.ID{}
|
|
|
|
if !calculateID {
|
|
|
|
inputID = id
|
|
|
|
}
|
|
|
|
sid, _, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, data, inputID, false)
|
2017-10-02 13:06:39 +00:00
|
|
|
rtest.OK(t, err)
|
2024-01-20 20:50:23 +00:00
|
|
|
rtest.Equals(t, id, sid)
|
2015-02-16 19:00:23 +00:00
|
|
|
|
2017-11-22 11:27:29 +00:00
|
|
|
rtest.OK(t, repo.Flush(context.Background()))
|
2015-04-26 15:44:38 +00:00
|
|
|
|
2015-02-16 19:00:23 +00:00
|
|
|
// read back
|
2020-03-10 15:41:22 +00:00
|
|
|
buf, err := repo.LoadBlob(context.TODO(), restic.DataBlob, id, nil)
|
2017-10-02 13:06:39 +00:00
|
|
|
rtest.OK(t, err)
|
2020-03-10 15:41:22 +00:00
|
|
|
rtest.Equals(t, size, len(buf))
|
2015-02-16 19:00:23 +00:00
|
|
|
|
2017-10-02 13:06:39 +00:00
|
|
|
rtest.Assert(t, len(buf) == len(data),
|
2015-02-16 19:00:23 +00:00
|
|
|
"number of bytes read back does not match: expected %d, got %d",
|
|
|
|
len(data), len(buf))
|
|
|
|
|
2017-10-02 13:06:39 +00:00
|
|
|
rtest.Assert(t, bytes.Equal(buf, data),
|
2015-02-16 19:00:23 +00:00
|
|
|
"data does not match: expected %02x, got %02x",
|
|
|
|
data, buf)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-05-08 11:13:29 +00:00
|
|
|
func BenchmarkSaveAndEncrypt(t *testing.B) {
|
2022-04-29 21:16:16 +00:00
|
|
|
repository.BenchmarkAllVersions(t, benchmarkSaveAndEncrypt)
|
|
|
|
}
|
|
|
|
|
|
|
|
func benchmarkSaveAndEncrypt(t *testing.B, version uint) {
|
2022-12-11 09:41:22 +00:00
|
|
|
repo := repository.TestRepositoryWithVersion(t, version)
|
2015-02-16 19:00:23 +00:00
|
|
|
size := 4 << 20 // 4MiB
|
|
|
|
|
|
|
|
data := make([]byte, size)
|
2017-01-13 11:20:37 +00:00
|
|
|
_, err := io.ReadFull(rnd, data)
|
2017-10-02 13:06:39 +00:00
|
|
|
rtest.OK(t, err)
|
2015-02-16 19:00:23 +00:00
|
|
|
|
2016-08-31 18:29:54 +00:00
|
|
|
id := restic.ID(sha256.Sum256(data))
|
2022-10-25 15:21:24 +00:00
|
|
|
var wg errgroup.Group
|
|
|
|
repo.StartPackUploader(context.Background(), &wg)
|
2015-02-16 19:00:23 +00:00
|
|
|
|
2020-02-26 22:26:11 +00:00
|
|
|
t.ReportAllocs()
|
2015-02-16 19:00:23 +00:00
|
|
|
t.ResetTimer()
|
|
|
|
t.SetBytes(int64(size))
|
|
|
|
|
|
|
|
for i := 0; i < t.N; i++ {
|
2022-05-01 12:26:57 +00:00
|
|
|
_, _, _, err = repo.SaveBlob(context.TODO(), restic.DataBlob, data, id, true)
|
2017-10-02 13:06:39 +00:00
|
|
|
rtest.OK(t, err)
|
2015-02-16 19:00:23 +00:00
|
|
|
}
|
|
|
|
}
|
2015-02-17 22:05:23 +00:00
|
|
|
|
2017-01-24 10:25:33 +00:00
|
|
|
func TestLoadBlob(t *testing.T) {
|
2022-04-29 21:16:16 +00:00
|
|
|
repository.TestAllVersions(t, testLoadBlob)
|
|
|
|
}
|
|
|
|
|
|
|
|
func testLoadBlob(t *testing.T, version uint) {
|
2022-12-11 09:41:22 +00:00
|
|
|
repo := repository.TestRepositoryWithVersion(t, version)
|
2017-01-24 10:25:33 +00:00
|
|
|
length := 1000000
|
2022-06-12 12:48:30 +00:00
|
|
|
buf := crypto.NewBlobBuffer(length)
|
2017-01-24 10:25:33 +00:00
|
|
|
_, err := io.ReadFull(rnd, buf)
|
2017-10-02 13:06:39 +00:00
|
|
|
rtest.OK(t, err)
|
2017-01-24 10:25:33 +00:00
|
|
|
|
2021-08-07 20:52:05 +00:00
|
|
|
var wg errgroup.Group
|
|
|
|
repo.StartPackUploader(context.TODO(), &wg)
|
|
|
|
|
2022-05-01 12:26:57 +00:00
|
|
|
id, _, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, buf, restic.ID{}, false)
|
2017-10-02 13:06:39 +00:00
|
|
|
rtest.OK(t, err)
|
2017-11-22 11:27:29 +00:00
|
|
|
rtest.OK(t, repo.Flush(context.Background()))
|
2017-01-24 10:25:33 +00:00
|
|
|
|
2022-06-12 12:48:30 +00:00
|
|
|
base := crypto.CiphertextLength(length)
|
2020-03-10 15:41:22 +00:00
|
|
|
for _, testlength := range []int{0, base - 20, base - 1, base, base + 7, base + 15, base + 1000} {
|
2017-01-24 10:25:33 +00:00
|
|
|
buf = make([]byte, 0, testlength)
|
2020-03-10 15:41:22 +00:00
|
|
|
buf, err := repo.LoadBlob(context.TODO(), restic.DataBlob, id, buf)
|
2017-01-24 10:25:33 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Errorf("LoadBlob() returned an error for buffer size %v: %v", testlength, err)
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2020-03-10 15:41:22 +00:00
|
|
|
if len(buf) != length {
|
|
|
|
t.Errorf("LoadBlob() returned the wrong number of bytes: want %v, got %v", length, len(buf))
|
2017-01-24 10:25:33 +00:00
|
|
|
continue
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2024-05-09 19:22:19 +00:00
|
|
|
func TestLoadBlobBroken(t *testing.T) {
|
|
|
|
be := mem.New()
|
|
|
|
repo := repository.TestRepositoryWithBackend(t, &damageOnceBackend{Backend: be}, restic.StableRepoVersion, repository.Options{}).(*repository.Repository)
|
|
|
|
buf := test.Random(42, 1000)
|
|
|
|
|
|
|
|
var wg errgroup.Group
|
|
|
|
repo.StartPackUploader(context.TODO(), &wg)
|
|
|
|
id, _, _, err := repo.SaveBlob(context.TODO(), restic.TreeBlob, buf, restic.ID{}, false)
|
|
|
|
rtest.OK(t, err)
|
|
|
|
rtest.OK(t, repo.Flush(context.Background()))
|
|
|
|
|
|
|
|
// setup cache after saving the blob to make sure that the damageOnceBackend damages the cached data
|
|
|
|
c := cache.TestNewCache(t)
|
|
|
|
repo.UseCache(c)
|
|
|
|
|
|
|
|
data, err := repo.LoadBlob(context.TODO(), restic.TreeBlob, id, nil)
|
|
|
|
rtest.OK(t, err)
|
|
|
|
rtest.Assert(t, bytes.Equal(buf, data), "data mismatch")
|
|
|
|
pack := repo.Index().Lookup(restic.BlobHandle{Type: restic.TreeBlob, ID: id})[0].PackID
|
|
|
|
rtest.Assert(t, c.Has(backend.Handle{Type: restic.PackFile, Name: pack.String()}), "expected tree pack to be cached")
|
|
|
|
}
|
|
|
|
|
2017-01-13 11:20:37 +00:00
|
|
|
func BenchmarkLoadBlob(b *testing.B) {
|
2022-04-29 21:16:16 +00:00
|
|
|
repository.BenchmarkAllVersions(b, benchmarkLoadBlob)
|
|
|
|
}
|
|
|
|
|
|
|
|
func benchmarkLoadBlob(b *testing.B, version uint) {
|
2022-12-11 09:41:22 +00:00
|
|
|
repo := repository.TestRepositoryWithVersion(b, version)
|
2017-01-13 11:20:37 +00:00
|
|
|
length := 1000000
|
2022-06-12 12:48:30 +00:00
|
|
|
buf := crypto.NewBlobBuffer(length)
|
2017-01-13 11:20:37 +00:00
|
|
|
_, err := io.ReadFull(rnd, buf)
|
2017-10-02 13:06:39 +00:00
|
|
|
rtest.OK(b, err)
|
2017-01-13 11:20:37 +00:00
|
|
|
|
2021-08-07 20:52:05 +00:00
|
|
|
var wg errgroup.Group
|
|
|
|
repo.StartPackUploader(context.TODO(), &wg)
|
|
|
|
|
2022-05-01 12:26:57 +00:00
|
|
|
id, _, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, buf, restic.ID{}, false)
|
2017-10-02 13:06:39 +00:00
|
|
|
rtest.OK(b, err)
|
2017-11-22 11:27:29 +00:00
|
|
|
rtest.OK(b, repo.Flush(context.Background()))
|
2017-01-13 11:20:37 +00:00
|
|
|
|
|
|
|
b.ResetTimer()
|
|
|
|
b.SetBytes(int64(length))
|
|
|
|
|
|
|
|
for i := 0; i < b.N; i++ {
|
2020-03-10 15:41:22 +00:00
|
|
|
var err error
|
|
|
|
buf, err = repo.LoadBlob(context.TODO(), restic.DataBlob, id, buf)
|
2020-04-28 05:57:29 +00:00
|
|
|
|
|
|
|
// Checking the SHA-256 with restic.Hash can make up 38% of the time
|
|
|
|
// spent in this loop, so pause the timer.
|
|
|
|
b.StopTimer()
|
2017-10-02 13:06:39 +00:00
|
|
|
rtest.OK(b, err)
|
2020-03-10 15:41:22 +00:00
|
|
|
if len(buf) != length {
|
|
|
|
b.Errorf("wanted %d bytes, got %d", length, len(buf))
|
2017-01-13 11:20:37 +00:00
|
|
|
}
|
|
|
|
|
2020-03-10 15:41:22 +00:00
|
|
|
id2 := restic.Hash(buf)
|
2017-01-13 11:20:37 +00:00
|
|
|
if !id.Equal(id2) {
|
|
|
|
b.Errorf("wrong data returned, wanted %v, got %v", id.Str(), id2.Str())
|
|
|
|
}
|
2020-04-28 05:57:29 +00:00
|
|
|
b.StartTimer()
|
2017-01-13 11:20:37 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-02-12 23:01:27 +00:00
|
|
|
func BenchmarkLoadUnpacked(b *testing.B) {
|
2022-04-29 21:16:16 +00:00
|
|
|
repository.BenchmarkAllVersions(b, benchmarkLoadUnpacked)
|
|
|
|
}
|
|
|
|
|
|
|
|
func benchmarkLoadUnpacked(b *testing.B, version uint) {
|
2022-12-11 09:41:22 +00:00
|
|
|
repo := repository.TestRepositoryWithVersion(b, version)
|
2017-01-13 19:56:50 +00:00
|
|
|
length := 1000000
|
2022-06-12 12:48:30 +00:00
|
|
|
buf := crypto.NewBlobBuffer(length)
|
2017-01-13 19:56:50 +00:00
|
|
|
_, err := io.ReadFull(rnd, buf)
|
2017-10-02 13:06:39 +00:00
|
|
|
rtest.OK(b, err)
|
2017-01-13 19:56:50 +00:00
|
|
|
|
|
|
|
dataID := restic.Hash(buf)
|
|
|
|
|
2020-08-16 09:16:38 +00:00
|
|
|
storageID, err := repo.SaveUnpacked(context.TODO(), restic.PackFile, buf)
|
2017-10-02 13:06:39 +00:00
|
|
|
rtest.OK(b, err)
|
|
|
|
// rtest.OK(b, repo.Flush())
|
2017-01-13 19:56:50 +00:00
|
|
|
|
|
|
|
b.ResetTimer()
|
|
|
|
b.SetBytes(int64(length))
|
|
|
|
|
|
|
|
for i := 0; i < b.N; i++ {
|
2023-01-27 14:01:54 +00:00
|
|
|
data, err := repo.LoadUnpacked(context.TODO(), restic.PackFile, storageID)
|
2017-10-02 13:06:39 +00:00
|
|
|
rtest.OK(b, err)
|
2020-04-28 05:57:29 +00:00
|
|
|
|
|
|
|
// See comment in BenchmarkLoadBlob.
|
|
|
|
b.StopTimer()
|
2017-01-13 19:56:50 +00:00
|
|
|
if len(data) != length {
|
|
|
|
b.Errorf("wanted %d bytes, got %d", length, len(data))
|
|
|
|
}
|
|
|
|
|
|
|
|
id2 := restic.Hash(data)
|
|
|
|
if !dataID.Equal(id2) {
|
|
|
|
b.Errorf("wrong data returned, wanted %v, got %v", storageID.Str(), id2.Str())
|
|
|
|
}
|
2020-04-28 05:57:29 +00:00
|
|
|
b.StartTimer()
|
2017-01-13 19:56:50 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-07-04 14:52:17 +00:00
|
|
|
var repoFixture = filepath.Join("testdata", "test-repo.tar.gz")
|
|
|
|
|
2015-08-08 10:22:17 +00:00
|
|
|
func TestRepositoryLoadIndex(t *testing.T) {
|
2024-02-24 20:45:24 +00:00
|
|
|
repo, cleanup := repository.TestFromFixture(t, repoFixture)
|
2016-09-04 12:29:04 +00:00
|
|
|
defer cleanup()
|
|
|
|
|
2023-07-16 02:48:30 +00:00
|
|
|
rtest.OK(t, repo.LoadIndex(context.TODO(), nil))
|
2015-07-04 14:52:17 +00:00
|
|
|
}
|
|
|
|
|
2020-11-07 17:53:59 +00:00
|
|
|
// loadIndex loads the index id from backend and returns it.
|
2024-01-19 21:44:50 +00:00
|
|
|
func loadIndex(ctx context.Context, repo restic.LoaderUnpacked, id restic.ID) (*index.Index, error) {
|
2023-01-27 14:01:54 +00:00
|
|
|
buf, err := repo.LoadUnpacked(ctx, restic.IndexFile, id)
|
2020-11-07 17:53:59 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2022-06-12 12:43:43 +00:00
|
|
|
idx, oldFormat, err := index.DecodeIndex(buf, id)
|
2020-11-07 17:53:59 +00:00
|
|
|
if oldFormat {
|
|
|
|
fmt.Fprintf(os.Stderr, "index %v has old format\n", id.Str())
|
|
|
|
}
|
|
|
|
return idx, err
|
|
|
|
}
|
|
|
|
|
2021-09-19 18:02:38 +00:00
|
|
|
func TestRepositoryLoadUnpackedBroken(t *testing.T) {
|
2024-04-14 09:48:40 +00:00
|
|
|
repo := repository.TestRepository(t)
|
2021-09-19 18:02:38 +00:00
|
|
|
|
|
|
|
data := rtest.Random(23, 12345)
|
|
|
|
id := restic.Hash(data)
|
2023-10-01 09:40:12 +00:00
|
|
|
h := backend.Handle{Type: restic.IndexFile, Name: id.String()}
|
2021-09-19 18:02:38 +00:00
|
|
|
// damage buffer
|
|
|
|
data[0] ^= 0xff
|
|
|
|
|
|
|
|
// store broken file
|
2024-04-14 09:48:40 +00:00
|
|
|
err := repo.Backend().Save(context.TODO(), h, backend.NewByteReader(data, repo.Backend().Hasher()))
|
2021-09-19 18:02:38 +00:00
|
|
|
rtest.OK(t, err)
|
|
|
|
|
2023-01-27 14:01:54 +00:00
|
|
|
_, err = repo.LoadUnpacked(context.TODO(), restic.IndexFile, id)
|
2024-05-09 16:59:29 +00:00
|
|
|
rtest.Assert(t, errors.Is(err, restic.ErrInvalidData), "unexpected error: %v", err)
|
2021-09-19 18:02:38 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
type damageOnceBackend struct {
|
2023-10-01 09:40:12 +00:00
|
|
|
backend.Backend
|
2024-05-09 13:26:41 +00:00
|
|
|
m sync.Map
|
2021-09-19 18:02:38 +00:00
|
|
|
}
|
|
|
|
|
2023-10-01 09:40:12 +00:00
|
|
|
func (be *damageOnceBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
|
2021-09-19 18:02:38 +00:00
|
|
|
// don't break the config file as we can't retry it
|
|
|
|
if h.Type == restic.ConfigFile {
|
|
|
|
return be.Backend.Load(ctx, h, length, offset, fn)
|
|
|
|
}
|
2024-05-09 13:26:41 +00:00
|
|
|
|
|
|
|
_, retry := be.m.Swap(h, true)
|
|
|
|
if !retry {
|
|
|
|
// return broken data on the first try
|
2024-05-09 19:22:19 +00:00
|
|
|
offset++
|
2021-09-19 18:02:38 +00:00
|
|
|
}
|
2024-05-09 13:26:41 +00:00
|
|
|
return be.Backend.Load(ctx, h, length, offset, fn)
|
2021-09-19 18:02:38 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func TestRepositoryLoadUnpackedRetryBroken(t *testing.T) {
|
|
|
|
repodir, cleanup := rtest.Env(t, repoFixture)
|
|
|
|
defer cleanup()
|
|
|
|
|
|
|
|
be, err := local.Open(context.TODO(), local.Config{Path: repodir, Connections: 2})
|
|
|
|
rtest.OK(t, err)
|
2024-02-24 20:45:24 +00:00
|
|
|
repo := repository.TestOpenBackend(t, &damageOnceBackend{Backend: be})
|
2021-09-19 18:02:38 +00:00
|
|
|
|
2023-07-16 02:48:30 +00:00
|
|
|
rtest.OK(t, repo.LoadIndex(context.TODO(), nil))
|
2021-09-19 18:02:38 +00:00
|
|
|
}
|
|
|
|
|
2015-07-04 14:52:17 +00:00
|
|
|
func BenchmarkLoadIndex(b *testing.B) {
|
2022-04-29 21:16:16 +00:00
|
|
|
repository.BenchmarkAllVersions(b, benchmarkLoadIndex)
|
|
|
|
}
|
|
|
|
|
|
|
|
func benchmarkLoadIndex(b *testing.B, version uint) {
|
2017-01-13 20:39:40 +00:00
|
|
|
repository.TestUseLowSecurityKDFParameters(b)
|
|
|
|
|
2022-12-11 09:41:22 +00:00
|
|
|
repo := repository.TestRepositoryWithVersion(b, version)
|
2022-06-12 12:43:43 +00:00
|
|
|
idx := index.NewIndex()
|
2017-01-13 20:39:40 +00:00
|
|
|
|
|
|
|
for i := 0; i < 5000; i++ {
|
2022-05-26 11:41:06 +00:00
|
|
|
idx.StorePack(restic.NewRandomID(), []restic.Blob{
|
|
|
|
{
|
2020-11-05 21:18:00 +00:00
|
|
|
BlobHandle: restic.NewRandomBlobHandle(),
|
|
|
|
Length: 1234,
|
|
|
|
Offset: 1235,
|
2017-01-13 20:39:40 +00:00
|
|
|
},
|
|
|
|
})
|
|
|
|
}
|
2022-09-11 11:51:08 +00:00
|
|
|
idx.Finalize()
|
2017-01-13 20:39:40 +00:00
|
|
|
|
2022-06-12 12:43:43 +00:00
|
|
|
id, err := index.SaveIndex(context.TODO(), repo, idx)
|
2017-10-02 13:06:39 +00:00
|
|
|
rtest.OK(b, err)
|
2017-01-13 20:39:40 +00:00
|
|
|
|
2022-05-26 13:05:29 +00:00
|
|
|
b.Logf("index saved as %v", id.Str())
|
2023-10-01 09:40:12 +00:00
|
|
|
fi, err := repo.Backend().Stat(context.TODO(), backend.Handle{Type: restic.IndexFile, Name: id.String()})
|
2017-10-02 13:06:39 +00:00
|
|
|
rtest.OK(b, err)
|
2017-01-13 20:39:40 +00:00
|
|
|
b.Logf("filesize is %v", fi.Size)
|
|
|
|
|
2016-09-04 12:29:04 +00:00
|
|
|
b.ResetTimer()
|
|
|
|
|
|
|
|
for i := 0; i < b.N; i++ {
|
2020-11-07 17:53:59 +00:00
|
|
|
_, err := loadIndex(context.TODO(), repo, id)
|
2017-10-02 13:06:39 +00:00
|
|
|
rtest.OK(b, err)
|
2016-09-04 12:29:04 +00:00
|
|
|
}
|
2015-07-04 14:52:17 +00:00
|
|
|
}
|
2015-10-25 14:35:33 +00:00
|
|
|
|
|
|
|
// saveRandomDataBlobs generates random data blobs and saves them to the repository.
|
2016-09-03 11:34:04 +00:00
|
|
|
func saveRandomDataBlobs(t testing.TB, repo restic.Repository, num int, sizeMax int) {
|
2021-08-07 20:52:05 +00:00
|
|
|
var wg errgroup.Group
|
|
|
|
repo.StartPackUploader(context.TODO(), &wg)
|
|
|
|
|
2015-10-25 14:35:33 +00:00
|
|
|
for i := 0; i < num; i++ {
|
2017-01-13 11:20:37 +00:00
|
|
|
size := rand.Int() % sizeMax
|
2015-10-25 14:35:33 +00:00
|
|
|
|
|
|
|
buf := make([]byte, size)
|
2017-01-13 11:20:37 +00:00
|
|
|
_, err := io.ReadFull(rnd, buf)
|
2017-10-02 13:06:39 +00:00
|
|
|
rtest.OK(t, err)
|
2015-10-25 14:35:33 +00:00
|
|
|
|
2022-05-01 12:26:57 +00:00
|
|
|
_, _, _, err = repo.SaveBlob(context.TODO(), restic.DataBlob, buf, restic.ID{}, false)
|
2017-10-02 13:06:39 +00:00
|
|
|
rtest.OK(t, err)
|
2015-10-25 14:35:33 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestRepositoryIncrementalIndex(t *testing.T) {
|
2022-04-29 21:16:16 +00:00
|
|
|
repository.TestAllVersions(t, testRepositoryIncrementalIndex)
|
|
|
|
}
|
|
|
|
|
|
|
|
func testRepositoryIncrementalIndex(t *testing.T, version uint) {
|
2022-12-11 09:41:22 +00:00
|
|
|
repo := repository.TestRepositoryWithVersion(t, version).(*repository.Repository)
|
2020-06-06 20:20:44 +00:00
|
|
|
|
2022-06-12 12:43:43 +00:00
|
|
|
index.IndexFull = func(*index.Index, bool) bool { return true }
|
2015-10-25 14:35:33 +00:00
|
|
|
|
2022-05-26 11:30:52 +00:00
|
|
|
// add a few rounds of packs
|
2015-10-25 14:35:33 +00:00
|
|
|
for j := 0; j < 5; j++ {
|
2022-05-26 11:30:52 +00:00
|
|
|
// add some packs, write intermediate index
|
|
|
|
saveRandomDataBlobs(t, repo, 20, 1<<15)
|
2022-05-26 10:38:18 +00:00
|
|
|
rtest.OK(t, repo.Flush(context.TODO()))
|
2015-10-25 14:35:33 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// save final index
|
2022-05-26 10:38:18 +00:00
|
|
|
rtest.OK(t, repo.Flush(context.TODO()))
|
2015-10-25 14:35:33 +00:00
|
|
|
|
2016-08-31 18:29:54 +00:00
|
|
|
packEntries := make(map[restic.ID]map[restic.ID]struct{})
|
2015-10-25 14:35:33 +00:00
|
|
|
|
2018-01-21 16:25:36 +00:00
|
|
|
err := repo.List(context.TODO(), restic.IndexFile, func(id restic.ID, size int64) error {
|
2020-11-07 17:53:59 +00:00
|
|
|
idx, err := loadIndex(context.TODO(), repo, id)
|
2017-10-02 13:06:39 +00:00
|
|
|
rtest.OK(t, err)
|
2015-10-25 14:35:33 +00:00
|
|
|
|
2024-04-05 20:20:14 +00:00
|
|
|
rtest.OK(t, idx.Each(context.TODO(), func(pb restic.PackedBlob) {
|
2015-10-25 14:35:33 +00:00
|
|
|
if _, ok := packEntries[pb.PackID]; !ok {
|
2016-08-31 18:29:54 +00:00
|
|
|
packEntries[pb.PackID] = make(map[restic.ID]struct{})
|
2015-10-25 14:35:33 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
packEntries[pb.PackID][id] = struct{}{}
|
2024-04-05 20:20:14 +00:00
|
|
|
}))
|
2018-01-21 16:25:36 +00:00
|
|
|
return nil
|
|
|
|
})
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
2015-10-25 14:35:33 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
for packID, ids := range packEntries {
|
|
|
|
if len(ids) > 1 {
|
|
|
|
t.Errorf("pack %v listed in %d indexes\n", packID, len(ids))
|
|
|
|
}
|
|
|
|
}
|
2022-03-21 19:38:53 +00:00
|
|
|
|
|
|
|
}
|
|
|
|
|
2022-10-29 20:03:39 +00:00
|
|
|
func TestInvalidCompression(t *testing.T) {
|
|
|
|
var comp repository.CompressionMode
|
|
|
|
err := comp.Set("nope")
|
|
|
|
rtest.Assert(t, err != nil, "missing error")
|
|
|
|
_, err = repository.New(nil, repository.Options{Compression: comp})
|
|
|
|
rtest.Assert(t, err != nil, "missing error")
|
|
|
|
}
|