repository: split index into a separate package

This commit is contained in:
Michael Eischer 2022-06-12 14:43:43 +02:00
parent 5760ba6989
commit 2e3f1c08c5
20 changed files with 101 additions and 80 deletions

View file

@ -22,6 +22,7 @@ import (
"github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/crypto"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/index"
"github.com/restic/restic/internal/pack" "github.com/restic/restic/internal/pack"
"github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
@ -129,7 +130,7 @@ func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer)
} }
func dumpIndexes(ctx context.Context, repo restic.Repository, wr io.Writer) error { func dumpIndexes(ctx context.Context, repo restic.Repository, wr io.Writer) error {
return repository.ForAllIndexes(ctx, repo, func(id restic.ID, idx *repository.Index, oldFormat bool, err error) error { return index.ForAllIndexes(ctx, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
Printf("index_id: %v\n", id) Printf("index_id: %v\n", id)
if err != nil { if err != nil {
return err return err

View file

@ -4,7 +4,7 @@ import (
"context" "context"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/index"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
"github.com/spf13/cobra" "github.com/spf13/cobra"
@ -63,7 +63,7 @@ func runList(ctx context.Context, cmd *cobra.Command, opts GlobalOptions, args [
case "locks": case "locks":
t = restic.LockFile t = restic.LockFile
case "blobs": case "blobs":
return repository.ForAllIndexes(ctx, repo, func(id restic.ID, idx *repository.Index, oldFormat bool, err error) error { return index.ForAllIndexes(ctx, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
if err != nil { if err != nil {
return err return err
} }

View file

@ -9,6 +9,7 @@ import (
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/index"
"github.com/restic/restic/internal/pack" "github.com/restic/restic/internal/pack"
"github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
@ -699,7 +700,7 @@ func doPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo r
if opts.unsafeRecovery { if opts.unsafeRecovery {
Verbosef("deleting index files\n") Verbosef("deleting index files\n")
indexFiles := repo.Index().(*repository.MasterIndex).IDs() indexFiles := repo.Index().(*index.MasterIndex).IDs()
err = DeleteFilesChecked(ctx, gopts, repo, indexFiles, restic.IndexFile) err = DeleteFilesChecked(ctx, gopts, repo, indexFiles, restic.IndexFile)
if err != nil { if err != nil {
return errors.Fatalf("%s", err) return errors.Fatalf("%s", err)

View file

@ -3,6 +3,7 @@ package main
import ( import (
"context" "context"
"github.com/restic/restic/internal/index"
"github.com/restic/restic/internal/pack" "github.com/restic/restic/internal/pack"
"github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
@ -74,8 +75,8 @@ func rebuildIndex(ctx context.Context, opts RebuildIndexOptions, gopts GlobalOpt
} }
} else { } else {
Verbosef("loading indexes...\n") Verbosef("loading indexes...\n")
mi := repository.NewMasterIndex() mi := index.NewMasterIndex()
err := repository.ForAllIndexes(ctx, repo, func(id restic.ID, idx *repository.Index, oldFormat bool, err error) error { err := index.ForAllIndexes(ctx, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
if err != nil { if err != nil {
Warnf("removing invalid index %v: %v\n", id, err) Warnf("removing invalid index %v: %v\n", id, err)
obsoleteIndexes = append(obsoleteIndexes, id) obsoleteIndexes = append(obsoleteIndexes, id)

View file

@ -23,6 +23,7 @@ import (
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/filter" "github.com/restic/restic/internal/filter"
"github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/index"
"github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test"
@ -1477,11 +1478,11 @@ func TestRebuildIndex(t *testing.T) {
} }
func TestRebuildIndexAlwaysFull(t *testing.T) { func TestRebuildIndexAlwaysFull(t *testing.T) {
indexFull := repository.IndexFull indexFull := index.IndexFull
defer func() { defer func() {
repository.IndexFull = indexFull index.IndexFull = indexFull
}() }()
repository.IndexFull = func(*repository.Index, bool) bool { return true } index.IndexFull = func(*index.Index, bool) bool { return true }
testRebuildIndex(t, nil) testRebuildIndex(t, nil)
} }

View file

@ -8,7 +8,7 @@ import (
"testing" "testing"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/index"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
"golang.org/x/sync/errgroup" "golang.org/x/sync/errgroup"
) )
@ -40,7 +40,7 @@ func TestBlobSaver(t *testing.T) {
wg, ctx := errgroup.WithContext(ctx) wg, ctx := errgroup.WithContext(ctx)
saver := &saveFail{ saver := &saveFail{
idx: repository.NewMasterIndex(), idx: index.NewMasterIndex(),
} }
b := NewBlobSaver(ctx, wg, saver, uint(runtime.NumCPU())) b := NewBlobSaver(ctx, wg, saver, uint(runtime.NumCPU()))
@ -86,7 +86,7 @@ func TestBlobSaverError(t *testing.T) {
wg, ctx := errgroup.WithContext(ctx) wg, ctx := errgroup.WithContext(ctx)
saver := &saveFail{ saver := &saveFail{
idx: repository.NewMasterIndex(), idx: index.NewMasterIndex(),
failAt: int32(test.failAt), failAt: int32(test.failAt),
} }

View file

@ -18,6 +18,7 @@ import (
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/hashing" "github.com/restic/restic/internal/hashing"
"github.com/restic/restic/internal/index"
"github.com/restic/restic/internal/pack" "github.com/restic/restic/internal/pack"
"github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
@ -38,7 +39,7 @@ type Checker struct {
} }
trackUnused bool trackUnused bool
masterIndex *repository.MasterIndex masterIndex *index.MasterIndex
snapshots restic.Lister snapshots restic.Lister
repo restic.Repository repo restic.Repository
@ -48,7 +49,7 @@ type Checker struct {
func New(repo restic.Repository, trackUnused bool) *Checker { func New(repo restic.Repository, trackUnused bool) *Checker {
c := &Checker{ c := &Checker{
packs: make(map[restic.ID]int64), packs: make(map[restic.ID]int64),
masterIndex: repository.NewMasterIndex(), masterIndex: index.NewMasterIndex(),
repo: repo, repo: repo,
trackUnused: trackUnused, trackUnused: trackUnused,
} }
@ -121,7 +122,7 @@ func (c *Checker) LoadIndex(ctx context.Context) (hints []error, errs []error) {
debug.Log("Start") debug.Log("Start")
packToIndex := make(map[restic.ID]restic.IDSet) packToIndex := make(map[restic.ID]restic.IDSet)
err := repository.ForAllIndexes(ctx, c.repo, func(id restic.ID, index *repository.Index, oldFormat bool, err error) error { err := index.ForAllIndexes(ctx, c.repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error {
debug.Log("process index %v, err %v", id, err) debug.Log("process index %v, err %v", id, err)
if oldFormat { if oldFormat {

View file

@ -1,4 +1,4 @@
package repository package index
import ( import (
"context" "context"

View file

@ -1,4 +1,4 @@
package repository package index
import ( import (
"context" "context"

View file

@ -1,15 +1,19 @@
package repository_test package index_test
import ( import (
"context" "context"
"path/filepath"
"testing" "testing"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/index"
"github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test"
) )
var repoFixture = filepath.Join("..", "repository", "testdata", "test-repo.tar.gz")
func TestRepositoryForAllIndexes(t *testing.T) { func TestRepositoryForAllIndexes(t *testing.T) {
repodir, cleanup := rtest.Env(t, repoFixture) repodir, cleanup := rtest.Env(t, repoFixture)
defer cleanup() defer cleanup()
@ -25,7 +29,7 @@ func TestRepositoryForAllIndexes(t *testing.T) {
// check that all expected indexes are loaded without errors // check that all expected indexes are loaded without errors
indexIDs := restic.NewIDSet() indexIDs := restic.NewIDSet()
var indexErr error var indexErr error
rtest.OK(t, repository.ForAllIndexes(context.TODO(), repo, func(id restic.ID, index *repository.Index, oldFormat bool, err error) error { rtest.OK(t, index.ForAllIndexes(context.TODO(), repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error {
if err != nil { if err != nil {
indexErr = err indexErr = err
} }
@ -38,7 +42,7 @@ func TestRepositoryForAllIndexes(t *testing.T) {
// must failed with the returned error // must failed with the returned error
iterErr := errors.New("error to pass upwards") iterErr := errors.New("error to pass upwards")
err := repository.ForAllIndexes(context.TODO(), repo, func(id restic.ID, index *repository.Index, oldFormat bool, err error) error { err := index.ForAllIndexes(context.TODO(), repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error {
return iterErr return iterErr
}) })

View file

@ -1,4 +1,4 @@
package repository_test package index_test
import ( import (
"bytes" "bytes"
@ -7,7 +7,7 @@ import (
"sync" "sync"
"testing" "testing"
"github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/index"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test"
) )
@ -15,7 +15,7 @@ import (
func TestIndexSerialize(t *testing.T) { func TestIndexSerialize(t *testing.T) {
tests := []restic.PackedBlob{} tests := []restic.PackedBlob{}
idx := repository.NewIndex() idx := index.NewIndex()
// create 50 packs with 20 blobs each // create 50 packs with 20 blobs each
for i := 0; i < 50; i++ { for i := 0; i < 50; i++ {
@ -51,7 +51,7 @@ func TestIndexSerialize(t *testing.T) {
rtest.OK(t, err) rtest.OK(t, err)
idx2ID := restic.NewRandomID() idx2ID := restic.NewRandomID()
idx2, oldFormat, err := repository.DecodeIndex(wr.Bytes(), idx2ID) idx2, oldFormat, err := index.DecodeIndex(wr.Bytes(), idx2ID)
rtest.OK(t, err) rtest.OK(t, err)
rtest.Assert(t, idx2 != nil, rtest.Assert(t, idx2 != nil,
"nil returned for decoded index") "nil returned for decoded index")
@ -121,7 +121,7 @@ func TestIndexSerialize(t *testing.T) {
rtest.OK(t, err) rtest.OK(t, err)
rtest.Equals(t, restic.IDs{id}, ids) rtest.Equals(t, restic.IDs{id}, ids)
idx3, oldFormat, err := repository.DecodeIndex(wr3.Bytes(), id) idx3, oldFormat, err := index.DecodeIndex(wr3.Bytes(), id)
rtest.OK(t, err) rtest.OK(t, err)
rtest.Assert(t, idx3 != nil, rtest.Assert(t, idx3 != nil,
"nil returned for decoded index") "nil returned for decoded index")
@ -143,7 +143,7 @@ func TestIndexSerialize(t *testing.T) {
} }
func TestIndexSize(t *testing.T) { func TestIndexSize(t *testing.T) {
idx := repository.NewIndex() idx := index.NewIndex()
packs := 200 packs := 200
blobCount := 100 blobCount := 100
@ -309,7 +309,7 @@ func TestIndexUnserialize(t *testing.T) {
} { } {
oldIdx := restic.IDs{restic.TestParseID("ed54ae36197f4745ebc4b54d10e0f623eaaaedd03013eb7ae90df881b7781452")} oldIdx := restic.IDs{restic.TestParseID("ed54ae36197f4745ebc4b54d10e0f623eaaaedd03013eb7ae90df881b7781452")}
idx, oldFormat, err := repository.DecodeIndex(task.idxBytes, restic.NewRandomID()) idx, oldFormat, err := index.DecodeIndex(task.idxBytes, restic.NewRandomID())
rtest.OK(t, err) rtest.OK(t, err)
rtest.Assert(t, !oldFormat, "new index format recognized as old format") rtest.Assert(t, !oldFormat, "new index format recognized as old format")
@ -354,7 +354,7 @@ func TestIndexUnserialize(t *testing.T) {
} }
} }
func listPack(idx *repository.Index, id restic.ID) (pbs []restic.PackedBlob) { func listPack(idx *index.Index, id restic.ID) (pbs []restic.PackedBlob) {
idx.Each(context.TODO(), func(pb restic.PackedBlob) { idx.Each(context.TODO(), func(pb restic.PackedBlob) {
if pb.PackID.Equal(id) { if pb.PackID.Equal(id) {
pbs = append(pbs, pb) pbs = append(pbs, pb)
@ -386,7 +386,7 @@ func BenchmarkDecodeIndex(b *testing.B) {
b.ResetTimer() b.ResetTimer()
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
_, _, err := repository.DecodeIndex(benchmarkIndexJSON, id) _, _, err := index.DecodeIndex(benchmarkIndexJSON, id)
rtest.OK(b, err) rtest.OK(b, err)
} }
} }
@ -399,14 +399,14 @@ func BenchmarkDecodeIndexParallel(b *testing.B) {
b.RunParallel(func(pb *testing.PB) { b.RunParallel(func(pb *testing.PB) {
for pb.Next() { for pb.Next() {
_, _, err := repository.DecodeIndex(benchmarkIndexJSON, id) _, _, err := index.DecodeIndex(benchmarkIndexJSON, id)
rtest.OK(b, err) rtest.OK(b, err)
} }
}) })
} }
func TestIndexUnserializeOld(t *testing.T) { func TestIndexUnserializeOld(t *testing.T) {
idx, oldFormat, err := repository.DecodeIndex(docOldExample, restic.NewRandomID()) idx, oldFormat, err := index.DecodeIndex(docOldExample, restic.NewRandomID())
rtest.OK(t, err) rtest.OK(t, err)
rtest.Assert(t, oldFormat, "old index format recognized as new format") rtest.Assert(t, oldFormat, "old index format recognized as new format")
@ -427,7 +427,7 @@ func TestIndexUnserializeOld(t *testing.T) {
} }
func TestIndexPacks(t *testing.T) { func TestIndexPacks(t *testing.T) {
idx := repository.NewIndex() idx := index.NewIndex()
packs := restic.NewIDSet() packs := restic.NewIDSet()
for i := 0; i < 20; i++ { for i := 0; i < 20; i++ {
@ -456,8 +456,8 @@ func NewRandomTestID(rng *rand.Rand) restic.ID {
return id return id
} }
func createRandomIndex(rng *rand.Rand, packfiles int) (idx *repository.Index, lookupBh restic.BlobHandle) { func createRandomIndex(rng *rand.Rand, packfiles int) (idx *index.Index, lookupBh restic.BlobHandle) {
idx = repository.NewIndex() idx = index.NewIndex()
// create index with given number of pack files // create index with given number of pack files
for i := 0; i < packfiles; i++ { for i := 0; i < packfiles; i++ {
@ -536,7 +536,7 @@ func BenchmarkIndexAllocParallel(b *testing.B) {
func TestIndexHas(t *testing.T) { func TestIndexHas(t *testing.T) {
tests := []restic.PackedBlob{} tests := []restic.PackedBlob{}
idx := repository.NewIndex() idx := index.NewIndex()
// create 50 packs with 20 blobs each // create 50 packs with 20 blobs each
for i := 0; i < 50; i++ { for i := 0; i < 50; i++ {
@ -576,7 +576,7 @@ func TestIndexHas(t *testing.T) {
} }
func TestMixedEachByPack(t *testing.T) { func TestMixedEachByPack(t *testing.T) {
idx := repository.NewIndex() idx := index.NewIndex()
expected := make(map[restic.ID]int) expected := make(map[restic.ID]int)
// create 50 packs with 2 blobs each // create 50 packs with 2 blobs each
@ -615,7 +615,7 @@ func TestMixedEachByPack(t *testing.T) {
} }
func TestEachByPackIgnoes(t *testing.T) { func TestEachByPackIgnoes(t *testing.T) {
idx := repository.NewIndex() idx := index.NewIndex()
ignores := restic.NewIDSet() ignores := restic.NewIDSet()
expected := make(map[restic.ID]int) expected := make(map[restic.ID]int)

View file

@ -1,4 +1,4 @@
package repository package index
import ( import (
"hash/maphash" "hash/maphash"

View file

@ -1,4 +1,4 @@
package repository package index
import ( import (
"math/rand" "math/rand"

View file

@ -1,4 +1,4 @@
package repository package index
import ( import (
"bytes" "bytes"
@ -31,7 +31,7 @@ func NewMasterIndex() *MasterIndex {
return &MasterIndex{idx: idx, pendingBlobs: restic.NewBlobSet()} return &MasterIndex{idx: idx, pendingBlobs: restic.NewBlobSet()}
} }
func (mi *MasterIndex) markCompressed() { func (mi *MasterIndex) MarkCompressed() {
mi.compress = true mi.compress = true
} }
@ -65,7 +65,7 @@ func (mi *MasterIndex) LookupSize(bh restic.BlobHandle) (uint, bool) {
// Before doing so it checks if this blob is already known. // Before doing so it checks if this blob is already known.
// Returns true if adding was successful and false if the blob // Returns true if adding was successful and false if the blob
// was already known // was already known
func (mi *MasterIndex) addPending(bh restic.BlobHandle) bool { func (mi *MasterIndex) AddPending(bh restic.BlobHandle) bool {
mi.idxMutex.Lock() mi.idxMutex.Lock()
defer mi.idxMutex.Unlock() defer mi.idxMutex.Unlock()

View file

@ -1,4 +1,4 @@
package repository_test package index_test
import ( import (
"context" "context"
@ -9,6 +9,7 @@ import (
"github.com/restic/restic/internal/checker" "github.com/restic/restic/internal/checker"
"github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/crypto"
"github.com/restic/restic/internal/index"
"github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test"
@ -57,15 +58,15 @@ func TestMasterIndex(t *testing.T) {
}, },
} }
idx1 := repository.NewIndex() idx1 := index.NewIndex()
idx1.StorePack(blob1.PackID, []restic.Blob{blob1.Blob}) idx1.StorePack(blob1.PackID, []restic.Blob{blob1.Blob})
idx1.StorePack(blob12a.PackID, []restic.Blob{blob12a.Blob}) idx1.StorePack(blob12a.PackID, []restic.Blob{blob12a.Blob})
idx2 := repository.NewIndex() idx2 := index.NewIndex()
idx2.StorePack(blob2.PackID, []restic.Blob{blob2.Blob}) idx2.StorePack(blob2.PackID, []restic.Blob{blob2.Blob})
idx2.StorePack(blob12b.PackID, []restic.Blob{blob12b.Blob}) idx2.StorePack(blob12b.PackID, []restic.Blob{blob12b.Blob})
mIdx := repository.NewMasterIndex() mIdx := index.NewMasterIndex()
mIdx.Insert(idx1) mIdx.Insert(idx1)
mIdx.Insert(idx2) mIdx.Insert(idx2)
@ -148,18 +149,18 @@ func TestMasterMergeFinalIndexes(t *testing.T) {
}, },
} }
idx1 := repository.NewIndex() idx1 := index.NewIndex()
idx1.StorePack(blob1.PackID, []restic.Blob{blob1.Blob}) idx1.StorePack(blob1.PackID, []restic.Blob{blob1.Blob})
idx2 := repository.NewIndex() idx2 := index.NewIndex()
idx2.StorePack(blob2.PackID, []restic.Blob{blob2.Blob}) idx2.StorePack(blob2.PackID, []restic.Blob{blob2.Blob})
mIdx := repository.NewMasterIndex() mIdx := index.NewMasterIndex()
mIdx.Insert(idx1) mIdx.Insert(idx1)
mIdx.Insert(idx2) mIdx.Insert(idx2)
finalIndexes, idxCount := repository.TestMergeIndex(t, mIdx) finalIndexes, idxCount := index.TestMergeIndex(t, mIdx)
rtest.Equals(t, []*repository.Index{idx1, idx2}, finalIndexes) rtest.Equals(t, []*index.Index{idx1, idx2}, finalIndexes)
rtest.Equals(t, 1, idxCount) rtest.Equals(t, 1, idxCount)
blobCount := 0 blobCount := 0
@ -178,13 +179,13 @@ func TestMasterMergeFinalIndexes(t *testing.T) {
rtest.Assert(t, blobs == nil, "Expected no blobs when fetching with a random id") rtest.Assert(t, blobs == nil, "Expected no blobs when fetching with a random id")
// merge another index containing identical blobs // merge another index containing identical blobs
idx3 := repository.NewIndex() idx3 := index.NewIndex()
idx3.StorePack(blob1.PackID, []restic.Blob{blob1.Blob}) idx3.StorePack(blob1.PackID, []restic.Blob{blob1.Blob})
idx3.StorePack(blob2.PackID, []restic.Blob{blob2.Blob}) idx3.StorePack(blob2.PackID, []restic.Blob{blob2.Blob})
mIdx.Insert(idx3) mIdx.Insert(idx3)
finalIndexes, idxCount = repository.TestMergeIndex(t, mIdx) finalIndexes, idxCount = index.TestMergeIndex(t, mIdx)
rtest.Equals(t, []*repository.Index{idx3}, finalIndexes) rtest.Equals(t, []*index.Index{idx3}, finalIndexes)
rtest.Equals(t, 1, idxCount) rtest.Equals(t, 1, idxCount)
// Index should have same entries as before! // Index should have same entries as before!
@ -201,8 +202,8 @@ func TestMasterMergeFinalIndexes(t *testing.T) {
rtest.Equals(t, 2, blobCount) rtest.Equals(t, 2, blobCount)
} }
func createRandomMasterIndex(t testing.TB, rng *rand.Rand, num, size int) (*repository.MasterIndex, restic.BlobHandle) { func createRandomMasterIndex(t testing.TB, rng *rand.Rand, num, size int) (*index.MasterIndex, restic.BlobHandle) {
mIdx := repository.NewMasterIndex() mIdx := index.NewMasterIndex()
for i := 0; i < num-1; i++ { for i := 0; i < num-1; i++ {
idx, _ := createRandomIndex(rng, size) idx, _ := createRandomIndex(rng, size)
mIdx.Insert(idx) mIdx.Insert(idx)
@ -210,7 +211,7 @@ func createRandomMasterIndex(t testing.TB, rng *rand.Rand, num, size int) (*repo
idx1, lookupBh := createRandomIndex(rng, size) idx1, lookupBh := createRandomIndex(rng, size)
mIdx.Insert(idx1) mIdx.Insert(idx1)
repository.TestMergeIndex(t, mIdx) index.TestMergeIndex(t, mIdx)
return mIdx, lookupBh return mIdx, lookupBh
} }

18
internal/index/testing.go Normal file
View file

@ -0,0 +1,18 @@
package index
import (
"testing"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/test"
)
func TestMergeIndex(t testing.TB, mi *MasterIndex) ([]*Index, int) {
finalIndexes := mi.finalizeNotFinalIndexes()
for _, idx := range finalIndexes {
test.OK(t, idx.SetID(restic.NewRandomID()))
}
test.OK(t, mi.MergeFinalIndexes())
return finalIndexes, len(mi.idx)
}

View file

@ -6,6 +6,7 @@ import (
"testing" "testing"
"time" "time"
"github.com/restic/restic/internal/index"
"github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test"
@ -170,7 +171,7 @@ func flush(t *testing.T, repo restic.Repository) {
} }
func rebuildIndex(t *testing.T, repo restic.Repository) { func rebuildIndex(t *testing.T, repo restic.Repository) {
err := repo.SetIndex(repository.NewMasterIndex()) err := repo.SetIndex(index.NewMasterIndex())
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -207,7 +208,7 @@ func rebuildIndex(t *testing.T, repo restic.Repository) {
} }
func reloadIndex(t *testing.T, repo restic.Repository) { func reloadIndex(t *testing.T, repo restic.Repository) {
err := repo.SetIndex(repository.NewMasterIndex()) err := repo.SetIndex(index.NewMasterIndex())
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }

View file

@ -19,6 +19,7 @@ import (
"github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/crypto"
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/index"
"github.com/restic/restic/internal/pack" "github.com/restic/restic/internal/pack"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui/progress" "github.com/restic/restic/internal/ui/progress"
@ -38,7 +39,7 @@ type Repository struct {
cfg restic.Config cfg restic.Config
key *crypto.Key key *crypto.Key
keyName string keyName string
idx *MasterIndex idx *index.MasterIndex
Cache *cache.Cache Cache *cache.Cache
opts Options opts Options
@ -118,7 +119,7 @@ func New(be restic.Backend, opts Options) (*Repository, error) {
repo := &Repository{ repo := &Repository{
be: be, be: be,
opts: opts, opts: opts,
idx: NewMasterIndex(), idx: index.NewMasterIndex(),
} }
return repo, nil return repo, nil
@ -134,7 +135,7 @@ func (r *Repository) DisableAutoIndexUpdate() {
func (r *Repository) setConfig(cfg restic.Config) { func (r *Repository) setConfig(cfg restic.Config) {
r.cfg = cfg r.cfg = cfg
if r.cfg.Version >= 2 { if r.cfg.Version >= 2 {
r.idx.markCompressed() r.idx.MarkCompressed()
} }
} }
@ -563,7 +564,7 @@ func (r *Repository) Index() restic.MasterIndex {
// SetIndex instructs the repository to use the given index. // SetIndex instructs the repository to use the given index.
func (r *Repository) SetIndex(i restic.MasterIndex) error { func (r *Repository) SetIndex(i restic.MasterIndex) error {
r.idx = i.(*MasterIndex) r.idx = i.(*index.MasterIndex)
return r.prepareCache() return r.prepareCache()
} }
@ -572,7 +573,7 @@ func (r *Repository) SetIndex(i restic.MasterIndex) error {
func (r *Repository) LoadIndex(ctx context.Context) error { func (r *Repository) LoadIndex(ctx context.Context) error {
debug.Log("Loading index") debug.Log("Loading index")
err := ForAllIndexes(ctx, r, func(id restic.ID, idx *Index, oldFormat bool, err error) error { err := index.ForAllIndexes(ctx, r, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
if err != nil { if err != nil {
return err return err
} }
@ -829,7 +830,7 @@ func (r *Repository) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte
} }
// first try to add to pending blobs; if not successful, this blob is already known // first try to add to pending blobs; if not successful, this blob is already known
known = !r.idx.addPending(restic.BlobHandle{ID: newID, Type: t}) known = !r.idx.AddPending(restic.BlobHandle{ID: newID, Type: t})
// only save when needed or explicitly told // only save when needed or explicitly told
if !known || storeDuplicate { if !known || storeDuplicate {

View file

@ -18,6 +18,7 @@ import (
"github.com/klauspost/compress/zstd" "github.com/klauspost/compress/zstd"
"github.com/restic/restic/internal/backend/local" "github.com/restic/restic/internal/backend/local"
"github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/crypto"
"github.com/restic/restic/internal/index"
"github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/test" "github.com/restic/restic/internal/test"
@ -267,13 +268,13 @@ func TestRepositoryLoadIndex(t *testing.T) {
} }
// loadIndex loads the index id from backend and returns it. // loadIndex loads the index id from backend and returns it.
func loadIndex(ctx context.Context, repo restic.Repository, id restic.ID) (*repository.Index, error) { func loadIndex(ctx context.Context, repo restic.Repository, id restic.ID) (*index.Index, error) {
buf, err := repo.LoadUnpacked(ctx, restic.IndexFile, id, nil) buf, err := repo.LoadUnpacked(ctx, restic.IndexFile, id, nil)
if err != nil { if err != nil {
return nil, err return nil, err
} }
idx, oldFormat, err := repository.DecodeIndex(buf, id) idx, oldFormat, err := index.DecodeIndex(buf, id)
if oldFormat { if oldFormat {
fmt.Fprintf(os.Stderr, "index %v has old format\n", id.Str()) fmt.Fprintf(os.Stderr, "index %v has old format\n", id.Str())
} }
@ -345,7 +346,7 @@ func benchmarkLoadIndex(b *testing.B, version uint) {
repo, cleanup := repository.TestRepositoryWithVersion(b, version) repo, cleanup := repository.TestRepositoryWithVersion(b, version)
defer cleanup() defer cleanup()
idx := repository.NewIndex() idx := index.NewIndex()
for i := 0; i < 5000; i++ { for i := 0; i < 5000; i++ {
idx.StorePack(restic.NewRandomID(), []restic.Blob{ idx.StorePack(restic.NewRandomID(), []restic.Blob{
@ -357,7 +358,7 @@ func benchmarkLoadIndex(b *testing.B, version uint) {
}) })
} }
id, err := repository.SaveIndex(context.TODO(), repo, idx) id, err := index.SaveIndex(context.TODO(), repo, idx)
rtest.OK(b, err) rtest.OK(b, err)
b.Logf("index saved as %v", id.Str()) b.Logf("index saved as %v", id.Str())
@ -400,7 +401,7 @@ func testRepositoryIncrementalIndex(t *testing.T, version uint) {
repo := r.(*repository.Repository) repo := r.(*repository.Repository)
repository.IndexFull = func(*repository.Index, bool) bool { return true } index.IndexFull = func(*index.Index, bool) bool { return true }
// add a few rounds of packs // add a few rounds of packs
for j := 0; j < 5; j++ { for j := 0; j < 5; j++ {

View file

@ -138,13 +138,3 @@ func BenchmarkAllVersions(b *testing.B, bench VersionedBenchmark) {
}) })
} }
} }
func TestMergeIndex(t testing.TB, mi *MasterIndex) ([]*Index, int) {
finalIndexes := mi.finalizeNotFinalIndexes()
for _, idx := range finalIndexes {
test.OK(t, idx.SetID(restic.NewRandomID()))
}
test.OK(t, mi.MergeFinalIndexes())
return finalIndexes, len(mi.idx)
}