forked from TrueCloudLab/restic
repository: split index into a separate package
This commit is contained in:
parent
5760ba6989
commit
2e3f1c08c5
20 changed files with 101 additions and 80 deletions
|
@ -22,6 +22,7 @@ import (
|
||||||
"github.com/restic/restic/internal/backend"
|
"github.com/restic/restic/internal/backend"
|
||||||
"github.com/restic/restic/internal/crypto"
|
"github.com/restic/restic/internal/crypto"
|
||||||
"github.com/restic/restic/internal/errors"
|
"github.com/restic/restic/internal/errors"
|
||||||
|
"github.com/restic/restic/internal/index"
|
||||||
"github.com/restic/restic/internal/pack"
|
"github.com/restic/restic/internal/pack"
|
||||||
"github.com/restic/restic/internal/repository"
|
"github.com/restic/restic/internal/repository"
|
||||||
"github.com/restic/restic/internal/restic"
|
"github.com/restic/restic/internal/restic"
|
||||||
|
@ -129,7 +130,7 @@ func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer)
|
||||||
}
|
}
|
||||||
|
|
||||||
func dumpIndexes(ctx context.Context, repo restic.Repository, wr io.Writer) error {
|
func dumpIndexes(ctx context.Context, repo restic.Repository, wr io.Writer) error {
|
||||||
return repository.ForAllIndexes(ctx, repo, func(id restic.ID, idx *repository.Index, oldFormat bool, err error) error {
|
return index.ForAllIndexes(ctx, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
|
||||||
Printf("index_id: %v\n", id)
|
Printf("index_id: %v\n", id)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
|
|
@ -4,7 +4,7 @@ import (
|
||||||
"context"
|
"context"
|
||||||
|
|
||||||
"github.com/restic/restic/internal/errors"
|
"github.com/restic/restic/internal/errors"
|
||||||
"github.com/restic/restic/internal/repository"
|
"github.com/restic/restic/internal/index"
|
||||||
"github.com/restic/restic/internal/restic"
|
"github.com/restic/restic/internal/restic"
|
||||||
|
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
|
@ -63,7 +63,7 @@ func runList(ctx context.Context, cmd *cobra.Command, opts GlobalOptions, args [
|
||||||
case "locks":
|
case "locks":
|
||||||
t = restic.LockFile
|
t = restic.LockFile
|
||||||
case "blobs":
|
case "blobs":
|
||||||
return repository.ForAllIndexes(ctx, repo, func(id restic.ID, idx *repository.Index, oldFormat bool, err error) error {
|
return index.ForAllIndexes(ctx, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
|
@ -9,6 +9,7 @@ import (
|
||||||
|
|
||||||
"github.com/restic/restic/internal/debug"
|
"github.com/restic/restic/internal/debug"
|
||||||
"github.com/restic/restic/internal/errors"
|
"github.com/restic/restic/internal/errors"
|
||||||
|
"github.com/restic/restic/internal/index"
|
||||||
"github.com/restic/restic/internal/pack"
|
"github.com/restic/restic/internal/pack"
|
||||||
"github.com/restic/restic/internal/repository"
|
"github.com/restic/restic/internal/repository"
|
||||||
"github.com/restic/restic/internal/restic"
|
"github.com/restic/restic/internal/restic"
|
||||||
|
@ -699,7 +700,7 @@ func doPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo r
|
||||||
|
|
||||||
if opts.unsafeRecovery {
|
if opts.unsafeRecovery {
|
||||||
Verbosef("deleting index files\n")
|
Verbosef("deleting index files\n")
|
||||||
indexFiles := repo.Index().(*repository.MasterIndex).IDs()
|
indexFiles := repo.Index().(*index.MasterIndex).IDs()
|
||||||
err = DeleteFilesChecked(ctx, gopts, repo, indexFiles, restic.IndexFile)
|
err = DeleteFilesChecked(ctx, gopts, repo, indexFiles, restic.IndexFile)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.Fatalf("%s", err)
|
return errors.Fatalf("%s", err)
|
||||||
|
|
|
@ -3,6 +3,7 @@ package main
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
|
||||||
|
"github.com/restic/restic/internal/index"
|
||||||
"github.com/restic/restic/internal/pack"
|
"github.com/restic/restic/internal/pack"
|
||||||
"github.com/restic/restic/internal/repository"
|
"github.com/restic/restic/internal/repository"
|
||||||
"github.com/restic/restic/internal/restic"
|
"github.com/restic/restic/internal/restic"
|
||||||
|
@ -74,8 +75,8 @@ func rebuildIndex(ctx context.Context, opts RebuildIndexOptions, gopts GlobalOpt
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
Verbosef("loading indexes...\n")
|
Verbosef("loading indexes...\n")
|
||||||
mi := repository.NewMasterIndex()
|
mi := index.NewMasterIndex()
|
||||||
err := repository.ForAllIndexes(ctx, repo, func(id restic.ID, idx *repository.Index, oldFormat bool, err error) error {
|
err := index.ForAllIndexes(ctx, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
Warnf("removing invalid index %v: %v\n", id, err)
|
Warnf("removing invalid index %v: %v\n", id, err)
|
||||||
obsoleteIndexes = append(obsoleteIndexes, id)
|
obsoleteIndexes = append(obsoleteIndexes, id)
|
||||||
|
|
|
@ -23,6 +23,7 @@ import (
|
||||||
"github.com/restic/restic/internal/errors"
|
"github.com/restic/restic/internal/errors"
|
||||||
"github.com/restic/restic/internal/filter"
|
"github.com/restic/restic/internal/filter"
|
||||||
"github.com/restic/restic/internal/fs"
|
"github.com/restic/restic/internal/fs"
|
||||||
|
"github.com/restic/restic/internal/index"
|
||||||
"github.com/restic/restic/internal/repository"
|
"github.com/restic/restic/internal/repository"
|
||||||
"github.com/restic/restic/internal/restic"
|
"github.com/restic/restic/internal/restic"
|
||||||
rtest "github.com/restic/restic/internal/test"
|
rtest "github.com/restic/restic/internal/test"
|
||||||
|
@ -1477,11 +1478,11 @@ func TestRebuildIndex(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRebuildIndexAlwaysFull(t *testing.T) {
|
func TestRebuildIndexAlwaysFull(t *testing.T) {
|
||||||
indexFull := repository.IndexFull
|
indexFull := index.IndexFull
|
||||||
defer func() {
|
defer func() {
|
||||||
repository.IndexFull = indexFull
|
index.IndexFull = indexFull
|
||||||
}()
|
}()
|
||||||
repository.IndexFull = func(*repository.Index, bool) bool { return true }
|
index.IndexFull = func(*index.Index, bool) bool { return true }
|
||||||
testRebuildIndex(t, nil)
|
testRebuildIndex(t, nil)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -8,7 +8,7 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/restic/restic/internal/errors"
|
"github.com/restic/restic/internal/errors"
|
||||||
"github.com/restic/restic/internal/repository"
|
"github.com/restic/restic/internal/index"
|
||||||
"github.com/restic/restic/internal/restic"
|
"github.com/restic/restic/internal/restic"
|
||||||
"golang.org/x/sync/errgroup"
|
"golang.org/x/sync/errgroup"
|
||||||
)
|
)
|
||||||
|
@ -40,7 +40,7 @@ func TestBlobSaver(t *testing.T) {
|
||||||
|
|
||||||
wg, ctx := errgroup.WithContext(ctx)
|
wg, ctx := errgroup.WithContext(ctx)
|
||||||
saver := &saveFail{
|
saver := &saveFail{
|
||||||
idx: repository.NewMasterIndex(),
|
idx: index.NewMasterIndex(),
|
||||||
}
|
}
|
||||||
|
|
||||||
b := NewBlobSaver(ctx, wg, saver, uint(runtime.NumCPU()))
|
b := NewBlobSaver(ctx, wg, saver, uint(runtime.NumCPU()))
|
||||||
|
@ -86,7 +86,7 @@ func TestBlobSaverError(t *testing.T) {
|
||||||
|
|
||||||
wg, ctx := errgroup.WithContext(ctx)
|
wg, ctx := errgroup.WithContext(ctx)
|
||||||
saver := &saveFail{
|
saver := &saveFail{
|
||||||
idx: repository.NewMasterIndex(),
|
idx: index.NewMasterIndex(),
|
||||||
failAt: int32(test.failAt),
|
failAt: int32(test.failAt),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -18,6 +18,7 @@ import (
|
||||||
"github.com/restic/restic/internal/debug"
|
"github.com/restic/restic/internal/debug"
|
||||||
"github.com/restic/restic/internal/errors"
|
"github.com/restic/restic/internal/errors"
|
||||||
"github.com/restic/restic/internal/hashing"
|
"github.com/restic/restic/internal/hashing"
|
||||||
|
"github.com/restic/restic/internal/index"
|
||||||
"github.com/restic/restic/internal/pack"
|
"github.com/restic/restic/internal/pack"
|
||||||
"github.com/restic/restic/internal/repository"
|
"github.com/restic/restic/internal/repository"
|
||||||
"github.com/restic/restic/internal/restic"
|
"github.com/restic/restic/internal/restic"
|
||||||
|
@ -38,7 +39,7 @@ type Checker struct {
|
||||||
}
|
}
|
||||||
trackUnused bool
|
trackUnused bool
|
||||||
|
|
||||||
masterIndex *repository.MasterIndex
|
masterIndex *index.MasterIndex
|
||||||
snapshots restic.Lister
|
snapshots restic.Lister
|
||||||
|
|
||||||
repo restic.Repository
|
repo restic.Repository
|
||||||
|
@ -48,7 +49,7 @@ type Checker struct {
|
||||||
func New(repo restic.Repository, trackUnused bool) *Checker {
|
func New(repo restic.Repository, trackUnused bool) *Checker {
|
||||||
c := &Checker{
|
c := &Checker{
|
||||||
packs: make(map[restic.ID]int64),
|
packs: make(map[restic.ID]int64),
|
||||||
masterIndex: repository.NewMasterIndex(),
|
masterIndex: index.NewMasterIndex(),
|
||||||
repo: repo,
|
repo: repo,
|
||||||
trackUnused: trackUnused,
|
trackUnused: trackUnused,
|
||||||
}
|
}
|
||||||
|
@ -121,7 +122,7 @@ func (c *Checker) LoadIndex(ctx context.Context) (hints []error, errs []error) {
|
||||||
debug.Log("Start")
|
debug.Log("Start")
|
||||||
|
|
||||||
packToIndex := make(map[restic.ID]restic.IDSet)
|
packToIndex := make(map[restic.ID]restic.IDSet)
|
||||||
err := repository.ForAllIndexes(ctx, c.repo, func(id restic.ID, index *repository.Index, oldFormat bool, err error) error {
|
err := index.ForAllIndexes(ctx, c.repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error {
|
||||||
debug.Log("process index %v, err %v", id, err)
|
debug.Log("process index %v, err %v", id, err)
|
||||||
|
|
||||||
if oldFormat {
|
if oldFormat {
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
package repository
|
package index
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
|
@ -1,4 +1,4 @@
|
||||||
package repository
|
package index
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
|
@ -1,15 +1,19 @@
|
||||||
package repository_test
|
package index_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"path/filepath"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/restic/restic/internal/errors"
|
"github.com/restic/restic/internal/errors"
|
||||||
|
"github.com/restic/restic/internal/index"
|
||||||
"github.com/restic/restic/internal/repository"
|
"github.com/restic/restic/internal/repository"
|
||||||
"github.com/restic/restic/internal/restic"
|
"github.com/restic/restic/internal/restic"
|
||||||
rtest "github.com/restic/restic/internal/test"
|
rtest "github.com/restic/restic/internal/test"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var repoFixture = filepath.Join("..", "repository", "testdata", "test-repo.tar.gz")
|
||||||
|
|
||||||
func TestRepositoryForAllIndexes(t *testing.T) {
|
func TestRepositoryForAllIndexes(t *testing.T) {
|
||||||
repodir, cleanup := rtest.Env(t, repoFixture)
|
repodir, cleanup := rtest.Env(t, repoFixture)
|
||||||
defer cleanup()
|
defer cleanup()
|
||||||
|
@ -25,7 +29,7 @@ func TestRepositoryForAllIndexes(t *testing.T) {
|
||||||
// check that all expected indexes are loaded without errors
|
// check that all expected indexes are loaded without errors
|
||||||
indexIDs := restic.NewIDSet()
|
indexIDs := restic.NewIDSet()
|
||||||
var indexErr error
|
var indexErr error
|
||||||
rtest.OK(t, repository.ForAllIndexes(context.TODO(), repo, func(id restic.ID, index *repository.Index, oldFormat bool, err error) error {
|
rtest.OK(t, index.ForAllIndexes(context.TODO(), repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
indexErr = err
|
indexErr = err
|
||||||
}
|
}
|
||||||
|
@ -38,7 +42,7 @@ func TestRepositoryForAllIndexes(t *testing.T) {
|
||||||
// must failed with the returned error
|
// must failed with the returned error
|
||||||
iterErr := errors.New("error to pass upwards")
|
iterErr := errors.New("error to pass upwards")
|
||||||
|
|
||||||
err := repository.ForAllIndexes(context.TODO(), repo, func(id restic.ID, index *repository.Index, oldFormat bool, err error) error {
|
err := index.ForAllIndexes(context.TODO(), repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error {
|
||||||
return iterErr
|
return iterErr
|
||||||
})
|
})
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
package repository_test
|
package index_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
|
@ -7,7 +7,7 @@ import (
|
||||||
"sync"
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/restic/restic/internal/repository"
|
"github.com/restic/restic/internal/index"
|
||||||
"github.com/restic/restic/internal/restic"
|
"github.com/restic/restic/internal/restic"
|
||||||
rtest "github.com/restic/restic/internal/test"
|
rtest "github.com/restic/restic/internal/test"
|
||||||
)
|
)
|
||||||
|
@ -15,7 +15,7 @@ import (
|
||||||
func TestIndexSerialize(t *testing.T) {
|
func TestIndexSerialize(t *testing.T) {
|
||||||
tests := []restic.PackedBlob{}
|
tests := []restic.PackedBlob{}
|
||||||
|
|
||||||
idx := repository.NewIndex()
|
idx := index.NewIndex()
|
||||||
|
|
||||||
// create 50 packs with 20 blobs each
|
// create 50 packs with 20 blobs each
|
||||||
for i := 0; i < 50; i++ {
|
for i := 0; i < 50; i++ {
|
||||||
|
@ -51,7 +51,7 @@ func TestIndexSerialize(t *testing.T) {
|
||||||
rtest.OK(t, err)
|
rtest.OK(t, err)
|
||||||
|
|
||||||
idx2ID := restic.NewRandomID()
|
idx2ID := restic.NewRandomID()
|
||||||
idx2, oldFormat, err := repository.DecodeIndex(wr.Bytes(), idx2ID)
|
idx2, oldFormat, err := index.DecodeIndex(wr.Bytes(), idx2ID)
|
||||||
rtest.OK(t, err)
|
rtest.OK(t, err)
|
||||||
rtest.Assert(t, idx2 != nil,
|
rtest.Assert(t, idx2 != nil,
|
||||||
"nil returned for decoded index")
|
"nil returned for decoded index")
|
||||||
|
@ -121,7 +121,7 @@ func TestIndexSerialize(t *testing.T) {
|
||||||
rtest.OK(t, err)
|
rtest.OK(t, err)
|
||||||
rtest.Equals(t, restic.IDs{id}, ids)
|
rtest.Equals(t, restic.IDs{id}, ids)
|
||||||
|
|
||||||
idx3, oldFormat, err := repository.DecodeIndex(wr3.Bytes(), id)
|
idx3, oldFormat, err := index.DecodeIndex(wr3.Bytes(), id)
|
||||||
rtest.OK(t, err)
|
rtest.OK(t, err)
|
||||||
rtest.Assert(t, idx3 != nil,
|
rtest.Assert(t, idx3 != nil,
|
||||||
"nil returned for decoded index")
|
"nil returned for decoded index")
|
||||||
|
@ -143,7 +143,7 @@ func TestIndexSerialize(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestIndexSize(t *testing.T) {
|
func TestIndexSize(t *testing.T) {
|
||||||
idx := repository.NewIndex()
|
idx := index.NewIndex()
|
||||||
|
|
||||||
packs := 200
|
packs := 200
|
||||||
blobCount := 100
|
blobCount := 100
|
||||||
|
@ -309,7 +309,7 @@ func TestIndexUnserialize(t *testing.T) {
|
||||||
} {
|
} {
|
||||||
oldIdx := restic.IDs{restic.TestParseID("ed54ae36197f4745ebc4b54d10e0f623eaaaedd03013eb7ae90df881b7781452")}
|
oldIdx := restic.IDs{restic.TestParseID("ed54ae36197f4745ebc4b54d10e0f623eaaaedd03013eb7ae90df881b7781452")}
|
||||||
|
|
||||||
idx, oldFormat, err := repository.DecodeIndex(task.idxBytes, restic.NewRandomID())
|
idx, oldFormat, err := index.DecodeIndex(task.idxBytes, restic.NewRandomID())
|
||||||
rtest.OK(t, err)
|
rtest.OK(t, err)
|
||||||
rtest.Assert(t, !oldFormat, "new index format recognized as old format")
|
rtest.Assert(t, !oldFormat, "new index format recognized as old format")
|
||||||
|
|
||||||
|
@ -354,7 +354,7 @@ func TestIndexUnserialize(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func listPack(idx *repository.Index, id restic.ID) (pbs []restic.PackedBlob) {
|
func listPack(idx *index.Index, id restic.ID) (pbs []restic.PackedBlob) {
|
||||||
idx.Each(context.TODO(), func(pb restic.PackedBlob) {
|
idx.Each(context.TODO(), func(pb restic.PackedBlob) {
|
||||||
if pb.PackID.Equal(id) {
|
if pb.PackID.Equal(id) {
|
||||||
pbs = append(pbs, pb)
|
pbs = append(pbs, pb)
|
||||||
|
@ -386,7 +386,7 @@ func BenchmarkDecodeIndex(b *testing.B) {
|
||||||
b.ResetTimer()
|
b.ResetTimer()
|
||||||
|
|
||||||
for i := 0; i < b.N; i++ {
|
for i := 0; i < b.N; i++ {
|
||||||
_, _, err := repository.DecodeIndex(benchmarkIndexJSON, id)
|
_, _, err := index.DecodeIndex(benchmarkIndexJSON, id)
|
||||||
rtest.OK(b, err)
|
rtest.OK(b, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -399,14 +399,14 @@ func BenchmarkDecodeIndexParallel(b *testing.B) {
|
||||||
|
|
||||||
b.RunParallel(func(pb *testing.PB) {
|
b.RunParallel(func(pb *testing.PB) {
|
||||||
for pb.Next() {
|
for pb.Next() {
|
||||||
_, _, err := repository.DecodeIndex(benchmarkIndexJSON, id)
|
_, _, err := index.DecodeIndex(benchmarkIndexJSON, id)
|
||||||
rtest.OK(b, err)
|
rtest.OK(b, err)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestIndexUnserializeOld(t *testing.T) {
|
func TestIndexUnserializeOld(t *testing.T) {
|
||||||
idx, oldFormat, err := repository.DecodeIndex(docOldExample, restic.NewRandomID())
|
idx, oldFormat, err := index.DecodeIndex(docOldExample, restic.NewRandomID())
|
||||||
rtest.OK(t, err)
|
rtest.OK(t, err)
|
||||||
rtest.Assert(t, oldFormat, "old index format recognized as new format")
|
rtest.Assert(t, oldFormat, "old index format recognized as new format")
|
||||||
|
|
||||||
|
@ -427,7 +427,7 @@ func TestIndexUnserializeOld(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestIndexPacks(t *testing.T) {
|
func TestIndexPacks(t *testing.T) {
|
||||||
idx := repository.NewIndex()
|
idx := index.NewIndex()
|
||||||
packs := restic.NewIDSet()
|
packs := restic.NewIDSet()
|
||||||
|
|
||||||
for i := 0; i < 20; i++ {
|
for i := 0; i < 20; i++ {
|
||||||
|
@ -456,8 +456,8 @@ func NewRandomTestID(rng *rand.Rand) restic.ID {
|
||||||
return id
|
return id
|
||||||
}
|
}
|
||||||
|
|
||||||
func createRandomIndex(rng *rand.Rand, packfiles int) (idx *repository.Index, lookupBh restic.BlobHandle) {
|
func createRandomIndex(rng *rand.Rand, packfiles int) (idx *index.Index, lookupBh restic.BlobHandle) {
|
||||||
idx = repository.NewIndex()
|
idx = index.NewIndex()
|
||||||
|
|
||||||
// create index with given number of pack files
|
// create index with given number of pack files
|
||||||
for i := 0; i < packfiles; i++ {
|
for i := 0; i < packfiles; i++ {
|
||||||
|
@ -536,7 +536,7 @@ func BenchmarkIndexAllocParallel(b *testing.B) {
|
||||||
func TestIndexHas(t *testing.T) {
|
func TestIndexHas(t *testing.T) {
|
||||||
tests := []restic.PackedBlob{}
|
tests := []restic.PackedBlob{}
|
||||||
|
|
||||||
idx := repository.NewIndex()
|
idx := index.NewIndex()
|
||||||
|
|
||||||
// create 50 packs with 20 blobs each
|
// create 50 packs with 20 blobs each
|
||||||
for i := 0; i < 50; i++ {
|
for i := 0; i < 50; i++ {
|
||||||
|
@ -576,7 +576,7 @@ func TestIndexHas(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMixedEachByPack(t *testing.T) {
|
func TestMixedEachByPack(t *testing.T) {
|
||||||
idx := repository.NewIndex()
|
idx := index.NewIndex()
|
||||||
|
|
||||||
expected := make(map[restic.ID]int)
|
expected := make(map[restic.ID]int)
|
||||||
// create 50 packs with 2 blobs each
|
// create 50 packs with 2 blobs each
|
||||||
|
@ -615,7 +615,7 @@ func TestMixedEachByPack(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestEachByPackIgnoes(t *testing.T) {
|
func TestEachByPackIgnoes(t *testing.T) {
|
||||||
idx := repository.NewIndex()
|
idx := index.NewIndex()
|
||||||
|
|
||||||
ignores := restic.NewIDSet()
|
ignores := restic.NewIDSet()
|
||||||
expected := make(map[restic.ID]int)
|
expected := make(map[restic.ID]int)
|
|
@ -1,4 +1,4 @@
|
||||||
package repository
|
package index
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"hash/maphash"
|
"hash/maphash"
|
|
@ -1,4 +1,4 @@
|
||||||
package repository
|
package index
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"math/rand"
|
"math/rand"
|
|
@ -1,4 +1,4 @@
|
||||||
package repository
|
package index
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
|
@ -31,7 +31,7 @@ func NewMasterIndex() *MasterIndex {
|
||||||
return &MasterIndex{idx: idx, pendingBlobs: restic.NewBlobSet()}
|
return &MasterIndex{idx: idx, pendingBlobs: restic.NewBlobSet()}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (mi *MasterIndex) markCompressed() {
|
func (mi *MasterIndex) MarkCompressed() {
|
||||||
mi.compress = true
|
mi.compress = true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -65,7 +65,7 @@ func (mi *MasterIndex) LookupSize(bh restic.BlobHandle) (uint, bool) {
|
||||||
// Before doing so it checks if this blob is already known.
|
// Before doing so it checks if this blob is already known.
|
||||||
// Returns true if adding was successful and false if the blob
|
// Returns true if adding was successful and false if the blob
|
||||||
// was already known
|
// was already known
|
||||||
func (mi *MasterIndex) addPending(bh restic.BlobHandle) bool {
|
func (mi *MasterIndex) AddPending(bh restic.BlobHandle) bool {
|
||||||
|
|
||||||
mi.idxMutex.Lock()
|
mi.idxMutex.Lock()
|
||||||
defer mi.idxMutex.Unlock()
|
defer mi.idxMutex.Unlock()
|
|
@ -1,4 +1,4 @@
|
||||||
package repository_test
|
package index_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
@ -9,6 +9,7 @@ import (
|
||||||
|
|
||||||
"github.com/restic/restic/internal/checker"
|
"github.com/restic/restic/internal/checker"
|
||||||
"github.com/restic/restic/internal/crypto"
|
"github.com/restic/restic/internal/crypto"
|
||||||
|
"github.com/restic/restic/internal/index"
|
||||||
"github.com/restic/restic/internal/repository"
|
"github.com/restic/restic/internal/repository"
|
||||||
"github.com/restic/restic/internal/restic"
|
"github.com/restic/restic/internal/restic"
|
||||||
rtest "github.com/restic/restic/internal/test"
|
rtest "github.com/restic/restic/internal/test"
|
||||||
|
@ -57,15 +58,15 @@ func TestMasterIndex(t *testing.T) {
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
idx1 := repository.NewIndex()
|
idx1 := index.NewIndex()
|
||||||
idx1.StorePack(blob1.PackID, []restic.Blob{blob1.Blob})
|
idx1.StorePack(blob1.PackID, []restic.Blob{blob1.Blob})
|
||||||
idx1.StorePack(blob12a.PackID, []restic.Blob{blob12a.Blob})
|
idx1.StorePack(blob12a.PackID, []restic.Blob{blob12a.Blob})
|
||||||
|
|
||||||
idx2 := repository.NewIndex()
|
idx2 := index.NewIndex()
|
||||||
idx2.StorePack(blob2.PackID, []restic.Blob{blob2.Blob})
|
idx2.StorePack(blob2.PackID, []restic.Blob{blob2.Blob})
|
||||||
idx2.StorePack(blob12b.PackID, []restic.Blob{blob12b.Blob})
|
idx2.StorePack(blob12b.PackID, []restic.Blob{blob12b.Blob})
|
||||||
|
|
||||||
mIdx := repository.NewMasterIndex()
|
mIdx := index.NewMasterIndex()
|
||||||
mIdx.Insert(idx1)
|
mIdx.Insert(idx1)
|
||||||
mIdx.Insert(idx2)
|
mIdx.Insert(idx2)
|
||||||
|
|
||||||
|
@ -148,18 +149,18 @@ func TestMasterMergeFinalIndexes(t *testing.T) {
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
idx1 := repository.NewIndex()
|
idx1 := index.NewIndex()
|
||||||
idx1.StorePack(blob1.PackID, []restic.Blob{blob1.Blob})
|
idx1.StorePack(blob1.PackID, []restic.Blob{blob1.Blob})
|
||||||
|
|
||||||
idx2 := repository.NewIndex()
|
idx2 := index.NewIndex()
|
||||||
idx2.StorePack(blob2.PackID, []restic.Blob{blob2.Blob})
|
idx2.StorePack(blob2.PackID, []restic.Blob{blob2.Blob})
|
||||||
|
|
||||||
mIdx := repository.NewMasterIndex()
|
mIdx := index.NewMasterIndex()
|
||||||
mIdx.Insert(idx1)
|
mIdx.Insert(idx1)
|
||||||
mIdx.Insert(idx2)
|
mIdx.Insert(idx2)
|
||||||
|
|
||||||
finalIndexes, idxCount := repository.TestMergeIndex(t, mIdx)
|
finalIndexes, idxCount := index.TestMergeIndex(t, mIdx)
|
||||||
rtest.Equals(t, []*repository.Index{idx1, idx2}, finalIndexes)
|
rtest.Equals(t, []*index.Index{idx1, idx2}, finalIndexes)
|
||||||
rtest.Equals(t, 1, idxCount)
|
rtest.Equals(t, 1, idxCount)
|
||||||
|
|
||||||
blobCount := 0
|
blobCount := 0
|
||||||
|
@ -178,13 +179,13 @@ func TestMasterMergeFinalIndexes(t *testing.T) {
|
||||||
rtest.Assert(t, blobs == nil, "Expected no blobs when fetching with a random id")
|
rtest.Assert(t, blobs == nil, "Expected no blobs when fetching with a random id")
|
||||||
|
|
||||||
// merge another index containing identical blobs
|
// merge another index containing identical blobs
|
||||||
idx3 := repository.NewIndex()
|
idx3 := index.NewIndex()
|
||||||
idx3.StorePack(blob1.PackID, []restic.Blob{blob1.Blob})
|
idx3.StorePack(blob1.PackID, []restic.Blob{blob1.Blob})
|
||||||
idx3.StorePack(blob2.PackID, []restic.Blob{blob2.Blob})
|
idx3.StorePack(blob2.PackID, []restic.Blob{blob2.Blob})
|
||||||
|
|
||||||
mIdx.Insert(idx3)
|
mIdx.Insert(idx3)
|
||||||
finalIndexes, idxCount = repository.TestMergeIndex(t, mIdx)
|
finalIndexes, idxCount = index.TestMergeIndex(t, mIdx)
|
||||||
rtest.Equals(t, []*repository.Index{idx3}, finalIndexes)
|
rtest.Equals(t, []*index.Index{idx3}, finalIndexes)
|
||||||
rtest.Equals(t, 1, idxCount)
|
rtest.Equals(t, 1, idxCount)
|
||||||
|
|
||||||
// Index should have same entries as before!
|
// Index should have same entries as before!
|
||||||
|
@ -201,8 +202,8 @@ func TestMasterMergeFinalIndexes(t *testing.T) {
|
||||||
rtest.Equals(t, 2, blobCount)
|
rtest.Equals(t, 2, blobCount)
|
||||||
}
|
}
|
||||||
|
|
||||||
func createRandomMasterIndex(t testing.TB, rng *rand.Rand, num, size int) (*repository.MasterIndex, restic.BlobHandle) {
|
func createRandomMasterIndex(t testing.TB, rng *rand.Rand, num, size int) (*index.MasterIndex, restic.BlobHandle) {
|
||||||
mIdx := repository.NewMasterIndex()
|
mIdx := index.NewMasterIndex()
|
||||||
for i := 0; i < num-1; i++ {
|
for i := 0; i < num-1; i++ {
|
||||||
idx, _ := createRandomIndex(rng, size)
|
idx, _ := createRandomIndex(rng, size)
|
||||||
mIdx.Insert(idx)
|
mIdx.Insert(idx)
|
||||||
|
@ -210,7 +211,7 @@ func createRandomMasterIndex(t testing.TB, rng *rand.Rand, num, size int) (*repo
|
||||||
idx1, lookupBh := createRandomIndex(rng, size)
|
idx1, lookupBh := createRandomIndex(rng, size)
|
||||||
mIdx.Insert(idx1)
|
mIdx.Insert(idx1)
|
||||||
|
|
||||||
repository.TestMergeIndex(t, mIdx)
|
index.TestMergeIndex(t, mIdx)
|
||||||
|
|
||||||
return mIdx, lookupBh
|
return mIdx, lookupBh
|
||||||
}
|
}
|
18
internal/index/testing.go
Normal file
18
internal/index/testing.go
Normal file
|
@ -0,0 +1,18 @@
|
||||||
|
package index
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/restic/restic/internal/restic"
|
||||||
|
"github.com/restic/restic/internal/test"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMergeIndex(t testing.TB, mi *MasterIndex) ([]*Index, int) {
|
||||||
|
finalIndexes := mi.finalizeNotFinalIndexes()
|
||||||
|
for _, idx := range finalIndexes {
|
||||||
|
test.OK(t, idx.SetID(restic.NewRandomID()))
|
||||||
|
}
|
||||||
|
|
||||||
|
test.OK(t, mi.MergeFinalIndexes())
|
||||||
|
return finalIndexes, len(mi.idx)
|
||||||
|
}
|
|
@ -6,6 +6,7 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/restic/restic/internal/index"
|
||||||
"github.com/restic/restic/internal/repository"
|
"github.com/restic/restic/internal/repository"
|
||||||
"github.com/restic/restic/internal/restic"
|
"github.com/restic/restic/internal/restic"
|
||||||
rtest "github.com/restic/restic/internal/test"
|
rtest "github.com/restic/restic/internal/test"
|
||||||
|
@ -170,7 +171,7 @@ func flush(t *testing.T, repo restic.Repository) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func rebuildIndex(t *testing.T, repo restic.Repository) {
|
func rebuildIndex(t *testing.T, repo restic.Repository) {
|
||||||
err := repo.SetIndex(repository.NewMasterIndex())
|
err := repo.SetIndex(index.NewMasterIndex())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
@ -207,7 +208,7 @@ func rebuildIndex(t *testing.T, repo restic.Repository) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func reloadIndex(t *testing.T, repo restic.Repository) {
|
func reloadIndex(t *testing.T, repo restic.Repository) {
|
||||||
err := repo.SetIndex(repository.NewMasterIndex())
|
err := repo.SetIndex(index.NewMasterIndex())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -19,6 +19,7 @@ import (
|
||||||
"github.com/restic/restic/internal/crypto"
|
"github.com/restic/restic/internal/crypto"
|
||||||
"github.com/restic/restic/internal/debug"
|
"github.com/restic/restic/internal/debug"
|
||||||
"github.com/restic/restic/internal/errors"
|
"github.com/restic/restic/internal/errors"
|
||||||
|
"github.com/restic/restic/internal/index"
|
||||||
"github.com/restic/restic/internal/pack"
|
"github.com/restic/restic/internal/pack"
|
||||||
"github.com/restic/restic/internal/restic"
|
"github.com/restic/restic/internal/restic"
|
||||||
"github.com/restic/restic/internal/ui/progress"
|
"github.com/restic/restic/internal/ui/progress"
|
||||||
|
@ -38,7 +39,7 @@ type Repository struct {
|
||||||
cfg restic.Config
|
cfg restic.Config
|
||||||
key *crypto.Key
|
key *crypto.Key
|
||||||
keyName string
|
keyName string
|
||||||
idx *MasterIndex
|
idx *index.MasterIndex
|
||||||
Cache *cache.Cache
|
Cache *cache.Cache
|
||||||
|
|
||||||
opts Options
|
opts Options
|
||||||
|
@ -118,7 +119,7 @@ func New(be restic.Backend, opts Options) (*Repository, error) {
|
||||||
repo := &Repository{
|
repo := &Repository{
|
||||||
be: be,
|
be: be,
|
||||||
opts: opts,
|
opts: opts,
|
||||||
idx: NewMasterIndex(),
|
idx: index.NewMasterIndex(),
|
||||||
}
|
}
|
||||||
|
|
||||||
return repo, nil
|
return repo, nil
|
||||||
|
@ -134,7 +135,7 @@ func (r *Repository) DisableAutoIndexUpdate() {
|
||||||
func (r *Repository) setConfig(cfg restic.Config) {
|
func (r *Repository) setConfig(cfg restic.Config) {
|
||||||
r.cfg = cfg
|
r.cfg = cfg
|
||||||
if r.cfg.Version >= 2 {
|
if r.cfg.Version >= 2 {
|
||||||
r.idx.markCompressed()
|
r.idx.MarkCompressed()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -563,7 +564,7 @@ func (r *Repository) Index() restic.MasterIndex {
|
||||||
|
|
||||||
// SetIndex instructs the repository to use the given index.
|
// SetIndex instructs the repository to use the given index.
|
||||||
func (r *Repository) SetIndex(i restic.MasterIndex) error {
|
func (r *Repository) SetIndex(i restic.MasterIndex) error {
|
||||||
r.idx = i.(*MasterIndex)
|
r.idx = i.(*index.MasterIndex)
|
||||||
return r.prepareCache()
|
return r.prepareCache()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -572,7 +573,7 @@ func (r *Repository) SetIndex(i restic.MasterIndex) error {
|
||||||
func (r *Repository) LoadIndex(ctx context.Context) error {
|
func (r *Repository) LoadIndex(ctx context.Context) error {
|
||||||
debug.Log("Loading index")
|
debug.Log("Loading index")
|
||||||
|
|
||||||
err := ForAllIndexes(ctx, r, func(id restic.ID, idx *Index, oldFormat bool, err error) error {
|
err := index.ForAllIndexes(ctx, r, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -829,7 +830,7 @@ func (r *Repository) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte
|
||||||
}
|
}
|
||||||
|
|
||||||
// first try to add to pending blobs; if not successful, this blob is already known
|
// first try to add to pending blobs; if not successful, this blob is already known
|
||||||
known = !r.idx.addPending(restic.BlobHandle{ID: newID, Type: t})
|
known = !r.idx.AddPending(restic.BlobHandle{ID: newID, Type: t})
|
||||||
|
|
||||||
// only save when needed or explicitly told
|
// only save when needed or explicitly told
|
||||||
if !known || storeDuplicate {
|
if !known || storeDuplicate {
|
||||||
|
|
|
@ -18,6 +18,7 @@ import (
|
||||||
"github.com/klauspost/compress/zstd"
|
"github.com/klauspost/compress/zstd"
|
||||||
"github.com/restic/restic/internal/backend/local"
|
"github.com/restic/restic/internal/backend/local"
|
||||||
"github.com/restic/restic/internal/crypto"
|
"github.com/restic/restic/internal/crypto"
|
||||||
|
"github.com/restic/restic/internal/index"
|
||||||
"github.com/restic/restic/internal/repository"
|
"github.com/restic/restic/internal/repository"
|
||||||
"github.com/restic/restic/internal/restic"
|
"github.com/restic/restic/internal/restic"
|
||||||
"github.com/restic/restic/internal/test"
|
"github.com/restic/restic/internal/test"
|
||||||
|
@ -267,13 +268,13 @@ func TestRepositoryLoadIndex(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// loadIndex loads the index id from backend and returns it.
|
// loadIndex loads the index id from backend and returns it.
|
||||||
func loadIndex(ctx context.Context, repo restic.Repository, id restic.ID) (*repository.Index, error) {
|
func loadIndex(ctx context.Context, repo restic.Repository, id restic.ID) (*index.Index, error) {
|
||||||
buf, err := repo.LoadUnpacked(ctx, restic.IndexFile, id, nil)
|
buf, err := repo.LoadUnpacked(ctx, restic.IndexFile, id, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
idx, oldFormat, err := repository.DecodeIndex(buf, id)
|
idx, oldFormat, err := index.DecodeIndex(buf, id)
|
||||||
if oldFormat {
|
if oldFormat {
|
||||||
fmt.Fprintf(os.Stderr, "index %v has old format\n", id.Str())
|
fmt.Fprintf(os.Stderr, "index %v has old format\n", id.Str())
|
||||||
}
|
}
|
||||||
|
@ -345,7 +346,7 @@ func benchmarkLoadIndex(b *testing.B, version uint) {
|
||||||
repo, cleanup := repository.TestRepositoryWithVersion(b, version)
|
repo, cleanup := repository.TestRepositoryWithVersion(b, version)
|
||||||
defer cleanup()
|
defer cleanup()
|
||||||
|
|
||||||
idx := repository.NewIndex()
|
idx := index.NewIndex()
|
||||||
|
|
||||||
for i := 0; i < 5000; i++ {
|
for i := 0; i < 5000; i++ {
|
||||||
idx.StorePack(restic.NewRandomID(), []restic.Blob{
|
idx.StorePack(restic.NewRandomID(), []restic.Blob{
|
||||||
|
@ -357,7 +358,7 @@ func benchmarkLoadIndex(b *testing.B, version uint) {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
id, err := repository.SaveIndex(context.TODO(), repo, idx)
|
id, err := index.SaveIndex(context.TODO(), repo, idx)
|
||||||
rtest.OK(b, err)
|
rtest.OK(b, err)
|
||||||
|
|
||||||
b.Logf("index saved as %v", id.Str())
|
b.Logf("index saved as %v", id.Str())
|
||||||
|
@ -400,7 +401,7 @@ func testRepositoryIncrementalIndex(t *testing.T, version uint) {
|
||||||
|
|
||||||
repo := r.(*repository.Repository)
|
repo := r.(*repository.Repository)
|
||||||
|
|
||||||
repository.IndexFull = func(*repository.Index, bool) bool { return true }
|
index.IndexFull = func(*index.Index, bool) bool { return true }
|
||||||
|
|
||||||
// add a few rounds of packs
|
// add a few rounds of packs
|
||||||
for j := 0; j < 5; j++ {
|
for j := 0; j < 5; j++ {
|
||||||
|
|
|
@ -138,13 +138,3 @@ func BenchmarkAllVersions(b *testing.B, bench VersionedBenchmark) {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMergeIndex(t testing.TB, mi *MasterIndex) ([]*Index, int) {
|
|
||||||
finalIndexes := mi.finalizeNotFinalIndexes()
|
|
||||||
for _, idx := range finalIndexes {
|
|
||||||
test.OK(t, idx.SetID(restic.NewRandomID()))
|
|
||||||
}
|
|
||||||
|
|
||||||
test.OK(t, mi.MergeFinalIndexes())
|
|
||||||
return finalIndexes, len(mi.idx)
|
|
||||||
}
|
|
||||||
|
|
Loading…
Reference in a new issue