From 6aed9f268ba2899e68ca580a06817d6e00b0897a Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 25 Oct 2015 17:24:52 +0100 Subject: [PATCH] Add command `rebuild-index` --- cmd/restic/cmd_check.go | 8 +++ cmd/restic/cmd_rebuild_index.go | 109 ++++++++++++++++++++++++++++++++ cmd/restic/integration_test.go | 36 +++++++++++ 3 files changed, 153 insertions(+) create mode 100644 cmd/restic/cmd_rebuild_index.go diff --git a/cmd/restic/cmd_check.go b/cmd/restic/cmd_check.go index a7439be4e..919568618 100644 --- a/cmd/restic/cmd_check.go +++ b/cmd/restic/cmd_check.go @@ -52,8 +52,16 @@ func (cmd CmdCheck) Execute(args []string) error { cmd.global.Verbosef("Load indexes\n") hints, errs := chkr.LoadIndex() + dupFound := false for _, hint := range hints { cmd.global.Printf("%v\n", hint) + if _, ok := hint.(checker.ErrDuplicatePacks); ok { + dupFound = true + } + } + + if dupFound { + cmd.global.Printf("\nrun `restic rebuild-index' to correct this\n") } if len(errs) > 0 { diff --git a/cmd/restic/cmd_rebuild_index.go b/cmd/restic/cmd_rebuild_index.go new file mode 100644 index 000000000..aceceecc2 --- /dev/null +++ b/cmd/restic/cmd_rebuild_index.go @@ -0,0 +1,109 @@ +package main + +import ( + "github.com/restic/restic/backend" + "github.com/restic/restic/debug" + "github.com/restic/restic/repository" +) + +type CmdRebuildIndex struct { + global *GlobalOptions + + repo *repository.Repository +} + +func init() { + _, err := parser.AddCommand("rebuild-index", + "rebuild the index", + "The rebuild-index command builds a new index", + &CmdRebuildIndex{global: &globalOpts}) + if err != nil { + panic(err) + } +} + +func (cmd CmdRebuildIndex) RebuildIndex() error { + debug.Log("RebuildIndex.RebuildIndex", "start") + + done := make(chan struct{}) + defer close(done) + + indexIDs := backend.NewIDSet() + for id := range cmd.repo.List(backend.Index, done) { + indexIDs.Insert(id) + } + + debug.Log("RebuildIndex.RebuildIndex", "found %v indexes", len(indexIDs)) + + var combinedIndex *repository.Index + + for indexID := range indexIDs { + debug.Log("RebuildIndex.RebuildIndex", "load index %v", indexID.Str()) + idx, err := repository.LoadIndex(cmd.repo, indexID.String()) + if err != nil { + return err + } + + debug.Log("RebuildIndex.RebuildIndex", "adding blobs from index %v", indexID.Str()) + + if combinedIndex == nil { + combinedIndex = repository.NewIndex() + } + + for packedBlob := range idx.Each(done) { + combinedIndex.Store(packedBlob.Type, packedBlob.ID, packedBlob.PackID, packedBlob.Offset, packedBlob.Length) + } + + combinedIndex.AddToSupersedes(indexID) + + if repository.IndexFull(combinedIndex) { + debug.Log("RebuildIndex.RebuildIndex", "saving full index") + + id, err := repository.SaveIndex(cmd.repo, combinedIndex) + if err != nil { + debug.Log("RebuildIndex.RebuildIndex", "error saving index: %v", err) + return err + } + + debug.Log("RebuildIndex.RebuildIndex", "index saved as %v", id.Str()) + combinedIndex = nil + } + } + + id, err := repository.SaveIndex(cmd.repo, combinedIndex) + if err != nil { + debug.Log("RebuildIndex.RebuildIndex", "error saving index: %v", err) + return err + } + + debug.Log("RebuildIndex.RebuildIndex", "last index saved as %v", id.Str()) + + for id := range indexIDs { + debug.Log("RebuildIndex.RebuildIndex", "remove index %v", id.Str()) + + err = cmd.repo.Backend().Remove(backend.Index, id.String()) + if err != nil { + debug.Log("RebuildIndex.RebuildIndex", "error removing index %v: %v", id.Str(), err) + return err + } + } + + debug.Log("RebuildIndex.RebuildIndex", "done") + return nil +} + +func (cmd CmdRebuildIndex) Execute(args []string) error { + repo, err := cmd.global.OpenRepository() + if err != nil { + return err + } + cmd.repo = repo + + lock, err := lockRepoExclusive(repo) + defer unlockRepo(lock) + if err != nil { + return err + } + + return cmd.RebuildIndex() +} diff --git a/cmd/restic/integration_test.go b/cmd/restic/integration_test.go index 360adca0f..24f8e52db 100644 --- a/cmd/restic/integration_test.go +++ b/cmd/restic/integration_test.go @@ -90,6 +90,19 @@ func cmdCheck(t testing.TB, global GlobalOptions) { OK(t, cmd.Execute(nil)) } +func cmdCheckOutput(t testing.TB, global GlobalOptions) string { + buf := bytes.NewBuffer(nil) + global.stdout = buf + cmd := &CmdCheck{global: &global, ReadData: true} + OK(t, cmd.Execute(nil)) + return string(buf.Bytes()) +} + +func cmdRebuildIndex(t testing.TB, global GlobalOptions) { + cmd := &CmdRebuildIndex{global: &global} + OK(t, cmd.Execute(nil)) +} + func cmdLs(t testing.TB, global GlobalOptions, snapshotID string) []string { var buf bytes.Buffer global.stdout = &buf @@ -646,3 +659,26 @@ func TestFind(t *testing.T) { Assert(t, len(results) < 2, "less than two file found in repo (%v)", datafile) }) } + +func TestRebuildIndex(t *testing.T) { + withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) { + datafile := filepath.Join("..", "..", "checker", "testdata", "duplicate-packs-in-index-test-repo.tar.gz") + SetupTarTestFixture(t, env.base, datafile) + + out := cmdCheckOutput(t, global) + if !strings.Contains(out, "contained in several indexes") { + t.Fatalf("did not find checker hint for packs in several indexes") + } + + if !strings.Contains(out, "restic rebuild-index") { + t.Fatalf("did not find hint for rebuild-index comman") + } + + cmdRebuildIndex(t, global) + + out = cmdCheckOutput(t, global) + if len(out) != 0 { + t.Fatalf("expected no output from the checker, got: %v", out) + } + }) +}