Add command rebuild-index

cmd/restic/cmd_check.go

@@ -52,8 +52,16 @@ func (cmd CmdCheck) Execute(args []string) error {
 	cmd.global.Verbosef("Load indexes\n")
 	hints, errs := chkr.LoadIndex()
 
+	dupFound := false
 	for _, hint := range hints {
 		cmd.global.Printf("%v\n", hint)
+		if _, ok := hint.(checker.ErrDuplicatePacks); ok {
+			dupFound = true
+		}
+	}
+
+	if dupFound {
+		cmd.global.Printf("\nrun `restic rebuild-index' to correct this\n")
 	}
 
 	if len(errs) > 0 {

cmd/restic/cmd_rebuild_index.go

@@ -0,0 +1,109 @@
+package main
+
+import (
+	"github.com/restic/restic/backend"
+	"github.com/restic/restic/debug"
+	"github.com/restic/restic/repository"
+)
+
+type CmdRebuildIndex struct {
+	global *GlobalOptions
+
+	repo *repository.Repository
+}
+
+func init() {
+	_, err := parser.AddCommand("rebuild-index",
+		"rebuild the index",
+		"The rebuild-index command builds a new index",
+		&CmdRebuildIndex{global: &globalOpts})
+	if err != nil {
+		panic(err)
+	}
+}
+
+func (cmd CmdRebuildIndex) RebuildIndex() error {
+	debug.Log("RebuildIndex.RebuildIndex", "start")
+
+	done := make(chan struct{})
+	defer close(done)
+
+	indexIDs := backend.NewIDSet()
+	for id := range cmd.repo.List(backend.Index, done) {
+		indexIDs.Insert(id)
+	}
+
+	debug.Log("RebuildIndex.RebuildIndex", "found %v indexes", len(indexIDs))
+
+	var combinedIndex *repository.Index
+
+	for indexID := range indexIDs {
+		debug.Log("RebuildIndex.RebuildIndex", "load index %v", indexID.Str())
+		idx, err := repository.LoadIndex(cmd.repo, indexID.String())
+		if err != nil {
+			return err
+		}
+
+		debug.Log("RebuildIndex.RebuildIndex", "adding blobs from index %v", indexID.Str())
+
+		if combinedIndex == nil {
+			combinedIndex = repository.NewIndex()
+		}
+
+		for packedBlob := range idx.Each(done) {
+			combinedIndex.Store(packedBlob.Type, packedBlob.ID, packedBlob.PackID, packedBlob.Offset, packedBlob.Length)
+		}
+
+		combinedIndex.AddToSupersedes(indexID)
+
+		if repository.IndexFull(combinedIndex) {
+			debug.Log("RebuildIndex.RebuildIndex", "saving full index")
+
+			id, err := repository.SaveIndex(cmd.repo, combinedIndex)
+			if err != nil {
+				debug.Log("RebuildIndex.RebuildIndex", "error saving index: %v", err)
+				return err
+			}
+
+			debug.Log("RebuildIndex.RebuildIndex", "index saved as %v", id.Str())
+			combinedIndex = nil
+		}
+	}
+
+	id, err := repository.SaveIndex(cmd.repo, combinedIndex)
+	if err != nil {
+		debug.Log("RebuildIndex.RebuildIndex", "error saving index: %v", err)
+		return err
+	}
+
+	debug.Log("RebuildIndex.RebuildIndex", "last index saved as %v", id.Str())
+
+	for id := range indexIDs {
+		debug.Log("RebuildIndex.RebuildIndex", "remove index %v", id.Str())
+
+		err = cmd.repo.Backend().Remove(backend.Index, id.String())
+		if err != nil {
+			debug.Log("RebuildIndex.RebuildIndex", "error removing index %v: %v", id.Str(), err)
+			return err
+		}
+	}
+
+	debug.Log("RebuildIndex.RebuildIndex", "done")
+	return nil
+}
+
+func (cmd CmdRebuildIndex) Execute(args []string) error {
+	repo, err := cmd.global.OpenRepository()
+	if err != nil {
+		return err
+	}
+	cmd.repo = repo
+
+	lock, err := lockRepoExclusive(repo)
+	defer unlockRepo(lock)
+	if err != nil {
+		return err
+	}
+
+	return cmd.RebuildIndex()
+}

cmd/restic/integration_test.go

@@ -90,6 +90,19 @@ func cmdCheck(t testing.TB, global GlobalOptions) {
 	OK(t, cmd.Execute(nil))
 }
 
+func cmdCheckOutput(t testing.TB, global GlobalOptions) string {
+	buf := bytes.NewBuffer(nil)
+	global.stdout = buf
+	cmd := &CmdCheck{global: &global, ReadData: true}
+	OK(t, cmd.Execute(nil))
+	return string(buf.Bytes())
+}
+
+func cmdRebuildIndex(t testing.TB, global GlobalOptions) {
+	cmd := &CmdRebuildIndex{global: &global}
+	OK(t, cmd.Execute(nil))
+}
+
 func cmdLs(t testing.TB, global GlobalOptions, snapshotID string) []string {
 	var buf bytes.Buffer
 	global.stdout = &buf
@@ -646,3 +659,26 @@ func TestFind(t *testing.T) {
 		Assert(t, len(results) < 2, "less than two file found in repo (%v)", datafile)
 	})
 }
+
+func TestRebuildIndex(t *testing.T) {
+	withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) {
+		datafile := filepath.Join("..", "..", "checker", "testdata", "duplicate-packs-in-index-test-repo.tar.gz")
+		SetupTarTestFixture(t, env.base, datafile)
+
+		out := cmdCheckOutput(t, global)
+		if !strings.Contains(out, "contained in several indexes") {
+			t.Fatalf("did not find checker hint for packs in several indexes")
+		}
+
+		if !strings.Contains(out, "restic rebuild-index") {
+			t.Fatalf("did not find hint for rebuild-index comman")
+		}
+
+		cmdRebuildIndex(t, global)
+
+		out = cmdCheckOutput(t, global)
+		if len(out) != 0 {
+			t.Fatalf("expected no output from the checker, got: %v", out)
+		}
+	})
+}