Merge pull request #325 from restic/fix-rebuild-index

rebuild-index: Remember already stored blobs

cmd/restic/cmd_rebuild_index.go

@@ -61,6 +61,12 @@ func (cmd CmdRebuildIndex) RebuildIndex() error {
 	combinedIndex := repository.NewIndex()
 	packsDone := backend.NewIDSet()
 
+	type Blob struct {
+		id  backend.ID
+		tpe pack.BlobType
+	}
+	blobsDone := make(map[Blob]struct{})
+
 	i := 0
 	for indexID := range indexIDs {
 		cmd.global.Printf("  loading index %v\n", i)
@@ -74,8 +80,17 @@ func (cmd CmdRebuildIndex) RebuildIndex() error {
 		debug.Log("RebuildIndex.RebuildIndex", "adding blobs from index %v", indexID.Str())
 
 		for packedBlob := range idx.Each(done) {
-			combinedIndex.Store(packedBlob.Type, packedBlob.ID, packedBlob.PackID, packedBlob.Offset, packedBlob.Length)
 			packsDone.Insert(packedBlob.PackID)
+			b := Blob{
+				id:  packedBlob.ID,
+				tpe: packedBlob.Type,
+			}
+			if _, ok := blobsDone[b]; ok {
+				continue
+			}
+
+			blobsDone[b] = struct{}{}
+			combinedIndex.Store(packedBlob.Type, packedBlob.ID, packedBlob.PackID, packedBlob.Offset, packedBlob.Length)
 		}
 
 		combinedIndex.AddToSupersedes(indexID)

cmd/restic/integration_test.go

@@ -18,6 +18,7 @@ import (
 	"github.com/restic/restic/backend"
 	"github.com/restic/restic/debug"
 	"github.com/restic/restic/filter"
+	"github.com/restic/restic/repository"
 	. "github.com/restic/restic/test"
 )
 
@@ -99,6 +100,7 @@ func cmdCheckOutput(t testing.TB, global GlobalOptions) string {
 }
 
 func cmdRebuildIndex(t testing.TB, global GlobalOptions) {
+	global.stdout = ioutil.Discard
 	cmd := &CmdRebuildIndex{global: &global}
 	OK(t, cmd.Execute(nil))
 }
@@ -682,3 +684,8 @@ func TestRebuildIndex(t *testing.T) {
 		}
 	})
 }
+
+func TestRebuildIndexAlwaysFull(t *testing.T) {
+	repository.IndexFull = func(*repository.Index) bool { return true }
+	TestRebuildIndex(t)
+}