index: deduplicate index loading of check and repository

internal/checker/checker.go

@@ -111,33 +111,10 @@ func computePackTypes(ctx context.Context, idx restic.ListBlobser) (map[restic.I
 func (c *Checker) LoadIndex(ctx context.Context, p *progress.Counter) (hints []error, errs []error) {
 	debug.Log("Start")
 
-	indexList, err := restic.MemorizeList(ctx, c.repo, restic.IndexFile)
-	if err != nil {
-		// abort if an error occurs while listing the indexes
-		return hints, append(errs, err)
-	}
-
-	if p != nil {
-		var numIndexFiles uint64
-		err := indexList.List(ctx, restic.IndexFile, func(_ restic.ID, _ int64) error {
-			numIndexFiles++
-			return nil
-		})
-		if err != nil {
-			return hints, append(errs, err)
-		}
-		p.SetMax(numIndexFiles)
-		defer p.Done()
-	}
-
 	packToIndex := make(map[restic.ID]restic.IDSet)
-	err = index.ForAllIndexes(ctx, indexList, c.repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error {
+	err := c.masterIndex.Load(ctx, c.repo, p, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
 		debug.Log("process index %v, err %v", id, err)
 
-		if p != nil {
-			p.Add(1)
-		}
-
 		if oldFormat {
 			debug.Log("index %v has old format", id)
 			hints = append(hints, &ErrOldIndexFormat{id})
@@ -150,11 +127,9 @@ func (c *Checker) LoadIndex(ctx context.Context, p *progress.Counter) (hints []e
 			return nil
 		}
 
-		c.masterIndex.Insert(index)
-
 		debug.Log("process blobs")
 		cnt := 0
-		err = index.Each(ctx, func(blob restic.PackedBlob) {
+		err = idx.Each(ctx, func(blob restic.PackedBlob) {
 			cnt++
 
 			if _, ok := packToIndex[blob.PackID]; !ok {
@@ -167,13 +142,7 @@ func (c *Checker) LoadIndex(ctx context.Context, p *progress.Counter) (hints []e
 		return err
 	})
 	if err != nil {
-		errs = append(errs, err)
+		// failed to load the index
-	}
-
-	// Merge index before computing pack sizes, as this needs removed duplicates
-	err = c.masterIndex.MergeFinalIndexes()
-	if err != nil {
-		// abort if an error occurs merging the indexes
 		return hints, append(errs, err)
 	}
 

internal/index/master_index.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/ui/progress"
 	"golang.org/x/sync/errgroup"
 )
 
@@ -267,6 +268,50 @@ func (mi *MasterIndex) MergeFinalIndexes() error {
 	return nil
 }
 
+func (mi *MasterIndex) Load(ctx context.Context, r restic.ListerLoaderUnpacked, p *progress.Counter, cb func(id restic.ID, idx *Index, oldFormat bool, err error) error) error {
+	indexList, err := restic.MemorizeList(ctx, r, restic.IndexFile)
+	if err != nil {
+		return err
+	}
+
+	if p != nil {
+		var numIndexFiles uint64
+		err := indexList.List(ctx, restic.IndexFile, func(_ restic.ID, _ int64) error {
+			numIndexFiles++
+			return nil
+		})
+		if err != nil {
+			return err
+		}
+		p.SetMax(numIndexFiles)
+		defer p.Done()
+	}
+
+	err = ForAllIndexes(ctx, indexList, r, func(id restic.ID, idx *Index, oldFormat bool, err error) error {
+		if p != nil {
+			p.Add(1)
+		}
+		if cb != nil {
+			err = cb(id, idx, oldFormat, err)
+		}
+		if err != nil {
+			return err
+		}
+		// special case to allow check to ignore index loading errors
+		if idx == nil {
+			return nil
+		}
+		mi.Insert(idx)
+		return nil
+	})
+
+	if err != nil {
+		return err
+	}
+
+	return mi.MergeFinalIndexes()
+}
+
 // Save saves all known indexes to index files, leaving out any
 // packs whose ID is contained in packBlacklist from finalized indexes.
 // It also removes the old index files and those listed in extraObsolete.

internal/repository/repository.go

@@ -623,43 +623,10 @@ func (r *Repository) configureIndex() {
 func (r *Repository) LoadIndex(ctx context.Context, p *progress.Counter) error {
 	debug.Log("Loading index")
 
-	indexList, err := restic.MemorizeList(ctx, r, restic.IndexFile)
-	if err != nil {
-		return err
-	}
-
-	if p != nil {
-		var numIndexFiles uint64
-		err := indexList.List(ctx, restic.IndexFile, func(_ restic.ID, _ int64) error {
-			numIndexFiles++
-			return nil
-		})
-		if err != nil {
-			return err
-		}
-		p.SetMax(numIndexFiles)
-		defer p.Done()
-	}
-
 	// reset in-memory index before loading it from the repository
 	r.clearIndex()
 
-	err = index.ForAllIndexes(ctx, indexList, r, func(_ restic.ID, idx *index.Index, _ bool, err error) error {
+	err := r.idx.Load(ctx, r, p, nil)
-		if err != nil {
-			return err
-		}
-		r.idx.Insert(idx)
-		if p != nil {
-			p.Add(1)
-		}
-		return nil
-	})
-
-	if err != nil {
-		return err
-	}
-
-	err = r.idx.MergeFinalIndexes()
 	if err != nil {
 		return err
 	}