repository/master_index: Optimize Index.Lookup()

When looking up a blob in the master index, with several
indexes present in the master index, a significant amount of time
is spent generating errors for each failed lookup.  However, these
errors are often used to check if a blob is present, but the contents
are not inspected making the overhead of the error not useful.

Instead, change Index.Lookup (and Index.LookupSize) to instead return
a boolean denoting if the blob was found instead of an error.  Also change
all the calls to these functions to handle the new function signature.

benchmark                                            old ns/op     new ns/op     delta
BenchmarkMasterIndexLookupSingleIndex-6              820           897           +9.39%
BenchmarkMasterIndexLookupMultipleIndex-6            12821         2001          -84.39%
BenchmarkMasterIndexLookupSingleIndexUnknown-6       5378          492           -90.85%
BenchmarkMasterIndexLookupMultipleIndexUnknown-6     17026         1649          -90.31%

benchmark                                            old allocs     new allocs     delta
BenchmarkMasterIndexLookupSingleIndex-6              9              9              +0.00%
BenchmarkMasterIndexLookupMultipleIndex-6            59             19             -67.80%
BenchmarkMasterIndexLookupSingleIndexUnknown-6       22             6              -72.73%
BenchmarkMasterIndexLookupMultipleIndexUnknown-6     72             16             -77.78%

benchmark                                            old bytes     new bytes     delta
BenchmarkMasterIndexLookupSingleIndex-6              160           160           +0.00%
BenchmarkMasterIndexLookupMultipleIndex-6            3200          240           -92.50%
BenchmarkMasterIndexLookupSingleIndexUnknown-6       1232          48            -96.10%
BenchmarkMasterIndexLookupMultipleIndexUnknown-6     4272          128           -97.00%

internal/repository/master_index_test.go

@@ -0,0 +1,123 @@
+package repository_test
+
+import (
+	"math/rand"
+	"testing"
+
+	"github.com/restic/restic/internal/repository"
+	"github.com/restic/restic/internal/restic"
+	rtest "github.com/restic/restic/internal/test"
+)
+
+func TestMasterIndexLookup(t *testing.T) {
+	idInIdx1 := restic.NewRandomID()
+	idInIdx2 := restic.NewRandomID()
+
+	blob1 := restic.PackedBlob{
+		PackID: restic.NewRandomID(),
+		Blob: restic.Blob{
+			Type:   restic.DataBlob,
+			ID:     idInIdx1,
+			Length: 10,
+			Offset: 0,
+		},
+	}
+
+	blob2 := restic.PackedBlob{
+		PackID: restic.NewRandomID(),
+		Blob: restic.Blob{
+			Type:   restic.DataBlob,
+			ID:     idInIdx2,
+			Length: 100,
+			Offset: 10,
+		},
+	}
+
+	idx1 := repository.NewIndex()
+	idx1.Store(blob1)
+
+	idx2 := repository.NewIndex()
+	idx2.Store(blob2)
+
+	mIdx := repository.NewMasterIndex()
+	mIdx.Insert(idx1)
+	mIdx.Insert(idx2)
+
+	blobs, found := mIdx.Lookup(idInIdx1, restic.DataBlob)
+	rtest.Assert(t, found, "Expected to find blob id %v from index 1", idInIdx1)
+	rtest.Equals(t, []restic.PackedBlob{blob1}, blobs)
+
+	blobs, found = mIdx.Lookup(idInIdx2, restic.DataBlob)
+	rtest.Assert(t, found, "Expected to find blob id %v from index 2", idInIdx2)
+	rtest.Equals(t, []restic.PackedBlob{blob2}, blobs)
+
+	blobs, found = mIdx.Lookup(restic.NewRandomID(), restic.DataBlob)
+	rtest.Assert(t, !found, "Expected to not find a blob when fetching with a random id")
+	rtest.Assert(t, blobs == nil, "Expected no blobs when fetching with a random id")
+}
+
+func BenchmarkMasterIndexLookupSingleIndex(b *testing.B) {
+	idx1, lookupID := createRandomIndex(rand.New(rand.NewSource(0)))
+
+	mIdx := repository.NewMasterIndex()
+	mIdx.Insert(idx1)
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		mIdx.Lookup(lookupID, restic.DataBlob)
+	}
+}
+
+func BenchmarkMasterIndexLookupMultipleIndex(b *testing.B) {
+	rng := rand.New(rand.NewSource(0))
+	mIdx := repository.NewMasterIndex()
+
+	for i := 0; i < 5; i++ {
+		idx, _ := createRandomIndex(rand.New(rng))
+		mIdx.Insert(idx)
+	}
+
+	idx1, lookupID := createRandomIndex(rand.New(rng))
+	mIdx.Insert(idx1)
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		mIdx.Lookup(lookupID, restic.DataBlob)
+	}
+}
+
+func BenchmarkMasterIndexLookupSingleIndexUnknown(b *testing.B) {
+	lookupID := restic.NewRandomID()
+	idx1, _ := createRandomIndex(rand.New(rand.NewSource(0)))
+
+	mIdx := repository.NewMasterIndex()
+	mIdx.Insert(idx1)
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		mIdx.Lookup(lookupID, restic.DataBlob)
+	}
+}
+
+func BenchmarkMasterIndexLookupMultipleIndexUnknown(b *testing.B) {
+	rng := rand.New(rand.NewSource(0))
+	lookupID := restic.NewRandomID()
+	mIdx := repository.NewMasterIndex()
+
+	for i := 0; i < 5; i++ {
+		idx, _ := createRandomIndex(rand.New(rng))
+		mIdx.Insert(idx)
+	}
+
+	idx1, _ := createRandomIndex(rand.New(rng))
+	mIdx.Insert(idx1)
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		mIdx.Lookup(lookupID, restic.DataBlob)
+	}
+}