Merge pull request #2622 from greatroar/optimize-packer-manager

Fix PackerManager benchmark and optimize hashing.Writer

internal/hashing/writer.go

@@ -15,13 +15,14 @@ type Writer struct {
 func NewWriter(w io.Writer, h hash.Hash) *Writer {
 	return &Writer{
 		h: h,
-		w: io.MultiWriter(w, h),
+		w: w,
 	}
 }
 
 // Write wraps the write method of the underlying writer and also hashes all data.
 func (h *Writer) Write(p []byte) (int, error) {
 	n, err := h.w.Write(p)
+	h.h.Write(p[:n])
 	return n, err
 }
 

internal/repository/packer_manager_test.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"math/rand"
 	"os"
+	"sync"
 	"testing"
 
 	"github.com/restic/restic/internal/backend/mem"
@@ -14,31 +15,6 @@ import (
 	"github.com/restic/restic/internal/restic"
 )
 
-type randReader struct {
-	src  rand.Source
-	rand *rand.Rand
-}
-
-func newRandReader(src rand.Source) *randReader {
-	return &randReader{
-		src:  src,
-		rand: rand.New(src),
-	}
-}
-
-// Read generates len(p) random bytes and writes them into p. It
-// always returns len(p) and a nil error.
-func (r *randReader) Read(p []byte) (n int, err error) {
-	for i := 0; i < len(p); i += 7 {
-		val := r.src.Int63()
-		for j := 0; i+j < len(p) && j < 7; j++ {
-			p[i+j] = byte(val)
-			val >>= 8
-		}
-	}
-	return len(p), nil
-}
-
 func randomID(rd io.Reader) restic.ID {
 	id := restic.ID{}
 	_, err := io.ReadFull(rd, id[:])
@@ -50,6 +26,13 @@ func randomID(rd io.Reader) restic.ID {
 
 const maxBlobSize = 1 << 20
 
+func min(a, b int) int {
+	if a < b {
+		return a
+	}
+	return b
+}
+
 func saveFile(t testing.TB, be Saver, length int, f *os.File, id restic.ID) {
 	h := restic.Handle{Type: restic.DataFile, Name: id.String()}
 	t.Logf("save file %v", h)
@@ -73,23 +56,19 @@ func saveFile(t testing.TB, be Saver, length int, f *os.File, id restic.ID) {
 	}
 }
 
-func fillPacks(t testing.TB, rnd *randReader, be Saver, pm *packerManager, buf []byte) (bytes int) {
+func fillPacks(t testing.TB, rnd *rand.Rand, be Saver, pm *packerManager, buf []byte) (bytes int) {
 	for i := 0; i < 100; i++ {
-		l := rnd.rand.Intn(1 << 20)
-		seed := rnd.rand.Int63()
+		l := rnd.Intn(maxBlobSize)
 
 		packer, err := pm.findPacker()
 		if err != nil {
 			t.Fatal(err)
 		}
 
-		rd := newRandReader(rand.NewSource(seed))
-		id := randomID(rd)
+		id := randomID(rnd)
 		buf = buf[:l]
-		_, err = io.ReadFull(rd, buf)
-		if err != nil {
-			t.Fatal(err)
-		}
+		// Only change a few bytes so we know we're not benchmarking the RNG.
+		rnd.Read(buf[:min(l, 4)])
 
 		n, err := packer.Add(restic.DataBlob, id, buf)
 		if err != nil {
@@ -117,7 +96,7 @@ func fillPacks(t testing.TB, rnd *randReader, be Saver, pm *packerManager, buf [
 	return bytes
 }
 
-func flushRemainingPacks(t testing.TB, rnd *randReader, be Saver, pm *packerManager) (bytes int) {
+func flushRemainingPacks(t testing.TB, be Saver, pm *packerManager) (bytes int) {
 	if pm.countPacker() > 0 {
 		for _, packer := range pm.packers {
 			n, err := packer.Finalize()
@@ -134,8 +113,20 @@ func flushRemainingPacks(t testing.TB, rnd *randReader, be Saver, pm *packerMana
 	return bytes
 }
 
+const randomSeed = 23
+
+var (
+	once      sync.Once
+	totalSize int64
+)
+
 func TestPackerManager(t *testing.T) {
-	rnd := newRandReader(rand.NewSource(23))
+	bytes := testPackerManager(t)
+	once.Do(func() { totalSize = bytes })
+}
+
+func testPackerManager(t testing.TB) int64 {
+	rnd := rand.New(rand.NewSource(randomSeed))
 
 	be := mem.New()
 	pm := newPackerManager(be, crypto.NewRandomKey())
@@ -143,26 +134,33 @@ func TestPackerManager(t *testing.T) {
 	blobBuf := make([]byte, maxBlobSize)
 
 	bytes := fillPacks(t, rnd, be, pm, blobBuf)
-	bytes += flushRemainingPacks(t, rnd, be, pm)
+	bytes += flushRemainingPacks(t, be, pm)
 
 	t.Logf("saved %d bytes", bytes)
+	return int64(bytes)
 }
 
 func BenchmarkPackerManager(t *testing.B) {
-	rnd := newRandReader(rand.NewSource(23))
+	// Run testPackerManager if it hasn't run already, to set totalSize.
+	once.Do(func() {
+		totalSize = testPackerManager(t)
+	})
+
+	rnd := rand.New(rand.NewSource(randomSeed))
 
 	be := &mock.Backend{
 		SaveFn: func(context.Context, restic.Handle, restic.RewindReader) error { return nil },
 	}
 	blobBuf := make([]byte, maxBlobSize)
 
+	t.ReportAllocs()
+	t.SetBytes(totalSize)
 	t.ResetTimer()
 
 	for i := 0; i < t.N; i++ {
-		bytes := 0
+		rnd.Seed(randomSeed)
 		pm := newPackerManager(be, crypto.NewRandomKey())
-		bytes += fillPacks(t, rnd, be, pm, blobBuf)
-		bytes += flushRemainingPacks(t, rnd, be, pm)
-		t.Logf("saved %d bytes", bytes)
+		fillPacks(t, rnd, be, pm, blobBuf)
+		flushRemainingPacks(t, be, pm)
 	}
 }