chunker: Add buffer to New()

archiver.go

@@ -170,6 +170,12 @@ func updateNodeContent(node *Node, results []saveResult) error {
 	return nil
 }
 
+const chunkerBufSize = 512 * chunker.KiB
+
+var chunkerBufPool = sync.Pool{
+	New: func() interface{} { return make([]byte, chunkerBufSize) },
+}
+
 // SaveFile stores the content of the file on the backend as a Blob by calling
 // Save for each chunk.
 func (arch *Archiver) SaveFile(p *Progress, node *Node) error {
@@ -184,7 +190,9 @@ func (arch *Archiver) SaveFile(p *Progress, node *Node) error {
 		return err
 	}
 
-	chnker := chunker.New(file, arch.s.Config.ChunkerPolynomial, sha256.New())
+	buf := chunkerBufPool.Get().([]byte)
+	chnker := chunker.New(file, arch.s.Config.ChunkerPolynomial, buf, sha256.New())
+	defer chunkerBufPool.Put(buf)
 	resultChannels := [](<-chan saveResult){}
 
 	for {

archiver_test.go

@@ -18,16 +18,16 @@ import (
 var benchArchiveDirectory = flag.String("test.benchdir", ".", "benchmark archiving a real directory (default: .)")
 var testPol = chunker.Pol(0x3DA3358B4DC173)
 
-const bufSize = chunker.MiB
+const chunkerBufSize = 512 * chunker.KiB
 
 type Rdr interface {
 	io.ReadSeeker
 	io.ReaderAt
 }
 
-func benchmarkChunkEncrypt(b testing.TB, buf, buf2 []byte, rd Rdr, key *crypto.Key) {
+func benchmarkChunkEncrypt(b testing.TB, buf, buf2, chunkBuf []byte, rd Rdr, key *crypto.Key) {
 	rd.Seek(0, 0)
-	ch := chunker.New(rd, testPol, sha256.New())
+	ch := chunker.New(rd, testPol, chunkBuf, sha256.New())
 
 	for {
 		chunk, err := ch.Next()
@@ -58,20 +58,21 @@ func BenchmarkChunkEncrypt(b *testing.B) {
 
 	buf := restic.GetChunkBuf("BenchmarkChunkEncrypt")
 	buf2 := restic.GetChunkBuf("BenchmarkChunkEncrypt")
+	chunkBuf := make([]byte, chunkerBufSize)
 
 	b.ResetTimer()
 	b.SetBytes(int64(len(data)))
 
 	for i := 0; i < b.N; i++ {
-		benchmarkChunkEncrypt(b, buf, buf2, rd, s.Key())
+		benchmarkChunkEncrypt(b, buf, buf2, chunkBuf, rd, s.Key())
 	}
 
 	restic.FreeChunkBuf("BenchmarkChunkEncrypt", buf)
 	restic.FreeChunkBuf("BenchmarkChunkEncrypt", buf2)
 }
 
-func benchmarkChunkEncryptP(b *testing.PB, buf []byte, rd Rdr, key *crypto.Key) {
-	ch := chunker.New(rd, testPol, sha256.New())
+func benchmarkChunkEncryptP(b *testing.PB, buf, chunkBuf []byte, rd Rdr, key *crypto.Key) {
+	ch := chunker.New(rd, testPol, chunkBuf, sha256.New())
 
 	for {
 		chunk, err := ch.Next()
@@ -93,6 +94,7 @@ func BenchmarkChunkEncryptParallel(b *testing.B) {
 	data := Random(23, 10<<20) // 10MiB
 
 	buf := restic.GetChunkBuf("BenchmarkChunkEncryptParallel")
+	chunkBuf := make([]byte, chunkerBufSize)
 
 	b.ResetTimer()
 	b.SetBytes(int64(len(data)))
@@ -100,7 +102,7 @@ func BenchmarkChunkEncryptParallel(b *testing.B) {
 	b.RunParallel(func(pb *testing.PB) {
 		for pb.Next() {
 			rd := bytes.NewReader(data)
-			benchmarkChunkEncryptP(pb, buf, rd, s.Key())
+			benchmarkChunkEncryptP(pb, buf, chunkBuf, rd, s.Key())
 		}
 	})
 

chunker/chunker.go

@@ -82,11 +82,18 @@ type Chunker struct {
 	h      hash.Hash
 }
 
-// New returns a new Chunker based on polynomial p that reads from data from rd
-// with bufsize and pass all data to hash along the way.
-func New(rd io.Reader, pol Pol, h hash.Hash) *Chunker {
+const minBufSize = 32
+
+// New returns a new Chunker based on polynomial p that reads from rd
+// with bufsize and pass all data to hash along the way, using buf for
+// buffering. Buf must at least hold 32 bytes.
+func New(rd io.Reader, pol Pol, buf []byte, h hash.Hash) *Chunker {
+	if len(buf) < minBufSize {
+		buf = make([]byte, minBufSize)
+	}
+
 	c := &Chunker{
-		buf: make([]byte, bufSize),
+		buf: buf,
 		h:   h,
 		pol: pol,
 		rd:  rd,

chunker/chunker_test.go

@@ -147,10 +147,13 @@ func getRandom(seed, count int) []byte {
 	return buf
 }
 
+const chunkerBufSize = 512 * chunker.KiB
+
 func TestChunker(t *testing.T) {
 	// setup data source
 	buf := getRandom(23, 32*1024*1024)
-	ch := chunker.New(bytes.NewReader(buf), testPol, sha256.New())
+	chunkBuf := make([]byte, chunkerBufSize)
+	ch := chunker.New(bytes.NewReader(buf), testPol, chunkBuf, sha256.New())
 	chunks := testWithData(t, ch, chunks1)
 
 	// test reader
@@ -177,7 +180,7 @@ func TestChunker(t *testing.T) {
 
 	// setup nullbyte data source
 	buf = bytes.Repeat([]byte{0}, len(chunks2)*chunker.MinSize)
-	ch = chunker.New(bytes.NewReader(buf), testPol, sha256.New())
+	ch = chunker.New(bytes.NewReader(buf), testPol, chunkBuf, sha256.New())
 
 	testWithData(t, ch, chunks2)
 }
@@ -185,6 +188,7 @@ func TestChunker(t *testing.T) {
 func TestChunkerWithRandomPolynomial(t *testing.T) {
 	// setup data source
 	buf := getRandom(23, 32*1024*1024)
+	chunkBuf := make([]byte, chunkerBufSize)
 
 	// generate a new random polynomial
 	start := time.Now()
@@ -193,7 +197,7 @@ func TestChunkerWithRandomPolynomial(t *testing.T) {
 	t.Logf("generating random polynomial took %v", time.Since(start))
 
 	start = time.Now()
-	ch := chunker.New(bytes.NewReader(buf), p, sha256.New())
+	ch := chunker.New(bytes.NewReader(buf), p, chunkBuf, sha256.New())
 	t.Logf("creating chunker took %v", time.Since(start))
 
 	// make sure that first chunk is different
@@ -210,7 +214,9 @@ func TestChunkerWithRandomPolynomial(t *testing.T) {
 func TestChunkerWithoutHash(t *testing.T) {
 	// setup data source
 	buf := getRandom(23, 32*1024*1024)
-	ch := chunker.New(bytes.NewReader(buf), testPol, nil)
+	chunkBuf := make([]byte, chunkerBufSize)
+
+	ch := chunker.New(bytes.NewReader(buf), testPol, chunkBuf, nil)
 	chunks := testWithData(t, ch, chunks1)
 
 	// test reader
@@ -240,7 +246,7 @@ func TestChunkerWithoutHash(t *testing.T) {
 
 	// setup nullbyte data source
 	buf = bytes.Repeat([]byte{0}, len(chunks2)*chunker.MinSize)
-	ch = chunker.New(bytes.NewReader(buf), testPol, sha256.New())
+	ch = chunker.New(bytes.NewReader(buf), testPol, chunkBuf, sha256.New())
 
 	testWithData(t, ch, chunks2)
 }
@@ -270,6 +276,8 @@ func benchmarkChunker(b *testing.B, hash hash.Hash) {
 		rd = bytes.NewReader(getRandom(23, size))
 	}
 
+	chunkBuf := make([]byte, chunkerBufSize)
+
 	b.ResetTimer()
 	b.SetBytes(int64(size))
 
@@ -278,7 +286,7 @@ func benchmarkChunker(b *testing.B, hash hash.Hash) {
 		chunks = 0
 
 		rd.Seek(0, 0)
-		ch := chunker.New(rd, testPol, hash)
+		ch := chunker.New(rd, testPol, chunkBuf, hash)
 
 		for {
 			_, err := ch.Next()
@@ -314,9 +322,11 @@ func BenchmarkNewChunker(b *testing.B) {
 	p, err := chunker.RandomPolynomial()
 	OK(b, err)
 
+	chunkBuf := make([]byte, chunkerBufSize)
+
 	b.ResetTimer()
 
 	for i := 0; i < b.N; i++ {
-		chunker.New(bytes.NewBuffer(nil), p, nil)
+		chunker.New(bytes.NewBuffer(nil), p, chunkBuf, nil)
 	}
 }