chunker: move buffer pool to chunker package
This commit is contained in:
Alexander Neumann
parent
bdbb3ab329
commit
ea22b2dfb1
4 changed files with 26 additions and 42 deletions
10
archiver.go
10
archiver.go
|
|
@ -170,12 +170,6 @@ func updateNodeContent(node *Node, results []saveResult) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
const chunkerBufSize = 512 * chunker.KiB
|
||||
|
||||
var chunkerBufPool = sync.Pool{
|
||||
New: func() interface{} { return make([]byte, chunkerBufSize) },
|
||||
}
|
||||
|
||||
// SaveFile stores the content of the file on the backend as a Blob by calling
|
||||
// Save for each chunk.
|
||||
func (arch *Archiver) SaveFile(p *Progress, node *Node) error {
|
||||
|
|
@ -190,9 +184,7 @@ func (arch *Archiver) SaveFile(p *Progress, node *Node) error {
|
|||
return err
|
||||
}
|
||||
|
||||
buf := chunkerBufPool.Get().([]byte)
|
||||
chnker := chunker.New(file, arch.s.Config.ChunkerPolynomial, buf, sha256.New())
|
||||
defer chunkerBufPool.Put(buf)
|
||||
chnker := chunker.New(file, arch.s.Config.ChunkerPolynomial, sha256.New())
|
||||
resultChannels := [](<-chan saveResult){}
|
||||
|
||||
for {
|
||||
|
|
|
|||
|
|
@ -25,9 +25,9 @@ type Rdr interface {
|
|||
io.ReaderAt
|
||||
}
|
||||
|
||||
func benchmarkChunkEncrypt(b testing.TB, buf, buf2, chunkBuf []byte, rd Rdr, key *crypto.Key) {
|
||||
func benchmarkChunkEncrypt(b testing.TB, buf, buf2 []byte, rd Rdr, key *crypto.Key) {
|
||||
rd.Seek(0, 0)
|
||||
ch := chunker.New(rd, testPol, chunkBuf, sha256.New())
|
||||
ch := chunker.New(rd, testPol, sha256.New())
|
||||
|
||||
for {
|
||||
chunk, err := ch.Next()
|
||||
|
|
@ -58,18 +58,17 @@ func BenchmarkChunkEncrypt(b *testing.B) {
|
|||
|
||||
buf := make([]byte, chunker.MaxSize)
|
||||
buf2 := make([]byte, chunker.MaxSize)
|
||||
chunkBuf := make([]byte, chunkerBufSize)
|
||||
|
||||
b.ResetTimer()
|
||||
b.SetBytes(int64(len(data)))
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
benchmarkChunkEncrypt(b, buf, buf2, chunkBuf, rd, s.Key())
|
||||
benchmarkChunkEncrypt(b, buf, buf2, rd, s.Key())
|
||||
}
|
||||
}
|
||||
|
||||
func benchmarkChunkEncryptP(b *testing.PB, buf, chunkBuf []byte, rd Rdr, key *crypto.Key) {
|
||||
ch := chunker.New(rd, testPol, chunkBuf, sha256.New())
|
||||
func benchmarkChunkEncryptP(b *testing.PB, buf []byte, rd Rdr, key *crypto.Key) {
|
||||
ch := chunker.New(rd, testPol, sha256.New())
|
||||
|
||||
for {
|
||||
chunk, err := ch.Next()
|
||||
|
|
@ -91,7 +90,6 @@ func BenchmarkChunkEncryptParallel(b *testing.B) {
|
|||
data := Random(23, 10<<20) // 10MiB
|
||||
|
||||
buf := make([]byte, chunker.MaxSize)
|
||||
chunkBuf := make([]byte, chunkerBufSize)
|
||||
|
||||
b.ResetTimer()
|
||||
b.SetBytes(int64(len(data)))
|
||||
|
|
@ -99,7 +97,7 @@ func BenchmarkChunkEncryptParallel(b *testing.B) {
|
|||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
rd := bytes.NewReader(data)
|
||||
benchmarkChunkEncryptP(pb, buf, chunkBuf, rd, s.Key())
|
||||
benchmarkChunkEncryptP(pb, buf, rd, s.Key())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,8 +23,14 @@ const (
|
|||
MaxSize = 8 * MiB
|
||||
|
||||
splitmask = (1 << averageBits) - 1
|
||||
|
||||
chunkerBufSize = 512 * KiB
|
||||
)
|
||||
|
||||
var bufPool = sync.Pool{
|
||||
New: func() interface{} { return make([]byte, chunkerBufSize) },
|
||||
}
|
||||
|
||||
type tables struct {
|
||||
out [256]Pol
|
||||
mod [256]Pol
|
||||
|
|
@ -79,18 +85,12 @@ type Chunker struct {
|
|||
h hash.Hash
|
||||
}
|
||||
|
||||
const minBufSize = 32
|
||||
|
||||
// New returns a new Chunker based on polynomial p that reads from rd
|
||||
// with bufsize and pass all data to hash along the way, using buf for
|
||||
// buffering. Buf must at least hold 32 bytes.
|
||||
func New(rd io.Reader, pol Pol, buf []byte, h hash.Hash) *Chunker {
|
||||
if len(buf) < minBufSize {
|
||||
buf = make([]byte, minBufSize)
|
||||
}
|
||||
|
||||
// buffering.
|
||||
func New(rd io.Reader, pol Pol, h hash.Hash) *Chunker {
|
||||
c := &Chunker{
|
||||
buf: buf,
|
||||
buf: bufPool.Get().([]byte),
|
||||
h: h,
|
||||
pol: pol,
|
||||
rd: rd,
|
||||
|
|
@ -204,6 +204,9 @@ func (c *Chunker) Next() (*Chunk, error) {
|
|||
if err == io.EOF && !c.closed {
|
||||
c.closed = true
|
||||
|
||||
// return the buffer to the pool
|
||||
bufPool.Put(c.buf)
|
||||
|
||||
// return current chunk, if any bytes have been processed
|
||||
if c.count > 0 {
|
||||
return &Chunk{
|
||||
|
|
|
|||
|
|
@ -147,13 +147,10 @@ func getRandom(seed, count int) []byte {
|
|||
return buf
|
||||
}
|
||||
|
||||
const chunkerBufSize = 512 * chunker.KiB
|
||||
|
||||
func TestChunker(t *testing.T) {
|
||||
// setup data source
|
||||
buf := getRandom(23, 32*1024*1024)
|
||||
chunkBuf := make([]byte, chunkerBufSize)
|
||||
ch := chunker.New(bytes.NewReader(buf), testPol, chunkBuf, sha256.New())
|
||||
ch := chunker.New(bytes.NewReader(buf), testPol, sha256.New())
|
||||
chunks := testWithData(t, ch, chunks1)
|
||||
|
||||
// test reader
|
||||
|
|
@ -180,7 +177,7 @@ func TestChunker(t *testing.T) {
|
|||
|
||||
// setup nullbyte data source
|
||||
buf = bytes.Repeat([]byte{0}, len(chunks2)*chunker.MinSize)
|
||||
ch = chunker.New(bytes.NewReader(buf), testPol, chunkBuf, sha256.New())
|
||||
ch = chunker.New(bytes.NewReader(buf), testPol, sha256.New())
|
||||
|
||||
testWithData(t, ch, chunks2)
|
||||
}
|
||||
|
|
@ -188,7 +185,6 @@ func TestChunker(t *testing.T) {
|
|||
func TestChunkerWithRandomPolynomial(t *testing.T) {
|
||||
// setup data source
|
||||
buf := getRandom(23, 32*1024*1024)
|
||||
chunkBuf := make([]byte, chunkerBufSize)
|
||||
|
||||
// generate a new random polynomial
|
||||
start := time.Now()
|
||||
|
|
@ -197,7 +193,7 @@ func TestChunkerWithRandomPolynomial(t *testing.T) {
|
|||
t.Logf("generating random polynomial took %v", time.Since(start))
|
||||
|
||||
start = time.Now()
|
||||
ch := chunker.New(bytes.NewReader(buf), p, chunkBuf, sha256.New())
|
||||
ch := chunker.New(bytes.NewReader(buf), p, sha256.New())
|
||||
t.Logf("creating chunker took %v", time.Since(start))
|
||||
|
||||
// make sure that first chunk is different
|
||||
|
|
@ -214,9 +210,8 @@ func TestChunkerWithRandomPolynomial(t *testing.T) {
|
|||
func TestChunkerWithoutHash(t *testing.T) {
|
||||
// setup data source
|
||||
buf := getRandom(23, 32*1024*1024)
|
||||
chunkBuf := make([]byte, chunkerBufSize)
|
||||
|
||||
ch := chunker.New(bytes.NewReader(buf), testPol, chunkBuf, nil)
|
||||
ch := chunker.New(bytes.NewReader(buf), testPol, nil)
|
||||
chunks := testWithData(t, ch, chunks1)
|
||||
|
||||
// test reader
|
||||
|
|
@ -246,7 +241,7 @@ func TestChunkerWithoutHash(t *testing.T) {
|
|||
|
||||
// setup nullbyte data source
|
||||
buf = bytes.Repeat([]byte{0}, len(chunks2)*chunker.MinSize)
|
||||
ch = chunker.New(bytes.NewReader(buf), testPol, chunkBuf, sha256.New())
|
||||
ch = chunker.New(bytes.NewReader(buf), testPol, sha256.New())
|
||||
|
||||
testWithData(t, ch, chunks2)
|
||||
}
|
||||
|
|
@ -276,8 +271,6 @@ func benchmarkChunker(b *testing.B, hash hash.Hash) {
|
|||
rd = bytes.NewReader(getRandom(23, size))
|
||||
}
|
||||
|
||||
chunkBuf := make([]byte, chunkerBufSize)
|
||||
|
||||
b.ResetTimer()
|
||||
b.SetBytes(int64(size))
|
||||
|
||||
|
|
@ -286,7 +279,7 @@ func benchmarkChunker(b *testing.B, hash hash.Hash) {
|
|||
chunks = 0
|
||||
|
||||
rd.Seek(0, 0)
|
||||
ch := chunker.New(rd, testPol, chunkBuf, hash)
|
||||
ch := chunker.New(rd, testPol, hash)
|
||||
|
||||
for {
|
||||
_, err := ch.Next()
|
||||
|
|
@ -322,11 +315,9 @@ func BenchmarkNewChunker(b *testing.B) {
|
|||
p, err := chunker.RandomPolynomial()
|
||||
OK(b, err)
|
||||
|
||||
chunkBuf := make([]byte, chunkerBufSize)
|
||||
|
||||
b.ResetTimer()
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
chunker.New(bytes.NewBuffer(nil), p, chunkBuf, nil)
|
||||
chunker.New(bytes.NewBuffer(nil), p, nil)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue