Re-use zlib writers to reduce memory usage

benchcmp:

    benchmark                       old ns/op       new ns/op       delta
    BenchmarkArchiveDirectory-4     29624960475     29511001504     -0.38%
    BenchmarkSaveJSON-4             379833          225609          -40.60%

    benchmark                       old allocs     new allocs     delta
    BenchmarkArchiveDirectory-4     546736         540642         -1.11%
    BenchmarkSaveJSON-4             150            126            -16.00%

    benchmark                       old bytes      new bytes     delta
    BenchmarkArchiveDirectory-4     1372476952     438519336     -68.05%
    BenchmarkSaveJSON-4             1462773        9087          -99.38%
This commit is contained in:
Alexander Neumann 2015-02-16 19:32:36 +01:00
parent 06ed5c12b8
commit 731e81ef06
2 changed files with 78 additions and 1 deletions

View file

@ -7,6 +7,7 @@ import (
"errors" "errors"
"fmt" "fmt"
"io" "io"
"sync"
"github.com/restic/restic/backend" "github.com/restic/restic/backend"
"github.com/restic/restic/debug" "github.com/restic/restic/debug"
@ -277,6 +278,12 @@ func (s Server) SaveFrom(t backend.Type, id backend.ID, length uint, rd io.Reade
return blob, nil return blob, nil
} }
var zWriterPool = sync.Pool{
New: func() interface{} {
return zlib.NewWriter(nil)
},
}
// SaveJSON serialises item as JSON and encrypts and saves it in the backend as // SaveJSON serialises item as JSON and encrypts and saves it in the backend as
// type t. // type t.
func (s Server) SaveJSON(t backend.Type, item interface{}) (Blob, error) { func (s Server) SaveJSON(t backend.Type, item interface{}) (Blob, error) {
@ -286,7 +293,9 @@ func (s Server) SaveJSON(t backend.Type, item interface{}) (Blob, error) {
} }
encWr := s.key.EncryptTo(backendBlob) encWr := s.key.EncryptTo(backendBlob)
wr := zlib.NewWriter(encWr) wr := zWriterPool.Get().(*zlib.Writer)
defer zWriterPool.Put(wr)
wr.Reset(encWr)
if err != nil { if err != nil {
return Blob{}, fmt.Errorf("zlib.NewWriter: %v", err) return Blob{}, fmt.Errorf("zlib.NewWriter: %v", err)
} }

68
server_test.go Normal file
View file

@ -0,0 +1,68 @@
package restic_test
import (
"bytes"
"crypto/sha256"
"encoding/json"
"testing"
"github.com/restic/restic"
"github.com/restic/restic/backend"
)
type testJSONStruct struct {
Foo uint32
Bar string
Baz []byte
}
var serverTests = []testJSONStruct{
testJSONStruct{Foo: 23, Bar: "Teststring", Baz: []byte("xx")},
}
func TestSaveJSON(t *testing.T) {
be := setupBackend(t)
defer teardownBackend(t, be)
key := setupKey(t, be, "geheim")
server := restic.NewServerWithKey(be, key)
for _, obj := range serverTests {
data, err := json.Marshal(obj)
ok(t, err)
data = append(data, '\n')
h := sha256.Sum256(data)
blob, err := server.SaveJSON(backend.Tree, obj)
ok(t, err)
assert(t, bytes.Equal(h[:], blob.ID),
"TestSaveJSON: wrong plaintext ID: expected %02x, got %02x",
h, blob.ID)
}
}
func BenchmarkSaveJSON(t *testing.B) {
be := setupBackend(t)
defer teardownBackend(t, be)
key := setupKey(t, be, "geheim")
server := restic.NewServerWithKey(be, key)
t.ResetTimer()
//t.SetBytes(int64(size))
obj := serverTests[0]
data, err := json.Marshal(obj)
ok(t, err)
data = append(data, '\n')
h := sha256.Sum256(data)
for i := 0; i < t.N; i++ {
blob, err := server.SaveJSON(backend.Tree, obj)
ok(t, err)
assert(t, bytes.Equal(h[:], blob.ID),
"TestSaveJSON: wrong plaintext ID: expected %02x, got %02x",
h, blob.ID)
}
}