alexvanin/restic
1
0
Fork 0
forked from TrueCloudLab/restic
Code Pull requests Activity

Improve memory usage (and speed) of crypto reader/writer

Browse source 
benchcmp:

    benchmark                         old ns/op     new ns/op     delta
    BenchmarkChunkEncrypt             260811933     261294510     +0.19%
    BenchmarkChunkEncryptParallel     262787948     259394601     -1.29%
    BenchmarkEncryptWriter            89769328      87108804      -2.96%
    BenchmarkEncrypt                  88586645      87489151      -1.24%
    BenchmarkDecryptReader            99669425      90019414      -9.68%
    BenchmarkEncryptDecryptReader     189263157     185185733     -2.15%
    BenchmarkDecrypt                  89631109      89172417      -0.51%

    benchmark                         old MB/s     new MB/s     speedup
    BenchmarkChunkEncrypt             40.20        40.13        1.00x
    BenchmarkChunkEncryptParallel     39.90        40.42        1.01x
    BenchmarkEncryptWriter            93.45        96.30        1.03x
    BenchmarkEncrypt                  94.69        95.88        1.01x
    BenchmarkDecryptReader            84.16        93.19        1.11x
    BenchmarkEncryptDecryptReader     44.32        45.30        1.02x
    BenchmarkDecrypt                  93.59        94.07        1.01x

    benchmark                         old allocs     new allocs     delta
    BenchmarkChunkEncrypt             113            113            +0.00%
    BenchmarkChunkEncryptParallel     104            104            +0.00%
    BenchmarkEncryptWriter            20             20             +0.00%
    BenchmarkEncrypt                  14             14             +0.00%
    BenchmarkDecryptReader            38             18             -52.63%
    BenchmarkEncryptDecryptReader     61             55             -9.84%
    BenchmarkDecrypt                  17             17             +0.00%

    benchmark                         old bytes     new bytes     delta
    BenchmarkChunkEncrypt             8515750       8515750       +0.00%
    BenchmarkChunkEncryptParallel     8515766       8515766       +0.00%
    BenchmarkEncryptWriter            8391305       28927         -99.66%
    BenchmarkEncrypt                  2475          2475          +0.00%
    BenchmarkDecryptReader            33563550      527827        -98.43%
    BenchmarkEncryptDecryptReader     50348456      35814894      -28.87%
    BenchmarkDecrypt                  8391127       8391127       +0.00%
This commit is contained in:
Alexander Neumann 2015-02-16 22:34:32 +01:00
parent d94d003165
commit 874b29b91f
1 changed files with 85 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

103
key.go
View file

@ -1,7 +1,6 @@
package restic
import (
"bytes"
"crypto/aes"
"crypto/cipher"
"crypto/hmac"
@ -15,6 +14,7 @@ import (
"io/ioutil"
"os"
"os/user"
"sync"
"time"
"github.com/restic/restic/backend"
@ -338,6 +338,7 @@ type encryptWriter struct {
iv []byte
wroteIV bool
h hash.Hash
s cipher.Stream
w io.Writer
origWr io.Writer
err error // remember error writing iv
@ -353,6 +354,13 @@ func (e *encryptWriter) Close() error {
return nil
}
const encryptWriterChunkSize = 512 * 1024 // 512 KiB
var encryptWriterBufPool = sync.Pool{
New: func() interface{} {
return make([]byte, encryptWriterChunkSize)
},
}
func (e *encryptWriter) Write(p []byte) (int, error) {
// write iv first
if !e.wroteIV {
@ -364,13 +372,34 @@ func (e *encryptWriter) Write(p []byte) (int, error) {
return 0, e.err
}
n, err := e.w.Write(p)
if err != nil {
e.err = err
return n, err
buf := encryptWriterBufPool.Get().([]byte)
defer encryptWriterBufPool.Put(buf)
written := 0
for len(p) > 0 {
max := len(p)
if max > encryptWriterChunkSize {
max = encryptWriterChunkSize
}
e.s.XORKeyStream(buf, p[:max])
n, err := e.w.Write(buf[:max])
if n != max {
if err == nil { // should never happen
err = io.ErrShortWrite
}
}
written += n
p = p[n:]
if err != nil {
e.err = err
return written, err
}
}
return n, nil
return written, nil
}
func (k *Key) encryptTo(ks *keys, wr io.Writer) io.WriteCloser {
@ -396,10 +425,8 @@ func (k *Key) encryptTo(ks *keys, wr io.Writer) io.WriteCloser {
panic(fmt.Sprintf("unable to create cipher: %v", err))
}
ew.w = cipher.StreamWriter{
S: cipher.NewCTR(c, ew.iv),
W: io.MultiWriter(ew.h, wr),
}
ew.s = cipher.NewCTR(c, ew.iv)
ew.w = io.MultiWriter(ew.h, wr)
return ew
}
@ -474,6 +501,34 @@ func (k *Key) DecryptUser(ciphertext []byte) ([]byte, error) {
return k.decrypt(k.user, ciphertext)
}
type decryptReader struct {
buf []byte
pos int
}
func (d *decryptReader) Read(dst []byte) (int, error) {
if d.buf == nil {
return 0, io.EOF
}
if len(dst) == 0 {
return 0, nil
}
remaining := len(d.buf) - d.pos
if len(dst) >= remaining {
n := copy(dst, d.buf[d.pos:])
FreeChunkBuf("decryptReader", d.buf)
d.buf = nil
return n, io.EOF
}
n := copy(dst, d.buf[d.pos:d.pos+len(dst)])
d.pos += n
return n, nil
}
// decryptFrom verifies and decrypts the ciphertext read from rd with ks and
// makes it available on the returned Reader. Ciphertext must be in the form IV
// || Ciphertext || HMAC. In order to correctly verify the ciphertext, rd is
@ -481,14 +536,28 @@ func (k *Key) DecryptUser(ciphertext []byte) ([]byte, error) {
// afterwards. If an HMAC verification failure is observed, it is returned
// immediately.
func (k *Key) decryptFrom(ks *keys, rd io.Reader) (io.Reader, error) {
ciphertext, err := ioutil.ReadAll(rd)
ciphertext := GetChunkBuf("decryptReader")
ciphertext = ciphertext[0:cap(ciphertext)]
n, err := io.ReadFull(rd, ciphertext)
if err != io.ErrUnexpectedEOF {
// read remaining data
buf, e := ioutil.ReadAll(rd)
ciphertext = append(ciphertext, buf...)
n += len(buf)
err = e
} else {
err = nil
}
if err != nil {
return nil, err
}
ciphertext = ciphertext[:n]
// check for plausible length
if len(ciphertext) < ivSize+hmacSize {
panic("trying to decryipt invalid data: ciphertext too small")
panic("trying to decrypt invalid data: ciphertext too small")
}
hm := hmac.New(sha256.New, ks.Sign)
@ -498,7 +567,7 @@ func (k *Key) decryptFrom(ks *keys, rd io.Reader) (io.Reader, error) {
ciphertext, mac := ciphertext[:l], ciphertext[l:]
// calculate new hmac
n, err := hm.Write(ciphertext)
n, err = hm.Write(ciphertext)
if err != nil || n != len(ciphertext) {
panic(fmt.Sprintf("unable to calculate hmac of ciphertext, err %v", err))
}
@ -519,12 +588,10 @@ func (k *Key) decryptFrom(ks *keys, rd io.Reader) (io.Reader, error) {
panic(fmt.Sprintf("unable to create cipher: %v", err))
}
r := cipher.StreamReader{
S: cipher.NewCTR(c, iv),
R: bytes.NewReader(ciphertext),
}
stream := cipher.NewCTR(c, iv)
stream.XORKeyStream(ciphertext, ciphertext)
return r, nil
return &decryptReader{buf: ciphertext}, nil
}
// DecryptFrom verifies and decrypts the ciphertext read from rd and makes it