Merge pull request #362 from restic/improvements

Cleanups

backend/backend_test.go

@@ -12,7 +12,34 @@ import (
 	. "github.com/restic/restic/test"
 )
 
+func testBackendConfig(b backend.Backend, t *testing.T) {
+	// create config and read it back
+	_, err := b.Get(backend.Config, "")
+	Assert(t, err != nil, "did not get expected error for non-existing config")
+
+	blob, err := b.Create()
+	OK(t, err)
+
+	_, err = blob.Write([]byte("Config"))
+	OK(t, err)
+	OK(t, blob.Finalize(backend.Config, ""))
+
+	// try accessing the config with different names, should all return the
+	// same config
+	for _, name := range []string{"", "foo", "bar", "0000000000000000000000000000000000000000000000000000000000000000"} {
+		rd, err := b.Get(backend.Config, name)
+		Assert(t, err == nil, "unable to read config")
+
+		buf, err := ioutil.ReadAll(rd)
+		OK(t, err)
+		OK(t, rd.Close())
+		Assert(t, string(buf) == "Config", "wrong data returned for config")
+	}
+}
+
 func testBackend(b backend.Backend, t *testing.T) {
+	testBackendConfig(b, t)
+
 	for _, tpe := range []backend.Type{
 		backend.Data, backend.Key, backend.Lock,
 		backend.Snapshot, backend.Index,
@@ -96,6 +123,11 @@ func testBackend(b backend.Backend, t *testing.T) {
 		err = b.Remove(tpe, test.id)
 		OK(t, err)
 
+		// test that the blob is gone
+		ok, err := b.Test(tpe, test.id)
+		OK(t, err)
+		Assert(t, ok == false, "removed blob still present")
+
 		// create blob
 		blob, err = b.Create()
 		OK(t, err)

backend/mem_backend.go

@@ -0,0 +1,231 @@
+package backend
+
+import (
+	"bytes"
+	"errors"
+	"io"
+	"sort"
+	"sync"
+
+	"github.com/restic/restic/debug"
+)
+
+type entry struct {
+	Type Type
+	Name string
+}
+
+type memMap map[entry][]byte
+
+// MemoryBackend is a mock backend that uses a map for storing all data in
+// memory. This should only be used for tests.
+type MemoryBackend struct {
+	data memMap
+	m    sync.Mutex
+
+	MockBackend
+}
+
+// NewMemoryBackend returns a new backend that saves all data in a map in
+// memory.
+func NewMemoryBackend() *MemoryBackend {
+	be := &MemoryBackend{
+		data: make(memMap),
+	}
+
+	be.MockBackend.TestFn = func(t Type, name string) (bool, error) {
+		return memTest(be, t, name)
+	}
+
+	be.MockBackend.CreateFn = func() (Blob, error) {
+		return memCreate(be)
+	}
+
+	be.MockBackend.GetFn = func(t Type, name string) (io.ReadCloser, error) {
+		return memGet(be, t, name)
+	}
+
+	be.MockBackend.GetReaderFn = func(t Type, name string, offset, length uint) (io.ReadCloser, error) {
+		return memGetReader(be, t, name, offset, length)
+	}
+
+	be.MockBackend.RemoveFn = func(t Type, name string) error {
+		return memRemove(be, t, name)
+	}
+
+	be.MockBackend.ListFn = func(t Type, done <-chan struct{}) <-chan string {
+		return memList(be, t, done)
+	}
+
+	be.MockBackend.DeleteFn = func() error {
+		be.m.Lock()
+		defer be.m.Unlock()
+
+		be.data = make(memMap)
+		return nil
+	}
+
+	debug.Log("MemoryBackend.New", "created new memory backend")
+
+	return be
+}
+
+func (be *MemoryBackend) insert(t Type, name string, data []byte) error {
+	be.m.Lock()
+	defer be.m.Unlock()
+
+	if _, ok := be.data[entry{t, name}]; ok {
+		return errors.New("already present")
+	}
+
+	be.data[entry{t, name}] = data
+	return nil
+}
+
+func memTest(be *MemoryBackend, t Type, name string) (bool, error) {
+	be.m.Lock()
+	defer be.m.Unlock()
+
+	debug.Log("MemoryBackend.Test", "test %v %v", t, name)
+
+	if _, ok := be.data[entry{t, name}]; ok {
+		return true, nil
+	}
+
+	return false, nil
+}
+
+// tempMemEntry temporarily holds data written to the memory backend before it
+// is finalized.
+type tempMemEntry struct {
+	be   *MemoryBackend
+	data bytes.Buffer
+}
+
+func (e *tempMemEntry) Write(p []byte) (int, error) {
+	return e.data.Write(p)
+}
+
+func (e *tempMemEntry) Size() uint {
+	return uint(len(e.data.Bytes()))
+}
+
+func (e *tempMemEntry) Finalize(t Type, name string) error {
+	if t == Config {
+		name = ""
+	}
+
+	debug.Log("MemoryBackend", "save blob %p (%d bytes) as %v %v", e, len(e.data.Bytes()), t, name)
+	return e.be.insert(t, name, e.data.Bytes())
+}
+
+func memCreate(be *MemoryBackend) (Blob, error) {
+	blob := &tempMemEntry{be: be}
+	debug.Log("MemoryBackend.Create", "create new blob %p", blob)
+	return blob, nil
+}
+
+// readCloser wraps a reader and adds a noop Close method.
+type readCloser struct {
+	io.Reader
+}
+
+func (rd readCloser) Close() error {
+	return nil
+}
+
+func memGet(be *MemoryBackend, t Type, name string) (io.ReadCloser, error) {
+	be.m.Lock()
+	defer be.m.Unlock()
+
+	if t == Config {
+		name = ""
+	}
+
+	debug.Log("MemoryBackend.Get", "get %v %v", t, name)
+
+	if _, ok := be.data[entry{t, name}]; !ok {
+		return nil, errors.New("no such data")
+	}
+
+	return readCloser{bytes.NewReader(be.data[entry{t, name}])}, nil
+}
+
+func memGetReader(be *MemoryBackend, t Type, name string, offset, length uint) (io.ReadCloser, error) {
+	be.m.Lock()
+	defer be.m.Unlock()
+
+	if t == Config {
+		name = ""
+	}
+
+	debug.Log("MemoryBackend.GetReader", "get %v %v offset %v len %v", t, name, offset, length)
+
+	if _, ok := be.data[entry{t, name}]; !ok {
+		return nil, errors.New("no such data")
+	}
+
+	buf := be.data[entry{t, name}]
+	if offset > uint(len(buf)) {
+		return nil, errors.New("offset beyond end of file")
+	}
+
+	buf = buf[offset:]
+
+	if length > 0 {
+		if length > uint(len(buf)) {
+			length = uint(len(buf))
+		}
+
+		buf = buf[:length]
+	}
+
+	return readCloser{bytes.NewReader(buf)}, nil
+}
+
+func memRemove(be *MemoryBackend, t Type, name string) error {
+	be.m.Lock()
+	defer be.m.Unlock()
+
+	debug.Log("MemoryBackend.Remove", "get %v %v", t, name)
+
+	if _, ok := be.data[entry{t, name}]; !ok {
+		return errors.New("no such data")
+	}
+
+	delete(be.data, entry{t, name})
+
+	return nil
+}
+
+func memList(be *MemoryBackend, t Type, done <-chan struct{}) <-chan string {
+	be.m.Lock()
+	defer be.m.Unlock()
+
+	ch := make(chan string)
+
+	var ids []string
+	for entry := range be.data {
+		if entry.Type != t {
+			continue
+		}
+		ids = append(ids, entry.Name)
+	}
+
+	sort.Strings(ids)
+
+	debug.Log("MemoryBackend.List", "list %v: %v", t, ids)
+
+	go func() {
+		defer close(ch)
+		for _, id := range ids {
+			select {
+			case ch <- id:
+			case <-done:
+				return
+			}
+		}
+	}()
+
+	return ch
+}

backend/mem_backend_test.go

@@ -0,0 +1,12 @@
+package backend_test
+
+import (
+	"testing"
+
+	"github.com/restic/restic/backend"
+)
+
+func TestMemoryBackend(t *testing.T) {
+	be := backend.NewMemoryBackend()
+	testBackend(be, t)
+}

backend/mock_backend.go

@@ -0,0 +1,97 @@
+package backend
+
+import (
+	"errors"
+	"io"
+)
+
+// MockBackend implements a backend whose functions can be specified. This
+// should only be used for tests.
+type MockBackend struct {
+	CloseFn     func() error
+	CreateFn    func() (Blob, error)
+	GetFn       func(Type, string) (io.ReadCloser, error)
+	GetReaderFn func(Type, string, uint, uint) (io.ReadCloser, error)
+	ListFn      func(Type, <-chan struct{}) <-chan string
+	RemoveFn    func(Type, string) error
+	TestFn      func(Type, string) (bool, error)
+	DeleteFn    func() error
+	LocationFn  func() string
+}
+
+func (m *MockBackend) Close() error {
+	if m.CloseFn == nil {
+		return nil
+	}
+
+	return m.CloseFn()
+}
+
+func (m *MockBackend) Location() string {
+	if m.LocationFn == nil {
+		return ""
+	}
+
+	return m.LocationFn()
+}
+
+func (m *MockBackend) Create() (Blob, error) {
+	if m.CreateFn == nil {
+		return nil, errors.New("not implemented")
+	}
+
+	return m.CreateFn()
+}
+
+func (m *MockBackend) Get(t Type, name string) (io.ReadCloser, error) {
+	if m.GetFn == nil {
+		return nil, errors.New("not implemented")
+	}
+
+	return m.GetFn(t, name)
+}
+
+func (m *MockBackend) GetReader(t Type, name string, offset, len uint) (io.ReadCloser, error) {
+	if m.GetReaderFn == nil {
+		return nil, errors.New("not implemented")
+	}
+
+	return m.GetReaderFn(t, name, offset, len)
+}
+
+func (m *MockBackend) List(t Type, done <-chan struct{}) <-chan string {
+	if m.ListFn == nil {
+		ch := make(chan string)
+		close(ch)
+		return ch
+	}
+
+	return m.ListFn(t, done)
+}
+
+func (m *MockBackend) Remove(t Type, name string) error {
+	if m.RemoveFn == nil {
+		return errors.New("not implemented")
+	}
+
+	return m.RemoveFn(t, name)
+}
+
+func (m *MockBackend) Test(t Type, name string) (bool, error) {
+	if m.TestFn == nil {
+		return false, errors.New("not implemented")
+	}
+
+	return m.TestFn(t, name)
+}
+
+func (m *MockBackend) Delete() error {
+	if m.DeleteFn == nil {
+		return errors.New("not implemented")
+	}
+
+	return m.DeleteFn()
+}
+
+// Make sure that MockBackend implements the backend interface.
+var _ Backend = &MockBackend{}

backend/writer.go

@@ -1,57 +1,18 @@
 package backend
 
 import (
-	"errors"
 	"hash"
 	"io"
 )
 
-type HashAppendWriter struct {
+// HashingWriter wraps an io.Writer to hashes all data that is written to it.
-	w      io.Writer
-	origWr io.Writer
-	h      hash.Hash
-	sum    []byte
-	closed bool
-}
-
-func NewHashAppendWriter(w io.Writer, h hash.Hash) *HashAppendWriter {
-	return &HashAppendWriter{
-		h:      h,
-		w:      io.MultiWriter(w, h),
-		origWr: w,
-		sum:    make([]byte, 0, h.Size()),
-	}
-}
-
-func (h *HashAppendWriter) Close() error {
-	if h == nil {
-		return nil
-	}
-
-	if !h.closed {
-		h.closed = true
-
-		_, err := h.origWr.Write(h.h.Sum(nil))
-		return err
-	}
-
-	return nil
-}
-
-func (h *HashAppendWriter) Write(p []byte) (n int, err error) {
-	if !h.closed {
-		return h.w.Write(p)
-	}
-
-	return 0, errors.New("Write() called on closed HashAppendWriter")
-}
-
 type HashingWriter struct {
 	w    io.Writer
 	h    hash.Hash
 	size int
 }
 
+// NewHashAppendWriter wraps the writer w and feeds all data written to the hash h.
 func NewHashingWriter(w io.Writer, h hash.Hash) *HashingWriter {
 	return &HashingWriter{
 		h: h,
@@ -59,16 +20,19 @@ func NewHashingWriter(w io.Writer, h hash.Hash) *HashingWriter {
 	}
 }
 
+// Write wraps the write method of the underlying writer and also hashes all data.
 func (h *HashingWriter) Write(p []byte) (int, error) {
 	n, err := h.w.Write(p)
 	h.size += n
 	return n, err
 }
 
+// Sum returns the hash of all data written so far.
 func (h *HashingWriter) Sum(d []byte) []byte {
 	return h.h.Sum(d)
 }
 
+// Size returns the number of bytes written to the underlying writer.
 func (h *HashingWriter) Size() int {
 	return h.size
 }

backend/writer_test.go

@@ -12,42 +12,6 @@ import (
 	. "github.com/restic/restic/test"
 )
 
-func TestHashAppendWriter(t *testing.T) {
-	tests := []int{5, 23, 2<<18 + 23, 1 << 20}
-
-	for _, size := range tests {
-		data := make([]byte, size)
-		_, err := io.ReadFull(rand.Reader, data)
-		if err != nil {
-			t.Fatalf("ReadFull: %v", err)
-		}
-
-		expectedHash := sha256.Sum256(data)
-
-		target := bytes.NewBuffer(nil)
-		wr := backend.NewHashAppendWriter(target, sha256.New())
-
-		_, err = wr.Write(data)
-		OK(t, err)
-		OK(t, wr.Close())
-
-		Assert(t, len(target.Bytes()) == size+len(expectedHash),
-			"HashAppendWriter: invalid number of bytes written: got %d, expected %d",
-			len(target.Bytes()), size+len(expectedHash))
-
-		r := target.Bytes()
-		resultingHash := r[len(r)-len(expectedHash):]
-		Assert(t, bytes.Equal(expectedHash[:], resultingHash),
-			"HashAppendWriter: hashes do not match: expected %02x, got %02x",
-			expectedHash, resultingHash)
-
-		// write again, this must return an error
-		_, err = wr.Write([]byte{23})
-		Assert(t, err != nil,
-			"HashAppendWriter: Write() after Close() did not return an error")
-	}
-}
-
 func TestHashingWriter(t *testing.T) {
 	tests := []int{5, 23, 2<<18 + 23, 1 << 20}
 

crypto/crypto.go

@@ -233,6 +233,10 @@ var ErrInvalidCiphertext = errors.New("invalid ciphertext, same slice used for p
 // necessary. ciphertext and plaintext may not point to (exactly) the same
 // slice or non-intersecting slices.
 func Encrypt(ks *Key, ciphertext []byte, plaintext []byte) ([]byte, error) {
+	if !ks.Valid() {
+		return nil, errors.New("invalid key")
+	}
+
 	ciphertext = ciphertext[:cap(ciphertext)]
 
 	// test for same slice, if possible
@@ -271,6 +275,10 @@ func Encrypt(ks *Key, ciphertext []byte, plaintext []byte) ([]byte, error) {
 // IV || Ciphertext || MAC. plaintext and ciphertext may point to (exactly) the
 // same slice.
 func Decrypt(ks *Key, plaintext []byte, ciphertextWithMac []byte) ([]byte, error) {
+	if !ks.Valid() {
+		return nil, errors.New("invalid key")
+	}
+
 	// check for plausible length
 	if len(ciphertextWithMac) < ivSize+macSize {
 		panic("trying to decrypt invalid data: ciphertext too small")

repository/decrypt_read_closer.go

@@ -0,0 +1,36 @@
+package repository
+
+import (
+	"io"
+
+	"github.com/restic/restic/crypto"
+)
+
+// decryptReadCloser couples an underlying reader with a DecryptReader and
+// implements io.ReadCloser. On Close(), both readers are closed.
+type decryptReadCloser struct {
+	r  io.ReadCloser
+	dr io.ReadCloser
+}
+
+func newDecryptReadCloser(key *crypto.Key, rd io.ReadCloser) (io.ReadCloser, error) {
+	dr, err := crypto.DecryptFrom(key, rd)
+	if err != nil {
+		return nil, err
+	}
+
+	return &decryptReadCloser{r: rd, dr: dr}, nil
+}
+
+func (dr *decryptReadCloser) Read(buf []byte) (int, error) {
+	return dr.dr.Read(buf)
+}
+
+func (dr *decryptReadCloser) Close() error {
+	err := dr.dr.Close()
+	if err != nil {
+		return err
+	}
+
+	return dr.r.Close()
+}

repository/doc.go

@@ -1,2 +1,28 @@
-// Package repository implements a restic repository on top of a backend.
+// Package repository implements a restic repository on top of a backend. In
+// the following the abstractions used for this package are listed. More
+// information can be found in the restic design document.
+//
+// File
+//
+// A file is a named handle for some data saved in the backend. For the local
+// backend, this corresponds to actual files saved to disk. Usually, the SHA256
+// hash of the content is used for a file's name (hexadecimal, in lower-case
+// ASCII characters). An exception is the file `config`. Most files are
+// encrypted before being saved in a backend. This means that the name is the
+// hash of the ciphertext.
+//
+// Blob
+//
+// A blob is a number of bytes that has a type (data or tree). Blobs are
+// identified by an ID, which is the SHA256 hash of the blobs' contents. One or
+// more blobs are bundled together in a Pack and then saved to the backend.
+// Blobs are always encrypted before being bundled in a Pack.
+//
+// Pack
+//
+// A Pack is a File in the backend that contains one or more (encrypted) blobs,
+// followed by a header at the end of the Pack. The header is encrypted and
+// contains the ID, type, length and offset for each blob contained in the
+// Pack.
+//
 package repository

repository/index.go

@@ -556,7 +556,7 @@ func DecodeOldIndex(rd io.Reader) (idx *Index, err error) {
 }
 
 // LoadIndexWithDecoder loads the index and decodes it with fn.
-func LoadIndexWithDecoder(repo *Repository, id string, fn func(io.Reader) (*Index, error)) (*Index, error) {
+func LoadIndexWithDecoder(repo *Repository, id string, fn func(io.Reader) (*Index, error)) (idx *Index, err error) {
 	debug.Log("LoadIndexWithDecoder", "Loading index %v", id[:8])
 
 	idxID, err := backend.ParseID(id)
@@ -568,9 +568,9 @@ func LoadIndexWithDecoder(repo *Repository, id string, fn func(io.Reader) (*Inde
 	if err != nil {
 		return nil, err
 	}
-	defer rd.Close()
+	defer closeOrErr(rd, &err)
 
-	idx, err := fn(rd)
+	idx, err = fn(rd)
 	if err != nil {
 		debug.Log("LoadIndexWithDecoder", "error while decoding index %v: %v", id, err)
 		return nil, err

repository/index_test.go

@@ -129,7 +129,7 @@ func TestIndexSerialize(t *testing.T) {
 		"index not final after encoding")
 
 	id := randomID()
-	idx.SetID(id)
+	OK(t, idx.SetID(id))
 	id2, err := idx.ID()
 	Assert(t, id2.Equal(id),
 		"wrong ID returned: want %v, got %v", id, id2)

repository/key.go

@@ -112,23 +112,23 @@ func SearchKey(s *Repository, password string) (*Key, error) {
 }
 
 // LoadKey loads a key from the backend.
-func LoadKey(s *Repository, name string) (*Key, error) {
+func LoadKey(s *Repository, name string) (k *Key, err error) {
 	// extract data from repo
 	rd, err := s.be.Get(backend.Key, name)
 	if err != nil {
 		return nil, err
 	}
-	defer rd.Close()
+	defer closeOrErr(rd, &err)
 
 	// restore json
 	dec := json.NewDecoder(rd)
-	k := Key{}
+	k = new(Key)
-	err = dec.Decode(&k)
+	err = dec.Decode(k)
 	if err != nil {
 		return nil, err
 	}
 
-	return &k, nil
+	return k, nil
 }
 
 // AddKey adds a new key to an already existing repository.

repository/packer_manager.go

@@ -0,0 +1,102 @@
+package repository
+
+import (
+	"sync"
+
+	"github.com/restic/chunker"
+	"github.com/restic/restic/backend"
+	"github.com/restic/restic/crypto"
+	"github.com/restic/restic/debug"
+	"github.com/restic/restic/pack"
+)
+
+// packerManager keeps a list of open packs and creates new on demand.
+type packerManager struct {
+	be    backend.Backend
+	key   *crypto.Key
+	pm    sync.Mutex
+	packs []*pack.Packer
+}
+
+const minPackSize = 4 * chunker.MiB
+const maxPackSize = 16 * chunker.MiB
+const maxPackers = 200
+
+// findPacker returns a packer for a new blob of size bytes. Either a new one is
+// created or one is returned that already has some blobs.
+func (r *packerManager) findPacker(size uint) (*pack.Packer, error) {
+	r.pm.Lock()
+	defer r.pm.Unlock()
+
+	// search for a suitable packer
+	if len(r.packs) > 0 {
+		debug.Log("Repo.findPacker", "searching packer for %d bytes\n", size)
+		for i, p := range r.packs {
+			if p.Size()+size < maxPackSize {
+				debug.Log("Repo.findPacker", "found packer %v", p)
+				// remove from list
+				r.packs = append(r.packs[:i], r.packs[i+1:]...)
+				return p, nil
+			}
+		}
+	}
+
+	// no suitable packer found, return new
+	blob, err := r.be.Create()
+	if err != nil {
+		return nil, err
+	}
+	debug.Log("Repo.findPacker", "create new pack %p for %d bytes", blob, size)
+	return pack.NewPacker(r.key, blob), nil
+}
+
+// insertPacker appends p to s.packs.
+func (r *packerManager) insertPacker(p *pack.Packer) {
+	r.pm.Lock()
+	defer r.pm.Unlock()
+
+	r.packs = append(r.packs, p)
+	debug.Log("Repo.insertPacker", "%d packers\n", len(r.packs))
+}
+
+// savePacker stores p in the backend.
+func (r *Repository) savePacker(p *pack.Packer) error {
+	debug.Log("Repo.savePacker", "save packer with %d blobs\n", p.Count())
+	_, err := p.Finalize()
+	if err != nil {
+		return err
+	}
+
+	// move file to the final location
+	sid := p.ID()
+	err = p.Writer().(backend.Blob).Finalize(backend.Data, sid.String())
+	if err != nil {
+		debug.Log("Repo.savePacker", "blob Finalize() error: %v", err)
+		return err
+	}
+
+	debug.Log("Repo.savePacker", "saved as %v", sid.Str())
+
+	// update blobs in the index
+	for _, b := range p.Blobs() {
+		debug.Log("Repo.savePacker", "  updating blob %v to pack %v", b.ID.Str(), sid.Str())
+		r.idx.Current().Store(PackedBlob{
+			Type:   b.Type,
+			ID:     b.ID,
+			PackID: sid,
+			Offset: b.Offset,
+			Length: uint(b.Length),
+		})
+		r.idx.RemoveFromInFlight(b.ID)
+	}
+
+	return nil
+}
+
+// countPacker returns the number of open (unfinished) packers.
+func (r *packerManager) countPacker() int {
+	r.pm.Lock()
+	defer r.pm.Unlock()
+
+	return len(r.packs)
+}

repository/repository.go

@@ -9,9 +9,7 @@ import (
 	"io"
 	"io/ioutil"
 	"os"
-	"sync"
 
-	"github.com/restic/chunker"
 	"github.com/restic/restic/backend"
 	"github.com/restic/restic/crypto"
 	"github.com/restic/restic/debug"
@@ -26,8 +24,7 @@ type Repository struct {
 	keyName string
 	idx     *MasterIndex
 
-	pm    sync.Mutex
+	*packerManager
-	packs []*pack.Packer
 }
 
 // New returns a new repository with backend be.
@@ -35,6 +32,9 @@ func New(be backend.Backend) *Repository {
 	return &Repository{
 		be:  be,
 		idx: NewMasterIndex(),
+		packerManager: &packerManager{
+			be: be,
+		},
 	}
 }
 
@@ -143,19 +143,29 @@ func (r *Repository) LoadBlob(t pack.BlobType, id backend.ID, plaintextBuf []byt
 	return plaintextBuf, nil
 }
 
+// closeOrErr calls cl.Close() and sets err to the returned error value if
+// itself is not yet set.
+func closeOrErr(cl io.Closer, err *error) {
+	e := cl.Close()
+	if *err != nil {
+		return
+	}
+	*err = e
+}
+
 // LoadJSONUnpacked decrypts the data and afterwards calls json.Unmarshal on
 // the item.
-func (r *Repository) LoadJSONUnpacked(t backend.Type, id backend.ID, item interface{}) error {
+func (r *Repository) LoadJSONUnpacked(t backend.Type, id backend.ID, item interface{}) (err error) {
 	// load blob from backend
 	rd, err := r.be.Get(t, id.String())
 	if err != nil {
 		return err
 	}
-	defer rd.Close()
+	defer closeOrErr(rd, &err)
 
 	// decrypt
 	decryptRd, err := crypto.DecryptFrom(r.key, rd)
-	defer decryptRd.Close()
+	defer closeOrErr(decryptRd, &err)
 	if err != nil {
 		return err
 	}
@@ -172,7 +182,7 @@ func (r *Repository) LoadJSONUnpacked(t backend.Type, id backend.ID, item interf
 
 // LoadJSONPack calls LoadBlob() to load a blob from the backend, decrypt the
 // data and afterwards call json.Unmarshal on the item.
-func (r *Repository) LoadJSONPack(t pack.BlobType, id backend.ID, item interface{}) error {
+func (r *Repository) LoadJSONPack(t pack.BlobType, id backend.ID, item interface{}) (err error) {
 	// lookup pack
 	blob, err := r.idx.Lookup(id)
 	if err != nil {
@@ -184,11 +194,11 @@ func (r *Repository) LoadJSONPack(t pack.BlobType, id backend.ID, item interface
 	if err != nil {
 		return err
 	}
-	defer rd.Close()
+	defer closeOrErr(rd, &err)
 
 	// decrypt
 	decryptRd, err := crypto.DecryptFrom(r.key, rd)
-	defer decryptRd.Close()
+	defer closeOrErr(decryptRd, &err)
 	if err != nil {
 		return err
 	}
@@ -208,90 +218,6 @@ func (r *Repository) LookupBlobSize(id backend.ID) (uint, error) {
 	return r.idx.LookupSize(id)
 }
 
-const minPackSize = 4 * chunker.MiB
-const maxPackSize = 16 * chunker.MiB
-const maxPackers = 200
-
-// findPacker returns a packer for a new blob of size bytes. Either a new one is
-// created or one is returned that already has some blobs.
-func (r *Repository) findPacker(size uint) (*pack.Packer, error) {
-	r.pm.Lock()
-	defer r.pm.Unlock()
-
-	// search for a suitable packer
-	if len(r.packs) > 0 {
-		debug.Log("Repo.findPacker", "searching packer for %d bytes\n", size)
-		for i, p := range r.packs {
-			if p.Size()+size < maxPackSize {
-				debug.Log("Repo.findPacker", "found packer %v", p)
-				// remove from list
-				r.packs = append(r.packs[:i], r.packs[i+1:]...)
-				return p, nil
-			}
-		}
-	}
-
-	// no suitable packer found, return new
-	blob, err := r.be.Create()
-	if err != nil {
-		return nil, err
-	}
-	debug.Log("Repo.findPacker", "create new pack %p for %d bytes", blob, size)
-	return pack.NewPacker(r.key, blob), nil
-}
-
-// insertPacker appends p to s.packs.
-func (r *Repository) insertPacker(p *pack.Packer) {
-	r.pm.Lock()
-	defer r.pm.Unlock()
-
-	r.packs = append(r.packs, p)
-	debug.Log("Repo.insertPacker", "%d packers\n", len(r.packs))
-}
-
-// savePacker stores p in the backend.
-func (r *Repository) savePacker(p *pack.Packer) error {
-	debug.Log("Repo.savePacker", "save packer with %d blobs\n", p.Count())
-	_, err := p.Finalize()
-	if err != nil {
-		return err
-	}
-
-	// move file to the final location
-	sid := p.ID()
-	err = p.Writer().(backend.Blob).Finalize(backend.Data, sid.String())
-	if err != nil {
-		debug.Log("Repo.savePacker", "blob Finalize() error: %v", err)
-		return err
-	}
-
-	debug.Log("Repo.savePacker", "saved as %v", sid.Str())
-
-	// update blobs in the index
-	var packedBlobs []PackedBlob
-	for _, b := range p.Blobs() {
-		packedBlobs = append(packedBlobs, PackedBlob{
-			Type:   b.Type,
-			ID:     b.ID,
-			PackID: sid,
-			Offset: b.Offset,
-			Length: uint(b.Length),
-		})
-		r.idx.RemoveFromInFlight(b.ID)
-	}
-	r.idx.Current().StoreBlobs(packedBlobs)
-
-	return nil
-}
-
-// countPacker returns the number of open (unfinished) packers.
-func (r *Repository) countPacker() int {
-	r.pm.Lock()
-	defer r.pm.Unlock()
-
-	return len(r.packs)
-}
-
 // SaveAndEncrypt encrypts data and stores it to the backend as type t. If data is small
 // enough, it will be packed together with other small blobs.
 func (r *Repository) SaveAndEncrypt(t pack.BlobType, data []byte, id *backend.ID) (backend.ID, error) {
@@ -618,35 +544,6 @@ func LoadIndex(repo *Repository, id string) (*Index, error) {
 	return nil, err
 }
 
-// decryptReadCloser couples an underlying reader with a DecryptReader and
-// implements io.ReadCloser. On Close(), both readers are closed.
-type decryptReadCloser struct {
-	r  io.ReadCloser
-	dr io.ReadCloser
-}
-
-func newDecryptReadCloser(key *crypto.Key, rd io.ReadCloser) (io.ReadCloser, error) {
-	dr, err := crypto.DecryptFrom(key, rd)
-	if err != nil {
-		return nil, err
-	}
-
-	return &decryptReadCloser{r: rd, dr: dr}, nil
-}
-
-func (dr *decryptReadCloser) Read(buf []byte) (int, error) {
-	return dr.dr.Read(buf)
-}
-
-func (dr *decryptReadCloser) Close() error {
-	err := dr.dr.Close()
-	if err != nil {
-		return err
-	}
-
-	return dr.r.Close()
-}
-
 // GetDecryptReader opens the file id stored in the backend and returns a
 // reader that yields the decrypted content. The reader must be closed.
 func (r *Repository) GetDecryptReader(t backend.Type, id string) (io.ReadCloser, error) {
@@ -667,6 +564,7 @@ func (r *Repository) SearchKey(password string) error {
 	}
 
 	r.key = key.master
+	r.packerManager.key = key.master
 	r.keyName = key.Name()
 	r.Config, err = LoadConfig(r)
 	return err
@@ -689,6 +587,7 @@ func (r *Repository) Init(password string) error {
 	}
 
 	r.key = key.master
+	r.packerManager.key = key.master
 	r.keyName = key.Name()
 	r.Config, err = CreateConfig(r)
 	return err

test/helpers.go

@@ -4,10 +4,10 @@ import (
 	"bytes"
 	"compress/bzip2"
 	"compress/gzip"
+	"crypto/rand"
 	"fmt"
 	"io"
 	"io/ioutil"
-	"math/rand"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -15,6 +15,8 @@ import (
 	"runtime"
 	"testing"
 
+	mrand "math/rand"
+
 	"github.com/restic/restic/backend"
 	"github.com/restic/restic/backend/local"
 	"github.com/restic/restic/repository"
@@ -76,7 +78,7 @@ func ParseID(s string) backend.ID {
 func Random(seed, count int) []byte {
 	buf := make([]byte, count)
 
-	rnd := rand.New(rand.NewSource(int64(seed)))
+	rnd := mrand.New(mrand.NewSource(int64(seed)))
 	for i := 0; i < count; i++ {
 		buf[i] = byte(rnd.Uint32())
 	}
@@ -90,6 +92,14 @@ func RandomReader(seed, size int) *bytes.Reader {
 	return bytes.NewReader(Random(seed, size))
 }
 
+// GenRandom returns a []byte filled with up to 1000 random bytes.
+func GenRandom(t testing.TB) []byte {
+	buf := make([]byte, mrand.Intn(1000))
+	_, err := io.ReadFull(rand.Reader, buf)
+	OK(t, err)
+	return buf
+}
+
 // SetupTarTestFixture extracts the tarFile to outputDir.
 func SetupTarTestFixture(t testing.TB, outputDir, tarFile string) {
 	input, err := os.Open(tarFile)