KurlesHS/restic
1
0
Fork 0
forked from TrueCloudLab/restic
Code Pull requests Activity

Merge pull request #362 from restic/improvements

Browse source 
Cleanups
This commit is contained in:
Alexander Neumann 2015-11-30 22:09:24 +01:00
commit 04cd318f6c
15 changed files with 593 additions and 212 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
backend/backend_test.go
View file

@ -12,7 +12,34 @@ import (
. "github.com/restic/restic/test"
)
func testBackendConfig(b backend.Backend, t *testing.T) {
// create config and read it back
_, err := b.Get(backend.Config, "")
Assert(t, err != nil, "did not get expected error for non-existing config")
blob, err := b.Create()
OK(t, err)
_, err = blob.Write([]byte("Config"))
OK(t, err)
OK(t, blob.Finalize(backend.Config, ""))
// try accessing the config with different names, should all return the
// same config
for _, name := range []string{"", "foo", "bar", "0000000000000000000000000000000000000000000000000000000000000000"} {
rd, err := b.Get(backend.Config, name)
Assert(t, err == nil, "unable to read config")
buf, err := ioutil.ReadAll(rd)
OK(t, err)
OK(t, rd.Close())
Assert(t, string(buf) == "Config", "wrong data returned for config")
}
}
func testBackend(b backend.Backend, t *testing.T) {
testBackendConfig(b, t)
for _, tpe := range []backend.Type{
backend.Data, backend.Key, backend.Lock,
backend.Snapshot, backend.Index,
@ -96,6 +123,11 @@ func testBackend(b backend.Backend, t *testing.T) {
err = b.Remove(tpe, test.id)
OK(t, err)
// test that the blob is gone
ok, err := b.Test(tpe, test.id)
OK(t, err)
Assert(t, ok == false, "removed blob still present")
// create blob
blob, err = b.Create()
OK(t, err)

231
backend/mem_backend.go Normal file
View file

@ -0,0 +1,231 @@
package backend
import (
"bytes"
"errors"
"io"
"sort"
"sync"
"github.com/restic/restic/debug"
)
type entry struct {
Type Type
Name string
}
type memMap map[entry][]byte
// MemoryBackend is a mock backend that uses a map for storing all data in
// memory. This should only be used for tests.
type MemoryBackend struct {
data memMap
m sync.Mutex
MockBackend
}
// NewMemoryBackend returns a new backend that saves all data in a map in
// memory.
func NewMemoryBackend() *MemoryBackend {
be := &MemoryBackend{
data: make(memMap),
}
be.MockBackend.TestFn = func(t Type, name string) (bool, error) {
return memTest(be, t, name)
}
be.MockBackend.CreateFn = func() (Blob, error) {
return memCreate(be)
}
be.MockBackend.GetFn = func(t Type, name string) (io.ReadCloser, error) {
return memGet(be, t, name)
}
be.MockBackend.GetReaderFn = func(t Type, name string, offset, length uint) (io.ReadCloser, error) {
return memGetReader(be, t, name, offset, length)
}
be.MockBackend.RemoveFn = func(t Type, name string) error {
return memRemove(be, t, name)
}
be.MockBackend.ListFn = func(t Type, done <-chan struct{}) <-chan string {
return memList(be, t, done)
}
be.MockBackend.DeleteFn = func() error {
be.m.Lock()
defer be.m.Unlock()
be.data = make(memMap)
return nil
}
debug.Log("MemoryBackend.New", "created new memory backend")
return be
}
func (be *MemoryBackend) insert(t Type, name string, data []byte) error {
be.m.Lock()
defer be.m.Unlock()
if _, ok := be.data[entry{t, name}]; ok {
return errors.New("already present")
}
be.data[entry{t, name}] = data
return nil
}
func memTest(be *MemoryBackend, t Type, name string) (bool, error) {
be.m.Lock()
defer be.m.Unlock()
debug.Log("MemoryBackend.Test", "test %v %v", t, name)
if _, ok := be.data[entry{t, name}]; ok {
return true, nil
}
return false, nil
}
// tempMemEntry temporarily holds data written to the memory backend before it
// is finalized.
type tempMemEntry struct {
be *MemoryBackend
data bytes.Buffer
}
func (e *tempMemEntry) Write(p []byte) (int, error) {
return e.data.Write(p)
}
func (e *tempMemEntry) Size() uint {
return uint(len(e.data.Bytes()))
}
func (e *tempMemEntry) Finalize(t Type, name string) error {
if t == Config {
name = ""
}
debug.Log("MemoryBackend", "save blob %p (%d bytes) as %v %v", e, len(e.data.Bytes()), t, name)
return e.be.insert(t, name, e.data.Bytes())
}
func memCreate(be *MemoryBackend) (Blob, error) {
blob := &tempMemEntry{be: be}
debug.Log("MemoryBackend.Create", "create new blob %p", blob)
return blob, nil
}
// readCloser wraps a reader and adds a noop Close method.
type readCloser struct {
io.Reader
}
func (rd readCloser) Close() error {
return nil
}
func memGet(be *MemoryBackend, t Type, name string) (io.ReadCloser, error) {
be.m.Lock()
defer be.m.Unlock()
if t == Config {
name = ""
}
debug.Log("MemoryBackend.Get", "get %v %v", t, name)
if _, ok := be.data[entry{t, name}]; !ok {
return nil, errors.New("no such data")
}
return readCloser{bytes.NewReader(be.data[entry{t, name}])}, nil
}
func memGetReader(be *MemoryBackend, t Type, name string, offset, length uint) (io.ReadCloser, error) {
be.m.Lock()
defer be.m.Unlock()
if t == Config {
name = ""
}
debug.Log("MemoryBackend.GetReader", "get %v %v offset %v len %v", t, name, offset, length)
if _, ok := be.data[entry{t, name}]; !ok {
return nil, errors.New("no such data")
}
buf := be.data[entry{t, name}]
if offset > uint(len(buf)) {
return nil, errors.New("offset beyond end of file")
}
buf = buf[offset:]
if length > 0 {
if length > uint(len(buf)) {
length = uint(len(buf))
}
buf = buf[:length]
}
return readCloser{bytes.NewReader(buf)}, nil
}
func memRemove(be *MemoryBackend, t Type, name string) error {
be.m.Lock()
defer be.m.Unlock()
debug.Log("MemoryBackend.Remove", "get %v %v", t, name)
if _, ok := be.data[entry{t, name}]; !ok {
return errors.New("no such data")
}
delete(be.data, entry{t, name})
return nil
}
func memList(be *MemoryBackend, t Type, done <-chan struct{}) <-chan string {
be.m.Lock()
defer be.m.Unlock()
ch := make(chan string)
var ids []string
for entry := range be.data {
if entry.Type != t {
continue
}
ids = append(ids, entry.Name)
}
sort.Strings(ids)
debug.Log("MemoryBackend.List", "list %v: %v", t, ids)
go func() {
defer close(ch)
for _, id := range ids {
select {
case ch <- id:
case <-done:
return
}
}
}()
return ch
}

12
backend/mem_backend_test.go Normal file
View file

@ -0,0 +1,12 @@
package backend_test
import (
"testing"
"github.com/restic/restic/backend"
)
func TestMemoryBackend(t *testing.T) {
be := backend.NewMemoryBackend()
testBackend(be, t)
}

97
backend/mock_backend.go Normal file
View file

@ -0,0 +1,97 @@
package backend
import (
"errors"
"io"
)
// MockBackend implements a backend whose functions can be specified. This
// should only be used for tests.
type MockBackend struct {
CloseFn func() error
CreateFn func() (Blob, error)
GetFn func(Type, string) (io.ReadCloser, error)
GetReaderFn func(Type, string, uint, uint) (io.ReadCloser, error)
ListFn func(Type, <-chan struct{}) <-chan string
RemoveFn func(Type, string) error
TestFn func(Type, string) (bool, error)
DeleteFn func() error
LocationFn func() string
}
func (m *MockBackend) Close() error {
if m.CloseFn == nil {
return nil
}
return m.CloseFn()
}
func (m *MockBackend) Location() string {
if m.LocationFn == nil {
return ""
}
return m.LocationFn()
}
func (m *MockBackend) Create() (Blob, error) {
if m.CreateFn == nil {
return nil, errors.New("not implemented")
}
return m.CreateFn()
}
func (m *MockBackend) Get(t Type, name string) (io.ReadCloser, error) {
if m.GetFn == nil {
return nil, errors.New("not implemented")
}
return m.GetFn(t, name)
}
func (m *MockBackend) GetReader(t Type, name string, offset, len uint) (io.ReadCloser, error) {
if m.GetReaderFn == nil {
return nil, errors.New("not implemented")
}
return m.GetReaderFn(t, name, offset, len)
}
func (m *MockBackend) List(t Type, done <-chan struct{}) <-chan string {
if m.ListFn == nil {
ch := make(chan string)
close(ch)
return ch
}
return m.ListFn(t, done)
}
func (m *MockBackend) Remove(t Type, name string) error {
if m.RemoveFn == nil {
return errors.New("not implemented")
}
return m.RemoveFn(t, name)
}
func (m *MockBackend) Test(t Type, name string) (bool, error) {
if m.TestFn == nil {
return false, errors.New("not implemented")
}
return m.TestFn(t, name)
}
func (m *MockBackend) Delete() error {
if m.DeleteFn == nil {
return errors.New("not implemented")
}
return m.DeleteFn()
}
// Make sure that MockBackend implements the backend interface.
var _ Backend = &MockBackend{}

46
backend/writer.go
View file

@ -1,57 +1,18 @@
package backend
import (
"errors"
"hash"
"io"
)
type HashAppendWriter struct {
w io.Writer
origWr io.Writer
h hash.Hash
sum []byte
closed bool
}
func NewHashAppendWriter(w io.Writer, h hash.Hash) *HashAppendWriter {
return &HashAppendWriter{
h: h,
w: io.MultiWriter(w, h),
origWr: w,
sum: make([]byte, 0, h.Size()),
}
}
func (h *HashAppendWriter) Close() error {
if h == nil {
return nil
}
if !h.closed {
h.closed = true
_, err := h.origWr.Write(h.h.Sum(nil))
return err
}
return nil
}
func (h *HashAppendWriter) Write(p []byte) (n int, err error) {
if !h.closed {
return h.w.Write(p)
}
return 0, errors.New("Write() called on closed HashAppendWriter")
}
// HashingWriter wraps an io.Writer to hashes all data that is written to it.
type HashingWriter struct {
w io.Writer
h hash.Hash
size int
}
// NewHashAppendWriter wraps the writer w and feeds all data written to the hash h.
func NewHashingWriter(w io.Writer, h hash.Hash) *HashingWriter {
return &HashingWriter{
h: h,
@ -59,16 +20,19 @@ func NewHashingWriter(w io.Writer, h hash.Hash) *HashingWriter {
}
}
// Write wraps the write method of the underlying writer and also hashes all data.
func (h *HashingWriter) Write(p []byte) (int, error) {
n, err := h.w.Write(p)
h.size += n
return n, err
}
// Sum returns the hash of all data written so far.
func (h *HashingWriter) Sum(d []byte) []byte {
return h.h.Sum(d)
}
// Size returns the number of bytes written to the underlying writer.
func (h *HashingWriter) Size() int {
return h.size
}

36
backend/writer_test.go
View file

@ -12,42 +12,6 @@ import (
. "github.com/restic/restic/test"
)
func TestHashAppendWriter(t *testing.T) {
tests := []int{5, 23, 2<<18 + 23, 1 << 20}
for _, size := range tests {
data := make([]byte, size)
_, err := io.ReadFull(rand.Reader, data)
if err != nil {
t.Fatalf("ReadFull: %v", err)
}
expectedHash := sha256.Sum256(data)
target := bytes.NewBuffer(nil)
wr := backend.NewHashAppendWriter(target, sha256.New())
_, err = wr.Write(data)
OK(t, err)
OK(t, wr.Close())
Assert(t, len(target.Bytes()) == size+len(expectedHash),
"HashAppendWriter: invalid number of bytes written: got %d, expected %d",
len(target.Bytes()), size+len(expectedHash))
r := target.Bytes()
resultingHash := r[len(r)-len(expectedHash):]
Assert(t, bytes.Equal(expectedHash[:], resultingHash),
"HashAppendWriter: hashes do not match: expected %02x, got %02x",
expectedHash, resultingHash)
// write again, this must return an error
_, err = wr.Write([]byte{23})
Assert(t, err != nil,
"HashAppendWriter: Write() after Close() did not return an error")
}
}
func TestHashingWriter(t *testing.T) {
tests := []int{5, 23, 2<<18 + 23, 1 << 20}

8
crypto/crypto.go
View file

@ -233,6 +233,10 @@ var ErrInvalidCiphertext = errors.New("invalid ciphertext, same slice used for p
// necessary. ciphertext and plaintext may not point to (exactly) the same
// slice or non-intersecting slices.
func Encrypt(ks *Key, ciphertext []byte, plaintext []byte) ([]byte, error) {
if !ks.Valid() {
return nil, errors.New("invalid key")
}
ciphertext = ciphertext[:cap(ciphertext)]
// test for same slice, if possible
@ -271,6 +275,10 @@ func Encrypt(ks *Key, ciphertext []byte, plaintext []byte) ([]byte, error) {
// IV || Ciphertext || MAC. plaintext and ciphertext may point to (exactly) the
// same slice.
func Decrypt(ks *Key, plaintext []byte, ciphertextWithMac []byte) ([]byte, error) {
if !ks.Valid() {
return nil, errors.New("invalid key")
}
// check for plausible length
if len(ciphertextWithMac) < ivSize+macSize {
panic("trying to decrypt invalid data: ciphertext too small")

36
repository/decrypt_read_closer.go Normal file
View file

@ -0,0 +1,36 @@
package repository
import (
"io"
"github.com/restic/restic/crypto"
)
// decryptReadCloser couples an underlying reader with a DecryptReader and
// implements io.ReadCloser. On Close(), both readers are closed.
type decryptReadCloser struct {
r io.ReadCloser
dr io.ReadCloser
}
func newDecryptReadCloser(key *crypto.Key, rd io.ReadCloser) (io.ReadCloser, error) {
dr, err := crypto.DecryptFrom(key, rd)
if err != nil {
return nil, err
}
return &decryptReadCloser{r: rd, dr: dr}, nil
}
func (dr *decryptReadCloser) Read(buf []byte) (int, error) {
return dr.dr.Read(buf)
}
func (dr *decryptReadCloser) Close() error {
err := dr.dr.Close()
if err != nil {
return err
}
return dr.r.Close()
}

28
repository/doc.go
View file

@ -1,2 +1,28 @@
// Package repository implements a restic repository on top of a backend.
// Package repository implements a restic repository on top of a backend. In
// the following the abstractions used for this package are listed. More
// information can be found in the restic design document.
//
// File
//
// A file is a named handle for some data saved in the backend. For the local
// backend, this corresponds to actual files saved to disk. Usually, the SHA256
// hash of the content is used for a file's name (hexadecimal, in lower-case
// ASCII characters). An exception is the file `config`. Most files are
// encrypted before being saved in a backend. This means that the name is the
// hash of the ciphertext.
//
// Blob
//
// A blob is a number of bytes that has a type (data or tree). Blobs are
// identified by an ID, which is the SHA256 hash of the blobs' contents. One or
// more blobs are bundled together in a Pack and then saved to the backend.
// Blobs are always encrypted before being bundled in a Pack.
//
// Pack
//
// A Pack is a File in the backend that contains one or more (encrypted) blobs,
// followed by a header at the end of the Pack. The header is encrypted and
// contains the ID, type, length and offset for each blob contained in the
// Pack.
//
package repository

6
repository/index.go
View file

@ -556,7 +556,7 @@ func DecodeOldIndex(rd io.Reader) (idx *Index, err error) {
}
// LoadIndexWithDecoder loads the index and decodes it with fn.
func LoadIndexWithDecoder(repo *Repository, id string, fn func(io.Reader) (*Index, error)) (*Index, error) {
func LoadIndexWithDecoder(repo *Repository, id string, fn func(io.Reader) (*Index, error)) (idx *Index, err error) {
debug.Log("LoadIndexWithDecoder", "Loading index %v", id[:8])
idxID, err := backend.ParseID(id)
@ -568,9 +568,9 @@ func LoadIndexWithDecoder(repo *Repository, id string, fn func(io.Reader) (*Inde
if err != nil {
return nil, err
}
defer rd.Close()
defer closeOrErr(rd, &err)
idx, err := fn(rd)
idx, err = fn(rd)
if err != nil {
debug.Log("LoadIndexWithDecoder", "error while decoding index %v: %v", id, err)
return nil, err

2
repository/index_test.go
View file

@ -129,7 +129,7 @@ func TestIndexSerialize(t *testing.T) {
"index not final after encoding")
id := randomID()
idx.SetID(id)
OK(t, idx.SetID(id))
id2, err := idx.ID()
Assert(t, id2.Equal(id),
"wrong ID returned: want %v, got %v", id, id2)

10
repository/key.go
View file

@ -112,23 +112,23 @@ func SearchKey(s *Repository, password string) (*Key, error) {
}
// LoadKey loads a key from the backend.
func LoadKey(s *Repository, name string) (*Key, error) {
func LoadKey(s *Repository, name string) (k *Key, err error) {
// extract data from repo
rd, err := s.be.Get(backend.Key, name)
if err != nil {
return nil, err
}
defer rd.Close()
defer closeOrErr(rd, &err)
// restore json
dec := json.NewDecoder(rd)
k := Key{}
err = dec.Decode(&k)
k = new(Key)
err = dec.Decode(k)
if err != nil {
return nil, err
}
return &k, nil
return k, nil
}
// AddKey adds a new key to an already existing repository.

102
repository/packer_manager.go Normal file
View file

@ -0,0 +1,102 @@
package repository
import (
"sync"
"github.com/restic/chunker"
"github.com/restic/restic/backend"
"github.com/restic/restic/crypto"
"github.com/restic/restic/debug"
"github.com/restic/restic/pack"
)
// packerManager keeps a list of open packs and creates new on demand.
type packerManager struct {
be backend.Backend
key *crypto.Key
pm sync.Mutex
packs []*pack.Packer
}
const minPackSize = 4 * chunker.MiB
const maxPackSize = 16 * chunker.MiB
const maxPackers = 200
// findPacker returns a packer for a new blob of size bytes. Either a new one is
// created or one is returned that already has some blobs.
func (r *packerManager) findPacker(size uint) (*pack.Packer, error) {
r.pm.Lock()
defer r.pm.Unlock()
// search for a suitable packer
if len(r.packs) > 0 {
debug.Log("Repo.findPacker", "searching packer for %d bytes\n", size)
for i, p := range r.packs {
if p.Size()+size < maxPackSize {
debug.Log("Repo.findPacker", "found packer %v", p)
// remove from list
r.packs = append(r.packs[:i], r.packs[i+1:]...)
return p, nil
}
}
}
// no suitable packer found, return new
blob, err := r.be.Create()
if err != nil {
return nil, err
}
debug.Log("Repo.findPacker", "create new pack %p for %d bytes", blob, size)
return pack.NewPacker(r.key, blob), nil
}
// insertPacker appends p to s.packs.
func (r *packerManager) insertPacker(p *pack.Packer) {
r.pm.Lock()
defer r.pm.Unlock()
r.packs = append(r.packs, p)
debug.Log("Repo.insertPacker", "%d packers\n", len(r.packs))
}
// savePacker stores p in the backend.
func (r *Repository) savePacker(p *pack.Packer) error {
debug.Log("Repo.savePacker", "save packer with %d blobs\n", p.Count())
_, err := p.Finalize()
if err != nil {
return err
}
// move file to the final location
sid := p.ID()
err = p.Writer().(backend.Blob).Finalize(backend.Data, sid.String())
if err != nil {
debug.Log("Repo.savePacker", "blob Finalize() error: %v", err)
return err
}
debug.Log("Repo.savePacker", "saved as %v", sid.Str())
// update blobs in the index
for _, b := range p.Blobs() {
debug.Log("Repo.savePacker", " updating blob %v to pack %v", b.ID.Str(), sid.Str())
r.idx.Current().Store(PackedBlob{
Type: b.Type,
ID: b.ID,
PackID: sid,
Offset: b.Offset,
Length: uint(b.Length),
})
r.idx.RemoveFromInFlight(b.ID)
}
return nil
}
// countPacker returns the number of open (unfinished) packers.
func (r *packerManager) countPacker() int {
r.pm.Lock()
defer r.pm.Unlock()
return len(r.packs)
}

145
repository/repository.go
View file

@ -9,9 +9,7 @@ import (
"io"
"io/ioutil"
"os"
"sync"
"github.com/restic/chunker"
"github.com/restic/restic/backend"
"github.com/restic/restic/crypto"
"github.com/restic/restic/debug"
@ -26,8 +24,7 @@ type Repository struct {
keyName string
idx *MasterIndex
pm sync.Mutex
packs []*pack.Packer
*packerManager
}
// New returns a new repository with backend be.
@ -35,6 +32,9 @@ func New(be backend.Backend) *Repository {
return &Repository{
be: be,
idx: NewMasterIndex(),
packerManager: &packerManager{
be: be,
},
}
}
@ -143,19 +143,29 @@ func (r *Repository) LoadBlob(t pack.BlobType, id backend.ID, plaintextBuf []byt
return plaintextBuf, nil
}
// closeOrErr calls cl.Close() and sets err to the returned error value if
// itself is not yet set.
func closeOrErr(cl io.Closer, err *error) {
e := cl.Close()
if *err != nil {
return
}
*err = e
}
// LoadJSONUnpacked decrypts the data and afterwards calls json.Unmarshal on
// the item.
func (r *Repository) LoadJSONUnpacked(t backend.Type, id backend.ID, item interface{}) error {
func (r *Repository) LoadJSONUnpacked(t backend.Type, id backend.ID, item interface{}) (err error) {
// load blob from backend
rd, err := r.be.Get(t, id.String())
if err != nil {
return err
}
defer rd.Close()
defer closeOrErr(rd, &err)
// decrypt
decryptRd, err := crypto.DecryptFrom(r.key, rd)
defer decryptRd.Close()
defer closeOrErr(decryptRd, &err)
if err != nil {
return err
}
@ -172,7 +182,7 @@ func (r *Repository) LoadJSONUnpacked(t backend.Type, id backend.ID, item interf
// LoadJSONPack calls LoadBlob() to load a blob from the backend, decrypt the
// data and afterwards call json.Unmarshal on the item.
func (r *Repository) LoadJSONPack(t pack.BlobType, id backend.ID, item interface{}) error {
func (r *Repository) LoadJSONPack(t pack.BlobType, id backend.ID, item interface{}) (err error) {
// lookup pack
blob, err := r.idx.Lookup(id)
if err != nil {
@ -184,11 +194,11 @@ func (r *Repository) LoadJSONPack(t pack.BlobType, id backend.ID, item interface
if err != nil {
return err
}
defer rd.Close()
defer closeOrErr(rd, &err)
// decrypt
decryptRd, err := crypto.DecryptFrom(r.key, rd)
defer decryptRd.Close()
defer closeOrErr(decryptRd, &err)
if err != nil {
return err
}
@ -208,90 +218,6 @@ func (r *Repository) LookupBlobSize(id backend.ID) (uint, error) {
return r.idx.LookupSize(id)
}
const minPackSize = 4 * chunker.MiB
const maxPackSize = 16 * chunker.MiB
const maxPackers = 200
// findPacker returns a packer for a new blob of size bytes. Either a new one is
// created or one is returned that already has some blobs.
func (r *Repository) findPacker(size uint) (*pack.Packer, error) {
r.pm.Lock()
defer r.pm.Unlock()
// search for a suitable packer
if len(r.packs) > 0 {
debug.Log("Repo.findPacker", "searching packer for %d bytes\n", size)
for i, p := range r.packs {
if p.Size()+size < maxPackSize {
debug.Log("Repo.findPacker", "found packer %v", p)
// remove from list
r.packs = append(r.packs[:i], r.packs[i+1:]...)
return p, nil
}
}
}
// no suitable packer found, return new
blob, err := r.be.Create()
if err != nil {
return nil, err
}
debug.Log("Repo.findPacker", "create new pack %p for %d bytes", blob, size)
return pack.NewPacker(r.key, blob), nil
}
// insertPacker appends p to s.packs.
func (r *Repository) insertPacker(p *pack.Packer) {
r.pm.Lock()
defer r.pm.Unlock()
r.packs = append(r.packs, p)
debug.Log("Repo.insertPacker", "%d packers\n", len(r.packs))
}
// savePacker stores p in the backend.
func (r *Repository) savePacker(p *pack.Packer) error {
debug.Log("Repo.savePacker", "save packer with %d blobs\n", p.Count())
_, err := p.Finalize()
if err != nil {
return err
}
// move file to the final location
sid := p.ID()
err = p.Writer().(backend.Blob).Finalize(backend.Data, sid.String())
if err != nil {
debug.Log("Repo.savePacker", "blob Finalize() error: %v", err)
return err
}
debug.Log("Repo.savePacker", "saved as %v", sid.Str())
// update blobs in the index
var packedBlobs []PackedBlob
for _, b := range p.Blobs() {
packedBlobs = append(packedBlobs, PackedBlob{
Type: b.Type,
ID: b.ID,
PackID: sid,
Offset: b.Offset,
Length: uint(b.Length),
})
r.idx.RemoveFromInFlight(b.ID)
}
r.idx.Current().StoreBlobs(packedBlobs)
return nil
}
// countPacker returns the number of open (unfinished) packers.
func (r *Repository) countPacker() int {
r.pm.Lock()
defer r.pm.Unlock()
return len(r.packs)
}
// SaveAndEncrypt encrypts data and stores it to the backend as type t. If data is small
// enough, it will be packed together with other small blobs.
func (r *Repository) SaveAndEncrypt(t pack.BlobType, data []byte, id *backend.ID) (backend.ID, error) {
@ -618,35 +544,6 @@ func LoadIndex(repo *Repository, id string) (*Index, error) {
return nil, err
}
// decryptReadCloser couples an underlying reader with a DecryptReader and
// implements io.ReadCloser. On Close(), both readers are closed.
type decryptReadCloser struct {
r io.ReadCloser
dr io.ReadCloser
}
func newDecryptReadCloser(key *crypto.Key, rd io.ReadCloser) (io.ReadCloser, error) {
dr, err := crypto.DecryptFrom(key, rd)
if err != nil {
return nil, err
}
return &decryptReadCloser{r: rd, dr: dr}, nil
}
func (dr *decryptReadCloser) Read(buf []byte) (int, error) {
return dr.dr.Read(buf)
}
func (dr *decryptReadCloser) Close() error {
err := dr.dr.Close()
if err != nil {
return err
}
return dr.r.Close()
}
// GetDecryptReader opens the file id stored in the backend and returns a
// reader that yields the decrypted content. The reader must be closed.
func (r *Repository) GetDecryptReader(t backend.Type, id string) (io.ReadCloser, error) {
@ -667,6 +564,7 @@ func (r *Repository) SearchKey(password string) error {
}
r.key = key.master
r.packerManager.key = key.master
r.keyName = key.Name()
r.Config, err = LoadConfig(r)
return err
@ -689,6 +587,7 @@ func (r *Repository) Init(password string) error {
}
r.key = key.master
r.packerManager.key = key.master
r.keyName = key.Name()
r.Config, err = CreateConfig(r)
return err

14
test/helpers.go
View file

@ -4,10 +4,10 @@ import (
"bytes"
"compress/bzip2"
"compress/gzip"
"crypto/rand"
"fmt"
"io"
"io/ioutil"
"math/rand"
"os"
"os/exec"
"path/filepath"
@ -15,6 +15,8 @@ import (
"runtime"
"testing"
mrand "math/rand"
"github.com/restic/restic/backend"
"github.com/restic/restic/backend/local"
"github.com/restic/restic/repository"
@ -76,7 +78,7 @@ func ParseID(s string) backend.ID {
func Random(seed, count int) []byte {
buf := make([]byte, count)
rnd := rand.New(rand.NewSource(int64(seed)))
rnd := mrand.New(mrand.NewSource(int64(seed)))
for i := 0; i < count; i++ {
buf[i] = byte(rnd.Uint32())
}
@ -90,6 +92,14 @@ func RandomReader(seed, size int) *bytes.Reader {
return bytes.NewReader(Random(seed, size))
}
// GenRandom returns a []byte filled with up to 1000 random bytes.
func GenRandom(t testing.TB) []byte {
buf := make([]byte, mrand.Intn(1000))
_, err := io.ReadFull(rand.Reader, buf)
OK(t, err)
return buf
}
// SetupTarTestFixture extracts the tarFile to outputDir.
func SetupTarTestFixture(t testing.TB, outputDir, tarFile string) {
input, err := os.Open(tarFile)