2015-04-26 13:36:49 +00:00
|
|
|
package pack
|
|
|
|
|
|
|
|
import (
|
2016-01-24 18:30:14 +00:00
|
|
|
"bytes"
|
2015-04-26 13:36:49 +00:00
|
|
|
"encoding/binary"
|
|
|
|
"errors"
|
|
|
|
"fmt"
|
|
|
|
"io"
|
|
|
|
"sync"
|
|
|
|
|
2016-02-14 14:29:28 +00:00
|
|
|
"restic/backend"
|
|
|
|
"restic/crypto"
|
2015-04-26 13:36:49 +00:00
|
|
|
)
|
|
|
|
|
2016-01-24 18:30:14 +00:00
|
|
|
// BlobType specifies what a blob stored in a pack is.
|
2015-04-26 13:36:49 +00:00
|
|
|
type BlobType uint8
|
|
|
|
|
2016-01-24 18:30:14 +00:00
|
|
|
// These are the blob types that can be stored in a pack.
|
2015-04-26 13:36:49 +00:00
|
|
|
const (
|
2016-08-03 20:38:05 +00:00
|
|
|
Invalid BlobType = iota
|
|
|
|
Data
|
|
|
|
Tree
|
2015-04-26 13:36:49 +00:00
|
|
|
)
|
|
|
|
|
2015-04-26 15:44:38 +00:00
|
|
|
func (t BlobType) String() string {
|
|
|
|
switch t {
|
|
|
|
case Data:
|
|
|
|
return "data"
|
|
|
|
case Tree:
|
|
|
|
return "tree"
|
|
|
|
}
|
|
|
|
|
|
|
|
return fmt.Sprintf("<BlobType %d>", t)
|
|
|
|
}
|
|
|
|
|
2016-01-24 18:30:14 +00:00
|
|
|
// MarshalJSON encodes the BlobType into JSON.
|
2015-04-26 13:36:49 +00:00
|
|
|
func (t BlobType) MarshalJSON() ([]byte, error) {
|
|
|
|
switch t {
|
|
|
|
case Data:
|
|
|
|
return []byte(`"data"`), nil
|
|
|
|
case Tree:
|
|
|
|
return []byte(`"tree"`), nil
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil, errors.New("unknown blob type")
|
|
|
|
}
|
|
|
|
|
2016-01-24 18:30:14 +00:00
|
|
|
// UnmarshalJSON decodes the BlobType from JSON.
|
2015-04-26 13:36:49 +00:00
|
|
|
func (t *BlobType) UnmarshalJSON(buf []byte) error {
|
|
|
|
switch string(buf) {
|
|
|
|
case `"data"`:
|
|
|
|
*t = Data
|
|
|
|
case `"tree"`:
|
|
|
|
*t = Tree
|
|
|
|
default:
|
|
|
|
return errors.New("unknown blob type")
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Blob is a blob within a pack.
|
|
|
|
type Blob struct {
|
|
|
|
Type BlobType
|
2015-08-08 11:47:08 +00:00
|
|
|
Length uint
|
2015-04-26 13:36:49 +00:00
|
|
|
ID backend.ID
|
|
|
|
Offset uint
|
|
|
|
}
|
|
|
|
|
|
|
|
// GetReader returns an io.Reader for the blob entry e.
|
|
|
|
func (e Blob) GetReader(rd io.ReadSeeker) (io.Reader, error) {
|
|
|
|
// seek to the correct location
|
|
|
|
_, err := rd.Seek(int64(e.Offset), 0)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
return io.LimitReader(rd, int64(e.Length)), nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Packer is used to create a new Pack.
|
|
|
|
type Packer struct {
|
|
|
|
blobs []Blob
|
|
|
|
|
|
|
|
bytes uint
|
|
|
|
k *crypto.Key
|
2016-03-06 11:26:25 +00:00
|
|
|
wr io.Writer
|
2015-04-26 13:36:49 +00:00
|
|
|
|
|
|
|
m sync.Mutex
|
|
|
|
}
|
|
|
|
|
|
|
|
// NewPacker returns a new Packer that can be used to pack blobs
|
2016-03-06 11:26:25 +00:00
|
|
|
// together. If wr is nil, a bytes.Buffer is used.
|
|
|
|
func NewPacker(k *crypto.Key, wr io.Writer) *Packer {
|
2016-03-06 11:34:23 +00:00
|
|
|
if wr == nil {
|
|
|
|
wr = bytes.NewBuffer(nil)
|
|
|
|
}
|
2016-03-06 11:26:25 +00:00
|
|
|
return &Packer{k: k, wr: wr}
|
2015-04-26 13:36:49 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Add saves the data read from rd as a new blob to the packer. Returned is the
|
|
|
|
// number of bytes written to the pack.
|
2016-03-06 11:26:25 +00:00
|
|
|
func (p *Packer) Add(t BlobType, id backend.ID, data []byte) (int, error) {
|
2015-04-26 13:36:49 +00:00
|
|
|
p.m.Lock()
|
|
|
|
defer p.m.Unlock()
|
|
|
|
|
|
|
|
c := Blob{Type: t, ID: id}
|
|
|
|
|
2016-03-06 11:26:25 +00:00
|
|
|
n, err := p.wr.Write(data)
|
2015-08-08 11:47:08 +00:00
|
|
|
c.Length = uint(n)
|
2015-04-26 13:36:49 +00:00
|
|
|
c.Offset = p.bytes
|
|
|
|
p.bytes += uint(n)
|
|
|
|
p.blobs = append(p.blobs, c)
|
|
|
|
|
|
|
|
return n, err
|
|
|
|
}
|
|
|
|
|
2015-04-29 22:19:56 +00:00
|
|
|
var entrySize = uint(binary.Size(BlobType(0)) + binary.Size(uint32(0)) + backend.IDSize)
|
2015-04-26 13:36:49 +00:00
|
|
|
|
|
|
|
// headerEntry is used with encoding/binary to read and write header entries
|
|
|
|
type headerEntry struct {
|
|
|
|
Type BlobType
|
|
|
|
Length uint32
|
|
|
|
ID [backend.IDSize]byte
|
|
|
|
}
|
|
|
|
|
|
|
|
// Finalize writes the header for all added blobs and finalizes the pack.
|
2016-03-06 11:26:25 +00:00
|
|
|
// Returned are the number of bytes written, including the header. If the
|
|
|
|
// underlying writer implements io.Closer, it is closed.
|
|
|
|
func (p *Packer) Finalize() (uint, error) {
|
2015-04-26 13:36:49 +00:00
|
|
|
p.m.Lock()
|
|
|
|
defer p.m.Unlock()
|
|
|
|
|
2016-01-24 18:30:14 +00:00
|
|
|
bytesWritten := p.bytes
|
2015-04-26 13:36:49 +00:00
|
|
|
|
2016-01-24 18:30:14 +00:00
|
|
|
hdrBuf := bytes.NewBuffer(nil)
|
|
|
|
bytesHeader, err := p.writeHeader(hdrBuf)
|
2015-04-29 22:36:36 +00:00
|
|
|
if err != nil {
|
2016-03-06 11:26:25 +00:00
|
|
|
return 0, err
|
2015-04-26 13:36:49 +00:00
|
|
|
}
|
|
|
|
|
2016-01-24 18:30:14 +00:00
|
|
|
encryptedHeader, err := crypto.Encrypt(p.k, nil, hdrBuf.Bytes())
|
|
|
|
if err != nil {
|
2016-03-06 11:26:25 +00:00
|
|
|
return 0, err
|
2016-01-24 18:30:14 +00:00
|
|
|
}
|
2015-04-29 22:41:11 +00:00
|
|
|
|
2016-01-24 18:30:14 +00:00
|
|
|
// append the header
|
2016-03-06 11:26:25 +00:00
|
|
|
n, err := p.wr.Write(encryptedHeader)
|
2015-04-26 13:36:49 +00:00
|
|
|
if err != nil {
|
2016-03-06 11:26:25 +00:00
|
|
|
return 0, err
|
2016-01-24 18:30:14 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
hdrBytes := bytesHeader + crypto.Extension
|
|
|
|
if uint(n) != hdrBytes {
|
2016-03-06 11:26:25 +00:00
|
|
|
return 0, errors.New("wrong number of bytes written")
|
2015-04-26 13:36:49 +00:00
|
|
|
}
|
|
|
|
|
2016-01-24 18:30:14 +00:00
|
|
|
bytesWritten += hdrBytes
|
2015-04-26 13:36:49 +00:00
|
|
|
|
|
|
|
// write length
|
2016-03-06 11:26:25 +00:00
|
|
|
err = binary.Write(p.wr, binary.LittleEndian, uint32(uint(len(p.blobs))*entrySize+crypto.Extension))
|
2015-04-26 13:36:49 +00:00
|
|
|
if err != nil {
|
2016-03-06 11:26:25 +00:00
|
|
|
return 0, err
|
2015-04-26 13:36:49 +00:00
|
|
|
}
|
2015-04-29 22:19:56 +00:00
|
|
|
bytesWritten += uint(binary.Size(uint32(0)))
|
2015-04-26 13:36:49 +00:00
|
|
|
|
2015-04-29 22:06:08 +00:00
|
|
|
p.bytes = uint(bytesWritten)
|
2015-04-26 13:36:49 +00:00
|
|
|
|
2016-03-06 11:26:25 +00:00
|
|
|
if w, ok := p.wr.(io.Closer); ok {
|
|
|
|
return bytesWritten, w.Close()
|
|
|
|
}
|
|
|
|
|
|
|
|
return bytesWritten, nil
|
2015-04-26 13:36:49 +00:00
|
|
|
}
|
|
|
|
|
2015-04-29 22:36:36 +00:00
|
|
|
// writeHeader constructs and writes the header to wr.
|
|
|
|
func (p *Packer) writeHeader(wr io.Writer) (bytesWritten uint, err error) {
|
|
|
|
for _, b := range p.blobs {
|
|
|
|
entry := headerEntry{
|
|
|
|
Type: b.Type,
|
2015-08-08 11:47:08 +00:00
|
|
|
Length: uint32(b.Length),
|
2015-07-25 15:05:45 +00:00
|
|
|
ID: b.ID,
|
2015-04-29 22:36:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
err := binary.Write(wr, binary.LittleEndian, entry)
|
|
|
|
if err != nil {
|
|
|
|
return bytesWritten, err
|
|
|
|
}
|
|
|
|
|
|
|
|
bytesWritten += entrySize
|
|
|
|
}
|
|
|
|
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2015-04-26 13:36:49 +00:00
|
|
|
// Size returns the number of bytes written so far.
|
|
|
|
func (p *Packer) Size() uint {
|
|
|
|
p.m.Lock()
|
|
|
|
defer p.m.Unlock()
|
|
|
|
|
|
|
|
return p.bytes
|
|
|
|
}
|
|
|
|
|
|
|
|
// Count returns the number of blobs in this packer.
|
|
|
|
func (p *Packer) Count() int {
|
|
|
|
p.m.Lock()
|
|
|
|
defer p.m.Unlock()
|
|
|
|
|
|
|
|
return len(p.blobs)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Blobs returns the slice of blobs that have been written.
|
|
|
|
func (p *Packer) Blobs() []Blob {
|
|
|
|
p.m.Lock()
|
|
|
|
defer p.m.Unlock()
|
|
|
|
|
|
|
|
return p.blobs
|
|
|
|
}
|
|
|
|
|
2016-03-06 11:26:25 +00:00
|
|
|
// Writer return the underlying writer.
|
|
|
|
func (p *Packer) Writer() io.Writer {
|
|
|
|
return p.wr
|
|
|
|
}
|
|
|
|
|
2015-04-26 13:36:49 +00:00
|
|
|
func (p *Packer) String() string {
|
|
|
|
return fmt.Sprintf("<Packer %d blobs, %d bytes>", len(p.blobs), p.bytes)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Unpacker is used to read individual blobs from a pack.
|
|
|
|
type Unpacker struct {
|
|
|
|
rd io.ReadSeeker
|
|
|
|
Entries []Blob
|
|
|
|
k *crypto.Key
|
|
|
|
}
|
|
|
|
|
|
|
|
// NewUnpacker returns a pointer to Unpacker which can be used to read
|
|
|
|
// individual Blobs from a pack.
|
2015-10-25 17:07:51 +00:00
|
|
|
func NewUnpacker(k *crypto.Key, rd io.ReadSeeker) (*Unpacker, error) {
|
2015-04-26 13:36:49 +00:00
|
|
|
var err error
|
|
|
|
ls := binary.Size(uint32(0))
|
|
|
|
|
|
|
|
// reset to the end to read header length
|
|
|
|
_, err = rd.Seek(-int64(ls), 2)
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("seeking to read header length failed: %v", err)
|
|
|
|
}
|
|
|
|
|
2015-04-29 22:06:08 +00:00
|
|
|
var length uint32
|
|
|
|
err = binary.Read(rd, binary.LittleEndian, &length)
|
2015-04-26 13:36:49 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("reading header length failed: %v", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
// reset to the beginning of the header
|
2015-04-29 22:06:08 +00:00
|
|
|
_, err = rd.Seek(-int64(ls)-int64(length), 2)
|
2015-04-26 13:36:49 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("seeking to read header length failed: %v", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
// read header
|
2015-04-29 22:06:08 +00:00
|
|
|
hrd, err := crypto.DecryptFrom(k, io.LimitReader(rd, int64(length)))
|
2015-04-26 13:36:49 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2015-10-25 17:07:51 +00:00
|
|
|
var entries []Blob
|
|
|
|
|
|
|
|
pos := uint(0)
|
|
|
|
for {
|
|
|
|
e := headerEntry{}
|
|
|
|
err = binary.Read(hrd, binary.LittleEndian, &e)
|
|
|
|
if err == io.EOF {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
2015-04-26 13:36:49 +00:00
|
|
|
}
|
2015-10-25 17:07:51 +00:00
|
|
|
|
|
|
|
entries = append(entries, Blob{
|
|
|
|
Type: e.Type,
|
|
|
|
Length: uint(e.Length),
|
|
|
|
ID: e.ID,
|
|
|
|
Offset: pos,
|
|
|
|
})
|
|
|
|
|
|
|
|
pos += uint(e.Length)
|
2015-04-26 13:36:49 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
p := &Unpacker{
|
|
|
|
rd: rd,
|
|
|
|
k: k,
|
|
|
|
Entries: entries,
|
|
|
|
}
|
|
|
|
|
|
|
|
return p, nil
|
|
|
|
}
|