forked from TrueCloudLab/restic
120ccc8754
Previously, SaveAndEncrypt would assemble blobs into packs and either return immediately if the pack is not yet full or upload the pack file otherwise. The upload will block the current goroutine until it finishes. Now, the upload is done using separate goroutines. This requires changes to the error handling. As uploads are no longer tied to a SaveAndEncrypt call, failed uploads are signaled using an errgroup. To count the uploaded amount of data, the pack header overhead is no longer returned by `packer.Finalize` but rather by `packer.HeaderOverhead`. This helper method is necessary to continue returning the pack header overhead directly to the responsible call to `repository.SaveBlob`. Without the method this would not be possible, as packs are finalized asynchronously.
145 lines
3.4 KiB
Go
145 lines
3.4 KiB
Go
package pack_test
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/rand"
|
|
"crypto/sha256"
|
|
"encoding/json"
|
|
"io"
|
|
"testing"
|
|
|
|
"github.com/restic/restic/internal/backend/mem"
|
|
"github.com/restic/restic/internal/crypto"
|
|
"github.com/restic/restic/internal/pack"
|
|
"github.com/restic/restic/internal/restic"
|
|
rtest "github.com/restic/restic/internal/test"
|
|
)
|
|
|
|
var testLens = []int{23, 31650, 25860, 10928, 13769, 19862, 5211, 127, 13690, 30231}
|
|
|
|
type Buf struct {
|
|
data []byte
|
|
id restic.ID
|
|
}
|
|
|
|
func newPack(t testing.TB, k *crypto.Key, lengths []int) ([]Buf, []byte, uint) {
|
|
bufs := []Buf{}
|
|
|
|
for _, l := range lengths {
|
|
b := make([]byte, l)
|
|
_, err := io.ReadFull(rand.Reader, b)
|
|
rtest.OK(t, err)
|
|
h := sha256.Sum256(b)
|
|
bufs = append(bufs, Buf{data: b, id: h})
|
|
}
|
|
|
|
// pack blobs
|
|
var buf bytes.Buffer
|
|
p := pack.NewPacker(k, &buf)
|
|
for _, b := range bufs {
|
|
_, err := p.Add(restic.TreeBlob, b.id, b.data, 2*len(b.data))
|
|
rtest.OK(t, err)
|
|
}
|
|
|
|
err := p.Finalize()
|
|
rtest.OK(t, err)
|
|
|
|
return bufs, buf.Bytes(), p.Size()
|
|
}
|
|
|
|
func verifyBlobs(t testing.TB, bufs []Buf, k *crypto.Key, rd io.ReaderAt, packSize uint) {
|
|
written := 0
|
|
for _, buf := range bufs {
|
|
written += len(buf.data)
|
|
}
|
|
|
|
// read and parse it again
|
|
entries, hdrSize, err := pack.List(k, rd, int64(packSize))
|
|
rtest.OK(t, err)
|
|
rtest.Equals(t, len(entries), len(bufs))
|
|
|
|
// check the head size calculation for consistency
|
|
headerSize := pack.CalculateHeaderSize(entries)
|
|
written += headerSize
|
|
|
|
// check length
|
|
rtest.Equals(t, uint(written), packSize)
|
|
rtest.Equals(t, headerSize, int(hdrSize))
|
|
|
|
var buf []byte
|
|
for i, b := range bufs {
|
|
e := entries[i]
|
|
rtest.Equals(t, b.id, e.ID)
|
|
|
|
if len(buf) < int(e.Length) {
|
|
buf = make([]byte, int(e.Length))
|
|
}
|
|
buf = buf[:int(e.Length)]
|
|
n, err := rd.ReadAt(buf, int64(e.Offset))
|
|
rtest.OK(t, err)
|
|
buf = buf[:n]
|
|
|
|
rtest.Assert(t, bytes.Equal(b.data, buf),
|
|
"data for blob %v doesn't match", i)
|
|
}
|
|
}
|
|
|
|
func TestCreatePack(t *testing.T) {
|
|
// create random keys
|
|
k := crypto.NewRandomKey()
|
|
|
|
bufs, packData, packSize := newPack(t, k, testLens)
|
|
rtest.Equals(t, uint(len(packData)), packSize)
|
|
verifyBlobs(t, bufs, k, bytes.NewReader(packData), packSize)
|
|
}
|
|
|
|
var blobTypeJSON = []struct {
|
|
t restic.BlobType
|
|
res string
|
|
}{
|
|
{restic.DataBlob, `"data"`},
|
|
{restic.TreeBlob, `"tree"`},
|
|
}
|
|
|
|
func TestBlobTypeJSON(t *testing.T) {
|
|
for _, test := range blobTypeJSON {
|
|
// test serialize
|
|
buf, err := json.Marshal(test.t)
|
|
rtest.OK(t, err)
|
|
rtest.Equals(t, test.res, string(buf))
|
|
|
|
// test unserialize
|
|
var v restic.BlobType
|
|
err = json.Unmarshal([]byte(test.res), &v)
|
|
rtest.OK(t, err)
|
|
rtest.Equals(t, test.t, v)
|
|
}
|
|
}
|
|
|
|
func TestUnpackReadSeeker(t *testing.T) {
|
|
// create random keys
|
|
k := crypto.NewRandomKey()
|
|
|
|
bufs, packData, packSize := newPack(t, k, testLens)
|
|
|
|
b := mem.New()
|
|
id := restic.Hash(packData)
|
|
|
|
handle := restic.Handle{Type: restic.PackFile, Name: id.String()}
|
|
rtest.OK(t, b.Save(context.TODO(), handle, restic.NewByteReader(packData, b.Hasher())))
|
|
verifyBlobs(t, bufs, k, restic.ReaderAt(context.TODO(), b, handle), packSize)
|
|
}
|
|
|
|
func TestShortPack(t *testing.T) {
|
|
k := crypto.NewRandomKey()
|
|
|
|
bufs, packData, packSize := newPack(t, k, []int{23})
|
|
|
|
b := mem.New()
|
|
id := restic.Hash(packData)
|
|
|
|
handle := restic.Handle{Type: restic.PackFile, Name: id.String()}
|
|
rtest.OK(t, b.Save(context.TODO(), handle, restic.NewByteReader(packData, b.Hasher())))
|
|
verifyBlobs(t, bufs, k, restic.ReaderAt(context.TODO(), b, handle), packSize)
|
|
}
|