restic/internal/fuse/file_test.go
Matthew Dawson df2c03a6a4
repository/master_index: Optimize Index.Lookup()
When looking up a blob in the master index, with several
indexes present in the master index, a significant amount of time
is spent generating errors for each failed lookup.  However, these
errors are often used to check if a blob is present, but the contents
are not inspected making the overhead of the error not useful.

Instead, change Index.Lookup (and Index.LookupSize) to instead return
a boolean denoting if the blob was found instead of an error.  Also change
all the calls to these functions to handle the new function signature.

benchmark                                            old ns/op     new ns/op     delta
BenchmarkMasterIndexLookupSingleIndex-6              820           897           +9.39%
BenchmarkMasterIndexLookupMultipleIndex-6            12821         2001          -84.39%
BenchmarkMasterIndexLookupSingleIndexUnknown-6       5378          492           -90.85%
BenchmarkMasterIndexLookupMultipleIndexUnknown-6     17026         1649          -90.31%

benchmark                                            old allocs     new allocs     delta
BenchmarkMasterIndexLookupSingleIndex-6              9              9              +0.00%
BenchmarkMasterIndexLookupMultipleIndex-6            59             19             -67.80%
BenchmarkMasterIndexLookupSingleIndexUnknown-6       22             6              -72.73%
BenchmarkMasterIndexLookupMultipleIndexUnknown-6     72             16             -77.78%

benchmark                                            old bytes     new bytes     delta
BenchmarkMasterIndexLookupSingleIndex-6              160           160           +0.00%
BenchmarkMasterIndexLookupMultipleIndex-6            3200          240           -92.50%
BenchmarkMasterIndexLookupSingleIndexUnknown-6       1232          48            -96.10%
BenchmarkMasterIndexLookupMultipleIndexUnknown-6     4272          128           -97.00%
2018-01-23 22:25:56 -05:00

146 lines
3.4 KiB
Go

// +build !openbsd
// +build !windows
package fuse
import (
"bytes"
"math/rand"
"testing"
"time"
"golang.org/x/net/context"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
"bazil.org/fuse"
"bazil.org/fuse/fs"
rtest "github.com/restic/restic/internal/test"
)
func testRead(t testing.TB, f *file, offset, length int, data []byte) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
req := &fuse.ReadRequest{
Offset: int64(offset),
Size: length,
}
resp := &fuse.ReadResponse{
Data: data,
}
rtest.OK(t, f.Read(ctx, req, resp))
}
func firstSnapshotID(t testing.TB, repo restic.Repository) (first restic.ID) {
for id := range repo.List(context.TODO(), restic.SnapshotFile) {
if first.IsNull() {
first = id
}
}
return first
}
func loadFirstSnapshot(t testing.TB, repo restic.Repository) *restic.Snapshot {
id := firstSnapshotID(t, repo)
sn, err := restic.LoadSnapshot(context.TODO(), repo, id)
rtest.OK(t, err)
return sn
}
func loadTree(t testing.TB, repo restic.Repository, id restic.ID) *restic.Tree {
tree, err := repo.LoadTree(context.TODO(), id)
rtest.OK(t, err)
return tree
}
func TestFuseFile(t *testing.T) {
repo, cleanup := repository.TestRepository(t)
defer cleanup()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
timestamp, err := time.Parse(time.RFC3339, "2017-01-24T10:42:56+01:00")
rtest.OK(t, err)
restic.TestCreateSnapshot(t, repo, timestamp, 2, 0.1)
sn := loadFirstSnapshot(t, repo)
tree := loadTree(t, repo, *sn.Tree)
var content restic.IDs
for _, node := range tree.Nodes {
content = append(content, node.Content...)
}
t.Logf("tree loaded, content: %v", content)
var (
filesize uint64
memfile []byte
)
for _, id := range content {
size, found := repo.LookupBlobSize(id, restic.DataBlob)
rtest.Assert(t, found, "Expected to find blob id %v", id)
filesize += uint64(size)
buf := restic.NewBlobBuffer(int(size))
n, err := repo.LoadBlob(context.TODO(), restic.DataBlob, id, buf)
rtest.OK(t, err)
if uint(n) != size {
t.Fatalf("not enough bytes read for id %v: want %v, got %v", id.Str(), size, n)
}
if uint(len(buf)) != size {
t.Fatalf("buffer has wrong length for id %v: want %v, got %v", id.Str(), size, len(buf))
}
memfile = append(memfile, buf...)
}
t.Logf("filesize is %v, memfile has size %v", filesize, len(memfile))
node := &restic.Node{
Name: "foo",
Inode: 23,
Mode: 0742,
Size: filesize,
Content: content,
}
root := &Root{
blobSizeCache: NewBlobSizeCache(context.TODO(), repo.Index()),
repo: repo,
}
t.Logf("blob cache has %d entries", len(root.blobSizeCache.m))
inode := fs.GenerateDynamicInode(1, "foo")
f, err := newFile(context.TODO(), root, inode, node)
rtest.OK(t, err)
attr := fuse.Attr{}
rtest.OK(t, f.Attr(ctx, &attr))
rtest.Equals(t, inode, attr.Inode)
rtest.Equals(t, node.Mode, attr.Mode)
rtest.Equals(t, node.Size, attr.Size)
rtest.Equals(t, (node.Size/uint64(attr.BlockSize))+1, attr.Blocks)
for i := 0; i < 200; i++ {
offset := rand.Intn(int(filesize))
length := rand.Intn(int(filesize)-offset) + 100
b := memfile[offset : offset+length]
buf := make([]byte, length)
testRead(t, f, offset, length, buf)
if !bytes.Equal(b, buf) {
t.Errorf("test %d failed, wrong data returned (offset %v, length %v)", i, offset, length)
}
}
rtest.OK(t, f.Release(ctx, nil))
}