KurlesHS/restic
1
0
Fork 0
forked from TrueCloudLab/restic
Code Pull requests Activity

Move Blobcache into dedicated internal package

Browse source
This commit is contained in:
Uli Martens 2021-09-11 13:26:10 +02:00 committed by greatroar
parent 4f33eca634
commit 718966a81a
5 changed files with 78 additions and 66 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
internal/fuse/blobcache.go → internal/bloblru/cache.go
View file

@ -1,4 +1,4 @@
package fuse
package bloblru
import (
"sync"
@ -10,12 +10,12 @@ import (
)
// Crude estimate of the overhead per blob: a SHA-256, a linked list node
// and some pointers. See comment in blobCache.add.
const cacheOverhead = len(restic.ID{}) + 64
// and some pointers. See comment in Cache.add.
const overhead = len(restic.ID{}) + 64
// A blobCache is a fixed-size cache of blob contents.
// A Cache is a fixed-size LRU cache of blob contents.
// It is safe for concurrent access.
type blobCache struct {
type Cache struct {
mu sync.Mutex
c *simplelru.LRU
@ -23,16 +23,16 @@ type blobCache struct {
}
// Construct a blob cache that stores at most size bytes worth of blobs.
func newBlobCache(size int) *blobCache {
c := &blobCache{
func New(size int) *Cache {
c := &Cache{
free: size,
size: size,
}
// NewLRU wants us to specify some max. number of entries, else it errors.
// The actual maximum will be smaller than size/cacheOverhead, because we
// The actual maximum will be smaller than size/overhead, because we
// evict entries (RemoveOldest in add) to maintain our size bound.
maxEntries := size / cacheOverhead
maxEntries := size / overhead
lru, err := simplelru.NewLRU(maxEntries, c.evict)
if err != nil {
panic(err) // Can only be maxEntries <= 0.
@ -42,10 +42,10 @@ func newBlobCache(size int) *blobCache {
return c
}
func (c *blobCache) add(id restic.ID, blob []byte) {
debug.Log("blobCache: add %v", id)
func (c *Cache) Add(id restic.ID, blob []byte) {
debug.Log("bloblru.Cache: add %v", id)
size := len(blob) + cacheOverhead
size := len(blob) + overhead
if size > c.size {
return
}
@ -59,7 +59,7 @@ func (c *blobCache) add(id restic.ID, blob []byte) {
return
}
// This loop takes at most min(maxEntries, maxchunksize/cacheOverhead)
// This loop takes at most min(maxEntries, maxchunksize/overhead)
// iterations.
for size > c.free {
c.c.RemoveOldest()
@ -69,19 +69,19 @@ func (c *blobCache) add(id restic.ID, blob []byte) {
c.free -= size
}
func (c *blobCache) get(id restic.ID) ([]byte, bool) {
func (c *Cache) Get(id restic.ID) ([]byte, bool) {
c.mu.Lock()
value, ok := c.c.Get(id)
c.mu.Unlock()
debug.Log("blobCache: get %v, hit %v", id, ok)
debug.Log("bloblru.Cache: get %v, hit %v", id, ok)
blob, ok := value.([]byte)
return blob, ok
}
func (c *blobCache) evict(key, value interface{}) {
func (c *Cache) evict(key, value interface{}) {
blob := value.([]byte)
debug.Log("blobCache: evict %v, %d bytes", key, len(blob))
c.free += len(blob) + cacheOverhead
debug.Log("bloblru.Cache: evict %v, %d bytes", key, len(blob))
c.free += len(blob) + overhead
}

50
internal/bloblru/cache_test.go Normal file
View file

@ -0,0 +1,50 @@
package bloblru
import (
"testing"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
func TestCache(t *testing.T) {
var id1, id2, id3 restic.ID
id1[0] = 1
id2[0] = 2
id3[0] = 3
const (
kiB = 1 << 10
cacheSize = 64*kiB + 3*overhead
)
c := New(cacheSize)
addAndCheck := func(id restic.ID, exp []byte) {
c.Add(id, exp)
blob, ok := c.Get(id)
rtest.Assert(t, ok, "blob %v added but not found in cache", id)
rtest.Equals(t, &exp[0], &blob[0])
rtest.Equals(t, exp, blob)
}
addAndCheck(id1, make([]byte, 32*kiB))
addAndCheck(id2, make([]byte, 30*kiB))
addAndCheck(id3, make([]byte, 10*kiB))
_, ok := c.Get(id2)
rtest.Assert(t, ok, "blob %v not present", id2)
_, ok = c.Get(id1)
rtest.Assert(t, !ok, "blob %v present, but should have been evicted", id1)
c.Add(id1, make([]byte, 1+c.size))
_, ok = c.Get(id1)
rtest.Assert(t, !ok, "blob %v too large but still added to cache")
c.c.Remove(id1)
c.c.Remove(id3)
c.c.Remove(id2)
rtest.Equals(t, cacheSize, c.size)
rtest.Equals(t, cacheSize, c.free)
}

4
internal/fuse/file.go
View file

@ -96,7 +96,7 @@ func (f *file) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenR
func (f *openFile) getBlobAt(ctx context.Context, i int) (blob []byte, err error) {
blob, ok := f.root.blobCache.get(f.node.Content[i])
blob, ok := f.root.blobCache.Get(f.node.Content[i])
if ok {
return blob, nil
}
@ -107,7 +107,7 @@ func (f *openFile) getBlobAt(ctx context.Context, i int) (blob []byte, err error
return nil, err
}
f.root.blobCache.add(f.node.Content[i], blob)
f.root.blobCache.Add(f.node.Content[i], blob)
return blob, nil
}

48
internal/fuse/fuse_test.go
View file

@ -1,3 +1,4 @@
//go:build darwin || freebsd || linux
// +build darwin freebsd linux
package fuse
@ -10,6 +11,7 @@ import (
"testing"
"time"
"github.com/restic/restic/internal/bloblru"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
@ -19,48 +21,6 @@ import (
rtest "github.com/restic/restic/internal/test"
)
func TestCache(t *testing.T) {
var id1, id2, id3 restic.ID
id1[0] = 1
id2[0] = 2
id3[0] = 3
const (
kiB = 1 << 10
cacheSize = 64*kiB + 3*cacheOverhead
)
c := newBlobCache(cacheSize)
addAndCheck := func(id restic.ID, exp []byte) {
c.add(id, exp)
blob, ok := c.get(id)
rtest.Assert(t, ok, "blob %v added but not found in cache", id)
rtest.Equals(t, &exp[0], &blob[0])
rtest.Equals(t, exp, blob)
}
addAndCheck(id1, make([]byte, 32*kiB))
addAndCheck(id2, make([]byte, 30*kiB))
addAndCheck(id3, make([]byte, 10*kiB))
_, ok := c.get(id2)
rtest.Assert(t, ok, "blob %v not present", id2)
_, ok = c.get(id1)
rtest.Assert(t, !ok, "blob %v present, but should have been evicted", id1)
c.add(id1, make([]byte, 1+c.size))
_, ok = c.get(id1)
rtest.Assert(t, !ok, "blob %v too large but still added to cache")
c.c.Remove(id1)
c.c.Remove(id3)
c.c.Remove(id2)
rtest.Equals(t, cacheSize, c.size)
rtest.Equals(t, cacheSize, c.free)
}
func testRead(t testing.TB, f fs.Handle, offset, length int, data []byte) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@ -156,7 +116,7 @@ func TestFuseFile(t *testing.T) {
Size: filesize,
Content: content,
}
root := &Root{repo: repo, blobCache: newBlobCache(blobCacheSize)}
root := &Root{repo: repo, blobCache: bloblru.New(blobCacheSize)}
inode := fs.GenerateDynamicInode(1, "foo")
f, err := newFile(context.TODO(), root, inode, node)
@ -191,7 +151,7 @@ func TestFuseDir(t *testing.T) {
repo, cleanup := repository.TestRepository(t)
defer cleanup()
root := &Root{repo: repo, blobCache: newBlobCache(blobCacheSize)}
root := &Root{repo: repo, blobCache: bloblru.New(blobCacheSize)}
node := &restic.Node{
Mode: 0755,

6
internal/fuse/root.go
View file

@ -1,3 +1,4 @@
//go:build darwin || freebsd || linux
// +build darwin freebsd linux
package fuse
@ -6,6 +7,7 @@ import (
"os"
"time"
"github.com/restic/restic/internal/bloblru"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/restic"
@ -27,7 +29,7 @@ type Root struct {
cfg Config
inode uint64
snapshots restic.Snapshots
blobCache *blobCache
blobCache *bloblru.Cache
snCount int
lastCheck time.Time
@ -54,7 +56,7 @@ func NewRoot(repo restic.Repository, cfg Config) *Root {
repo: repo,
inode: rootInode,
cfg: cfg,
blobCache: newBlobCache(blobCacheSize),
blobCache: bloblru.New(blobCacheSize),
}
if !cfg.OwnerIsRoot {