From 718966a81a6f59e79ef3f5fd38af864c1d770691 Mon Sep 17 00:00:00 2001 From: Uli Martens Date: Sat, 11 Sep 2021 13:26:10 +0200 Subject: [PATCH] Move Blobcache into dedicated internal package --- .../{fuse/blobcache.go => bloblru/cache.go} | 36 ++++++------- internal/bloblru/cache_test.go | 50 +++++++++++++++++++ internal/fuse/file.go | 4 +- internal/fuse/fuse_test.go | 48 ++---------------- internal/fuse/root.go | 6 ++- 5 files changed, 78 insertions(+), 66 deletions(-) rename internal/{fuse/blobcache.go => bloblru/cache.go} (58%) create mode 100644 internal/bloblru/cache_test.go diff --git a/internal/fuse/blobcache.go b/internal/bloblru/cache.go similarity index 58% rename from internal/fuse/blobcache.go rename to internal/bloblru/cache.go index 728457b11..473a4c2e6 100644 --- a/internal/fuse/blobcache.go +++ b/internal/bloblru/cache.go @@ -1,4 +1,4 @@ -package fuse +package bloblru import ( "sync" @@ -10,12 +10,12 @@ import ( ) // Crude estimate of the overhead per blob: a SHA-256, a linked list node -// and some pointers. See comment in blobCache.add. -const cacheOverhead = len(restic.ID{}) + 64 +// and some pointers. See comment in Cache.add. +const overhead = len(restic.ID{}) + 64 -// A blobCache is a fixed-size cache of blob contents. +// A Cache is a fixed-size LRU cache of blob contents. // It is safe for concurrent access. -type blobCache struct { +type Cache struct { mu sync.Mutex c *simplelru.LRU @@ -23,16 +23,16 @@ type blobCache struct { } // Construct a blob cache that stores at most size bytes worth of blobs. -func newBlobCache(size int) *blobCache { - c := &blobCache{ +func New(size int) *Cache { + c := &Cache{ free: size, size: size, } // NewLRU wants us to specify some max. number of entries, else it errors. - // The actual maximum will be smaller than size/cacheOverhead, because we + // The actual maximum will be smaller than size/overhead, because we // evict entries (RemoveOldest in add) to maintain our size bound. - maxEntries := size / cacheOverhead + maxEntries := size / overhead lru, err := simplelru.NewLRU(maxEntries, c.evict) if err != nil { panic(err) // Can only be maxEntries <= 0. @@ -42,10 +42,10 @@ func newBlobCache(size int) *blobCache { return c } -func (c *blobCache) add(id restic.ID, blob []byte) { - debug.Log("blobCache: add %v", id) +func (c *Cache) Add(id restic.ID, blob []byte) { + debug.Log("bloblru.Cache: add %v", id) - size := len(blob) + cacheOverhead + size := len(blob) + overhead if size > c.size { return } @@ -59,7 +59,7 @@ func (c *blobCache) add(id restic.ID, blob []byte) { return } - // This loop takes at most min(maxEntries, maxchunksize/cacheOverhead) + // This loop takes at most min(maxEntries, maxchunksize/overhead) // iterations. for size > c.free { c.c.RemoveOldest() @@ -69,19 +69,19 @@ func (c *blobCache) add(id restic.ID, blob []byte) { c.free -= size } -func (c *blobCache) get(id restic.ID) ([]byte, bool) { +func (c *Cache) Get(id restic.ID) ([]byte, bool) { c.mu.Lock() value, ok := c.c.Get(id) c.mu.Unlock() - debug.Log("blobCache: get %v, hit %v", id, ok) + debug.Log("bloblru.Cache: get %v, hit %v", id, ok) blob, ok := value.([]byte) return blob, ok } -func (c *blobCache) evict(key, value interface{}) { +func (c *Cache) evict(key, value interface{}) { blob := value.([]byte) - debug.Log("blobCache: evict %v, %d bytes", key, len(blob)) - c.free += len(blob) + cacheOverhead + debug.Log("bloblru.Cache: evict %v, %d bytes", key, len(blob)) + c.free += len(blob) + overhead } diff --git a/internal/bloblru/cache_test.go b/internal/bloblru/cache_test.go new file mode 100644 index 000000000..c257a95e2 --- /dev/null +++ b/internal/bloblru/cache_test.go @@ -0,0 +1,50 @@ +package bloblru + +import ( + "testing" + + "github.com/restic/restic/internal/restic" + rtest "github.com/restic/restic/internal/test" +) + +func TestCache(t *testing.T) { + var id1, id2, id3 restic.ID + id1[0] = 1 + id2[0] = 2 + id3[0] = 3 + + const ( + kiB = 1 << 10 + cacheSize = 64*kiB + 3*overhead + ) + + c := New(cacheSize) + + addAndCheck := func(id restic.ID, exp []byte) { + c.Add(id, exp) + blob, ok := c.Get(id) + rtest.Assert(t, ok, "blob %v added but not found in cache", id) + rtest.Equals(t, &exp[0], &blob[0]) + rtest.Equals(t, exp, blob) + } + + addAndCheck(id1, make([]byte, 32*kiB)) + addAndCheck(id2, make([]byte, 30*kiB)) + addAndCheck(id3, make([]byte, 10*kiB)) + + _, ok := c.Get(id2) + rtest.Assert(t, ok, "blob %v not present", id2) + _, ok = c.Get(id1) + rtest.Assert(t, !ok, "blob %v present, but should have been evicted", id1) + + c.Add(id1, make([]byte, 1+c.size)) + _, ok = c.Get(id1) + rtest.Assert(t, !ok, "blob %v too large but still added to cache") + + c.c.Remove(id1) + c.c.Remove(id3) + c.c.Remove(id2) + + rtest.Equals(t, cacheSize, c.size) + rtest.Equals(t, cacheSize, c.free) +} diff --git a/internal/fuse/file.go b/internal/fuse/file.go index def7ae85e..2de2660e3 100644 --- a/internal/fuse/file.go +++ b/internal/fuse/file.go @@ -96,7 +96,7 @@ func (f *file) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenR func (f *openFile) getBlobAt(ctx context.Context, i int) (blob []byte, err error) { - blob, ok := f.root.blobCache.get(f.node.Content[i]) + blob, ok := f.root.blobCache.Get(f.node.Content[i]) if ok { return blob, nil } @@ -107,7 +107,7 @@ func (f *openFile) getBlobAt(ctx context.Context, i int) (blob []byte, err error return nil, err } - f.root.blobCache.add(f.node.Content[i], blob) + f.root.blobCache.Add(f.node.Content[i], blob) return blob, nil } diff --git a/internal/fuse/fuse_test.go b/internal/fuse/fuse_test.go index 24d8cf03d..690df770a 100644 --- a/internal/fuse/fuse_test.go +++ b/internal/fuse/fuse_test.go @@ -1,3 +1,4 @@ +//go:build darwin || freebsd || linux // +build darwin freebsd linux package fuse @@ -10,6 +11,7 @@ import ( "testing" "time" + "github.com/restic/restic/internal/bloblru" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" @@ -19,48 +21,6 @@ import ( rtest "github.com/restic/restic/internal/test" ) -func TestCache(t *testing.T) { - var id1, id2, id3 restic.ID - id1[0] = 1 - id2[0] = 2 - id3[0] = 3 - - const ( - kiB = 1 << 10 - cacheSize = 64*kiB + 3*cacheOverhead - ) - - c := newBlobCache(cacheSize) - - addAndCheck := func(id restic.ID, exp []byte) { - c.add(id, exp) - blob, ok := c.get(id) - rtest.Assert(t, ok, "blob %v added but not found in cache", id) - rtest.Equals(t, &exp[0], &blob[0]) - rtest.Equals(t, exp, blob) - } - - addAndCheck(id1, make([]byte, 32*kiB)) - addAndCheck(id2, make([]byte, 30*kiB)) - addAndCheck(id3, make([]byte, 10*kiB)) - - _, ok := c.get(id2) - rtest.Assert(t, ok, "blob %v not present", id2) - _, ok = c.get(id1) - rtest.Assert(t, !ok, "blob %v present, but should have been evicted", id1) - - c.add(id1, make([]byte, 1+c.size)) - _, ok = c.get(id1) - rtest.Assert(t, !ok, "blob %v too large but still added to cache") - - c.c.Remove(id1) - c.c.Remove(id3) - c.c.Remove(id2) - - rtest.Equals(t, cacheSize, c.size) - rtest.Equals(t, cacheSize, c.free) -} - func testRead(t testing.TB, f fs.Handle, offset, length int, data []byte) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -156,7 +116,7 @@ func TestFuseFile(t *testing.T) { Size: filesize, Content: content, } - root := &Root{repo: repo, blobCache: newBlobCache(blobCacheSize)} + root := &Root{repo: repo, blobCache: bloblru.New(blobCacheSize)} inode := fs.GenerateDynamicInode(1, "foo") f, err := newFile(context.TODO(), root, inode, node) @@ -191,7 +151,7 @@ func TestFuseDir(t *testing.T) { repo, cleanup := repository.TestRepository(t) defer cleanup() - root := &Root{repo: repo, blobCache: newBlobCache(blobCacheSize)} + root := &Root{repo: repo, blobCache: bloblru.New(blobCacheSize)} node := &restic.Node{ Mode: 0755, diff --git a/internal/fuse/root.go b/internal/fuse/root.go index ecbd10546..bed760f02 100644 --- a/internal/fuse/root.go +++ b/internal/fuse/root.go @@ -1,3 +1,4 @@ +//go:build darwin || freebsd || linux // +build darwin freebsd linux package fuse @@ -6,6 +7,7 @@ import ( "os" "time" + "github.com/restic/restic/internal/bloblru" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/restic" @@ -27,7 +29,7 @@ type Root struct { cfg Config inode uint64 snapshots restic.Snapshots - blobCache *blobCache + blobCache *bloblru.Cache snCount int lastCheck time.Time @@ -54,7 +56,7 @@ func NewRoot(repo restic.Repository, cfg Config) *Root { repo: repo, inode: rootInode, cfg: cfg, - blobCache: newBlobCache(blobCacheSize), + blobCache: bloblru.New(blobCacheSize), } if !cfg.OwnerIsRoot {