From 7456c8556a8f924f065c7e18f1b80b1624847dd0 Mon Sep 17 00:00:00 2001
From: Dmitrii Stepanov <d.stepanov@yadro.com>
Date: Thu, 31 Aug 2023 11:32:09 +0300
Subject: [PATCH] [#536] blobovnicza: Add blobovniczatree DB cache

Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
---
 .../blobstor/blobovniczatree/blobovnicza.go   |   2 +
 .../blobstor/blobovniczatree/cache.go         | 103 ++++++++++++++++++
 .../blobovniczatree/concurrency_test.go       |   2 +-
 .../blobstor/blobovniczatree/control.go       |  18 +--
 .../blobstor/blobovniczatree/delete.go        |   4 +-
 .../blobstor/blobovniczatree/exists.go        |   2 +-
 .../blobstor/blobovniczatree/get.go           |   4 +-
 .../blobstor/blobovniczatree/get_range.go     |   4 +-
 .../blobstor/blobovniczatree/iterate.go       |   2 +-
 .../blobstor/blobovniczatree/manager.go       |   3 +-
 10 files changed, 124 insertions(+), 20 deletions(-)
 create mode 100644 pkg/local_object_storage/blobstor/blobovniczatree/cache.go

diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go
index 46a8df685..fd5155ee3 100644
--- a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go
+++ b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go
@@ -56,6 +56,7 @@ type Blobovniczas struct {
 
 	commondbManager *dbManager
 	activeDBManager *activeDBManager
+	dbCache         *dbCache
 }
 
 var _ common.Storage = (*Blobovniczas)(nil)
@@ -77,6 +78,7 @@ func NewBlobovniczaTree(opts ...Option) (blz *Blobovniczas) {
 
 	blz.commondbManager = newDBManager(blz.rootPath, blz.blzOpts, blz.blzLeafWidth, blz.readOnly, blz.metrics.Blobovnicza(), blz.log)
 	blz.activeDBManager = newActiveDBManager(blz.commondbManager, blz.blzLeafWidth)
+	blz.dbCache = newDBCache(blz.openedCacheSize, blz.commondbManager)
 
 	return blz
 }
diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/cache.go b/pkg/local_object_storage/blobstor/blobovniczatree/cache.go
new file mode 100644
index 000000000..9bdee6df3
--- /dev/null
+++ b/pkg/local_object_storage/blobstor/blobovniczatree/cache.go
@@ -0,0 +1,103 @@
+package blobovniczatree
+
+import (
+	"fmt"
+	"sync"
+
+	utilSync "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/sync"
+	"github.com/hashicorp/golang-lru/v2/simplelru"
+)
+
+// dbCache caches sharedDB instances that are NOT open for Put.
+//
+// Uses dbManager for opening/closing sharedDB instances.
+// Stores a reference to an cached sharedDB, so dbManager does not close it.
+type dbCache struct {
+	cacheGuard *sync.RWMutex
+	cache      simplelru.LRUCache[string, *sharedDB]
+	pathLock   *utilSync.KeyLocker[string]
+	closed     bool
+
+	dbManager *dbManager
+}
+
+func newDBCache(size int, dbManager *dbManager) *dbCache {
+	cache, err := simplelru.NewLRU[string, *sharedDB](size, func(_ string, evictedDB *sharedDB) {
+		evictedDB.Close()
+	})
+	if err != nil {
+		// occurs only if the size is not positive
+		panic(fmt.Errorf("could not create LRU cache of size %d: %w", size, err))
+	}
+	return &dbCache{
+		cacheGuard: &sync.RWMutex{},
+		cache:      cache,
+		dbManager:  dbManager,
+		pathLock:   utilSync.NewKeyLocker[string](),
+	}
+}
+
+func (c *dbCache) Open() {
+	c.cacheGuard.Lock()
+	defer c.cacheGuard.Unlock()
+
+	c.closed = false
+}
+
+func (c *dbCache) Close() {
+	c.cacheGuard.Lock()
+	defer c.cacheGuard.Unlock()
+	c.cache.Purge()
+	c.closed = true
+}
+
+func (c *dbCache) GetOrCreate(path string) *sharedDB {
+	value := c.getExisted(path)
+	if value != nil {
+		return value
+	}
+	return c.create(path)
+}
+
+func (c *dbCache) getExisted(path string) *sharedDB {
+	c.cacheGuard.Lock()
+	defer c.cacheGuard.Unlock()
+
+	if value, ok := c.cache.Get(path); ok {
+		return value
+	}
+	return nil
+}
+
+func (c *dbCache) create(path string) *sharedDB {
+	c.pathLock.Lock(path)
+	defer c.pathLock.Unlock(path)
+
+	value := c.getExisted(path)
+	if value != nil {
+		return value
+	}
+
+	value = c.dbManager.GetByPath(path)
+
+	_, err := value.Open() //open db to hold reference, closed by evictedDB.Close() or if cache closed
+	if err != nil {
+		return value
+	}
+	if added := c.put(path, value); !added {
+		value.Close()
+	}
+	return value
+}
+
+func (c *dbCache) put(path string, db *sharedDB) bool {
+	c.cacheGuard.Lock()
+	defer c.cacheGuard.Unlock()
+
+	if !c.closed {
+		c.cache.Add(path, db)
+		return true
+	}
+
+	return false
+}
diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go
index 018ecce3c..24cb93865 100644
--- a/pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go
+++ b/pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go
@@ -50,7 +50,7 @@ func TestBlobovniczaTree_Concurrency(t *testing.T) {
 				require.NoError(t, err)
 
 				_, err = st.Get(context.Background(), common.GetPrm{Address: addr})
-				require.NoError(t, err) // fails very often, correlated to how many goroutines are started
+				require.NoError(t, err)
 			}
 		}()
 	}
diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/control.go b/pkg/local_object_storage/blobstor/blobovniczatree/control.go
index c3d12088a..5ea426e1c 100644
--- a/pkg/local_object_storage/blobstor/blobovniczatree/control.go
+++ b/pkg/local_object_storage/blobstor/blobovniczatree/control.go
@@ -21,15 +21,13 @@ func (b *Blobovniczas) Open(readOnly bool) error {
 func (b *Blobovniczas) Init() error {
 	b.log.Debug(logs.BlobovniczatreeInitializingBlobovniczas)
 
-	b.openManagers()
-
 	if b.readOnly {
 		b.log.Debug(logs.BlobovniczatreeReadonlyModeSkipBlobovniczasInitialization)
 		return nil
 	}
 
 	return b.iterateLeaves(context.TODO(), func(p string) (bool, error) {
-		shBlz := b.openBlobovniczaNoCache(p)
+		shBlz := b.getBlobovniczaWithoutCaching(p)
 		_, err := shBlz.Open()
 		if err != nil {
 			return true, err
@@ -44,23 +42,25 @@ func (b *Blobovniczas) Init() error {
 func (b *Blobovniczas) openManagers() {
 	b.commondbManager.Open() //order important
 	b.activeDBManager.Open()
+	b.dbCache.Open()
 }
 
 // Close implements common.Storage.
 func (b *Blobovniczas) Close() error {
-	b.activeDBManager.Close() //order important
+	b.dbCache.Close() //order important
+	b.activeDBManager.Close()
 	b.commondbManager.Close()
 
 	return nil
 }
 
-// opens and returns blobovnicza with path p.
+// returns blobovnicza with path p
 //
-// If blobovnicza is already opened and cached, instance from cache is returned w/o changes.
-func (b *Blobovniczas) openBlobovnicza(p string) *sharedDB {
-	return b.openBlobovniczaNoCache(p)
+// If blobovnicza is already cached, instance from cache is returned w/o changes.
+func (b *Blobovniczas) getBlobovnicza(p string) *sharedDB {
+	return b.dbCache.GetOrCreate(p)
 }
 
-func (b *Blobovniczas) openBlobovniczaNoCache(p string) *sharedDB {
+func (b *Blobovniczas) getBlobovniczaWithoutCaching(p string) *sharedDB {
 	return b.commondbManager.GetByPath(p)
 }
diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go
index bc7735e74..ca1e46a15 100644
--- a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go
+++ b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go
@@ -47,7 +47,7 @@ func (b *Blobovniczas) Delete(ctx context.Context, prm common.DeletePrm) (res co
 
 	if prm.StorageID != nil {
 		id := blobovnicza.NewIDFromBytes(prm.StorageID)
-		shBlz := b.openBlobovnicza(id.String())
+		shBlz := b.getBlobovnicza(id.String())
 		blz, err := shBlz.Open()
 		if err != nil {
 			return res, err
@@ -94,7 +94,7 @@ func (b *Blobovniczas) Delete(ctx context.Context, prm common.DeletePrm) (res co
 //
 // returns no error if object was removed from some blobovnicza of the same level.
 func (b *Blobovniczas) deleteObjectFromLevel(ctx context.Context, prm blobovnicza.DeletePrm, blzPath string) (common.DeleteRes, error) {
-	shBlz := b.openBlobovnicza(blzPath)
+	shBlz := b.getBlobovnicza(blzPath)
 	blz, err := shBlz.Open()
 	if err != nil {
 		return common.DeleteRes{}, err
diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/exists.go b/pkg/local_object_storage/blobstor/blobovniczatree/exists.go
index f7a192526..c94881168 100644
--- a/pkg/local_object_storage/blobstor/blobovniczatree/exists.go
+++ b/pkg/local_object_storage/blobstor/blobovniczatree/exists.go
@@ -36,7 +36,7 @@ func (b *Blobovniczas) Exists(ctx context.Context, prm common.ExistsPrm) (common
 
 	if prm.StorageID != nil {
 		id := blobovnicza.NewIDFromBytes(prm.StorageID)
-		shBlz := b.openBlobovnicza(id.String())
+		shBlz := b.getBlobovnicza(id.String())
 		blz, err := shBlz.Open()
 		if err != nil {
 			return common.ExistsRes{}, err
diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get.go b/pkg/local_object_storage/blobstor/blobovniczatree/get.go
index aca39bf3b..f2399e414 100644
--- a/pkg/local_object_storage/blobstor/blobovniczatree/get.go
+++ b/pkg/local_object_storage/blobstor/blobovniczatree/get.go
@@ -47,7 +47,7 @@ func (b *Blobovniczas) Get(ctx context.Context, prm common.GetPrm) (res common.G
 
 	if prm.StorageID != nil {
 		id := blobovnicza.NewIDFromBytes(prm.StorageID)
-		shBlz := b.openBlobovnicza(id.String())
+		shBlz := b.getBlobovnicza(id.String())
 		blz, err := shBlz.Open()
 		if err != nil {
 			return res, err
@@ -93,7 +93,7 @@ func (b *Blobovniczas) Get(ctx context.Context, prm common.GetPrm) (res common.G
 // returns error if object could not be read from any blobovnicza of the same level.
 func (b *Blobovniczas) getObjectFromLevel(ctx context.Context, prm blobovnicza.GetPrm, blzPath string) (common.GetRes, error) {
 	// open blobovnicza (cached inside)
-	shBlz := b.openBlobovnicza(blzPath)
+	shBlz := b.getBlobovnicza(blzPath)
 	blz, err := shBlz.Open()
 	if err != nil {
 		return common.GetRes{}, err
diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go
index 3d1a6b988..56b41351b 100644
--- a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go
+++ b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go
@@ -46,7 +46,7 @@ func (b *Blobovniczas) GetRange(ctx context.Context, prm common.GetRangePrm) (re
 
 	if prm.StorageID != nil {
 		id := blobovnicza.NewIDFromBytes(prm.StorageID)
-		shBlz := b.openBlobovnicza(id.String())
+		shBlz := b.getBlobovnicza(id.String())
 		blz, err := shBlz.Open()
 		if err != nil {
 			return common.GetRangeRes{}, err
@@ -102,7 +102,7 @@ func (b *Blobovniczas) GetRange(ctx context.Context, prm common.GetRangePrm) (re
 // returns error if object could not be read from any blobovnicza of the same level.
 func (b *Blobovniczas) getRangeFromLevel(ctx context.Context, prm common.GetRangePrm, blzPath string) (common.GetRangeRes, error) {
 	// open blobovnicza (cached inside)
-	shBlz := b.openBlobovnicza(blzPath)
+	shBlz := b.getBlobovnicza(blzPath)
 	blz, err := shBlz.Open()
 	if err != nil {
 		return common.GetRangeRes{}, err
diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go
index cc273839a..a2afbb8aa 100644
--- a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go
+++ b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go
@@ -68,7 +68,7 @@ func (b *Blobovniczas) Iterate(ctx context.Context, prm common.IteratePrm) (comm
 // iterator over all Blobovniczas in unsorted order. Break on f's error return.
 func (b *Blobovniczas) iterateBlobovniczas(ctx context.Context, ignoreErrors bool, f func(string, *blobovnicza.Blobovnicza) error) error {
 	return b.iterateLeaves(ctx, func(p string) (bool, error) {
-		shBlz := b.openBlobovnicza(p)
+		shBlz := b.getBlobovnicza(p)
 		blz, err := shBlz.Open()
 		if err != nil {
 			if ignoreErrors {
diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/manager.go b/pkg/local_object_storage/blobstor/blobovniczatree/manager.go
index 941d9961d..7bc8e2827 100644
--- a/pkg/local_object_storage/blobstor/blobovniczatree/manager.go
+++ b/pkg/local_object_storage/blobstor/blobovniczatree/manager.go
@@ -114,8 +114,7 @@ func newLevelDBManager(width uint64, options []blobovnicza.Option, rootPath stri
 	result := &levelDbManager{
 		databases: make([]*sharedDB, width),
 	}
-	var idx uint64
-	for idx = 0; idx < width; idx++ {
+	for idx := uint64(0); idx < width; idx++ {
 		result.databases[idx] = newSharedDB(options, filepath.Join(rootPath, lvlPath, u64ToHexString(idx)), readOnly, metrics, openDBCounter, closedFlog, log)
 	}
 	return result