[#866] Use TTL for blobovnicza tree cache

Signed-off-by: Anton Nikiforov <an.nikiforov@yadro.com>

cmd/frostfs-adm/internal/modules/storagecfg/config.go

@@ -61,6 +61,7 @@ storage:
           depth: 1  # max depth of object tree storage in key-value DB
           width: 4   # max width of object tree storage in key-value DB
           opened_cache_capacity: 50  # maximum number of opened database files
+          opened_cache_ttl: 5m  # ttl for opened database file
 
       gc:
         remover_batch_size: 200  # number of objects to be removed by the garbage collector

cmd/frostfs-node/config.go

@@ -185,6 +185,7 @@ type subStorageCfg struct {
 	width           uint64
 	leafWidth       uint64
 	openedCacheSize int
+	openedCacheTTL  time.Duration
 	initWorkerCount int
 	initInAdvance   bool
 }
@@ -305,6 +306,7 @@ func (a *applicationConfiguration) setShardStorageConfig(newConfig *shardCfg, ol
 			sCfg.width = sub.ShallowWidth()
 			sCfg.leafWidth = sub.LeafWidth()
 			sCfg.openedCacheSize = sub.OpenedCacheSize()
+			sCfg.openedCacheTTL = sub.OpenedCacheTTL()
 			sCfg.initWorkerCount = sub.InitWorkerCount()
 			sCfg.initInAdvance = sub.InitInAdvance()
 		case fstree.Type:
@@ -898,6 +900,7 @@ func (c *cfg) getSubstorageOpts(shCfg shardCfg) []blobstor.SubStorage {
 				blobovniczatree.WithBlobovniczaShallowWidth(sRead.width),
 				blobovniczatree.WithBlobovniczaLeafWidth(sRead.leafWidth),
 				blobovniczatree.WithOpenedCacheSize(sRead.openedCacheSize),
+				blobovniczatree.WithOpenedCacheTTL(sRead.openedCacheTTL),
 				blobovniczatree.WithInitWorkerCount(sRead.initWorkerCount),
 				blobovniczatree.WithInitInAdvance(sRead.initInAdvance),
 				blobovniczatree.WithLogger(c.log),

cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go

@@ -1,6 +1,8 @@
 package blobovniczaconfig
 
 import (
+	"time"
+
 	"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config"
 	boltdbconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/boltdb"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree"
@@ -23,6 +25,9 @@ const (
 	// OpenedCacheSizeDefault is a default cache size of opened Blobovnicza's.
 	OpenedCacheSizeDefault = 16
 
+	// OpenedCacheTTLDefault is a default cache ttl of opened Blobovnicza's.
+	OpenedCacheTTLDefault = 0 // means expiring is off
+
 	// InitWorkerCountDefault is a default workers count to initialize Blobovnicza's.
 	InitWorkerCountDefault = 5
 )
@@ -101,6 +106,22 @@ func (x *Config) OpenedCacheSize() int {
 	return OpenedCacheSizeDefault
 }
 
+// OpenedCacheTTL returns the value of "opened_cache_ttl" config parameter.
+//
+// Returns OpenedCacheTTLDefault if the value is not a positive number.
+func (x *Config) OpenedCacheTTL() time.Duration {
+	d := config.DurationSafe(
+		(*config.Config)(x),
+		"opened_cache_ttl",
+	)
+
+	if d > 0 {
+		return d
+	}
+
+	return OpenedCacheTTLDefault
+}
+
 // BoltDB returns config instance for querying bolt db specific parameters.
 func (x *Config) BoltDB() *boltdbconfig.Config {
 	return (*boltdbconfig.Config)(x)

config/example/node.env

@@ -125,6 +125,7 @@ FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_SIZE=4194304
 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_DEPTH=1
 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_WIDTH=4
 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_OPENED_CACHE_CAPACITY=50
+FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_OPENED_CACHE_TTL=5m
 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_LEAF_WIDTH=10
 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_INIT_WORKER_COUNT=10
 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_INIT_IN_ADVANCE=TRUE
@@ -174,6 +175,7 @@ FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_SIZE=4194304
 FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_DEPTH=1
 FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_WIDTH=4
 FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_OPENED_CACHE_CAPACITY=50
+FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_OPENED_CACHE_TTL=5m
 FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_LEAF_WIDTH=10
 ### FSTree config
 FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_TYPE=fstree

config/example/node.json

@@ -173,6 +173,7 @@
             "depth": 1,
             "width": 4,
             "opened_cache_capacity": 50,
+            "opened_cache_ttl": "5m",
             "leaf_width": 10,
             "init_worker_count": 10,
             "init_in_advance": true
@@ -226,6 +227,7 @@
             "depth": 1,
             "width": 4,
             "opened_cache_capacity": 50,
+            "opened_cache_ttl": "5m",
             "leaf_width": 10
           },
           {

config/example/node.yaml

@@ -150,6 +150,7 @@ storage:
           depth: 1  # max depth of object tree storage in key-value DB
           width: 4   # max width of object tree storage in key-value DB
           opened_cache_capacity: 50  # maximum number of opened database files
+          opened_cache_ttl: 5m  # ttl for opened database file
           leaf_width: 10 # max count of key-value DB on leafs of object tree storage
         - perm: 0644  # permissions for blobstor files(directories: +x for current user and group)
           depth: 5  # max depth of object tree storage in FS

docs/storage-node-configuration.md

@@ -209,6 +209,7 @@ blobstor:
     depth: 1
     width: 4
     opened_cache_capacity: 50
+    opened_cache_ttl: 5m
 ```
 
 #### Common options for sub-storages
@@ -226,13 +227,14 @@ blobstor:
 
 #### `blobovnicza` type options
 | Parameter               | Type       | Default value | Description                                                                                                                                                                                                                                                                                                                                                                                                                              |
-|-------------------------|-----------|---------------|-------------------------------------------------------|
+|-------------------------|------------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
 | `path`                  | `string`   |               | Path to the root of the blobstor.                                                                                                                                                                                                                                                                                                                                                                                                        |
 | `perm`                  | file mode  | `0660`        | Default permission for created files and directories.                                                                                                                                                                                                                                                                                                                                                                                    |
 | `size`                  | `size`     | `1 G`         | Maximum size of a single blobovnicza                                                                                                                                                                                                                                                                                                                                                                                                     |
 | `depth`                 | `int`      | `2`           | Blobovnicza tree depth.                                                                                                                                                                                                                                                                                                                                                                                                                  |
 | `width`                 | `int`      | `16`          | Blobovnicza tree width.                                                                                                                                                                                                                                                                                                                                                                                                                  |
 | `opened_cache_capacity` | `int`      | `16`          | Maximum number of simultaneously opened blobovniczas.                                                                                                                                                                                                                                                                                                                                                                                    |
+| `opened_cache_ttl`      | `duration` | `0`           | TTL in cache for opened blobovniczas(disabled by default). In case of heavy random-read and 10 shards each with 10_000 databases and accessing 400 objects per-second we will access each db approximately once per ((10 * 10_000 / 400) = 250 seconds <= 300 seconds = 5 min). Also take in mind that in this scenario they will probably be closed earlier because of the cache capacity, so bigger values are likely to be of no use. |
 
 ### `gc` subsection
 

pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go

@@ -87,7 +87,7 @@ func NewBlobovniczaTree(opts ...Option) (blz *Blobovniczas) {
 
 	blz.commondbManager = newDBManager(blz.rootPath, blz.blzOpts, blz.readOnly, blz.metrics.Blobovnicza(), blz.log)
 	blz.activeDBManager = newActiveDBManager(blz.commondbManager, blz.blzLeafWidth)
-	blz.dbCache = newDBCache(blz.openedCacheSize, blz.commondbManager)
+	blz.dbCache = newDBCache(blz.openedCacheSize, blz.openedCacheTTL, blz.commondbManager)
 	blz.deleteProtectedObjects = newAddressMap()
 	blz.dbFilesGuard = &sync.RWMutex{}
 	blz.rebuildGuard = &sync.RWMutex{}

pkg/local_object_storage/blobstor/blobovniczatree/cache.go

@@ -1,10 +1,11 @@
 package blobovniczatree
 
 import (
-	"fmt"
 	"sync"
+	"time"
 
 	utilSync "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/sync"
+	"github.com/hashicorp/golang-lru/v2/expirable"
 	"github.com/hashicorp/golang-lru/v2/simplelru"
 )
 
@@ -22,14 +23,10 @@ type dbCache struct {
 	dbManager *dbManager
 }
 
-func newDBCache(size int, dbManager *dbManager) *dbCache {
+func newDBCache(size int, ttl time.Duration, dbManager *dbManager) *dbCache {
-	cache, err := simplelru.NewLRU[string, *sharedDB](size, func(_ string, evictedDB *sharedDB) {
+	cache := expirable.NewLRU[string, *sharedDB](size, func(_ string, evictedDB *sharedDB) {
 		evictedDB.Close()
-	})
+	}, ttl)
-	if err != nil {
-		// occurs only if the size is not positive
-		panic(fmt.Errorf("could not create LRU cache of size %d: %w", size, err))
-	}
 	return &dbCache{
 		cacheGuard: &sync.RWMutex{},
 		cache:      cache,

pkg/local_object_storage/blobstor/blobovniczatree/option.go

@@ -16,6 +16,7 @@ type cfg struct {
 	readOnly           bool
 	rootPath           string
 	openedCacheSize    int
+	openedCacheTTL     time.Duration
 	blzShallowDepth    uint64
 	blzShallowWidth    uint64
 	blzLeafWidth       uint64
@@ -34,6 +35,7 @@ type Option func(*cfg)
 const (
 	defaultPerm               = 0o700
 	defaultOpenedCacheSize    = 50
+	defaultOpenedCacheTTL     = 0 // means expiring is off
 	defaultBlzShallowDepth    = 2
 	defaultBlzShallowWidth    = 16
 	defaultWaitBeforeDropDB   = 10 * time.Second
@@ -46,6 +48,7 @@ func initConfig(c *cfg) {
 		log:                &logger.Logger{Logger: zap.L()},
 		perm:               defaultPerm,
 		openedCacheSize:    defaultOpenedCacheSize,
+		openedCacheTTL:     defaultOpenedCacheTTL,
 		blzShallowDepth:    defaultBlzShallowDepth,
 		blzShallowWidth:    defaultBlzShallowWidth,
 		reportError:        func(string, error) {},
@@ -105,6 +108,12 @@ func WithOpenedCacheSize(sz int) Option {
 	}
 }
 
+func WithOpenedCacheTTL(ttl time.Duration) Option {
+	return func(c *cfg) {
+		c.openedCacheTTL = ttl
+	}
+}
+
 func WithObjectSizeLimit(sz uint64) Option {
 	return func(c *cfg) {
 		c.blzOpts = append(c.blzOpts, blobovnicza.WithObjectSizeLimit(sz))