From d9cbb16bd3cec547ba9621a51ac3d8b0d75b211b Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Mon, 18 Dec 2023 12:47:32 +0300 Subject: [PATCH] [#866] Use TTL for blobovnicza tree cache Signed-off-by: Anton Nikiforov --- .../internal/modules/storagecfg/config.go | 1 + cmd/frostfs-node/config.go | 3 +++ .../shard/blobstor/blobovnicza/config.go | 21 +++++++++++++++++++ config/example/node.env | 2 ++ config/example/node.json | 2 ++ config/example/node.yaml | 1 + docs/storage-node-configuration.md | 18 +++++++++------- .../blobstor/blobovniczatree/blobovnicza.go | 2 +- .../blobstor/blobovniczatree/cache.go | 13 +++++------- .../blobstor/blobovniczatree/option.go | 9 ++++++++ 10 files changed, 55 insertions(+), 17 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/storagecfg/config.go b/cmd/frostfs-adm/internal/modules/storagecfg/config.go index a07ce32c6..9f7efe2df 100644 --- a/cmd/frostfs-adm/internal/modules/storagecfg/config.go +++ b/cmd/frostfs-adm/internal/modules/storagecfg/config.go @@ -61,6 +61,7 @@ storage: depth: 1 # max depth of object tree storage in key-value DB width: 4 # max width of object tree storage in key-value DB opened_cache_capacity: 50 # maximum number of opened database files + opened_cache_ttl: 5m # ttl for opened database file gc: remover_batch_size: 200 # number of objects to be removed by the garbage collector diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index e36081ba6..88b93e6aa 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -185,6 +185,7 @@ type subStorageCfg struct { width uint64 leafWidth uint64 openedCacheSize int + openedCacheTTL time.Duration initWorkerCount int initInAdvance bool } @@ -305,6 +306,7 @@ func (a *applicationConfiguration) setShardStorageConfig(newConfig *shardCfg, ol sCfg.width = sub.ShallowWidth() sCfg.leafWidth = sub.LeafWidth() sCfg.openedCacheSize = sub.OpenedCacheSize() + sCfg.openedCacheTTL = sub.OpenedCacheTTL() sCfg.initWorkerCount = sub.InitWorkerCount() sCfg.initInAdvance = sub.InitInAdvance() case fstree.Type: @@ -898,6 +900,7 @@ func (c *cfg) getSubstorageOpts(shCfg shardCfg) []blobstor.SubStorage { blobovniczatree.WithBlobovniczaShallowWidth(sRead.width), blobovniczatree.WithBlobovniczaLeafWidth(sRead.leafWidth), blobovniczatree.WithOpenedCacheSize(sRead.openedCacheSize), + blobovniczatree.WithOpenedCacheTTL(sRead.openedCacheTTL), blobovniczatree.WithInitWorkerCount(sRead.initWorkerCount), blobovniczatree.WithInitInAdvance(sRead.initInAdvance), blobovniczatree.WithLogger(c.log), diff --git a/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go b/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go index 37191cc72..0e007b425 100644 --- a/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go +++ b/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go @@ -1,6 +1,8 @@ package blobovniczaconfig import ( + "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" boltdbconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/boltdb" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" @@ -23,6 +25,9 @@ const ( // OpenedCacheSizeDefault is a default cache size of opened Blobovnicza's. OpenedCacheSizeDefault = 16 + // OpenedCacheTTLDefault is a default cache ttl of opened Blobovnicza's. + OpenedCacheTTLDefault = 0 // means expiring is off + // InitWorkerCountDefault is a default workers count to initialize Blobovnicza's. InitWorkerCountDefault = 5 ) @@ -101,6 +106,22 @@ func (x *Config) OpenedCacheSize() int { return OpenedCacheSizeDefault } +// OpenedCacheTTL returns the value of "opened_cache_ttl" config parameter. +// +// Returns OpenedCacheTTLDefault if the value is not a positive number. +func (x *Config) OpenedCacheTTL() time.Duration { + d := config.DurationSafe( + (*config.Config)(x), + "opened_cache_ttl", + ) + + if d > 0 { + return d + } + + return OpenedCacheTTLDefault +} + // BoltDB returns config instance for querying bolt db specific parameters. func (x *Config) BoltDB() *boltdbconfig.Config { return (*boltdbconfig.Config)(x) diff --git a/config/example/node.env b/config/example/node.env index be16f62bf..2972f56e9 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -125,6 +125,7 @@ FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_SIZE=4194304 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_DEPTH=1 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_WIDTH=4 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_OPENED_CACHE_CAPACITY=50 +FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_OPENED_CACHE_TTL=5m FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_LEAF_WIDTH=10 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_INIT_WORKER_COUNT=10 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_INIT_IN_ADVANCE=TRUE @@ -174,6 +175,7 @@ FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_SIZE=4194304 FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_DEPTH=1 FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_WIDTH=4 FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_OPENED_CACHE_CAPACITY=50 +FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_OPENED_CACHE_TTL=5m FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_LEAF_WIDTH=10 ### FSTree config FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_TYPE=fstree diff --git a/config/example/node.json b/config/example/node.json index ff38e3894..c803e11fb 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -173,6 +173,7 @@ "depth": 1, "width": 4, "opened_cache_capacity": 50, + "opened_cache_ttl": "5m", "leaf_width": 10, "init_worker_count": 10, "init_in_advance": true @@ -226,6 +227,7 @@ "depth": 1, "width": 4, "opened_cache_capacity": 50, + "opened_cache_ttl": "5m", "leaf_width": 10 }, { diff --git a/config/example/node.yaml b/config/example/node.yaml index c6a2be9ac..d4c3516bc 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -150,6 +150,7 @@ storage: depth: 1 # max depth of object tree storage in key-value DB width: 4 # max width of object tree storage in key-value DB opened_cache_capacity: 50 # maximum number of opened database files + opened_cache_ttl: 5m # ttl for opened database file leaf_width: 10 # max count of key-value DB on leafs of object tree storage - perm: 0644 # permissions for blobstor files(directories: +x for current user and group) depth: 5 # max depth of object tree storage in FS diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 71eabc042..613be1288 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -209,6 +209,7 @@ blobstor: depth: 1 width: 4 opened_cache_capacity: 50 + opened_cache_ttl: 5m ``` #### Common options for sub-storages @@ -225,14 +226,15 @@ blobstor: | `depth` | `int` | `4` | File-system tree depth. | #### `blobovnicza` type options -| Parameter | Type | Default value | Description | -|-------------------------|-----------|---------------|-------------------------------------------------------| -| `path` | `string` | | Path to the root of the blobstor. | -| `perm` | file mode | `0660` | Default permission for created files and directories. | -| `size` | `size` | `1 G` | Maximum size of a single blobovnicza | -| `depth` | `int` | `2` | Blobovnicza tree depth. | -| `width` | `int` | `16` | Blobovnicza tree width. | -| `opened_cache_capacity` | `int` | `16` | Maximum number of simultaneously opened blobovniczas. | +| Parameter | Type | Default value | Description | +|-------------------------|------------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `path` | `string` | | Path to the root of the blobstor. | +| `perm` | file mode | `0660` | Default permission for created files and directories. | +| `size` | `size` | `1 G` | Maximum size of a single blobovnicza | +| `depth` | `int` | `2` | Blobovnicza tree depth. | +| `width` | `int` | `16` | Blobovnicza tree width. | +| `opened_cache_capacity` | `int` | `16` | Maximum number of simultaneously opened blobovniczas. | +| `opened_cache_ttl` | `duration` | `0` | TTL in cache for opened blobovniczas(disabled by default). In case of heavy random-read and 10 shards each with 10_000 databases and accessing 400 objects per-second we will access each db approximately once per ((10 * 10_000 / 400) = 250 seconds <= 300 seconds = 5 min). Also take in mind that in this scenario they will probably be closed earlier because of the cache capacity, so bigger values are likely to be of no use. | ### `gc` subsection diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go index 9f2950094..460aebbba 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go @@ -87,7 +87,7 @@ func NewBlobovniczaTree(opts ...Option) (blz *Blobovniczas) { blz.commondbManager = newDBManager(blz.rootPath, blz.blzOpts, blz.readOnly, blz.metrics.Blobovnicza(), blz.log) blz.activeDBManager = newActiveDBManager(blz.commondbManager, blz.blzLeafWidth) - blz.dbCache = newDBCache(blz.openedCacheSize, blz.commondbManager) + blz.dbCache = newDBCache(blz.openedCacheSize, blz.openedCacheTTL, blz.commondbManager) blz.deleteProtectedObjects = newAddressMap() blz.dbFilesGuard = &sync.RWMutex{} blz.rebuildGuard = &sync.RWMutex{} diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/cache.go b/pkg/local_object_storage/blobstor/blobovniczatree/cache.go index 317bac5ff..d8fe6e8b0 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/cache.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/cache.go @@ -1,10 +1,11 @@ package blobovniczatree import ( - "fmt" "sync" + "time" utilSync "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/sync" + "github.com/hashicorp/golang-lru/v2/expirable" "github.com/hashicorp/golang-lru/v2/simplelru" ) @@ -22,14 +23,10 @@ type dbCache struct { dbManager *dbManager } -func newDBCache(size int, dbManager *dbManager) *dbCache { - cache, err := simplelru.NewLRU[string, *sharedDB](size, func(_ string, evictedDB *sharedDB) { +func newDBCache(size int, ttl time.Duration, dbManager *dbManager) *dbCache { + cache := expirable.NewLRU[string, *sharedDB](size, func(_ string, evictedDB *sharedDB) { evictedDB.Close() - }) - if err != nil { - // occurs only if the size is not positive - panic(fmt.Errorf("could not create LRU cache of size %d: %w", size, err)) - } + }, ttl) return &dbCache{ cacheGuard: &sync.RWMutex{}, cache: cache, diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/option.go b/pkg/local_object_storage/blobstor/blobovniczatree/option.go index d36074dc8..e93640fae 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/option.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/option.go @@ -16,6 +16,7 @@ type cfg struct { readOnly bool rootPath string openedCacheSize int + openedCacheTTL time.Duration blzShallowDepth uint64 blzShallowWidth uint64 blzLeafWidth uint64 @@ -34,6 +35,7 @@ type Option func(*cfg) const ( defaultPerm = 0o700 defaultOpenedCacheSize = 50 + defaultOpenedCacheTTL = 0 // means expiring is off defaultBlzShallowDepth = 2 defaultBlzShallowWidth = 16 defaultWaitBeforeDropDB = 10 * time.Second @@ -46,6 +48,7 @@ func initConfig(c *cfg) { log: &logger.Logger{Logger: zap.L()}, perm: defaultPerm, openedCacheSize: defaultOpenedCacheSize, + openedCacheTTL: defaultOpenedCacheTTL, blzShallowDepth: defaultBlzShallowDepth, blzShallowWidth: defaultBlzShallowWidth, reportError: func(string, error) {}, @@ -105,6 +108,12 @@ func WithOpenedCacheSize(sz int) Option { } } +func WithOpenedCacheTTL(ttl time.Duration) Option { + return func(c *cfg) { + c.openedCacheTTL = ttl + } +} + func WithObjectSizeLimit(sz uint64) Option { return func(c *cfg) { c.blzOpts = append(c.blzOpts, blobovnicza.WithObjectSizeLimit(sz))