From ae5bb87e7087245d775352eb2c1a74dd5d38468e Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Fri, 1 Mar 2024 19:29:33 +0300 Subject: [PATCH] Revert "[#866] Use TTL for blobovnicza tree cache" This reverts commit d9cbb16bd3cec547ba9621a51ac3d8b0d75b211b. Signed-off-by: Anton Nikiforov --- .../internal/modules/storagecfg/config.go | 1 - cmd/frostfs-node/config.go | 3 --- .../shard/blobstor/blobovnicza/config.go | 21 ------------------- config/example/node.env | 2 -- config/example/node.json | 2 -- config/example/node.yaml | 1 - docs/storage-node-configuration.md | 18 +++++++--------- .../blobstor/blobovniczatree/blobovnicza.go | 2 +- .../blobstor/blobovniczatree/cache.go | 13 +++++++----- .../blobstor/blobovniczatree/option.go | 9 -------- 10 files changed, 17 insertions(+), 55 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/storagecfg/config.go b/cmd/frostfs-adm/internal/modules/storagecfg/config.go index 9f7efe2df..a07ce32c6 100644 --- a/cmd/frostfs-adm/internal/modules/storagecfg/config.go +++ b/cmd/frostfs-adm/internal/modules/storagecfg/config.go @@ -61,7 +61,6 @@ storage: depth: 1 # max depth of object tree storage in key-value DB width: 4 # max width of object tree storage in key-value DB opened_cache_capacity: 50 # maximum number of opened database files - opened_cache_ttl: 5m # ttl for opened database file gc: remover_batch_size: 200 # number of objects to be removed by the garbage collector diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 50219a8c7..360fdbec6 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -182,7 +182,6 @@ type subStorageCfg struct { width uint64 leafWidth uint64 openedCacheSize int - openedCacheTTL time.Duration initWorkerCount int initInAdvance bool } @@ -302,7 +301,6 @@ func (a *applicationConfiguration) setShardStorageConfig(newConfig *shardCfg, ol sCfg.width = sub.ShallowWidth() sCfg.leafWidth = sub.LeafWidth() sCfg.openedCacheSize = sub.OpenedCacheSize() - sCfg.openedCacheTTL = sub.OpenedCacheTTL() sCfg.initWorkerCount = sub.InitWorkerCount() sCfg.initInAdvance = sub.InitInAdvance() case fstree.Type: @@ -887,7 +885,6 @@ func (c *cfg) getSubstorageOpts(shCfg shardCfg) []blobstor.SubStorage { blobovniczatree.WithBlobovniczaShallowWidth(sRead.width), blobovniczatree.WithBlobovniczaLeafWidth(sRead.leafWidth), blobovniczatree.WithOpenedCacheSize(sRead.openedCacheSize), - blobovniczatree.WithOpenedCacheTTL(sRead.openedCacheTTL), blobovniczatree.WithInitWorkerCount(sRead.initWorkerCount), blobovniczatree.WithInitInAdvance(sRead.initInAdvance), blobovniczatree.WithLogger(c.log), diff --git a/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go b/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go index 0e007b425..37191cc72 100644 --- a/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go +++ b/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go @@ -1,8 +1,6 @@ package blobovniczaconfig import ( - "time" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" boltdbconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/boltdb" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" @@ -25,9 +23,6 @@ const ( // OpenedCacheSizeDefault is a default cache size of opened Blobovnicza's. OpenedCacheSizeDefault = 16 - // OpenedCacheTTLDefault is a default cache ttl of opened Blobovnicza's. - OpenedCacheTTLDefault = 0 // means expiring is off - // InitWorkerCountDefault is a default workers count to initialize Blobovnicza's. InitWorkerCountDefault = 5 ) @@ -106,22 +101,6 @@ func (x *Config) OpenedCacheSize() int { return OpenedCacheSizeDefault } -// OpenedCacheTTL returns the value of "opened_cache_ttl" config parameter. -// -// Returns OpenedCacheTTLDefault if the value is not a positive number. -func (x *Config) OpenedCacheTTL() time.Duration { - d := config.DurationSafe( - (*config.Config)(x), - "opened_cache_ttl", - ) - - if d > 0 { - return d - } - - return OpenedCacheTTLDefault -} - // BoltDB returns config instance for querying bolt db specific parameters. func (x *Config) BoltDB() *boltdbconfig.Config { return (*boltdbconfig.Config)(x) diff --git a/config/example/node.env b/config/example/node.env index a1db0c876..b0364b741 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -126,7 +126,6 @@ FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_SIZE=4194304 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_DEPTH=1 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_WIDTH=4 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_OPENED_CACHE_CAPACITY=50 -FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_OPENED_CACHE_TTL=5m FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_LEAF_WIDTH=10 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_INIT_WORKER_COUNT=10 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_INIT_IN_ADVANCE=TRUE @@ -176,7 +175,6 @@ FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_SIZE=4194304 FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_DEPTH=1 FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_WIDTH=4 FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_OPENED_CACHE_CAPACITY=50 -FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_OPENED_CACHE_TTL=5m FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_LEAF_WIDTH=10 ### FSTree config FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_TYPE=fstree diff --git a/config/example/node.json b/config/example/node.json index a7d7a3651..495fcb8ce 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -174,7 +174,6 @@ "depth": 1, "width": 4, "opened_cache_capacity": 50, - "opened_cache_ttl": "5m", "leaf_width": 10, "init_worker_count": 10, "init_in_advance": true @@ -227,7 +226,6 @@ "depth": 1, "width": 4, "opened_cache_capacity": 50, - "opened_cache_ttl": "5m", "leaf_width": 10 }, { diff --git a/config/example/node.yaml b/config/example/node.yaml index 678ee1a87..96af3b208 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -150,7 +150,6 @@ storage: depth: 1 # max depth of object tree storage in key-value DB width: 4 # max width of object tree storage in key-value DB opened_cache_capacity: 50 # maximum number of opened database files - opened_cache_ttl: 5m # ttl for opened database file leaf_width: 10 # max count of key-value DB on leafs of object tree storage - perm: 0644 # permissions for blobstor files(directories: +x for current user and group) depth: 5 # max depth of object tree storage in FS diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index e2334f8fb..6573e3ea2 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -209,7 +209,6 @@ blobstor: depth: 1 width: 4 opened_cache_capacity: 50 - opened_cache_ttl: 5m ``` #### Common options for sub-storages @@ -226,15 +225,14 @@ blobstor: | `depth` | `int` | `4` | File-system tree depth. | #### `blobovnicza` type options -| Parameter | Type | Default value | Description | -|-------------------------|------------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `path` | `string` | | Path to the root of the blobstor. | -| `perm` | file mode | `0660` | Default permission for created files and directories. | -| `size` | `size` | `1 G` | Maximum size of a single blobovnicza | -| `depth` | `int` | `2` | Blobovnicza tree depth. | -| `width` | `int` | `16` | Blobovnicza tree width. | -| `opened_cache_capacity` | `int` | `16` | Maximum number of simultaneously opened blobovniczas. | -| `opened_cache_ttl` | `duration` | `0` | TTL in cache for opened blobovniczas(disabled by default). In case of heavy random-read and 10 shards each with 10_000 databases and accessing 400 objects per-second we will access each db approximately once per ((10 * 10_000 / 400) = 250 seconds <= 300 seconds = 5 min). Also take in mind that in this scenario they will probably be closed earlier because of the cache capacity, so bigger values are likely to be of no use. | +| Parameter | Type | Default value | Description | +|-------------------------|-----------|---------------|-------------------------------------------------------| +| `path` | `string` | | Path to the root of the blobstor. | +| `perm` | file mode | `0660` | Default permission for created files and directories. | +| `size` | `size` | `1 G` | Maximum size of a single blobovnicza | +| `depth` | `int` | `2` | Blobovnicza tree depth. | +| `width` | `int` | `16` | Blobovnicza tree width. | +| `opened_cache_capacity` | `int` | `16` | Maximum number of simultaneously opened blobovniczas. | ### `gc` subsection diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go index 460aebbba..9f2950094 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go @@ -87,7 +87,7 @@ func NewBlobovniczaTree(opts ...Option) (blz *Blobovniczas) { blz.commondbManager = newDBManager(blz.rootPath, blz.blzOpts, blz.readOnly, blz.metrics.Blobovnicza(), blz.log) blz.activeDBManager = newActiveDBManager(blz.commondbManager, blz.blzLeafWidth) - blz.dbCache = newDBCache(blz.openedCacheSize, blz.openedCacheTTL, blz.commondbManager) + blz.dbCache = newDBCache(blz.openedCacheSize, blz.commondbManager) blz.deleteProtectedObjects = newAddressMap() blz.dbFilesGuard = &sync.RWMutex{} blz.rebuildGuard = &sync.RWMutex{} diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/cache.go b/pkg/local_object_storage/blobstor/blobovniczatree/cache.go index ef1793edf..766f5ef0a 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/cache.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/cache.go @@ -1,11 +1,10 @@ package blobovniczatree import ( + "fmt" "sync" - "time" utilSync "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/sync" - "github.com/hashicorp/golang-lru/v2/expirable" "github.com/hashicorp/golang-lru/v2/simplelru" ) @@ -23,10 +22,14 @@ type dbCache struct { dbManager *dbManager } -func newDBCache(size int, ttl time.Duration, dbManager *dbManager) *dbCache { - cache := expirable.NewLRU(size, func(_ string, evictedDB *sharedDB) { +func newDBCache(size int, dbManager *dbManager) *dbCache { + cache, err := simplelru.NewLRU(size, func(_ string, evictedDB *sharedDB) { evictedDB.Close() - }, ttl) + }) + if err != nil { + // occurs only if the size is not positive + panic(fmt.Errorf("could not create LRU cache of size %d: %w", size, err)) + } return &dbCache{ cacheGuard: &sync.RWMutex{}, cache: cache, diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/option.go b/pkg/local_object_storage/blobstor/blobovniczatree/option.go index e93640fae..d36074dc8 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/option.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/option.go @@ -16,7 +16,6 @@ type cfg struct { readOnly bool rootPath string openedCacheSize int - openedCacheTTL time.Duration blzShallowDepth uint64 blzShallowWidth uint64 blzLeafWidth uint64 @@ -35,7 +34,6 @@ type Option func(*cfg) const ( defaultPerm = 0o700 defaultOpenedCacheSize = 50 - defaultOpenedCacheTTL = 0 // means expiring is off defaultBlzShallowDepth = 2 defaultBlzShallowWidth = 16 defaultWaitBeforeDropDB = 10 * time.Second @@ -48,7 +46,6 @@ func initConfig(c *cfg) { log: &logger.Logger{Logger: zap.L()}, perm: defaultPerm, openedCacheSize: defaultOpenedCacheSize, - openedCacheTTL: defaultOpenedCacheTTL, blzShallowDepth: defaultBlzShallowDepth, blzShallowWidth: defaultBlzShallowWidth, reportError: func(string, error) {}, @@ -108,12 +105,6 @@ func WithOpenedCacheSize(sz int) Option { } } -func WithOpenedCacheTTL(ttl time.Duration) Option { - return func(c *cfg) { - c.openedCacheTTL = ttl - } -} - func WithObjectSizeLimit(sz uint64) Option { return func(c *cfg) { c.blzOpts = append(c.blzOpts, blobovnicza.WithObjectSizeLimit(sz))