TrueCloudLab/frostfs-node
21
1
Fork 27
Code Issues 108 Pull requests 3 Projects Releases 1 Activity Actions

Revert "[#866] Use TTL for blobovnicza tree cache" #1018

Merged
dstepanov-yadro merged 1 commit from acid-ant/frostfs-node:bugfix/blob-cache-ttl-revert into master 2024-03-04 06:55:52 +00:00
Conversation 0 Commits 1 Files changed 10 +17 -55
10 changed files with 17 additions and 55 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
cmd/frostfs-adm/internal/modules/storagecfg/config.go
View file

@ -61,7 +61,6 @@ storage:
depth: 1 # max depth of object tree storage in key-value DB depth: 1 # max depth of object tree storage in key-value DB
width: 4 # max width of object tree storage in key-value DB width: 4 # max width of object tree storage in key-value DB
opened_cache_capacity: 50 # maximum number of opened database files opened_cache_capacity: 50 # maximum number of opened database files
opened_cache_ttl: 5m # ttl for opened database file
gc: gc:
remover_batch_size: 200 # number of objects to be removed by the garbage collector remover_batch_size: 200 # number of objects to be removed by the garbage collector

3
cmd/frostfs-node/config.go
View file

@ -182,7 +182,6 @@ type subStorageCfg struct {
width uint64 width uint64
leafWidth uint64 leafWidth uint64
openedCacheSize int openedCacheSize int
openedCacheTTL time.Duration
initWorkerCount int initWorkerCount int
initInAdvance bool initInAdvance bool
} }
@ -302,7 +301,6 @@ func (a *applicationConfiguration) setShardStorageConfig(newConfig *shardCfg, ol
sCfg.width = sub.ShallowWidth() sCfg.width = sub.ShallowWidth()
sCfg.leafWidth = sub.LeafWidth() sCfg.leafWidth = sub.LeafWidth()
sCfg.openedCacheSize = sub.OpenedCacheSize() sCfg.openedCacheSize = sub.OpenedCacheSize()
sCfg.openedCacheTTL = sub.OpenedCacheTTL()
sCfg.initWorkerCount = sub.InitWorkerCount() sCfg.initWorkerCount = sub.InitWorkerCount()
sCfg.initInAdvance = sub.InitInAdvance() sCfg.initInAdvance = sub.InitInAdvance()
case fstree.Type: case fstree.Type:
@ -887,7 +885,6 @@ func (c *cfg) getSubstorageOpts(shCfg shardCfg) []blobstor.SubStorage {
blobovniczatree.WithBlobovniczaShallowWidth(sRead.width), blobovniczatree.WithBlobovniczaShallowWidth(sRead.width),
blobovniczatree.WithBlobovniczaLeafWidth(sRead.leafWidth), blobovniczatree.WithBlobovniczaLeafWidth(sRead.leafWidth),
blobovniczatree.WithOpenedCacheSize(sRead.openedCacheSize), blobovniczatree.WithOpenedCacheSize(sRead.openedCacheSize),
blobovniczatree.WithOpenedCacheTTL(sRead.openedCacheTTL),
blobovniczatree.WithInitWorkerCount(sRead.initWorkerCount), blobovniczatree.WithInitWorkerCount(sRead.initWorkerCount),
blobovniczatree.WithInitInAdvance(sRead.initInAdvance), blobovniczatree.WithInitInAdvance(sRead.initInAdvance),
blobovniczatree.WithLogger(c.log), blobovniczatree.WithLogger(c.log),

21
cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go
View file

@ -1,8 +1,6 @@
package blobovniczaconfig package blobovniczaconfig
import ( import (
"time"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config"
boltdbconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/boltdb" boltdbconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/boltdb"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree"
@ -25,9 +23,6 @@ const (
// OpenedCacheSizeDefault is a default cache size of opened Blobovnicza's. // OpenedCacheSizeDefault is a default cache size of opened Blobovnicza's.
OpenedCacheSizeDefault = 16 OpenedCacheSizeDefault = 16
// OpenedCacheTTLDefault is a default cache ttl of opened Blobovnicza's.
OpenedCacheTTLDefault = 0 // means expiring is off
// InitWorkerCountDefault is a default workers count to initialize Blobovnicza's. // InitWorkerCountDefault is a default workers count to initialize Blobovnicza's.
InitWorkerCountDefault = 5 InitWorkerCountDefault = 5
) )
@ -106,22 +101,6 @@ func (x *Config) OpenedCacheSize() int {
return OpenedCacheSizeDefault return OpenedCacheSizeDefault
} }
// OpenedCacheTTL returns the value of "opened_cache_ttl" config parameter.
//
// Returns OpenedCacheTTLDefault if the value is not a positive number.
func (x *Config) OpenedCacheTTL() time.Duration {
d := config.DurationSafe(
(*config.Config)(x),
"opened_cache_ttl",
)
if d > 0 {
return d
}
return OpenedCacheTTLDefault
}
// BoltDB returns config instance for querying bolt db specific parameters. // BoltDB returns config instance for querying bolt db specific parameters.
func (x *Config) BoltDB() *boltdbconfig.Config { func (x *Config) BoltDB() *boltdbconfig.Config {
return (*boltdbconfig.Config)(x) return (*boltdbconfig.Config)(x)

2
config/example/node.env
View file

@ -126,7 +126,6 @@ FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_SIZE=4194304
FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_DEPTH=1 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_DEPTH=1
FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_WIDTH=4 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_WIDTH=4
FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_OPENED_CACHE_CAPACITY=50 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_OPENED_CACHE_CAPACITY=50
FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_OPENED_CACHE_TTL=5m
FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_LEAF_WIDTH=10 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_LEAF_WIDTH=10
FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_INIT_WORKER_COUNT=10 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_INIT_WORKER_COUNT=10
FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_INIT_IN_ADVANCE=TRUE FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_INIT_IN_ADVANCE=TRUE
@ -176,7 +175,6 @@ FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_SIZE=4194304
FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_DEPTH=1 FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_DEPTH=1
FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_WIDTH=4 FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_WIDTH=4
FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_OPENED_CACHE_CAPACITY=50 FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_OPENED_CACHE_CAPACITY=50
FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_OPENED_CACHE_TTL=5m
FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_LEAF_WIDTH=10 FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_LEAF_WIDTH=10
### FSTree config ### FSTree config
FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_TYPE=fstree FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_TYPE=fstree

2
config/example/node.json
View file

@ -174,7 +174,6 @@
"depth": 1, "depth": 1,
"width": 4, "width": 4,
"opened_cache_capacity": 50, "opened_cache_capacity": 50,
"opened_cache_ttl": "5m",
"leaf_width": 10, "leaf_width": 10,
"init_worker_count": 10, "init_worker_count": 10,
"init_in_advance": true "init_in_advance": true
@ -227,7 +226,6 @@
"depth": 1, "depth": 1,
"width": 4, "width": 4,
"opened_cache_capacity": 50, "opened_cache_capacity": 50,
"opened_cache_ttl": "5m",
"leaf_width": 10 "leaf_width": 10
}, },
{ {

1
config/example/node.yaml
View file

@ -150,7 +150,6 @@ storage:
depth: 1 # max depth of object tree storage in key-value DB depth: 1 # max depth of object tree storage in key-value DB
width: 4 # max width of object tree storage in key-value DB width: 4 # max width of object tree storage in key-value DB
opened_cache_capacity: 50 # maximum number of opened database files opened_cache_capacity: 50 # maximum number of opened database files
opened_cache_ttl: 5m # ttl for opened database file
leaf_width: 10 # max count of key-value DB on leafs of object tree storage leaf_width: 10 # max count of key-value DB on leafs of object tree storage
- perm: 0644 # permissions for blobstor files(directories: +x for current user and group) - perm: 0644 # permissions for blobstor files(directories: +x for current user and group)
depth: 5 # max depth of object tree storage in FS depth: 5 # max depth of object tree storage in FS

18
docs/storage-node-configuration.md
View file

@ -209,7 +209,6 @@ blobstor:
depth: 1 depth: 1
width: 4 width: 4
opened_cache_capacity: 50 opened_cache_capacity: 50
opened_cache_ttl: 5m
``` ```
#### Common options for sub-storages #### Common options for sub-storages
@ -226,15 +225,14 @@ blobstor:
| `depth` | `int` | `4` | File-system tree depth. | | `depth` | `int` | `4` | File-system tree depth. |
#### `blobovnicza` type options #### `blobovnicza` type options
| Parameter | Type | Default value | Description | | Parameter | Type | Default value | Description |
|-------------------------|------------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| |-------------------------|-----------|---------------|-------------------------------------------------------|
| `path` | `string` | | Path to the root of the blobstor. | | `path` | `string` | | Path to the root of the blobstor. |
| `perm` | file mode | `0660` | Default permission for created files and directories. | | `perm` | file mode | `0660` | Default permission for created files and directories. |
| `size` | `size` | `1 G` | Maximum size of a single blobovnicza | | `size` | `size` | `1 G` | Maximum size of a single blobovnicza |
| `depth` | `int` | `2` | Blobovnicza tree depth. | | `depth` | `int` | `2` | Blobovnicza tree depth. |
| `width` | `int` | `16` | Blobovnicza tree width. | | `width` | `int` | `16` | Blobovnicza tree width. |
| `opened_cache_capacity` | `int` | `16` | Maximum number of simultaneously opened blobovniczas. | | `opened_cache_capacity` | `int` | `16` | Maximum number of simultaneously opened blobovniczas. |
| `opened_cache_ttl` | `duration` | `0` | TTL in cache for opened blobovniczas(disabled by default). In case of heavy random-read and 10 shards each with 10_000 databases and accessing 400 objects per-second we will access each db approximately once per ((10 * 10_000 / 400) = 250 seconds <= 300 seconds = 5 min). Also take in mind that in this scenario they will probably be closed earlier because of the cache capacity, so bigger values are likely to be of no use. |
### `gc` subsection ### `gc` subsection

2
pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go
View file

@ -87,7 +87,7 @@ func NewBlobovniczaTree(opts ...Option) (blz *Blobovniczas) {
blz.commondbManager = newDBManager(blz.rootPath, blz.blzOpts, blz.readOnly, blz.metrics.Blobovnicza(), blz.log) blz.commondbManager = newDBManager(blz.rootPath, blz.blzOpts, blz.readOnly, blz.metrics.Blobovnicza(), blz.log)
blz.activeDBManager = newActiveDBManager(blz.commondbManager, blz.blzLeafWidth) blz.activeDBManager = newActiveDBManager(blz.commondbManager, blz.blzLeafWidth)
blz.dbCache = newDBCache(blz.openedCacheSize, blz.openedCacheTTL, blz.commondbManager) blz.dbCache = newDBCache(blz.openedCacheSize, blz.commondbManager)
blz.deleteProtectedObjects = newAddressMap() blz.deleteProtectedObjects = newAddressMap()
blz.dbFilesGuard = &sync.RWMutex{} blz.dbFilesGuard = &sync.RWMutex{}
blz.rebuildGuard = &sync.RWMutex{} blz.rebuildGuard = &sync.RWMutex{}

13
pkg/local_object_storage/blobstor/blobovniczatree/cache.go
View file

@ -1,11 +1,10 @@
package blobovniczatree package blobovniczatree
import ( import (
"fmt"
"sync" "sync"
"time"
utilSync "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/sync" utilSync "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/sync"
"github.com/hashicorp/golang-lru/v2/expirable"
"github.com/hashicorp/golang-lru/v2/simplelru" "github.com/hashicorp/golang-lru/v2/simplelru"
) )
@ -23,10 +22,14 @@ type dbCache struct {
dbManager *dbManager dbManager *dbManager
} }
func newDBCache(size int, ttl time.Duration, dbManager *dbManager) *dbCache { func newDBCache(size int, dbManager *dbManager) *dbCache {
cache := expirable.NewLRU(size, func(_ string, evictedDB *sharedDB) { cache, err := simplelru.NewLRU(size, func(_ string, evictedDB *sharedDB) {
evictedDB.Close() evictedDB.Close()
}, ttl) })
if err != nil {
// occurs only if the size is not positive
panic(fmt.Errorf("could not create LRU cache of size %d: %w", size, err))
}
return &dbCache{ return &dbCache{
cacheGuard: &sync.RWMutex{}, cacheGuard: &sync.RWMutex{},
cache: cache, cache: cache,

9
pkg/local_object_storage/blobstor/blobovniczatree/option.go
View file

@ -16,7 +16,6 @@ type cfg struct {
readOnly bool readOnly bool
rootPath string rootPath string
openedCacheSize int openedCacheSize int
openedCacheTTL time.Duration
blzShallowDepth uint64 blzShallowDepth uint64
blzShallowWidth uint64 blzShallowWidth uint64
blzLeafWidth uint64 blzLeafWidth uint64
@ -35,7 +34,6 @@ type Option func(*cfg)
const ( const (
defaultPerm = 0o700 defaultPerm = 0o700
defaultOpenedCacheSize = 50 defaultOpenedCacheSize = 50
defaultOpenedCacheTTL = 0 // means expiring is off
defaultBlzShallowDepth = 2 defaultBlzShallowDepth = 2
defaultBlzShallowWidth = 16 defaultBlzShallowWidth = 16
defaultWaitBeforeDropDB = 10 * time.Second defaultWaitBeforeDropDB = 10 * time.Second
@ -48,7 +46,6 @@ func initConfig(c *cfg) {
log: &logger.Logger{Logger: zap.L()}, log: &logger.Logger{Logger: zap.L()},
perm: defaultPerm, perm: defaultPerm,
openedCacheSize: defaultOpenedCacheSize, openedCacheSize: defaultOpenedCacheSize,
openedCacheTTL: defaultOpenedCacheTTL,
blzShallowDepth: defaultBlzShallowDepth, blzShallowDepth: defaultBlzShallowDepth,
blzShallowWidth: defaultBlzShallowWidth, blzShallowWidth: defaultBlzShallowWidth,
reportError: func(string, error) {}, reportError: func(string, error) {},
@ -108,12 +105,6 @@ func WithOpenedCacheSize(sz int) Option {
} }
} }
func WithOpenedCacheTTL(ttl time.Duration) Option {
return func(c *cfg) {
c.openedCacheTTL = ttl
}
}
func WithObjectSizeLimit(sz uint64) Option { func WithObjectSizeLimit(sz uint64) Option {
return func(c *cfg) { return func(c *cfg) {
c.blzOpts = append(c.blzOpts, blobovnicza.WithObjectSizeLimit(sz)) c.blzOpts = append(c.blzOpts, blobovnicza.WithObjectSizeLimit(sz))