TrueCloudLab/frostfs-node
21
1
Fork 27
Code Issues 103 Pull requests 3 Projects Releases 2 Activity Actions 1

node: Use TTL for blobovnicza tree cache #875

Merged
fyrchik merged 1 commit from acid-ant/frostfs-node:bugfix/ttl-for-blobz-tree-cache into master 2023-12-19 16:36:34 +00:00
Conversation 0 Commits 1 Files changed 10 +55 -17
10 changed files with 55 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
cmd/frostfs-adm/internal/modules/storagecfg/config.go
View file

@ -61,6 +61,7 @@ storage:
depth: 1 # max depth of object tree storage in key-value DB
width: 4 # max width of object tree storage in key-value DB
opened_cache_capacity: 50 # maximum number of opened database files
opened_cache_ttl: 5m # ttl for opened database file
acid-ant commented 2023-12-18 09:57:56 +00:00
Outdated
Review
Copy link

20m used here just to start discussion.

`20m` used here just to start discussion.
fyrchik commented 2023-12-18 11:20:54 +00:00
Outdated
Review
Copy link

I would use smaller value here (5m or even 1m). PUT is frequently followed by GET, so we have some time to retain non-active blobovnicza open.
In case of heavy random-read and 10 shards each with 10_000 databases and accessing 400 objects per-second we will access each db approximately once per ((10 * 10_000 / 400) = 250 seconds <= 300 seconds = 5 min). Also take in mind that in this scenario they will probably be closed earlier because of the cache capacity, so bigger values are likely to be of no use.

I would use smaller value here (`5m` or even `1m`). PUT is frequently followed by GET, so we have some time to retain non-active blobovnicza open. In case of heavy random-read and 10 shards each with 10_000 databases and accessing 400 objects per-second we will access each db approximately once per ((10 * 10_000 / 400) = 250 seconds <= 300 seconds = 5 min). Also take in mind that in this scenario they will probably be closed earlier because of the cache capacity, so bigger values are likely to be of no use.
acid-ant commented 2023-12-19 08:13:51 +00:00
Outdated
Review
Copy link

Agree, I've added your comment in doc.

Agree, I've added your comment in doc.
gc:
remover_batch_size: 200 # number of objects to be removed by the garbage collector

3
cmd/frostfs-node/config.go
View file

@ -185,6 +185,7 @@ type subStorageCfg struct {
width uint64
leafWidth uint64
openedCacheSize int
openedCacheTTL time.Duration
initWorkerCount int
initInAdvance bool
}
@ -305,6 +306,7 @@ func (a *applicationConfiguration) setShardStorageConfig(newConfig *shardCfg, ol
sCfg.width = sub.ShallowWidth()
sCfg.leafWidth = sub.LeafWidth()
sCfg.openedCacheSize = sub.OpenedCacheSize()
sCfg.openedCacheTTL = sub.OpenedCacheTTL()
sCfg.initWorkerCount = sub.InitWorkerCount()
sCfg.initInAdvance = sub.InitInAdvance()
case fstree.Type:
@ -898,6 +900,7 @@ func (c *cfg) getSubstorageOpts(shCfg shardCfg) []blobstor.SubStorage {
blobovniczatree.WithBlobovniczaShallowWidth(sRead.width),
blobovniczatree.WithBlobovniczaLeafWidth(sRead.leafWidth),
blobovniczatree.WithOpenedCacheSize(sRead.openedCacheSize),
blobovniczatree.WithOpenedCacheTTL(sRead.openedCacheTTL),
blobovniczatree.WithInitWorkerCount(sRead.initWorkerCount),
blobovniczatree.WithInitInAdvance(sRead.initInAdvance),
blobovniczatree.WithLogger(c.log),

21
cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go
View file

@ -1,6 +1,8 @@
package blobovniczaconfig
import (
"time"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config"
boltdbconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/boltdb"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree"
@ -23,6 +25,9 @@ const (
// OpenedCacheSizeDefault is a default cache size of opened Blobovnicza's.
OpenedCacheSizeDefault = 16
// OpenedCacheTTLDefault is a default cache ttl of opened Blobovnicza's.
OpenedCacheTTLDefault = 0 // means expiring is off
fyrchik commented 2023-12-18 11:08:14 +00:00
Outdated
Review
Copy link

Maybe not close at all by default?

Maybe not close at all by default?
acid-ant commented 2023-12-18 12:28:47 +00:00
Outdated
Review
Copy link

Agree, let's leave current behavior.

Agree, let's leave current behavior.
// InitWorkerCountDefault is a default workers count to initialize Blobovnicza's.
InitWorkerCountDefault = 5
)
@ -101,6 +106,22 @@ func (x *Config) OpenedCacheSize() int {
return OpenedCacheSizeDefault
}
// OpenedCacheTTL returns the value of "opened_cache_ttl" config parameter.
//
// Returns OpenedCacheTTLDefault if the value is not a positive number.
func (x *Config) OpenedCacheTTL() time.Duration {
d := config.DurationSafe(
(*config.Config)(x),
"opened_cache_ttl",
)
if d > 0 {
return d
}
return OpenedCacheTTLDefault
}
// BoltDB returns config instance for querying bolt db specific parameters.
func (x *Config) BoltDB() *boltdbconfig.Config {
return (*boltdbconfig.Config)(x)

2
config/example/node.env
View file

@ -125,6 +125,7 @@ FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_SIZE=4194304
FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_DEPTH=1
FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_WIDTH=4
FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_OPENED_CACHE_CAPACITY=50
FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_OPENED_CACHE_TTL=5m
FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_LEAF_WIDTH=10
FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_INIT_WORKER_COUNT=10
FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_INIT_IN_ADVANCE=TRUE
@ -174,6 +175,7 @@ FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_SIZE=4194304
FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_DEPTH=1
FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_WIDTH=4
FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_OPENED_CACHE_CAPACITY=50
FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_OPENED_CACHE_TTL=5m
FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_LEAF_WIDTH=10
### FSTree config
FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_TYPE=fstree

2
config/example/node.json
View file

@ -173,6 +173,7 @@
"depth": 1,
"width": 4,
"opened_cache_capacity": 50,
"opened_cache_ttl": "5m",
"leaf_width": 10,
"init_worker_count": 10,
"init_in_advance": true
@ -226,6 +227,7 @@
"depth": 1,
"width": 4,
"opened_cache_capacity": 50,
"opened_cache_ttl": "5m",
"leaf_width": 10
},
{

1
config/example/node.yaml
View file

@ -150,6 +150,7 @@ storage:
depth: 1 # max depth of object tree storage in key-value DB
width: 4 # max width of object tree storage in key-value DB
opened_cache_capacity: 50 # maximum number of opened database files
opened_cache_ttl: 5m # ttl for opened database file
leaf_width: 10 # max count of key-value DB on leafs of object tree storage
- perm: 0644 # permissions for blobstor files(directories: +x for current user and group)
depth: 5 # max depth of object tree storage in FS

18
docs/storage-node-configuration.md
View file

@ -209,6 +209,7 @@ blobstor:
depth: 1
width: 4
opened_cache_capacity: 50
opened_cache_ttl: 5m
```
#### Common options for sub-storages
@ -225,14 +226,15 @@ blobstor:
| `depth` | `int` | `4` | File-system tree depth. |
#### `blobovnicza` type options
| Parameter | Type | Default value | Description |
|-------------------------|-----------|---------------|-------------------------------------------------------|
| `path` | `string` | | Path to the root of the blobstor. |
| `perm` | file mode | `0660` | Default permission for created files and directories. |
| `size` | `size` | `1 G` | Maximum size of a single blobovnicza |
| `depth` | `int` | `2` | Blobovnicza tree depth. |
| `width` | `int` | `16` | Blobovnicza tree width. |
| `opened_cache_capacity` | `int` | `16` | Maximum number of simultaneously opened blobovniczas. |
| Parameter | Type | Default value | Description |
|-------------------------|------------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `path` | `string` | | Path to the root of the blobstor. |
| `perm` | file mode | `0660` | Default permission for created files and directories. |
| `size` | `size` | `1 G` | Maximum size of a single blobovnicza |
| `depth` | `int` | `2` | Blobovnicza tree depth. |
| `width` | `int` | `16` | Blobovnicza tree width. |
| `opened_cache_capacity` | `int` | `16` | Maximum number of simultaneously opened blobovniczas. |
| `opened_cache_ttl` | `duration` | `0` | TTL in cache for opened blobovniczas(disabled by default). In case of heavy random-read and 10 shards each with 10_000 databases and accessing 400 objects per-second we will access each db approximately once per ((10 * 10_000 / 400) = 250 seconds <= 300 seconds = 5 min). Also take in mind that in this scenario they will probably be closed earlier because of the cache capacity, so bigger values are likely to be of no use. |
### `gc` subsection

2
pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go
View file

@ -87,7 +87,7 @@ func NewBlobovniczaTree(opts ...Option) (blz *Blobovniczas) {
blz.commondbManager = newDBManager(blz.rootPath, blz.blzOpts, blz.readOnly, blz.metrics.Blobovnicza(), blz.log)
blz.activeDBManager = newActiveDBManager(blz.commondbManager, blz.blzLeafWidth)
blz.dbCache = newDBCache(blz.openedCacheSize, blz.commondbManager)
blz.dbCache = newDBCache(blz.openedCacheSize, blz.openedCacheTTL, blz.commondbManager)
blz.deleteProtectedObjects = newAddressMap()
blz.dbFilesGuard = &sync.RWMutex{}
blz.rebuildGuard = &sync.RWMutex{}

13
pkg/local_object_storage/blobstor/blobovniczatree/cache.go
View file

@ -1,10 +1,11 @@
package blobovniczatree
import (
"fmt"
"sync"
"time"
utilSync "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/sync"
"github.com/hashicorp/golang-lru/v2/expirable"
"github.com/hashicorp/golang-lru/v2/simplelru"
)
@ -22,14 +23,10 @@ type dbCache struct {
dbManager *dbManager
}
func newDBCache(size int, dbManager *dbManager) *dbCache {
cache, err := simplelru.NewLRU[string, *sharedDB](size, func(_ string, evictedDB *sharedDB) {
func newDBCache(size int, ttl time.Duration, dbManager *dbManager) *dbCache {
cache := expirable.NewLRU[string, *sharedDB](size, func(_ string, evictedDB *sharedDB) {
fyrchik commented 2023-12-18 11:11:15 +00:00
Outdated
Review
Copy link

I am not sure expireable.LRU evicts and not just invalidates entries on access.
Have you checked that it has some sort of a timer?
Can we be sure that if get from cache fails the following Open() will succeed (i.e. database is surely closed)?

I am not sure `expireable.LRU` _evicts_ and not just invalidates entries on access. Have you checked that it has some sort of a timer? Can we be sure that if get from cache fails the following `Open()` will succeed (i.e. database is surely closed)?
acid-ant commented 2023-12-18 12:05:40 +00:00
Outdated
Review
Copy link

For the test purposes I've added debug log in evictcallback, so I'm sure that expirable cache exec this callback. Also, according to code, all access to the cache was done under the internal mutex - so onEvict will be executed before getting from cache.

For the test purposes I've added debug log in `evictcallback`, so I'm sure that expirable cache exec this callback. Also, according to code, all access to the cache was done under the internal [mutex](https://github.com/hashicorp/golang-lru/blob/97f49b45d00745bd1bbc7a64d9363e5edf216713/expirable/expirable_lru.go#L24) - so `onEvict` will be executed before getting from cache.
evictedDB.Close()
})
if err != nil {
// occurs only if the size is not positive
panic(fmt.Errorf("could not create LRU cache of size %d: %w", size, err))
}
}, ttl)
return &dbCache{
cacheGuard: &sync.RWMutex{},
cache: cache,

9
pkg/local_object_storage/blobstor/blobovniczatree/option.go
View file

@ -16,6 +16,7 @@ type cfg struct {
readOnly bool
rootPath string
openedCacheSize int
openedCacheTTL time.Duration
blzShallowDepth uint64
blzShallowWidth uint64
blzLeafWidth uint64
@ -34,6 +35,7 @@ type Option func(*cfg)
const (
defaultPerm = 0o700
defaultOpenedCacheSize = 50
defaultOpenedCacheTTL = 0 // means expiring is off
defaultBlzShallowDepth = 2
defaultBlzShallowWidth = 16
defaultWaitBeforeDropDB = 10 * time.Second
@ -46,6 +48,7 @@ func initConfig(c *cfg) {
log: &logger.Logger{Logger: zap.L()},
perm: defaultPerm,
openedCacheSize: defaultOpenedCacheSize,
openedCacheTTL: defaultOpenedCacheTTL,
blzShallowDepth: defaultBlzShallowDepth,
blzShallowWidth: defaultBlzShallowWidth,
reportError: func(string, error) {},
@ -105,6 +108,12 @@ func WithOpenedCacheSize(sz int) Option {
}
}
func WithOpenedCacheTTL(ttl time.Duration) Option {
return func(c *cfg) {
c.openedCacheTTL = ttl
}
}
func WithObjectSizeLimit(sz uint64) Option {
return func(c *cfg) {
c.blzOpts = append(c.blzOpts, blobovnicza.WithObjectSizeLimit(sz))