dstepanov-yadro/frostfs-node
1
0
Fork 0
forked from TrueCloudLab/frostfs-node
Code Issues Pull requests Projects Releases Packages Wiki Activity

[#9999] config: Add objectstore section

Browse source 
Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
This commit is contained in:
Dmitrii Stepanov 2025-02-25 10:07:30 +03:00
parent 82f64ed2bc
commit 2fae2cd39c
Signed by: dstepanov-yadro
GPG key ID: 237AF1A763293BC0
8 changed files with 81 additions and 233 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
cmd/frostfs-node/config.go
View file

@ -137,6 +137,12 @@ type shardCfg struct {
maxBatchSize int
maxBatchDelay time.Duration
}
objectStoreCfg struct {
metaPath string
blobPath string
walPath string
}
}
// id returns persistent id of a shard. It is different from the ID used in runtime
@ -229,6 +235,7 @@ func (a *applicationConfiguration) updateShardConfig(c *config.Config, oldConfig
a.setShardPiloramaConfig(c, &newConfig, oldConfig)
a.setGCConfig(&newConfig, oldConfig)
a.setObjectStoreConfig(&newConfig, oldConfig)
a.EngineCfg.shards = append(a.EngineCfg.shards, newConfig)
@ -257,6 +264,13 @@ func (a *applicationConfiguration) setGCConfig(newConfig *shardCfg, oldConfig *s
newConfig.gcCfg.expiredCollectorWorkerCount = gcCfg.ExpiredCollectorWorkerCount()
}
func (a *applicationConfiguration) setObjectStoreConfig(target *shardCfg, source *shardconfig.Config) {
osCfg := source.ObjectStore()
target.objectStoreCfg.blobPath = osCfg.BlobPath()
target.objectStoreCfg.metaPath = osCfg.MetaPath()
target.objectStoreCfg.walPath = osCfg.WALPath()
}
// internals contains application-specific internals that are created
// on application startup and are shared b/w the components during
// the application life cycle.

2
cmd/frostfs-node/config/engine/config.go
View file

@ -49,7 +49,7 @@ func IterateShards(c *config.Config, required bool, f func(*shardconfig.Config)
// must have different paths, so if it is missing, the shard is not here.
// At the same time checking for "blobstor" section doesn't work proper
// with configuration via the environment.
if (*config.Config)(sc).Value("metabase.path") == nil {
if (*config.Config)(sc).Value("objectstore.blob_path") == nil {
break
}
(*config.Config)(sc).SetDefault(def)

11
cmd/frostfs-node/config/engine/config_test.go
View file

@ -66,6 +66,7 @@ func TestEngineSection(t *testing.T) {
pl := sc.Pilorama()
gc := sc.GC()
os := sc.ObjectStore()
switch num {
case 0:
@ -79,7 +80,6 @@ func TestEngineSection(t *testing.T) {
require.Equal(t, []string{"audio/*", "video/*"}, sc.UncompressableContentTypes())
require.Equal(t, true, sc.EstimateCompressibility())
require.Equal(t, float64(0.7), sc.EstimateCompressibilityThreshold())
require.EqualValues(t, 102400, sc.SmallSizeLimit())
require.EqualValues(t, 150, gc.RemoverBatchSize())
require.Equal(t, 2*time.Minute, gc.RemoverSleepInterval())
@ -87,6 +87,10 @@ func TestEngineSection(t *testing.T) {
require.Equal(t, 15, gc.ExpiredCollectorWorkerCount())
require.Equal(t, mode.ReadOnly, sc.Mode())
require.Equal(t, "tmp/0/blob", os.BlobPath())
require.Equal(t, "tmp/0/meta", os.MetaPath())
require.Equal(t, "tmp/0/wal", os.WALPath())
case 1:
require.Equal(t, "tmp/1/blob/pilorama.db", pl.Path())
require.Equal(t, fs.FileMode(0o644), pl.Perm())
@ -96,7 +100,6 @@ func TestEngineSection(t *testing.T) {
require.Equal(t, false, sc.Compress())
require.Equal(t, []string(nil), sc.UncompressableContentTypes())
require.EqualValues(t, 102400, sc.SmallSizeLimit())
require.EqualValues(t, 200, gc.RemoverBatchSize())
require.Equal(t, 5*time.Minute, gc.RemoverSleepInterval())
@ -104,6 +107,10 @@ func TestEngineSection(t *testing.T) {
require.Equal(t, gcconfig.ExpiredCollectorWorkersCountDefault, gc.ExpiredCollectorWorkerCount())
require.Equal(t, mode.ReadWrite, sc.Mode())
require.Equal(t, "tmp/1/blob", os.BlobPath())
require.Equal(t, "tmp/1/meta", os.MetaPath())
require.Equal(t, "tmp/1/wal", os.WALPath())
}
return nil
})

26
cmd/frostfs-node/config/engine/shard/config.go
View file

@ -3,6 +3,7 @@ package shardconfig
import (
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config"
gcconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/gc"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/objectstore"
piloramaconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/pilorama"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
)
@ -12,8 +13,6 @@ import (
type Config config.Config
const (
// SmallSizeLimitDefault is a default limit of small objects payload in bytes.
SmallSizeLimitDefault = 1 << 20
EstimateCompressibilityThresholdDefault = 0.1
RefillMetabaseWorkersCountDefault = 500
)
@ -66,22 +65,6 @@ func (x *Config) EstimateCompressibilityThreshold() float64 {
return v
}
// SmallSizeLimit returns the value of "small_object_size" config parameter.
//
// Returns SmallSizeLimitDefault if the value is not a positive number.
func (x *Config) SmallSizeLimit() uint64 {
l := config.SizeInBytesSafe(
(*config.Config)(x),
"small_object_size",
)
if l > 0 {
return l
}
return SmallSizeLimitDefault
}
// Pilorama returns "pilorama" subsection as a piloramaconfig.Config.
func (x *Config) Pilorama() *piloramaconfig.Config {
return piloramaconfig.From(
@ -98,6 +81,13 @@ func (x *Config) GC() *gcconfig.Config {
)
}
func (x *Config) ObjectStore() *objectstore.Config {
return objectstore.From(
(*config.Config)(x).
Sub("objectstore"),
)
}
// Mode return the value of "mode" config parameter.
//
// Panics if read the value is not one of predefined

23
cmd/frostfs-node/config/engine/shard/objectstore/config.go Normal file
View file

@ -0,0 +1,23 @@
package objectstore
import (
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config"
)
type Config config.Config
func From(c *config.Config) *Config {
return (*Config)(c)
}
func (x *Config) MetaPath() string {
return config.String((*config.Config)(x), "meta_path")
}
func (x *Config) BlobPath() string {
return config.String((*config.Config)(x), "blob_path")
}
func (x *Config) WALPath() string {
return config.String((*config.Config)(x), "wal_path")
}

75
config/example/node.env
View file

@ -97,50 +97,12 @@ FROSTFS_OBJECT_GET_PRIORITY="$attribute:ClusterName $attribute:UN-LOCODE"
FROSTFS_STORAGE_SHARD_POOL_SIZE=15
FROSTFS_STORAGE_SHARD_RO_ERROR_THRESHOLD=100
## 0 shard
### Flag to refill Metabase from BlobStor
FROSTFS_STORAGE_SHARD_0_RESYNC_METABASE=false
FROSTFS_STORAGE_SHARD_0_RESYNC_METABASE_WORKER_COUNT=100
### Flag to set shard mode
FROSTFS_STORAGE_SHARD_0_MODE=read-only
### Write cache config
FROSTFS_STORAGE_SHARD_0_WRITECACHE_ENABLED=false
FROSTFS_STORAGE_SHARD_0_WRITECACHE_NO_SYNC=true
FROSTFS_STORAGE_SHARD_0_WRITECACHE_PATH=tmp/0/cache
FROSTFS_STORAGE_SHARD_0_WRITECACHE_SMALL_OBJECT_SIZE=16384
FROSTFS_STORAGE_SHARD_0_WRITECACHE_MAX_OBJECT_SIZE=134217728
FROSTFS_STORAGE_SHARD_0_WRITECACHE_FLUSH_WORKER_COUNT=30
FROSTFS_STORAGE_SHARD_0_WRITECACHE_CAPACITY=3221225472
FROSTFS_STORAGE_SHARD_0_WRITECACHE_PAGE_SIZE=4096
FROSTFS_STORAGE_SHARD_0_WRITECACHE_MAX_OBJECT_COUNT=49
FROSTFS_STORAGE_SHARD_0_WRITECACHE_MAX_FLUSHING_OBJECTS_SIZE=100
### Metabase config
FROSTFS_STORAGE_SHARD_0_METABASE_PATH=tmp/0/meta
FROSTFS_STORAGE_SHARD_0_METABASE_PERM=0644
FROSTFS_STORAGE_SHARD_0_METABASE_MAX_BATCH_SIZE=100
FROSTFS_STORAGE_SHARD_0_METABASE_MAX_BATCH_DELAY=10ms
### Blobstor config
FROSTFS_STORAGE_SHARD_0_COMPRESS=true
FROSTFS_STORAGE_SHARD_0_COMPRESSION_EXCLUDE_CONTENT_TYPES="audio/* video/*"
FROSTFS_STORAGE_SHARD_0_COMPRESSION_ESTIMATE_COMPRESSIBILITY=true
FROSTFS_STORAGE_SHARD_0_COMPRESSION_ESTIMATE_COMPRESSIBILITY_THRESHOLD=0.7
FROSTFS_STORAGE_SHARD_0_SMALL_OBJECT_SIZE=102400
### Blobovnicza config
FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_PATH=tmp/0/blob/blobovnicza
FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_PERM=0644
FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_TYPE=blobovnicza
FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_SIZE=4194304
FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_DEPTH=1
FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_WIDTH=4
FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_OPENED_CACHE_CAPACITY=50
FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_OPENED_CACHE_TTL=1m
FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_OPENED_CACHE_EXP_INTERVAL=30s
FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_INIT_WORKER_COUNT=10
FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_REBUILD_DROP_TIMEOUT=30s
### FSTree config
FROSTFS_STORAGE_SHARD_0_BLOBSTOR_1_TYPE=fstree
FROSTFS_STORAGE_SHARD_0_BLOBSTOR_1_PATH=tmp/0/blob
FROSTFS_STORAGE_SHARD_0_BLOBSTOR_1_PERM=0644
FROSTFS_STORAGE_SHARD_0_BLOBSTOR_1_DEPTH=5
### Pilorama config
FROSTFS_STORAGE_SHARD_0_PILORAMA_PATH="tmp/0/blob/pilorama.db"
FROSTFS_STORAGE_SHARD_0_PILORAMA_MAX_BATCH_DELAY=10ms
@ -154,42 +116,14 @@ FROSTFS_STORAGE_SHARD_0_GC_REMOVER_SLEEP_INTERVAL=2m
FROSTFS_STORAGE_SHARD_0_GC_EXPIRED_COLLECTOR_BATCH_SIZE=1500
#### Limit of concurrent workers collecting expired objects by the garbage collector
FROSTFS_STORAGE_SHARD_0_GC_EXPIRED_COLLECTOR_WORKER_COUNT=15
FROSTFS_STORAGE_SHARD_0_OBJECTSTORE_META_PATH="tmp/0/meta"
FROSTFS_STORAGE_SHARD_0_OBJECTSTORE_BLOB_PATH="tmp/0/blob"
FROSTFS_STORAGE_SHARD_0_OBJECTSTORE_WAL_PATH="tmp/0/wal"
## 1 shard
### Flag to refill Metabase from BlobStor
FROSTFS_STORAGE_SHARD_1_RESYNC_METABASE=true
### Flag to set shard mode
FROSTFS_STORAGE_SHARD_1_MODE=read-write
### Write cache config
FROSTFS_STORAGE_SHARD_1_WRITECACHE_ENABLED=true
FROSTFS_STORAGE_SHARD_1_WRITECACHE_PATH=tmp/1/cache
FROSTFS_STORAGE_SHARD_1_WRITECACHE_SMALL_OBJECT_SIZE=16384
FROSTFS_STORAGE_SHARD_1_WRITECACHE_MAX_OBJECT_SIZE=134217728
FROSTFS_STORAGE_SHARD_1_WRITECACHE_FLUSH_WORKER_COUNT=30
FROSTFS_STORAGE_SHARD_1_WRITECACHE_CAPACITY=4294967296
### Metabase config
FROSTFS_STORAGE_SHARD_1_METABASE_PATH=tmp/1/meta
FROSTFS_STORAGE_SHARD_1_METABASE_PERM=0644
FROSTFS_STORAGE_SHARD_1_METABASE_MAX_BATCH_SIZE=200
FROSTFS_STORAGE_SHARD_1_METABASE_MAX_BATCH_DELAY=20ms
### Blobstor config
FROSTFS_STORAGE_SHARD_1_COMPRESS=false
FROSTFS_STORAGE_SHARD_1_SMALL_OBJECT_SIZE=102400
### Blobovnicza config
FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_TYPE=blobovnicza
FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_PATH=tmp/1/blob/blobovnicza
FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_SIZE=4194304
FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_DEPTH=1
FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_WIDTH=4
FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_OPENED_CACHE_CAPACITY=50
FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_OPENED_CACHE_TTL=5m
FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_OPENED_CACHE_EXP_INTERVAL=15s
### FSTree config
FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_TYPE=fstree
FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_PATH=tmp/1/blob
FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_PERM=0644
FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_NO_SYNC=true
FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_DEPTH=5
### Pilorama config
FROSTFS_STORAGE_SHARD_1_PILORAMA_PATH="tmp/1/blob/pilorama.db"
FROSTFS_STORAGE_SHARD_1_PILORAMA_PERM=0644
@ -201,6 +135,9 @@ FROSTFS_STORAGE_SHARD_1_PILORAMA_MAX_BATCH_SIZE=100
FROSTFS_STORAGE_SHARD_1_GC_REMOVER_BATCH_SIZE=200
#### Sleep interval between data remover tacts
FROSTFS_STORAGE_SHARD_1_GC_REMOVER_SLEEP_INTERVAL=5m
FROSTFS_STORAGE_SHARD_1_OBJECTSTORE_META_PATH="tmp/1/meta"
FROSTFS_STORAGE_SHARD_1_OBJECTSTORE_BLOB_PATH="tmp/1/blob"
FROSTFS_STORAGE_SHARD_1_OBJECTSTORE_WAL_PATH="tmp/1/wal"
FROSTFS_TRACING_ENABLED=true
FROSTFS_TRACING_ENDPOINT="localhost"

89
config/example/node.json
View file

@ -148,54 +148,12 @@
"shard": {
"0": {
"mode": "read-only",
"resync_metabase": false,
"resync_metabase_worker_count": 100,
"writecache": {
"enabled": false,
"no_sync": true,
"path": "tmp/0/cache",
"small_object_size": 16384,
"max_object_size": 134217728,
"flush_worker_count": 30,
"capacity": 3221225472,
"page_size": 4096,
"max_object_count": 49,
"max_flushing_objects_size": 100
},
"metabase": {
"path": "tmp/0/meta",
"perm": "0644",
"max_batch_size": 100,
"max_batch_delay": "10ms"
},
"compress": true,
"compression_exclude_content_types": [
"audio/*", "video/*"
],
"compression_estimate_compressibility": true,
"compression_estimate_compressibility_threshold": 0.7,
"small_object_size": 102400,
"blobstor": [
{
"type": "blobovnicza",
"path": "tmp/0/blob/blobovnicza",
"perm": "0644",
"size": 4194304,
"depth": 1,
"width": 4,
"opened_cache_capacity": 50,
"opened_cache_ttl": "1m",
"opened_cache_exp_interval": "30s",
"init_worker_count": 10,
"rebuild_drop_timeout": "30s"
},
{
"type": "fstree",
"path": "tmp/0/blob",
"perm": "0644",
"depth": 5
}
],
"pilorama": {
"path": "tmp/0/blob/pilorama.db",
"max_batch_delay": "10ms",
@ -206,48 +164,16 @@
"remover_sleep_interval": "2m",
"expired_collector_batch_size": 1500,
"expired_collector_worker_count": 15
},
"objectstore": {
"wal_path": "tmp/0/wal",
"blob_path": "tmp/0/blob",
"meta_path": "tmp/0/meta"
}
},
"1": {
"mode": "read-write",
"resync_metabase": true,
"writecache": {
"enabled": true,
"path": "tmp/1/cache",
"memcache_capacity": 2147483648,
"small_object_size": 16384,
"max_object_size": 134217728,
"flush_worker_count": 30,
"capacity": 4294967296
},
"metabase": {
"path": "tmp/1/meta",
"perm": "0644",
"max_batch_size": 200,
"max_batch_delay": "20ms"
},
"compress": false,
"small_object_size": 102400,
"blobstor": [
{
"type": "blobovnicza",
"path": "tmp/1/blob/blobovnicza",
"perm": "0644",
"size": 4194304,
"depth": 1,
"width": 4,
"opened_cache_capacity": 50,
"opened_cache_ttl": "5m",
"opened_cache_exp_interval": "15s"
},
{
"type": "fstree",
"path": "tmp/1/blob",
"no_sync": true,
"perm": "0644",
"depth": 5
}
],
"pilorama": {
"path": "tmp/1/blob/pilorama.db",
"perm": "0644",
@ -258,6 +184,11 @@
"gc": {
"remover_batch_size": 200,
"remover_sleep_interval": "5m"
},
"objectstore": {
"wal_path": "tmp/1/wal",
"blob_path": "tmp/1/blob",
"meta_path": "tmp/1/meta"
}
}
}

74
config/example/node.yaml
View file

@ -132,36 +132,12 @@ storage:
shard:
default: # section with the default shard parameters
resync_metabase: true # sync metabase with blobstor on start, expensive, leave false until complete understanding
writecache:
enabled: true
small_object_size: 16k # size threshold for "small" objects which are cached in key-value DB, not in FS, bytes
max_object_size: 134217728 # size threshold for "big" objects which bypass write-cache and go to the storage directly, bytes
flush_worker_count: 30 # number of write-cache flusher threads
metabase:
perm: 0644 # permissions for metabase files(directories: +x for current user and group)
max_batch_size: 200
max_batch_delay: 20ms
pilorama:
max_batch_delay: 5ms # maximum delay for a batch of operations to be executed
max_batch_size: 100 # maximum amount of operations in a single batch
compress: false # turn on/off zstd(level 3) compression of stored objects
small_object_size: 100 kb # size threshold for "small" objects which are cached in key-value DB, not in FS, bytes
blobstor:
- size: 4m # approximate size limit of single blobovnicza instance, total size will be: size*width^(depth+1), bytes
perm: 0644 # permissions for blobstor files(directories: +x for current user and group)
depth: 1 # max depth of object tree storage in key-value DB
width: 4 # max width of object tree storage in key-value DB
opened_cache_capacity: 50 # maximum number of opened database files
opened_cache_ttl: 5m # ttl for opened database file
opened_cache_exp_interval: 15s # cache cleanup interval for expired blobovnicza's
- perm: 0644 # permissions for blobstor files(directories: +x for current user and group)
depth: 5 # max depth of object tree storage in FS
gc:
remover_batch_size: 200 # number of objects to be removed by the garbage collector
@ -174,22 +150,6 @@ storage:
# degraded
# degraded-read-only
# disabled (do not work with the shard, allows to not remove it from the config)
resync_metabase: false # sync metabase with blobstor on start, expensive, leave false until complete understanding
resync_metabase_worker_count: 100
writecache:
enabled: false
no_sync: true
path: tmp/0/cache # write-cache root directory
capacity: 3221225472 # approximate write-cache total size, bytes
max_object_count: 49
page_size: 4k
max_flushing_objects_size: 100b
metabase:
path: tmp/0/meta # metabase path
max_batch_size: 100
max_batch_delay: 10ms
compress: true # turn on/off zstd(level 3) compression of stored objects
compression_exclude_content_types:
@ -198,16 +158,6 @@ storage:
compression_estimate_compressibility: true
compression_estimate_compressibility_threshold: 0.7
blobstor:
- type: blobovnicza
path: tmp/0/blob/blobovnicza
init_worker_count: 10 #count of workers to initialize blobovniczas
rebuild_drop_timeout: 30s # timeout before drop single blobovnicza
opened_cache_ttl: 1m
opened_cache_exp_interval: 30s
- type: fstree
path: tmp/0/blob # blobstor path
pilorama:
path: tmp/0/blob/pilorama.db # path to the pilorama database. If omitted, `pilorama.db` file is created blobstor.path
max_batch_delay: 10ms
@ -219,26 +169,22 @@ storage:
expired_collector_batch_size: 1500 # number of objects to be marked expired by the garbage collector
expired_collector_worker_count: 15 # number of concurrent workers collecting expired objects by the garbage collector
objectstore:
wal_path: tmp/0/wal
blob_path: tmp/0/blob
meta_path: tmp/0/meta
1:
writecache:
path: tmp/1/cache # write-cache root directory
capacity: 4 G # approximate write-cache total size, bytes
metabase:
path: tmp/1/meta # metabase path
blobstor:
- type: blobovnicza
path: tmp/1/blob/blobovnicza
- type: fstree
path: tmp/1/blob # blobstor path
no_sync: true
pilorama:
path: tmp/1/blob/pilorama.db
no_sync: true # USE WITH CAUTION. Return to user before pages have been persisted.
perm: 0644 # permission to use for the database file and intermediate directories
objectstore:
wal_path: tmp/1/wal
blob_path: tmp/1/blob
meta_path: tmp/1/meta
tracing:
enabled: true
exporter: "otlp_grpc"