From 078b5fdd6e09a63f881d5707edd05306ee434f69 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 8 Aug 2024 13:32:18 +0300 Subject: [PATCH] [#1302] writecache: Allow to specify custom page size Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-lens/internal/writecache/inspect.go | 2 +- cmd/frostfs-lens/internal/writecache/list.go | 2 +- cmd/frostfs-node/config.go | 3 +++ cmd/frostfs-node/config/engine/config_test.go | 2 ++ cmd/frostfs-node/config/engine/shard/boltdb/boltdb.go | 11 +++++++++++ config/example/node.env | 1 + config/example/node.json | 3 ++- config/example/node.yaml | 1 + docs/storage-node-configuration.md | 2 ++ pkg/local_object_storage/writecache/options.go | 9 +++++++++ pkg/local_object_storage/writecache/storage.go | 2 +- pkg/local_object_storage/writecache/util.go | 3 ++- 12 files changed, 36 insertions(+), 5 deletions(-) diff --git a/cmd/frostfs-lens/internal/writecache/inspect.go b/cmd/frostfs-lens/internal/writecache/inspect.go index afc986c8b..63c669a35 100644 --- a/cmd/frostfs-lens/internal/writecache/inspect.go +++ b/cmd/frostfs-lens/internal/writecache/inspect.go @@ -25,7 +25,7 @@ func init() { func inspectFunc(cmd *cobra.Command, _ []string) { var data []byte - db, err := writecache.OpenDB(vPath, true, os.OpenFile) + db, err := writecache.OpenDB(vPath, true, os.OpenFile, 0) common.ExitOnErr(cmd, common.Errf("could not open write-cache db: %w", err)) defer db.Close() diff --git a/cmd/frostfs-lens/internal/writecache/list.go b/cmd/frostfs-lens/internal/writecache/list.go index bcbae0ec9..9c8fa6138 100644 --- a/cmd/frostfs-lens/internal/writecache/list.go +++ b/cmd/frostfs-lens/internal/writecache/list.go @@ -31,7 +31,7 @@ func listFunc(cmd *cobra.Command, _ []string) { return err } - db, err := writecache.OpenDB(vPath, true, os.OpenFile) + db, err := writecache.OpenDB(vPath, true, os.OpenFile, 0) common.ExitOnErr(cmd, common.Errf("could not open write-cache db: %w", err)) defer db.Close() diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 5b91e7819..6b79b9a3f 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -153,6 +153,7 @@ type shardCfg struct { flushWorkerCount int sizeLimit uint64 noSync bool + pageSize int } piloramaCfg struct { @@ -271,6 +272,7 @@ func (a *applicationConfiguration) setShardWriteCacheConfig(newConfig *shardCfg, wc.path = writeCacheCfg.Path() wc.maxBatchSize = writeCacheCfg.BoltDB().MaxBatchSize() wc.maxBatchDelay = writeCacheCfg.BoltDB().MaxBatchDelay() + wc.pageSize = writeCacheCfg.BoltDB().PageSize() wc.maxObjSize = writeCacheCfg.MaxObjectSize() wc.smallObjectSize = writeCacheCfg.SmallObjectSize() wc.flushWorkerCount = writeCacheCfg.WorkerCount() @@ -863,6 +865,7 @@ func (c *cfg) getWriteCacheOpts(shCfg shardCfg) []writecache.Option { writecache.WithPath(wcRead.path), writecache.WithMaxBatchSize(wcRead.maxBatchSize), writecache.WithMaxBatchDelay(wcRead.maxBatchDelay), + writecache.WithPageSize(wcRead.pageSize), writecache.WithMaxObjectSize(wcRead.maxObjSize), writecache.WithSmallObjectSize(wcRead.smallObjectSize), writecache.WithFlushWorkersCount(wcRead.flushWorkerCount), diff --git a/cmd/frostfs-node/config/engine/config_test.go b/cmd/frostfs-node/config/engine/config_test.go index 7473afefb..ed6f0f694 100644 --- a/cmd/frostfs-node/config/engine/config_test.go +++ b/cmd/frostfs-node/config/engine/config_test.go @@ -78,6 +78,7 @@ func TestEngineSection(t *testing.T) { require.EqualValues(t, 134217728, wc.MaxObjectSize()) require.EqualValues(t, 30, wc.WorkerCount()) require.EqualValues(t, 3221225472, wc.SizeLimit()) + require.EqualValues(t, 4096, wc.BoltDB().PageSize()) require.Equal(t, "tmp/0/meta", meta.Path()) require.Equal(t, fs.FileMode(0o644), meta.BoltDB().Perm()) @@ -133,6 +134,7 @@ func TestEngineSection(t *testing.T) { require.EqualValues(t, 134217728, wc.MaxObjectSize()) require.EqualValues(t, 30, wc.WorkerCount()) require.EqualValues(t, 4294967296, wc.SizeLimit()) + require.EqualValues(t, 0, wc.BoltDB().PageSize()) require.Equal(t, "tmp/1/meta", meta.Path()) require.Equal(t, fs.FileMode(0o644), meta.BoltDB().Perm()) diff --git a/cmd/frostfs-node/config/engine/shard/boltdb/boltdb.go b/cmd/frostfs-node/config/engine/shard/boltdb/boltdb.go index 9e334cd8f..a51308b5b 100644 --- a/cmd/frostfs-node/config/engine/shard/boltdb/boltdb.go +++ b/cmd/frostfs-node/config/engine/shard/boltdb/boltdb.go @@ -60,3 +60,14 @@ func (x *Config) MaxBatchSize() int { func (x *Config) NoSync() bool { return config.BoolSafe((*config.Config)(x), "no_sync") } + +// PageSize returns the value of "page_size" config parameter. +// +// Returns 0 if the value is not a positive number. +func (x *Config) PageSize() int { + s := int(config.SizeInBytesSafe((*config.Config)(x), "page_size")) + if s < 0 { + s = 0 + } + return s +} diff --git a/config/example/node.env b/config/example/node.env index 00190eb39..3beb4658c 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -105,6 +105,7 @@ FROSTFS_STORAGE_SHARD_0_WRITECACHE_SMALL_OBJECT_SIZE=16384 FROSTFS_STORAGE_SHARD_0_WRITECACHE_MAX_OBJECT_SIZE=134217728 FROSTFS_STORAGE_SHARD_0_WRITECACHE_FLUSH_WORKER_COUNT=30 FROSTFS_STORAGE_SHARD_0_WRITECACHE_CAPACITY=3221225472 +FROSTFS_STORAGE_SHARD_0_WRITECACHE_PAGE_SIZE=4096 ### Metabase config FROSTFS_STORAGE_SHARD_0_METABASE_PATH=tmp/0/meta FROSTFS_STORAGE_SHARD_0_METABASE_PERM=0644 diff --git a/config/example/node.json b/config/example/node.json index 9051d2bb7..763f7c169 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -148,7 +148,8 @@ "small_object_size": 16384, "max_object_size": 134217728, "flush_worker_count": 30, - "capacity": 3221225472 + "capacity": 3221225472, + "page_size": 4096 }, "metabase": { "path": "tmp/0/meta", diff --git a/config/example/node.yaml b/config/example/node.yaml index bcc8552b3..8c5793bac 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -171,6 +171,7 @@ storage: no_sync: true path: tmp/0/cache # write-cache root directory capacity: 3221225472 # approximate write-cache total size, bytes + page_size: 4k metabase: path: tmp/0/meta # metabase path diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 4a6e5ba6d..0b4fb5431 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -290,6 +290,7 @@ writecache: small_object_size: 16384 max_object_size: 134217728 flush_worker_count: 30 + page_size: '4k' ``` | Parameter | Type | Default value | Description | @@ -301,6 +302,7 @@ writecache: | `flush_worker_count` | `int` | `20` | Amount of background workers that move data from the writecache to the blobstor. | | `max_batch_size` | `int` | `1000` | Maximum amount of small object `PUT` operations to perform in a single transaction. | | `max_batch_delay` | `duration` | `10ms` | Maximum delay before a batch starts. | +| `page_size` | `size` | `0` | Page size overrides the default OS page size for small objects storage. Does not affect the existing storage. | # `node` section diff --git a/pkg/local_object_storage/writecache/options.go b/pkg/local_object_storage/writecache/options.go index c8eb1bc45..585cf2533 100644 --- a/pkg/local_object_storage/writecache/options.go +++ b/pkg/local_object_storage/writecache/options.go @@ -45,6 +45,8 @@ type options struct { metrics Metrics // disableBackgroundFlush is for testing purposes only. disableBackgroundFlush bool + // pageSize is bbolt's page size config value + pageSize int } // WithLogger sets logger. @@ -163,3 +165,10 @@ func WithDisableBackgroundFlush() Option { o.disableBackgroundFlush = true } } + +// WithPageSize sets bbolt's page size. +func WithPageSize(s int) Option { + return func(o *options) { + o.pageSize = s + } +} diff --git a/pkg/local_object_storage/writecache/storage.go b/pkg/local_object_storage/writecache/storage.go index caf997af8..57021cc17 100644 --- a/pkg/local_object_storage/writecache/storage.go +++ b/pkg/local_object_storage/writecache/storage.go @@ -32,7 +32,7 @@ func (c *cache) openStore(mod mode.ComponentMode) error { return err } - c.db, err = OpenDB(c.path, mod.ReadOnly(), c.openFile) + c.db, err = OpenDB(c.path, mod.ReadOnly(), c.openFile, c.pageSize) if err != nil { return fmt.Errorf("could not open database: %w", err) } diff --git a/pkg/local_object_storage/writecache/util.go b/pkg/local_object_storage/writecache/util.go index 0ed4a954e..ad3b443f3 100644 --- a/pkg/local_object_storage/writecache/util.go +++ b/pkg/local_object_storage/writecache/util.go @@ -10,11 +10,12 @@ import ( ) // OpenDB opens BoltDB instance for write-cache. Opens in read-only mode if ro is true. -func OpenDB(p string, ro bool, openFile func(string, int, fs.FileMode) (*os.File, error)) (*bbolt.DB, error) { +func OpenDB(p string, ro bool, openFile func(string, int, fs.FileMode) (*os.File, error), pageSize int) (*bbolt.DB, error) { return bbolt.Open(filepath.Join(p, dbName), os.ModePerm, &bbolt.Options{ NoFreelistSync: true, ReadOnly: ro, Timeout: 100 * time.Millisecond, OpenFile: openFile, + PageSize: pageSize, }) }