diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go
index 88d55ae23..0458e0eb4 100644
--- a/cmd/frostfs-node/config.go
+++ b/cmd/frostfs-node/config.go
@@ -107,11 +107,12 @@ type applicationConfiguration struct {
}
type shardCfg struct {
- compress bool
- smallSizeObjectLimit uint64
- uncompressableContentType []string
- refillMetabase bool
- mode shardmode.Mode
+ compress bool
+ smallSizeObjectLimit uint64
+ uncompressableContentType []string
+ refillMetabase bool
+ refillMetabaseWorkersCount int
+ mode shardmode.Mode
metaCfg struct {
path string
@@ -223,6 +224,7 @@ func (a *applicationConfiguration) updateShardConfig(c *config.Config, oldConfig
var newConfig shardCfg
newConfig.refillMetabase = oldConfig.RefillMetabase()
+ newConfig.refillMetabaseWorkersCount = oldConfig.RefillMetabaseWorkersCount()
newConfig.mode = oldConfig.Mode()
newConfig.compress = oldConfig.Compress()
newConfig.uncompressableContentType = oldConfig.UncompressableContentTypes()
@@ -874,6 +876,7 @@ func (c *cfg) getShardOpts(shCfg shardCfg) shardOptsWithID {
sh.shOpts = []shard.Option{
shard.WithLogger(c.log),
shard.WithRefillMetabase(shCfg.refillMetabase),
+ shard.WithRefillMetabaseWorkersCount(shCfg.refillMetabaseWorkersCount),
shard.WithMode(shCfg.mode),
shard.WithBlobStorOptions(blobstoreOpts...),
shard.WithMetaBaseOptions(mbOptions...),
diff --git a/cmd/frostfs-node/config/engine/config_test.go b/cmd/frostfs-node/config/engine/config_test.go
index 620b55d20..e049c5a1f 100644
--- a/cmd/frostfs-node/config/engine/config_test.go
+++ b/cmd/frostfs-node/config/engine/config_test.go
@@ -114,6 +114,7 @@ func TestEngineSection(t *testing.T) {
require.Equal(t, false, sc.RefillMetabase())
require.Equal(t, mode.ReadOnly, sc.Mode())
+ require.Equal(t, 100, sc.RefillMetabaseWorkersCount())
case 1:
require.Equal(t, "tmp/1/blob/pilorama.db", pl.Path())
require.Equal(t, fs.FileMode(0o644), pl.Perm())
@@ -164,6 +165,7 @@ func TestEngineSection(t *testing.T) {
require.Equal(t, true, sc.RefillMetabase())
require.Equal(t, mode.ReadWrite, sc.Mode())
+ require.Equal(t, shardconfig.RefillMetabaseWorkersCountDefault, sc.RefillMetabaseWorkersCount())
}
return nil
})
diff --git a/cmd/frostfs-node/config/engine/shard/config.go b/cmd/frostfs-node/config/engine/shard/config.go
index 1dc32fb86..5ac2f8272 100644
--- a/cmd/frostfs-node/config/engine/shard/config.go
+++ b/cmd/frostfs-node/config/engine/shard/config.go
@@ -16,8 +16,11 @@ import (
// which provides access to Shard configurations.
type Config config.Config
-// SmallSizeLimitDefault is a default limit of small objects payload in bytes.
-const SmallSizeLimitDefault = 1 << 20
+const (
+ // SmallSizeLimitDefault is a default limit of small objects payload in bytes.
+ SmallSizeLimitDefault = 1 << 20
+ RefillMetabaseWorkersCountDefault = 500
+)
// From wraps config section into Config.
func From(c *config.Config) *Config {
@@ -109,6 +112,20 @@ func (x *Config) RefillMetabase() bool {
)
}
+// RefillMetabaseWorkersCount returns the value of "resync_metabase_worker_count" config parameter.
+//
+// Returns RefillMetabaseWorkersCountDefault if the value is not a positive number.
+func (x *Config) RefillMetabaseWorkersCount() int {
+ v := config.IntSafe(
+ (*config.Config)(x),
+ "resync_metabase_worker_count",
+ )
+ if v > 0 {
+ return int(v)
+ }
+ return RefillMetabaseWorkersCountDefault
+}
+
// Mode return the value of "mode" config parameter.
//
// Panics if read the value is not one of predefined
diff --git a/config/example/node.env b/config/example/node.env
index 9ea73a11c..d9176fbb9 100644
--- a/config/example/node.env
+++ b/config/example/node.env
@@ -96,6 +96,7 @@ FROSTFS_STORAGE_REBUILD_WORKERS_COUNT=1000
## 0 shard
### Flag to refill Metabase from BlobStor
FROSTFS_STORAGE_SHARD_0_RESYNC_METABASE=false
+FROSTFS_STORAGE_SHARD_0_RESYNC_METABASE_WORKER_COUNT=100
### Flag to set shard mode
FROSTFS_STORAGE_SHARD_0_MODE=read-only
### Write cache config
diff --git a/config/example/node.json b/config/example/node.json
index 854bc1303..000ad7931 100644
--- a/config/example/node.json
+++ b/config/example/node.json
@@ -142,6 +142,7 @@
"0": {
"mode": "read-only",
"resync_metabase": false,
+ "resync_metabase_worker_count": 100,
"writecache": {
"enabled": false,
"no_sync": true,
diff --git a/config/example/node.yaml b/config/example/node.yaml
index fc006eaa7..727df5f3e 100644
--- a/config/example/node.yaml
+++ b/config/example/node.yaml
@@ -163,6 +163,7 @@ storage:
# degraded-read-only
# disabled (do not work with the shard, allows to not remove it from the config)
resync_metabase: false # sync metabase with blobstor on start, expensive, leave false until complete understanding
+ resync_metabase_worker_count: 100
writecache:
enabled: false
diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md
index 2e2d04088..2d267ceb2 100644
--- a/docs/storage-node-configuration.md
+++ b/docs/storage-node-configuration.md
@@ -177,17 +177,20 @@ Contains configuration for each shard. Keys must be consecutive numbers starting
`default` subsection has the same format and specifies defaults for missing values.
The following table describes configuration for each shard.
-| Parameter | Type | Default value | Description |
-|-------------------------------------|---------------------------------------------|---------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| `compress` | `bool` | `false` | Flag to enable compression. |
-| `compression_exclude_content_types` | `[]string` | | List of content-types to disable compression for. Content-type is taken from `Content-Type` object attribute. Each element can contain a star `*` as a first (last) character, which matches any prefix (suffix). |
-| `mode` | `string` | `read-write` | Shard Mode.
Possible values: `read-write`, `read-only`, `degraded`, `degraded-read-only`, `disabled` |
-| `resync_metabase` | `bool` | `false` | Flag to enable metabase resync on start. |
-| `writecache` | [Writecache config](#writecache-subsection) | | Write-cache configuration. |
-| `metabase` | [Metabase config](#metabase-subsection) | | Metabase configuration. |
-| `blobstor` | [Blobstor config](#blobstor-subsection) | | Blobstor configuration. |
-| `small_object_size` | `size` | `1M` | Maximum size of an object stored in blobovnicza tree. |
-| `gc` | [GC config](#gc-subsection) | | GC configuration. |
+| Parameter | Type | Default value | Description |
+| ------------------------------------------------ | ------------------------------------------- | ------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `compress` | `bool` | `false` | Flag to enable compression. |
+| `compression_exclude_content_types` | `[]string` | | List of content-types to disable compression for. Content-type is taken from `Content-Type` object attribute. Each element can contain a star `*` as a first (last) character, which matches any prefix (suffix). |
+| `compression_estimate_compressibility` | `bool` | `false` | If `true`, then noramalized compressibility estimation is used to decide compress data or not. |
+| `compression_estimate_compressibility_threshold` | `float` | `0.1` | Normilized compressibility estimate threshold: data will compress if estimation if greater than this value. |
+| `mode` | `string` | `read-write` | Shard Mode.
Possible values: `read-write`, `read-only`, `degraded`, `degraded-read-only`, `disabled` |
+| `resync_metabase` | `bool` | `false` | Flag to enable metabase resync on start. |
+| `resync_metabase_worker_count` | `int` | `1000` | Count of concurrent workers to resync metabase. |
+| `writecache` | [Writecache config](#writecache-subsection) | | Write-cache configuration. |
+| `metabase` | [Metabase config](#metabase-subsection) | | Metabase configuration. |
+| `blobstor` | [Blobstor config](#blobstor-subsection) | | Blobstor configuration. |
+| `small_object_size` | `size` | `1M` | Maximum size of an object stored in blobovnicza tree. |
+| `gc` | [GC config](#gc-subsection) | | GC configuration. |
### `blobstor` subsection
diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go
index f103ebc2b..3d209e703 100644
--- a/pkg/local_object_storage/shard/control.go
+++ b/pkg/local_object_storage/shard/control.go
@@ -15,6 +15,7 @@ import (
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
"go.uber.org/zap"
+ "golang.org/x/sync/errgroup"
)
func (s *Shard) handleMetabaseFailure(stage string, err error) error {
@@ -174,39 +175,56 @@ func (s *Shard) refillMetabase(ctx context.Context) error {
return fmt.Errorf("could not reset metabase: %w", err)
}
- obj := objectSDK.New()
+ eg, egCtx := errgroup.WithContext(ctx)
+ if s.cfg.refillMetabaseWorkersCount > 0 {
+ eg.SetLimit(s.cfg.refillMetabaseWorkersCount)
+ }
- err = blobstor.IterateBinaryObjects(ctx, s.blobStor, func(addr oid.Address, data []byte, descriptor []byte) error {
- if err := obj.Unmarshal(data); err != nil {
- s.log.Warn(logs.ShardCouldNotUnmarshalObject,
- zap.Stringer("address", addr),
- zap.String("err", err.Error()))
+ itErr := blobstor.IterateBinaryObjects(egCtx, s.blobStor, func(addr oid.Address, data []byte, descriptor []byte) error {
+ eg.Go(func() error {
+ obj := objectSDK.New()
+ if err := obj.Unmarshal(data); err != nil {
+ s.log.Warn(logs.ShardCouldNotUnmarshalObject,
+ zap.Stringer("address", addr),
+ zap.String("err", err.Error()))
+ return nil
+ }
+
+ var err error
+ switch obj.Type() {
+ case objectSDK.TypeTombstone:
+ err = s.refillTombstoneObject(egCtx, obj)
+ case objectSDK.TypeLock:
+ err = s.refillLockObject(egCtx, obj)
+ default:
+ }
+ if err != nil {
+ return err
+ }
+
+ var mPrm meta.PutPrm
+ mPrm.SetObject(obj)
+ mPrm.SetStorageID(descriptor)
+
+ _, err = s.metaBase.Put(egCtx, mPrm)
+ if err != nil && !client.IsErrObjectAlreadyRemoved(err) && !errors.Is(err, meta.ErrObjectIsExpired) {
+ return err
+ }
+
+ return nil
+ })
+
+ select {
+ case <-egCtx.Done():
+ return egCtx.Err()
+ default:
return nil
}
-
- var err error
- switch obj.Type() {
- case objectSDK.TypeTombstone:
- err = s.refillTombstoneObject(ctx, obj)
- case objectSDK.TypeLock:
- err = s.refillLockObject(ctx, obj)
- default:
- }
- if err != nil {
- return err
- }
-
- var mPrm meta.PutPrm
- mPrm.SetObject(obj)
- mPrm.SetStorageID(descriptor)
-
- _, err = s.metaBase.Put(ctx, mPrm)
- if err != nil && !client.IsErrObjectAlreadyRemoved(err) && !errors.Is(err, meta.ErrObjectIsExpired) {
- return err
- }
-
- return nil
})
+
+ egErr := eg.Wait()
+
+ err = errors.Join(egErr, itErr)
if err != nil {
return fmt.Errorf("could not put objects to the meta: %w", err)
}
diff --git a/pkg/local_object_storage/shard/refill_test.go b/pkg/local_object_storage/shard/refill_test.go
new file mode 100644
index 000000000..07ec9180c
--- /dev/null
+++ b/pkg/local_object_storage/shard/refill_test.go
@@ -0,0 +1,72 @@
+package shard
+
+import (
+ "context"
+ "os"
+ "testing"
+
+ "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
+ oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
+ "github.com/stretchr/testify/require"
+)
+
+func BenchmarkRefillMetabase(b *testing.B) {
+ b.Run("100 objects", func(b *testing.B) {
+ benchRefillMetabase(b, 100)
+ })
+
+ b.Run("1000 objects", func(b *testing.B) {
+ benchRefillMetabase(b, 1000)
+ })
+
+ b.Run("2000 objects", func(b *testing.B) {
+ benchRefillMetabase(b, 2000)
+ })
+
+ b.Run("5000 objects", func(b *testing.B) {
+ benchRefillMetabase(b, 5000)
+ })
+}
+
+func benchRefillMetabase(b *testing.B, objectsCount int) {
+ sh := newShard(b, false)
+ defer func() { require.NoError(b, sh.Close()) }()
+
+ var putPrm PutPrm
+
+ for i := 0; i < objectsCount/2; i++ {
+ obj := testutil.GenerateObject()
+ testutil.AddAttribute(obj, "foo", "bar")
+ testutil.AddPayload(obj, 1<<5) // blobvnicza tree obj
+
+ putPrm.SetObject(obj)
+
+ _, err := sh.Put(context.Background(), putPrm)
+ require.NoError(b, err)
+ }
+
+ for i := 0; i < objectsCount/2; i++ {
+ obj := testutil.GenerateObject()
+ testutil.AddAttribute(obj, "foo", "bar")
+ obj.SetID(oidtest.ID())
+ testutil.AddPayload(obj, 1<<20) // fstree obj
+
+ putPrm.SetObject(obj)
+
+ _, err := sh.Put(context.Background(), putPrm)
+ require.NoError(b, err)
+ }
+
+ require.NoError(b, sh.Close())
+ require.NoError(b, os.Remove(sh.metaBase.DumpInfo().Path))
+
+ require.NoError(b, sh.Open(context.Background()))
+ sh.cfg.refillMetabase = true
+
+ b.ReportAllocs()
+ b.ResetTimer()
+
+ require.NoError(b, sh.Init(context.Background()))
+
+ require.NoError(b, sh.Close())
+}
diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go
index 2aa01469b..378d77177 100644
--- a/pkg/local_object_storage/shard/shard.go
+++ b/pkg/local_object_storage/shard/shard.go
@@ -92,7 +92,8 @@ type MetricsWriter interface {
type cfg struct {
m sync.RWMutex
- refillMetabase bool
+ refillMetabase bool
+ refillMetabaseWorkersCount int
rmBatchSize int
@@ -308,6 +309,13 @@ func WithRefillMetabase(v bool) Option {
}
}
+// WithRefillMetabaseWorkersCount returns option to set count of workers to refill the Metabase on Shard's initialization step.
+func WithRefillMetabaseWorkersCount(v int) Option {
+ return func(c *cfg) {
+ c.refillMetabaseWorkersCount = v
+ }
+}
+
// WithMode returns option to set shard's mode. Mode must be one of the predefined:
// - mode.ReadWrite;
// - mode.ReadOnly.