From 73bb590cb1f13918bac6a470f39b06a115158def Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Wed, 22 Feb 2023 11:31:50 +0300 Subject: [PATCH] [#64] node: Use pool_size_local and separate pool for local puts Signed-off-by: Alejandro Lopez --- CHANGELOG.md | 1 + cmd/frostfs-node/config.go | 9 ++++++++- cmd/frostfs-node/config/object/config.go | 12 ++++++++++++ cmd/frostfs-node/config/object/config_test.go | 2 ++ cmd/frostfs-node/object.go | 2 +- config/example/node.env | 1 + config/example/node.json | 3 ++- config/example/node.yaml | 1 + docs/storage-node-configuration.md | 1 + pkg/services/object/put/service.go | 4 ++-- 10 files changed, 31 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7544f3249..9228226df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,7 @@ Changelog for FrostFS Node - Possible panic during write-cache initialization (#2234) - Do not fetch an object if `meta` is missing it (#61) - Create contract wallet only by `init` and `update-config` command (#63) +- Actually use `object.put.pool_size_local` and independent pool for local puts (#64). ### Removed ### Updated diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 75cdfd7fa..16204be49 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -496,6 +496,10 @@ type cfgObjectRoutines struct { putRemoteCapacity int + putLocal *ants.Pool + + putLocalCapacity int + replicatorPoolSize int replication *ants.Pool @@ -834,10 +838,13 @@ func initObjectPool(cfg *config.Config) (pool cfgObjectRoutines) { optNonBlocking := ants.WithNonblocking(true) pool.putRemoteCapacity = objectconfig.Put(cfg).PoolSizeRemote() - pool.putRemote, err = ants.NewPool(pool.putRemoteCapacity, optNonBlocking) fatalOnErr(err) + pool.putLocalCapacity = objectconfig.Put(cfg).PoolSizeLocal() + pool.putLocal, err = ants.NewPool(pool.putLocalCapacity, optNonBlocking) + fatalOnErr(err) + pool.replicatorPoolSize = replicatorconfig.PoolSize(cfg) if pool.replicatorPoolSize <= 0 { pool.replicatorPoolSize = pool.putRemoteCapacity diff --git a/cmd/frostfs-node/config/object/config.go b/cmd/frostfs-node/config/object/config.go index b675b39e0..9a36980e8 100644 --- a/cmd/frostfs-node/config/object/config.go +++ b/cmd/frostfs-node/config/object/config.go @@ -39,3 +39,15 @@ func (g PutConfig) PoolSizeRemote() int { return PutPoolSizeDefault } + +// PoolSizeLocal returns the value of "pool_size_local" config parameter. +// +// Returns PutPoolSizeDefault if the value is not a positive number. +func (g PutConfig) PoolSizeLocal() int { + v := config.Int(g.cfg, "pool_size_local") + if v > 0 { + return int(v) + } + + return PutPoolSizeDefault +} diff --git a/cmd/frostfs-node/config/object/config_test.go b/cmd/frostfs-node/config/object/config_test.go index 5ff504171..b154ed444 100644 --- a/cmd/frostfs-node/config/object/config_test.go +++ b/cmd/frostfs-node/config/object/config_test.go @@ -14,6 +14,7 @@ func TestObjectSection(t *testing.T) { empty := configtest.EmptyConfig() require.Equal(t, objectconfig.PutPoolSizeDefault, objectconfig.Put(empty).PoolSizeRemote()) + require.Equal(t, objectconfig.PutPoolSizeDefault, objectconfig.Put(empty).PoolSizeLocal()) require.EqualValues(t, objectconfig.DefaultTombstoneLifetime, objectconfig.TombstoneLifetime(empty)) }) @@ -21,6 +22,7 @@ func TestObjectSection(t *testing.T) { var fileConfigTest = func(c *config.Config) { require.Equal(t, 100, objectconfig.Put(c).PoolSizeRemote()) + require.Equal(t, 200, objectconfig.Put(c).PoolSizeLocal()) require.EqualValues(t, 10, objectconfig.TombstoneLifetime(c)) } diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 602e8e25f..23dc62721 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -275,7 +275,7 @@ func initObjectService(c *cfg) { putsvc.WithNetworkMapSource(c.netMapSource), putsvc.WithNetmapKeys(c), putsvc.WithNetworkState(c.cfgNetmap.state), - putsvc.WithWorkerPools(c.cfgObject.pool.putRemote), + putsvc.WithWorkerPools(c.cfgObject.pool.putRemote, c.cfgObject.pool.putLocal), putsvc.WithLogger(c.log), ) diff --git a/config/example/node.env b/config/example/node.env index ae51d473e..247178d60 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -85,6 +85,7 @@ FROSTFS_REPLICATOR_POOL_SIZE=10 # Object service section FROSTFS_OBJECT_PUT_POOL_SIZE_REMOTE=100 +FROSTFS_OBJECT_PUT_POOL_SIZE_LOCAL=200 FROSTFS_OBJECT_DELETE_TOMBSTONE_LIFETIME=10 # Storage engine section diff --git a/config/example/node.json b/config/example/node.json index 2930bf9d1..9fbd0f9bd 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -132,7 +132,8 @@ "tombstone_lifetime": 10 }, "put": { - "pool_size_remote": 100 + "pool_size_remote": 100, + "pool_size_local": 200 } }, "storage": { diff --git a/config/example/node.yaml b/config/example/node.yaml index 110e54206..bf1d65451 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -111,6 +111,7 @@ object: tombstone_lifetime: 10 # tombstone "local" lifetime in epochs put: pool_size_remote: 100 # number of async workers for remote PUT operations + pool_size_local: 200 # number of async workers for local PUT operations storage: # note: shard configuration can be omitted for relay node (see `node.relay`) diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index b0f79c257..b77253561 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -427,3 +427,4 @@ object: |-----------------------------|-------|---------------|------------------------------------------------------------------------------------------------| | `delete.tombstone_lifetime` | `int` | `5` | Tombstone lifetime for removed objects in epochs. | | `put.pool_size_remote` | `int` | `10` | Max pool size for performing remote `PUT` operations. Used by Policer and Replicator services. | +| `put.pool_size_local` | `int` | `10` | Max pool size for performing local `PUT` operations. Used by Policer and Replicator services. | diff --git a/pkg/services/object/put/service.go b/pkg/services/object/put/service.go index 2895eb796..7e72df18a 100644 --- a/pkg/services/object/put/service.go +++ b/pkg/services/object/put/service.go @@ -116,9 +116,9 @@ func WithNetworkMapSource(v netmap.Source) Option { } } -func WithWorkerPools(remote util.WorkerPool) Option { +func WithWorkerPools(remote, local util.WorkerPool) Option { return func(c *cfg) { - c.remotePool = remote + c.remotePool, c.localPool = remote, local } }