[#2159] node: Add tree replication timeout configuration

Signed-off-by: Pavel Karpy <p.karpy@yadro.com>

CHANGELOG.md

@@ -14,6 +14,7 @@ Changelog for NeoFS Node
 - `neofs-cli neofs-cli acl basic/extended print` commands (#2012)
 - `neofs_node_object_*_req_count_success` prometheus metrics for tracking successfully executed requests (#1984)
 - Metric 'readonly' to get shards mode (#2022)
+- Tree service replication timeout (#2159)
 
 ### Changed
 - `object lock` command reads CID and OID the same way other commands do (#1971)
@@ -80,6 +81,8 @@ Added `neofs_node_object_*_req_count_success` metrics for tracking successfully
 `neofs-cli container delete` command now requires given account or session issuer
 to match the container owner. Use `--force` (`-f`) flag to bypass this requirement.
 
+Tree service network replication can now be fine-tuned with `tree.replication_timeout` config field.
+
 ## [0.34.0] - 2022-10-31 - Marado (마라도, 馬羅島)
 
 ### Added

cmd/frostfs-node/config/tree/config.go

@@ -1,6 +1,10 @@
 package treeconfig
 
-import "github.com/TrueCloudLab/frostfs-node/cmd/frostfs-node/config"
+import (
+	"time"
+
+	"github.com/TrueCloudLab/frostfs-node/cmd/frostfs-node/config"
+)
 
 const (
 	subsection = "tree"
@@ -36,6 +40,14 @@ func (c TreeConfig) CacheSize() int {
 	return int(config.IntSafe(c.cfg, "cache_size"))
 }
 
+// ReplicationTimeout returns the value of "replication_timeout"
+// config parameter from the "tree" section.
+//
+// Returns `0` if config value is not specified.
+func (c TreeConfig) ReplicationTimeout() time.Duration {
+	return config.DurationSafe(c.cfg, "replication_timeout")
+}
+
 // ReplicationChannelCapacity returns the value of "replication_channel_capacity"
 // config parameter from the "tree" section.
 //

cmd/frostfs-node/config/tree/config_test.go

@@ -2,6 +2,7 @@ package treeconfig_test
 
 import (
 	"testing"
+	"time"
 
 	"github.com/TrueCloudLab/frostfs-node/cmd/frostfs-node/config"
 	configtest "github.com/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/test"
@@ -19,6 +20,7 @@ func TestTreeSection(t *testing.T) {
 		require.Equal(t, 0, treeSec.CacheSize())
 		require.Equal(t, 0, treeSec.ReplicationChannelCapacity())
 		require.Equal(t, 0, treeSec.ReplicationWorkerCount())
+		require.Equal(t, time.Duration(0), treeSec.ReplicationTimeout())
 	})
 
 	const path = "../../../../config/example/node"
@@ -30,6 +32,7 @@ func TestTreeSection(t *testing.T) {
 		require.Equal(t, 15, treeSec.CacheSize())
 		require.Equal(t, 32, treeSec.ReplicationChannelCapacity())
 		require.Equal(t, 32, treeSec.ReplicationWorkerCount())
+		require.Equal(t, 5*time.Second, treeSec.ReplicationTimeout())
 	}
 
 	configtest.ForEachFileType(path, fileConfigTest)

cmd/frostfs-node/tree.go

@@ -51,6 +51,7 @@ func initTreeService(c *cfg) {
 		tree.WithLogger(c.log),
 		tree.WithStorage(c.cfgObject.cfgLocalStorage.localStorage),
 		tree.WithContainerCacheSize(treeConfig.CacheSize()),
+		tree.WithReplicationTimeout(treeConfig.ReplicationTimeout()),
 		tree.WithReplicationChannelCapacity(treeConfig.ReplicationChannelCapacity()),
 		tree.WithReplicationWorkerCount(treeConfig.ReplicationWorkerCount()))
 

config/example/node.env

@@ -34,6 +34,7 @@ NEOFS_TREE_ENABLED=true
 NEOFS_TREE_CACHE_SIZE=15
 NEOFS_TREE_REPLICATION_CHANNEL_CAPACITY=32
 NEOFS_TREE_REPLICATION_WORKER_COUNT=32
+NEOFS_TREE_REPLICATION_TIMEOUT=5s
 
 # gRPC section
 ## 0 server

config/example/node.json

@@ -79,7 +79,8 @@
     "enabled": true,
     "cache_size": 15,
     "replication_channel_capacity": 32,
-    "replication_worker_count": 32
+    "replication_worker_count": 32,
+    "replication_timeout": "5s"
   },
   "control": {
     "authorized_keys": [

config/example/node.yaml

@@ -64,6 +64,7 @@ tree:
   cache_size: 15
   replication_worker_count: 32
   replication_channel_capacity: 32
+  replication_timeout: 5s
 
 control:
   authorized_keys:  # list of hex-encoded public keys that have rights to use the Control Service

pkg/services/tree/options.go

@@ -2,6 +2,7 @@ package tree
 
 import (
 	"crypto/ecdsa"
+	"time"
 
 	"github.com/TrueCloudLab/frostfs-node/pkg/core/container"
 	"github.com/TrueCloudLab/frostfs-node/pkg/core/netmap"
@@ -30,6 +31,7 @@ type cfg struct {
 	// replication-related parameters
 	replicatorChannelCapacity int
 	replicatorWorkerCount     int
+	replicatorTimeout         time.Duration
 	containerCacheSize        int
 }
 
@@ -106,3 +108,11 @@ func WithContainerCacheSize(n int) Option {
 		}
 	}
 }
+
+func WithReplicationTimeout(t time.Duration) Option {
+	return func(c *cfg) {
+		if t > 0 {
+			c.replicatorTimeout = t
+		}
+	}
+}

pkg/services/tree/replicator.go

@@ -49,7 +49,7 @@ func (s *Service) replicationWorker() {
 					return false
 				}
 
-				ctx, cancel := context.WithTimeout(context.Background(), defaultReplicatorSendTimeout)
+				ctx, cancel := context.WithTimeout(context.Background(), s.replicatorTimeout)
 				_, lastErr = c.Apply(ctx, task.req)
 				cancel()
 

pkg/services/tree/service.go

@@ -37,6 +37,7 @@ func New(opts ...Option) *Service {
 	s.containerCacheSize = defaultContainerCacheSize
 	s.replicatorChannelCapacity = defaultReplicatorCapacity
 	s.replicatorWorkerCount = defaultReplicatorWorkerCount
+	s.replicatorTimeout = defaultReplicatorSendTimeout
 
 	for i := range opts {
 		opts[i](&s.cfg)