[#2164] neofs-node: Allow to set reconnect interval

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>

CHANGELOG.md

@@ -15,6 +15,7 @@ Changelog for NeoFS Node
 - `neofs_node_object_*_req_count_success` prometheus metrics for tracking successfully executed requests (#1984)
 - Metric 'readonly' to get shards mode (#2022)
 - Tree service replication timeout (#2159)
+- `apiclient.reconnect_timeout` setting allowing to ignore failed clients for some time (#2164)
 
 ### Changed
 - `object lock` command reads CID and OID the same way other commands do (#1971)

cmd/frostfs-node/config.go

@@ -561,10 +561,11 @@ func initCfg(appCfg *config.Config) *cfg {
 	}
 
 	cacheOpts := cache.ClientCacheOpts{
-		DialTimeout:   apiclientconfig.DialTimeout(appCfg),
-		StreamTimeout: apiclientconfig.StreamTimeout(appCfg),
-		Key:           &key.PrivateKey,
-		AllowExternal: apiclientconfig.AllowExternal(appCfg),
+		DialTimeout:      apiclientconfig.DialTimeout(appCfg),
+		StreamTimeout:    apiclientconfig.StreamTimeout(appCfg),
+		Key:              &key.PrivateKey,
+		AllowExternal:    apiclientconfig.AllowExternal(appCfg),
+		ReconnectTimeout: apiclientconfig.ReconnectTimeout(appCfg),
 	}
 	c.shared = shared{
 		key:           key,

cmd/frostfs-node/config/apiclient/config.go

@@ -42,6 +42,19 @@ func StreamTimeout(c *config.Config) time.Duration {
 	return StreamTimeoutDefault
 }
 
+// ReconnectTimeout returns the value of "reconnect_timeout" config parameter
+// from "apiclient" section.
+//
+// Returns 0 if the value is not positive duration.
+func ReconnectTimeout(c *config.Config) time.Duration {
+	v := config.DurationSafe(c.Sub(subsection), "reconnect_timeout")
+	if v > 0 {
+		return v
+	}
+
+	return 0
+}
+
 // AllowExternal returns the value of "allow_external" config parameter
 // from "apiclient" section.
 //

cmd/frostfs-node/config/apiclient/config_test.go

@@ -16,6 +16,7 @@ func TestApiclientSection(t *testing.T) {
 
 		require.Equal(t, apiclientconfig.DialTimeoutDefault, apiclientconfig.DialTimeout(empty))
 		require.Equal(t, apiclientconfig.StreamTimeoutDefault, apiclientconfig.StreamTimeout(empty))
+		require.Equal(t, time.Duration(0), apiclientconfig.ReconnectTimeout(empty))
 		require.False(t, apiclientconfig.AllowExternal(empty))
 	})
 
@@ -24,6 +25,7 @@ func TestApiclientSection(t *testing.T) {
 	var fileConfigTest = func(c *config.Config) {
 		require.Equal(t, 15*time.Second, apiclientconfig.DialTimeout(c))
 		require.Equal(t, 20*time.Second, apiclientconfig.StreamTimeout(c))
+		require.Equal(t, 30*time.Second, apiclientconfig.ReconnectTimeout(c))
 		require.True(t, apiclientconfig.AllowExternal(c))
 	}
 

config/example/node.env

@@ -73,6 +73,7 @@ NEOFS_MORPH_RPC_ENDPOINT_1_PRIORITY=2
 # API Client section
 NEOFS_APICLIENT_DIAL_TIMEOUT=15s
 NEOFS_APICLIENT_STREAM_TIMEOUT=20s
+NEOFS_APICLIENT_RECONNECT_TIMEOUT=30s
 NEOFS_APICLIENT_ALLOW_EXTERNAL=true
 
 # Policer section

config/example/node.json

@@ -117,6 +117,7 @@
   "apiclient": {
     "dial_timeout": "15s",
     "stream_timeout": "20s",
+    "reconnect_timeout": "30s",
     "allow_external": true
   },
   "policer": {

config/example/node.yaml

@@ -97,6 +97,7 @@ apiclient:
   dial_timeout: 15s  # timeout for NEOFS API client connection
   stream_timeout: 20s # timeout for individual operations in a streaming RPC
   allow_external: true # allow to fallback to addresses in `ExternalAddr` attribute
+  reconnect_timeout: 30s # time to wait before reconnecting to a failed node
 
 policer:
   head_timeout: 15s  # timeout for the Policer HEAD remote operation

docs/storage-node-configuration.md

@@ -378,11 +378,13 @@ Configuration for the NeoFS API client used for communication with other NeoFS n
 apiclient:
   dial_timeout: 15s
   stream_timeout: 20s
+  reconnect_timeout: 30s
 ```
-| Parameter      | Type     | Default value | Description                                                           |
-|----------------|----------|---------------|-----------------------------------------------------------------------|
-| dial_timeout   | duration | `5s`          | Timeout for dialing connections to other storage or inner ring nodes. |
-| stream_timeout | duration | `15s`         | Timeout for individual operations in a streaming RPC.                 |
+| Parameter         | Type     | Default value | Description                                                           |
+|-------------------|----------|---------------|-----------------------------------------------------------------------|
+| dial_timeout      | duration | `5s`          | Timeout for dialing connections to other storage or inner ring nodes. |
+| stream_timeout    | duration | `15s`         | Timeout for individual operations in a streaming RPC.                 |
+| reconnect_timeout | duration | `30s`         | Time to wait before reconnecting to a failed node.                    |
 
 # `policer` section
 

pkg/network/cache/client.go

@@ -21,6 +21,7 @@ type (
 	ClientCacheOpts struct {
 		DialTimeout      time.Duration
 		StreamTimeout    time.Duration
+		ReconnectTimeout time.Duration
 		Key              *ecdsa.PrivateKey
 		ResponseCallback func(client.ResponseMetaInfo) error
 		AllowExternal    bool

pkg/network/cache/multi.go

@@ -36,6 +36,9 @@ type multiClient struct {
 const defaultReconnectInterval = time.Second * 30
 
 func newMultiClient(addr network.AddressGroup, opts ClientCacheOpts) *multiClient {
+	if opts.ReconnectTimeout <= 0 {
+		opts.ReconnectTimeout = defaultReconnectInterval
+	}
 	return &multiClient{
 		clients:           make(map[string]*singleClient),
 		addr:              addr,