diff --git a/CHANGELOG.md b/CHANGELOG.md index b7c97ea3..15af341f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ Changelog for NeoFS Node - `neofs_node_object_*_req_count_success` prometheus metrics for tracking successfully executed requests (#1984) - Metric 'readonly' to get shards mode (#2022) - Tree service replication timeout (#2159) +- `apiclient.reconnect_timeout` setting allowing to ignore failed clients for some time (#2164) ### Changed - `object lock` command reads CID and OID the same way other commands do (#1971) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index d9fe14da..e090f8b3 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -561,10 +561,11 @@ func initCfg(appCfg *config.Config) *cfg { } cacheOpts := cache.ClientCacheOpts{ - DialTimeout: apiclientconfig.DialTimeout(appCfg), - StreamTimeout: apiclientconfig.StreamTimeout(appCfg), - Key: &key.PrivateKey, - AllowExternal: apiclientconfig.AllowExternal(appCfg), + DialTimeout: apiclientconfig.DialTimeout(appCfg), + StreamTimeout: apiclientconfig.StreamTimeout(appCfg), + Key: &key.PrivateKey, + AllowExternal: apiclientconfig.AllowExternal(appCfg), + ReconnectTimeout: apiclientconfig.ReconnectTimeout(appCfg), } c.shared = shared{ key: key, diff --git a/cmd/frostfs-node/config/apiclient/config.go b/cmd/frostfs-node/config/apiclient/config.go index f2262283..8123ba17 100644 --- a/cmd/frostfs-node/config/apiclient/config.go +++ b/cmd/frostfs-node/config/apiclient/config.go @@ -42,6 +42,19 @@ func StreamTimeout(c *config.Config) time.Duration { return StreamTimeoutDefault } +// ReconnectTimeout returns the value of "reconnect_timeout" config parameter +// from "apiclient" section. +// +// Returns 0 if the value is not positive duration. +func ReconnectTimeout(c *config.Config) time.Duration { + v := config.DurationSafe(c.Sub(subsection), "reconnect_timeout") + if v > 0 { + return v + } + + return 0 +} + // AllowExternal returns the value of "allow_external" config parameter // from "apiclient" section. // diff --git a/cmd/frostfs-node/config/apiclient/config_test.go b/cmd/frostfs-node/config/apiclient/config_test.go index 0b16b077..589f043a 100644 --- a/cmd/frostfs-node/config/apiclient/config_test.go +++ b/cmd/frostfs-node/config/apiclient/config_test.go @@ -16,6 +16,7 @@ func TestApiclientSection(t *testing.T) { require.Equal(t, apiclientconfig.DialTimeoutDefault, apiclientconfig.DialTimeout(empty)) require.Equal(t, apiclientconfig.StreamTimeoutDefault, apiclientconfig.StreamTimeout(empty)) + require.Equal(t, time.Duration(0), apiclientconfig.ReconnectTimeout(empty)) require.False(t, apiclientconfig.AllowExternal(empty)) }) @@ -24,6 +25,7 @@ func TestApiclientSection(t *testing.T) { var fileConfigTest = func(c *config.Config) { require.Equal(t, 15*time.Second, apiclientconfig.DialTimeout(c)) require.Equal(t, 20*time.Second, apiclientconfig.StreamTimeout(c)) + require.Equal(t, 30*time.Second, apiclientconfig.ReconnectTimeout(c)) require.True(t, apiclientconfig.AllowExternal(c)) } diff --git a/config/example/node.env b/config/example/node.env index dc58d3d6..4df2d35d 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -73,6 +73,7 @@ NEOFS_MORPH_RPC_ENDPOINT_1_PRIORITY=2 # API Client section NEOFS_APICLIENT_DIAL_TIMEOUT=15s NEOFS_APICLIENT_STREAM_TIMEOUT=20s +NEOFS_APICLIENT_RECONNECT_TIMEOUT=30s NEOFS_APICLIENT_ALLOW_EXTERNAL=true # Policer section diff --git a/config/example/node.json b/config/example/node.json index bd85e34b..e739b057 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -117,6 +117,7 @@ "apiclient": { "dial_timeout": "15s", "stream_timeout": "20s", + "reconnect_timeout": "30s", "allow_external": true }, "policer": { diff --git a/config/example/node.yaml b/config/example/node.yaml index 55f8bd4f..bf2f0a27 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -97,6 +97,7 @@ apiclient: dial_timeout: 15s # timeout for NEOFS API client connection stream_timeout: 20s # timeout for individual operations in a streaming RPC allow_external: true # allow to fallback to addresses in `ExternalAddr` attribute + reconnect_timeout: 30s # time to wait before reconnecting to a failed node policer: head_timeout: 15s # timeout for the Policer HEAD remote operation diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index fdc0444d..f9ef2cc0 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -378,11 +378,13 @@ Configuration for the NeoFS API client used for communication with other NeoFS n apiclient: dial_timeout: 15s stream_timeout: 20s + reconnect_timeout: 30s ``` -| Parameter | Type | Default value | Description | -|----------------|----------|---------------|-----------------------------------------------------------------------| -| dial_timeout | duration | `5s` | Timeout for dialing connections to other storage or inner ring nodes. | -| stream_timeout | duration | `15s` | Timeout for individual operations in a streaming RPC. | +| Parameter | Type | Default value | Description | +|-------------------|----------|---------------|-----------------------------------------------------------------------| +| dial_timeout | duration | `5s` | Timeout for dialing connections to other storage or inner ring nodes. | +| stream_timeout | duration | `15s` | Timeout for individual operations in a streaming RPC. | +| reconnect_timeout | duration | `30s` | Time to wait before reconnecting to a failed node. | # `policer` section diff --git a/pkg/network/cache/client.go b/pkg/network/cache/client.go index c032c2af..56d33859 100644 --- a/pkg/network/cache/client.go +++ b/pkg/network/cache/client.go @@ -21,6 +21,7 @@ type ( ClientCacheOpts struct { DialTimeout time.Duration StreamTimeout time.Duration + ReconnectTimeout time.Duration Key *ecdsa.PrivateKey ResponseCallback func(client.ResponseMetaInfo) error AllowExternal bool diff --git a/pkg/network/cache/multi.go b/pkg/network/cache/multi.go index 64b14282..b251c5ce 100644 --- a/pkg/network/cache/multi.go +++ b/pkg/network/cache/multi.go @@ -36,6 +36,9 @@ type multiClient struct { const defaultReconnectInterval = time.Second * 30 func newMultiClient(addr network.AddressGroup, opts ClientCacheOpts) *multiClient { + if opts.ReconnectTimeout <= 0 { + opts.ReconnectTimeout = defaultReconnectInterval + } return &multiClient{ clients: make(map[string]*singleClient), addr: addr,