From 597ed182694125cb62c9e515f58291416bac2b03 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Sat, 12 Nov 2022 10:46:23 +0300 Subject: [PATCH] [#2048] neofs-node: Use a separate client cache for client operations Background workers can prevent user operations to complete because of locking in cache. Signed-off-by: Evgenii Stratonikov --- CHANGELOG.md | 1 + cmd/neofs-node/config.go | 33 +++++++++++++++++++-------------- cmd/neofs-node/container.go | 2 +- cmd/neofs-node/object.go | 10 ++++++++-- cmd/neofs-node/reputation.go | 4 ++-- 5 files changed, 31 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7259f6d01..7a6140df6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ Changelog for NeoFS Node - `--binary` flag in `neofs-cli object put/get/delete` commands (#1338) - `session` flag support to `neofs-cli object hash` (#2029) - Shard can now change mode when encountering background disk errors (#2035) +- Background workers and object service now use separate client caches (#2048) ### Changed - `object lock` command reads CID and OID the same way other commands do (#1971) diff --git a/cmd/neofs-node/config.go b/cmd/neofs-node/config.go index 9c4bbce14..07a544204 100644 --- a/cmd/neofs-node/config.go +++ b/cmd/neofs-node/config.go @@ -341,8 +341,9 @@ type shared struct { privateTokenStore sessionStorage persistate *state.PersistentStorage - clientCache *cache.ClientCache - localAddr network.AddressGroup + clientCache *cache.ClientCache + bgClientCache *cache.ClientCache + localAddr network.AddressGroup key *keys.PrivateKey binPublicKey []byte @@ -555,18 +556,21 @@ func initCfg(appCfg *config.Config) *cfg { apiVersion: version.Current(), healthStatus: atomic.NewInt32(int32(control.HealthStatus_HEALTH_STATUS_UNDEFINED)), } + + cacheOpts := cache.ClientCacheOpts{ + DialTimeout: apiclientconfig.DialTimeout(appCfg), + StreamTimeout: apiclientconfig.StreamTimeout(appCfg), + Key: &key.PrivateKey, + AllowExternal: apiclientconfig.AllowExternal(appCfg), + } c.shared = shared{ - key: key, - binPublicKey: key.PublicKey().Bytes(), - localAddr: netAddr, - respSvc: response.NewService(response.WithNetworkState(netState)), - clientCache: cache.NewSDKClientCache(cache.ClientCacheOpts{ - DialTimeout: apiclientconfig.DialTimeout(appCfg), - StreamTimeout: apiclientconfig.StreamTimeout(appCfg), - Key: &key.PrivateKey, - AllowExternal: apiclientconfig.AllowExternal(appCfg), - }), - persistate: persistate, + key: key, + binPublicKey: key.PublicKey().Bytes(), + localAddr: netAddr, + respSvc: response.NewService(response.WithNetworkState(netState)), + clientCache: cache.NewSDKClientCache(cacheOpts), + bgClientCache: cache.NewSDKClientCache(cacheOpts), + persistate: persistate, } c.cfgAccounting = cfgAccounting{ scriptHash: contractsconfig.Balance(appCfg), @@ -604,7 +608,8 @@ func initCfg(appCfg *config.Config) *cfg { netState.metrics = c.metricsCollector } - c.onShutdown(c.clientCache.CloseAll) // clean up connections + c.onShutdown(c.clientCache.CloseAll) // clean up connections + c.onShutdown(c.bgClientCache.CloseAll) // clean up connections c.onShutdown(func() { _ = c.persistate.Close() }) return c diff --git a/cmd/neofs-node/container.go b/cmd/neofs-node/container.go index 9116e46b9..0ad924d3b 100644 --- a/cmd/neofs-node/container.go +++ b/cmd/neofs-node/container.go @@ -162,7 +162,7 @@ func initContainerService(c *cfg) { RemoteWriterProvider: &remoteLoadAnnounceProvider{ key: &c.key.PrivateKey, netmapKeys: c, - clientCache: c.clientCache, + clientCache: c.bgClientCache, deadEndProvider: loadcontroller.SimpleWriterProvider(loadAccumulator), }, Builder: routeBuilder, diff --git a/cmd/neofs-node/object.go b/cmd/neofs-node/object.go index c1f35ebfa..0289f0212 100644 --- a/cmd/neofs-node/object.go +++ b/cmd/neofs-node/object.go @@ -171,10 +171,16 @@ func initObjectService(c *cfg) { nmSrc: c.netMapSource, netState: c.cfgNetmap.state, trustStorage: c.cfgReputation.localTrustStorage, - basicConstructor: c.clientCache, + basicConstructor: c.bgClientCache, } - coreConstructor := (*coreClientConstructor)(clientConstructor) + coreConstructor := &coreClientConstructor{ + log: c.log, + nmSrc: c.netMapSource, + netState: c.cfgNetmap.state, + trustStorage: c.cfgReputation.localTrustStorage, + basicConstructor: c.clientCache, + } var irFetcher v2.InnerRingFetcher diff --git a/cmd/neofs-node/reputation.go b/cmd/neofs-node/reputation.go index 620ea466d..739dacdf1 100644 --- a/cmd/neofs-node/reputation.go +++ b/cmd/neofs-node/reputation.go @@ -95,7 +95,7 @@ func initReputationService(c *cfg) { common.RemoteProviderPrm{ NetmapKeys: c, DeadEndProvider: daughterStorageWriterProvider, - ClientCache: c.clientCache, + ClientCache: c.bgClientCache, WriterProvider: localreputation.NewRemoteProvider( localreputation.RemoteProviderPrm{ Key: &c.key.PrivateKey, @@ -110,7 +110,7 @@ func initReputationService(c *cfg) { common.RemoteProviderPrm{ NetmapKeys: c, DeadEndProvider: consumerStorageWriterProvider, - ClientCache: c.clientCache, + ClientCache: c.bgClientCache, WriterProvider: intermediatereputation.NewRemoteProvider( intermediatereputation.RemoteProviderPrm{ Key: &c.key.PrivateKey,