[#2048] neofs-node: Use a separate client cache for client operations

Background workers can prevent user operations to complete because of
locking in cache.

Signed-off-by: Evgenii Stratonikov <evgeniy@morphbits.ru>

CHANGELOG.md

@@ -8,6 +8,7 @@ Changelog for NeoFS Node
 - `--binary` flag in `neofs-cli object put/get/delete` commands (#1338)
 - `session` flag support to `neofs-cli object hash` (#2029)
 - Shard can now change mode when encountering background disk errors (#2035)
+- Background workers and object service now use separate client caches (#2048)
 
 ### Changed
 - `object lock` command reads CID and OID the same way other commands do (#1971)

cmd/neofs-node/config.go

@@ -342,6 +342,7 @@ type shared struct {
 	persistate        *state.PersistentStorage
 
 	clientCache   *cache.ClientCache
+	bgClientCache *cache.ClientCache
 	localAddr     network.AddressGroup
 
 	key            *keys.PrivateKey
@@ -555,17 +556,20 @@ func initCfg(appCfg *config.Config) *cfg {
 		apiVersion:   version.Current(),
 		healthStatus: atomic.NewInt32(int32(control.HealthStatus_HEALTH_STATUS_UNDEFINED)),
 	}
+
+	cacheOpts := cache.ClientCacheOpts{
+		DialTimeout:   apiclientconfig.DialTimeout(appCfg),
+		StreamTimeout: apiclientconfig.StreamTimeout(appCfg),
+		Key:           &key.PrivateKey,
+		AllowExternal: apiclientconfig.AllowExternal(appCfg),
+	}
 	c.shared = shared{
 		key:           key,
 		binPublicKey:  key.PublicKey().Bytes(),
 		localAddr:     netAddr,
 		respSvc:       response.NewService(response.WithNetworkState(netState)),
-		clientCache: cache.NewSDKClientCache(cache.ClientCacheOpts{
+		clientCache:   cache.NewSDKClientCache(cacheOpts),
-			DialTimeout:   apiclientconfig.DialTimeout(appCfg),
+		bgClientCache: cache.NewSDKClientCache(cacheOpts),
-			StreamTimeout: apiclientconfig.StreamTimeout(appCfg),
-			Key:           &key.PrivateKey,
-			AllowExternal: apiclientconfig.AllowExternal(appCfg),
-		}),
 		persistate:    persistate,
 	}
 	c.cfgAccounting = cfgAccounting{
@@ -605,6 +609,7 @@ func initCfg(appCfg *config.Config) *cfg {
 	}
 
 	c.onShutdown(c.clientCache.CloseAll)   // clean up connections
+	c.onShutdown(c.bgClientCache.CloseAll) // clean up connections
 	c.onShutdown(func() { _ = c.persistate.Close() })
 
 	return c

cmd/neofs-node/container.go

@@ -162,7 +162,7 @@ func initContainerService(c *cfg) {
 			RemoteWriterProvider: &remoteLoadAnnounceProvider{
 				key:             &c.key.PrivateKey,
 				netmapKeys:      c,
-				clientCache:     c.clientCache,
+				clientCache:     c.bgClientCache,
 				deadEndProvider: loadcontroller.SimpleWriterProvider(loadAccumulator),
 			},
 			Builder: routeBuilder,

cmd/neofs-node/object.go

@@ -171,10 +171,16 @@ func initObjectService(c *cfg) {
 		nmSrc:            c.netMapSource,
 		netState:         c.cfgNetmap.state,
 		trustStorage:     c.cfgReputation.localTrustStorage,
-		basicConstructor: c.clientCache,
+		basicConstructor: c.bgClientCache,
 	}
 
-	coreConstructor := (*coreClientConstructor)(clientConstructor)
+	coreConstructor := &coreClientConstructor{
+		log:              c.log,
+		nmSrc:            c.netMapSource,
+		netState:         c.cfgNetmap.state,
+		trustStorage:     c.cfgReputation.localTrustStorage,
+		basicConstructor: c.clientCache,
+	}
 
 	var irFetcher v2.InnerRingFetcher
 

cmd/neofs-node/reputation.go

@@ -95,7 +95,7 @@ func initReputationService(c *cfg) {
 		common.RemoteProviderPrm{
 			NetmapKeys:      c,
 			DeadEndProvider: daughterStorageWriterProvider,
-			ClientCache:     c.clientCache,
+			ClientCache:     c.bgClientCache,
 			WriterProvider: localreputation.NewRemoteProvider(
 				localreputation.RemoteProviderPrm{
 					Key: &c.key.PrivateKey,
@@ -110,7 +110,7 @@ func initReputationService(c *cfg) {
 		common.RemoteProviderPrm{
 			NetmapKeys:      c,
 			DeadEndProvider: consumerStorageWriterProvider,
-			ClientCache:     c.clientCache,
+			ClientCache:     c.bgClientCache,
 			WriterProvider: intermediatereputation.NewRemoteProvider(
 				intermediatereputation.RemoteProviderPrm{
 					Key: &c.key.PrivateKey,