TrueCloudLab/frostfs-node
20
1
Fork 27
Code Issues 96 Pull requests 4 Projects Releases 1 Activity Actions

[#2260] node: Use a separate client cache for PUT service

Browse source 
Currently, under a mixed load one failed PUT can lead to closing
connection for all concurrent GETs. For PUT it does no harm: we have
many other nodes to choose from. For GET we are limited by `REP N`
factor, so in case of failover we can close the connection with the only
node posessing an object, which leads to failing the whole operation.

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
This commit is contained in:
Evgenii Stratonikov 2023-02-16 16:04:53 +03:00 committed by fyrchik
parent 0b61a3c961
commit 2b755ddb12
2 changed files with 24 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
cmd/frostfs-node/config.go
View file

@ -342,9 +342,10 @@ type shared struct {
privateTokenStore sessionStorage privateTokenStore sessionStorage
persistate *state.PersistentStorage persistate *state.PersistentStorage
clientCache *cache.ClientCache clientCache *cache.ClientCache
bgClientCache *cache.ClientCache bgClientCache *cache.ClientCache
localAddr network.AddressGroup putClientCache *cache.ClientCache
localAddr network.AddressGroup
key *keys.PrivateKey key *keys.PrivateKey
binPublicKey []byte binPublicKey []byte
@ -570,13 +571,14 @@ func initCfg(appCfg *config.Config) *cfg {
ReconnectTimeout: apiclientconfig.ReconnectTimeout(appCfg), ReconnectTimeout: apiclientconfig.ReconnectTimeout(appCfg),
} }
c.shared = shared{ c.shared = shared{
key: key, key: key,
binPublicKey: key.PublicKey().Bytes(), binPublicKey: key.PublicKey().Bytes(),
localAddr: netAddr, localAddr: netAddr,
respSvc: response.NewService(response.WithNetworkState(netState)), respSvc: response.NewService(response.WithNetworkState(netState)),
clientCache: cache.NewSDKClientCache(cacheOpts), clientCache: cache.NewSDKClientCache(cacheOpts),
bgClientCache: cache.NewSDKClientCache(cacheOpts), bgClientCache: cache.NewSDKClientCache(cacheOpts),
persistate: persistate, putClientCache: cache.NewSDKClientCache(cacheOpts),
persistate: persistate,
} }
c.cfgAccounting = cfgAccounting{ c.cfgAccounting = cfgAccounting{
scriptHash: contractsconfig.Balance(appCfg), scriptHash: contractsconfig.Balance(appCfg),
@ -615,8 +617,9 @@ func initCfg(appCfg *config.Config) *cfg {
netState.metrics = c.metricsCollector netState.metrics = c.metricsCollector
} }
c.onShutdown(c.clientCache.CloseAll) // clean up connections c.onShutdown(c.clientCache.CloseAll) // clean up connections
c.onShutdown(c.bgClientCache.CloseAll) // clean up connections c.onShutdown(c.bgClientCache.CloseAll) // clean up connections
c.onShutdown(c.putClientCache.CloseAll) // clean up connections
c.onShutdown(func() { _ = c.persistate.Close() }) c.onShutdown(func() { _ = c.persistate.Close() })
return c return c

10
cmd/frostfs-node/object.go
View file

@ -182,6 +182,14 @@ func initObjectService(c *cfg) {
basicConstructor: c.clientCache, basicConstructor: c.clientCache,
} }
putConstructor := &coreClientConstructor{
log: c.log,
nmSrc: c.netMapSource,
netState: c.cfgNetmap.state,
trustStorage: c.cfgReputation.localTrustStorage,
basicConstructor: c.putClientCache,
}
var irFetcher v2.InnerRingFetcher var irFetcher v2.InnerRingFetcher
if c.cfgMorph.client.ProbeNotary() { if c.cfgMorph.client.ProbeNotary() {
@ -255,7 +263,7 @@ func initObjectService(c *cfg) {
sPut := putsvc.NewService( sPut := putsvc.NewService(
putsvc.WithKeyStorage(keyStorage), putsvc.WithKeyStorage(keyStorage),
putsvc.WithClientConstructor(coreConstructor), putsvc.WithClientConstructor(putConstructor),
putsvc.WithMaxSizeSource(newCachedMaxObjectSizeSource(c)), putsvc.WithMaxSizeSource(newCachedMaxObjectSizeSource(c)),
putsvc.WithObjectStorage(os), putsvc.WithObjectStorage(os),
putsvc.WithContainerSource(c.cfgObject.cnrSource), putsvc.WithContainerSource(c.cfgObject.cnrSource),