From 6bf77cabd452bad85bb3418969a38b8fc24acb59 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 15 Mar 2024 10:10:03 +0300 Subject: [PATCH] [#1044] ape: Add morph chain cache Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 10 ++- cmd/frostfs-node/config/morph/config.go | 15 +++++ cmd/frostfs-node/config/morph/config_test.go | 2 + cmd/frostfs-node/policy_engine.go | 67 ++++++++++++++++++++ config/example/node.env | 1 + config/example/node.json | 3 +- config/example/node.yaml | 1 + docs/storage-node-configuration.md | 40 ++++++------ 8 files changed, 118 insertions(+), 21 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 48b76ac0f..424970b8f 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -24,6 +24,7 @@ import ( blobovniczaconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza" fstreeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/blobstor/fstree" loggerconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/logger" + morphconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/morph" nodeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/node" objectconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/object" replicatorconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/replicator" @@ -68,6 +69,7 @@ import ( objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" + policy_engine "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" policy_client "git.frostfs.info/TrueCloudLab/policy-engine/pkg/morph/policy" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" neogoutil "github.com/nspcc-dev/neo-go/pkg/util" @@ -1074,10 +1076,16 @@ func initAccessPolicyEngine(_ context.Context, c *cfg) { ) } - morphRuleStorage := policy_client.NewContractStorage( + var morphRuleStorage policy_engine.MorphRuleChainStorage + morphRuleStorage = policy_client.NewContractStorage( client.NewSwitchRPCGuardedActor(c.cfgMorph.client), c.cfgObject.cfgAccessPolicyEngine.policyContractHash) + cacheSize := morphconfig.APEChainCacheSize(c.appCfg) + if cacheSize > 0 { + morphRuleStorage = newMorphCache(morphRuleStorage, int(cacheSize), morphconfig.CacheTTL(c.appCfg)) + } + ape := newAccessPolicyEngine(morphRuleStorage, localOverrideDB) c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine = ape diff --git a/cmd/frostfs-node/config/morph/config.go b/cmd/frostfs-node/config/morph/config.go index 866223073..c32940625 100644 --- a/cmd/frostfs-node/config/morph/config.go +++ b/cmd/frostfs-node/config/morph/config.go @@ -25,6 +25,9 @@ const ( // SwitchIntervalDefault is a default Neo RPCs switch interval. SwitchIntervalDefault = 2 * time.Minute + + // APEChainCacheSizeDefault is a default value of APE chain cache. + APEChainCacheSizeDefault = 10_000 ) var errNoMorphEndpoints = errors.New("no morph chain RPC endpoints, see `morph.rpc_endpoint` section") @@ -99,3 +102,15 @@ func SwitchInterval(c *config.Config) time.Duration { return SwitchIntervalDefault } + +// APEChainCacheSize returns the value of "ape_chain_cache_size" config parameter +// from "morph" section. +// +// Returns 0 if the value is not positive integer. +// Returns APEChainCacheSizeDefault if the value is missing. +func APEChainCacheSize(c *config.Config) uint32 { + if c.Sub(subsection).Value("ape_chain_cache_size") == nil { + return APEChainCacheSizeDefault + } + return config.Uint32Safe(c.Sub(subsection), "ape_chain_cache_size") +} diff --git a/cmd/frostfs-node/config/morph/config_test.go b/cmd/frostfs-node/config/morph/config_test.go index a30733cd0..13da3d70d 100644 --- a/cmd/frostfs-node/config/morph/config_test.go +++ b/cmd/frostfs-node/config/morph/config_test.go @@ -19,6 +19,7 @@ func TestMorphSection(t *testing.T) { require.Equal(t, morphconfig.DialTimeoutDefault, morphconfig.DialTimeout(empty)) require.Equal(t, morphconfig.CacheTTLDefault, morphconfig.CacheTTL(empty)) require.Equal(t, morphconfig.SwitchIntervalDefault, morphconfig.SwitchInterval(empty)) + require.Equal(t, uint32(morphconfig.APEChainCacheSizeDefault), morphconfig.APEChainCacheSize(empty)) }) const path = "../../../../config/example/node" @@ -39,6 +40,7 @@ func TestMorphSection(t *testing.T) { require.Equal(t, 30*time.Second, morphconfig.DialTimeout(c)) require.Equal(t, 15*time.Second, morphconfig.CacheTTL(c)) require.Equal(t, 3*time.Minute, morphconfig.SwitchInterval(c)) + require.Equal(t, uint32(100000), morphconfig.APEChainCacheSize(c)) } configtest.ForEachFileType(path, fileConfigTest) diff --git a/cmd/frostfs-node/policy_engine.go b/cmd/frostfs-node/policy_engine.go index 74cbf841c..100520f50 100644 --- a/cmd/frostfs-node/policy_engine.go +++ b/cmd/frostfs-node/policy_engine.go @@ -2,11 +2,16 @@ package main import ( "sync" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/chainbase" "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" "git.frostfs.info/TrueCloudLab/policy-engine/pkg/resource" + "github.com/google/uuid" + "github.com/hashicorp/golang-lru/v2/expirable" + "github.com/nspcc-dev/neo-go/pkg/neorpc/result" + "github.com/nspcc-dev/neo-go/pkg/util" ) type accessPolicyEngine struct { @@ -21,6 +26,68 @@ type accessPolicyEngine struct { var _ engine.LocalOverrideEngine = (*accessPolicyEngine)(nil) +var _ engine.MorphRuleChainStorage = (*morphAPEChainCache)(nil) + +type morphAPEChainCacheKey struct { + name chain.Name + target engine.Target +} + +type morphAPEChainCache struct { + source engine.MorphRuleChainStorage + cache *expirable.LRU[morphAPEChainCacheKey, []*chain.Chain] +} + +func newMorphCache(source engine.MorphRuleChainStorage, size int, ttl time.Duration) engine.MorphRuleChainStorage { + return &morphAPEChainCache{ + source: source, + cache: expirable.NewLRU(size, func(morphAPEChainCacheKey, []*chain.Chain) {}, ttl), + } +} + +func (m *morphAPEChainCache) AddMorphRuleChain(name chain.Name, target engine.Target, c *chain.Chain) (util.Uint256, uint32, error) { + m.cache.Remove(morphAPEChainCacheKey{name: name, target: target}) + return m.source.AddMorphRuleChain(name, target, c) +} + +func (m *morphAPEChainCache) GetAdmin() (util.Uint160, error) { + return m.source.GetAdmin() +} + +func (m *morphAPEChainCache) ListMorphRuleChains(name chain.Name, target engine.Target) ([]*chain.Chain, error) { + key := morphAPEChainCacheKey{name: name, target: target} + result, found := m.cache.Get(key) + if found { + return result, nil + } + + result, err := m.source.ListMorphRuleChains(name, target) + if err != nil { + return nil, err + } + + m.cache.Add(key, result) + return result, nil +} + +func (m *morphAPEChainCache) RemoveMorphRuleChain(name chain.Name, target engine.Target, chainID chain.ID) (util.Uint256, uint32, error) { + m.cache.Remove(morphAPEChainCacheKey{name: name, target: target}) + return m.source.RemoveMorphRuleChain(name, target, chainID) +} + +func (m *morphAPEChainCache) SetAdmin(addr util.Uint160) (util.Uint256, uint32, error) { + return m.source.SetAdmin(addr) +} + +func (m *morphAPEChainCache) ListTargetsIterator(targetType engine.TargetType) (uuid.UUID, result.Iterator, error) { + return m.source.ListTargetsIterator(targetType) +} + +func (m *morphAPEChainCache) RemoveMorphRuleChainsByTarget(name chain.Name, target engine.Target) (util.Uint256, uint32, error) { + m.cache.Remove(morphAPEChainCacheKey{name: name, target: target}) + return m.source.RemoveMorphRuleChainsByTarget(name, target) +} + func newAccessPolicyEngine( morphChainStorage engine.MorphRuleChainStorage, localOverrideDatabase chainbase.LocalOverrideDatabase, diff --git a/config/example/node.env b/config/example/node.env index 810a2842b..976c42629 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -70,6 +70,7 @@ FROSTFS_MORPH_RPC_ENDPOINT_0_ADDRESS="wss://rpc1.morph.frostfs.info:40341/ws" FROSTFS_MORPH_RPC_ENDPOINT_0_PRIORITY=0 FROSTFS_MORPH_RPC_ENDPOINT_1_ADDRESS="wss://rpc2.morph.frostfs.info:40341/ws" FROSTFS_MORPH_RPC_ENDPOINT_1_PRIORITY=2 +FROSTFS_MORPH_APE_CHAIN_CACHE_SIZE=100000 # API Client section FROSTFS_APICLIENT_DIAL_TIMEOUT=15s diff --git a/config/example/node.json b/config/example/node.json index e24a12f5f..648fb77b0 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -110,7 +110,8 @@ "address": "wss://rpc2.morph.frostfs.info:40341/ws", "priority": 2 } - ] + ], + "ape_chain_cache_size": 100000 }, "apiclient": { "dial_timeout": "15s", diff --git a/config/example/node.yaml b/config/example/node.yaml index c9886355f..2dcf7c4d9 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -94,6 +94,7 @@ morph: priority: 0 - address: wss://rpc2.morph.frostfs.info:40341/ws priority: 2 + ape_chain_cache_size: 100000 apiclient: dial_timeout: 15s # timeout for FrostFS API client connection diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 15071f23d..02ead3020 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -12,19 +12,19 @@ There are some custom types used for brevity: # Structure -| Section | Description | -|--------------|---------------------------------------------------------| -| `logger` | [Logging parameters](#logger-section) | -| `pprof` | [PProf configuration](#pprof-section) | -| `prometheus` | [Prometheus metrics configuration](#prometheus-section) | -| `control` | [Control service configuration](#control-section) | -| `contracts` | [Override FrostFS contracts hashes](#contracts-section) | -| `morph` | [N3 blockchain client configuration](#morph-section) | -| `apiclient` | [FrostFS API client configuration](#apiclient-section) | -| `policer` | [Policer service configuration](#policer-section) | -| `replicator` | [Replicator service configuration](#replicator-section) | -| `storage` | [Storage engine configuration](#storage-section) | -| `runtime` | [Runtime configuration](#runtime-section) | +| Section | Description | +|------------------------|---------------------------------------------------------------------| +| `logger` | [Logging parameters](#logger-section) | +| `pprof` | [PProf configuration](#pprof-section) | +| `prometheus` | [Prometheus metrics configuration](#prometheus-section) | +| `control` | [Control service configuration](#control-section) | +| `contracts` | [Override FrostFS contracts hashes](#contracts-section) | +| `morph` | [N3 blockchain client configuration](#morph-section) | +| `apiclient` | [FrostFS API client configuration](#apiclient-section) | +| `policer` | [Policer service configuration](#policer-section) | +| `replicator` | [Replicator service configuration](#replicator-section) | +| `storage` | [Storage engine configuration](#storage-section) | +| `runtime` | [Runtime configuration](#runtime-section) | # `control` section @@ -139,6 +139,7 @@ contracts: morph: dial_timeout: 30s cache_ttl: 15s + ape_chain_cache_size: 10000 rpc_endpoint: - address: wss://rpc1.morph.frostfs.info:40341/ws priority: 1 @@ -147,12 +148,13 @@ morph: switch_interval: 2m ``` -| Parameter | Type | Default value | Description | -|-------------------|-----------------------------------------------------------|------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `dial_timeout` | `duration` | `5s` | Timeout for dialing connections to N3 RPCs. | -| `cache_ttl` | `duration` | Morph block time | Sidechain cache TTL value (min interval between similar calls).
Negative value disables caching.
Cached entities: containers, container lists, eACL tables. | -| `rpc_endpoint` | list of [endpoint descriptions](#rpc_endpoint-subsection) | | Array of endpoint descriptions. | -| `switch_interval` | `duration` | `2m` | Time interval between the attempts to connect to the highest priority RPC node if the connection is not established yet. | +| Parameter | Type | Default value | Description | +| ---------------------- | --------------------------------------------------------- | ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `dial_timeout` | `duration` | `5s` | Timeout for dialing connections to N3 RPCs. | +| `cache_ttl` | `duration` | Morph block time | Sidechain cache TTL value (min interval between similar calls).
Negative value disables caching.
Cached entities: containers, container lists, eACL tables. | +| `rpc_endpoint` | list of [endpoint descriptions](#rpc_endpoint-subsection) | | Array of endpoint descriptions. | +| `switch_interval` | `duration` | `2m` | Time interval between the attempts to connect to the highest priority RPC node if the connection is not established yet. | +| `ape_chain_cache_size` | `int` | `10000` | Size of the morph cache for APE chains. | ## `rpc_endpoint` subsection | Parameter | Type | Default value | Description |