dstepanov-yadro/frostfs-node
1
0
Fork 0
forked from TrueCloudLab/frostfs-node
Code Issues Pull requests Projects Releases Packages Wiki Activity

[#1044] ape: Add morph chain cache

Browse source 
Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
This commit is contained in:
Dmitrii Stepanov 2024-03-15 10:10:03 +03:00
parent 6959e617c4
commit 6bf77cabd4
8 changed files with 118 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
cmd/frostfs-node/config.go
View file

@ -24,6 +24,7 @@ import (
blobovniczaconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza"
fstreeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/blobstor/fstree"
loggerconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/logger"
morphconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/morph"
nodeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/node"
objectconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/object"
replicatorconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/replicator"
@ -68,6 +69,7 @@ import (
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version"
policy_engine "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine"
policy_client "git.frostfs.info/TrueCloudLab/policy-engine/pkg/morph/policy"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
neogoutil "github.com/nspcc-dev/neo-go/pkg/util"
@ -1074,10 +1076,16 @@ func initAccessPolicyEngine(_ context.Context, c *cfg) {
)
}
morphRuleStorage := policy_client.NewContractStorage(
var morphRuleStorage policy_engine.MorphRuleChainStorage
morphRuleStorage = policy_client.NewContractStorage(
client.NewSwitchRPCGuardedActor(c.cfgMorph.client),
c.cfgObject.cfgAccessPolicyEngine.policyContractHash)
cacheSize := morphconfig.APEChainCacheSize(c.appCfg)
if cacheSize > 0 {
morphRuleStorage = newMorphCache(morphRuleStorage, int(cacheSize), morphconfig.CacheTTL(c.appCfg))
}
ape := newAccessPolicyEngine(morphRuleStorage, localOverrideDB)
c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine = ape

15
cmd/frostfs-node/config/morph/config.go
View file

@ -25,6 +25,9 @@ const (
// SwitchIntervalDefault is a default Neo RPCs switch interval.
SwitchIntervalDefault = 2 * time.Minute
// APEChainCacheSizeDefault is a default value of APE chain cache.
APEChainCacheSizeDefault = 10_000
)
var errNoMorphEndpoints = errors.New("no morph chain RPC endpoints, see `morph.rpc_endpoint` section")
@ -99,3 +102,15 @@ func SwitchInterval(c *config.Config) time.Duration {
return SwitchIntervalDefault
}
// APEChainCacheSize returns the value of "ape_chain_cache_size" config parameter
// from "morph" section.
//
// Returns 0 if the value is not positive integer.
// Returns APEChainCacheSizeDefault if the value is missing.
func APEChainCacheSize(c *config.Config) uint32 {
if c.Sub(subsection).Value("ape_chain_cache_size") == nil {
return APEChainCacheSizeDefault
}
return config.Uint32Safe(c.Sub(subsection), "ape_chain_cache_size")
}

2
cmd/frostfs-node/config/morph/config_test.go
View file

@ -19,6 +19,7 @@ func TestMorphSection(t *testing.T) {
require.Equal(t, morphconfig.DialTimeoutDefault, morphconfig.DialTimeout(empty))
require.Equal(t, morphconfig.CacheTTLDefault, morphconfig.CacheTTL(empty))
require.Equal(t, morphconfig.SwitchIntervalDefault, morphconfig.SwitchInterval(empty))
require.Equal(t, uint32(morphconfig.APEChainCacheSizeDefault), morphconfig.APEChainCacheSize(empty))
})
const path = "../../../../config/example/node"
@ -39,6 +40,7 @@ func TestMorphSection(t *testing.T) {
require.Equal(t, 30*time.Second, morphconfig.DialTimeout(c))
require.Equal(t, 15*time.Second, morphconfig.CacheTTL(c))
require.Equal(t, 3*time.Minute, morphconfig.SwitchInterval(c))
require.Equal(t, uint32(100000), morphconfig.APEChainCacheSize(c))
}
configtest.ForEachFileType(path, fileConfigTest)

67
cmd/frostfs-node/policy_engine.go
View file

@ -2,11 +2,16 @@ package main
import (
"sync"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/chainbase"
"git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain"
"git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine"
"git.frostfs.info/TrueCloudLab/policy-engine/pkg/resource"
"github.com/google/uuid"
"github.com/hashicorp/golang-lru/v2/expirable"
"github.com/nspcc-dev/neo-go/pkg/neorpc/result"
"github.com/nspcc-dev/neo-go/pkg/util"
)
type accessPolicyEngine struct {
@ -21,6 +26,68 @@ type accessPolicyEngine struct {
var _ engine.LocalOverrideEngine = (*accessPolicyEngine)(nil)
var _ engine.MorphRuleChainStorage = (*morphAPEChainCache)(nil)
type morphAPEChainCacheKey struct {
name chain.Name
target engine.Target
}
type morphAPEChainCache struct {
source engine.MorphRuleChainStorage
cache *expirable.LRU[morphAPEChainCacheKey, []*chain.Chain]
}
func newMorphCache(source engine.MorphRuleChainStorage, size int, ttl time.Duration) engine.MorphRuleChainStorage {
return &morphAPEChainCache{
source: source,
cache: expirable.NewLRU(size, func(morphAPEChainCacheKey, []*chain.Chain) {}, ttl),
}
}
func (m *morphAPEChainCache) AddMorphRuleChain(name chain.Name, target engine.Target, c *chain.Chain) (util.Uint256, uint32, error) {
m.cache.Remove(morphAPEChainCacheKey{name: name, target: target})
return m.source.AddMorphRuleChain(name, target, c)
}
func (m *morphAPEChainCache) GetAdmin() (util.Uint160, error) {
return m.source.GetAdmin()
}
func (m *morphAPEChainCache) ListMorphRuleChains(name chain.Name, target engine.Target) ([]*chain.Chain, error) {
key := morphAPEChainCacheKey{name: name, target: target}
result, found := m.cache.Get(key)
if found {
return result, nil
}
result, err := m.source.ListMorphRuleChains(name, target)
if err != nil {
return nil, err
}
m.cache.Add(key, result)
return result, nil
}
func (m *morphAPEChainCache) RemoveMorphRuleChain(name chain.Name, target engine.Target, chainID chain.ID) (util.Uint256, uint32, error) {
m.cache.Remove(morphAPEChainCacheKey{name: name, target: target})
return m.source.RemoveMorphRuleChain(name, target, chainID)
}
func (m *morphAPEChainCache) SetAdmin(addr util.Uint160) (util.Uint256, uint32, error) {
return m.source.SetAdmin(addr)
}
func (m *morphAPEChainCache) ListTargetsIterator(targetType engine.TargetType) (uuid.UUID, result.Iterator, error) {
return m.source.ListTargetsIterator(targetType)
}
func (m *morphAPEChainCache) RemoveMorphRuleChainsByTarget(name chain.Name, target engine.Target) (util.Uint256, uint32, error) {
m.cache.Remove(morphAPEChainCacheKey{name: name, target: target})
return m.source.RemoveMorphRuleChainsByTarget(name, target)
}
func newAccessPolicyEngine(
morphChainStorage engine.MorphRuleChainStorage,
localOverrideDatabase chainbase.LocalOverrideDatabase,

1
config/example/node.env
View file

@ -70,6 +70,7 @@ FROSTFS_MORPH_RPC_ENDPOINT_0_ADDRESS="wss://rpc1.morph.frostfs.info:40341/ws"
FROSTFS_MORPH_RPC_ENDPOINT_0_PRIORITY=0
FROSTFS_MORPH_RPC_ENDPOINT_1_ADDRESS="wss://rpc2.morph.frostfs.info:40341/ws"
FROSTFS_MORPH_RPC_ENDPOINT_1_PRIORITY=2
FROSTFS_MORPH_APE_CHAIN_CACHE_SIZE=100000
# API Client section
FROSTFS_APICLIENT_DIAL_TIMEOUT=15s

3
config/example/node.json
View file

@ -110,7 +110,8 @@
"address": "wss://rpc2.morph.frostfs.info:40341/ws",
"priority": 2
}
]
],
"ape_chain_cache_size": 100000
},
"apiclient": {
"dial_timeout": "15s",

1
config/example/node.yaml
View file

@ -94,6 +94,7 @@ morph:
priority: 0
- address: wss://rpc2.morph.frostfs.info:40341/ws
priority: 2
ape_chain_cache_size: 100000
apiclient:
dial_timeout: 15s # timeout for FrostFS API client connection

40
docs/storage-node-configuration.md
View file

@ -12,19 +12,19 @@ There are some custom types used for brevity:
# Structure
| Section | Description |
|--------------|---------------------------------------------------------|
| `logger` | [Logging parameters](#logger-section) |
| `pprof` | [PProf configuration](#pprof-section) |
| `prometheus` | [Prometheus metrics configuration](#prometheus-section) |
| `control` | [Control service configuration](#control-section) |
| `contracts` | [Override FrostFS contracts hashes](#contracts-section) |
| `morph` | [N3 blockchain client configuration](#morph-section) |
| `apiclient` | [FrostFS API client configuration](#apiclient-section) |
| `policer` | [Policer service configuration](#policer-section) |
| `replicator` | [Replicator service configuration](#replicator-section) |
| `storage` | [Storage engine configuration](#storage-section) |
| `runtime` | [Runtime configuration](#runtime-section) |
| Section | Description |
|------------------------|---------------------------------------------------------------------|
| `logger` | [Logging parameters](#logger-section) |
| `pprof` | [PProf configuration](#pprof-section) |
| `prometheus` | [Prometheus metrics configuration](#prometheus-section) |
| `control` | [Control service configuration](#control-section) |
| `contracts` | [Override FrostFS contracts hashes](#contracts-section) |
| `morph` | [N3 blockchain client configuration](#morph-section) |
| `apiclient` | [FrostFS API client configuration](#apiclient-section) |
| `policer` | [Policer service configuration](#policer-section) |
| `replicator` | [Replicator service configuration](#replicator-section) |
| `storage` | [Storage engine configuration](#storage-section) |
| `runtime` | [Runtime configuration](#runtime-section) |
# `control` section
@ -139,6 +139,7 @@ contracts:
morph:
dial_timeout: 30s
cache_ttl: 15s
ape_chain_cache_size: 10000
rpc_endpoint:
- address: wss://rpc1.morph.frostfs.info:40341/ws
priority: 1
@ -147,12 +148,13 @@ morph:
switch_interval: 2m
```
| Parameter | Type | Default value | Description |
|-------------------|-----------------------------------------------------------|------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `dial_timeout` | `duration` | `5s` | Timeout for dialing connections to N3 RPCs. |
| `cache_ttl` | `duration` | Morph block time | Sidechain cache TTL value (min interval between similar calls).<br/>Negative value disables caching.<br/>Cached entities: containers, container lists, eACL tables. |
| `rpc_endpoint` | list of [endpoint descriptions](#rpc_endpoint-subsection) | | Array of endpoint descriptions. |
| `switch_interval` | `duration` | `2m` | Time interval between the attempts to connect to the highest priority RPC node if the connection is not established yet. |
| Parameter | Type | Default value | Description |
| ---------------------- | --------------------------------------------------------- | ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `dial_timeout` | `duration` | `5s` | Timeout for dialing connections to N3 RPCs. |
| `cache_ttl` | `duration` | Morph block time | Sidechain cache TTL value (min interval between similar calls).<br/>Negative value disables caching.<br/>Cached entities: containers, container lists, eACL tables. |
| `rpc_endpoint` | list of [endpoint descriptions](#rpc_endpoint-subsection) | | Array of endpoint descriptions. |
| `switch_interval` | `duration` | `2m` | Time interval between the attempts to connect to the highest priority RPC node if the connection is not established yet. |
| `ape_chain_cache_size` | `int` | `10000` | Size of the morph cache for APE chains. |
## `rpc_endpoint` subsection
| Parameter | Type | Default value | Description |