[#804] ape: Implement boltdb storage for local overrides

Signed-off-by: Airat Arifullin <a.arifullin@yadro.com>

cmd/frostfs-node/config.go

@@ -29,6 +29,7 @@ import (
 	replicatorconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/replicator"
 	tracingconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/tracing"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/chainbase"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container"
 	netmapCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor"
@@ -68,6 +69,7 @@ import (
 	objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version"
+	"git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine/inmemory"
 	"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
 	neogoutil "github.com/nspcc-dev/neo-go/pkg/util"
 	"github.com/panjf2000/ants/v2"
@@ -518,10 +520,7 @@ type cfgObject struct {
 
 	eaclSource container.EACLSource
 
-	// Access policy chain source is used by object service to
-	// check for operation permissions but this source is also shared with
-	// control service that dispatches local overrides.
-	apeChainSource container.AccessPolicyEngineChainSource
+	cfgAccessPolicyEngine cfgAccessPolicyEngine
 
 	pool cfgObjectRoutines
 
@@ -542,6 +541,10 @@ type cfgLocalStorage struct {
 	localStorage *engine.StorageEngine
 }
 
+type cfgAccessPolicyEngine struct {
+	accessPolicyEngine *accessPolicyEngine
+}
+
 type cfgObjectRoutines struct {
 	putRemote *ants.Pool
 
@@ -970,6 +973,34 @@ func initLocalStorage(ctx context.Context, c *cfg) {
 	})
 }
 
+func initAccessPolicyEngine(_ context.Context, c *cfg) {
+	var localOverrideDB chainbase.LocalOverrideDatabase
+	if nodeconfig.PersistentPolicyRules(c.appCfg).Path() == "" {
+		c.log.Warn(logs.FrostFSNodePersistentRuleStorageDBPathIsNotSetInmemoryWillBeUsed)
+		localOverrideDB = chainbase.NewInmemoryLocalOverrideDatabase()
+	} else {
+		localOverrideDB = chainbase.NewBoltLocalOverrideDatabase(
+			chainbase.WithLogger(c.log),
+			chainbase.WithPath(nodeconfig.PersistentPolicyRules(c.appCfg).Path()),
+			chainbase.WithPerm(nodeconfig.PersistentPolicyRules(c.appCfg).Perm()),
+			chainbase.WithNoSync(nodeconfig.PersistentPolicyRules(c.appCfg).NoSync()),
+		)
+	}
+
+	morphRuleStorage := inmemory.NewInmemoryMorphRuleChainStorage()
+
+	ape := newAccessPolicyEngine(morphRuleStorage, localOverrideDB)
+	c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine = ape
+
+	c.onShutdown(func() {
+		if err := ape.LocalOverrideDatabaseCore().Close(); err != nil {
+			c.log.Warn(logs.FrostFSNodeAccessPolicyEngineClosingFailure,
+				zap.Error(err),
+			)
+		}
+	})
+}
+
 func initObjectPool(cfg *config.Config) (pool cfgObjectRoutines) {
 	var err error
 

cmd/frostfs-node/config/node/config.go

@@ -2,6 +2,7 @@ package nodeconfig
 
 import (
 	"fmt"
+	"io/fs"
 	"os"
 	"strconv"
 	"time"
@@ -30,11 +31,18 @@ type NotificationConfig struct {
 	cfg *config.Config
 }
 
+// PersistentPolicyRulesConfig is a wrapper over "persistent_policy_rules" config section
+// which provides access to persistent policy rules storage configuration of node.
+type PersistentPolicyRulesConfig struct {
+	cfg *config.Config
+}
+
 const (
-	subsection                   = "node"
-	persistentSessionsSubsection = "persistent_sessions"
-	persistentStateSubsection    = "persistent_state"
-	notificationSubsection       = "notification"
+	subsection                      = "node"
+	persistentSessionsSubsection    = "persistent_sessions"
+	persistentStateSubsection       = "persistent_state"
+	notificationSubsection          = "notification"
+	persistentPolicyRulesSubsection = "persistent_policy_rules"
 
 	attributePrefix = "attribute"
 
@@ -245,3 +253,42 @@ func (n NotificationConfig) KeyPath() string {
 func (n NotificationConfig) CAPath() string {
 	return config.StringSafe(n.cfg, "ca")
 }
+
+const (
+	// PermDefault is a default permission bits for local override storage file.
+	PermDefault = 0o644
+)
+
+// PersistentPolicyRules returns structure that provides access to "persistent_policy_rules"
+// subsection of "node" section.
+func PersistentPolicyRules(c *config.Config) PersistentPolicyRulesConfig {
+	return PersistentPolicyRulesConfig{
+		c.Sub(subsection).Sub(persistentPolicyRulesSubsection),
+	}
+}
+
+// Path returns the value of "path" config parameter.
+//
+// Returns empty string if missing, for compatibility with older configurations.
+func (l PersistentPolicyRulesConfig) Path() string {
+	return config.StringSafe(l.cfg, "path")
+}
+
+// Perm returns the value of "perm" config parameter as a fs.FileMode.
+//
+// Returns PermDefault if the value is not a positive number.
+func (l PersistentPolicyRulesConfig) Perm() fs.FileMode {
+	p := config.UintSafe((*config.Config)(l.cfg), "perm")
+	if p == 0 {
+		p = PermDefault
+	}
+
+	return fs.FileMode(p)
+}
+
+// NoSync returns the value of "no_sync" config parameter as a bool value.
+//
+// Returns false if the value is not a boolean.
+func (l PersistentPolicyRulesConfig) NoSync() bool {
+	return config.BoolSafe((*config.Config)(l.cfg), "no_sync")
+}

cmd/frostfs-node/control.go

@@ -51,7 +51,7 @@ func initControlService(c *cfg) {
 		controlSvc.WithTreeService(treeSynchronizer{
 			c.treeService,
 		}),
-		controlSvc.WithAPEChainSource(c.cfgObject.apeChainSource),
+		controlSvc.WithLocalOverrideStorage(c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine),
 	)
 
 	lis, err := net.Listen("tcp", endpoint)

cmd/frostfs-node/main.go

@@ -98,6 +98,12 @@ func initApp(ctx context.Context, c *cfg) {
 		fatalOnErr(c.cfgObject.cfgLocalStorage.localStorage.Init(ctx))
 	})
 
+	initAccessPolicyEngine(ctx, c)
+	initAndLog(c, "access policy engine", func(c *cfg) {
+		fatalOnErr(c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine.LocalOverrideDatabaseCore().Open(ctx))
+		fatalOnErr(c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine.LocalOverrideDatabaseCore().Init())
+	})
+
 	initAndLog(c, "gRPC", initGRPC)
 	initAndLog(c, "netmap", func(c *cfg) { initNetmapService(ctx, c) })
 	initAndLog(c, "accounting", func(c *cfg) { initAccountingService(ctx, c) })

cmd/frostfs-node/object.go

@@ -157,8 +157,6 @@ func initObjectService(c *cfg) {
 
 	c.replicator = createReplicator(c, keyStorage, c.bgClientCache)
 
-	c.cfgObject.apeChainSource = NewAPESource()
-
 	addPolicer(c, keyStorage, c.bgClientCache)
 
 	traverseGen := util.NewTraverserGenerator(c.netMapSource, c.cfgObject.cnrSource, c)
@@ -426,7 +424,7 @@ func createACLServiceV2(c *cfg, splitSvc *objectService.TransportSplitter, irFet
 			c.cfgObject.eaclSource,
 			eaclSDK.NewValidator(),
 			ls),
-		acl.NewAPEChecker(c.log, c.cfgObject.apeChainSource),
+		acl.NewAPEChecker(c.log, c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine.chainRouter),
 		c.cfgObject.cnrSource,
 		v2.WithLogger(c.log),
 	)

cmd/frostfs-node/policy_engine.go

@@ -3,33 +3,63 @@ package main
 import (
 	"sync"
 
-	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container"
-	cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/chainbase"
+	"git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain"
 	"git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine"
-	"git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine/inmemory"
+	"git.frostfs.info/TrueCloudLab/policy-engine/pkg/resource"
 )
 
-type apeChainSourceImpl struct {
-	mtx               sync.Mutex
-	localChainStorage map[cid.ID]engine.LocalOverrideEngine
+type accessPolicyEngine struct {
+	mtx sync.RWMutex
+
+	chainRouter engine.ChainRouter
+
+	morphChainStorage engine.MorphRuleChainStorage
+
+	localOverrideDatabase chainbase.LocalOverrideDatabase
 }
 
-func NewAPESource() container.AccessPolicyEngineChainSource {
-	return &apeChainSourceImpl{
-		localChainStorage: make(map[cid.ID]engine.LocalOverrideEngine),
+var _ engine.LocalOverrideEngine = (*accessPolicyEngine)(nil)
+
+func newAccessPolicyEngine(
+	morphChainStorage engine.MorphRuleChainStorage,
+	localOverrideDatabase chainbase.LocalOverrideDatabase) *accessPolicyEngine {
+	return &accessPolicyEngine{
+		chainRouter: engine.NewDefaultChainRouterWithLocalOverrides(
+			morphChainStorage,
+			localOverrideDatabase,
+		),
+
+		morphChainStorage: morphChainStorage,
+
+		localOverrideDatabase: localOverrideDatabase,
 	}
 }
 
-var _ container.AccessPolicyEngineChainSource = (*apeChainSourceImpl)(nil)
+func (a *accessPolicyEngine) IsAllowed(name chain.Name, target engine.RequestTarget, r resource.Request) (status chain.Status, found bool, err error) {
+	a.mtx.RLock()
+	defer a.mtx.RUnlock()
 
-func (c *apeChainSourceImpl) GetChainSource(cid cid.ID) (engine.LocalOverrideEngine, error) {
-	c.mtx.Lock()
-	defer c.mtx.Unlock()
-
-	s, ok := c.localChainStorage[cid]
-	if ok {
-		return s, nil
-	}
-	c.localChainStorage[cid] = inmemory.NewInMemoryLocalOverrides()
-	return c.localChainStorage[cid], nil
+	return a.chainRouter.IsAllowed(name, target, r)
+}
+
+func (a *accessPolicyEngine) MorphRuleChainStorage() engine.MorphRuleChainStorage {
+	a.mtx.Lock()
+	defer a.mtx.Unlock()
+
+	return a.morphChainStorage
+}
+
+func (a *accessPolicyEngine) LocalStorage() engine.LocalOverrideStorage {
+	a.mtx.Lock()
+	defer a.mtx.Unlock()
+
+	return a.localOverrideDatabase
+}
+
+func (a *accessPolicyEngine) LocalOverrideDatabaseCore() chainbase.DatabaseCore {
+	a.mtx.Lock()
+	defer a.mtx.Unlock()
+
+	return a.localOverrideDatabase
 }