policy: Introduce ListTargets method for Policy contract #79

Closed
aarifullin wants to merge 1 commit from aarifullin/frostfs-contract:feat/78-list_targets into master
5 changed files with 112 additions and 12 deletions
Showing only changes of commit 1d4deffb7d - Show all commits

View file

@ -4,5 +4,6 @@ safemethods:
- "listChains" - "listChains"
- "getChain" - "getChain"
- "listChainsByPrefix" - "listChainsByPrefix"
- "listTargets"
- "iteratorChainsByPrefix" - "iteratorChainsByPrefix"
- "version" - "version"

View file

@ -8,6 +8,7 @@
| 'n' + uint16(len(namespace)) + namespace | []byte | Container chain | | 'n' + uint16(len(namespace)) + namespace | []byte | Container chain |
| 'm' + entity name (namespace/container) | []byte | Mapped name to an encoded number | | 'm' + entity name (namespace/container) | []byte | Mapped name to an encoded number |
| 'counter' | uint64 | Integer counter used for mapping | | 'counter' | uint64 | Integer counter used for mapping |
| 'rc' | uint64 | Actvie rules count for a target |
*/ */

View file

@ -23,8 +23,9 @@ const (
) )
const ( const (
mappingKeyPrefix = 'm' activeRulesCounterKeyPrefix = "rc"
counterKey = "counter" mappingKeyPrefix = 'm'
counterKey = "counter"
) )
const ( const (
@ -90,11 +91,15 @@ func storageKey(prefix Kind, counter int, name []byte) []byte {
return append(key, name...) return append(key, name...)
} }
func numericMapKeyPrefix(kind Kind) []byte {
return []byte{mappingKeyPrefix, byte(kind)}
}
// mapToNumeric maps a name to a number. That allows to keep more space in // mapToNumeric maps a name to a number. That allows to keep more space in
// a storage key shortening long names. Short entity // a storage key shortening long names. Short entity
// names are also mapped to prevent collisions in the map. // names are also mapped to prevent collisions in the map.
func mapToNumeric(ctx storage.Context, name []byte) (mapped int, mappingExists bool) { func mapToNumeric(ctx storage.Context, kind Kind, name []byte) (mapped int, mappingExists bool) {
mKey := append([]byte{mappingKeyPrefix}, name...) mKey := append(numericMapKeyPrefix(kind), name...)
numericID := storage.Get(ctx, mKey) numericID := storage.Get(ctx, mKey)
if numericID == nil { if numericID == nil {
return return
@ -109,8 +114,8 @@ func mapToNumeric(ctx storage.Context, name []byte) (mapped int, mappingExists b
// names are also mapped to prevent collisions in the map. // names are also mapped to prevent collisions in the map.
// If a mapping cannot be found, then the method creates and returns it. // If a mapping cannot be found, then the method creates and returns it.
// mapToNumericCreateIfNotExists is NOT applicable for a read-only context. // mapToNumericCreateIfNotExists is NOT applicable for a read-only context.
func mapToNumericCreateIfNotExists(ctx storage.Context, name []byte) int { func mapToNumericCreateIfNotExists(ctx storage.Context, kind Kind, name []byte) int {
mKey := append([]byte{mappingKeyPrefix}, name...) mKey := append(numericMapKeyPrefix(kind), name...)
numericID := storage.Get(ctx, mKey) numericID := storage.Get(ctx, mKey)
if numericID == nil { if numericID == nil {
counter := storage.Get(ctx, counterKey).(int) counter := storage.Get(ctx, counterKey).(int)
@ -126,15 +131,17 @@ func AddChain(entity Kind, entityName string, name []byte, chain []byte) {
ctx := storage.GetContext() ctx := storage.GetContext()
checkAuthorization(ctx) checkAuthorization(ctx)
entityNameBytes := mapToNumericCreateIfNotExists(ctx, []byte(entityName)) entityNameBytes := mapToNumericCreateIfNotExists(ctx, entity, []byte(entityName))
key := storageKey(entity, entityNameBytes, name) key := storageKey(entity, entityNameBytes, name)
storage.Put(ctx, key, chain) storage.Put(ctx, key, chain)
incTargetRulesCount(ctx, append(numericMapKeyPrefix(entity), entityName...))
} }
func GetChain(entity Kind, entityName string, name []byte) []byte { func GetChain(entity Kind, entityName string, name []byte) []byte {
ctx := storage.GetReadOnlyContext() ctx := storage.GetReadOnlyContext()
entityNameBytes, exists := mapToNumeric(ctx, []byte(entityName)) entityNameBytes, exists := mapToNumeric(ctx, entity, []byte(entityName))
if !exists { if !exists {
panic("not found") panic("not found")
} }
@ -152,20 +159,21 @@ func RemoveChain(entity Kind, entityName string, name []byte) {
ctx := storage.GetContext() ctx := storage.GetContext()
checkAuthorization(ctx) checkAuthorization(ctx)
entityNameBytes, exists := mapToNumeric(ctx, []byte(entityName)) entityNameBytes, exists := mapToNumeric(ctx, entity, []byte(entityName))
if !exists { if !exists {
return return
} }
key := storageKey(entity, entityNameBytes, name) key := storageKey(entity, entityNameBytes, name)
storage.Delete(ctx, key) storage.Delete(ctx, key)
decTargetRulesCount(ctx, append(numericMapKeyPrefix(entity), entityName...))
} }
func RemoveChainsByPrefix(entity Kind, entityName string, name []byte) { func RemoveChainsByPrefix(entity Kind, entityName string, name []byte) {
ctx := storage.GetContext() ctx := storage.GetContext()
checkAuthorization(ctx) checkAuthorization(ctx)
entityNameBytes, exists := mapToNumeric(ctx, []byte(entityName)) entityNameBytes, exists := mapToNumeric(ctx, entity, []byte(entityName))
if !exists { if !exists {
return return
} }
@ -174,6 +182,7 @@ func RemoveChainsByPrefix(entity Kind, entityName string, name []byte) {
it := storage.Find(ctx, key, storage.KeysOnly) it := storage.Find(ctx, key, storage.KeysOnly)
for iterator.Next(it) { for iterator.Next(it) {
storage.Delete(ctx, iterator.Value(it).([]byte)) storage.Delete(ctx, iterator.Value(it).([]byte))
decTargetRulesCount(ctx, key)
} }
} }
@ -195,7 +204,7 @@ func ListChainsByPrefix(entity Kind, entityName string, prefix []byte) [][]byte
result := [][]byte{} result := [][]byte{}
entityNameBytes, exists := mapToNumeric(ctx, []byte(entityName)) entityNameBytes, exists := mapToNumeric(ctx, entity, []byte(entityName))
if !exists { if !exists {
return result return result
} }
@ -211,7 +220,63 @@ func ListChainsByPrefix(entity Kind, entityName string, prefix []byte) [][]byte
func IteratorChainsByPrefix(entity Kind, entityName string, prefix []byte) iterator.Iterator { func IteratorChainsByPrefix(entity Kind, entityName string, prefix []byte) iterator.Iterator {
ctx := storage.GetReadOnlyContext() ctx := storage.GetReadOnlyContext()
id, _ := mapToNumeric(ctx, []byte(entityName)) id, _ := mapToNumeric(ctx, entity, []byte(entityName))

@fyrchik Since we have decided a container/namespace will get default policy with deny, we should consider a migration for already created containers. len(targets) == len(containers) + len(namespaces). That means listing containers may be a problem if the container number is huge. But we can't use iterators - we need iterators with filters, because we should check if at least one chain is defined for a name

@fyrchik Since we have decided a container/namespace will get default policy with `deny`, we should consider a migration for already created containers. `len(targets) == len(containers) + len(namespaces)`. That means listing containers may be a problem if the container number is huge. But we can't use iterators - we need iterators with filters, because we should check if at least one chain is defined for a name

Why do we need any filters? If a key exist, there must be some chain.

Why do we need any filters? If a key exist, there must be some chain.

If a key exist, there must be some chain.

This cannot be always true. If a chain existed for a while and then removed, then the mapping is left anyway (if want to remove the mapping we need to impose remove methods to check if no chains are defined for a target. Also, this is helpful because a new chain can be added to a target again).
So, briefly, even the mapping is found it does not mean a chain is defined for a target

> If a key exist, there must be some chain. This cannot be _always_ true. If a chain existed for a while and then removed, then the mapping is left anyway (if want to remove the mapping we need to impose remove methods to check if no chains are defined for a target. Also, this is helpful because a new chain can be added to a target again). So, briefly, even the mapping is found it does not mean a chain is defined for a target

Is it possible to add batchSize and last as arguments? Then you can use something like this:

for k, _ := c.Seek(prm.NextPageToken); k != nil; k, _ = c.Next() {

Is it possible to add `batchSize` and `last` as arguments? Then you can use something like this: https://git.frostfs.info/TrueCloudLab/frostfs-node/src/commit/3a41858a0f2b02f426d37abe9d9a39ca1835436f/pkg/local_object_storage/pilorama/boltdb.go#L1224

I suppose such result "paging" is possible only if some isolation can be supported to not get range invalidated. @fyrchik WDYT, is it possible in contracts?

I suppose such result "paging" is possible only if some isolation can be supported to not get range invalidated. @fyrchik WDYT, is it possible in contracts?

Sessions are the only way to do paging correctly.

Sessions are the only way to do paging correctly.

we need to impose remove methods to check if no chains are defined for a target

TBH I do not see a problem here, besides counter issues.
To make possible number reuse we can even store rule count=0 instead of removal

>we need to impose remove methods to check if no chains are defined for a target TBH I do not see a problem here, besides counter issues. To make possible number reuse we can even store rule count=0 instead of removal

I hope I properly got your idea. I introduced active rules count for a target. That helps to just check counter instead listing chains

I hope I properly got your idea. I introduced active rules count for a target. That helps to just check counter instead listing chains
keyPrefix := storageKey(entity, id, prefix) keyPrefix := storageKey(entity, id, prefix)
return storage.Find(ctx, keyPrefix, storage.ValuesOnly) return storage.Find(ctx, keyPrefix, storage.ValuesOnly)
} }
// ListTargets lists targets for which rules are defined by kind.
func ListTargets(entity Kind) [][]byte {
ctx := storage.GetReadOnlyContext()
targetNames := [][]byte{}
keyPrefix := numericMapKeyPrefix(entity)
it := storage.Find(ctx, keyPrefix, storage.KeysOnly)
for iterator.Next(it) {
key := iterator.Value(it).([]byte)
entityName := key[len(keyPrefix):]
if getTargetRulesCount(ctx, key) == 0 {
continue
}
targetNames = append(targetNames, entityName)
}
return targetNames
}
func incTargetRulesCount(ctx storage.Context, key []byte) {
ruleCounterKey := append([]byte(activeRulesCounterKeyPrefix), key...)
counter := storage.Get(ctx, ruleCounterKey)
if counter == nil {
storage.Put(ctx, ruleCounterKey, 1)
} else {
val := counter.(int)
val++
storage.Put(ctx, ruleCounterKey, val)
}
}
func decTargetRulesCount(ctx storage.Context, key []byte) {
ruleCounterKey := append([]byte(activeRulesCounterKeyPrefix), key...)
counter := storage.Get(ctx, ruleCounterKey)
if counter != nil {
val := counter.(int)
if val == 0 {
return
}
val--
storage.Put(ctx, ruleCounterKey, val)
}
}
func getTargetRulesCount(ctx storage.Context, key []byte) int {
ruleCounterKey := append([]byte(activeRulesCounterKeyPrefix), key...)
counter := storage.Get(ctx, ruleCounterKey)
if counter != nil {
return counter.(int)
}
return 0
}

View file

@ -90,6 +90,11 @@ func (c *ContractReader) ListChainsByPrefix(entity *big.Int, entityName string,
return unwrap.Array(c.invoker.Call(c.hash, "listChainsByPrefix", entity, entityName, prefix)) return unwrap.Array(c.invoker.Call(c.hash, "listChainsByPrefix", entity, entityName, prefix))
} }
// ListTargets invokes `listTargets` method of contract.
func (c *ContractReader) ListTargets(entity *big.Int) ([]stackitem.Item, error) {
return unwrap.Array(c.invoker.Call(c.hash, "listTargets", entity))
}
// Version invokes `version` method of contract. // Version invokes `version` method of contract.
func (c *ContractReader) Version() (*big.Int, error) { func (c *ContractReader) Version() (*big.Int, error) {
return unwrap.BigInt(c.invoker.Call(c.hash, "version")) return unwrap.BigInt(c.invoker.Call(c.hash, "version"))

View file

@ -39,19 +39,26 @@ func TestPolicy(t *testing.T) {
p33 := []byte("chain33") p33 := []byte("chain33")
e.Invoke(t, stackitem.Null{}, "addChain", policy.Namespace, "mynamespace", "ingress:123", p1) e.Invoke(t, stackitem.Null{}, "addChain", policy.Namespace, "mynamespace", "ingress:123", p1)
checkTargets(t, e, policy.Namespace, [][]byte{[]byte("mynamespace")})
checkChains(t, e, "mynamespace", "", "ingress", [][]byte{p1}) checkChains(t, e, "mynamespace", "", "ingress", [][]byte{p1})
checkChains(t, e, "mynamespace", "", "all", nil) checkChains(t, e, "mynamespace", "", "all", nil)
e.Invoke(t, stackitem.Null{}, "addChain", policy.Container, "cnr1", "ingress:myrule2", p2) e.Invoke(t, stackitem.Null{}, "addChain", policy.Container, "cnr1", "ingress:myrule2", p2)
checkTargets(t, e, policy.Namespace, [][]byte{[]byte("mynamespace")})
checkTargets(t, e, policy.Container, [][]byte{[]byte("cnr1")})
checkChains(t, e, "mynamespace", "", "ingress", [][]byte{p1}) // Only namespace chains. checkChains(t, e, "mynamespace", "", "ingress", [][]byte{p1}) // Only namespace chains.
checkChains(t, e, "mynamespace", "cnr1", "ingress", [][]byte{p1, p2}) checkChains(t, e, "mynamespace", "cnr1", "ingress", [][]byte{p1, p2})
checkChains(t, e, "mynamespace", "cnr1", "all", nil) // No chains attached to 'all'. checkChains(t, e, "mynamespace", "cnr1", "all", nil) // No chains attached to 'all'.
checkChains(t, e, "mynamespace", "cnr2", "ingress", [][]byte{p1}) // Only namespace, no chains for the container. checkChains(t, e, "mynamespace", "cnr2", "ingress", [][]byte{p1}) // Only namespace, no chains for the container.
e.Invoke(t, stackitem.Null{}, "addChain", policy.Container, "cnr1", "ingress:myrule3", p3) e.Invoke(t, stackitem.Null{}, "addChain", policy.Container, "cnr1", "ingress:myrule3", p3)
checkTargets(t, e, policy.Namespace, [][]byte{[]byte("mynamespace")})
checkTargets(t, e, policy.Container, [][]byte{[]byte("cnr1")})
checkChains(t, e, "mynamespace", "cnr1", "ingress", [][]byte{p1, p2, p3}) checkChains(t, e, "mynamespace", "cnr1", "ingress", [][]byte{p1, p2, p3})
e.Invoke(t, stackitem.Null{}, "addChain", policy.Container, "cnr1", "ingress:myrule3", p33) e.Invoke(t, stackitem.Null{}, "addChain", policy.Container, "cnr1", "ingress:myrule3", p33)
checkTargets(t, e, policy.Namespace, [][]byte{[]byte("mynamespace")})
checkTargets(t, e, policy.Container, [][]byte{[]byte("cnr1")})
checkChain(t, e, policy.Container, "cnr1", "ingress:myrule3", p33) checkChain(t, e, policy.Container, "cnr1", "ingress:myrule3", p33)
checkChains(t, e, "mynamespace", "cnr1", "ingress", [][]byte{p1, p2, p33}) // Override chain. checkChains(t, e, "mynamespace", "cnr1", "ingress", [][]byte{p1, p2, p33}) // Override chain.
checkChainsByPrefix(t, e, policy.Container, "cnr1", "", [][]byte{p2, p33}) checkChainsByPrefix(t, e, policy.Container, "cnr1", "", [][]byte{p2, p33})
@ -73,6 +80,9 @@ func TestPolicy(t *testing.T) {
// Remove by prefix. // Remove by prefix.
e.Invoke(t, stackitem.Null{}, "removeChainsByPrefix", policy.Container, "cnr1", "ingress") e.Invoke(t, stackitem.Null{}, "removeChainsByPrefix", policy.Container, "cnr1", "ingress")
checkChains(t, e, "mynamespace", "cnr1", "ingress", nil) checkChains(t, e, "mynamespace", "cnr1", "ingress", nil)
checkTargets(t, e, policy.Namespace, [][]byte{})
checkTargets(t, e, policy.Container, [][]byte{})
}) })
} }
@ -148,3 +158,21 @@ func checkChain(t *testing.T, e *neotest.ContractInvoker, kind byte, entityName,
require.True(t, bytes.Equal(expected, s.Pop().Bytes())) require.True(t, bytes.Equal(expected, s.Pop().Bytes()))
} }
func checkTargets(t *testing.T, e *neotest.ContractInvoker, kind byte, expected [][]byte) {
s, err := e.TestInvoke(t, "listTargets", kind)
require.NoError(t, err)
var targets [][]byte
for _, item := range s.Pop().Array() {
target, err := item.TryBytes()
require.NoError(t, err)
targets = append(targets, target)
}
require.Len(t, targets, len(expected))
for _, exp := range expected {
require.Contains(t, targets, exp)
}
}