Allow to parse single unnamed selectors #162

Merged
fyrchik merged 3 commits from fyrchik/frostfs-sdk-go:fix-netmap into master 2024-09-04 19:51:15 +00:00
10 changed files with 81 additions and 23 deletions

File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 12 KiB

After

Width:  |  Height:  |  Size: 12 KiB

File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 23 KiB

After

Width:  |  Height:  |  Size: 23 KiB

File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 8.8 KiB

After

Width:  |  Height:  |  Size: 8.8 KiB

File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 12 KiB

After

Width:  |  Height:  |  Size: 12 KiB

File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 11 KiB

After

Width:  |  Height:  |  Size: 11 KiB

View file

@ -101,16 +101,16 @@ In a nutshell, a `SELECT` takes a filter result as input and outputs a specific
Let's see some examples Let's see some examples
```sql ```sql
-- Selects exactly one node from the entire netmap -- Selects exactly one node from the entire netmap
SELECT 1 FROM * SELECT 1 FROM *
-- Same as above, but with an identifier for the selection -- Same as above, but with an identifier for the selection
SELECT 1 FROM * AS ONE SELECT 1 FROM * AS ONE
-- Selects two nodes from the RedOrBlueNodes filter, such that both selected nodes -- Selects two nodes from the RedOrBlueNodes filter, such that both selected nodes
-- share the same value for the Color attribute, i.e. both red or both blue. -- share the same value for the Color attribute, i.e. both red or both blue.
SELECT 2 IN SAME Color FROM RedOrBlueNodes SELECT 2 IN SAME Color FROM RedOrBlueNodes
-- Selects two nodes from the RedOrBlueNodes filter, such that the selected nodes -- Selects two nodes from the RedOrBlueNodes filter, such that the selected nodes
-- have distinct values for the Color attribute, i.e. one red and one blue. -- have distinct values for the Color attribute, i.e. one red and one blue.
-- The selection is also given an identifier. -- The selection is also given an identifier.
SELECT 2 IN DISTINCT Color FROM RedOrBlueNodes AS MyNodes SELECT 2 IN DISTINCT Color FROM RedOrBlueNodes AS MyNodes
@ -131,7 +131,8 @@ Its basic syntax is as follows:
REP <count> {IN <select>} REP <count> {IN <select>}
``` ```
If a select is not specified, then the entire netmap is used as input. The resulting nodes will be used to actually store objects and they constitute a replica group (or simply, "a replica"). If a select is not specified, then the entire netmap is used as input. The only exception to this rule is when exactly 1 replica and 1 selector are being present: in this case the only selector is being used instead of the whole netmap.
The resulting nodes will be used to actually store objects and they constitute a replica group (or simply, "a replica").
Examples Examples
```sql ```sql
@ -173,18 +174,18 @@ In additional to this basic syntax, there are a couple of additional useful opti
### The policy playground ### The policy playground
> This section assumes you have an up-to-date version of the `frostfs-cli`. > This section assumes you have an up-to-date version of the `frostfs-cli`.
While simple placement policies have predictable results that can be understood at a glance, more complex ones need careful consideration before deployment. In order to simplify understanding a policy's outcome and experimenting while learning, a builtin tool is provided as part of the `frostfs-cli` for this purpose: the policy playground. While simple placement policies have predictable results that can be understood at a glance, more complex ones need careful consideration before deployment. In order to simplify understanding a policy's outcome and experimenting while learning, a builtin tool is provided as part of the `frostfs-cli` for this purpose: the policy playground.
For the remainder of this guide, we will use the policy playground to setup a virtual netmap (that is, one that doesn't require any networking or deployment) and test various policies. In order to visualize this netmap easily, each node will have three attributes: a character, a shape and a color For the remainder of this guide, we will use the policy playground to setup a virtual netmap (that is, one that doesn't require any networking or deployment) and test various policies. In order to visualize this netmap easily, each node will have three attributes: a character, a shape and a color
![Sample Netmap](./image/sample_netmap.svg) ![Sample Netmap](./image/sample_netmap.svg)
We can start the policy playground as follows: We can start the policy playground as follows:
```sh ```sh
$ frostfs-cli container policy-playground $ frostfs-cli container policy-playground
> >
``` ```
Since we didn't pass any endpoint, the initial netmap is empty, which we can verify with the `ls` command (to list the nodes in the netmap): Since we didn't pass any endpoint, the initial netmap is empty, which we can verify with the `ls` command (to list the nodes in the netmap):
@ -400,7 +401,7 @@ FILTER Color EQ 'Green' AS GreenNodes
#### Example #6 #### Example #6
```sql ```sql
REP 1 IN MyNodes REP 1 IN MyNodes
REP 2 REP 2
CBF 2 CBF 2
SELECT 1 FROM CuteNodes AS MyNodes SELECT 1 FROM CuteNodes AS MyNodes
FILTER (Color EQ 'Blue') AND NOT (Shape EQ 'Circle' OR Shape EQ 'Square') AS CuteNodes FILTER (Color EQ 'Blue') AND NOT (Shape EQ 'Circle' OR Shape EQ 'Square') AS CuteNodes
@ -442,4 +443,4 @@ Others:
- `ls`: list nodes in the current netmap and their attributes - `ls`: list nodes in the current netmap and their attributes
- `add`: add a node to the current netmap. If it already exists, it will be overwritten. - `add`: add a node to the current netmap. If it already exists, it will be overwritten.
- `remove`: remove a node from the current netmap. - `remove`: remove a node from the current netmap.
- `eval`: evaluate a placement policy on the current netmap. - `eval`: evaluate a placement policy on the current netmap.

View file

@ -238,7 +238,7 @@ func (m NetMap) ContainerNodes(p PlacementPolicy, pivot []byte) ([][]NodeInfo, e
// marked as used by earlier replicas. // marked as used by earlier replicas.
for i := range p.replicas { for i := range p.replicas {
sName := p.replicas[i].GetSelector() sName := p.replicas[i].GetSelector()
if sName == "" { if sName == "" && !(len(p.replicas) == 1 && len(p.selectors) == 1) {
var s netmap.Selector var s netmap.Selector
s.SetCount(p.replicas[i].GetCount()) s.SetCount(p.replicas[i].GetCount())
s.SetFilter(mainFilterName) s.SetFilter(mainFilterName)
@ -258,6 +258,9 @@ func (m NetMap) ContainerNodes(p PlacementPolicy, pivot []byte) ([][]NodeInfo, e
} }
if p.unique { if p.unique {
if c.processedSelectors[sName] == nil {
return nil, fmt.Errorf("selector not found: '%s'", sName)
}
nodes, err := c.getSelection(*c.processedSelectors[sName]) nodes, err := c.getSelection(*c.processedSelectors[sName])
if err != nil { if err != nil {
return nil, err return nil, err

View file

@ -852,6 +852,7 @@ func (p *policyVisitor) VisitExpr(ctx *parser.ExprContext) any {
// validatePolicy checks high-level constraints such as filter link in SELECT // validatePolicy checks high-level constraints such as filter link in SELECT
// being actually defined in FILTER section. // being actually defined in FILTER section.
func validatePolicy(p PlacementPolicy) error { func validatePolicy(p PlacementPolicy) error {
canOmitNames := len(p.selectors) == 1 && len(p.replicas) == 1
seenFilters := map[string]bool{} seenFilters := map[string]bool{}
expectedFilters := map[string]struct{}{} expectedFilters := map[string]struct{}{}
for i := range p.filters { for i := range p.filters {
@ -865,7 +866,7 @@ func validatePolicy(p PlacementPolicy) error {
seenSelectors := map[string]*netmap.Selector{} seenSelectors := map[string]*netmap.Selector{}
for i := range p.selectors { for i := range p.selectors {
if p.selectors[i].GetName() == "" { if p.selectors[i].GetName() == "" && !canOmitNames {
return errUnnamedSelector return errUnnamedSelector
} }
if flt := p.selectors[i].GetFilter(); flt != mainFilterName { if flt := p.selectors[i].GetFilter(); flt != mainFilterName {
@ -886,7 +887,7 @@ func validatePolicy(p PlacementPolicy) error {
expectedSelectors := map[string]struct{}{} expectedSelectors := map[string]struct{}{}
for i := range p.replicas { for i := range p.replicas {
selName := p.replicas[i].GetSelector() selName := p.replicas[i].GetSelector()
if selName != "" { if selName != "" || canOmitNames {
expectedSelectors[selName] = struct{}{} expectedSelectors[selName] = struct{}{}
if seenSelectors[selName] == nil { if seenSelectors[selName] == nil {
return fmt.Errorf("%w: '%s'", errUnknownSelector, selName) return fmt.Errorf("%w: '%s'", errUnknownSelector, selName)

View file

@ -9,6 +9,9 @@ import (
func TestDecodeString(t *testing.T) { func TestDecodeString(t *testing.T) {
testCases := []string{ testCases := []string{
`REP 2
CBF 2
SELECT 2 FROM *`,
`REP 1 IN X `REP 1 IN X
CBF 1 CBF 1
SELECT 2 IN SAME Location FROM * AS X`, SELECT 2 IN SAME Location FROM * AS X`,
@ -48,10 +51,11 @@ REP 1`,
} }
invalidTestCases := map[string]error{ invalidTestCases := map[string]error{
`?REP 1`: errSyntaxError, `?REP 1`: errSyntaxError,
`REP 1 trailing garbage`: errSyntaxError, `REP 1 trailing garbage`: errSyntaxError,
`REP 1 SELECT 4 FROM * `: errUnnamedSelector, `REP 1 REP 1 SELECT 4 FROM *`: errUnnamedSelector,
`REP 1 SELECT 4 FROM * AS X`: errRedundantSelector, `REP 1 SELECT 4 FROM * SELECT 1 FROM *`: errUnnamedSelector,
`REP 1 IN X SELECT 4 FROM *`: errUnknownSelector,
`REP 1 IN X REP 2 SELECT 4 FROM * AS X FILTER 'UN-LOCODE' EQ 'RU LED' AS F`: errRedundantFilter, `REP 1 IN X REP 2 SELECT 4 FROM * AS X FILTER 'UN-LOCODE' EQ 'RU LED' AS F`: errRedundantFilter,
} }

View file

@ -283,6 +283,55 @@ func TestPlacementPolicy_Unique(t *testing.T) {
} }
} }
func TestPlacementPolicy_SingleOmitNames(t *testing.T) {
nodes := []NodeInfo{
nodeInfoFromAttributes("ID", "1", "Country", "Russia", "City", "SPB"),
nodeInfoFromAttributes("ID", "2", "Country", "Germany", "City", "Berlin"),
nodeInfoFromAttributes("ID", "3", "Country", "Russia", "City", "Moscow"),
nodeInfoFromAttributes("ID", "4", "Country", "France", "City", "Paris"),
nodeInfoFromAttributes("ID", "5", "Country", "France", "City", "Lyon"),
nodeInfoFromAttributes("ID", "6", "Country", "Russia", "City", "SPB"),
nodeInfoFromAttributes("ID", "7", "Country", "Russia", "City", "Moscow"),
nodeInfoFromAttributes("ID", "8", "Country", "Germany", "City", "Darmstadt"),
nodeInfoFromAttributes("ID", "9", "Country", "Germany", "City", "Frankfurt"),
nodeInfoFromAttributes("ID", "10", "Country", "Russia", "City", "SPB"),
nodeInfoFromAttributes("ID", "11", "Country", "Russia", "City", "Moscow"),
nodeInfoFromAttributes("ID", "12", "Country", "Germany", "City", "London"),
}
for i := range nodes {
pub := make([]byte, 33)
rand.Read(pub)
nodes[i].SetPublicKey(pub)
}
var nm NetMap
nm.SetNodes(nodes)
for _, unique := range []bool{false, true} {
t.Run(fmt.Sprintf("unique=%t", unique), func(t *testing.T) {
ssNamed := []Selector{newSelector("X", "City", 2, "FromRU", (*Selector).SelectDistinct)}
fsNamed := []Filter{newFilter("FromRU", "Country", "Russia", netmap.EQ)}
rsNamed := []ReplicaDescriptor{newReplica(1, "X")}
pNamed := newPlacementPolicy(3, rsNamed, ssNamed, fsNamed)
pNamed.unique = unique
vNamed, err := nm.ContainerNodes(pNamed, []byte{1})
require.NoError(t, err)
ssUnnamed := []Selector{newSelector("", "City", 2, "FromRU", (*Selector).SelectDistinct)}
fsUnnamed := []Filter{newFilter("FromRU", "Country", "Russia", netmap.EQ)}
rsUnnamed := []ReplicaDescriptor{newReplica(1, "")}
pUnnamed := newPlacementPolicy(3, rsUnnamed, ssUnnamed, fsUnnamed)
pUnnamed.unique = unique
vUnnamed, err := nm.ContainerNodes(pUnnamed, []byte{1})
require.NoError(t, err)
require.Equal(t, vNamed, vUnnamed)
})
}
}
func TestPlacementPolicy_MultiREP(t *testing.T) { func TestPlacementPolicy_MultiREP(t *testing.T) {
nodes := []NodeInfo{ nodes := []NodeInfo{
nodeInfoFromAttributes("ID", "1", "Country", "Russia", "City", "SPB"), nodeInfoFromAttributes("ID", "1", "Country", "Russia", "City", "SPB"),