Allow to parse single unnamed selectors #162
Before Width: | Height: | Size: 12 KiB After Width: | Height: | Size: 12 KiB |
Before Width: | Height: | Size: 23 KiB After Width: | Height: | Size: 23 KiB |
Before Width: | Height: | Size: 8.8 KiB After Width: | Height: | Size: 8.8 KiB |
Before Width: | Height: | Size: 12 KiB After Width: | Height: | Size: 12 KiB |
Before Width: | Height: | Size: 11 KiB After Width: | Height: | Size: 11 KiB |
|
@ -101,16 +101,16 @@ In a nutshell, a `SELECT` takes a filter result as input and outputs a specific
|
|||
Let's see some examples
|
||||
```sql
|
||||
-- Selects exactly one node from the entire netmap
|
||||
SELECT 1 FROM *
|
||||
SELECT 1 FROM *
|
||||
|
||||
-- Same as above, but with an identifier for the selection
|
||||
SELECT 1 FROM * AS ONE
|
||||
SELECT 1 FROM * AS ONE
|
||||
|
||||
-- Selects two nodes from the RedOrBlueNodes filter, such that both selected nodes
|
||||
-- Selects two nodes from the RedOrBlueNodes filter, such that both selected nodes
|
||||
-- share the same value for the Color attribute, i.e. both red or both blue.
|
||||
SELECT 2 IN SAME Color FROM RedOrBlueNodes
|
||||
SELECT 2 IN SAME Color FROM RedOrBlueNodes
|
||||
|
||||
-- Selects two nodes from the RedOrBlueNodes filter, such that the selected nodes
|
||||
-- Selects two nodes from the RedOrBlueNodes filter, such that the selected nodes
|
||||
-- have distinct values for the Color attribute, i.e. one red and one blue.
|
||||
-- The selection is also given an identifier.
|
||||
SELECT 2 IN DISTINCT Color FROM RedOrBlueNodes AS MyNodes
|
||||
|
@ -131,7 +131,8 @@ Its basic syntax is as follows:
|
|||
REP <count> {IN <select>}
|
||||
```
|
||||
|
||||
If a select is not specified, then the entire netmap is used as input. The resulting nodes will be used to actually store objects and they constitute a replica group (or simply, "a replica").
|
||||
If a select is not specified, then the entire netmap is used as input. The only exception to this rule is when exactly 1 replica and 1 selector are being present: in this case the only selector is being used instead of the whole netmap.
|
||||
The resulting nodes will be used to actually store objects and they constitute a replica group (or simply, "a replica").
|
||||
|
||||
Examples
|
||||
```sql
|
||||
|
@ -173,18 +174,18 @@ In additional to this basic syntax, there are a couple of additional useful opti
|
|||
|
||||
### The policy playground
|
||||
|
||||
> ℹ️ This section assumes you have an up-to-date version of the `frostfs-cli`.
|
||||
> ℹ️ This section assumes you have an up-to-date version of the `frostfs-cli`.
|
||||
|
||||
While simple placement policies have predictable results that can be understood at a glance, more complex ones need careful consideration before deployment. In order to simplify understanding a policy's outcome and experimenting while learning, a builtin tool is provided as part of the `frostfs-cli` for this purpose: the policy playground.
|
||||
|
||||
For the remainder of this guide, we will use the policy playground to setup a virtual netmap (that is, one that doesn't require any networking or deployment) and test various policies. In order to visualize this netmap easily, each node will have three attributes: a character, a shape and a color
|
||||
For the remainder of this guide, we will use the policy playground to setup a virtual netmap (that is, one that doesn't require any networking or deployment) and test various policies. In order to visualize this netmap easily, each node will have three attributes: a character, a shape and a color
|
||||
|
||||
![Sample Netmap](./image/sample_netmap.svg)
|
||||
|
||||
We can start the policy playground as follows:
|
||||
```sh
|
||||
$ frostfs-cli container policy-playground
|
||||
>
|
||||
>
|
||||
```
|
||||
|
||||
Since we didn't pass any endpoint, the initial netmap is empty, which we can verify with the `ls` command (to list the nodes in the netmap):
|
||||
|
@ -400,7 +401,7 @@ FILTER Color EQ 'Green' AS GreenNodes
|
|||
#### Example #6
|
||||
```sql
|
||||
REP 1 IN MyNodes
|
||||
REP 2
|
||||
REP 2
|
||||
CBF 2
|
||||
SELECT 1 FROM CuteNodes AS MyNodes
|
||||
FILTER (Color EQ 'Blue') AND NOT (Shape EQ 'Circle' OR Shape EQ 'Square') AS CuteNodes
|
||||
|
@ -442,4 +443,4 @@ Others:
|
|||
- `ls`: list nodes in the current netmap and their attributes
|
||||
- `add`: add a node to the current netmap. If it already exists, it will be overwritten.
|
||||
- `remove`: remove a node from the current netmap.
|
||||
- `eval`: evaluate a placement policy on the current netmap.
|
||||
- `eval`: evaluate a placement policy on the current netmap.
|
||||
|
|
|
@ -238,7 +238,7 @@ func (m NetMap) ContainerNodes(p PlacementPolicy, pivot []byte) ([][]NodeInfo, e
|
|||
// marked as used by earlier replicas.
|
||||
for i := range p.replicas {
|
||||
sName := p.replicas[i].GetSelector()
|
||||
if sName == "" {
|
||||
if sName == "" && !(len(p.replicas) == 1 && len(p.selectors) == 1) {
|
||||
var s netmap.Selector
|
||||
s.SetCount(p.replicas[i].GetCount())
|
||||
s.SetFilter(mainFilterName)
|
||||
|
@ -258,6 +258,9 @@ func (m NetMap) ContainerNodes(p PlacementPolicy, pivot []byte) ([][]NodeInfo, e
|
|||
}
|
||||
|
||||
if p.unique {
|
||||
if c.processedSelectors[sName] == nil {
|
||||
return nil, fmt.Errorf("selector not found: '%s'", sName)
|
||||
}
|
||||
nodes, err := c.getSelection(*c.processedSelectors[sName])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
|
@ -852,6 +852,7 @@ func (p *policyVisitor) VisitExpr(ctx *parser.ExprContext) any {
|
|||
// validatePolicy checks high-level constraints such as filter link in SELECT
|
||||
// being actually defined in FILTER section.
|
||||
func validatePolicy(p PlacementPolicy) error {
|
||||
canOmitNames := len(p.selectors) == 1 && len(p.replicas) == 1
|
||||
seenFilters := map[string]bool{}
|
||||
expectedFilters := map[string]struct{}{}
|
||||
for i := range p.filters {
|
||||
|
@ -865,7 +866,7 @@ func validatePolicy(p PlacementPolicy) error {
|
|||
|
||||
seenSelectors := map[string]*netmap.Selector{}
|
||||
for i := range p.selectors {
|
||||
if p.selectors[i].GetName() == "" {
|
||||
if p.selectors[i].GetName() == "" && !canOmitNames {
|
||||
return errUnnamedSelector
|
||||
}
|
||||
if flt := p.selectors[i].GetFilter(); flt != mainFilterName {
|
||||
|
@ -886,7 +887,7 @@ func validatePolicy(p PlacementPolicy) error {
|
|||
expectedSelectors := map[string]struct{}{}
|
||||
for i := range p.replicas {
|
||||
selName := p.replicas[i].GetSelector()
|
||||
if selName != "" {
|
||||
if selName != "" || canOmitNames {
|
||||
expectedSelectors[selName] = struct{}{}
|
||||
if seenSelectors[selName] == nil {
|
||||
return fmt.Errorf("%w: '%s'", errUnknownSelector, selName)
|
||||
|
|
|
@ -9,6 +9,9 @@ import (
|
|||
|
||||
func TestDecodeString(t *testing.T) {
|
||||
testCases := []string{
|
||||
`REP 2
|
||||
CBF 2
|
||||
SELECT 2 FROM *`,
|
||||
`REP 1 IN X
|
||||
CBF 1
|
||||
SELECT 2 IN SAME Location FROM * AS X`,
|
||||
|
@ -48,10 +51,11 @@ REP 1`,
|
|||
}
|
||||
|
||||
invalidTestCases := map[string]error{
|
||||
`?REP 1`: errSyntaxError,
|
||||
`REP 1 trailing garbage`: errSyntaxError,
|
||||
`REP 1 SELECT 4 FROM * `: errUnnamedSelector,
|
||||
`REP 1 SELECT 4 FROM * AS X`: errRedundantSelector,
|
||||
`?REP 1`: errSyntaxError,
|
||||
`REP 1 trailing garbage`: errSyntaxError,
|
||||
`REP 1 REP 1 SELECT 4 FROM *`: errUnnamedSelector,
|
||||
`REP 1 SELECT 4 FROM * SELECT 1 FROM *`: errUnnamedSelector,
|
||||
`REP 1 IN X SELECT 4 FROM *`: errUnknownSelector,
|
||||
`REP 1 IN X REP 2 SELECT 4 FROM * AS X FILTER 'UN-LOCODE' EQ 'RU LED' AS F`: errRedundantFilter,
|
||||
}
|
||||
|
||||
|
|
|
@ -283,6 +283,55 @@ func TestPlacementPolicy_Unique(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestPlacementPolicy_SingleOmitNames(t *testing.T) {
|
||||
nodes := []NodeInfo{
|
||||
nodeInfoFromAttributes("ID", "1", "Country", "Russia", "City", "SPB"),
|
||||
nodeInfoFromAttributes("ID", "2", "Country", "Germany", "City", "Berlin"),
|
||||
nodeInfoFromAttributes("ID", "3", "Country", "Russia", "City", "Moscow"),
|
||||
nodeInfoFromAttributes("ID", "4", "Country", "France", "City", "Paris"),
|
||||
nodeInfoFromAttributes("ID", "5", "Country", "France", "City", "Lyon"),
|
||||
nodeInfoFromAttributes("ID", "6", "Country", "Russia", "City", "SPB"),
|
||||
nodeInfoFromAttributes("ID", "7", "Country", "Russia", "City", "Moscow"),
|
||||
nodeInfoFromAttributes("ID", "8", "Country", "Germany", "City", "Darmstadt"),
|
||||
nodeInfoFromAttributes("ID", "9", "Country", "Germany", "City", "Frankfurt"),
|
||||
nodeInfoFromAttributes("ID", "10", "Country", "Russia", "City", "SPB"),
|
||||
nodeInfoFromAttributes("ID", "11", "Country", "Russia", "City", "Moscow"),
|
||||
nodeInfoFromAttributes("ID", "12", "Country", "Germany", "City", "London"),
|
||||
}
|
||||
for i := range nodes {
|
||||
pub := make([]byte, 33)
|
||||
rand.Read(pub)
|
||||
nodes[i].SetPublicKey(pub)
|
||||
}
|
||||
|
||||
var nm NetMap
|
||||
nm.SetNodes(nodes)
|
||||
|
||||
for _, unique := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("unique=%t", unique), func(t *testing.T) {
|
||||
ssNamed := []Selector{newSelector("X", "City", 2, "FromRU", (*Selector).SelectDistinct)}
|
||||
fsNamed := []Filter{newFilter("FromRU", "Country", "Russia", netmap.EQ)}
|
||||
rsNamed := []ReplicaDescriptor{newReplica(1, "X")}
|
||||
pNamed := newPlacementPolicy(3, rsNamed, ssNamed, fsNamed)
|
||||
pNamed.unique = unique
|
||||
|
||||
vNamed, err := nm.ContainerNodes(pNamed, []byte{1})
|
||||
require.NoError(t, err)
|
||||
|
||||
ssUnnamed := []Selector{newSelector("", "City", 2, "FromRU", (*Selector).SelectDistinct)}
|
||||
fsUnnamed := []Filter{newFilter("FromRU", "Country", "Russia", netmap.EQ)}
|
||||
rsUnnamed := []ReplicaDescriptor{newReplica(1, "")}
|
||||
pUnnamed := newPlacementPolicy(3, rsUnnamed, ssUnnamed, fsUnnamed)
|
||||
pUnnamed.unique = unique
|
||||
|
||||
vUnnamed, err := nm.ContainerNodes(pUnnamed, []byte{1})
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, vNamed, vUnnamed)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlacementPolicy_MultiREP(t *testing.T) {
|
||||
nodes := []NodeInfo{
|
||||
nodeInfoFromAttributes("ID", "1", "Country", "Russia", "City", "SPB"),
|
||||
|
|