[#23] Support SQL-like queries for placement

JSON format is rather verbose an inconvenient to be
edited by hand. This commit implements SQL-like
language for representing placement policy.

Signed-off-by: Evgenii Stratonikov <evgeniy@nspcc.ru>
This commit is contained in:
Evgenii Stratonikov 2020-09-11 17:44:34 +03:00 committed by Alex Vanin
parent 246a15de35
commit 001a23eb31
7 changed files with 531 additions and 0 deletions

1
go.mod
View file

@ -4,6 +4,7 @@ go 1.14
require ( require (
bou.ke/monkey v1.0.2 bou.ke/monkey v1.0.2
github.com/alecthomas/participle v0.6.0
github.com/golang/protobuf v1.4.2 github.com/golang/protobuf v1.4.2
github.com/google/uuid v1.1.1 github.com/google/uuid v1.1.1
github.com/mitchellh/go-homedir v1.1.0 github.com/mitchellh/go-homedir v1.1.0

5
go.sum
View file

@ -34,6 +34,9 @@ github.com/abiosoft/ishell v2.0.0+incompatible/go.mod h1:HQR9AqF2R3P4XXpMpI0NAzg
github.com/abiosoft/readline v0.0.0-20180607040430-155bce2042db h1:CjPUSXOiYptLbTdr1RceuZgSFDQ7U15ITERUGrUORx8= github.com/abiosoft/readline v0.0.0-20180607040430-155bce2042db h1:CjPUSXOiYptLbTdr1RceuZgSFDQ7U15ITERUGrUORx8=
github.com/abiosoft/readline v0.0.0-20180607040430-155bce2042db/go.mod h1:rB3B4rKii8V21ydCbIzH5hZiCQE7f5E9SzUb/ZZx530= github.com/abiosoft/readline v0.0.0-20180607040430-155bce2042db/go.mod h1:rB3B4rKii8V21ydCbIzH5hZiCQE7f5E9SzUb/ZZx530=
github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
github.com/alecthomas/participle v0.6.0 h1:Pvo8XUCQKgIywVjz/+Ci3IsjGg+g/TdKkMcfgghKCEw=
github.com/alecthomas/participle v0.6.0/go.mod h1:HfdmEuwvr12HXQN44HPWXR0lHmVolVYe4dyL6lQ3duY=
github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4gioE62mJzwPIB8+Tee4RNCL9ulrY= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4gioE62mJzwPIB8+Tee4RNCL9ulrY=
@ -297,6 +300,8 @@ github.com/nspcc-dev/dbft v0.0.0-20200219114139-199d286ed6c1 h1:yEx9WznS+rjE0jl0
github.com/nspcc-dev/dbft v0.0.0-20200219114139-199d286ed6c1/go.mod h1:O0qtn62prQSqizzoagHmuuKoz8QMkU3SzBoKdEvm3aQ= github.com/nspcc-dev/dbft v0.0.0-20200219114139-199d286ed6c1/go.mod h1:O0qtn62prQSqizzoagHmuuKoz8QMkU3SzBoKdEvm3aQ=
github.com/nspcc-dev/dbft v0.0.0-20200711144034-c526ccc6f570 h1:EHBwlOyd2m06C3dnxhpPokpYqlNg7u5ZX/uPBhjYuZ4= github.com/nspcc-dev/dbft v0.0.0-20200711144034-c526ccc6f570 h1:EHBwlOyd2m06C3dnxhpPokpYqlNg7u5ZX/uPBhjYuZ4=
github.com/nspcc-dev/dbft v0.0.0-20200711144034-c526ccc6f570/go.mod h1:1FYQXSbb6/9HQIkoF8XO7W/S8N7AZRkBsgwbcXRvk0E= github.com/nspcc-dev/dbft v0.0.0-20200711144034-c526ccc6f570/go.mod h1:1FYQXSbb6/9HQIkoF8XO7W/S8N7AZRkBsgwbcXRvk0E=
github.com/nspcc-dev/hrw v1.0.9 h1:17VcAuTtrstmFppBjfRiia4K2wA/ukXZhLFS8Y8rz5Y=
github.com/nspcc-dev/hrw v1.0.9/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkPG06MU=
github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1:pPYwPZ2ks+uMnlRLUyXOpLieaDQSEaf4NM3zHVbRjmg= github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1:pPYwPZ2ks+uMnlRLUyXOpLieaDQSEaf4NM3zHVbRjmg=
github.com/nspcc-dev/neo-go v0.91.0 h1:KKOPMKs0fm8JIau1SuwxiLdrZ+1kDPBiVRlWwzfebWE= github.com/nspcc-dev/neo-go v0.91.0 h1:KKOPMKs0fm8JIau1SuwxiLdrZ+1kDPBiVRlWwzfebWE=
github.com/nspcc-dev/neo-go v0.91.0/go.mod h1:G6HdOWvzQ6tlvFdvFSN/PgCzLPN/X/X4d5hTjFRUDcc= github.com/nspcc-dev/neo-go v0.91.0/go.mod h1:G6HdOWvzQ6tlvFdvFSN/PgCzLPN/X/X4d5hTjFRUDcc=

20
pkg/policy/doc.go Normal file
View file

@ -0,0 +1,20 @@
// Package policy provides facilities for creating policy from SQL-like language.
// eBNF grammar is provided in `grammar.ebnf` for illustration.
//
// Current limitations:
// 1. Grouping filter expressions in parenthesis is not supported right now.
// Requiring this will make query too verbose, making it optional makes
// our grammar not LL(1). This can be supported in future.
// 2. Filters must be defined before they are used.
// This requirement may be relaxed in future.
//
// Example query:
// REP 1 in SPB
// REP 2 in Americas
// CBF 4
// SELECT 1 Node IN City FROM SPBSSD AS SPB
// SELECT 2 Node IN SAME City FROM Americas AS Americas
// FILTER SSD EQ true AS IsSSD
// FILTER @IsSSD AND Country eq "RU" AND City eq "St.Petersburg" AS SPBSSD
// FILTER 'Continent' == 'North America' OR Continent == 'South America' AS Americas
package policy

52
pkg/policy/grammar.ebnf Normal file
View file

@ -0,0 +1,52 @@
Policy ::=
RepStmt, [RepStmt],
CbtStmt?,
[SelectStmt],
[FilterStmt],
;
RepStmt ::=
'REP', Number1, (* number of object replicas *)
('AS', Ident)? (* optional selector name *)
;
CbtStmt ::= 'CBF', Number1 (* container backup factor *)
;
SelectStmt ::=
'SELECT', Number1, (* number of nodes to select without container backup factor *)
'IN', Clause?, Ident, (* bucket name *)
FROM, (Ident | '*'), (* filter reference or whole netmap *)
('AS', Ident)? (* optional selector name *)
;
Clause ::=
'SAME' (* nodes from the same bucket *)
| 'DISTINCT' (* nodes from distinct buckets *)
;
FilterStmt ::=
'FILTER', AndChain, ['OR', AndChain],
'AS', Ident (* obligatory filter name *)
;
AndChain ::=
Expr, ['AND', Expr]
;
Expr ::=
'@' Ident (* filter reference *)
| Ident, Op, Value (* attribute filter *)
;
Op ::= 'EQ' | 'NE' | 'GE' | 'GT' | 'LT' | 'LE'
;
Value ::= Ident | Number | String
;
Number1 ::= Digit1 [Digit];
Number ::= Digit [Digit];
Digit1 ::= '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' ;
Digit ::= '0' | Digit1;

60
pkg/policy/grammar.go Normal file
View file

@ -0,0 +1,60 @@
package policy
import (
"github.com/alecthomas/participle"
)
var parser *participle.Parser
func init() {
p, err := participle.Build(&query{})
if err != nil {
panic(err)
}
parser = p
}
type query struct {
Replicas []*replicaStmt `@@+`
CBF uint32 `("CBF" @Int)?`
Selectors []*selectorStmt `@@*`
Filters []*filterStmt `@@*`
}
type replicaStmt struct {
Count int `"REP" @Int`
Selector string `("IN" @Ident)?`
}
type selectorStmt struct {
Count uint32 `"SELECT" @Int`
Clause string `"IN" @("SAME" | "DISTINCT")?`
Bucket string `@Ident`
Filter string `"FROM" @(Ident | "*")`
Name string `("AS" @Ident)?`
}
type filterStmt struct {
Value *orChain `"FILTER" @@`
Name string `"AS" @Ident`
}
type filterOrExpr struct {
Reference string `"@"@Ident`
Expr *simpleExpr `| @@`
}
type orChain struct {
Clauses []*andChain `@@ ("OR" @@)*`
}
type andChain struct {
Clauses []*filterOrExpr `@@ ("AND" @@)*`
}
type simpleExpr struct {
Key string `@Ident`
// We don't use literals here to improve error messages.
Op string `@Ident`
Value string `@(Ident | String | Int)`
}

163
pkg/policy/query.go Normal file
View file

@ -0,0 +1,163 @@
package policy
import (
"errors"
"fmt"
"strings"
sdknm "github.com/nspcc-dev/neofs-api-go/pkg/netmap"
"github.com/nspcc-dev/neofs-api-go/v2/netmap"
)
var (
ErrInvalidNumber = errors.New("policy: expected positive integer")
ErrUnknownOp = errors.New("policy: unknown operation")
ErrUnknownFilter = errors.New("policy: filter not found")
ErrUnknownSelector = errors.New("policy: selector not found")
)
func parse(s string) (*query, error) {
q := new(query)
err := parser.Parse(strings.NewReader(s), q)
if err != nil {
return nil, err
}
return q, nil
}
// Parse parses s into a placement policy.
func Parse(s string) (*netmap.PlacementPolicy, error) {
q, err := parse(s)
if err != nil {
return nil, err
}
seenFilters := map[string]bool{}
fs := make([]*netmap.Filter, 0, len(q.Filters))
for _, qf := range q.Filters {
f, err := filterFromOrChain(qf.Value, seenFilters)
if err != nil {
return nil, err
}
f.SetName(qf.Name)
fs = append(fs, f)
seenFilters[qf.Name] = true
}
seenSelectors := map[string]bool{}
ss := make([]*netmap.Selector, 0, len(q.Selectors))
for _, qs := range q.Selectors {
if qs.Filter != sdknm.MainFilterName && !seenFilters[qs.Filter] {
return nil, fmt.Errorf("%w: '%s'", ErrUnknownFilter, qs.Filter)
}
s := new(netmap.Selector)
switch qs.Clause {
case "SAME":
s.SetClause(netmap.Same)
case "DISTINCT":
s.SetClause(netmap.Distinct)
default:
s.SetClause(netmap.UnspecifiedClause)
}
s.SetName(qs.Name)
seenSelectors[qs.Name] = true
s.SetFilter(qs.Filter)
s.SetAttribute(qs.Bucket)
if qs.Count == 0 {
return nil, fmt.Errorf("%w: SELECT", ErrInvalidNumber)
}
s.SetCount(qs.Count)
ss = append(ss, s)
}
rs := make([]*netmap.Replica, 0, len(q.Replicas))
for _, qr := range q.Replicas {
r := new(netmap.Replica)
if qr.Selector != "" {
if !seenSelectors[qr.Selector] {
return nil, fmt.Errorf("%w: '%s'", ErrUnknownSelector, qr.Selector)
}
r.SetSelector(qr.Selector)
}
if qr.Count == 0 {
return nil, fmt.Errorf("%w: REP", ErrInvalidNumber)
}
r.SetCount(uint32(qr.Count))
rs = append(rs, r)
}
p := new(netmap.PlacementPolicy)
p.SetFilters(fs)
p.SetSelectors(ss)
p.SetReplicas(rs)
p.SetContainerBackupFactor(q.CBF)
return p, nil
}
func filterFromOrChain(expr *orChain, seen map[string]bool) (*netmap.Filter, error) {
var fs []*netmap.Filter
for _, ac := range expr.Clauses {
f, err := filterFromAndChain(ac, seen)
if err != nil {
return nil, err
}
fs = append(fs, f)
}
if len(fs) == 1 {
return fs[0], nil
}
f := new(netmap.Filter)
f.SetOp(netmap.OR)
f.SetFilters(fs)
return f, nil
}
func filterFromAndChain(expr *andChain, seen map[string]bool) (*netmap.Filter, error) {
var fs []*netmap.Filter
for _, fe := range expr.Clauses {
var f *netmap.Filter
var err error
if fe.Expr != nil {
f, err = filterFromSimpleExpr(fe.Expr, seen)
} else {
f = new(netmap.Filter)
f.SetName(fe.Reference)
}
if err != nil {
return nil, err
}
fs = append(fs, f)
}
if len(fs) == 1 {
return fs[0], nil
}
f := new(netmap.Filter)
f.SetOp(netmap.AND)
f.SetFilters(fs)
return f, nil
}
func filterFromSimpleExpr(se *simpleExpr, seen map[string]bool) (*netmap.Filter, error) {
f := new(netmap.Filter)
f.SetKey(se.Key)
switch se.Op {
case "EQ":
f.SetOp(netmap.EQ)
case "NE":
f.SetOp(netmap.NE)
case "GE":
f.SetOp(netmap.GE)
case "GT":
f.SetOp(netmap.GT)
case "LE":
f.SetOp(netmap.LE)
case "LT":
f.SetOp(netmap.LT)
default:
return nil, fmt.Errorf("%w: '%s'", ErrUnknownOp, se.Op)
}
f.SetValue(se.Value)
return f, nil
}

230
pkg/policy/query_test.go Normal file
View file

@ -0,0 +1,230 @@
package policy
import (
"errors"
"testing"
"github.com/nspcc-dev/neofs-api-go/v2/netmap"
"github.com/stretchr/testify/require"
)
func TestSimple(t *testing.T) {
q := `REP 3`
expected := new(netmap.PlacementPolicy)
expected.SetFilters([]*netmap.Filter{})
expected.SetSelectors([]*netmap.Selector{})
expected.SetReplicas([]*netmap.Replica{newReplica("", 3)})
r, err := Parse(q)
require.NoError(t, err)
require.Equal(t, expected, r)
}
func TestSimpleWithHRWB(t *testing.T) {
q := `REP 3 CBF 4`
expected := new(netmap.PlacementPolicy)
expected.SetFilters([]*netmap.Filter{})
expected.SetSelectors([]*netmap.Selector{})
expected.SetReplicas([]*netmap.Replica{newReplica("", 3)})
expected.SetContainerBackupFactor(4)
r, err := Parse(q)
require.NoError(t, err)
require.Equal(t, expected, r)
}
func TestFromSelect(t *testing.T) {
q := `REP 1 IN SPB
SELECT 1 IN City FROM * AS SPB`
expected := new(netmap.PlacementPolicy)
expected.SetFilters([]*netmap.Filter{})
expected.SetSelectors([]*netmap.Selector{
newSelector(1, netmap.UnspecifiedClause, "City", "*", "SPB"),
})
expected.SetReplicas([]*netmap.Replica{newReplica("SPB", 1)})
r, err := Parse(q)
require.NoError(t, err)
require.Equal(t, expected, r)
}
func TestFromSelectClause(t *testing.T) {
q := `REP 4
SELECT 3 IN Country FROM *
SELECT 2 IN SAME City FROM *
SELECT 1 IN DISTINCT Continent FROM *`
expected := new(netmap.PlacementPolicy)
expected.SetFilters([]*netmap.Filter{})
expected.SetSelectors([]*netmap.Selector{
newSelector(3, netmap.UnspecifiedClause, "Country", "*", ""),
newSelector(2, netmap.Same, "City", "*", ""),
newSelector(1, netmap.Distinct, "Continent", "*", ""),
})
expected.SetReplicas([]*netmap.Replica{newReplica("", 4)})
r, err := Parse(q)
require.NoError(t, err)
require.Equal(t, expected, r)
}
func TestSimpleFilter(t *testing.T) {
q := `REP 1
SELECT 1 IN City FROM Good
FILTER Rating GT 7 AS Good`
expected := new(netmap.PlacementPolicy)
expected.SetReplicas([]*netmap.Replica{newReplica("", 1)})
expected.SetSelectors([]*netmap.Selector{
newSelector(1, netmap.UnspecifiedClause, "City", "Good", ""),
})
expected.SetFilters([]*netmap.Filter{
newFilter("Good", "Rating", "7", netmap.GT),
})
r, err := Parse(q)
require.NoError(t, err)
require.Equal(t, expected, r)
}
func TestFilterReference(t *testing.T) {
q := `REP 1
SELECT 2 IN City FROM Good
FILTER Country EQ "RU" AS FromRU
FILTER @FromRU AND Rating GT 7 AS Good`
expected := new(netmap.PlacementPolicy)
expected.SetReplicas([]*netmap.Replica{newReplica("", 1)})
expected.SetSelectors([]*netmap.Selector{
newSelector(2, netmap.UnspecifiedClause, "City", "Good", ""),
})
expected.SetFilters([]*netmap.Filter{
newFilter("FromRU", "Country", "RU", netmap.EQ),
newFilter("Good", "", "", netmap.AND,
newFilter("FromRU", "", "", netmap.UnspecifiedOperation),
newFilter("", "Rating", "7", netmap.GT)),
})
r, err := Parse(q)
require.NoError(t, err)
require.Equal(t, expected, r)
}
func TestFilterOps(t *testing.T) {
q := `REP 1
SELECT 2 IN City FROM Good
FILTER A GT 1 AND B GE 2 AND C LT 3 AND D LE 4
AND E EQ 5 AND F NE 6 AS Good`
expected := new(netmap.PlacementPolicy)
expected.SetReplicas([]*netmap.Replica{newReplica("", 1)})
expected.SetSelectors([]*netmap.Selector{
newSelector(2, netmap.UnspecifiedClause, "City", "Good", ""),
})
expected.SetFilters([]*netmap.Filter{
newFilter("Good", "", "", netmap.AND,
newFilter("", "A", "1", netmap.GT),
newFilter("", "B", "2", netmap.GE),
newFilter("", "C", "3", netmap.LT),
newFilter("", "D", "4", netmap.LE),
newFilter("", "E", "5", netmap.EQ),
newFilter("", "F", "6", netmap.NE)),
})
r, err := Parse(q)
require.NoError(t, err)
require.Equal(t, expected, r)
}
func TestWithFilterPrecedence(t *testing.T) {
q := `REP 7 IN SPB
SELECT 1 IN City FROM SPBSSD AS SPB
FILTER City EQ "SPB" AND SSD EQ true OR City EQ "SPB" AND Rating GE 5 AS SPBSSD`
expected := new(netmap.PlacementPolicy)
expected.SetReplicas([]*netmap.Replica{newReplica("SPB", 7)})
expected.SetSelectors([]*netmap.Selector{
newSelector(1, netmap.UnspecifiedClause, "City", "SPBSSD", "SPB"),
})
expected.SetFilters([]*netmap.Filter{
newFilter("SPBSSD", "", "", netmap.OR,
newFilter("", "", "", netmap.AND,
newFilter("", "City", "SPB", netmap.EQ),
newFilter("", "SSD", "true", netmap.EQ)),
newFilter("", "", "", netmap.AND,
newFilter("", "City", "SPB", netmap.EQ),
newFilter("", "Rating", "5", netmap.GE))),
})
r, err := Parse(q)
require.NoError(t, err)
require.Equal(t, expected, r)
}
func TestValidation(t *testing.T) {
t.Run("MissingSelector", func(t *testing.T) {
q := `REP 3 IN RU`
_, err := Parse(q)
require.True(t, errors.Is(err, ErrUnknownSelector), "got: %v", err)
})
t.Run("MissingFilter", func(t *testing.T) {
q := `REP 3
SELECT 1 IN City FROM MissingFilter`
_, err := Parse(q)
require.True(t, errors.Is(err, ErrUnknownFilter), "got: %v", err)
})
t.Run("UnknownOp", func(t *testing.T) {
q := `REP 3
SELECT 1 IN City FROM F
FILTER Country KEK RU AS F`
_, err := Parse(q)
require.True(t, errors.Is(err, ErrUnknownOp), "got: %v", err)
})
t.Run("TypoInREP", func(t *testing.T) {
q := `REK 3`
_, err := Parse(q)
require.Error(t, err)
})
t.Run("InvalidFilterName", func(t *testing.T) {
q := `REP 3
SELECT 1 IN City FROM F
FILTER Good AND Country EQ RU AS F
FILTER Rating EQ 5 AS Good`
_, err := Parse(q)
require.Error(t, err)
})
t.Run("InvalidNumberInREP", func(t *testing.T) {
q := `REP 0`
_, err := Parse(q)
require.True(t, errors.Is(err, ErrInvalidNumber), "got: %v", err)
})
t.Run("InvalidNumberInREP", func(t *testing.T) {
q := `REP 1 IN Good
SELECT 0 IN City FROM *`
_, err := Parse(q)
require.True(t, errors.Is(err, ErrInvalidNumber), "got: %v", err)
})
}
func newFilter(name, key, value string, op netmap.Operation, sub ...*netmap.Filter) *netmap.Filter {
f := new(netmap.Filter)
f.SetName(name)
f.SetKey(key)
f.SetValue(value)
f.SetOp(op)
f.SetFilters(sub)
return f
}
func newReplica(s string, c uint32) *netmap.Replica {
r := new(netmap.Replica)
r.SetSelector(s)
r.SetCount(c)
return r
}
func newSelector(count uint32, c netmap.Clause, attr, f, name string) *netmap.Selector {
s := new(netmap.Selector)
s.SetCount(count)
s.SetClause(c)
s.SetAttribute(attr)
s.SetFilter(f)
s.SetName(name)
return s
}