From 001a23eb31950f82c85b6a38e31c8580a19626a7 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 11 Sep 2020 17:44:34 +0300 Subject: [PATCH] [#23] Support SQL-like queries for placement JSON format is rather verbose an inconvenient to be edited by hand. This commit implements SQL-like language for representing placement policy. Signed-off-by: Evgenii Stratonikov --- go.mod | 1 + go.sum | 5 + pkg/policy/doc.go | 20 ++++ pkg/policy/grammar.ebnf | 52 +++++++++ pkg/policy/grammar.go | 60 ++++++++++ pkg/policy/query.go | 163 +++++++++++++++++++++++++++ pkg/policy/query_test.go | 230 +++++++++++++++++++++++++++++++++++++++ 7 files changed, 531 insertions(+) create mode 100644 pkg/policy/doc.go create mode 100644 pkg/policy/grammar.ebnf create mode 100644 pkg/policy/grammar.go create mode 100644 pkg/policy/query.go create mode 100644 pkg/policy/query_test.go diff --git a/go.mod b/go.mod index 3483090f0..f6f94549e 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.14 require ( bou.ke/monkey v1.0.2 + github.com/alecthomas/participle v0.6.0 github.com/golang/protobuf v1.4.2 github.com/google/uuid v1.1.1 github.com/mitchellh/go-homedir v1.1.0 diff --git a/go.sum b/go.sum index c54ce5c71..1f49860d7 100644 --- a/go.sum +++ b/go.sum @@ -34,6 +34,9 @@ github.com/abiosoft/ishell v2.0.0+incompatible/go.mod h1:HQR9AqF2R3P4XXpMpI0NAzg github.com/abiosoft/readline v0.0.0-20180607040430-155bce2042db h1:CjPUSXOiYptLbTdr1RceuZgSFDQ7U15ITERUGrUORx8= github.com/abiosoft/readline v0.0.0-20180607040430-155bce2042db/go.mod h1:rB3B4rKii8V21ydCbIzH5hZiCQE7f5E9SzUb/ZZx530= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= +github.com/alecthomas/participle v0.6.0 h1:Pvo8XUCQKgIywVjz/+Ci3IsjGg+g/TdKkMcfgghKCEw= +github.com/alecthomas/participle v0.6.0/go.mod h1:HfdmEuwvr12HXQN44HPWXR0lHmVolVYe4dyL6lQ3duY= +github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4gioE62mJzwPIB8+Tee4RNCL9ulrY= @@ -297,6 +300,8 @@ github.com/nspcc-dev/dbft v0.0.0-20200219114139-199d286ed6c1 h1:yEx9WznS+rjE0jl0 github.com/nspcc-dev/dbft v0.0.0-20200219114139-199d286ed6c1/go.mod h1:O0qtn62prQSqizzoagHmuuKoz8QMkU3SzBoKdEvm3aQ= github.com/nspcc-dev/dbft v0.0.0-20200711144034-c526ccc6f570 h1:EHBwlOyd2m06C3dnxhpPokpYqlNg7u5ZX/uPBhjYuZ4= github.com/nspcc-dev/dbft v0.0.0-20200711144034-c526ccc6f570/go.mod h1:1FYQXSbb6/9HQIkoF8XO7W/S8N7AZRkBsgwbcXRvk0E= +github.com/nspcc-dev/hrw v1.0.9 h1:17VcAuTtrstmFppBjfRiia4K2wA/ukXZhLFS8Y8rz5Y= +github.com/nspcc-dev/hrw v1.0.9/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkPG06MU= github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1:pPYwPZ2ks+uMnlRLUyXOpLieaDQSEaf4NM3zHVbRjmg= github.com/nspcc-dev/neo-go v0.91.0 h1:KKOPMKs0fm8JIau1SuwxiLdrZ+1kDPBiVRlWwzfebWE= github.com/nspcc-dev/neo-go v0.91.0/go.mod h1:G6HdOWvzQ6tlvFdvFSN/PgCzLPN/X/X4d5hTjFRUDcc= diff --git a/pkg/policy/doc.go b/pkg/policy/doc.go new file mode 100644 index 000000000..e2f94bb77 --- /dev/null +++ b/pkg/policy/doc.go @@ -0,0 +1,20 @@ +// Package policy provides facilities for creating policy from SQL-like language. +// eBNF grammar is provided in `grammar.ebnf` for illustration. +// +// Current limitations: +// 1. Grouping filter expressions in parenthesis is not supported right now. +// Requiring this will make query too verbose, making it optional makes +// our grammar not LL(1). This can be supported in future. +// 2. Filters must be defined before they are used. +// This requirement may be relaxed in future. +// +// Example query: +// REP 1 in SPB +// REP 2 in Americas +// CBF 4 +// SELECT 1 Node IN City FROM SPBSSD AS SPB +// SELECT 2 Node IN SAME City FROM Americas AS Americas +// FILTER SSD EQ true AS IsSSD +// FILTER @IsSSD AND Country eq "RU" AND City eq "St.Petersburg" AS SPBSSD +// FILTER 'Continent' == 'North America' OR Continent == 'South America' AS Americas +package policy diff --git a/pkg/policy/grammar.ebnf b/pkg/policy/grammar.ebnf new file mode 100644 index 000000000..1b2774732 --- /dev/null +++ b/pkg/policy/grammar.ebnf @@ -0,0 +1,52 @@ +Policy ::= + RepStmt, [RepStmt], + CbtStmt?, + [SelectStmt], + [FilterStmt], +; + +RepStmt ::= + 'REP', Number1, (* number of object replicas *) + ('AS', Ident)? (* optional selector name *) +; + +CbtStmt ::= 'CBF', Number1 (* container backup factor *) +; + +SelectStmt ::= + 'SELECT', Number1, (* number of nodes to select without container backup factor *) + 'IN', Clause?, Ident, (* bucket name *) + FROM, (Ident | '*'), (* filter reference or whole netmap *) + ('AS', Ident)? (* optional selector name *) +; + +Clause ::= + 'SAME' (* nodes from the same bucket *) + | 'DISTINCT' (* nodes from distinct buckets *) +; + +FilterStmt ::= + 'FILTER', AndChain, ['OR', AndChain], + 'AS', Ident (* obligatory filter name *) +; + +AndChain ::= + Expr, ['AND', Expr] +; + +Expr ::= + '@' Ident (* filter reference *) + | Ident, Op, Value (* attribute filter *) +; + +Op ::= 'EQ' | 'NE' | 'GE' | 'GT' | 'LT' | 'LE' +; + +Value ::= Ident | Number | String +; + +Number1 ::= Digit1 [Digit]; +Number ::= Digit [Digit]; + +Digit1 ::= '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' ; +Digit ::= '0' | Digit1; diff --git a/pkg/policy/grammar.go b/pkg/policy/grammar.go new file mode 100644 index 000000000..1bbbd61a7 --- /dev/null +++ b/pkg/policy/grammar.go @@ -0,0 +1,60 @@ +package policy + +import ( + "github.com/alecthomas/participle" +) + +var parser *participle.Parser + +func init() { + p, err := participle.Build(&query{}) + if err != nil { + panic(err) + } + parser = p +} + +type query struct { + Replicas []*replicaStmt `@@+` + CBF uint32 `("CBF" @Int)?` + Selectors []*selectorStmt `@@*` + Filters []*filterStmt `@@*` +} + +type replicaStmt struct { + Count int `"REP" @Int` + Selector string `("IN" @Ident)?` +} + +type selectorStmt struct { + Count uint32 `"SELECT" @Int` + Clause string `"IN" @("SAME" | "DISTINCT")?` + Bucket string `@Ident` + Filter string `"FROM" @(Ident | "*")` + Name string `("AS" @Ident)?` +} + +type filterStmt struct { + Value *orChain `"FILTER" @@` + Name string `"AS" @Ident` +} + +type filterOrExpr struct { + Reference string `"@"@Ident` + Expr *simpleExpr `| @@` +} + +type orChain struct { + Clauses []*andChain `@@ ("OR" @@)*` +} + +type andChain struct { + Clauses []*filterOrExpr `@@ ("AND" @@)*` +} + +type simpleExpr struct { + Key string `@Ident` + // We don't use literals here to improve error messages. + Op string `@Ident` + Value string `@(Ident | String | Int)` +} diff --git a/pkg/policy/query.go b/pkg/policy/query.go new file mode 100644 index 000000000..b7e055e80 --- /dev/null +++ b/pkg/policy/query.go @@ -0,0 +1,163 @@ +package policy + +import ( + "errors" + "fmt" + "strings" + + sdknm "github.com/nspcc-dev/neofs-api-go/pkg/netmap" + "github.com/nspcc-dev/neofs-api-go/v2/netmap" +) + +var ( + ErrInvalidNumber = errors.New("policy: expected positive integer") + ErrUnknownOp = errors.New("policy: unknown operation") + ErrUnknownFilter = errors.New("policy: filter not found") + ErrUnknownSelector = errors.New("policy: selector not found") +) + +func parse(s string) (*query, error) { + q := new(query) + err := parser.Parse(strings.NewReader(s), q) + if err != nil { + return nil, err + } + return q, nil +} + +// Parse parses s into a placement policy. +func Parse(s string) (*netmap.PlacementPolicy, error) { + q, err := parse(s) + if err != nil { + return nil, err + } + + seenFilters := map[string]bool{} + fs := make([]*netmap.Filter, 0, len(q.Filters)) + for _, qf := range q.Filters { + f, err := filterFromOrChain(qf.Value, seenFilters) + if err != nil { + return nil, err + } + f.SetName(qf.Name) + fs = append(fs, f) + seenFilters[qf.Name] = true + } + + seenSelectors := map[string]bool{} + ss := make([]*netmap.Selector, 0, len(q.Selectors)) + for _, qs := range q.Selectors { + if qs.Filter != sdknm.MainFilterName && !seenFilters[qs.Filter] { + return nil, fmt.Errorf("%w: '%s'", ErrUnknownFilter, qs.Filter) + } + s := new(netmap.Selector) + switch qs.Clause { + case "SAME": + s.SetClause(netmap.Same) + case "DISTINCT": + s.SetClause(netmap.Distinct) + default: + s.SetClause(netmap.UnspecifiedClause) + } + s.SetName(qs.Name) + seenSelectors[qs.Name] = true + s.SetFilter(qs.Filter) + s.SetAttribute(qs.Bucket) + if qs.Count == 0 { + return nil, fmt.Errorf("%w: SELECT", ErrInvalidNumber) + } + s.SetCount(qs.Count) + ss = append(ss, s) + } + + rs := make([]*netmap.Replica, 0, len(q.Replicas)) + for _, qr := range q.Replicas { + r := new(netmap.Replica) + if qr.Selector != "" { + if !seenSelectors[qr.Selector] { + return nil, fmt.Errorf("%w: '%s'", ErrUnknownSelector, qr.Selector) + } + r.SetSelector(qr.Selector) + } + if qr.Count == 0 { + return nil, fmt.Errorf("%w: REP", ErrInvalidNumber) + } + r.SetCount(uint32(qr.Count)) + rs = append(rs, r) + } + + p := new(netmap.PlacementPolicy) + p.SetFilters(fs) + p.SetSelectors(ss) + p.SetReplicas(rs) + p.SetContainerBackupFactor(q.CBF) + return p, nil +} + +func filterFromOrChain(expr *orChain, seen map[string]bool) (*netmap.Filter, error) { + var fs []*netmap.Filter + for _, ac := range expr.Clauses { + f, err := filterFromAndChain(ac, seen) + if err != nil { + return nil, err + } + fs = append(fs, f) + } + if len(fs) == 1 { + return fs[0], nil + } + + f := new(netmap.Filter) + f.SetOp(netmap.OR) + f.SetFilters(fs) + return f, nil +} + +func filterFromAndChain(expr *andChain, seen map[string]bool) (*netmap.Filter, error) { + var fs []*netmap.Filter + for _, fe := range expr.Clauses { + var f *netmap.Filter + var err error + if fe.Expr != nil { + f, err = filterFromSimpleExpr(fe.Expr, seen) + } else { + f = new(netmap.Filter) + f.SetName(fe.Reference) + } + if err != nil { + return nil, err + } + fs = append(fs, f) + } + if len(fs) == 1 { + return fs[0], nil + } + + f := new(netmap.Filter) + f.SetOp(netmap.AND) + f.SetFilters(fs) + return f, nil +} + +func filterFromSimpleExpr(se *simpleExpr, seen map[string]bool) (*netmap.Filter, error) { + f := new(netmap.Filter) + f.SetKey(se.Key) + switch se.Op { + case "EQ": + f.SetOp(netmap.EQ) + case "NE": + f.SetOp(netmap.NE) + case "GE": + f.SetOp(netmap.GE) + case "GT": + f.SetOp(netmap.GT) + case "LE": + f.SetOp(netmap.LE) + case "LT": + f.SetOp(netmap.LT) + default: + return nil, fmt.Errorf("%w: '%s'", ErrUnknownOp, se.Op) + } + f.SetValue(se.Value) + return f, nil +} diff --git a/pkg/policy/query_test.go b/pkg/policy/query_test.go new file mode 100644 index 000000000..b15c33f3e --- /dev/null +++ b/pkg/policy/query_test.go @@ -0,0 +1,230 @@ +package policy + +import ( + "errors" + "testing" + + "github.com/nspcc-dev/neofs-api-go/v2/netmap" + "github.com/stretchr/testify/require" +) + +func TestSimple(t *testing.T) { + q := `REP 3` + expected := new(netmap.PlacementPolicy) + expected.SetFilters([]*netmap.Filter{}) + expected.SetSelectors([]*netmap.Selector{}) + expected.SetReplicas([]*netmap.Replica{newReplica("", 3)}) + + r, err := Parse(q) + require.NoError(t, err) + require.Equal(t, expected, r) +} + +func TestSimpleWithHRWB(t *testing.T) { + q := `REP 3 CBF 4` + expected := new(netmap.PlacementPolicy) + expected.SetFilters([]*netmap.Filter{}) + expected.SetSelectors([]*netmap.Selector{}) + expected.SetReplicas([]*netmap.Replica{newReplica("", 3)}) + expected.SetContainerBackupFactor(4) + + r, err := Parse(q) + require.NoError(t, err) + require.Equal(t, expected, r) +} + +func TestFromSelect(t *testing.T) { + q := `REP 1 IN SPB +SELECT 1 IN City FROM * AS SPB` + expected := new(netmap.PlacementPolicy) + expected.SetFilters([]*netmap.Filter{}) + expected.SetSelectors([]*netmap.Selector{ + newSelector(1, netmap.UnspecifiedClause, "City", "*", "SPB"), + }) + expected.SetReplicas([]*netmap.Replica{newReplica("SPB", 1)}) + + r, err := Parse(q) + require.NoError(t, err) + require.Equal(t, expected, r) +} + +func TestFromSelectClause(t *testing.T) { + q := `REP 4 +SELECT 3 IN Country FROM * +SELECT 2 IN SAME City FROM * +SELECT 1 IN DISTINCT Continent FROM *` + expected := new(netmap.PlacementPolicy) + expected.SetFilters([]*netmap.Filter{}) + expected.SetSelectors([]*netmap.Selector{ + newSelector(3, netmap.UnspecifiedClause, "Country", "*", ""), + newSelector(2, netmap.Same, "City", "*", ""), + newSelector(1, netmap.Distinct, "Continent", "*", ""), + }) + expected.SetReplicas([]*netmap.Replica{newReplica("", 4)}) + + r, err := Parse(q) + require.NoError(t, err) + require.Equal(t, expected, r) +} + +func TestSimpleFilter(t *testing.T) { + q := `REP 1 +SELECT 1 IN City FROM Good +FILTER Rating GT 7 AS Good` + expected := new(netmap.PlacementPolicy) + expected.SetReplicas([]*netmap.Replica{newReplica("", 1)}) + expected.SetSelectors([]*netmap.Selector{ + newSelector(1, netmap.UnspecifiedClause, "City", "Good", ""), + }) + expected.SetFilters([]*netmap.Filter{ + newFilter("Good", "Rating", "7", netmap.GT), + }) + + r, err := Parse(q) + require.NoError(t, err) + require.Equal(t, expected, r) +} + +func TestFilterReference(t *testing.T) { + q := `REP 1 +SELECT 2 IN City FROM Good +FILTER Country EQ "RU" AS FromRU +FILTER @FromRU AND Rating GT 7 AS Good` + expected := new(netmap.PlacementPolicy) + expected.SetReplicas([]*netmap.Replica{newReplica("", 1)}) + expected.SetSelectors([]*netmap.Selector{ + newSelector(2, netmap.UnspecifiedClause, "City", "Good", ""), + }) + expected.SetFilters([]*netmap.Filter{ + newFilter("FromRU", "Country", "RU", netmap.EQ), + newFilter("Good", "", "", netmap.AND, + newFilter("FromRU", "", "", netmap.UnspecifiedOperation), + newFilter("", "Rating", "7", netmap.GT)), + }) + + r, err := Parse(q) + require.NoError(t, err) + require.Equal(t, expected, r) +} + +func TestFilterOps(t *testing.T) { + q := `REP 1 +SELECT 2 IN City FROM Good +FILTER A GT 1 AND B GE 2 AND C LT 3 AND D LE 4 + AND E EQ 5 AND F NE 6 AS Good` + expected := new(netmap.PlacementPolicy) + expected.SetReplicas([]*netmap.Replica{newReplica("", 1)}) + expected.SetSelectors([]*netmap.Selector{ + newSelector(2, netmap.UnspecifiedClause, "City", "Good", ""), + }) + expected.SetFilters([]*netmap.Filter{ + newFilter("Good", "", "", netmap.AND, + newFilter("", "A", "1", netmap.GT), + newFilter("", "B", "2", netmap.GE), + newFilter("", "C", "3", netmap.LT), + newFilter("", "D", "4", netmap.LE), + newFilter("", "E", "5", netmap.EQ), + newFilter("", "F", "6", netmap.NE)), + }) + + r, err := Parse(q) + require.NoError(t, err) + require.Equal(t, expected, r) +} + +func TestWithFilterPrecedence(t *testing.T) { + q := `REP 7 IN SPB +SELECT 1 IN City FROM SPBSSD AS SPB +FILTER City EQ "SPB" AND SSD EQ true OR City EQ "SPB" AND Rating GE 5 AS SPBSSD` + expected := new(netmap.PlacementPolicy) + expected.SetReplicas([]*netmap.Replica{newReplica("SPB", 7)}) + expected.SetSelectors([]*netmap.Selector{ + newSelector(1, netmap.UnspecifiedClause, "City", "SPBSSD", "SPB"), + }) + expected.SetFilters([]*netmap.Filter{ + newFilter("SPBSSD", "", "", netmap.OR, + newFilter("", "", "", netmap.AND, + newFilter("", "City", "SPB", netmap.EQ), + newFilter("", "SSD", "true", netmap.EQ)), + newFilter("", "", "", netmap.AND, + newFilter("", "City", "SPB", netmap.EQ), + newFilter("", "Rating", "5", netmap.GE))), + }) + + r, err := Parse(q) + require.NoError(t, err) + require.Equal(t, expected, r) +} + +func TestValidation(t *testing.T) { + t.Run("MissingSelector", func(t *testing.T) { + q := `REP 3 IN RU` + _, err := Parse(q) + require.True(t, errors.Is(err, ErrUnknownSelector), "got: %v", err) + }) + t.Run("MissingFilter", func(t *testing.T) { + q := `REP 3 + SELECT 1 IN City FROM MissingFilter` + _, err := Parse(q) + require.True(t, errors.Is(err, ErrUnknownFilter), "got: %v", err) + }) + t.Run("UnknownOp", func(t *testing.T) { + q := `REP 3 + SELECT 1 IN City FROM F + FILTER Country KEK RU AS F` + _, err := Parse(q) + require.True(t, errors.Is(err, ErrUnknownOp), "got: %v", err) + }) + t.Run("TypoInREP", func(t *testing.T) { + q := `REK 3` + _, err := Parse(q) + require.Error(t, err) + }) + t.Run("InvalidFilterName", func(t *testing.T) { + q := `REP 3 + SELECT 1 IN City FROM F + FILTER Good AND Country EQ RU AS F + FILTER Rating EQ 5 AS Good` + _, err := Parse(q) + require.Error(t, err) + }) + t.Run("InvalidNumberInREP", func(t *testing.T) { + q := `REP 0` + _, err := Parse(q) + require.True(t, errors.Is(err, ErrInvalidNumber), "got: %v", err) + }) + t.Run("InvalidNumberInREP", func(t *testing.T) { + q := `REP 1 IN Good + SELECT 0 IN City FROM *` + _, err := Parse(q) + require.True(t, errors.Is(err, ErrInvalidNumber), "got: %v", err) + }) + +} + +func newFilter(name, key, value string, op netmap.Operation, sub ...*netmap.Filter) *netmap.Filter { + f := new(netmap.Filter) + f.SetName(name) + f.SetKey(key) + f.SetValue(value) + f.SetOp(op) + f.SetFilters(sub) + return f +} + +func newReplica(s string, c uint32) *netmap.Replica { + r := new(netmap.Replica) + r.SetSelector(s) + r.SetCount(c) + return r +} + +func newSelector(count uint32, c netmap.Clause, attr, f, name string) *netmap.Selector { + s := new(netmap.Selector) + s.SetCount(count) + s.SetClause(c) + s.SetAttribute(attr) + s.SetFilter(f) + s.SetName(name) + return s +}