From 0eeddf6dfec7eae824fa9c22e3f3c56bca8f8a07 Mon Sep 17 00:00:00 2001 From: Angira Kekteeva Date: Thu, 20 May 2021 22:35:03 +0300 Subject: [PATCH] [nspcc-dev/neofs-node#116] policy: Move files from neofs-node repo Took from commit on neofs-node: f81ecbe8db17e63674a47b4b6d49575aced3c16d Signed-off-by: Angira Kekteeva --- pkg/policy/doc.go | 20 +++ pkg/policy/encode.go | 140 +++++++++++++++++++ pkg/policy/encode_test.go | 36 +++++ pkg/policy/grammar.ebnf | 55 ++++++++ pkg/policy/grammar.go | 59 ++++++++ pkg/policy/json.go | 225 ++++++++++++++++++++++++++++++ pkg/policy/json_test.go | 84 +++++++++++ pkg/policy/query.go | 173 +++++++++++++++++++++++ pkg/policy/query_test.go | 284 ++++++++++++++++++++++++++++++++++++++ 9 files changed, 1076 insertions(+) create mode 100644 pkg/policy/doc.go create mode 100644 pkg/policy/encode.go create mode 100644 pkg/policy/encode_test.go create mode 100644 pkg/policy/grammar.ebnf create mode 100644 pkg/policy/grammar.go create mode 100644 pkg/policy/json.go create mode 100644 pkg/policy/json_test.go create mode 100644 pkg/policy/query.go create mode 100644 pkg/policy/query_test.go diff --git a/pkg/policy/doc.go b/pkg/policy/doc.go new file mode 100644 index 00000000..e2f94bb7 --- /dev/null +++ b/pkg/policy/doc.go @@ -0,0 +1,20 @@ +// Package policy provides facilities for creating policy from SQL-like language. +// eBNF grammar is provided in `grammar.ebnf` for illustration. +// +// Current limitations: +// 1. Grouping filter expressions in parenthesis is not supported right now. +// Requiring this will make query too verbose, making it optional makes +// our grammar not LL(1). This can be supported in future. +// 2. Filters must be defined before they are used. +// This requirement may be relaxed in future. +// +// Example query: +// REP 1 in SPB +// REP 2 in Americas +// CBF 4 +// SELECT 1 Node IN City FROM SPBSSD AS SPB +// SELECT 2 Node IN SAME City FROM Americas AS Americas +// FILTER SSD EQ true AS IsSSD +// FILTER @IsSSD AND Country eq "RU" AND City eq "St.Petersburg" AS SPBSSD +// FILTER 'Continent' == 'North America' OR Continent == 'South America' AS Americas +package policy diff --git a/pkg/policy/encode.go b/pkg/policy/encode.go new file mode 100644 index 00000000..a7d3ef08 --- /dev/null +++ b/pkg/policy/encode.go @@ -0,0 +1,140 @@ +package policy + +import ( + "fmt" + "strconv" + "strings" + + "github.com/nspcc-dev/neofs-api-go/pkg/netmap" +) + +func Encode(p *netmap.PlacementPolicy) []string { + if p == nil { + return nil + } + + var ( + replicas = p.Replicas() + selectors = p.Selectors() + filters = p.Filters() + ) + + // 1 for container backup factor + result := make([]string, 0, len(replicas)+len(selectors)+len(filters)+1) + + // first print replicas + encodeReplicas(replicas, &result) + + // then backup factor + if backupFactor := p.ContainerBackupFactor(); backupFactor != 0 { + result = append(result, fmt.Sprintf("CBF %d", backupFactor)) + } + + // then selectors + encodeSelectors(selectors, &result) + + // then filters + encodeFilters(filters, &result) + + return result +} + +func encodeReplicas(replicas []*netmap.Replica, dst *[]string) { + builder := new(strings.Builder) + + for _, replica := range replicas { + builder.WriteString("REP ") + builder.WriteString(strconv.FormatUint(uint64(replica.Count()), 10)) + + if s := replica.Selector(); s != "" { + builder.WriteString(" IN ") + builder.WriteString(s) + } + + *dst = append(*dst, builder.String()) + builder.Reset() + } +} + +func encodeSelectors(selectors []*netmap.Selector, dst *[]string) { + builder := new(strings.Builder) + + for _, selector := range selectors { + builder.WriteString("SELECT ") + builder.WriteString(strconv.FormatUint(uint64(selector.Count()), 10)) + + if a := selector.Attribute(); a != "" { + builder.WriteString(" IN") + + switch selector.Clause() { + case netmap.ClauseSame: + builder.WriteString(" SAME ") + case netmap.ClauseDistinct: + builder.WriteString(" DISTINCT ") + default: + builder.WriteString(" ") + } + + builder.WriteString(a) + } + + if f := selector.Filter(); f != "" { + builder.WriteString(" FROM ") + builder.WriteString(f) + } + + if n := selector.Name(); n != "" { + builder.WriteString(" AS ") + builder.WriteString(n) + } + + *dst = append(*dst, builder.String()) + builder.Reset() + } +} + +func encodeFilters(filters []*netmap.Filter, dst *[]string) { + builder := new(strings.Builder) + + for _, filter := range filters { + builder.WriteString("FILTER ") + + builder.WriteString(encodeFilter(filter)) + + *dst = append(*dst, builder.String()) + builder.Reset() + } +} + +func encodeFilter(filter *netmap.Filter) string { + builder := new(strings.Builder) + unspecified := filter.Operation() == 0 + + if k := filter.Key(); k != "" { + builder.WriteString(k) + builder.WriteString(" ") + builder.WriteString(filter.Operation().String()) + builder.WriteString(" ") + builder.WriteString(filter.Value()) + } else if n := filter.Name(); unspecified && n != "" { + builder.WriteString("@") + builder.WriteString(n) + } + + for i, subfilter := range filter.InnerFilters() { + if i != 0 { + builder.WriteString(" ") + builder.WriteString(filter.Operation().String()) + builder.WriteString(" ") + } + + builder.WriteString(encodeFilter(subfilter)) + } + + if n := filter.Name(); n != "" && !unspecified { + builder.WriteString(" AS ") + builder.WriteString(n) + } + + return builder.String() +} diff --git a/pkg/policy/encode_test.go b/pkg/policy/encode_test.go new file mode 100644 index 00000000..968c6903 --- /dev/null +++ b/pkg/policy/encode_test.go @@ -0,0 +1,36 @@ +package policy_test + +import ( + "fmt" + "strings" + "testing" + + "github.com/nspcc-dev/neofs-node/pkg/policy" + "github.com/stretchr/testify/require" +) + +func TestEncode(t *testing.T) { + testCases := []string{ + `REP 1 IN X +CBF 1 +SELECT 2 IN SAME Location FROM * AS X`, + + `REP 1 +SELECT 2 IN City FROM Good +FILTER Country EQ RU AS FromRU +FILTER @FromRU AND Rating GT 7 AS Good`, + + `REP 7 IN SPB +SELECT 1 IN City FROM SPBSSD AS SPB +FILTER City EQ SPB AND SSD EQ true OR City EQ SPB AND Rating GE 5 AS SPBSSD`, + } + + for _, testCase := range testCases { + q, err := policy.Parse(testCase) + require.NoError(t, err) + + got := policy.Encode(q) + fmt.Println(strings.Join(got, "\n")) + require.Equal(t, testCase, strings.Join(got, "\n")) + } +} diff --git a/pkg/policy/grammar.ebnf b/pkg/policy/grammar.ebnf new file mode 100644 index 00000000..2bcd3a20 --- /dev/null +++ b/pkg/policy/grammar.ebnf @@ -0,0 +1,55 @@ +Policy ::= + RepStmt, [RepStmt], + CbtStmt?, + [SelectStmt], + [FilterStmt], +; + +RepStmt ::= + 'REP', Number1, (* number of object replicas *) + ('AS', Ident)? (* optional selector name *) +; + +CbtStmt ::= 'CBF', Number1 (* container backup factor *) +; + +SelectStmt ::= + 'SELECT', Number1, (* number of nodes to select without container backup factor *) + ('IN', Clause?, Ident)?, (* bucket name *) + FROM, (Ident | '*'), (* filter reference or whole netmap *) + ('AS', Ident)? (* optional selector name *) +; + +Clause ::= + 'SAME' (* nodes from the same bucket *) + | 'DISTINCT' (* nodes from distinct buckets *) +; + +FilterStmt ::= + 'FILTER', AndChain, ['OR', AndChain], + 'AS', Ident (* obligatory filter name *) +; + +AndChain ::= + Expr, ['AND', Expr] +; + +Expr ::= + '@' Ident (* filter reference *) + | Key, Op, Value (* attribute filter *) +; + +Op ::= 'EQ' | 'NE' | 'GE' | 'GT' | 'LT' | 'LE' +; + +Key ::= Ident | String +; + +Value ::= Ident | Number | String +; + +Number1 ::= Digit1 [Digit]; +Number ::= Digit [Digit]; + +Digit1 ::= '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' ; +Digit ::= '0' | Digit1; diff --git a/pkg/policy/grammar.go b/pkg/policy/grammar.go new file mode 100644 index 00000000..3c695277 --- /dev/null +++ b/pkg/policy/grammar.go @@ -0,0 +1,59 @@ +package policy + +import ( + "github.com/alecthomas/participle" +) + +var parser *participle.Parser + +func init() { + p, err := participle.Build(&query{}) + if err != nil { + panic(err) + } + parser = p +} + +type query struct { + Replicas []*replicaStmt `@@+` + CBF uint32 `("CBF" @Int)?` + Selectors []*selectorStmt `@@*` + Filters []*filterStmt `@@*` +} + +type replicaStmt struct { + Count int `"REP" @Int` + Selector string `("IN" @Ident)?` +} + +type selectorStmt struct { + Count uint32 `"SELECT" @Int` + Bucket []string `("IN" @(("SAME" | "DISTINCT")? Ident))?` + Filter string `"FROM" @(Ident | "*")` + Name string `("AS" @Ident)?` +} + +type filterStmt struct { + Value *orChain `"FILTER" @@` + Name string `"AS" @Ident` +} + +type filterOrExpr struct { + Reference string `"@"@Ident` + Expr *simpleExpr `| @@` +} + +type orChain struct { + Clauses []*andChain `@@ ("OR" @@)*` +} + +type andChain struct { + Clauses []*filterOrExpr `@@ ("AND" @@)*` +} + +type simpleExpr struct { + Key string `@(Ident | String)` + // We don't use literals here to improve error messages. + Op string `@Ident` + Value string `@(Ident | String | Int)` +} diff --git a/pkg/policy/json.go b/pkg/policy/json.go new file mode 100644 index 00000000..a0b474fd --- /dev/null +++ b/pkg/policy/json.go @@ -0,0 +1,225 @@ +package policy + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/nspcc-dev/neofs-api-go/v2/netmap" +) + +type ( + filter struct { + Name string `json:"name,omitempty"` + Key string `json:"key,omitempty"` + Op string `json:"op,omitempty"` + Value string `json:"value,omitempty"` + Filters []filter `json:"filters,omitempty"` + } + replica struct { + Count uint32 `json:"count"` + Selector string `json:"selector,omitempty"` + } + selector struct { + Count uint32 `json:"count"` + Attribute string `json:"attribute"` + Filter string `json:"filter,omitempty"` + Name string `json:"name,omitempty"` + Clause string `json:"clause,omitempty"` + } + placement struct { + Replicas []replica `json:"replicas"` + CBF uint32 `json:"container_backup_factor,omitempty"` + Selectors []selector `json:"selectors,omitempty"` + Filters []filter `json:"filters,omitempty"` + } +) + +// ToJSON converts placement policy to JSON. +func ToJSON(np *netmap.PlacementPolicy) ([]byte, error) { + p := new(placement) + p.CBF = np.GetContainerBackupFactor() + p.Filters = make([]filter, len(np.GetFilters())) + for i, f := range np.GetFilters() { + p.Filters[i].fromNetmap(f) + } + p.Selectors = make([]selector, len(np.GetSelectors())) + for i, s := range np.GetSelectors() { + p.Selectors[i].fromNetmap(s) + } + p.Replicas = make([]replica, len(np.GetReplicas())) + for i, r := range np.GetReplicas() { + p.Replicas[i].fromNetmap(r) + } + return json.Marshal(p) +} + +// ToJSON creates placement policy from JSON. +func FromJSON(data []byte) (*netmap.PlacementPolicy, error) { + p := new(placement) + if err := json.Unmarshal(data, p); err != nil { + return nil, err + } + + rs := make([]*netmap.Replica, len(p.Replicas)) + for i := range p.Replicas { + rs[i] = p.Replicas[i].toNetmap() + } + + var fs []*netmap.Filter + if len(p.Filters) != 0 { + fs = make([]*netmap.Filter, len(p.Filters)) + for i := range p.Filters { + f, err := p.Filters[i].toNetmap() + if err != nil { + return nil, err + } + fs[i] = f + } + } + + var ss []*netmap.Selector + if len(p.Selectors) != 0 { + ss = make([]*netmap.Selector, len(p.Selectors)) + for i := range p.Selectors { + s, err := p.Selectors[i].toNetmap() + if err != nil { + return nil, err + } + ss[i] = s + } + } + + pp := new(netmap.PlacementPolicy) + pp.SetReplicas(rs) + pp.SetContainerBackupFactor(p.CBF) + pp.SetFilters(fs) + pp.SetSelectors(ss) + return pp, nil +} + +func (r *replica) toNetmap() *netmap.Replica { + nr := new(netmap.Replica) + nr.SetCount(r.Count) + nr.SetSelector(r.Selector) + return nr +} + +func (r *replica) fromNetmap(nr *netmap.Replica) { + r.Count = nr.GetCount() + r.Selector = nr.GetSelector() +} + +func (f *filter) toNetmap() (*netmap.Filter, error) { + var op netmap.Operation + switch strings.ToUpper(f.Op) { + case "EQ": + op = netmap.EQ + case "NE": + op = netmap.NE + case "GT": + op = netmap.GT + case "GE": + op = netmap.GE + case "LT": + op = netmap.LT + case "LE": + op = netmap.LE + case "AND": + op = netmap.AND + case "OR": + op = netmap.OR + case "": + op = netmap.UnspecifiedOperation + default: + return nil, fmt.Errorf("%w: '%s'", ErrUnknownOp, f.Op) + } + + var fs []*netmap.Filter + if len(f.Filters) != 0 { + fs = make([]*netmap.Filter, len(f.Filters)) + for i := range f.Filters { + var err error + fs[i], err = f.Filters[i].toNetmap() + if err != nil { + return nil, err + } + } + } + + nf := new(netmap.Filter) + nf.SetFilters(fs) + nf.SetOp(op) + nf.SetName(f.Name) + nf.SetValue(f.Value) + nf.SetKey(f.Key) + return nf, nil +} + +func (f *filter) fromNetmap(nf *netmap.Filter) { + f.Name = nf.GetName() + f.Key = nf.GetKey() + f.Value = nf.GetValue() + switch nf.GetOp() { + case netmap.EQ: + f.Op = "EQ" + case netmap.NE: + f.Op = "NE" + case netmap.GT: + f.Op = "GT" + case netmap.GE: + f.Op = "GE" + case netmap.LT: + f.Op = "LT" + case netmap.LE: + f.Op = "LE" + case netmap.AND: + f.Op = "AND" + case netmap.OR: + f.Op = "OR" + default: + // do nothing + } + if nf.GetFilters() != nil { + f.Filters = make([]filter, len(nf.GetFilters())) + for i, sf := range nf.GetFilters() { + f.Filters[i].fromNetmap(sf) + } + } +} + +func (s *selector) toNetmap() (*netmap.Selector, error) { + var c netmap.Clause + switch strings.ToUpper(s.Clause) { + case "SAME": + c = netmap.Same + case "DISTINCT": + c = netmap.Distinct + case "": + default: + return nil, fmt.Errorf("%w: '%s'", ErrUnknownClause, s.Clause) + } + ns := new(netmap.Selector) + ns.SetName(s.Name) + ns.SetAttribute(s.Attribute) + ns.SetCount(s.Count) + ns.SetClause(c) + ns.SetFilter(s.Filter) + return ns, nil +} + +func (s *selector) fromNetmap(ns *netmap.Selector) { + s.Name = ns.GetName() + s.Filter = ns.GetFilter() + s.Count = ns.GetCount() + s.Attribute = ns.GetAttribute() + switch ns.GetClause() { + case netmap.Same: + s.Clause = "same" + case netmap.Distinct: + s.Clause = "distinct" + default: + // do nothing + } + s.Name = ns.GetName() +} diff --git a/pkg/policy/json_test.go b/pkg/policy/json_test.go new file mode 100644 index 00000000..60b1a15d --- /dev/null +++ b/pkg/policy/json_test.go @@ -0,0 +1,84 @@ +package policy + +import ( + "testing" + + "github.com/nspcc-dev/neofs-api-go/v2/netmap" + "github.com/stretchr/testify/require" +) + +func TestToJSON(t *testing.T) { + check := func(t *testing.T, p *netmap.PlacementPolicy, json string) { + data, err := ToJSON(p) + require.NoError(t, err) + require.JSONEq(t, json, string(data)) + + np, err := FromJSON(data) + require.NoError(t, err) + require.Equal(t, p, np) + } + t.Run("SimpleREP", func(t *testing.T) { + p := new(netmap.PlacementPolicy) + p.SetReplicas([]*netmap.Replica{newReplica("", 3)}) + check(t, p, `{"replicas":[{"count":3}]}`) + }) + t.Run("REPWithCBF", func(t *testing.T) { + p := new(netmap.PlacementPolicy) + p.SetReplicas([]*netmap.Replica{newReplica("", 3)}) + p.SetContainerBackupFactor(3) + check(t, p, `{"replicas":[{"count":3}],"container_backup_factor":3}`) + }) + t.Run("REPFromSelector", func(t *testing.T) { + p := new(netmap.PlacementPolicy) + p.SetReplicas([]*netmap.Replica{newReplica("Nodes", 3)}) + p.SetContainerBackupFactor(3) + p.SetSelectors([]*netmap.Selector{ + newSelector(1, netmap.Distinct, "City", "", "Nodes"), + }) + check(t, p, `{ + "replicas":[{"count":3,"selector":"Nodes"}], + "container_backup_factor":3, + "selectors": [{ + "name":"Nodes", + "attribute":"City", + "clause":"distinct", + "count":1 + }]}`) + }) + t.Run("FilterOps", func(t *testing.T) { + p := new(netmap.PlacementPolicy) + p.SetReplicas([]*netmap.Replica{newReplica("Nodes", 3)}) + p.SetContainerBackupFactor(3) + p.SetSelectors([]*netmap.Selector{ + newSelector(1, netmap.Same, "City", "Good", "Nodes"), + }) + p.SetFilters([]*netmap.Filter{ + newFilter("GoodRating", "Rating", "5", netmap.GE), + newFilter("Good", "", "", netmap.OR, + newFilter("GoodRating", "", "", netmap.UnspecifiedOperation), + newFilter("", "Attr1", "Val1", netmap.EQ), + newFilter("", "Attr2", "Val2", netmap.NE), + newFilter("", "", "", netmap.AND, + newFilter("", "Attr4", "2", netmap.LT), + newFilter("", "Attr5", "3", netmap.LE)), + newFilter("", "Attr3", "1", netmap.GT)), + }) + check(t, p, `{ + "replicas":[{"count":3,"selector":"Nodes"}], + "container_backup_factor":3, + "selectors": [{"name":"Nodes","attribute":"City","clause":"same","count":1,"filter":"Good"}], + "filters": [ + {"name":"GoodRating","key":"Rating","op":"GE","value":"5"}, + {"name":"Good","op":"OR","filters":[ + {"name":"GoodRating"}, + {"key":"Attr1","op":"EQ","value":"Val1"}, + {"key":"Attr2","op":"NE","value":"Val2"}, + {"op":"AND","filters":[ + {"key":"Attr4","op":"LT","value":"2"}, + {"key":"Attr5","op":"LE","value":"3"} + ]}, + {"key":"Attr3","op":"GT","value":"1"} + ]} + ]}`) + }) +} diff --git a/pkg/policy/query.go b/pkg/policy/query.go new file mode 100644 index 00000000..49042790 --- /dev/null +++ b/pkg/policy/query.go @@ -0,0 +1,173 @@ +package policy + +import ( + "errors" + "fmt" + "strings" + + "github.com/nspcc-dev/neofs-api-go/pkg/netmap" +) + +var ( + ErrInvalidNumber = errors.New("policy: expected positive integer") + ErrUnknownClause = errors.New("policy: unknown clause") + ErrUnknownOp = errors.New("policy: unknown operation") + ErrUnknownFilter = errors.New("policy: filter not found") + ErrUnknownSelector = errors.New("policy: selector not found") +) + +func parse(s string) (*query, error) { + q := new(query) + err := parser.Parse(strings.NewReader(s), q) + if err != nil { + return nil, err + } + return q, nil +} + +// Parse parses s into a placement policy. +func Parse(s string) (*netmap.PlacementPolicy, error) { + q, err := parse(s) + if err != nil { + return nil, err + } + + seenFilters := map[string]bool{} + fs := make([]*netmap.Filter, 0, len(q.Filters)) + for _, qf := range q.Filters { + f, err := filterFromOrChain(qf.Value, seenFilters) + if err != nil { + return nil, err + } + f.SetName(qf.Name) + fs = append(fs, f) + seenFilters[qf.Name] = true + } + + seenSelectors := map[string]bool{} + ss := make([]*netmap.Selector, 0, len(q.Selectors)) + for _, qs := range q.Selectors { + if qs.Filter != netmap.MainFilterName && !seenFilters[qs.Filter] { + return nil, fmt.Errorf("%w: '%s'", ErrUnknownFilter, qs.Filter) + } + s := netmap.NewSelector() + switch len(qs.Bucket) { + case 1: // only bucket + s.SetAttribute(qs.Bucket[0]) + case 2: // clause + bucket + s.SetClause(clauseFromString(qs.Bucket[0])) + s.SetAttribute(qs.Bucket[1]) + } + s.SetName(qs.Name) + seenSelectors[qs.Name] = true + s.SetFilter(qs.Filter) + if qs.Count == 0 { + return nil, fmt.Errorf("%w: SELECT", ErrInvalidNumber) + } + s.SetCount(qs.Count) + ss = append(ss, s) + } + + rs := make([]*netmap.Replica, 0, len(q.Replicas)) + for _, qr := range q.Replicas { + r := netmap.NewReplica() + if qr.Selector != "" { + if !seenSelectors[qr.Selector] { + return nil, fmt.Errorf("%w: '%s'", ErrUnknownSelector, qr.Selector) + } + r.SetSelector(qr.Selector) + } + if qr.Count == 0 { + return nil, fmt.Errorf("%w: REP", ErrInvalidNumber) + } + r.SetCount(uint32(qr.Count)) + rs = append(rs, r) + } + + p := new(netmap.PlacementPolicy) + p.SetFilters(fs...) + p.SetSelectors(ss...) + p.SetReplicas(rs...) + p.SetContainerBackupFactor(q.CBF) + + return p, nil +} + +func clauseFromString(s string) netmap.Clause { + switch strings.ToUpper(s) { + case "SAME": + return netmap.ClauseSame + case "DISTINCT": + return netmap.ClauseDistinct + default: + return 0 + } +} + +func filterFromOrChain(expr *orChain, seen map[string]bool) (*netmap.Filter, error) { + var fs []*netmap.Filter + for _, ac := range expr.Clauses { + f, err := filterFromAndChain(ac, seen) + if err != nil { + return nil, err + } + fs = append(fs, f) + } + if len(fs) == 1 { + return fs[0], nil + } + + f := netmap.NewFilter() + f.SetOperation(netmap.OpOR) + f.SetInnerFilters(fs...) + return f, nil +} + +func filterFromAndChain(expr *andChain, seen map[string]bool) (*netmap.Filter, error) { + var fs []*netmap.Filter + for _, fe := range expr.Clauses { + var f *netmap.Filter + var err error + if fe.Expr != nil { + f, err = filterFromSimpleExpr(fe.Expr, seen) + } else { + f = netmap.NewFilter() + f.SetName(fe.Reference) + } + if err != nil { + return nil, err + } + fs = append(fs, f) + } + if len(fs) == 1 { + return fs[0], nil + } + + f := netmap.NewFilter() + f.SetOperation(netmap.OpAND) + f.SetInnerFilters(fs...) + return f, nil +} + +func filterFromSimpleExpr(se *simpleExpr, seen map[string]bool) (*netmap.Filter, error) { + f := netmap.NewFilter() + f.SetKey(se.Key) + switch se.Op { + case "EQ": + f.SetOperation(netmap.OpEQ) + case "NE": + f.SetOperation(netmap.OpNE) + case "GE": + f.SetOperation(netmap.OpGE) + case "GT": + f.SetOperation(netmap.OpGT) + case "LE": + f.SetOperation(netmap.OpLE) + case "LT": + f.SetOperation(netmap.OpLT) + default: + return nil, fmt.Errorf("%w: '%s'", ErrUnknownOp, se.Op) + } + f.SetValue(se.Value) + return f, nil +} diff --git a/pkg/policy/query_test.go b/pkg/policy/query_test.go new file mode 100644 index 00000000..b7c4cc76 --- /dev/null +++ b/pkg/policy/query_test.go @@ -0,0 +1,284 @@ +package policy + +import ( + "errors" + "testing" + + "github.com/nspcc-dev/neofs-api-go/v2/netmap" + "github.com/stretchr/testify/require" +) + +func TestSimple(t *testing.T) { + q := `REP 3` + expected := new(netmap.PlacementPolicy) + expected.SetFilters([]*netmap.Filter{}) + expected.SetSelectors([]*netmap.Selector{}) + expected.SetReplicas([]*netmap.Replica{newReplica("", 3)}) + + r, err := Parse(q) + require.NoError(t, err) + require.EqualValues(t, expected, r) +} + +func TestSimpleWithHRWB(t *testing.T) { + q := `REP 3 CBF 4` + expected := new(netmap.PlacementPolicy) + expected.SetFilters([]*netmap.Filter{}) + expected.SetSelectors([]*netmap.Selector{}) + expected.SetReplicas([]*netmap.Replica{newReplica("", 3)}) + expected.SetContainerBackupFactor(4) + + r, err := Parse(q) + require.NoError(t, err) + require.EqualValues(t, expected, r) +} + +func TestFromSelect(t *testing.T) { + q := `REP 1 IN SPB +SELECT 1 IN City FROM * AS SPB` + expected := new(netmap.PlacementPolicy) + expected.SetFilters([]*netmap.Filter{}) + expected.SetSelectors([]*netmap.Selector{ + newSelector(1, netmap.UnspecifiedClause, "City", "*", "SPB"), + }) + expected.SetReplicas([]*netmap.Replica{newReplica("SPB", 1)}) + + r, err := Parse(q) + require.NoError(t, err) + require.EqualValues(t, expected, r) +} + +// https://github.com/nspcc-dev/neofs-node/issues/46 +func TestFromSelectNoAttribute(t *testing.T) { + t.Run("Simple", func(t *testing.T) { + q := `REP 2 + SELECT 6 FROM *` + + expected := new(netmap.PlacementPolicy) + expected.SetFilters([]*netmap.Filter{}) + expected.SetSelectors([]*netmap.Selector{newSelector(6, netmap.UnspecifiedClause, "", "*", "")}) + expected.SetReplicas([]*netmap.Replica{newReplica("", 2)}) + + r, err := Parse(q) + require.NoError(t, err) + require.EqualValues(t, expected, r) + + }) + + t.Run("with filter", func(t *testing.T) { + q := `REP 2 + SELECT 6 FROM F + FILTER StorageType EQ SSD AS F` + + expected := new(netmap.PlacementPolicy) + expected.SetFilters([]*netmap.Filter{newFilter("F", "StorageType", "SSD", netmap.EQ)}) + expected.SetSelectors([]*netmap.Selector{newSelector(6, netmap.UnspecifiedClause, "", "F", "")}) + expected.SetReplicas([]*netmap.Replica{newReplica("", 2)}) + + r, err := Parse(q) + require.NoError(t, err) + require.EqualValues(t, expected, r) + }) +} + +func TestFromSelectClause(t *testing.T) { + q := `REP 4 +SELECT 3 IN Country FROM * +SELECT 2 IN SAME City FROM * +SELECT 1 IN DISTINCT Continent FROM *` + expected := new(netmap.PlacementPolicy) + expected.SetFilters([]*netmap.Filter{}) + expected.SetSelectors([]*netmap.Selector{ + newSelector(3, netmap.UnspecifiedClause, "Country", "*", ""), + newSelector(2, netmap.Same, "City", "*", ""), + newSelector(1, netmap.Distinct, "Continent", "*", ""), + }) + expected.SetReplicas([]*netmap.Replica{newReplica("", 4)}) + + r, err := Parse(q) + require.NoError(t, err) + require.EqualValues(t, expected, r) +} + +func TestSimpleFilter(t *testing.T) { + q := `REP 1 +SELECT 1 IN City FROM Good +FILTER Rating GT 7 AS Good` + expected := new(netmap.PlacementPolicy) + expected.SetReplicas([]*netmap.Replica{newReplica("", 1)}) + expected.SetSelectors([]*netmap.Selector{ + newSelector(1, netmap.UnspecifiedClause, "City", "Good", ""), + }) + expected.SetFilters([]*netmap.Filter{ + newFilter("Good", "Rating", "7", netmap.GT), + }) + + r, err := Parse(q) + require.NoError(t, err) + require.EqualValues(t, expected, r) +} + +func TestFilterReference(t *testing.T) { + q := `REP 1 +SELECT 2 IN City FROM Good +FILTER Country EQ "RU" AS FromRU +FILTER @FromRU AND Rating GT 7 AS Good` + expected := new(netmap.PlacementPolicy) + expected.SetReplicas([]*netmap.Replica{newReplica("", 1)}) + expected.SetSelectors([]*netmap.Selector{ + newSelector(2, netmap.UnspecifiedClause, "City", "Good", ""), + }) + expected.SetFilters([]*netmap.Filter{ + newFilter("FromRU", "Country", "RU", netmap.EQ), + newFilter("Good", "", "", netmap.AND, + newFilter("FromRU", "", "", netmap.UnspecifiedOperation), + newFilter("", "Rating", "7", netmap.GT)), + }) + + r, err := Parse(q) + require.NoError(t, err) + require.EqualValues(t, expected, r) +} + +func TestFilterOps(t *testing.T) { + q := `REP 1 +SELECT 2 IN City FROM Good +FILTER A GT 1 AND B GE 2 AND C LT 3 AND D LE 4 + AND E EQ 5 AND F NE 6 AS Good` + expected := new(netmap.PlacementPolicy) + expected.SetReplicas([]*netmap.Replica{newReplica("", 1)}) + expected.SetSelectors([]*netmap.Selector{ + newSelector(2, netmap.UnspecifiedClause, "City", "Good", ""), + }) + expected.SetFilters([]*netmap.Filter{ + newFilter("Good", "", "", netmap.AND, + newFilter("", "A", "1", netmap.GT), + newFilter("", "B", "2", netmap.GE), + newFilter("", "C", "3", netmap.LT), + newFilter("", "D", "4", netmap.LE), + newFilter("", "E", "5", netmap.EQ), + newFilter("", "F", "6", netmap.NE)), + }) + + r, err := Parse(q) + require.NoError(t, err) + require.EqualValues(t, expected, r) +} + +func TestWithFilterPrecedence(t *testing.T) { + q := `REP 7 IN SPB +SELECT 1 IN City FROM SPBSSD AS SPB +FILTER City EQ "SPB" AND SSD EQ true OR City EQ "SPB" AND Rating GE 5 AS SPBSSD` + expected := new(netmap.PlacementPolicy) + expected.SetReplicas([]*netmap.Replica{newReplica("SPB", 7)}) + expected.SetSelectors([]*netmap.Selector{ + newSelector(1, netmap.UnspecifiedClause, "City", "SPBSSD", "SPB"), + }) + expected.SetFilters([]*netmap.Filter{ + newFilter("SPBSSD", "", "", netmap.OR, + newFilter("", "", "", netmap.AND, + newFilter("", "City", "SPB", netmap.EQ), + newFilter("", "SSD", "true", netmap.EQ)), + newFilter("", "", "", netmap.AND, + newFilter("", "City", "SPB", netmap.EQ), + newFilter("", "Rating", "5", netmap.GE))), + }) + + r, err := Parse(q) + require.NoError(t, err) + require.EqualValues(t, expected, r) +} + +func TestValidation(t *testing.T) { + t.Run("MissingSelector", func(t *testing.T) { + q := `REP 3 IN RU` + _, err := Parse(q) + require.True(t, errors.Is(err, ErrUnknownSelector), "got: %v", err) + }) + t.Run("MissingFilter", func(t *testing.T) { + q := `REP 3 + SELECT 1 IN City FROM MissingFilter` + _, err := Parse(q) + require.True(t, errors.Is(err, ErrUnknownFilter), "got: %v", err) + }) + t.Run("UnknownOp", func(t *testing.T) { + q := `REP 3 + SELECT 1 IN City FROM F + FILTER Country KEK RU AS F` + _, err := Parse(q) + require.True(t, errors.Is(err, ErrUnknownOp), "got: %v", err) + }) + t.Run("TypoInREP", func(t *testing.T) { + q := `REK 3` + _, err := Parse(q) + require.Error(t, err) + }) + t.Run("InvalidFilterName", func(t *testing.T) { + q := `REP 3 + SELECT 1 IN City FROM F + FILTER Good AND Country EQ RU AS F + FILTER Rating EQ 5 AS Good` + _, err := Parse(q) + require.Error(t, err) + }) + t.Run("InvalidNumberInREP", func(t *testing.T) { + q := `REP 0` + _, err := Parse(q) + require.True(t, errors.Is(err, ErrInvalidNumber), "got: %v", err) + }) + t.Run("InvalidNumberInREP", func(t *testing.T) { + q := `REP 1 IN Good + SELECT 0 IN City FROM *` + _, err := Parse(q) + require.True(t, errors.Is(err, ErrInvalidNumber), "got: %v", err) + }) + +} + +func TestFilterStringSymbols(t *testing.T) { + q := `REP 1 IN S +SELECT 1 FROM F AS S +FILTER "UN-LOCODE" EQ "RU LED" AS F` + + expected := new(netmap.PlacementPolicy) + expected.SetReplicas([]*netmap.Replica{ + newReplica("S", 1), + }) + expected.SetSelectors([]*netmap.Selector{ + newSelector(1, netmap.UnspecifiedClause, "", "F", "S"), + }) + expected.SetFilters([]*netmap.Filter{ + newFilter("F", "UN-LOCODE", "RU LED", netmap.EQ), + }) + + r, err := Parse(q) + require.NoError(t, err) + require.EqualValues(t, expected, r) +} + +func newFilter(name, key, value string, op netmap.Operation, sub ...*netmap.Filter) *netmap.Filter { + f := new(netmap.Filter) + f.SetName(name) + f.SetKey(key) + f.SetValue(value) + f.SetOp(op) + f.SetFilters(sub) + return f +} + +func newReplica(s string, c uint32) *netmap.Replica { + r := new(netmap.Replica) + r.SetSelector(s) + r.SetCount(c) + return r +} + +func newSelector(count uint32, c netmap.Clause, attr, f, name string) *netmap.Selector { + s := new(netmap.Selector) + s.SetCount(count) + s.SetClause(c) + s.SetAttribute(attr) + s.SetFilter(f) + s.SetName(name) + return s +}