From 7eab7529234bd9e00fb1b3a0fcc9dcc1830eff32 Mon Sep 17 00:00:00 2001 From: Angira Kekteeva Date: Fri, 21 May 2021 16:35:24 +0300 Subject: [PATCH] [#116] *: Replace pkg policy by neofs-sdk policy Processing transfer of policy pkg to neofs-sdk repository. Replace local dependency in neofs-cli. Signed-off-by: Angira Kekteeva --- cmd/neofs-cli/modules/container.go | 2 +- go.mod | 2 +- go.sum | 6 +- pkg/policy/doc.go | 20 -- pkg/policy/encode.go | 140 -------------- pkg/policy/encode_test.go | 36 ---- pkg/policy/grammar.ebnf | 55 ------ pkg/policy/grammar.go | 59 ------ pkg/policy/json.go | 225 ----------------------- pkg/policy/json_test.go | 84 --------- pkg/policy/query.go | 173 ------------------ pkg/policy/query_test.go | 284 ----------------------------- 12 files changed, 6 insertions(+), 1080 deletions(-) delete mode 100644 pkg/policy/doc.go delete mode 100644 pkg/policy/encode.go delete mode 100644 pkg/policy/encode_test.go delete mode 100644 pkg/policy/grammar.ebnf delete mode 100644 pkg/policy/grammar.go delete mode 100644 pkg/policy/json.go delete mode 100644 pkg/policy/json_test.go delete mode 100644 pkg/policy/query.go delete mode 100644 pkg/policy/query_test.go diff --git a/cmd/neofs-cli/modules/container.go b/cmd/neofs-cli/modules/container.go index 8b2f50d7f..6a7b9c89b 100644 --- a/cmd/neofs-cli/modules/container.go +++ b/cmd/neofs-cli/modules/container.go @@ -22,7 +22,7 @@ import ( "github.com/nspcc-dev/neofs-api-go/pkg/netmap" "github.com/nspcc-dev/neofs-api-go/pkg/object" "github.com/nspcc-dev/neofs-api-go/pkg/owner" - "github.com/nspcc-dev/neofs-node/pkg/policy" + "github.com/nspcc-dev/neofs-sdk-go/pkg/policy" "github.com/spf13/cobra" ) diff --git a/go.mod b/go.mod index 8f43e5a17..3c595127d 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,6 @@ go 1.14 require ( code.cloudfoundry.org/bytefmt v0.0.0-20200131002437-cf55d5288a48 - github.com/alecthomas/participle v0.6.0 github.com/golang/protobuf v1.4.3 github.com/google/uuid v1.1.1 github.com/hashicorp/golang-lru v0.5.4 @@ -16,6 +15,7 @@ require ( github.com/nspcc-dev/neo-go v0.95.0 github.com/nspcc-dev/neofs-api-go v1.26.1 github.com/nspcc-dev/neofs-crypto v0.3.0 + github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210520210714-9dee13f0d556 github.com/nspcc-dev/tzhash v1.4.0 github.com/panjf2000/ants/v2 v2.3.0 github.com/paulmach/orb v0.2.1 diff --git a/go.sum b/go.sum index ad9a71ae5..2e1e83bd1 100644 --- a/go.sum +++ b/go.sum @@ -29,8 +29,8 @@ github.com/Workiva/go-datastructures v1.0.50/go.mod h1:Z+F2Rca0qCsVYDS8z7bAGm8f3 github.com/abiosoft/ishell v2.0.0+incompatible/go.mod h1:HQR9AqF2R3P4XXpMpI0NAzgHf/aS6+zVXRj14cVk9qg= github.com/abiosoft/readline v0.0.0-20180607040430-155bce2042db/go.mod h1:rB3B4rKii8V21ydCbIzH5hZiCQE7f5E9SzUb/ZZx530= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= -github.com/alecthomas/participle v0.6.0 h1:Pvo8XUCQKgIywVjz/+Ci3IsjGg+g/TdKkMcfgghKCEw= -github.com/alecthomas/participle v0.6.0/go.mod h1:HfdmEuwvr12HXQN44HPWXR0lHmVolVYe4dyL6lQ3duY= +github.com/alecthomas/participle v0.7.1 h1:2bN7reTw//5f0cugJcTOnY/NYZcWQOaajW+BwZB5xWs= +github.com/alecthomas/participle v0.7.1/go.mod h1:HfdmEuwvr12HXQN44HPWXR0lHmVolVYe4dyL6lQ3duY= github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -290,6 +290,8 @@ github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9K github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM= github.com/nspcc-dev/neofs-crypto v0.3.0/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= +github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210520210714-9dee13f0d556 h1:mxbTyGzDrcFonKSboCHSHr0lvJaxYdnLf1VrymFhVHE= +github.com/nspcc-dev/neofs-sdk-go v0.0.0-20210520210714-9dee13f0d556/go.mod h1:s/+72j1JGevJje75KhOrUmzmy8Y3X2QaLRwkrTmhoMw= github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= diff --git a/pkg/policy/doc.go b/pkg/policy/doc.go deleted file mode 100644 index e2f94bb77..000000000 --- a/pkg/policy/doc.go +++ /dev/null @@ -1,20 +0,0 @@ -// Package policy provides facilities for creating policy from SQL-like language. -// eBNF grammar is provided in `grammar.ebnf` for illustration. -// -// Current limitations: -// 1. Grouping filter expressions in parenthesis is not supported right now. -// Requiring this will make query too verbose, making it optional makes -// our grammar not LL(1). This can be supported in future. -// 2. Filters must be defined before they are used. -// This requirement may be relaxed in future. -// -// Example query: -// REP 1 in SPB -// REP 2 in Americas -// CBF 4 -// SELECT 1 Node IN City FROM SPBSSD AS SPB -// SELECT 2 Node IN SAME City FROM Americas AS Americas -// FILTER SSD EQ true AS IsSSD -// FILTER @IsSSD AND Country eq "RU" AND City eq "St.Petersburg" AS SPBSSD -// FILTER 'Continent' == 'North America' OR Continent == 'South America' AS Americas -package policy diff --git a/pkg/policy/encode.go b/pkg/policy/encode.go deleted file mode 100644 index a7d3ef08f..000000000 --- a/pkg/policy/encode.go +++ /dev/null @@ -1,140 +0,0 @@ -package policy - -import ( - "fmt" - "strconv" - "strings" - - "github.com/nspcc-dev/neofs-api-go/pkg/netmap" -) - -func Encode(p *netmap.PlacementPolicy) []string { - if p == nil { - return nil - } - - var ( - replicas = p.Replicas() - selectors = p.Selectors() - filters = p.Filters() - ) - - // 1 for container backup factor - result := make([]string, 0, len(replicas)+len(selectors)+len(filters)+1) - - // first print replicas - encodeReplicas(replicas, &result) - - // then backup factor - if backupFactor := p.ContainerBackupFactor(); backupFactor != 0 { - result = append(result, fmt.Sprintf("CBF %d", backupFactor)) - } - - // then selectors - encodeSelectors(selectors, &result) - - // then filters - encodeFilters(filters, &result) - - return result -} - -func encodeReplicas(replicas []*netmap.Replica, dst *[]string) { - builder := new(strings.Builder) - - for _, replica := range replicas { - builder.WriteString("REP ") - builder.WriteString(strconv.FormatUint(uint64(replica.Count()), 10)) - - if s := replica.Selector(); s != "" { - builder.WriteString(" IN ") - builder.WriteString(s) - } - - *dst = append(*dst, builder.String()) - builder.Reset() - } -} - -func encodeSelectors(selectors []*netmap.Selector, dst *[]string) { - builder := new(strings.Builder) - - for _, selector := range selectors { - builder.WriteString("SELECT ") - builder.WriteString(strconv.FormatUint(uint64(selector.Count()), 10)) - - if a := selector.Attribute(); a != "" { - builder.WriteString(" IN") - - switch selector.Clause() { - case netmap.ClauseSame: - builder.WriteString(" SAME ") - case netmap.ClauseDistinct: - builder.WriteString(" DISTINCT ") - default: - builder.WriteString(" ") - } - - builder.WriteString(a) - } - - if f := selector.Filter(); f != "" { - builder.WriteString(" FROM ") - builder.WriteString(f) - } - - if n := selector.Name(); n != "" { - builder.WriteString(" AS ") - builder.WriteString(n) - } - - *dst = append(*dst, builder.String()) - builder.Reset() - } -} - -func encodeFilters(filters []*netmap.Filter, dst *[]string) { - builder := new(strings.Builder) - - for _, filter := range filters { - builder.WriteString("FILTER ") - - builder.WriteString(encodeFilter(filter)) - - *dst = append(*dst, builder.String()) - builder.Reset() - } -} - -func encodeFilter(filter *netmap.Filter) string { - builder := new(strings.Builder) - unspecified := filter.Operation() == 0 - - if k := filter.Key(); k != "" { - builder.WriteString(k) - builder.WriteString(" ") - builder.WriteString(filter.Operation().String()) - builder.WriteString(" ") - builder.WriteString(filter.Value()) - } else if n := filter.Name(); unspecified && n != "" { - builder.WriteString("@") - builder.WriteString(n) - } - - for i, subfilter := range filter.InnerFilters() { - if i != 0 { - builder.WriteString(" ") - builder.WriteString(filter.Operation().String()) - builder.WriteString(" ") - } - - builder.WriteString(encodeFilter(subfilter)) - } - - if n := filter.Name(); n != "" && !unspecified { - builder.WriteString(" AS ") - builder.WriteString(n) - } - - return builder.String() -} diff --git a/pkg/policy/encode_test.go b/pkg/policy/encode_test.go deleted file mode 100644 index 968c69039..000000000 --- a/pkg/policy/encode_test.go +++ /dev/null @@ -1,36 +0,0 @@ -package policy_test - -import ( - "fmt" - "strings" - "testing" - - "github.com/nspcc-dev/neofs-node/pkg/policy" - "github.com/stretchr/testify/require" -) - -func TestEncode(t *testing.T) { - testCases := []string{ - `REP 1 IN X -CBF 1 -SELECT 2 IN SAME Location FROM * AS X`, - - `REP 1 -SELECT 2 IN City FROM Good -FILTER Country EQ RU AS FromRU -FILTER @FromRU AND Rating GT 7 AS Good`, - - `REP 7 IN SPB -SELECT 1 IN City FROM SPBSSD AS SPB -FILTER City EQ SPB AND SSD EQ true OR City EQ SPB AND Rating GE 5 AS SPBSSD`, - } - - for _, testCase := range testCases { - q, err := policy.Parse(testCase) - require.NoError(t, err) - - got := policy.Encode(q) - fmt.Println(strings.Join(got, "\n")) - require.Equal(t, testCase, strings.Join(got, "\n")) - } -} diff --git a/pkg/policy/grammar.ebnf b/pkg/policy/grammar.ebnf deleted file mode 100644 index 2bcd3a20c..000000000 --- a/pkg/policy/grammar.ebnf +++ /dev/null @@ -1,55 +0,0 @@ -Policy ::= - RepStmt, [RepStmt], - CbtStmt?, - [SelectStmt], - [FilterStmt], -; - -RepStmt ::= - 'REP', Number1, (* number of object replicas *) - ('AS', Ident)? (* optional selector name *) -; - -CbtStmt ::= 'CBF', Number1 (* container backup factor *) -; - -SelectStmt ::= - 'SELECT', Number1, (* number of nodes to select without container backup factor *) - ('IN', Clause?, Ident)?, (* bucket name *) - FROM, (Ident | '*'), (* filter reference or whole netmap *) - ('AS', Ident)? (* optional selector name *) -; - -Clause ::= - 'SAME' (* nodes from the same bucket *) - | 'DISTINCT' (* nodes from distinct buckets *) -; - -FilterStmt ::= - 'FILTER', AndChain, ['OR', AndChain], - 'AS', Ident (* obligatory filter name *) -; - -AndChain ::= - Expr, ['AND', Expr] -; - -Expr ::= - '@' Ident (* filter reference *) - | Key, Op, Value (* attribute filter *) -; - -Op ::= 'EQ' | 'NE' | 'GE' | 'GT' | 'LT' | 'LE' -; - -Key ::= Ident | String -; - -Value ::= Ident | Number | String -; - -Number1 ::= Digit1 [Digit]; -Number ::= Digit [Digit]; - -Digit1 ::= '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' ; -Digit ::= '0' | Digit1; diff --git a/pkg/policy/grammar.go b/pkg/policy/grammar.go deleted file mode 100644 index 3c695277b..000000000 --- a/pkg/policy/grammar.go +++ /dev/null @@ -1,59 +0,0 @@ -package policy - -import ( - "github.com/alecthomas/participle" -) - -var parser *participle.Parser - -func init() { - p, err := participle.Build(&query{}) - if err != nil { - panic(err) - } - parser = p -} - -type query struct { - Replicas []*replicaStmt `@@+` - CBF uint32 `("CBF" @Int)?` - Selectors []*selectorStmt `@@*` - Filters []*filterStmt `@@*` -} - -type replicaStmt struct { - Count int `"REP" @Int` - Selector string `("IN" @Ident)?` -} - -type selectorStmt struct { - Count uint32 `"SELECT" @Int` - Bucket []string `("IN" @(("SAME" | "DISTINCT")? Ident))?` - Filter string `"FROM" @(Ident | "*")` - Name string `("AS" @Ident)?` -} - -type filterStmt struct { - Value *orChain `"FILTER" @@` - Name string `"AS" @Ident` -} - -type filterOrExpr struct { - Reference string `"@"@Ident` - Expr *simpleExpr `| @@` -} - -type orChain struct { - Clauses []*andChain `@@ ("OR" @@)*` -} - -type andChain struct { - Clauses []*filterOrExpr `@@ ("AND" @@)*` -} - -type simpleExpr struct { - Key string `@(Ident | String)` - // We don't use literals here to improve error messages. - Op string `@Ident` - Value string `@(Ident | String | Int)` -} diff --git a/pkg/policy/json.go b/pkg/policy/json.go deleted file mode 100644 index a0b474fdb..000000000 --- a/pkg/policy/json.go +++ /dev/null @@ -1,225 +0,0 @@ -package policy - -import ( - "encoding/json" - "fmt" - "strings" - - "github.com/nspcc-dev/neofs-api-go/v2/netmap" -) - -type ( - filter struct { - Name string `json:"name,omitempty"` - Key string `json:"key,omitempty"` - Op string `json:"op,omitempty"` - Value string `json:"value,omitempty"` - Filters []filter `json:"filters,omitempty"` - } - replica struct { - Count uint32 `json:"count"` - Selector string `json:"selector,omitempty"` - } - selector struct { - Count uint32 `json:"count"` - Attribute string `json:"attribute"` - Filter string `json:"filter,omitempty"` - Name string `json:"name,omitempty"` - Clause string `json:"clause,omitempty"` - } - placement struct { - Replicas []replica `json:"replicas"` - CBF uint32 `json:"container_backup_factor,omitempty"` - Selectors []selector `json:"selectors,omitempty"` - Filters []filter `json:"filters,omitempty"` - } -) - -// ToJSON converts placement policy to JSON. -func ToJSON(np *netmap.PlacementPolicy) ([]byte, error) { - p := new(placement) - p.CBF = np.GetContainerBackupFactor() - p.Filters = make([]filter, len(np.GetFilters())) - for i, f := range np.GetFilters() { - p.Filters[i].fromNetmap(f) - } - p.Selectors = make([]selector, len(np.GetSelectors())) - for i, s := range np.GetSelectors() { - p.Selectors[i].fromNetmap(s) - } - p.Replicas = make([]replica, len(np.GetReplicas())) - for i, r := range np.GetReplicas() { - p.Replicas[i].fromNetmap(r) - } - return json.Marshal(p) -} - -// ToJSON creates placement policy from JSON. -func FromJSON(data []byte) (*netmap.PlacementPolicy, error) { - p := new(placement) - if err := json.Unmarshal(data, p); err != nil { - return nil, err - } - - rs := make([]*netmap.Replica, len(p.Replicas)) - for i := range p.Replicas { - rs[i] = p.Replicas[i].toNetmap() - } - - var fs []*netmap.Filter - if len(p.Filters) != 0 { - fs = make([]*netmap.Filter, len(p.Filters)) - for i := range p.Filters { - f, err := p.Filters[i].toNetmap() - if err != nil { - return nil, err - } - fs[i] = f - } - } - - var ss []*netmap.Selector - if len(p.Selectors) != 0 { - ss = make([]*netmap.Selector, len(p.Selectors)) - for i := range p.Selectors { - s, err := p.Selectors[i].toNetmap() - if err != nil { - return nil, err - } - ss[i] = s - } - } - - pp := new(netmap.PlacementPolicy) - pp.SetReplicas(rs) - pp.SetContainerBackupFactor(p.CBF) - pp.SetFilters(fs) - pp.SetSelectors(ss) - return pp, nil -} - -func (r *replica) toNetmap() *netmap.Replica { - nr := new(netmap.Replica) - nr.SetCount(r.Count) - nr.SetSelector(r.Selector) - return nr -} - -func (r *replica) fromNetmap(nr *netmap.Replica) { - r.Count = nr.GetCount() - r.Selector = nr.GetSelector() -} - -func (f *filter) toNetmap() (*netmap.Filter, error) { - var op netmap.Operation - switch strings.ToUpper(f.Op) { - case "EQ": - op = netmap.EQ - case "NE": - op = netmap.NE - case "GT": - op = netmap.GT - case "GE": - op = netmap.GE - case "LT": - op = netmap.LT - case "LE": - op = netmap.LE - case "AND": - op = netmap.AND - case "OR": - op = netmap.OR - case "": - op = netmap.UnspecifiedOperation - default: - return nil, fmt.Errorf("%w: '%s'", ErrUnknownOp, f.Op) - } - - var fs []*netmap.Filter - if len(f.Filters) != 0 { - fs = make([]*netmap.Filter, len(f.Filters)) - for i := range f.Filters { - var err error - fs[i], err = f.Filters[i].toNetmap() - if err != nil { - return nil, err - } - } - } - - nf := new(netmap.Filter) - nf.SetFilters(fs) - nf.SetOp(op) - nf.SetName(f.Name) - nf.SetValue(f.Value) - nf.SetKey(f.Key) - return nf, nil -} - -func (f *filter) fromNetmap(nf *netmap.Filter) { - f.Name = nf.GetName() - f.Key = nf.GetKey() - f.Value = nf.GetValue() - switch nf.GetOp() { - case netmap.EQ: - f.Op = "EQ" - case netmap.NE: - f.Op = "NE" - case netmap.GT: - f.Op = "GT" - case netmap.GE: - f.Op = "GE" - case netmap.LT: - f.Op = "LT" - case netmap.LE: - f.Op = "LE" - case netmap.AND: - f.Op = "AND" - case netmap.OR: - f.Op = "OR" - default: - // do nothing - } - if nf.GetFilters() != nil { - f.Filters = make([]filter, len(nf.GetFilters())) - for i, sf := range nf.GetFilters() { - f.Filters[i].fromNetmap(sf) - } - } -} - -func (s *selector) toNetmap() (*netmap.Selector, error) { - var c netmap.Clause - switch strings.ToUpper(s.Clause) { - case "SAME": - c = netmap.Same - case "DISTINCT": - c = netmap.Distinct - case "": - default: - return nil, fmt.Errorf("%w: '%s'", ErrUnknownClause, s.Clause) - } - ns := new(netmap.Selector) - ns.SetName(s.Name) - ns.SetAttribute(s.Attribute) - ns.SetCount(s.Count) - ns.SetClause(c) - ns.SetFilter(s.Filter) - return ns, nil -} - -func (s *selector) fromNetmap(ns *netmap.Selector) { - s.Name = ns.GetName() - s.Filter = ns.GetFilter() - s.Count = ns.GetCount() - s.Attribute = ns.GetAttribute() - switch ns.GetClause() { - case netmap.Same: - s.Clause = "same" - case netmap.Distinct: - s.Clause = "distinct" - default: - // do nothing - } - s.Name = ns.GetName() -} diff --git a/pkg/policy/json_test.go b/pkg/policy/json_test.go deleted file mode 100644 index 60b1a15d5..000000000 --- a/pkg/policy/json_test.go +++ /dev/null @@ -1,84 +0,0 @@ -package policy - -import ( - "testing" - - "github.com/nspcc-dev/neofs-api-go/v2/netmap" - "github.com/stretchr/testify/require" -) - -func TestToJSON(t *testing.T) { - check := func(t *testing.T, p *netmap.PlacementPolicy, json string) { - data, err := ToJSON(p) - require.NoError(t, err) - require.JSONEq(t, json, string(data)) - - np, err := FromJSON(data) - require.NoError(t, err) - require.Equal(t, p, np) - } - t.Run("SimpleREP", func(t *testing.T) { - p := new(netmap.PlacementPolicy) - p.SetReplicas([]*netmap.Replica{newReplica("", 3)}) - check(t, p, `{"replicas":[{"count":3}]}`) - }) - t.Run("REPWithCBF", func(t *testing.T) { - p := new(netmap.PlacementPolicy) - p.SetReplicas([]*netmap.Replica{newReplica("", 3)}) - p.SetContainerBackupFactor(3) - check(t, p, `{"replicas":[{"count":3}],"container_backup_factor":3}`) - }) - t.Run("REPFromSelector", func(t *testing.T) { - p := new(netmap.PlacementPolicy) - p.SetReplicas([]*netmap.Replica{newReplica("Nodes", 3)}) - p.SetContainerBackupFactor(3) - p.SetSelectors([]*netmap.Selector{ - newSelector(1, netmap.Distinct, "City", "", "Nodes"), - }) - check(t, p, `{ - "replicas":[{"count":3,"selector":"Nodes"}], - "container_backup_factor":3, - "selectors": [{ - "name":"Nodes", - "attribute":"City", - "clause":"distinct", - "count":1 - }]}`) - }) - t.Run("FilterOps", func(t *testing.T) { - p := new(netmap.PlacementPolicy) - p.SetReplicas([]*netmap.Replica{newReplica("Nodes", 3)}) - p.SetContainerBackupFactor(3) - p.SetSelectors([]*netmap.Selector{ - newSelector(1, netmap.Same, "City", "Good", "Nodes"), - }) - p.SetFilters([]*netmap.Filter{ - newFilter("GoodRating", "Rating", "5", netmap.GE), - newFilter("Good", "", "", netmap.OR, - newFilter("GoodRating", "", "", netmap.UnspecifiedOperation), - newFilter("", "Attr1", "Val1", netmap.EQ), - newFilter("", "Attr2", "Val2", netmap.NE), - newFilter("", "", "", netmap.AND, - newFilter("", "Attr4", "2", netmap.LT), - newFilter("", "Attr5", "3", netmap.LE)), - newFilter("", "Attr3", "1", netmap.GT)), - }) - check(t, p, `{ - "replicas":[{"count":3,"selector":"Nodes"}], - "container_backup_factor":3, - "selectors": [{"name":"Nodes","attribute":"City","clause":"same","count":1,"filter":"Good"}], - "filters": [ - {"name":"GoodRating","key":"Rating","op":"GE","value":"5"}, - {"name":"Good","op":"OR","filters":[ - {"name":"GoodRating"}, - {"key":"Attr1","op":"EQ","value":"Val1"}, - {"key":"Attr2","op":"NE","value":"Val2"}, - {"op":"AND","filters":[ - {"key":"Attr4","op":"LT","value":"2"}, - {"key":"Attr5","op":"LE","value":"3"} - ]}, - {"key":"Attr3","op":"GT","value":"1"} - ]} - ]}`) - }) -} diff --git a/pkg/policy/query.go b/pkg/policy/query.go deleted file mode 100644 index 490427901..000000000 --- a/pkg/policy/query.go +++ /dev/null @@ -1,173 +0,0 @@ -package policy - -import ( - "errors" - "fmt" - "strings" - - "github.com/nspcc-dev/neofs-api-go/pkg/netmap" -) - -var ( - ErrInvalidNumber = errors.New("policy: expected positive integer") - ErrUnknownClause = errors.New("policy: unknown clause") - ErrUnknownOp = errors.New("policy: unknown operation") - ErrUnknownFilter = errors.New("policy: filter not found") - ErrUnknownSelector = errors.New("policy: selector not found") -) - -func parse(s string) (*query, error) { - q := new(query) - err := parser.Parse(strings.NewReader(s), q) - if err != nil { - return nil, err - } - return q, nil -} - -// Parse parses s into a placement policy. -func Parse(s string) (*netmap.PlacementPolicy, error) { - q, err := parse(s) - if err != nil { - return nil, err - } - - seenFilters := map[string]bool{} - fs := make([]*netmap.Filter, 0, len(q.Filters)) - for _, qf := range q.Filters { - f, err := filterFromOrChain(qf.Value, seenFilters) - if err != nil { - return nil, err - } - f.SetName(qf.Name) - fs = append(fs, f) - seenFilters[qf.Name] = true - } - - seenSelectors := map[string]bool{} - ss := make([]*netmap.Selector, 0, len(q.Selectors)) - for _, qs := range q.Selectors { - if qs.Filter != netmap.MainFilterName && !seenFilters[qs.Filter] { - return nil, fmt.Errorf("%w: '%s'", ErrUnknownFilter, qs.Filter) - } - s := netmap.NewSelector() - switch len(qs.Bucket) { - case 1: // only bucket - s.SetAttribute(qs.Bucket[0]) - case 2: // clause + bucket - s.SetClause(clauseFromString(qs.Bucket[0])) - s.SetAttribute(qs.Bucket[1]) - } - s.SetName(qs.Name) - seenSelectors[qs.Name] = true - s.SetFilter(qs.Filter) - if qs.Count == 0 { - return nil, fmt.Errorf("%w: SELECT", ErrInvalidNumber) - } - s.SetCount(qs.Count) - ss = append(ss, s) - } - - rs := make([]*netmap.Replica, 0, len(q.Replicas)) - for _, qr := range q.Replicas { - r := netmap.NewReplica() - if qr.Selector != "" { - if !seenSelectors[qr.Selector] { - return nil, fmt.Errorf("%w: '%s'", ErrUnknownSelector, qr.Selector) - } - r.SetSelector(qr.Selector) - } - if qr.Count == 0 { - return nil, fmt.Errorf("%w: REP", ErrInvalidNumber) - } - r.SetCount(uint32(qr.Count)) - rs = append(rs, r) - } - - p := new(netmap.PlacementPolicy) - p.SetFilters(fs...) - p.SetSelectors(ss...) - p.SetReplicas(rs...) - p.SetContainerBackupFactor(q.CBF) - - return p, nil -} - -func clauseFromString(s string) netmap.Clause { - switch strings.ToUpper(s) { - case "SAME": - return netmap.ClauseSame - case "DISTINCT": - return netmap.ClauseDistinct - default: - return 0 - } -} - -func filterFromOrChain(expr *orChain, seen map[string]bool) (*netmap.Filter, error) { - var fs []*netmap.Filter - for _, ac := range expr.Clauses { - f, err := filterFromAndChain(ac, seen) - if err != nil { - return nil, err - } - fs = append(fs, f) - } - if len(fs) == 1 { - return fs[0], nil - } - - f := netmap.NewFilter() - f.SetOperation(netmap.OpOR) - f.SetInnerFilters(fs...) - return f, nil -} - -func filterFromAndChain(expr *andChain, seen map[string]bool) (*netmap.Filter, error) { - var fs []*netmap.Filter - for _, fe := range expr.Clauses { - var f *netmap.Filter - var err error - if fe.Expr != nil { - f, err = filterFromSimpleExpr(fe.Expr, seen) - } else { - f = netmap.NewFilter() - f.SetName(fe.Reference) - } - if err != nil { - return nil, err - } - fs = append(fs, f) - } - if len(fs) == 1 { - return fs[0], nil - } - - f := netmap.NewFilter() - f.SetOperation(netmap.OpAND) - f.SetInnerFilters(fs...) - return f, nil -} - -func filterFromSimpleExpr(se *simpleExpr, seen map[string]bool) (*netmap.Filter, error) { - f := netmap.NewFilter() - f.SetKey(se.Key) - switch se.Op { - case "EQ": - f.SetOperation(netmap.OpEQ) - case "NE": - f.SetOperation(netmap.OpNE) - case "GE": - f.SetOperation(netmap.OpGE) - case "GT": - f.SetOperation(netmap.OpGT) - case "LE": - f.SetOperation(netmap.OpLE) - case "LT": - f.SetOperation(netmap.OpLT) - default: - return nil, fmt.Errorf("%w: '%s'", ErrUnknownOp, se.Op) - } - f.SetValue(se.Value) - return f, nil -} diff --git a/pkg/policy/query_test.go b/pkg/policy/query_test.go deleted file mode 100644 index b7c4cc76f..000000000 --- a/pkg/policy/query_test.go +++ /dev/null @@ -1,284 +0,0 @@ -package policy - -import ( - "errors" - "testing" - - "github.com/nspcc-dev/neofs-api-go/v2/netmap" - "github.com/stretchr/testify/require" -) - -func TestSimple(t *testing.T) { - q := `REP 3` - expected := new(netmap.PlacementPolicy) - expected.SetFilters([]*netmap.Filter{}) - expected.SetSelectors([]*netmap.Selector{}) - expected.SetReplicas([]*netmap.Replica{newReplica("", 3)}) - - r, err := Parse(q) - require.NoError(t, err) - require.EqualValues(t, expected, r) -} - -func TestSimpleWithHRWB(t *testing.T) { - q := `REP 3 CBF 4` - expected := new(netmap.PlacementPolicy) - expected.SetFilters([]*netmap.Filter{}) - expected.SetSelectors([]*netmap.Selector{}) - expected.SetReplicas([]*netmap.Replica{newReplica("", 3)}) - expected.SetContainerBackupFactor(4) - - r, err := Parse(q) - require.NoError(t, err) - require.EqualValues(t, expected, r) -} - -func TestFromSelect(t *testing.T) { - q := `REP 1 IN SPB -SELECT 1 IN City FROM * AS SPB` - expected := new(netmap.PlacementPolicy) - expected.SetFilters([]*netmap.Filter{}) - expected.SetSelectors([]*netmap.Selector{ - newSelector(1, netmap.UnspecifiedClause, "City", "*", "SPB"), - }) - expected.SetReplicas([]*netmap.Replica{newReplica("SPB", 1)}) - - r, err := Parse(q) - require.NoError(t, err) - require.EqualValues(t, expected, r) -} - -// https://github.com/nspcc-dev/neofs-node/issues/46 -func TestFromSelectNoAttribute(t *testing.T) { - t.Run("Simple", func(t *testing.T) { - q := `REP 2 - SELECT 6 FROM *` - - expected := new(netmap.PlacementPolicy) - expected.SetFilters([]*netmap.Filter{}) - expected.SetSelectors([]*netmap.Selector{newSelector(6, netmap.UnspecifiedClause, "", "*", "")}) - expected.SetReplicas([]*netmap.Replica{newReplica("", 2)}) - - r, err := Parse(q) - require.NoError(t, err) - require.EqualValues(t, expected, r) - - }) - - t.Run("with filter", func(t *testing.T) { - q := `REP 2 - SELECT 6 FROM F - FILTER StorageType EQ SSD AS F` - - expected := new(netmap.PlacementPolicy) - expected.SetFilters([]*netmap.Filter{newFilter("F", "StorageType", "SSD", netmap.EQ)}) - expected.SetSelectors([]*netmap.Selector{newSelector(6, netmap.UnspecifiedClause, "", "F", "")}) - expected.SetReplicas([]*netmap.Replica{newReplica("", 2)}) - - r, err := Parse(q) - require.NoError(t, err) - require.EqualValues(t, expected, r) - }) -} - -func TestFromSelectClause(t *testing.T) { - q := `REP 4 -SELECT 3 IN Country FROM * -SELECT 2 IN SAME City FROM * -SELECT 1 IN DISTINCT Continent FROM *` - expected := new(netmap.PlacementPolicy) - expected.SetFilters([]*netmap.Filter{}) - expected.SetSelectors([]*netmap.Selector{ - newSelector(3, netmap.UnspecifiedClause, "Country", "*", ""), - newSelector(2, netmap.Same, "City", "*", ""), - newSelector(1, netmap.Distinct, "Continent", "*", ""), - }) - expected.SetReplicas([]*netmap.Replica{newReplica("", 4)}) - - r, err := Parse(q) - require.NoError(t, err) - require.EqualValues(t, expected, r) -} - -func TestSimpleFilter(t *testing.T) { - q := `REP 1 -SELECT 1 IN City FROM Good -FILTER Rating GT 7 AS Good` - expected := new(netmap.PlacementPolicy) - expected.SetReplicas([]*netmap.Replica{newReplica("", 1)}) - expected.SetSelectors([]*netmap.Selector{ - newSelector(1, netmap.UnspecifiedClause, "City", "Good", ""), - }) - expected.SetFilters([]*netmap.Filter{ - newFilter("Good", "Rating", "7", netmap.GT), - }) - - r, err := Parse(q) - require.NoError(t, err) - require.EqualValues(t, expected, r) -} - -func TestFilterReference(t *testing.T) { - q := `REP 1 -SELECT 2 IN City FROM Good -FILTER Country EQ "RU" AS FromRU -FILTER @FromRU AND Rating GT 7 AS Good` - expected := new(netmap.PlacementPolicy) - expected.SetReplicas([]*netmap.Replica{newReplica("", 1)}) - expected.SetSelectors([]*netmap.Selector{ - newSelector(2, netmap.UnspecifiedClause, "City", "Good", ""), - }) - expected.SetFilters([]*netmap.Filter{ - newFilter("FromRU", "Country", "RU", netmap.EQ), - newFilter("Good", "", "", netmap.AND, - newFilter("FromRU", "", "", netmap.UnspecifiedOperation), - newFilter("", "Rating", "7", netmap.GT)), - }) - - r, err := Parse(q) - require.NoError(t, err) - require.EqualValues(t, expected, r) -} - -func TestFilterOps(t *testing.T) { - q := `REP 1 -SELECT 2 IN City FROM Good -FILTER A GT 1 AND B GE 2 AND C LT 3 AND D LE 4 - AND E EQ 5 AND F NE 6 AS Good` - expected := new(netmap.PlacementPolicy) - expected.SetReplicas([]*netmap.Replica{newReplica("", 1)}) - expected.SetSelectors([]*netmap.Selector{ - newSelector(2, netmap.UnspecifiedClause, "City", "Good", ""), - }) - expected.SetFilters([]*netmap.Filter{ - newFilter("Good", "", "", netmap.AND, - newFilter("", "A", "1", netmap.GT), - newFilter("", "B", "2", netmap.GE), - newFilter("", "C", "3", netmap.LT), - newFilter("", "D", "4", netmap.LE), - newFilter("", "E", "5", netmap.EQ), - newFilter("", "F", "6", netmap.NE)), - }) - - r, err := Parse(q) - require.NoError(t, err) - require.EqualValues(t, expected, r) -} - -func TestWithFilterPrecedence(t *testing.T) { - q := `REP 7 IN SPB -SELECT 1 IN City FROM SPBSSD AS SPB -FILTER City EQ "SPB" AND SSD EQ true OR City EQ "SPB" AND Rating GE 5 AS SPBSSD` - expected := new(netmap.PlacementPolicy) - expected.SetReplicas([]*netmap.Replica{newReplica("SPB", 7)}) - expected.SetSelectors([]*netmap.Selector{ - newSelector(1, netmap.UnspecifiedClause, "City", "SPBSSD", "SPB"), - }) - expected.SetFilters([]*netmap.Filter{ - newFilter("SPBSSD", "", "", netmap.OR, - newFilter("", "", "", netmap.AND, - newFilter("", "City", "SPB", netmap.EQ), - newFilter("", "SSD", "true", netmap.EQ)), - newFilter("", "", "", netmap.AND, - newFilter("", "City", "SPB", netmap.EQ), - newFilter("", "Rating", "5", netmap.GE))), - }) - - r, err := Parse(q) - require.NoError(t, err) - require.EqualValues(t, expected, r) -} - -func TestValidation(t *testing.T) { - t.Run("MissingSelector", func(t *testing.T) { - q := `REP 3 IN RU` - _, err := Parse(q) - require.True(t, errors.Is(err, ErrUnknownSelector), "got: %v", err) - }) - t.Run("MissingFilter", func(t *testing.T) { - q := `REP 3 - SELECT 1 IN City FROM MissingFilter` - _, err := Parse(q) - require.True(t, errors.Is(err, ErrUnknownFilter), "got: %v", err) - }) - t.Run("UnknownOp", func(t *testing.T) { - q := `REP 3 - SELECT 1 IN City FROM F - FILTER Country KEK RU AS F` - _, err := Parse(q) - require.True(t, errors.Is(err, ErrUnknownOp), "got: %v", err) - }) - t.Run("TypoInREP", func(t *testing.T) { - q := `REK 3` - _, err := Parse(q) - require.Error(t, err) - }) - t.Run("InvalidFilterName", func(t *testing.T) { - q := `REP 3 - SELECT 1 IN City FROM F - FILTER Good AND Country EQ RU AS F - FILTER Rating EQ 5 AS Good` - _, err := Parse(q) - require.Error(t, err) - }) - t.Run("InvalidNumberInREP", func(t *testing.T) { - q := `REP 0` - _, err := Parse(q) - require.True(t, errors.Is(err, ErrInvalidNumber), "got: %v", err) - }) - t.Run("InvalidNumberInREP", func(t *testing.T) { - q := `REP 1 IN Good - SELECT 0 IN City FROM *` - _, err := Parse(q) - require.True(t, errors.Is(err, ErrInvalidNumber), "got: %v", err) - }) - -} - -func TestFilterStringSymbols(t *testing.T) { - q := `REP 1 IN S -SELECT 1 FROM F AS S -FILTER "UN-LOCODE" EQ "RU LED" AS F` - - expected := new(netmap.PlacementPolicy) - expected.SetReplicas([]*netmap.Replica{ - newReplica("S", 1), - }) - expected.SetSelectors([]*netmap.Selector{ - newSelector(1, netmap.UnspecifiedClause, "", "F", "S"), - }) - expected.SetFilters([]*netmap.Filter{ - newFilter("F", "UN-LOCODE", "RU LED", netmap.EQ), - }) - - r, err := Parse(q) - require.NoError(t, err) - require.EqualValues(t, expected, r) -} - -func newFilter(name, key, value string, op netmap.Operation, sub ...*netmap.Filter) *netmap.Filter { - f := new(netmap.Filter) - f.SetName(name) - f.SetKey(key) - f.SetValue(value) - f.SetOp(op) - f.SetFilters(sub) - return f -} - -func newReplica(s string, c uint32) *netmap.Replica { - r := new(netmap.Replica) - r.SetSelector(s) - r.SetCount(c) - return r -} - -func newSelector(count uint32, c netmap.Clause, attr, f, name string) *netmap.Selector { - s := new(netmap.Selector) - s.SetCount(count) - s.SetClause(c) - s.SetAttribute(attr) - s.SetFilter(f) - s.SetName(name) - return s -}