[#23] Support SQL-like queries for placement

JSON format is rather verbose an inconvenient to be
edited by hand. This commit implements SQL-like
language for representing placement policy.

Signed-off-by: Evgenii Stratonikov <evgeniy@nspcc.ru>

go.mod

@@ -4,6 +4,7 @@ go 1.14
 
 require (
 	bou.ke/monkey v1.0.2
+	github.com/alecthomas/participle v0.6.0
 	github.com/golang/protobuf v1.4.2
 	github.com/google/uuid v1.1.1
 	github.com/mitchellh/go-homedir v1.1.0

go.sum

@@ -34,6 +34,9 @@ github.com/abiosoft/ishell v2.0.0+incompatible/go.mod h1:HQR9AqF2R3P4XXpMpI0NAzg
 github.com/abiosoft/readline v0.0.0-20180607040430-155bce2042db h1:CjPUSXOiYptLbTdr1RceuZgSFDQ7U15ITERUGrUORx8=
 github.com/abiosoft/readline v0.0.0-20180607040430-155bce2042db/go.mod h1:rB3B4rKii8V21ydCbIzH5hZiCQE7f5E9SzUb/ZZx530=
 github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
+github.com/alecthomas/participle v0.6.0 h1:Pvo8XUCQKgIywVjz/+Ci3IsjGg+g/TdKkMcfgghKCEw=
+github.com/alecthomas/participle v0.6.0/go.mod h1:HfdmEuwvr12HXQN44HPWXR0lHmVolVYe4dyL6lQ3duY=
+github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4gioE62mJzwPIB8+Tee4RNCL9ulrY=
@@ -297,6 +300,8 @@ github.com/nspcc-dev/dbft v0.0.0-20200219114139-199d286ed6c1 h1:yEx9WznS+rjE0jl0
 github.com/nspcc-dev/dbft v0.0.0-20200219114139-199d286ed6c1/go.mod h1:O0qtn62prQSqizzoagHmuuKoz8QMkU3SzBoKdEvm3aQ=
 github.com/nspcc-dev/dbft v0.0.0-20200711144034-c526ccc6f570 h1:EHBwlOyd2m06C3dnxhpPokpYqlNg7u5ZX/uPBhjYuZ4=
 github.com/nspcc-dev/dbft v0.0.0-20200711144034-c526ccc6f570/go.mod h1:1FYQXSbb6/9HQIkoF8XO7W/S8N7AZRkBsgwbcXRvk0E=
+github.com/nspcc-dev/hrw v1.0.9 h1:17VcAuTtrstmFppBjfRiia4K2wA/ukXZhLFS8Y8rz5Y=
+github.com/nspcc-dev/hrw v1.0.9/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkPG06MU=
 github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1:pPYwPZ2ks+uMnlRLUyXOpLieaDQSEaf4NM3zHVbRjmg=
 github.com/nspcc-dev/neo-go v0.91.0 h1:KKOPMKs0fm8JIau1SuwxiLdrZ+1kDPBiVRlWwzfebWE=
 github.com/nspcc-dev/neo-go v0.91.0/go.mod h1:G6HdOWvzQ6tlvFdvFSN/PgCzLPN/X/X4d5hTjFRUDcc=

pkg/policy/doc.go

@@ -0,0 +1,20 @@
+// Package policy provides facilities for creating policy from SQL-like language.
+//   eBNF grammar is provided in `grammar.ebnf` for illustration.
+//
+// Current limitations:
+// 1. Grouping filter expressions in parenthesis is not supported right now.
+//    Requiring this will make query too verbose, making it optional makes
+//    our grammar not LL(1). This can be supported in future.
+// 2. Filters must be defined before they are used.
+// 	This requirement may be relaxed in future.
+//
+// Example query:
+// REP 1 in SPB
+// REP 2 in Americas
+// CBF 4
+// SELECT 1 Node IN City FROM SPBSSD AS SPB
+// SELECT 2 Node IN SAME City FROM Americas AS Americas
+// FILTER SSD EQ true AS IsSSD
+// FILTER @IsSSD AND Country eq "RU" AND City eq "St.Petersburg" AS SPBSSD
+// FILTER 'Continent' == 'North America' OR Continent == 'South America' AS Americas
+package policy

pkg/policy/grammar.ebnf

@@ -0,0 +1,52 @@
+Policy ::=
+    RepStmt, [RepStmt],
+    CbtStmt?,
+    [SelectStmt],
+    [FilterStmt],
+;
+
+RepStmt ::=
+    'REP', Number1, (* number of object replicas *)
+    ('AS', Ident)?  (* optional selector name *)
+;
+
+CbtStmt ::= 'CBF', Number1 (* container backup factor *)
+;
+
+SelectStmt ::=
+    'SELECT', Number1,    (* number of nodes to select without container backup factor *)
+    'IN', Clause?, Ident, (* bucket name *)
+    FROM, (Ident | '*'),  (* filter reference or whole netmap *)
+    ('AS', Ident)?        (* optional selector name *)
+;
+
+Clause ::=
+      'SAME'     (* nodes from the same bucket *)
+    | 'DISTINCT' (* nodes from distinct buckets *)
+;
+
+FilterStmt ::=
+    'FILTER', AndChain, ['OR', AndChain],
+    'AS', Ident (* obligatory filter name *)
+;
+
+AndChain ::=
+    Expr, ['AND', Expr]
+;
+
+Expr ::=
+      '@' Ident		   (* filter reference *)
+    | Ident, Op, Value (* attribute filter *)
+;
+
+Op ::= 'EQ' | 'NE' | 'GE' | 'GT' | 'LT' | 'LE'
+;
+
+Value ::= Ident | Number | String
+;
+
+Number1 ::= Digit1 [Digit];
+Number ::= Digit [Digit];
+
+Digit1 ::= '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' ;
+Digit ::= '0' | Digit1;

pkg/policy/grammar.go

@@ -0,0 +1,60 @@
+package policy
+
+import (
+	"github.com/alecthomas/participle"
+)
+
+var parser *participle.Parser
+
+func init() {
+	p, err := participle.Build(&query{})
+	if err != nil {
+		panic(err)
+	}
+	parser = p
+}
+
+type query struct {
+	Replicas  []*replicaStmt  `@@+`
+	CBF       uint32          `("CBF" @Int)?`
+	Selectors []*selectorStmt `@@*`
+	Filters   []*filterStmt   `@@*`
+}
+
+type replicaStmt struct {
+	Count    int    `"REP" @Int`
+	Selector string `("IN" @Ident)?`
+}
+
+type selectorStmt struct {
+	Count  uint32 `"SELECT" @Int`
+	Clause string `"IN" @("SAME" | "DISTINCT")?`
+	Bucket string `@Ident`
+	Filter string `"FROM" @(Ident | "*")`
+	Name   string `("AS" @Ident)?`
+}
+
+type filterStmt struct {
+	Value *orChain `"FILTER" @@`
+	Name  string   `"AS" @Ident`
+}
+
+type filterOrExpr struct {
+	Reference string      `"@"@Ident`
+	Expr      *simpleExpr `| @@`
+}
+
+type orChain struct {
+	Clauses []*andChain `@@ ("OR" @@)*`
+}
+
+type andChain struct {
+	Clauses []*filterOrExpr `@@ ("AND" @@)*`
+}
+
+type simpleExpr struct {
+	Key string `@Ident`
+	// We don't use literals here to improve error messages.
+	Op    string `@Ident`
+	Value string `@(Ident | String | Int)`
+}

pkg/policy/query.go

@@ -0,0 +1,163 @@
+package policy
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+
+	sdknm "github.com/nspcc-dev/neofs-api-go/pkg/netmap"
+	"github.com/nspcc-dev/neofs-api-go/v2/netmap"
+)
+
+var (
+	ErrInvalidNumber   = errors.New("policy: expected positive integer")
+	ErrUnknownOp       = errors.New("policy: unknown operation")
+	ErrUnknownFilter   = errors.New("policy: filter not found")
+	ErrUnknownSelector = errors.New("policy: selector not found")
+)
+
+func parse(s string) (*query, error) {
+	q := new(query)
+	err := parser.Parse(strings.NewReader(s), q)
+	if err != nil {
+		return nil, err
+	}
+	return q, nil
+}
+
+// Parse parses s into a placement policy.
+func Parse(s string) (*netmap.PlacementPolicy, error) {
+	q, err := parse(s)
+	if err != nil {
+		return nil, err
+	}
+
+	seenFilters := map[string]bool{}
+	fs := make([]*netmap.Filter, 0, len(q.Filters))
+	for _, qf := range q.Filters {
+		f, err := filterFromOrChain(qf.Value, seenFilters)
+		if err != nil {
+			return nil, err
+		}
+		f.SetName(qf.Name)
+		fs = append(fs, f)
+		seenFilters[qf.Name] = true
+	}
+
+	seenSelectors := map[string]bool{}
+	ss := make([]*netmap.Selector, 0, len(q.Selectors))
+	for _, qs := range q.Selectors {
+		if qs.Filter != sdknm.MainFilterName && !seenFilters[qs.Filter] {
+			return nil, fmt.Errorf("%w: '%s'", ErrUnknownFilter, qs.Filter)
+		}
+		s := new(netmap.Selector)
+		switch qs.Clause {
+		case "SAME":
+			s.SetClause(netmap.Same)
+		case "DISTINCT":
+			s.SetClause(netmap.Distinct)
+		default:
+			s.SetClause(netmap.UnspecifiedClause)
+		}
+		s.SetName(qs.Name)
+		seenSelectors[qs.Name] = true
+		s.SetFilter(qs.Filter)
+		s.SetAttribute(qs.Bucket)
+		if qs.Count == 0 {
+			return nil, fmt.Errorf("%w: SELECT", ErrInvalidNumber)
+		}
+		s.SetCount(qs.Count)
+		ss = append(ss, s)
+	}
+
+	rs := make([]*netmap.Replica, 0, len(q.Replicas))
+	for _, qr := range q.Replicas {
+		r := new(netmap.Replica)
+		if qr.Selector != "" {
+			if !seenSelectors[qr.Selector] {
+				return nil, fmt.Errorf("%w: '%s'", ErrUnknownSelector, qr.Selector)
+			}
+			r.SetSelector(qr.Selector)
+		}
+		if qr.Count == 0 {
+			return nil, fmt.Errorf("%w: REP", ErrInvalidNumber)
+		}
+		r.SetCount(uint32(qr.Count))
+		rs = append(rs, r)
+	}
+
+	p := new(netmap.PlacementPolicy)
+	p.SetFilters(fs)
+	p.SetSelectors(ss)
+	p.SetReplicas(rs)
+	p.SetContainerBackupFactor(q.CBF)
+	return p, nil
+}
+
+func filterFromOrChain(expr *orChain, seen map[string]bool) (*netmap.Filter, error) {
+	var fs []*netmap.Filter
+	for _, ac := range expr.Clauses {
+		f, err := filterFromAndChain(ac, seen)
+		if err != nil {
+			return nil, err
+		}
+		fs = append(fs, f)
+	}
+	if len(fs) == 1 {
+		return fs[0], nil
+	}
+
+	f := new(netmap.Filter)
+	f.SetOp(netmap.OR)
+	f.SetFilters(fs)
+	return f, nil
+}
+
+func filterFromAndChain(expr *andChain, seen map[string]bool) (*netmap.Filter, error) {
+	var fs []*netmap.Filter
+	for _, fe := range expr.Clauses {
+		var f *netmap.Filter
+		var err error
+		if fe.Expr != nil {
+			f, err = filterFromSimpleExpr(fe.Expr, seen)
+		} else {
+			f = new(netmap.Filter)
+			f.SetName(fe.Reference)
+		}
+		if err != nil {
+			return nil, err
+		}
+		fs = append(fs, f)
+	}
+	if len(fs) == 1 {
+		return fs[0], nil
+	}
+
+	f := new(netmap.Filter)
+	f.SetOp(netmap.AND)
+	f.SetFilters(fs)
+	return f, nil
+}
+
+func filterFromSimpleExpr(se *simpleExpr, seen map[string]bool) (*netmap.Filter, error) {
+	f := new(netmap.Filter)
+	f.SetKey(se.Key)
+	switch se.Op {
+	case "EQ":
+		f.SetOp(netmap.EQ)
+	case "NE":
+		f.SetOp(netmap.NE)
+	case "GE":
+		f.SetOp(netmap.GE)
+	case "GT":
+		f.SetOp(netmap.GT)
+	case "LE":
+		f.SetOp(netmap.LE)
+	case "LT":
+		f.SetOp(netmap.LT)
+	default:
+		return nil, fmt.Errorf("%w: '%s'", ErrUnknownOp, se.Op)
+	}
+	f.SetValue(se.Value)
+	return f, nil
+}

pkg/policy/query_test.go

@@ -0,0 +1,230 @@
+package policy
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/nspcc-dev/neofs-api-go/v2/netmap"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSimple(t *testing.T) {
+	q := `REP 3`
+	expected := new(netmap.PlacementPolicy)
+	expected.SetFilters([]*netmap.Filter{})
+	expected.SetSelectors([]*netmap.Selector{})
+	expected.SetReplicas([]*netmap.Replica{newReplica("", 3)})
+
+	r, err := Parse(q)
+	require.NoError(t, err)
+	require.Equal(t, expected, r)
+}
+
+func TestSimpleWithHRWB(t *testing.T) {
+	q := `REP 3 CBF 4`
+	expected := new(netmap.PlacementPolicy)
+	expected.SetFilters([]*netmap.Filter{})
+	expected.SetSelectors([]*netmap.Selector{})
+	expected.SetReplicas([]*netmap.Replica{newReplica("", 3)})
+	expected.SetContainerBackupFactor(4)
+
+	r, err := Parse(q)
+	require.NoError(t, err)
+	require.Equal(t, expected, r)
+}
+
+func TestFromSelect(t *testing.T) {
+	q := `REP 1 IN SPB
+SELECT 1 IN City FROM * AS SPB`
+	expected := new(netmap.PlacementPolicy)
+	expected.SetFilters([]*netmap.Filter{})
+	expected.SetSelectors([]*netmap.Selector{
+		newSelector(1, netmap.UnspecifiedClause, "City", "*", "SPB"),
+	})
+	expected.SetReplicas([]*netmap.Replica{newReplica("SPB", 1)})
+
+	r, err := Parse(q)
+	require.NoError(t, err)
+	require.Equal(t, expected, r)
+}
+
+func TestFromSelectClause(t *testing.T) {
+	q := `REP 4
+SELECT 3 IN Country FROM *
+SELECT 2 IN SAME City FROM *
+SELECT 1 IN DISTINCT Continent FROM *`
+	expected := new(netmap.PlacementPolicy)
+	expected.SetFilters([]*netmap.Filter{})
+	expected.SetSelectors([]*netmap.Selector{
+		newSelector(3, netmap.UnspecifiedClause, "Country", "*", ""),
+		newSelector(2, netmap.Same, "City", "*", ""),
+		newSelector(1, netmap.Distinct, "Continent", "*", ""),
+	})
+	expected.SetReplicas([]*netmap.Replica{newReplica("", 4)})
+
+	r, err := Parse(q)
+	require.NoError(t, err)
+	require.Equal(t, expected, r)
+}
+
+func TestSimpleFilter(t *testing.T) {
+	q := `REP 1
+SELECT 1 IN City FROM Good
+FILTER Rating GT 7 AS Good`
+	expected := new(netmap.PlacementPolicy)
+	expected.SetReplicas([]*netmap.Replica{newReplica("", 1)})
+	expected.SetSelectors([]*netmap.Selector{
+		newSelector(1, netmap.UnspecifiedClause, "City", "Good", ""),
+	})
+	expected.SetFilters([]*netmap.Filter{
+		newFilter("Good", "Rating", "7", netmap.GT),
+	})
+
+	r, err := Parse(q)
+	require.NoError(t, err)
+	require.Equal(t, expected, r)
+}
+
+func TestFilterReference(t *testing.T) {
+	q := `REP 1
+SELECT 2 IN City FROM Good
+FILTER Country EQ "RU" AS FromRU
+FILTER @FromRU AND Rating GT 7 AS Good`
+	expected := new(netmap.PlacementPolicy)
+	expected.SetReplicas([]*netmap.Replica{newReplica("", 1)})
+	expected.SetSelectors([]*netmap.Selector{
+		newSelector(2, netmap.UnspecifiedClause, "City", "Good", ""),
+	})
+	expected.SetFilters([]*netmap.Filter{
+		newFilter("FromRU", "Country", "RU", netmap.EQ),
+		newFilter("Good", "", "", netmap.AND,
+			newFilter("FromRU", "", "", netmap.UnspecifiedOperation),
+			newFilter("", "Rating", "7", netmap.GT)),
+	})
+
+	r, err := Parse(q)
+	require.NoError(t, err)
+	require.Equal(t, expected, r)
+}
+
+func TestFilterOps(t *testing.T) {
+	q := `REP 1
+SELECT 2 IN City FROM Good
+FILTER A GT 1 AND B GE 2 AND C LT 3 AND D LE 4
+  AND E EQ 5 AND F NE 6 AS Good`
+	expected := new(netmap.PlacementPolicy)
+	expected.SetReplicas([]*netmap.Replica{newReplica("", 1)})
+	expected.SetSelectors([]*netmap.Selector{
+		newSelector(2, netmap.UnspecifiedClause, "City", "Good", ""),
+	})
+	expected.SetFilters([]*netmap.Filter{
+		newFilter("Good", "", "", netmap.AND,
+			newFilter("", "A", "1", netmap.GT),
+			newFilter("", "B", "2", netmap.GE),
+			newFilter("", "C", "3", netmap.LT),
+			newFilter("", "D", "4", netmap.LE),
+			newFilter("", "E", "5", netmap.EQ),
+			newFilter("", "F", "6", netmap.NE)),
+	})
+
+	r, err := Parse(q)
+	require.NoError(t, err)
+	require.Equal(t, expected, r)
+}
+
+func TestWithFilterPrecedence(t *testing.T) {
+	q := `REP 7 IN SPB
+SELECT 1 IN City FROM SPBSSD AS SPB
+FILTER City EQ "SPB" AND SSD EQ true OR City EQ "SPB" AND Rating GE 5 AS SPBSSD`
+	expected := new(netmap.PlacementPolicy)
+	expected.SetReplicas([]*netmap.Replica{newReplica("SPB", 7)})
+	expected.SetSelectors([]*netmap.Selector{
+		newSelector(1, netmap.UnspecifiedClause, "City", "SPBSSD", "SPB"),
+	})
+	expected.SetFilters([]*netmap.Filter{
+		newFilter("SPBSSD", "", "", netmap.OR,
+			newFilter("", "", "", netmap.AND,
+				newFilter("", "City", "SPB", netmap.EQ),
+				newFilter("", "SSD", "true", netmap.EQ)),
+			newFilter("", "", "", netmap.AND,
+				newFilter("", "City", "SPB", netmap.EQ),
+				newFilter("", "Rating", "5", netmap.GE))),
+	})
+
+	r, err := Parse(q)
+	require.NoError(t, err)
+	require.Equal(t, expected, r)
+}
+
+func TestValidation(t *testing.T) {
+	t.Run("MissingSelector", func(t *testing.T) {
+		q := `REP 3 IN RU`
+		_, err := Parse(q)
+		require.True(t, errors.Is(err, ErrUnknownSelector), "got: %v", err)
+	})
+	t.Run("MissingFilter", func(t *testing.T) {
+		q := `REP 3
+              SELECT 1 IN City FROM MissingFilter`
+		_, err := Parse(q)
+		require.True(t, errors.Is(err, ErrUnknownFilter), "got: %v", err)
+	})
+	t.Run("UnknownOp", func(t *testing.T) {
+		q := `REP 3
+              SELECT 1 IN City FROM F
+			  FILTER Country KEK RU AS F`
+		_, err := Parse(q)
+		require.True(t, errors.Is(err, ErrUnknownOp), "got: %v", err)
+	})
+	t.Run("TypoInREP", func(t *testing.T) {
+		q := `REK 3`
+		_, err := Parse(q)
+		require.Error(t, err)
+	})
+	t.Run("InvalidFilterName", func(t *testing.T) {
+		q := `REP 3
+			  SELECT 1 IN City FROM F
+			  FILTER Good AND Country EQ RU AS F
+			  FILTER Rating EQ 5 AS Good`
+		_, err := Parse(q)
+		require.Error(t, err)
+	})
+	t.Run("InvalidNumberInREP", func(t *testing.T) {
+		q := `REP 0`
+		_, err := Parse(q)
+		require.True(t, errors.Is(err, ErrInvalidNumber), "got: %v", err)
+	})
+	t.Run("InvalidNumberInREP", func(t *testing.T) {
+		q := `REP 1 IN Good
+			  SELECT 0 IN City FROM *`
+		_, err := Parse(q)
+		require.True(t, errors.Is(err, ErrInvalidNumber), "got: %v", err)
+	})
+
+}
+
+func newFilter(name, key, value string, op netmap.Operation, sub ...*netmap.Filter) *netmap.Filter {
+	f := new(netmap.Filter)
+	f.SetName(name)
+	f.SetKey(key)
+	f.SetValue(value)
+	f.SetOp(op)
+	f.SetFilters(sub)
+	return f
+}
+
+func newReplica(s string, c uint32) *netmap.Replica {
+	r := new(netmap.Replica)
+	r.SetSelector(s)
+	r.SetCount(c)
+	return r
+}
+
+func newSelector(count uint32, c netmap.Clause, attr, f, name string) *netmap.Selector {
+	s := new(netmap.Selector)
+	s.SetCount(count)
+	s.SetClause(c)
+	s.SetAttribute(attr)
+	s.SetFilter(f)
+	s.SetName(name)
+	return s
+}