TrueCloudLab/frostfs-sdk-go
33
0
Fork 21
Code Issues 34 Pull requests 1 Releases Activity Actions

netmap: Support non-ascii attributes in SELECT IN #273

Merged
acid-ant merged 6 commits from a-savchuk/frostfs-sdk-go:antlr-bug into master 2024-11-02 14:21:46 +00:00
Conversation 1 Commits 6 Files changed 13 +112 -45
13 changed files with 112 additions and 45 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.gitignore vendored
View file

@ -23,7 +23,7 @@ coverage.txt
coverage.html
# antlr tool jar
antlr-*.jar
antlr*.jar
# tempfiles
.cache

5
Makefile
View file

@ -1,6 +1,6 @@
#!/usr/bin/make -f
ANTLR_VERSION="4.13.0"
ANTLR_VERSION=4.13.1
TMP_DIR := .cache
LINT_VERSION ?= 1.60.1
TRUECLOUDLAB_LINT_VERSION ?= 0.0.6
@ -53,7 +53,8 @@ format:
policy:
@wget -q https://www.antlr.org/download/antlr-${ANTLR_VERSION}-complete.jar -O antlr4-tool.jar
@java -Xmx500M -cp "`pwd`/antlr4-tool.jar" "org.antlr.v4.Tool" -o `pwd`/netmap/parser/ -Dlanguage=Go -no-listener -visitor `pwd`/netmap/parser/Query.g4 `pwd`/netmap/parser/QueryLexer.g4
@java -Xmx500M -cp antlr4-tool.jar org.antlr.v4.Tool -Dlanguage=Go \
-no-listener -visitor netmap/parser/Query.g4 netmap/parser/QueryLexer.g4
# Run `make %` in truecloudlab/frostfs-sdk-go container(Golang+Java)
docker/%:

4
go.mod
View file

@ -7,7 +7,7 @@ require (
git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e
git.frostfs.info/TrueCloudLab/hrw v1.2.1
git.frostfs.info/TrueCloudLab/tzhash v1.8.0
github.com/antlr4-go/antlr/v4 v4.13.0
github.com/antlr4-go/antlr/v4 v4.13.1
github.com/google/uuid v1.6.0
github.com/hashicorp/golang-lru/v2 v2.0.7
github.com/klauspost/reedsolomon v1.12.1
@ -40,7 +40,7 @@ require (
go.etcd.io/bbolt v1.3.9 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/crypto v0.24.0 // indirect
golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect
golang.org/x/net v0.26.0 // indirect
golang.org/x/sync v0.7.0 // indirect
golang.org/x/sys v0.21.0 // indirect

BIN
go.sum
View file

Binary file not shown.

8
netmap/parser/Query.g4
View file

@ -19,10 +19,10 @@ repStmt:
cbfStmt: CBF BackupFactor = NUMBER1; // container backup factor
selectStmt:
SELECT Count = NUMBER1 // number of nodes to select without container backup factor *)
(IN clause? Bucket = ident)? // bucket name
FROM Filter = identWC // filter reference or whole netmap
(AS Name = ident)? // optional selector name
SELECT Count = NUMBER1 // number of nodes to select without container backup factor *)
(IN clause? Bucket = filterKey)? // bucket name
FROM Filter = identWC // filter reference or whole netmap
(AS Name = ident)? // optional selector name
;
clause: CLAUSE_SAME | CLAUSE_DISTINCT; // nodes from distinct buckets

BIN
netmap/parser/Query.interp
View file

Binary file not shown.

5
netmap/parser/generate.go
View file

@ -1,4 +1,5 @@
package parser
// ANTLR can be downloaded from https://www.antlr.org/download/antlr-4.13.0-complete.jar
//go:generate java -Xmx500M -cp "./antlr-4.13.0-complete.jar:$CLASSPATH" org.antlr.v4.Tool -Dlanguage=Go -no-listener -visitor QueryLexer.g4 Query.g4
// You can download ANTLR from https://www.antlr.org/download/antlr-4.13.1-complete.jar,
// then run generate or simply run the dedicated Makefile target like this `make policy`.
//go:generate java -Xmx500M -cp "./antlr-4.13.1-complete.jar:$CLASSPATH" org.antlr.v4.Tool -Dlanguage=Go -no-listener -visitor QueryLexer.g4 Query.g4

2
netmap/parser/query_base_visitor.go
View file

@ -1,4 +1,4 @@
// Code generated from /repo/frostfs/sdk-go/netmap/parser/Query.g4 by ANTLR 4.13.0. DO NOT EDIT.
// Code generated from netmap/parser/Query.g4 by ANTLR 4.13.1. DO NOT EDIT.
package parser // Query

2
netmap/parser/query_lexer.go
View file

@ -1,4 +1,4 @@
// Code generated from /repo/frostfs/sdk-go/netmap/parser/QueryLexer.g4 by ANTLR 4.13.0. DO NOT EDIT.
// Code generated from netmap/parser/QueryLexer.g4 by ANTLR 4.13.1. DO NOT EDIT.
package parser

53
netmap/parser/query_parser.go
View file

@ -1,4 +1,4 @@
// Code generated from /repo/frostfs/sdk-go/netmap/parser/Query.g4 by ANTLR 4.13.0. DO NOT EDIT.
// Code generated from netmap/parser/Query.g4 by ANTLR 4.13.1. DO NOT EDIT.
package parser // Query
@ -93,7 +93,7 @@ func queryParserInit() {
85, 1, 0, 0, 0, 85, 7, 1, 0, 0, 0, 86, 87, 5, 10, 0, 0, 87, 88, 5, 22,
0, 0, 88, 9, 1, 0, 0, 0, 89, 90, 5, 11, 0, 0, 90, 96, 5, 22, 0, 0, 91,
93, 5, 8, 0, 0, 92, 94, 3, 12, 6, 0, 93, 92, 1, 0, 0, 0, 93, 94, 1, 0,
0, 0, 94, 95, 1, 0, 0, 0, 95, 97, 3, 28, 14, 0, 96, 91, 1, 0, 0, 0, 96,
0, 0, 94, 95, 1, 0, 0, 0, 95, 97, 3, 20, 10, 0, 96, 91, 1, 0, 0, 0, 96,
97, 1, 0, 0, 0, 97, 98, 1, 0, 0, 0, 98, 99, 5, 12, 0, 0, 99, 102, 3, 30,
15, 0, 100, 101, 5, 9, 0, 0, 101, 103, 3, 28, 14, 0, 102, 100, 1, 0, 0,
0, 102, 103, 1, 0, 0, 0, 103, 11, 1, 0, 0, 0, 104, 105, 7, 0, 0, 0, 105,
@ -1364,7 +1364,7 @@ type ISelectStmtContext interface {
SetCount(antlr.Token)
// GetBucket returns the Bucket rule contexts.
GetBucket() IIdentContext
GetBucket() IFilterKeyContext
// GetFilter returns the Filter rule contexts.
GetFilter() IIdentWCContext
@ -1373,7 +1373,7 @@ type ISelectStmtContext interface {
GetName() IIdentContext
// SetBucket sets the Bucket rule contexts.
SetBucket(IIdentContext)
SetBucket(IFilterKeyContext)
// SetFilter sets the Filter rule contexts.
SetFilter(IIdentWCContext)
@ -1388,8 +1388,8 @@ type ISelectStmtContext interface {
IdentWC() IIdentWCContext
IN() antlr.TerminalNode
AS() antlr.TerminalNode
AllIdent() []IIdentContext
Ident(i int) IIdentContext
FilterKey() IFilterKeyContext
Ident() IIdentContext
Clause() IClauseContext
// IsSelectStmtContext differentiates from other interfaces.
@ -1400,7 +1400,7 @@ type SelectStmtContext struct {
antlr.BaseParserRuleContext
parser antlr.Parser
Count antlr.Token
Bucket IIdentContext
Bucket IFilterKeyContext
Filter IIdentWCContext
Name IIdentContext
}
@ -1436,13 +1436,13 @@ func (s *SelectStmtContext) GetCount() antlr.Token { return s.Count }
func (s *SelectStmtContext) SetCount(v antlr.Token) { s.Count = v }
func (s *SelectStmtContext) GetBucket() IIdentContext { return s.Bucket }
func (s *SelectStmtContext) GetBucket() IFilterKeyContext { return s.Bucket }
func (s *SelectStmtContext) GetFilter() IIdentWCContext { return s.Filter }
func (s *SelectStmtContext) GetName() IIdentContext { return s.Name }
func (s *SelectStmtContext) SetBucket(v IIdentContext) { s.Bucket = v }
func (s *SelectStmtContext) SetBucket(v IFilterKeyContext) { s.Bucket = v }
func (s *SelectStmtContext) SetFilter(v IIdentWCContext) { s.Filter = v }
@ -1484,37 +1484,28 @@ func (s *SelectStmtContext) AS() antlr.TerminalNode {
return s.GetToken(QueryAS, 0)
}
func (s *SelectStmtContext) AllIdent() []IIdentContext {
children := s.GetChildren()
len := 0
for _, ctx := range children {
if _, ok := ctx.(IIdentContext); ok {
len++
func (s *SelectStmtContext) FilterKey() IFilterKeyContext {
var t antlr.RuleContext
for _, ctx := range s.GetChildren() {
if _, ok := ctx.(IFilterKeyContext); ok {
t = ctx.(antlr.RuleContext)
break
}
}
tst := make([]IIdentContext, len)
i := 0
for _, ctx := range children {
if t, ok := ctx.(IIdentContext); ok {
tst[i] = t.(IIdentContext)
i++
}
if t == nil {
return nil
}
return tst
return t.(IFilterKeyContext)
}
func (s *SelectStmtContext) Ident(i int) IIdentContext {
func (s *SelectStmtContext) Ident() IIdentContext {
var t antlr.RuleContext
j := 0
for _, ctx := range s.GetChildren() {
if _, ok := ctx.(IIdentContext); ok {
if j == i {
t = ctx.(antlr.RuleContext)
break
}
j++
t = ctx.(antlr.RuleContext)
break
}
}
@ -1617,7 +1608,7 @@ func (p *Query) SelectStmt() (localctx ISelectStmtContext) {
{
p.SetState(95)
var _x = p.Ident()
var _x = p.FilterKey()
localctx.(*SelectStmtContext).Bucket = _x
}

2
netmap/parser/query_visitor.go
View file

@ -1,4 +1,4 @@
// Code generated from /repo/frostfs/sdk-go/netmap/parser/Query.g4 by ANTLR 4.13.0. DO NOT EDIT.
// Code generated from netmap/parser/Query.g4 by ANTLR 4.13.1. DO NOT EDIT.
package parser // Query

12
netmap/policy_test.go
View file

@ -82,6 +82,13 @@ CBF 1
SELECT 1 FROM Color
FILTER (Color EQ Red OR Color EQ Blue OR Color EQ Yellow) AND Color NE Green AS Color`,
},
{
name: "non-ascii attributes in SELECT IN",
input: `REP 1
CBF 1
SELECT 1 IN SAME 'Цвет' FROM Colorful
FILTER 'Цвет' EQ 'Красный' OR 'Цвет' EQ 'Синий' AS Colorful`,
},
}
for _, tc := range testCases {
@ -127,6 +134,11 @@ func TestDecodeSelectFilterExpr(t *testing.T) {
SELECT 1 FROM R
FILTER Color LIKE 'R' AS R
`,
`
CBF 1
SELECT 1 IN SAME 'Цвет' FROM Colorful
FILTER 'Цвет' EQ 'Красный' OR 'Цвет' EQ 'Синий' AS Colorful
`,
} {
_, err := DecodeSelectFilterString(s)
require.NoError(t, err)

62
netmap/selector_test.go
View file

@ -6,6 +6,7 @@ import (
"encoding/binary"
"fmt"
mrand "math/rand"
"reflect"
"slices"
"strconv"
"strings"
@ -13,6 +14,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap"
"git.frostfs.info/TrueCloudLab/hrw"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@ -542,6 +544,66 @@ func TestPlacementPolicy_ProcessSelectorsExceptForNodes(t *testing.T) {
}
}
func TestPlacementPolicy_NonAsciiAttributes(t *testing.T) {
p := newPlacementPolicy(
1,
[]ReplicaDescriptor{
newReplica(2, "Nodes"),
newReplica(2, "Nodes"),
},
[]Selector{
newSelector("Nodes", "Цвет", 2, "Colorful", (*Selector).SelectSame),
},
[]Filter{
newFilter("Colorful", "", "", netmap.OR,
newFilter("", "Цвет", "Красный", netmap.EQ),
newFilter("", "Цвет", "Синий", netmap.EQ),
),
},
)
p.SetUnique(true)
nodes := []NodeInfo{
nodeInfoFromAttributes("Цвет", "Красный", "Форма", "Треугольник"),
nodeInfoFromAttributes("Цвет", "Красный", "Форма", "Круг"),
nodeInfoFromAttributes("Цвет", "Синий", "Форма", "Треугольник"),
nodeInfoFromAttributes("Цвет", "Синий", "Форма", "Круг"),
nodeInfoFromAttributes("Свойство", "Мягкий", "Форма", "Треугольник"),
nodeInfoFromAttributes("Свойство", "Теплый", "Форма", "Круг"),
}
for i := range nodes {
nodes[i].SetPublicKey([]byte{byte(i)})
}
redNodes := nodes[:2]
blueNodes := nodes[2:4]
var nm NetMap
nm.SetNodes(nodes)
pivot := make([]byte, 42)
_, _ = rand.Read(pivot)
nodesPerReplica, err := nm.ContainerNodes(p, pivot)
require.NoError(t, err)
require.Len(t, nodesPerReplica, 2)
for i := range nodesPerReplica {
slices.SortFunc(nodesPerReplica[i], func(n1, n2 NodeInfo) int {
pk1, pk2 := string(n1.PublicKey()), string(n2.PublicKey())
return cmp.Compare(pk1, pk2)
})
}
redMatchFirst := reflect.DeepEqual(redNodes, nodesPerReplica[0])
blueMatchFirst := reflect.DeepEqual(blueNodes, nodesPerReplica[0])
redMatchSecond := reflect.DeepEqual(redNodes, nodesPerReplica[1])
blueMatchSecond := reflect.DeepEqual(blueNodes, nodesPerReplica[1])
assert.True(t, redMatchFirst && blueMatchSecond || blueMatchFirst && redMatchSecond)
}
func TestSelector_SetName(t *testing.T) {
const name = "some name"
var s Selector