TrueCloudLab/frostfs-sdk-go
33
0
Fork 21
Code Issues 34 Pull requests 2 Releases Activity Actions

Initial EC implementation #205

Merged
fyrchik merged 4 commits from fyrchik/frostfs-sdk-go:erasure-code into master 2024-03-22 10:14:13 +00:00
Conversation 0 Commits 4 Files changed 27 +728 -344
13 changed files with 728 additions and 344 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files
Showing only changes of commit 31ba724610 - Show all commits

26
netmap/netmap.go
View file

@ -209,6 +209,25 @@ func (m NetMap) SelectFilterNodes(expr *SelectFilterExpr) ([][]NodeInfo, error)
return ret, nil return ret, nil
} }
func countNodes(r netmap.Replica) uint32 {
if r.GetCount() != 0 {
return r.GetCount()
}
return r.GetECDataCount() + r.GetECParityCount()
}
func (p PlacementPolicy) isUnique() bool {
if p.unique {
return true
}
for _, r := range p.replicas {
if r.GetECDataCount() != 0 || r.GetECParityCount() != 0 {
return true
}
}
return false
}
// ContainerNodes returns two-dimensional list of nodes as a result of applying // ContainerNodes returns two-dimensional list of nodes as a result of applying
// given PlacementPolicy to the NetMap. Each line of the list corresponds to a // given PlacementPolicy to the NetMap. Each line of the list corresponds to a
// replica descriptor. Line order corresponds to order of ReplicaDescriptor list // replica descriptor. Line order corresponds to order of ReplicaDescriptor list
@ -230,6 +249,7 @@ func (m NetMap) ContainerNodes(p PlacementPolicy, pivot []byte) ([][]NodeInfo, e
return nil, err return nil, err
} }
unique := p.isUnique()
result := make([][]NodeInfo, len(p.replicas)) result := make([][]NodeInfo, len(p.replicas))
// Note that the cached selectors are not used when the policy contains the UNIQUE flag. // Note that the cached selectors are not used when the policy contains the UNIQUE flag.
@ -240,7 +260,7 @@ func (m NetMap) ContainerNodes(p PlacementPolicy, pivot []byte) ([][]NodeInfo, e
sName := p.replicas[i].GetSelector() sName := p.replicas[i].GetSelector()
if sName == "" && !(len(p.replicas) == 1 && len(p.selectors) == 1) { if sName == "" && !(len(p.replicas) == 1 && len(p.selectors) == 1) {
var s netmap.Selector var s netmap.Selector
s.SetCount(p.replicas[i].GetCount()) s.SetCount(countNodes(p.replicas[i]))
s.SetFilter(mainFilterName) s.SetFilter(mainFilterName)
nodes, err := c.getSelection(s) nodes, err := c.getSelection(s)
@ -250,14 +270,14 @@ func (m NetMap) ContainerNodes(p PlacementPolicy, pivot []byte) ([][]NodeInfo, e
result[i] = append(result[i], flattenNodes(nodes)...) result[i] = append(result[i], flattenNodes(nodes)...)
if p.unique { if unique {
c.addUsedNodes(result[i]...) c.addUsedNodes(result[i]...)
} }
continue continue
} }
if p.unique { if unique {
if c.processedSelectors[sName] == nil { if c.processedSelectors[sName] == nil {
return nil, fmt.Errorf("selector not found: '%s'", sName) return nil, fmt.Errorf("selector not found: '%s'", sName)
} }

6
netmap/parser/Query.g4
View file

@ -4,10 +4,14 @@ options {
tokenVocab = QueryLexer; tokenVocab = QueryLexer;
} }
policy: UNIQUE? repStmt+ cbfStmt? selectStmt* filterStmt* EOF; policy: UNIQUE? (repStmt | ecStmt)+ cbfStmt? selectStmt* filterStmt* EOF;
selectFilterExpr: cbfStmt? selectStmt? filterStmt* EOF; selectFilterExpr: cbfStmt? selectStmt? filterStmt* EOF;
ecStmt:
EC Data = NUMBER1 DOT Parity = NUMBER1 // erasure code configuration
(IN Selector = ident)?; // optional selector name
repStmt: repStmt:
REP Count = NUMBER1 // number of object replicas REP Count = NUMBER1 // number of object replicas
(IN Selector = ident)?; // optional selector name (IN Selector = ident)?; // optional selector name

BIN
netmap/parser/Query.interp
View file

Binary file not shown.

BIN
netmap/parser/Query.tokens
View file

Binary file not shown.

2
netmap/parser/QueryLexer.g4
View file

@ -7,6 +7,7 @@ SIMPLE_OP : 'EQ' | 'NE' | 'GE' | 'GT' | 'LT' | 'LE';
UNIQUE : 'UNIQUE'; UNIQUE : 'UNIQUE';
REP : 'REP'; REP : 'REP';
EC : 'EC';
IN : 'IN'; IN : 'IN';
AS : 'AS'; AS : 'AS';
CBF : 'CBF'; CBF : 'CBF';
@ -14,6 +15,7 @@ SELECT : 'SELECT';
FROM : 'FROM'; FROM : 'FROM';
FILTER : 'FILTER'; FILTER : 'FILTER';
WILDCARD : '*'; WILDCARD : '*';
DOT : '.';
CLAUSE_SAME : 'SAME'; CLAUSE_SAME : 'SAME';
CLAUSE_DISTINCT : 'DISTINCT'; CLAUSE_DISTINCT : 'DISTINCT';

BIN
netmap/parser/QueryLexer.interp
View file

Binary file not shown.

BIN
netmap/parser/QueryLexer.tokens
View file

Binary file not shown.

6
netmap/parser/query_base_visitor.go
View file

@ -1,4 +1,4 @@
// Code generated from Query.g4 by ANTLR 4.13.0. DO NOT EDIT. // Code generated from /repo/frostfs/sdk-go/netmap/parser/Query.g4 by ANTLR 4.13.0. DO NOT EDIT.
package parser // Query package parser // Query
@ -16,6 +16,10 @@ func (v *BaseQueryVisitor) VisitSelectFilterExpr(ctx *SelectFilterExprContext) i
return v.VisitChildren(ctx) return v.VisitChildren(ctx)
} }
func (v *BaseQueryVisitor) VisitEcStmt(ctx *EcStmtContext) interface{} {
return v.VisitChildren(ctx)
}
func (v *BaseQueryVisitor) VisitRepStmt(ctx *RepStmtContext) interface{} { func (v *BaseQueryVisitor) VisitRepStmt(ctx *RepStmtContext) interface{} {
return v.VisitChildren(ctx) return v.VisitChildren(ctx)
} }

242
netmap/parser/query_lexer.go
View file

@ -1,4 +1,4 @@
// Code generated from QueryLexer.g4 by ANTLR 4.13.0. DO NOT EDIT. // Code generated from /repo/frostfs/sdk-go/netmap/parser/QueryLexer.g4 by ANTLR 4.13.0. DO NOT EDIT.
package parser package parser
@ -43,119 +43,123 @@ func querylexerLexerInit() {
"DEFAULT_MODE", "DEFAULT_MODE",
} }
staticData.LiteralNames = []string{ staticData.LiteralNames = []string{
"", "'NOT'", "'AND'", "'OR'", "", "'UNIQUE'", "'REP'", "'IN'", "'AS'", "", "'NOT'", "'AND'", "'OR'", "", "'UNIQUE'", "'REP'", "'EC'", "'IN'",
"'CBF'", "'SELECT'", "'FROM'", "'FILTER'", "'*'", "'SAME'", "'DISTINCT'", "'AS'", "'CBF'", "'SELECT'", "'FROM'", "'FILTER'", "'*'", "'.'", "'SAME'",
"'('", "')'", "'@'", "", "", "'0'", "'DISTINCT'", "'('", "')'", "'@'", "", "", "'0'",
} }
staticData.SymbolicNames = []string{ staticData.SymbolicNames = []string{
"", "NOT_OP", "AND_OP", "OR_OP", "SIMPLE_OP", "UNIQUE", "REP", "IN", "", "NOT_OP", "AND_OP", "OR_OP", "SIMPLE_OP", "UNIQUE", "REP", "EC",
"AS", "CBF", "SELECT", "FROM", "FILTER", "WILDCARD", "CLAUSE_SAME", "IN", "AS", "CBF", "SELECT", "FROM", "FILTER", "WILDCARD", "DOT", "CLAUSE_SAME",
"CLAUSE_DISTINCT", "L_PAREN", "R_PAREN", "AT", "IDENT", "NUMBER1", "ZERO", "CLAUSE_DISTINCT", "L_PAREN", "R_PAREN", "AT", "IDENT", "NUMBER1", "ZERO",
"STRING", "WS", "STRING", "WS",
} }
staticData.RuleNames = []string{ staticData.RuleNames = []string{
"NOT_OP", "AND_OP", "OR_OP", "SIMPLE_OP", "UNIQUE", "REP", "IN", "AS", "NOT_OP", "AND_OP", "OR_OP", "SIMPLE_OP", "UNIQUE", "REP", "EC", "IN",
"CBF", "SELECT", "FROM", "FILTER", "WILDCARD", "CLAUSE_SAME", "CLAUSE_DISTINCT", "AS", "CBF", "SELECT", "FROM", "FILTER", "WILDCARD", "DOT", "CLAUSE_SAME",
"L_PAREN", "R_PAREN", "AT", "IDENT", "Digit", "Nondigit", "NUMBER1", "CLAUSE_DISTINCT", "L_PAREN", "R_PAREN", "AT", "IDENT", "Digit", "Nondigit",
"ZERO", "STRING", "ESC", "UNICODE", "HEX", "SAFECODEPOINTSINGLE", "SAFECODEPOINTDOUBLE", "NUMBER1", "ZERO", "STRING", "ESC", "UNICODE", "HEX", "SAFECODEPOINTSINGLE",
"WS", "SAFECODEPOINTDOUBLE", "WS",
} }
staticData.PredictionContextCache = antlr.NewPredictionContextCache() staticData.PredictionContextCache = antlr.NewPredictionContextCache()
staticData.serializedATN = []int32{ staticData.serializedATN = []int32{
4, 0, 23, 213, 6, -1, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 0, 25, 222, 6, -1, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2,
4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2,
10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15,
7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7,
20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25,
2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 1, 0, 1, 0, 1, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2,
0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 2, 1, 3, 1, 3, 1, 3, 1, 31, 7, 31, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2,
3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 3, 3, 85, 8, 3, 1, 4, 1, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3,
1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 6, 1, 6, 1, 3, 3, 3, 89, 8, 3, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 5, 1,
1, 6, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 9, 1, 9, 1, 9, 1, 9, 5, 1, 5, 1, 5, 1, 6, 1, 6, 1, 6, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1,
1, 9, 1, 9, 1, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 11, 1, 11, 1, 11, 9, 1, 9, 1, 9, 1, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1,
1, 11, 1, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 1, 13, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12,
13, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 15, 1, 12, 1, 13, 1, 13, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1,
1, 15, 1, 16, 1, 16, 1, 17, 1, 17, 1, 18, 1, 18, 1, 18, 5, 18, 152, 8, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 17, 1, 17,
18, 10, 18, 12, 18, 155, 9, 18, 1, 19, 1, 19, 1, 20, 1, 20, 1, 21, 1, 21, 1, 18, 1, 18, 1, 19, 1, 19, 1, 20, 1, 20, 1, 20, 5, 20, 161, 8, 20, 10,
5, 21, 163, 8, 21, 10, 21, 12, 21, 166, 9, 21, 1, 22, 1, 22, 1, 23, 1, 20, 12, 20, 164, 9, 20, 1, 21, 1, 21, 1, 22, 1, 22, 1, 23, 1, 23, 5, 23,
23, 1, 23, 5, 23, 173, 8, 23, 10, 23, 12, 23, 176, 9, 23, 1, 23, 1, 23, 172, 8, 23, 10, 23, 12, 23, 175, 9, 23, 1, 24, 1, 24, 1, 25, 1, 25, 1,
1, 23, 1, 23, 5, 23, 182, 8, 23, 10, 23, 12, 23, 185, 9, 23, 1, 23, 3, 25, 5, 25, 182, 8, 25, 10, 25, 12, 25, 185, 9, 25, 1, 25, 1, 25, 1, 25,
23, 188, 8, 23, 1, 24, 1, 24, 1, 24, 3, 24, 193, 8, 24, 1, 25, 1, 25, 1, 1, 25, 5, 25, 191, 8, 25, 10, 25, 12, 25, 194, 9, 25, 1, 25, 3, 25, 197,
25, 1, 25, 1, 25, 1, 25, 1, 26, 1, 26, 1, 27, 1, 27, 1, 28, 1, 28, 1, 29, 8, 25, 1, 26, 1, 26, 1, 26, 3, 26, 202, 8, 26, 1, 27, 1, 27, 1, 27, 1,
4, 29, 208, 8, 29, 11, 29, 12, 29, 209, 1, 29, 1, 29, 0, 0, 30, 1, 1, 3, 27, 1, 27, 1, 27, 1, 28, 1, 28, 1, 29, 1, 29, 1, 30, 1, 30, 1, 31, 4, 31,
2, 5, 3, 7, 4, 9, 5, 11, 6, 13, 7, 15, 8, 17, 9, 19, 10, 21, 11, 23, 12, 217, 8, 31, 11, 31, 12, 31, 218, 1, 31, 1, 31, 0, 0, 32, 1, 1, 3, 2, 5,
25, 13, 27, 14, 29, 15, 31, 16, 33, 17, 35, 18, 37, 19, 39, 0, 41, 0, 43, 3, 7, 4, 9, 5, 11, 6, 13, 7, 15, 8, 17, 9, 19, 10, 21, 11, 23, 12, 25,
20, 45, 21, 47, 22, 49, 0, 51, 0, 53, 0, 55, 0, 57, 0, 59, 23, 1, 0, 8, 13, 27, 14, 29, 15, 31, 16, 33, 17, 35, 18, 37, 19, 39, 20, 41, 21, 43,
1, 0, 48, 57, 3, 0, 65, 90, 95, 95, 97, 122, 1, 0, 49, 57, 9, 0, 34, 34, 0, 45, 0, 47, 22, 49, 23, 51, 24, 53, 0, 55, 0, 57, 0, 59, 0, 61, 0, 63,
39, 39, 47, 47, 92, 92, 98, 98, 102, 102, 110, 110, 114, 114, 116, 116, 25, 1, 0, 8, 1, 0, 48, 57, 3, 0, 65, 90, 95, 95, 97, 122, 1, 0, 49, 57,
3, 0, 48, 57, 65, 70, 97, 102, 3, 0, 0, 31, 39, 39, 92, 92, 3, 0, 0, 31, 9, 0, 34, 34, 39, 39, 47, 47, 92, 92, 98, 98, 102, 102, 110, 110, 114,
34, 34, 92, 92, 3, 0, 9, 10, 13, 13, 32, 32, 220, 0, 1, 1, 0, 0, 0, 0, 114, 116, 116, 3, 0, 48, 57, 65, 70, 97, 102, 3, 0, 0, 31, 39, 39, 92,
3, 1, 0, 0, 0, 0, 5, 1, 0, 0, 0, 0, 7, 1, 0, 0, 0, 0, 9, 1, 0, 0, 0, 0, 92, 3, 0, 0, 31, 34, 34, 92, 92, 3, 0, 9, 10, 13, 13, 32, 32, 229, 0, 1,
11, 1, 0, 0, 0, 0, 13, 1, 0, 0, 0, 0, 15, 1, 0, 0, 0, 0, 17, 1, 0, 0, 0, 1, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 5, 1, 0, 0, 0, 0, 7, 1, 0, 0, 0, 0, 9,
0, 19, 1, 0, 0, 0, 0, 21, 1, 0, 0, 0, 0, 23, 1, 0, 0, 0, 0, 25, 1, 0, 0, 1, 0, 0, 0, 0, 11, 1, 0, 0, 0, 0, 13, 1, 0, 0, 0, 0, 15, 1, 0, 0, 0, 0,
0, 0, 27, 1, 0, 0, 0, 0, 29, 1, 0, 0, 0, 0, 31, 1, 0, 0, 0, 0, 33, 1, 0, 17, 1, 0, 0, 0, 0, 19, 1, 0, 0, 0, 0, 21, 1, 0, 0, 0, 0, 23, 1, 0, 0, 0,
0, 0, 0, 35, 1, 0, 0, 0, 0, 37, 1, 0, 0, 0, 0, 43, 1, 0, 0, 0, 0, 45, 1, 0, 25, 1, 0, 0, 0, 0, 27, 1, 0, 0, 0, 0, 29, 1, 0, 0, 0, 0, 31, 1, 0, 0,
0, 0, 0, 0, 47, 1, 0, 0, 0, 0, 59, 1, 0, 0, 0, 1, 61, 1, 0, 0, 0, 3, 65, 0, 0, 33, 1, 0, 0, 0, 0, 35, 1, 0, 0, 0, 0, 37, 1, 0, 0, 0, 0, 39, 1, 0,
1, 0, 0, 0, 5, 69, 1, 0, 0, 0, 7, 84, 1, 0, 0, 0, 9, 86, 1, 0, 0, 0, 11, 0, 0, 0, 41, 1, 0, 0, 0, 0, 47, 1, 0, 0, 0, 0, 49, 1, 0, 0, 0, 0, 51, 1,
93, 1, 0, 0, 0, 13, 97, 1, 0, 0, 0, 15, 100, 1, 0, 0, 0, 17, 103, 1, 0, 0, 0, 0, 0, 63, 1, 0, 0, 0, 1, 65, 1, 0, 0, 0, 3, 69, 1, 0, 0, 0, 5, 73,
0, 0, 19, 107, 1, 0, 0, 0, 21, 114, 1, 0, 0, 0, 23, 119, 1, 0, 0, 0, 25, 1, 0, 0, 0, 7, 88, 1, 0, 0, 0, 9, 90, 1, 0, 0, 0, 11, 97, 1, 0, 0, 0, 13,
126, 1, 0, 0, 0, 27, 128, 1, 0, 0, 0, 29, 133, 1, 0, 0, 0, 31, 142, 1, 101, 1, 0, 0, 0, 15, 104, 1, 0, 0, 0, 17, 107, 1, 0, 0, 0, 19, 110, 1,
0, 0, 0, 33, 144, 1, 0, 0, 0, 35, 146, 1, 0, 0, 0, 37, 148, 1, 0, 0, 0, 0, 0, 0, 21, 114, 1, 0, 0, 0, 23, 121, 1, 0, 0, 0, 25, 126, 1, 0, 0, 0,
39, 156, 1, 0, 0, 0, 41, 158, 1, 0, 0, 0, 43, 160, 1, 0, 0, 0, 45, 167, 27, 133, 1, 0, 0, 0, 29, 135, 1, 0, 0, 0, 31, 137, 1, 0, 0, 0, 33, 142,
1, 0, 0, 0, 47, 187, 1, 0, 0, 0, 49, 189, 1, 0, 0, 0, 51, 194, 1, 0, 0, 1, 0, 0, 0, 35, 151, 1, 0, 0, 0, 37, 153, 1, 0, 0, 0, 39, 155, 1, 0, 0,
0, 53, 200, 1, 0, 0, 0, 55, 202, 1, 0, 0, 0, 57, 204, 1, 0, 0, 0, 59, 207, 0, 41, 157, 1, 0, 0, 0, 43, 165, 1, 0, 0, 0, 45, 167, 1, 0, 0, 0, 47, 169,
1, 0, 0, 0, 61, 62, 5, 78, 0, 0, 62, 63, 5, 79, 0, 0, 63, 64, 5, 84, 0, 1, 0, 0, 0, 49, 176, 1, 0, 0, 0, 51, 196, 1, 0, 0, 0, 53, 198, 1, 0, 0,
0, 64, 2, 1, 0, 0, 0, 65, 66, 5, 65, 0, 0, 66, 67, 5, 78, 0, 0, 67, 68, 0, 55, 203, 1, 0, 0, 0, 57, 209, 1, 0, 0, 0, 59, 211, 1, 0, 0, 0, 61, 213,
5, 68, 0, 0, 68, 4, 1, 0, 0, 0, 69, 70, 5, 79, 0, 0, 70, 71, 5, 82, 0, 1, 0, 0, 0, 63, 216, 1, 0, 0, 0, 65, 66, 5, 78, 0, 0, 66, 67, 5, 79, 0,
0, 71, 6, 1, 0, 0, 0, 72, 73, 5, 69, 0, 0, 73, 85, 5, 81, 0, 0, 74, 75, 0, 67, 68, 5, 84, 0, 0, 68, 2, 1, 0, 0, 0, 69, 70, 5, 65, 0, 0, 70, 71,
5, 78, 0, 0, 75, 85, 5, 69, 0, 0, 76, 77, 5, 71, 0, 0, 77, 85, 5, 69, 0, 5, 78, 0, 0, 71, 72, 5, 68, 0, 0, 72, 4, 1, 0, 0, 0, 73, 74, 5, 79, 0,
0, 78, 79, 5, 71, 0, 0, 79, 85, 5, 84, 0, 0, 80, 81, 5, 76, 0, 0, 81, 85, 0, 74, 75, 5, 82, 0, 0, 75, 6, 1, 0, 0, 0, 76, 77, 5, 69, 0, 0, 77, 89,
5, 84, 0, 0, 82, 83, 5, 76, 0, 0, 83, 85, 5, 69, 0, 0, 84, 72, 1, 0, 0, 5, 81, 0, 0, 78, 79, 5, 78, 0, 0, 79, 89, 5, 69, 0, 0, 80, 81, 5, 71, 0,
0, 84, 74, 1, 0, 0, 0, 84, 76, 1, 0, 0, 0, 84, 78, 1, 0, 0, 0, 84, 80, 0, 81, 89, 5, 69, 0, 0, 82, 83, 5, 71, 0, 0, 83, 89, 5, 84, 0, 0, 84, 85,
1, 0, 0, 0, 84, 82, 1, 0, 0, 0, 85, 8, 1, 0, 0, 0, 86, 87, 5, 85, 0, 0, 5, 76, 0, 0, 85, 89, 5, 84, 0, 0, 86, 87, 5, 76, 0, 0, 87, 89, 5, 69, 0,
87, 88, 5, 78, 0, 0, 88, 89, 5, 73, 0, 0, 89, 90, 5, 81, 0, 0, 90, 91, 0, 88, 76, 1, 0, 0, 0, 88, 78, 1, 0, 0, 0, 88, 80, 1, 0, 0, 0, 88, 82,
5, 85, 0, 0, 91, 92, 5, 69, 0, 0, 92, 10, 1, 0, 0, 0, 93, 94, 5, 82, 0, 1, 0, 0, 0, 88, 84, 1, 0, 0, 0, 88, 86, 1, 0, 0, 0, 89, 8, 1, 0, 0, 0,
0, 94, 95, 5, 69, 0, 0, 95, 96, 5, 80, 0, 0, 96, 12, 1, 0, 0, 0, 97, 98, 90, 91, 5, 85, 0, 0, 91, 92, 5, 78, 0, 0, 92, 93, 5, 73, 0, 0, 93, 94,
5, 73, 0, 0, 98, 99, 5, 78, 0, 0, 99, 14, 1, 0, 0, 0, 100, 101, 5, 65, 5, 81, 0, 0, 94, 95, 5, 85, 0, 0, 95, 96, 5, 69, 0, 0, 96, 10, 1, 0, 0,
0, 0, 101, 102, 5, 83, 0, 0, 102, 16, 1, 0, 0, 0, 103, 104, 5, 67, 0, 0, 0, 97, 98, 5, 82, 0, 0, 98, 99, 5, 69, 0, 0, 99, 100, 5, 80, 0, 0, 100,
104, 105, 5, 66, 0, 0, 105, 106, 5, 70, 0, 0, 106, 18, 1, 0, 0, 0, 107, 12, 1, 0, 0, 0, 101, 102, 5, 69, 0, 0, 102, 103, 5, 67, 0, 0, 103, 14,
108, 5, 83, 0, 0, 108, 109, 5, 69, 0, 0, 109, 110, 5, 76, 0, 0, 110, 111, 1, 0, 0, 0, 104, 105, 5, 73, 0, 0, 105, 106, 5, 78, 0, 0, 106, 16, 1, 0,
5, 69, 0, 0, 111, 112, 5, 67, 0, 0, 112, 113, 5, 84, 0, 0, 113, 20, 1, 0, 0, 107, 108, 5, 65, 0, 0, 108, 109, 5, 83, 0, 0, 109, 18, 1, 0, 0, 0,
0, 0, 0, 114, 115, 5, 70, 0, 0, 115, 116, 5, 82, 0, 0, 116, 117, 5, 79, 110, 111, 5, 67, 0, 0, 111, 112, 5, 66, 0, 0, 112, 113, 5, 70, 0, 0, 113,
0, 0, 117, 118, 5, 77, 0, 0, 118, 22, 1, 0, 0, 0, 119, 120, 5, 70, 0, 0, 20, 1, 0, 0, 0, 114, 115, 5, 83, 0, 0, 115, 116, 5, 69, 0, 0, 116, 117,
120, 121, 5, 73, 0, 0, 121, 122, 5, 76, 0, 0, 122, 123, 5, 84, 0, 0, 123, 5, 76, 0, 0, 117, 118, 5, 69, 0, 0, 118, 119, 5, 67, 0, 0, 119, 120, 5,
124, 5, 69, 0, 0, 124, 125, 5, 82, 0, 0, 125, 24, 1, 0, 0, 0, 126, 127, 84, 0, 0, 120, 22, 1, 0, 0, 0, 121, 122, 5, 70, 0, 0, 122, 123, 5, 82,
5, 42, 0, 0, 127, 26, 1, 0, 0, 0, 128, 129, 5, 83, 0, 0, 129, 130, 5, 65, 0, 0, 123, 124, 5, 79, 0, 0, 124, 125, 5, 77, 0, 0, 125, 24, 1, 0, 0, 0,
0, 0, 130, 131, 5, 77, 0, 0, 131, 132, 5, 69, 0, 0, 132, 28, 1, 0, 0, 0, 126, 127, 5, 70, 0, 0, 127, 128, 5, 73, 0, 0, 128, 129, 5, 76, 0, 0, 129,
133, 134, 5, 68, 0, 0, 134, 135, 5, 73, 0, 0, 135, 136, 5, 83, 0, 0, 136, 130, 5, 84, 0, 0, 130, 131, 5, 69, 0, 0, 131, 132, 5, 82, 0, 0, 132, 26,
137, 5, 84, 0, 0, 137, 138, 5, 73, 0, 0, 138, 139, 5, 78, 0, 0, 139, 140, 1, 0, 0, 0, 133, 134, 5, 42, 0, 0, 134, 28, 1, 0, 0, 0, 135, 136, 5, 46,
5, 67, 0, 0, 140, 141, 5, 84, 0, 0, 141, 30, 1, 0, 0, 0, 142, 143, 5, 40, 0, 0, 136, 30, 1, 0, 0, 0, 137, 138, 5, 83, 0, 0, 138, 139, 5, 65, 0, 0,
0, 0, 143, 32, 1, 0, 0, 0, 144, 145, 5, 41, 0, 0, 145, 34, 1, 0, 0, 0, 139, 140, 5, 77, 0, 0, 140, 141, 5, 69, 0, 0, 141, 32, 1, 0, 0, 0, 142,
146, 147, 5, 64, 0, 0, 147, 36, 1, 0, 0, 0, 148, 153, 3, 41, 20, 0, 149, 143, 5, 68, 0, 0, 143, 144, 5, 73, 0, 0, 144, 145, 5, 83, 0, 0, 145, 146,
152, 3, 39, 19, 0, 150, 152, 3, 41, 20, 0, 151, 149, 1, 0, 0, 0, 151, 150, 5, 84, 0, 0, 146, 147, 5, 73, 0, 0, 147, 148, 5, 78, 0, 0, 148, 149, 5,
1, 0, 0, 0, 152, 155, 1, 0, 0, 0, 153, 151, 1, 0, 0, 0, 153, 154, 1, 0, 67, 0, 0, 149, 150, 5, 84, 0, 0, 150, 34, 1, 0, 0, 0, 151, 152, 5, 40,
0, 0, 154, 38, 1, 0, 0, 0, 155, 153, 1, 0, 0, 0, 156, 157, 7, 0, 0, 0, 0, 0, 152, 36, 1, 0, 0, 0, 153, 154, 5, 41, 0, 0, 154, 38, 1, 0, 0, 0,
157, 40, 1, 0, 0, 0, 158, 159, 7, 1, 0, 0, 159, 42, 1, 0, 0, 0, 160, 164, 155, 156, 5, 64, 0, 0, 156, 40, 1, 0, 0, 0, 157, 162, 3, 45, 22, 0, 158,
7, 2, 0, 0, 161, 163, 3, 39, 19, 0, 162, 161, 1, 0, 0, 0, 163, 166, 1, 161, 3, 43, 21, 0, 159, 161, 3, 45, 22, 0, 160, 158, 1, 0, 0, 0, 160, 159,
0, 0, 0, 164, 162, 1, 0, 0, 0, 164, 165, 1, 0, 0, 0, 165, 44, 1, 0, 0, 1, 0, 0, 0, 161, 164, 1, 0, 0, 0, 162, 160, 1, 0, 0, 0, 162, 163, 1, 0,
0, 166, 164, 1, 0, 0, 0, 167, 168, 5, 48, 0, 0, 168, 46, 1, 0, 0, 0, 169, 0, 0, 163, 42, 1, 0, 0, 0, 164, 162, 1, 0, 0, 0, 165, 166, 7, 0, 0, 0,
174, 5, 34, 0, 0, 170, 173, 3, 49, 24, 0, 171, 173, 3, 57, 28, 0, 172, 166, 44, 1, 0, 0, 0, 167, 168, 7, 1, 0, 0, 168, 46, 1, 0, 0, 0, 169, 173,
170, 1, 0, 0, 0, 172, 171, 1, 0, 0, 0, 173, 176, 1, 0, 0, 0, 174, 172, 7, 2, 0, 0, 170, 172, 3, 43, 21, 0, 171, 170, 1, 0, 0, 0, 172, 175, 1,
1, 0, 0, 0, 174, 175, 1, 0, 0, 0, 175, 177, 1, 0, 0, 0, 176, 174, 1, 0, 0, 0, 0, 173, 171, 1, 0, 0, 0, 173, 174, 1, 0, 0, 0, 174, 48, 1, 0, 0,
0, 0, 177, 188, 5, 34, 0, 0, 178, 183, 5, 39, 0, 0, 179, 182, 3, 49, 24, 0, 175, 173, 1, 0, 0, 0, 176, 177, 5, 48, 0, 0, 177, 50, 1, 0, 0, 0, 178,
0, 180, 182, 3, 55, 27, 0, 181, 179, 1, 0, 0, 0, 181, 180, 1, 0, 0, 0, 183, 5, 34, 0, 0, 179, 182, 3, 53, 26, 0, 180, 182, 3, 61, 30, 0, 181,
182, 185, 1, 0, 0, 0, 183, 181, 1, 0, 0, 0, 183, 184, 1, 0, 0, 0, 184, 179, 1, 0, 0, 0, 181, 180, 1, 0, 0, 0, 182, 185, 1, 0, 0, 0, 183, 181,
186, 1, 0, 0, 0, 185, 183, 1, 0, 0, 0, 186, 188, 5, 39, 0, 0, 187, 169, 1, 0, 0, 0, 183, 184, 1, 0, 0, 0, 184, 186, 1, 0, 0, 0, 185, 183, 1, 0,
1, 0, 0, 0, 187, 178, 1, 0, 0, 0, 188, 48, 1, 0, 0, 0, 189, 192, 5, 92, 0, 0, 186, 197, 5, 34, 0, 0, 187, 192, 5, 39, 0, 0, 188, 191, 3, 53, 26,
0, 0, 190, 193, 7, 3, 0, 0, 191, 193, 3, 51, 25, 0, 192, 190, 1, 0, 0, 0, 189, 191, 3, 59, 29, 0, 190, 188, 1, 0, 0, 0, 190, 189, 1, 0, 0, 0,
0, 192, 191, 1, 0, 0, 0, 193, 50, 1, 0, 0, 0, 194, 195, 5, 117, 0, 0, 195, 191, 194, 1, 0, 0, 0, 192, 190, 1, 0, 0, 0, 192, 193, 1, 0, 0, 0, 193,
196, 3, 53, 26, 0, 196, 197, 3, 53, 26, 0, 197, 198, 3, 53, 26, 0, 198, 195, 1, 0, 0, 0, 194, 192, 1, 0, 0, 0, 195, 197, 5, 39, 0, 0, 196, 178,
199, 3, 53, 26, 0, 199, 52, 1, 0, 0, 0, 200, 201, 7, 4, 0, 0, 201, 54, 1, 0, 0, 0, 196, 187, 1, 0, 0, 0, 197, 52, 1, 0, 0, 0, 198, 201, 5, 92,
1, 0, 0, 0, 202, 203, 8, 5, 0, 0, 203, 56, 1, 0, 0, 0, 204, 205, 8, 6, 0, 0, 199, 202, 7, 3, 0, 0, 200, 202, 3, 55, 27, 0, 201, 199, 1, 0, 0,
0, 0, 205, 58, 1, 0, 0, 0, 206, 208, 7, 7, 0, 0, 207, 206, 1, 0, 0, 0, 0, 201, 200, 1, 0, 0, 0, 202, 54, 1, 0, 0, 0, 203, 204, 5, 117, 0, 0, 204,
208, 209, 1, 0, 0, 0, 209, 207, 1, 0, 0, 0, 209, 210, 1, 0, 0, 0, 210, 205, 3, 57, 28, 0, 205, 206, 3, 57, 28, 0, 206, 207, 3, 57, 28, 0, 207,
211, 1, 0, 0, 0, 211, 212, 6, 29, 0, 0, 212, 60, 1, 0, 0, 0, 12, 0, 84, 208, 3, 57, 28, 0, 208, 56, 1, 0, 0, 0, 209, 210, 7, 4, 0, 0, 210, 58,
151, 153, 164, 172, 174, 181, 183, 187, 192, 209, 1, 6, 0, 0, 1, 0, 0, 0, 211, 212, 8, 5, 0, 0, 212, 60, 1, 0, 0, 0, 213, 214, 8, 6,
0, 0, 214, 62, 1, 0, 0, 0, 215, 217, 7, 7, 0, 0, 216, 215, 1, 0, 0, 0,
217, 218, 1, 0, 0, 0, 218, 216, 1, 0, 0, 0, 218, 219, 1, 0, 0, 0, 219,
220, 1, 0, 0, 0, 220, 221, 6, 31, 0, 0, 221, 64, 1, 0, 0, 0, 12, 0, 88,
160, 162, 173, 181, 183, 190, 192, 196, 201, 218, 1, 6, 0, 0,
} }
deserializer := antlr.NewATNDeserializer(nil) deserializer := antlr.NewATNDeserializer(nil)
staticData.atn = deserializer.Deserialize(staticData.serializedATN) staticData.atn = deserializer.Deserialize(staticData.serializedATN)
@ -202,21 +206,23 @@ const (
QueryLexerSIMPLE_OP = 4 QueryLexerSIMPLE_OP = 4
QueryLexerUNIQUE = 5 QueryLexerUNIQUE = 5
QueryLexerREP = 6 QueryLexerREP = 6
QueryLexerIN = 7 QueryLexerEC = 7
QueryLexerAS = 8 QueryLexerIN = 8
QueryLexerCBF = 9 QueryLexerAS = 9
QueryLexerSELECT = 10 QueryLexerCBF = 10
QueryLexerFROM = 11 QueryLexerSELECT = 11
QueryLexerFILTER = 12 QueryLexerFROM = 12
QueryLexerWILDCARD = 13 QueryLexerFILTER = 13
QueryLexerCLAUSE_SAME = 14 QueryLexerWILDCARD = 14
QueryLexerCLAUSE_DISTINCT = 15 QueryLexerDOT = 15
QueryLexerL_PAREN = 16 QueryLexerCLAUSE_SAME = 16
QueryLexerR_PAREN = 17 QueryLexerCLAUSE_DISTINCT = 17
QueryLexerAT = 18 QueryLexerL_PAREN = 18
QueryLexerIDENT = 19 QueryLexerR_PAREN = 19
QueryLexerNUMBER1 = 20 QueryLexerAT = 20
QueryLexerZERO = 21 QueryLexerIDENT = 21
QueryLexerSTRING = 22 QueryLexerNUMBER1 = 22
QueryLexerWS = 23 QueryLexerZERO = 23
QueryLexerSTRING = 24
QueryLexerWS = 25
) )

705
netmap/parser/query_parser.go
View file

File diff suppressed because it is too large Load diff

5
netmap/parser/query_visitor.go
View file

@ -1,4 +1,4 @@
// Code generated from Query.g4 by ANTLR 4.13.0. DO NOT EDIT. // Code generated from /repo/frostfs/sdk-go/netmap/parser/Query.g4 by ANTLR 4.13.0. DO NOT EDIT.
package parser // Query package parser // Query
@ -14,6 +14,9 @@ type QueryVisitor interface {
// Visit a parse tree produced by Query#selectFilterExpr. // Visit a parse tree produced by Query#selectFilterExpr.
VisitSelectFilterExpr(ctx *SelectFilterExprContext) interface{} VisitSelectFilterExpr(ctx *SelectFilterExprContext) interface{}
// Visit a parse tree produced by Query#ecStmt.
VisitEcStmt(ctx *EcStmtContext) interface{}
// Visit a parse tree produced by Query#repStmt. // Visit a parse tree produced by Query#repStmt.
VisitRepStmt(ctx *RepStmtContext) interface{} VisitRepStmt(ctx *RepStmtContext) interface{}

70
netmap/policy.go
View file

@ -34,9 +34,18 @@ type PlacementPolicy struct {
func (p *PlacementPolicy) readFromV2(m netmap.PlacementPolicy, checkFieldPresence bool) error { func (p *PlacementPolicy) readFromV2(m netmap.PlacementPolicy, checkFieldPresence bool) error {
p.replicas = m.GetReplicas() p.replicas = m.GetReplicas()
if checkFieldPresence && len(p.replicas) == 0 { if checkFieldPresence {
if len(p.replicas) == 0 {
return errors.New("missing replicas") return errors.New("missing replicas")
} }
if len(p.replicas) != 1 {
for i := range p.replicas {
if p.replicas[i].GetECDataCount() != 0 || p.replicas[i].GetECParityCount() != 0 {
return errors.New("erasure code group must be used exclusively")
}
}
}
}
p.backupFactor = m.GetContainerBackupFactor() p.backupFactor = m.GetContainerBackupFactor()
p.selectors = m.GetSelectors() p.selectors = m.GetSelectors()
@ -393,10 +402,14 @@ func (p PlacementPolicy) WriteStringTo(w io.StringWriter) (err error) {
c := p.replicas[i].GetCount() c := p.replicas[i].GetCount()
s := p.replicas[i].GetSelector() s := p.replicas[i].GetSelector()
if s != "" { if c != 0 {
_, err = w.WriteString(fmt.Sprintf("%sREP %d IN %s", delim, c, s))
} else {
_, err = w.WriteString(fmt.Sprintf("%sREP %d", delim, c)) _, err = w.WriteString(fmt.Sprintf("%sREP %d", delim, c))
} else {
ecx, ecy := p.replicas[i].GetECDataCount(), p.replicas[i].GetECParityCount()
_, err = w.WriteString(fmt.Sprintf("%sEC %d.%d", delim, ecx, ecy))
}
if s != "" {
_, err = w.WriteString(fmt.Sprintf(" IN %s", s))
} }
if err != nil { if err != nil {
@ -630,6 +643,8 @@ var (
"make sure to pair REP and SELECT clauses: \"REP .. IN X\" + \"SELECT ... AS X\"") "make sure to pair REP and SELECT clauses: \"REP .. IN X\" + \"SELECT ... AS X\"")
// errRedundantSelector is returned for errors found by filters policy validator. // errRedundantSelector is returned for errors found by filters policy validator.
errRedundantFilter = errors.New("policy: found redundant filter") errRedundantFilter = errors.New("policy: found redundant filter")
// errECFewSelectors is returned when EC keyword is used without UNIQUE keyword.
errECFewSelectors = errors.New("policy: too few nodes to select")
) )
type policyVisitor struct { type policyVisitor struct {
@ -657,11 +672,18 @@ func (p *policyVisitor) VisitPolicy(ctx *parser.PolicyContext) any {
pl.unique = ctx.UNIQUE() != nil pl.unique = ctx.UNIQUE() != nil
repStmts := ctx.AllRepStmt() stmts := ctx.GetChildren()
pl.replicas = make([]netmap.Replica, 0, len(repStmts)) for _, r := range stmts {
var res *netmap.Replica
for _, r := range repStmts { var ok bool
res, ok := r.Accept(p).(*netmap.Replica) switch r := r.(type) {
case parser.IRepStmtContext:
res, ok = r.Accept(p).(*netmap.Replica)
case parser.IEcStmtContext:
res, ok = r.Accept(p).(*netmap.Replica)
default:
continue
dstepanov-yadro commented 2024-03-06 12:38:16 +00:00
Outdated
Review
Copy link

This means current statement will be skipped. Why?

This means current statement will be skipped. Why?
fyrchik commented 2024-03-06 12:45:51 +00:00
Outdated
Review
Copy link

We are interested only in EC and REP statements here and in their order.
Previously it was AllRepStmt, now we get all children and filter all non-REP and non-EC statements, they will be processed below.

We are interested only in EC and REP statements here and in their _order_. Previously it was `AllRepStmt`, now we get all children and filter all non-REP and non-EC statements, they will be processed below.
}
if !ok { if !ok {
return nil return nil
} }
@ -758,6 +780,28 @@ func (p *policyVisitor) VisitRepStmt(ctx *parser.RepStmtContext) any {
return rs return rs
} }
// VisitRepStmt implements parser.QueryVisitor interface.
func (p *policyVisitor) VisitEcStmt(ctx *parser.EcStmtContext) any {
dataCount, err := strconv.ParseUint(ctx.GetData().GetText(), 10, 32)
if err != nil {
return p.reportError(errInvalidNumber)
}
parityCount, err := strconv.ParseUint(ctx.GetParity().GetText(), 10, 32)
if err != nil {
return p.reportError(errInvalidNumber)
}
rs := new(netmap.Replica)
rs.SetECDataCount(uint32(dataCount))
rs.SetECParityCount(uint32(parityCount))
if sel := ctx.GetSelector(); sel != nil {
rs.SetSelector(sel.GetText())
}
return rs
}
// VisitSelectStmt implements parser.QueryVisitor interface. // VisitSelectStmt implements parser.QueryVisitor interface.
func (p *policyVisitor) VisitSelectStmt(ctx *parser.SelectStmtContext) any { func (p *policyVisitor) VisitSelectStmt(ctx *parser.SelectStmtContext) any {
res, err := strconv.ParseUint(ctx.GetCount().GetText(), 10, 32) res, err := strconv.ParseUint(ctx.GetCount().GetText(), 10, 32)
@ -910,6 +954,14 @@ func validatePolicy(p PlacementPolicy) error {
if seenSelectors[selName] == nil { if seenSelectors[selName] == nil {
return fmt.Errorf("%w: '%s'", errUnknownSelector, selName) return fmt.Errorf("%w: '%s'", errUnknownSelector, selName)
} }
dataCount := p.replicas[i].GetECDataCount()
parityCount := p.replicas[i].GetECParityCount()
if dataCount != 0 || parityCount != 0 {
if c := seenSelectors[selName].GetCount(); c < dataCount+parityCount {
dstepanov-yadro marked this conversation as resolved Outdated
dstepanov-yadro commented 2024-03-06 13:50:27 +00:00
Outdated
Review
Copy link

Also we can check that X+Y <= Z for EC X.Y SELECT Z FROM ... policy.

Also we can check that `X+Y <= Z` for `EC X.Y SELECT Z FROM ... ` policy.
fyrchik commented 2024-03-12 13:51:07 +00:00
Outdated
Review
Copy link

Fixed.

Fixed.
return fmt.Errorf("%w: %d < %d + %d", errECFewSelectors, c, dataCount, parityCount)
}
}
} }
} }

2
netmap/policy_decode_test.go
View file

@ -44,6 +44,8 @@ FILTER Node EQ '10.78.8.11' AS F`,
`UNIQUE `UNIQUE
REP 1 REP 1
REP 1`, REP 1`,
`EC 1.2 IN X
SELECT 3 IN City FROM * AS X`,
} }
var p PlacementPolicy var p PlacementPolicy