Initial EC implementation #205
27 changed files with 1715 additions and 345 deletions
4
go.mod
4
go.mod
|
|
@ -3,7 +3,7 @@ module git.frostfs.info/TrueCloudLab/frostfs-sdk-go
|
|||
go 1.20
|
||||
|
||||
require (
|
||||
git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240306101814-c1c7b344b9c0
|
||||
git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240319122301-1772b921826b
|
||||
git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb
|
||||
git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0
|
||||
git.frostfs.info/TrueCloudLab/hrw v1.2.1
|
||||
|
|
@ -11,6 +11,7 @@ require (
|
|||
github.com/antlr4-go/antlr/v4 v4.13.0
|
||||
github.com/google/uuid v1.3.0
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.2
|
||||
github.com/klauspost/reedsolomon v1.12.1
|
||||
github.com/mr-tron/base58 v1.2.0
|
||||
github.com/nspcc-dev/neo-go v0.101.2-0.20230601131642-a0117042e8fc
|
||||
github.com/stretchr/testify v1.8.3
|
||||
|
|
@ -28,6 +29,7 @@ require (
|
|||
github.com/golang/protobuf v1.5.3 // indirect
|
||||
github.com/gorilla/websocket v1.5.0 // indirect
|
||||
github.com/hashicorp/golang-lru v0.6.0 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.6 // indirect
|
||||
github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22 // indirect
|
||||
github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20230615193820-9185820289ce // indirect
|
||||
github.com/nspcc-dev/rfc6979 v0.2.0 // indirect
|
||||
|
|
|
|||
BIN
go.sum
BIN
go.sum
Binary file not shown.
|
|
@ -209,6 +209,25 @@ func (m NetMap) SelectFilterNodes(expr *SelectFilterExpr) ([][]NodeInfo, error)
|
|||
return ret, nil
|
||||
}
|
||||
|
||||
func countNodes(r netmap.Replica) uint32 {
|
||||
if r.GetCount() != 0 {
|
||||
return r.GetCount()
|
||||
}
|
||||
return r.GetECDataCount() + r.GetECParityCount()
|
||||
}
|
||||
|
||||
func (p PlacementPolicy) isUnique() bool {
|
||||
if p.unique {
|
||||
return true
|
||||
}
|
||||
for _, r := range p.replicas {
|
||||
if r.GetECDataCount() != 0 || r.GetECParityCount() != 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ContainerNodes returns two-dimensional list of nodes as a result of applying
|
||||
// given PlacementPolicy to the NetMap. Each line of the list corresponds to a
|
||||
// replica descriptor. Line order corresponds to order of ReplicaDescriptor list
|
||||
|
|
@ -230,6 +249,7 @@ func (m NetMap) ContainerNodes(p PlacementPolicy, pivot []byte) ([][]NodeInfo, e
|
|||
return nil, err
|
||||
}
|
||||
|
||||
unique := p.isUnique()
|
||||
result := make([][]NodeInfo, len(p.replicas))
|
||||
|
||||
// Note that the cached selectors are not used when the policy contains the UNIQUE flag.
|
||||
|
|
@ -240,7 +260,7 @@ func (m NetMap) ContainerNodes(p PlacementPolicy, pivot []byte) ([][]NodeInfo, e
|
|||
sName := p.replicas[i].GetSelector()
|
||||
if sName == "" && !(len(p.replicas) == 1 && len(p.selectors) == 1) {
|
||||
var s netmap.Selector
|
||||
s.SetCount(p.replicas[i].GetCount())
|
||||
s.SetCount(countNodes(p.replicas[i]))
|
||||
s.SetFilter(mainFilterName)
|
||||
|
||||
nodes, err := c.getSelection(s)
|
||||
|
|
@ -250,14 +270,14 @@ func (m NetMap) ContainerNodes(p PlacementPolicy, pivot []byte) ([][]NodeInfo, e
|
|||
|
||||
result[i] = append(result[i], flattenNodes(nodes)...)
|
||||
|
||||
if p.unique {
|
||||
if unique {
|
||||
c.addUsedNodes(result[i]...)
|
||||
}
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
if p.unique {
|
||||
if unique {
|
||||
if c.processedSelectors[sName] == nil {
|
||||
return nil, fmt.Errorf("selector not found: '%s'", sName)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -62,6 +62,8 @@ func (x *NetworkInfo) readFromV2(m netmap.NetworkInfo, checkFieldPresence bool)
|
|||
configEpochDuration,
|
||||
configIRCandidateFee,
|
||||
configMaxObjSize,
|
||||
configMaxECDataCount,
|
||||
configMaxECParityCount,
|
||||
configWithdrawalFee:
|
||||
_, err = decodeConfigValueUint64(prm.GetValue())
|
||||
case configHomomorphicHashingDisabled,
|
||||
|
|
@ -234,6 +236,8 @@ func (x *NetworkInfo) IterateRawNetworkParameters(f func(name string, value []by
|
|||
configEpochDuration,
|
||||
configIRCandidateFee,
|
||||
configMaxObjSize,
|
||||
configMaxECDataCount,
|
||||
configMaxECParityCount,
|
||||
configWithdrawalFee,
|
||||
configHomomorphicHashingDisabled,
|
||||
configMaintenanceModeAllowed:
|
||||
|
|
@ -432,6 +436,34 @@ func (x NetworkInfo) MaxObjectSize() uint64 {
|
|||
return x.configUint64(configMaxObjSize)
|
||||
}
|
||||
|
||||
const configMaxECDataCount = "MaxECDataCount"
|
||||
|
||||
// SetMaxECDataCount sets maximum number of data shards for erasure codes.
|
||||
//
|
||||
// Zero means no restrictions.
|
||||
func (x *NetworkInfo) SetMaxECDataCount(dataCount uint64) {
|
||||
x.setConfigUint64(configMaxECDataCount, dataCount)
|
||||
}
|
||||
|
||||
// MaxECDataCount returns maximum number of data shards for erasure codes.
|
||||
func (x NetworkInfo) MaxECDataCount() uint64 {
|
||||
return x.configUint64(configMaxECDataCount)
|
||||
}
|
||||
|
||||
const configMaxECParityCount = "MaxECParityCount"
|
||||
|
||||
// SetMaxECParityCount sets maximum number of parity shards for erasure codes.
|
||||
//
|
||||
// Zero means no restrictions.
|
||||
func (x *NetworkInfo) SetMaxECParityCount(parityCount uint64) {
|
||||
x.setConfigUint64(configMaxECParityCount, parityCount)
|
||||
}
|
||||
|
||||
// MaxECParityCount returns maximum number of parity shards for erasure codes.
|
||||
func (x NetworkInfo) MaxECParityCount() uint64 {
|
||||
return x.configUint64(configMaxECParityCount)
|
||||
}
|
||||
|
||||
const configWithdrawalFee = "WithdrawFee"
|
||||
|
||||
// SetWithdrawalFee sets fee for withdrawals from the FrostFS accounts that
|
||||
|
|
|
|||
|
|
@ -173,6 +173,32 @@ func TestNetworkInfo_MaxObjectSize(t *testing.T) {
|
|||
)
|
||||
}
|
||||
|
||||
func TestNetworkInfo_MaxECDataCount(t *testing.T) {
|
||||
testConfigValue(t,
|
||||
func(x NetworkInfo) any { return x.MaxECDataCount() },
|
||||
func(info *NetworkInfo, val any) { info.SetMaxECDataCount(val.(uint64)) },
|
||||
uint64(1), uint64(2),
|
||||
"MaxECDataCount", func(val any) []byte {
|
||||
data := make([]byte, 8)
|
||||
binary.LittleEndian.PutUint64(data, val.(uint64))
|
||||
return data
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func TestNetworkInfo_MaxECParityCount(t *testing.T) {
|
||||
testConfigValue(t,
|
||||
func(x NetworkInfo) any { return x.MaxECParityCount() },
|
||||
func(info *NetworkInfo, val any) { info.SetMaxECParityCount(val.(uint64)) },
|
||||
uint64(1), uint64(2),
|
||||
"MaxECParityCount", func(val any) []byte {
|
||||
data := make([]byte, 8)
|
||||
binary.LittleEndian.PutUint64(data, val.(uint64))
|
||||
return data
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func TestNetworkInfo_WithdrawalFee(t *testing.T) {
|
||||
testConfigValue(t,
|
||||
func(x NetworkInfo) any { return x.WithdrawalFee() },
|
||||
|
|
|
|||
|
|
@ -4,10 +4,14 @@ options {
|
|||
tokenVocab = QueryLexer;
|
||||
}
|
||||
|
||||
policy: UNIQUE? repStmt+ cbfStmt? selectStmt* filterStmt* EOF;
|
||||
policy: UNIQUE? (repStmt | ecStmt)+ cbfStmt? selectStmt* filterStmt* EOF;
|
||||
|
||||
selectFilterExpr: cbfStmt? selectStmt? filterStmt* EOF;
|
||||
|
||||
ecStmt:
|
||||
EC Data = NUMBER1 DOT Parity = NUMBER1 // erasure code configuration
|
||||
(IN Selector = ident)?; // optional selector name
|
||||
|
||||
repStmt:
|
||||
REP Count = NUMBER1 // number of object replicas
|
||||
(IN Selector = ident)?; // optional selector name
|
||||
|
|
|
|||
Binary file not shown.
Binary file not shown.
|
|
@ -7,6 +7,7 @@ SIMPLE_OP : 'EQ' | 'NE' | 'GE' | 'GT' | 'LT' | 'LE';
|
|||
|
||||
UNIQUE : 'UNIQUE';
|
||||
REP : 'REP';
|
||||
EC : 'EC';
|
||||
IN : 'IN';
|
||||
AS : 'AS';
|
||||
CBF : 'CBF';
|
||||
|
|
@ -14,6 +15,7 @@ SELECT : 'SELECT';
|
|||
FROM : 'FROM';
|
||||
FILTER : 'FILTER';
|
||||
WILDCARD : '*';
|
||||
DOT : '.';
|
||||
|
||||
CLAUSE_SAME : 'SAME';
|
||||
CLAUSE_DISTINCT : 'DISTINCT';
|
||||
|
|
|
|||
Binary file not shown.
Binary file not shown.
|
|
@ -1,4 +1,4 @@
|
|||
// Code generated from Query.g4 by ANTLR 4.13.0. DO NOT EDIT.
|
||||
// Code generated from /repo/frostfs/sdk-go/netmap/parser/Query.g4 by ANTLR 4.13.0. DO NOT EDIT.
|
||||
|
||||
package parser // Query
|
||||
|
||||
|
|
@ -16,6 +16,10 @@ func (v *BaseQueryVisitor) VisitSelectFilterExpr(ctx *SelectFilterExprContext) i
|
|||
return v.VisitChildren(ctx)
|
||||
}
|
||||
|
||||
func (v *BaseQueryVisitor) VisitEcStmt(ctx *EcStmtContext) interface{} {
|
||||
return v.VisitChildren(ctx)
|
||||
}
|
||||
|
||||
func (v *BaseQueryVisitor) VisitRepStmt(ctx *RepStmtContext) interface{} {
|
||||
return v.VisitChildren(ctx)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
// Code generated from QueryLexer.g4 by ANTLR 4.13.0. DO NOT EDIT.
|
||||
// Code generated from /repo/frostfs/sdk-go/netmap/parser/QueryLexer.g4 by ANTLR 4.13.0. DO NOT EDIT.
|
||||
|
||||
package parser
|
||||
|
||||
|
|
@ -43,119 +43,123 @@ func querylexerLexerInit() {
|
|||
"DEFAULT_MODE",
|
||||
}
|
||||
staticData.LiteralNames = []string{
|
||||
"", "'NOT'", "'AND'", "'OR'", "", "'UNIQUE'", "'REP'", "'IN'", "'AS'",
|
||||
"'CBF'", "'SELECT'", "'FROM'", "'FILTER'", "'*'", "'SAME'", "'DISTINCT'",
|
||||
"'('", "')'", "'@'", "", "", "'0'",
|
||||
"", "'NOT'", "'AND'", "'OR'", "", "'UNIQUE'", "'REP'", "'EC'", "'IN'",
|
||||
"'AS'", "'CBF'", "'SELECT'", "'FROM'", "'FILTER'", "'*'", "'.'", "'SAME'",
|
||||
"'DISTINCT'", "'('", "')'", "'@'", "", "", "'0'",
|
||||
}
|
||||
staticData.SymbolicNames = []string{
|
||||
"", "NOT_OP", "AND_OP", "OR_OP", "SIMPLE_OP", "UNIQUE", "REP", "IN",
|
||||
"AS", "CBF", "SELECT", "FROM", "FILTER", "WILDCARD", "CLAUSE_SAME",
|
||||
"", "NOT_OP", "AND_OP", "OR_OP", "SIMPLE_OP", "UNIQUE", "REP", "EC",
|
||||
"IN", "AS", "CBF", "SELECT", "FROM", "FILTER", "WILDCARD", "DOT", "CLAUSE_SAME",
|
||||
"CLAUSE_DISTINCT", "L_PAREN", "R_PAREN", "AT", "IDENT", "NUMBER1", "ZERO",
|
||||
"STRING", "WS",
|
||||
}
|
||||
staticData.RuleNames = []string{
|
||||
"NOT_OP", "AND_OP", "OR_OP", "SIMPLE_OP", "UNIQUE", "REP", "IN", "AS",
|
||||
"CBF", "SELECT", "FROM", "FILTER", "WILDCARD", "CLAUSE_SAME", "CLAUSE_DISTINCT",
|
||||
"L_PAREN", "R_PAREN", "AT", "IDENT", "Digit", "Nondigit", "NUMBER1",
|
||||
"ZERO", "STRING", "ESC", "UNICODE", "HEX", "SAFECODEPOINTSINGLE", "SAFECODEPOINTDOUBLE",
|
||||
"WS",
|
||||
"NOT_OP", "AND_OP", "OR_OP", "SIMPLE_OP", "UNIQUE", "REP", "EC", "IN",
|
||||
"AS", "CBF", "SELECT", "FROM", "FILTER", "WILDCARD", "DOT", "CLAUSE_SAME",
|
||||
"CLAUSE_DISTINCT", "L_PAREN", "R_PAREN", "AT", "IDENT", "Digit", "Nondigit",
|
||||
"NUMBER1", "ZERO", "STRING", "ESC", "UNICODE", "HEX", "SAFECODEPOINTSINGLE",
|
||||
"SAFECODEPOINTDOUBLE", "WS",
|
||||
}
|
||||
staticData.PredictionContextCache = antlr.NewPredictionContextCache()
|
||||
staticData.serializedATN = []int32{
|
||||
4, 0, 23, 213, 6, -1, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2,
|
||||
4, 0, 25, 222, 6, -1, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2,
|
||||
4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2,
|
||||
10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15,
|
||||
7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7,
|
||||
20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25,
|
||||
2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 1, 0, 1, 0, 1,
|
||||
0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 2, 1, 3, 1, 3, 1, 3, 1,
|
||||
3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 3, 3, 85, 8, 3, 1, 4,
|
||||
1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 6, 1, 6,
|
||||
1, 6, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 9, 1, 9, 1, 9, 1, 9,
|
||||
1, 9, 1, 9, 1, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 11, 1, 11, 1, 11,
|
||||
1, 11, 1, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 1, 13, 1,
|
||||
13, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 15,
|
||||
1, 15, 1, 16, 1, 16, 1, 17, 1, 17, 1, 18, 1, 18, 1, 18, 5, 18, 152, 8,
|
||||
18, 10, 18, 12, 18, 155, 9, 18, 1, 19, 1, 19, 1, 20, 1, 20, 1, 21, 1, 21,
|
||||
5, 21, 163, 8, 21, 10, 21, 12, 21, 166, 9, 21, 1, 22, 1, 22, 1, 23, 1,
|
||||
23, 1, 23, 5, 23, 173, 8, 23, 10, 23, 12, 23, 176, 9, 23, 1, 23, 1, 23,
|
||||
1, 23, 1, 23, 5, 23, 182, 8, 23, 10, 23, 12, 23, 185, 9, 23, 1, 23, 3,
|
||||
23, 188, 8, 23, 1, 24, 1, 24, 1, 24, 3, 24, 193, 8, 24, 1, 25, 1, 25, 1,
|
||||
25, 1, 25, 1, 25, 1, 25, 1, 26, 1, 26, 1, 27, 1, 27, 1, 28, 1, 28, 1, 29,
|
||||
4, 29, 208, 8, 29, 11, 29, 12, 29, 209, 1, 29, 1, 29, 0, 0, 30, 1, 1, 3,
|
||||
2, 5, 3, 7, 4, 9, 5, 11, 6, 13, 7, 15, 8, 17, 9, 19, 10, 21, 11, 23, 12,
|
||||
25, 13, 27, 14, 29, 15, 31, 16, 33, 17, 35, 18, 37, 19, 39, 0, 41, 0, 43,
|
||||
20, 45, 21, 47, 22, 49, 0, 51, 0, 53, 0, 55, 0, 57, 0, 59, 23, 1, 0, 8,
|
||||
1, 0, 48, 57, 3, 0, 65, 90, 95, 95, 97, 122, 1, 0, 49, 57, 9, 0, 34, 34,
|
||||
39, 39, 47, 47, 92, 92, 98, 98, 102, 102, 110, 110, 114, 114, 116, 116,
|
||||
3, 0, 48, 57, 65, 70, 97, 102, 3, 0, 0, 31, 39, 39, 92, 92, 3, 0, 0, 31,
|
||||
34, 34, 92, 92, 3, 0, 9, 10, 13, 13, 32, 32, 220, 0, 1, 1, 0, 0, 0, 0,
|
||||
3, 1, 0, 0, 0, 0, 5, 1, 0, 0, 0, 0, 7, 1, 0, 0, 0, 0, 9, 1, 0, 0, 0, 0,
|
||||
11, 1, 0, 0, 0, 0, 13, 1, 0, 0, 0, 0, 15, 1, 0, 0, 0, 0, 17, 1, 0, 0, 0,
|
||||
0, 19, 1, 0, 0, 0, 0, 21, 1, 0, 0, 0, 0, 23, 1, 0, 0, 0, 0, 25, 1, 0, 0,
|
||||
0, 0, 27, 1, 0, 0, 0, 0, 29, 1, 0, 0, 0, 0, 31, 1, 0, 0, 0, 0, 33, 1, 0,
|
||||
0, 0, 0, 35, 1, 0, 0, 0, 0, 37, 1, 0, 0, 0, 0, 43, 1, 0, 0, 0, 0, 45, 1,
|
||||
0, 0, 0, 0, 47, 1, 0, 0, 0, 0, 59, 1, 0, 0, 0, 1, 61, 1, 0, 0, 0, 3, 65,
|
||||
1, 0, 0, 0, 5, 69, 1, 0, 0, 0, 7, 84, 1, 0, 0, 0, 9, 86, 1, 0, 0, 0, 11,
|
||||
93, 1, 0, 0, 0, 13, 97, 1, 0, 0, 0, 15, 100, 1, 0, 0, 0, 17, 103, 1, 0,
|
||||
0, 0, 19, 107, 1, 0, 0, 0, 21, 114, 1, 0, 0, 0, 23, 119, 1, 0, 0, 0, 25,
|
||||
126, 1, 0, 0, 0, 27, 128, 1, 0, 0, 0, 29, 133, 1, 0, 0, 0, 31, 142, 1,
|
||||
0, 0, 0, 33, 144, 1, 0, 0, 0, 35, 146, 1, 0, 0, 0, 37, 148, 1, 0, 0, 0,
|
||||
39, 156, 1, 0, 0, 0, 41, 158, 1, 0, 0, 0, 43, 160, 1, 0, 0, 0, 45, 167,
|
||||
1, 0, 0, 0, 47, 187, 1, 0, 0, 0, 49, 189, 1, 0, 0, 0, 51, 194, 1, 0, 0,
|
||||
0, 53, 200, 1, 0, 0, 0, 55, 202, 1, 0, 0, 0, 57, 204, 1, 0, 0, 0, 59, 207,
|
||||
1, 0, 0, 0, 61, 62, 5, 78, 0, 0, 62, 63, 5, 79, 0, 0, 63, 64, 5, 84, 0,
|
||||
0, 64, 2, 1, 0, 0, 0, 65, 66, 5, 65, 0, 0, 66, 67, 5, 78, 0, 0, 67, 68,
|
||||
5, 68, 0, 0, 68, 4, 1, 0, 0, 0, 69, 70, 5, 79, 0, 0, 70, 71, 5, 82, 0,
|
||||
0, 71, 6, 1, 0, 0, 0, 72, 73, 5, 69, 0, 0, 73, 85, 5, 81, 0, 0, 74, 75,
|
||||
5, 78, 0, 0, 75, 85, 5, 69, 0, 0, 76, 77, 5, 71, 0, 0, 77, 85, 5, 69, 0,
|
||||
0, 78, 79, 5, 71, 0, 0, 79, 85, 5, 84, 0, 0, 80, 81, 5, 76, 0, 0, 81, 85,
|
||||
5, 84, 0, 0, 82, 83, 5, 76, 0, 0, 83, 85, 5, 69, 0, 0, 84, 72, 1, 0, 0,
|
||||
0, 84, 74, 1, 0, 0, 0, 84, 76, 1, 0, 0, 0, 84, 78, 1, 0, 0, 0, 84, 80,
|
||||
1, 0, 0, 0, 84, 82, 1, 0, 0, 0, 85, 8, 1, 0, 0, 0, 86, 87, 5, 85, 0, 0,
|
||||
87, 88, 5, 78, 0, 0, 88, 89, 5, 73, 0, 0, 89, 90, 5, 81, 0, 0, 90, 91,
|
||||
5, 85, 0, 0, 91, 92, 5, 69, 0, 0, 92, 10, 1, 0, 0, 0, 93, 94, 5, 82, 0,
|
||||
0, 94, 95, 5, 69, 0, 0, 95, 96, 5, 80, 0, 0, 96, 12, 1, 0, 0, 0, 97, 98,
|
||||
5, 73, 0, 0, 98, 99, 5, 78, 0, 0, 99, 14, 1, 0, 0, 0, 100, 101, 5, 65,
|
||||
0, 0, 101, 102, 5, 83, 0, 0, 102, 16, 1, 0, 0, 0, 103, 104, 5, 67, 0, 0,
|
||||
104, 105, 5, 66, 0, 0, 105, 106, 5, 70, 0, 0, 106, 18, 1, 0, 0, 0, 107,
|
||||
108, 5, 83, 0, 0, 108, 109, 5, 69, 0, 0, 109, 110, 5, 76, 0, 0, 110, 111,
|
||||
5, 69, 0, 0, 111, 112, 5, 67, 0, 0, 112, 113, 5, 84, 0, 0, 113, 20, 1,
|
||||
0, 0, 0, 114, 115, 5, 70, 0, 0, 115, 116, 5, 82, 0, 0, 116, 117, 5, 79,
|
||||
0, 0, 117, 118, 5, 77, 0, 0, 118, 22, 1, 0, 0, 0, 119, 120, 5, 70, 0, 0,
|
||||
120, 121, 5, 73, 0, 0, 121, 122, 5, 76, 0, 0, 122, 123, 5, 84, 0, 0, 123,
|
||||
124, 5, 69, 0, 0, 124, 125, 5, 82, 0, 0, 125, 24, 1, 0, 0, 0, 126, 127,
|
||||
5, 42, 0, 0, 127, 26, 1, 0, 0, 0, 128, 129, 5, 83, 0, 0, 129, 130, 5, 65,
|
||||
0, 0, 130, 131, 5, 77, 0, 0, 131, 132, 5, 69, 0, 0, 132, 28, 1, 0, 0, 0,
|
||||
133, 134, 5, 68, 0, 0, 134, 135, 5, 73, 0, 0, 135, 136, 5, 83, 0, 0, 136,
|
||||
137, 5, 84, 0, 0, 137, 138, 5, 73, 0, 0, 138, 139, 5, 78, 0, 0, 139, 140,
|
||||
5, 67, 0, 0, 140, 141, 5, 84, 0, 0, 141, 30, 1, 0, 0, 0, 142, 143, 5, 40,
|
||||
0, 0, 143, 32, 1, 0, 0, 0, 144, 145, 5, 41, 0, 0, 145, 34, 1, 0, 0, 0,
|
||||
146, 147, 5, 64, 0, 0, 147, 36, 1, 0, 0, 0, 148, 153, 3, 41, 20, 0, 149,
|
||||
152, 3, 39, 19, 0, 150, 152, 3, 41, 20, 0, 151, 149, 1, 0, 0, 0, 151, 150,
|
||||
1, 0, 0, 0, 152, 155, 1, 0, 0, 0, 153, 151, 1, 0, 0, 0, 153, 154, 1, 0,
|
||||
0, 0, 154, 38, 1, 0, 0, 0, 155, 153, 1, 0, 0, 0, 156, 157, 7, 0, 0, 0,
|
||||
157, 40, 1, 0, 0, 0, 158, 159, 7, 1, 0, 0, 159, 42, 1, 0, 0, 0, 160, 164,
|
||||
7, 2, 0, 0, 161, 163, 3, 39, 19, 0, 162, 161, 1, 0, 0, 0, 163, 166, 1,
|
||||
0, 0, 0, 164, 162, 1, 0, 0, 0, 164, 165, 1, 0, 0, 0, 165, 44, 1, 0, 0,
|
||||
0, 166, 164, 1, 0, 0, 0, 167, 168, 5, 48, 0, 0, 168, 46, 1, 0, 0, 0, 169,
|
||||
174, 5, 34, 0, 0, 170, 173, 3, 49, 24, 0, 171, 173, 3, 57, 28, 0, 172,
|
||||
170, 1, 0, 0, 0, 172, 171, 1, 0, 0, 0, 173, 176, 1, 0, 0, 0, 174, 172,
|
||||
1, 0, 0, 0, 174, 175, 1, 0, 0, 0, 175, 177, 1, 0, 0, 0, 176, 174, 1, 0,
|
||||
0, 0, 177, 188, 5, 34, 0, 0, 178, 183, 5, 39, 0, 0, 179, 182, 3, 49, 24,
|
||||
0, 180, 182, 3, 55, 27, 0, 181, 179, 1, 0, 0, 0, 181, 180, 1, 0, 0, 0,
|
||||
182, 185, 1, 0, 0, 0, 183, 181, 1, 0, 0, 0, 183, 184, 1, 0, 0, 0, 184,
|
||||
186, 1, 0, 0, 0, 185, 183, 1, 0, 0, 0, 186, 188, 5, 39, 0, 0, 187, 169,
|
||||
1, 0, 0, 0, 187, 178, 1, 0, 0, 0, 188, 48, 1, 0, 0, 0, 189, 192, 5, 92,
|
||||
0, 0, 190, 193, 7, 3, 0, 0, 191, 193, 3, 51, 25, 0, 192, 190, 1, 0, 0,
|
||||
0, 192, 191, 1, 0, 0, 0, 193, 50, 1, 0, 0, 0, 194, 195, 5, 117, 0, 0, 195,
|
||||
196, 3, 53, 26, 0, 196, 197, 3, 53, 26, 0, 197, 198, 3, 53, 26, 0, 198,
|
||||
199, 3, 53, 26, 0, 199, 52, 1, 0, 0, 0, 200, 201, 7, 4, 0, 0, 201, 54,
|
||||
1, 0, 0, 0, 202, 203, 8, 5, 0, 0, 203, 56, 1, 0, 0, 0, 204, 205, 8, 6,
|
||||
0, 0, 205, 58, 1, 0, 0, 0, 206, 208, 7, 7, 0, 0, 207, 206, 1, 0, 0, 0,
|
||||
208, 209, 1, 0, 0, 0, 209, 207, 1, 0, 0, 0, 209, 210, 1, 0, 0, 0, 210,
|
||||
211, 1, 0, 0, 0, 211, 212, 6, 29, 0, 0, 212, 60, 1, 0, 0, 0, 12, 0, 84,
|
||||
151, 153, 164, 172, 174, 181, 183, 187, 192, 209, 1, 6, 0, 0,
|
||||
2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2,
|
||||
31, 7, 31, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2,
|
||||
1, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3,
|
||||
1, 3, 3, 3, 89, 8, 3, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 5, 1,
|
||||
5, 1, 5, 1, 5, 1, 6, 1, 6, 1, 6, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1,
|
||||
9, 1, 9, 1, 9, 1, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1,
|
||||
11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12,
|
||||
1, 12, 1, 13, 1, 13, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1,
|
||||
16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 17, 1, 17,
|
||||
1, 18, 1, 18, 1, 19, 1, 19, 1, 20, 1, 20, 1, 20, 5, 20, 161, 8, 20, 10,
|
||||
20, 12, 20, 164, 9, 20, 1, 21, 1, 21, 1, 22, 1, 22, 1, 23, 1, 23, 5, 23,
|
||||
172, 8, 23, 10, 23, 12, 23, 175, 9, 23, 1, 24, 1, 24, 1, 25, 1, 25, 1,
|
||||
25, 5, 25, 182, 8, 25, 10, 25, 12, 25, 185, 9, 25, 1, 25, 1, 25, 1, 25,
|
||||
1, 25, 5, 25, 191, 8, 25, 10, 25, 12, 25, 194, 9, 25, 1, 25, 3, 25, 197,
|
||||
8, 25, 1, 26, 1, 26, 1, 26, 3, 26, 202, 8, 26, 1, 27, 1, 27, 1, 27, 1,
|
||||
27, 1, 27, 1, 27, 1, 28, 1, 28, 1, 29, 1, 29, 1, 30, 1, 30, 1, 31, 4, 31,
|
||||
217, 8, 31, 11, 31, 12, 31, 218, 1, 31, 1, 31, 0, 0, 32, 1, 1, 3, 2, 5,
|
||||
3, 7, 4, 9, 5, 11, 6, 13, 7, 15, 8, 17, 9, 19, 10, 21, 11, 23, 12, 25,
|
||||
13, 27, 14, 29, 15, 31, 16, 33, 17, 35, 18, 37, 19, 39, 20, 41, 21, 43,
|
||||
0, 45, 0, 47, 22, 49, 23, 51, 24, 53, 0, 55, 0, 57, 0, 59, 0, 61, 0, 63,
|
||||
25, 1, 0, 8, 1, 0, 48, 57, 3, 0, 65, 90, 95, 95, 97, 122, 1, 0, 49, 57,
|
||||
9, 0, 34, 34, 39, 39, 47, 47, 92, 92, 98, 98, 102, 102, 110, 110, 114,
|
||||
114, 116, 116, 3, 0, 48, 57, 65, 70, 97, 102, 3, 0, 0, 31, 39, 39, 92,
|
||||
92, 3, 0, 0, 31, 34, 34, 92, 92, 3, 0, 9, 10, 13, 13, 32, 32, 229, 0, 1,
|
||||
1, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 5, 1, 0, 0, 0, 0, 7, 1, 0, 0, 0, 0, 9,
|
||||
1, 0, 0, 0, 0, 11, 1, 0, 0, 0, 0, 13, 1, 0, 0, 0, 0, 15, 1, 0, 0, 0, 0,
|
||||
17, 1, 0, 0, 0, 0, 19, 1, 0, 0, 0, 0, 21, 1, 0, 0, 0, 0, 23, 1, 0, 0, 0,
|
||||
0, 25, 1, 0, 0, 0, 0, 27, 1, 0, 0, 0, 0, 29, 1, 0, 0, 0, 0, 31, 1, 0, 0,
|
||||
0, 0, 33, 1, 0, 0, 0, 0, 35, 1, 0, 0, 0, 0, 37, 1, 0, 0, 0, 0, 39, 1, 0,
|
||||
0, 0, 0, 41, 1, 0, 0, 0, 0, 47, 1, 0, 0, 0, 0, 49, 1, 0, 0, 0, 0, 51, 1,
|
||||
0, 0, 0, 0, 63, 1, 0, 0, 0, 1, 65, 1, 0, 0, 0, 3, 69, 1, 0, 0, 0, 5, 73,
|
||||
1, 0, 0, 0, 7, 88, 1, 0, 0, 0, 9, 90, 1, 0, 0, 0, 11, 97, 1, 0, 0, 0, 13,
|
||||
101, 1, 0, 0, 0, 15, 104, 1, 0, 0, 0, 17, 107, 1, 0, 0, 0, 19, 110, 1,
|
||||
0, 0, 0, 21, 114, 1, 0, 0, 0, 23, 121, 1, 0, 0, 0, 25, 126, 1, 0, 0, 0,
|
||||
27, 133, 1, 0, 0, 0, 29, 135, 1, 0, 0, 0, 31, 137, 1, 0, 0, 0, 33, 142,
|
||||
1, 0, 0, 0, 35, 151, 1, 0, 0, 0, 37, 153, 1, 0, 0, 0, 39, 155, 1, 0, 0,
|
||||
0, 41, 157, 1, 0, 0, 0, 43, 165, 1, 0, 0, 0, 45, 167, 1, 0, 0, 0, 47, 169,
|
||||
1, 0, 0, 0, 49, 176, 1, 0, 0, 0, 51, 196, 1, 0, 0, 0, 53, 198, 1, 0, 0,
|
||||
0, 55, 203, 1, 0, 0, 0, 57, 209, 1, 0, 0, 0, 59, 211, 1, 0, 0, 0, 61, 213,
|
||||
1, 0, 0, 0, 63, 216, 1, 0, 0, 0, 65, 66, 5, 78, 0, 0, 66, 67, 5, 79, 0,
|
||||
0, 67, 68, 5, 84, 0, 0, 68, 2, 1, 0, 0, 0, 69, 70, 5, 65, 0, 0, 70, 71,
|
||||
5, 78, 0, 0, 71, 72, 5, 68, 0, 0, 72, 4, 1, 0, 0, 0, 73, 74, 5, 79, 0,
|
||||
0, 74, 75, 5, 82, 0, 0, 75, 6, 1, 0, 0, 0, 76, 77, 5, 69, 0, 0, 77, 89,
|
||||
5, 81, 0, 0, 78, 79, 5, 78, 0, 0, 79, 89, 5, 69, 0, 0, 80, 81, 5, 71, 0,
|
||||
0, 81, 89, 5, 69, 0, 0, 82, 83, 5, 71, 0, 0, 83, 89, 5, 84, 0, 0, 84, 85,
|
||||
5, 76, 0, 0, 85, 89, 5, 84, 0, 0, 86, 87, 5, 76, 0, 0, 87, 89, 5, 69, 0,
|
||||
0, 88, 76, 1, 0, 0, 0, 88, 78, 1, 0, 0, 0, 88, 80, 1, 0, 0, 0, 88, 82,
|
||||
1, 0, 0, 0, 88, 84, 1, 0, 0, 0, 88, 86, 1, 0, 0, 0, 89, 8, 1, 0, 0, 0,
|
||||
90, 91, 5, 85, 0, 0, 91, 92, 5, 78, 0, 0, 92, 93, 5, 73, 0, 0, 93, 94,
|
||||
5, 81, 0, 0, 94, 95, 5, 85, 0, 0, 95, 96, 5, 69, 0, 0, 96, 10, 1, 0, 0,
|
||||
0, 97, 98, 5, 82, 0, 0, 98, 99, 5, 69, 0, 0, 99, 100, 5, 80, 0, 0, 100,
|
||||
12, 1, 0, 0, 0, 101, 102, 5, 69, 0, 0, 102, 103, 5, 67, 0, 0, 103, 14,
|
||||
1, 0, 0, 0, 104, 105, 5, 73, 0, 0, 105, 106, 5, 78, 0, 0, 106, 16, 1, 0,
|
||||
0, 0, 107, 108, 5, 65, 0, 0, 108, 109, 5, 83, 0, 0, 109, 18, 1, 0, 0, 0,
|
||||
110, 111, 5, 67, 0, 0, 111, 112, 5, 66, 0, 0, 112, 113, 5, 70, 0, 0, 113,
|
||||
20, 1, 0, 0, 0, 114, 115, 5, 83, 0, 0, 115, 116, 5, 69, 0, 0, 116, 117,
|
||||
5, 76, 0, 0, 117, 118, 5, 69, 0, 0, 118, 119, 5, 67, 0, 0, 119, 120, 5,
|
||||
84, 0, 0, 120, 22, 1, 0, 0, 0, 121, 122, 5, 70, 0, 0, 122, 123, 5, 82,
|
||||
0, 0, 123, 124, 5, 79, 0, 0, 124, 125, 5, 77, 0, 0, 125, 24, 1, 0, 0, 0,
|
||||
126, 127, 5, 70, 0, 0, 127, 128, 5, 73, 0, 0, 128, 129, 5, 76, 0, 0, 129,
|
||||
130, 5, 84, 0, 0, 130, 131, 5, 69, 0, 0, 131, 132, 5, 82, 0, 0, 132, 26,
|
||||
1, 0, 0, 0, 133, 134, 5, 42, 0, 0, 134, 28, 1, 0, 0, 0, 135, 136, 5, 46,
|
||||
0, 0, 136, 30, 1, 0, 0, 0, 137, 138, 5, 83, 0, 0, 138, 139, 5, 65, 0, 0,
|
||||
139, 140, 5, 77, 0, 0, 140, 141, 5, 69, 0, 0, 141, 32, 1, 0, 0, 0, 142,
|
||||
143, 5, 68, 0, 0, 143, 144, 5, 73, 0, 0, 144, 145, 5, 83, 0, 0, 145, 146,
|
||||
5, 84, 0, 0, 146, 147, 5, 73, 0, 0, 147, 148, 5, 78, 0, 0, 148, 149, 5,
|
||||
67, 0, 0, 149, 150, 5, 84, 0, 0, 150, 34, 1, 0, 0, 0, 151, 152, 5, 40,
|
||||
0, 0, 152, 36, 1, 0, 0, 0, 153, 154, 5, 41, 0, 0, 154, 38, 1, 0, 0, 0,
|
||||
155, 156, 5, 64, 0, 0, 156, 40, 1, 0, 0, 0, 157, 162, 3, 45, 22, 0, 158,
|
||||
161, 3, 43, 21, 0, 159, 161, 3, 45, 22, 0, 160, 158, 1, 0, 0, 0, 160, 159,
|
||||
1, 0, 0, 0, 161, 164, 1, 0, 0, 0, 162, 160, 1, 0, 0, 0, 162, 163, 1, 0,
|
||||
0, 0, 163, 42, 1, 0, 0, 0, 164, 162, 1, 0, 0, 0, 165, 166, 7, 0, 0, 0,
|
||||
166, 44, 1, 0, 0, 0, 167, 168, 7, 1, 0, 0, 168, 46, 1, 0, 0, 0, 169, 173,
|
||||
7, 2, 0, 0, 170, 172, 3, 43, 21, 0, 171, 170, 1, 0, 0, 0, 172, 175, 1,
|
||||
0, 0, 0, 173, 171, 1, 0, 0, 0, 173, 174, 1, 0, 0, 0, 174, 48, 1, 0, 0,
|
||||
0, 175, 173, 1, 0, 0, 0, 176, 177, 5, 48, 0, 0, 177, 50, 1, 0, 0, 0, 178,
|
||||
183, 5, 34, 0, 0, 179, 182, 3, 53, 26, 0, 180, 182, 3, 61, 30, 0, 181,
|
||||
179, 1, 0, 0, 0, 181, 180, 1, 0, 0, 0, 182, 185, 1, 0, 0, 0, 183, 181,
|
||||
1, 0, 0, 0, 183, 184, 1, 0, 0, 0, 184, 186, 1, 0, 0, 0, 185, 183, 1, 0,
|
||||
0, 0, 186, 197, 5, 34, 0, 0, 187, 192, 5, 39, 0, 0, 188, 191, 3, 53, 26,
|
||||
0, 189, 191, 3, 59, 29, 0, 190, 188, 1, 0, 0, 0, 190, 189, 1, 0, 0, 0,
|
||||
191, 194, 1, 0, 0, 0, 192, 190, 1, 0, 0, 0, 192, 193, 1, 0, 0, 0, 193,
|
||||
195, 1, 0, 0, 0, 194, 192, 1, 0, 0, 0, 195, 197, 5, 39, 0, 0, 196, 178,
|
||||
1, 0, 0, 0, 196, 187, 1, 0, 0, 0, 197, 52, 1, 0, 0, 0, 198, 201, 5, 92,
|
||||
0, 0, 199, 202, 7, 3, 0, 0, 200, 202, 3, 55, 27, 0, 201, 199, 1, 0, 0,
|
||||
0, 201, 200, 1, 0, 0, 0, 202, 54, 1, 0, 0, 0, 203, 204, 5, 117, 0, 0, 204,
|
||||
205, 3, 57, 28, 0, 205, 206, 3, 57, 28, 0, 206, 207, 3, 57, 28, 0, 207,
|
||||
208, 3, 57, 28, 0, 208, 56, 1, 0, 0, 0, 209, 210, 7, 4, 0, 0, 210, 58,
|
||||
1, 0, 0, 0, 211, 212, 8, 5, 0, 0, 212, 60, 1, 0, 0, 0, 213, 214, 8, 6,
|
||||
0, 0, 214, 62, 1, 0, 0, 0, 215, 217, 7, 7, 0, 0, 216, 215, 1, 0, 0, 0,
|
||||
217, 218, 1, 0, 0, 0, 218, 216, 1, 0, 0, 0, 218, 219, 1, 0, 0, 0, 219,
|
||||
220, 1, 0, 0, 0, 220, 221, 6, 31, 0, 0, 221, 64, 1, 0, 0, 0, 12, 0, 88,
|
||||
160, 162, 173, 181, 183, 190, 192, 196, 201, 218, 1, 6, 0, 0,
|
||||
}
|
||||
deserializer := antlr.NewATNDeserializer(nil)
|
||||
staticData.atn = deserializer.Deserialize(staticData.serializedATN)
|
||||
|
|
@ -202,21 +206,23 @@ const (
|
|||
QueryLexerSIMPLE_OP = 4
|
||||
QueryLexerUNIQUE = 5
|
||||
QueryLexerREP = 6
|
||||
QueryLexerIN = 7
|
||||
QueryLexerAS = 8
|
||||
QueryLexerCBF = 9
|
||||
QueryLexerSELECT = 10
|
||||
QueryLexerFROM = 11
|
||||
QueryLexerFILTER = 12
|
||||
QueryLexerWILDCARD = 13
|
||||
QueryLexerCLAUSE_SAME = 14
|
||||
QueryLexerCLAUSE_DISTINCT = 15
|
||||
QueryLexerL_PAREN = 16
|
||||
QueryLexerR_PAREN = 17
|
||||
QueryLexerAT = 18
|
||||
QueryLexerIDENT = 19
|
||||
QueryLexerNUMBER1 = 20
|
||||
QueryLexerZERO = 21
|
||||
QueryLexerSTRING = 22
|
||||
QueryLexerWS = 23
|
||||
QueryLexerEC = 7
|
||||
QueryLexerIN = 8
|
||||
QueryLexerAS = 9
|
||||
QueryLexerCBF = 10
|
||||
QueryLexerSELECT = 11
|
||||
QueryLexerFROM = 12
|
||||
QueryLexerFILTER = 13
|
||||
QueryLexerWILDCARD = 14
|
||||
QueryLexerDOT = 15
|
||||
QueryLexerCLAUSE_SAME = 16
|
||||
QueryLexerCLAUSE_DISTINCT = 17
|
||||
QueryLexerL_PAREN = 18
|
||||
QueryLexerR_PAREN = 19
|
||||
QueryLexerAT = 20
|
||||
QueryLexerIDENT = 21
|
||||
QueryLexerNUMBER1 = 22
|
||||
QueryLexerZERO = 23
|
||||
QueryLexerSTRING = 24
|
||||
QueryLexerWS = 25
|
||||
)
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -1,4 +1,4 @@
|
|||
// Code generated from Query.g4 by ANTLR 4.13.0. DO NOT EDIT.
|
||||
// Code generated from /repo/frostfs/sdk-go/netmap/parser/Query.g4 by ANTLR 4.13.0. DO NOT EDIT.
|
||||
|
||||
package parser // Query
|
||||
|
||||
|
|
@ -14,6 +14,9 @@ type QueryVisitor interface {
|
|||
// Visit a parse tree produced by Query#selectFilterExpr.
|
||||
VisitSelectFilterExpr(ctx *SelectFilterExprContext) interface{}
|
||||
|
||||
// Visit a parse tree produced by Query#ecStmt.
|
||||
VisitEcStmt(ctx *EcStmtContext) interface{}
|
||||
|
||||
// Visit a parse tree produced by Query#repStmt.
|
||||
VisitRepStmt(ctx *RepStmtContext) interface{}
|
||||
|
||||
|
|
|
|||
|
|
@ -34,8 +34,17 @@ type PlacementPolicy struct {
|
|||
|
||||
func (p *PlacementPolicy) readFromV2(m netmap.PlacementPolicy, checkFieldPresence bool) error {
|
||||
p.replicas = m.GetReplicas()
|
||||
if checkFieldPresence && len(p.replicas) == 0 {
|
||||
return errors.New("missing replicas")
|
||||
if checkFieldPresence {
|
||||
if len(p.replicas) == 0 {
|
||||
return errors.New("missing replicas")
|
||||
}
|
||||
if len(p.replicas) != 1 {
|
||||
for i := range p.replicas {
|
||||
if p.replicas[i].GetECDataCount() != 0 || p.replicas[i].GetECParityCount() != 0 {
|
||||
return errors.New("erasure code group must be used exclusively")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
p.backupFactor = m.GetContainerBackupFactor()
|
||||
|
|
@ -393,10 +402,14 @@ func (p PlacementPolicy) WriteStringTo(w io.StringWriter) (err error) {
|
|||
c := p.replicas[i].GetCount()
|
||||
s := p.replicas[i].GetSelector()
|
||||
|
||||
if s != "" {
|
||||
_, err = w.WriteString(fmt.Sprintf("%sREP %d IN %s", delim, c, s))
|
||||
} else {
|
||||
if c != 0 {
|
||||
_, err = w.WriteString(fmt.Sprintf("%sREP %d", delim, c))
|
||||
} else {
|
||||
ecx, ecy := p.replicas[i].GetECDataCount(), p.replicas[i].GetECParityCount()
|
||||
_, err = w.WriteString(fmt.Sprintf("%sEC %d.%d", delim, ecx, ecy))
|
||||
}
|
||||
if s != "" {
|
||||
_, err = w.WriteString(fmt.Sprintf(" IN %s", s))
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
|
|
@ -630,6 +643,8 @@ var (
|
|||
"make sure to pair REP and SELECT clauses: \"REP .. IN X\" + \"SELECT ... AS X\"")
|
||||
// errRedundantSelector is returned for errors found by filters policy validator.
|
||||
errRedundantFilter = errors.New("policy: found redundant filter")
|
||||
// errECFewSelectors is returned when EC keyword is used without UNIQUE keyword.
|
||||
errECFewSelectors = errors.New("policy: too few nodes to select")
|
||||
)
|
||||
|
||||
type policyVisitor struct {
|
||||
|
|
@ -657,11 +672,18 @@ func (p *policyVisitor) VisitPolicy(ctx *parser.PolicyContext) any {
|
|||
|
||||
pl.unique = ctx.UNIQUE() != nil
|
||||
|
||||
repStmts := ctx.AllRepStmt()
|
||||
pl.replicas = make([]netmap.Replica, 0, len(repStmts))
|
||||
|
||||
for _, r := range repStmts {
|
||||
res, ok := r.Accept(p).(*netmap.Replica)
|
||||
stmts := ctx.GetChildren()
|
||||
for _, r := range stmts {
|
||||
var res *netmap.Replica
|
||||
var ok bool
|
||||
switch r := r.(type) {
|
||||
case parser.IRepStmtContext:
|
||||
res, ok = r.Accept(p).(*netmap.Replica)
|
||||
case parser.IEcStmtContext:
|
||||
res, ok = r.Accept(p).(*netmap.Replica)
|
||||
default:
|
||||
continue
|
||||
|
|
||||
}
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
|
@ -758,6 +780,28 @@ func (p *policyVisitor) VisitRepStmt(ctx *parser.RepStmtContext) any {
|
|||
return rs
|
||||
}
|
||||
|
||||
// VisitRepStmt implements parser.QueryVisitor interface.
|
||||
func (p *policyVisitor) VisitEcStmt(ctx *parser.EcStmtContext) any {
|
||||
dataCount, err := strconv.ParseUint(ctx.GetData().GetText(), 10, 32)
|
||||
if err != nil {
|
||||
return p.reportError(errInvalidNumber)
|
||||
}
|
||||
parityCount, err := strconv.ParseUint(ctx.GetParity().GetText(), 10, 32)
|
||||
if err != nil {
|
||||
return p.reportError(errInvalidNumber)
|
||||
}
|
||||
|
||||
rs := new(netmap.Replica)
|
||||
rs.SetECDataCount(uint32(dataCount))
|
||||
rs.SetECParityCount(uint32(parityCount))
|
||||
|
||||
if sel := ctx.GetSelector(); sel != nil {
|
||||
rs.SetSelector(sel.GetText())
|
||||
}
|
||||
|
||||
return rs
|
||||
}
|
||||
|
||||
// VisitSelectStmt implements parser.QueryVisitor interface.
|
||||
func (p *policyVisitor) VisitSelectStmt(ctx *parser.SelectStmtContext) any {
|
||||
res, err := strconv.ParseUint(ctx.GetCount().GetText(), 10, 32)
|
||||
|
|
@ -910,6 +954,14 @@ func validatePolicy(p PlacementPolicy) error {
|
|||
if seenSelectors[selName] == nil {
|
||||
return fmt.Errorf("%w: '%s'", errUnknownSelector, selName)
|
||||
}
|
||||
|
||||
dataCount := p.replicas[i].GetECDataCount()
|
||||
parityCount := p.replicas[i].GetECParityCount()
|
||||
if dataCount != 0 || parityCount != 0 {
|
||||
if c := seenSelectors[selName].GetCount(); c < dataCount+parityCount {
|
||||
|
dstepanov-yadro marked this conversation as resolved
Outdated
dstepanov-yadro
commented
Also we can check that Also we can check that `X+Y <= Z` for `EC X.Y SELECT Z FROM ... ` policy.
fyrchik
commented
Fixed. Fixed.
|
||||
return fmt.Errorf("%w: %d < %d + %d", errECFewSelectors, c, dataCount, parityCount)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -44,6 +44,8 @@ FILTER Node EQ '10.78.8.11' AS F`,
|
|||
`UNIQUE
|
||||
REP 1
|
||||
REP 1`,
|
||||
`EC 1.2 IN X
|
||||
SELECT 3 IN City FROM * AS X`,
|
||||
}
|
||||
|
||||
var p PlacementPolicy
|
||||
|
|
|
|||
121
object/erasure_code.go
Normal file
121
object/erasure_code.go
Normal file
|
|
@ -0,0 +1,121 @@
|
|||
package object
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
|
||||
refs "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
)
|
||||
|
||||
// ECHeader represents erasure coding header.
|
||||
type ECHeader struct {
|
||||
parent oid.ID
|
||||
index uint32
|
||||
total uint32
|
||||
header []byte
|
||||
headerLength uint32
|
||||
}
|
||||
|
||||
// NewECHeader constructs new erasure coding header.
|
||||
func NewECHeader(parent oid.ID, index, total uint32, header []byte, headerLength uint32) *ECHeader {
|
||||
return &ECHeader{
|
||||
parent: parent,
|
||||
index: index,
|
||||
total: total,
|
||||
header: header,
|
||||
headerLength: headerLength,
|
||||
}
|
||||
}
|
||||
|
||||
// WriteToV2 converts SDK structure to v2-api one.
|
||||
func (e *ECHeader) WriteToV2(h *object.ECHeader) {
|
||||
var parent refs.ObjectID
|
||||
e.parent.WriteToV2(&parent)
|
||||
h.Parent = &parent
|
||||
h.Index = e.index
|
||||
h.Total = e.total
|
||||
h.Header = e.header
|
||||
h.HeaderLength = e.headerLength
|
||||
}
|
||||
|
||||
// ReadFromV2 converts v2-api structure to SDK one.
|
||||
func (e *ECHeader) ReadFromV2(h *object.ECHeader) error {
|
||||
if h == nil {
|
||||
return nil
|
||||
}
|
||||
if h.Parent == nil {
|
||||
return errors.New("empty parent")
|
||||
}
|
||||
|
||||
_ = e.parent.ReadFromV2(*h.Parent)
|
||||
e.index = h.Index
|
||||
e.total = h.Total
|
||||
e.header = h.Header
|
||||
e.headerLength = h.HeaderLength
|
||||
return nil
|
||||
}
|
||||
|
||||
func (o *Object) ECHeader() *ECHeader {
|
||||
ec := (*object.Object)(o).GetHeader().GetEC()
|
||||
if ec == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
h := new(ECHeader)
|
||||
_ = h.ReadFromV2(ec)
|
||||
return h
|
||||
}
|
||||
|
||||
func (o *Object) SetECHeader(ec *ECHeader) {
|
||||
o.setHeaderField(func(h *object.Header) {
|
||||
if ec == nil {
|
||||
h.SetEC(nil)
|
||||
return
|
||||
}
|
||||
|
||||
v2 := new(object.ECHeader)
|
||||
ec.WriteToV2(v2)
|
||||
h.SetEC(v2)
|
||||
})
|
||||
}
|
||||
|
||||
func (e *ECHeader) Parent() oid.ID {
|
||||
return e.parent
|
||||
}
|
||||
|
||||
func (e *ECHeader) SetParent(id oid.ID) {
|
||||
e.parent = id
|
||||
}
|
||||
|
||||
func (e *ECHeader) Index() uint32 {
|
||||
return e.index
|
||||
}
|
||||
|
||||
func (e *ECHeader) SetIndex(i uint32) {
|
||||
e.index = i
|
||||
}
|
||||
|
||||
func (e *ECHeader) Total() uint32 {
|
||||
return e.total
|
||||
}
|
||||
|
||||
func (e *ECHeader) SetTotal(i uint32) {
|
||||
e.total = i
|
||||
}
|
||||
|
||||
func (e *ECHeader) Header() []byte {
|
||||
return e.header
|
||||
}
|
||||
|
||||
func (e *ECHeader) SetHeader(header []byte) {
|
||||
e.header = header
|
||||
}
|
||||
|
||||
func (e *ECHeader) HeaderLength() uint32 {
|
||||
return e.headerLength
|
||||
}
|
||||
|
||||
func (e *ECHeader) SetHeaderLength(l uint32) {
|
||||
e.headerLength = l
|
||||
}
|
||||
87
object/erasurecode/constructor.go
Normal file
87
object/erasurecode/constructor.go
Normal file
|
|
@ -0,0 +1,87 @@
|
|||
package erasurecode
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
"github.com/klauspost/reedsolomon"
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrMalformedSlice is returned when a slice of EC chunks is inconsistent.
|
||||
ErrMalformedSlice = errors.New("inconsistent EC headers")
|
||||
// ErrInvShardNum is returned from NewConstructor when the number of shards is invalid.
|
||||
ErrInvShardNum = reedsolomon.ErrInvShardNum
|
||||
// ErrMaxShardNum is returned from NewConstructor when the number of shards is too big.
|
||||
ErrMaxShardNum = reedsolomon.ErrMaxShardNum
|
||||
)
|
||||
|
||||
// MaxShardCount is the maximum number of shards.
|
||||
const MaxShardCount = 256
|
||||
|
||||
// Constructor is a wrapper around encoder allowing to reconstruct objects.
|
||||
// It's methods are not thread-safe.
|
||||
type Constructor struct {
|
||||
enc reedsolomon.Encoder
|
||||
headerLength uint32
|
||||
payloadShards [][]byte
|
||||
headerShards [][]byte
|
||||
}
|
||||
|
||||
// NewConstructor returns new constructor instance.
|
||||
func NewConstructor(dataCount int, parityCount int) (*Constructor, error) {
|
||||
// The library supports up to 65536 shards with some restrictions.
|
||||
// This can easily result in OOM or panic, thus SDK declares it's own restriction.
|
||||
if dataCount+parityCount > MaxShardCount {
|
||||
return nil, ErrMaxShardNum
|
||||
}
|
||||
|
||||
enc, err := reedsolomon.New(dataCount, parityCount)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Constructor{enc: enc}, nil
|
||||
}
|
||||
|
||||
|
dstepanov-yadro marked this conversation as resolved
Outdated
dstepanov-yadro
commented
This looks like invalid usage. I think error must be returned. This looks like invalid usage. I think error must be returned.
|
||||
// clear clears internal state of the constructor, so it can be reused.
|
||||
func (c *Constructor) clear() {
|
||||
c.headerLength = 0
|
||||
c.payloadShards = nil
|
||||
c.headerShards = nil
|
||||
}
|
||||
|
||||
func (c *Constructor) fillHeader(parts []*objectSDK.Object) error {
|
||||
shards := make([][]byte, len(parts))
|
||||
headerLength := 0
|
||||
for i := range parts {
|
||||
if parts[i] == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
var err error
|
||||
headerLength, err = validatePart(parts, i, headerLength)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
shards[i] = parts[i].GetECHeader().Header()
|
||||
}
|
||||
|
dstepanov-yadro marked this conversation as resolved
Outdated
dstepanov-yadro
commented
Again, looks like invalid usage. Again, looks like invalid usage.
fyrchik
commented
These functions accept collected parts from other nodes, some of the parts may be nil -- these are the ones we need to reconstruct. These functions accept collected parts from other nodes, some of the parts may be nil -- these are the ones we need to reconstruct.
|
||||
|
||||
c.headerLength = uint32(headerLength)
|
||||
c.headerShards = shards
|
||||
return nil
|
||||
}
|
||||
|
||||
// fillPayload fills the payload shards.
|
||||
// Currently there is no case when it can be called without reconstructing header,
|
||||
// thus fillHeader() must be called before and this function performs no validation.
|
||||
func (c *Constructor) fillPayload(parts []*objectSDK.Object) {
|
||||
shards := make([][]byte, len(parts))
|
||||
for i := range parts {
|
||||
if parts[i] == nil {
|
||||
continue
|
||||
}
|
||||
shards[i] = parts[i].Payload()
|
||||
}
|
||||
c.payloadShards = shards
|
||||
}
|
||||
31
object/erasurecode/constructor_test.go
Normal file
31
object/erasurecode/constructor_test.go
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
package erasurecode_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/erasurecode"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestErasureConstruct(t *testing.T) {
|
||||
t.Run("negative, no panic", func(t *testing.T) {
|
||||
_, err := erasurecode.NewConstructor(-1, 2)
|
||||
require.ErrorIs(t, err, erasurecode.ErrInvShardNum)
|
||||
})
|
||||
t.Run("negative, no panic", func(t *testing.T) {
|
||||
_, err := erasurecode.NewConstructor(2, -1)
|
||||
require.ErrorIs(t, err, erasurecode.ErrInvShardNum)
|
||||
})
|
||||
t.Run("zero parity", func(t *testing.T) {
|
||||
_, err := erasurecode.NewConstructor(1, 0)
|
||||
require.NoError(t, err)
|
||||
})
|
||||
t.Run("max shard num", func(t *testing.T) {
|
||||
_, err := erasurecode.NewConstructor(erasurecode.MaxShardCount, 0)
|
||||
require.NoError(t, err)
|
||||
})
|
||||
t.Run("max+1 shard num", func(t *testing.T) {
|
||||
_, err := erasurecode.NewConstructor(erasurecode.MaxShardCount+1, 0)
|
||||
require.ErrorIs(t, err, erasurecode.ErrMaxShardNum)
|
||||
})
|
||||
}
|
||||
142
object/erasurecode/reconstruct.go
Normal file
142
object/erasurecode/reconstruct.go
Normal file
|
|
@ -0,0 +1,142 @@
|
|||
package erasurecode
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/ecdsa"
|
||||
"fmt"
|
||||
|
||||
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
"github.com/klauspost/reedsolomon"
|
||||
)
|
||||
|
||||
// Reconstruct returns full object reconstructed from parts.
|
||||
// All non-nil objects in parts must have EC header with the same `total` field equal to len(parts).
|
||||
// The slice must contain at least one non nil object.
|
||||
// Index of the objects in parts must be equal to it's index field in the EC header.
|
||||
// The parts slice isn't changed and can be used concurrently for reading.
|
||||
func (c *Constructor) Reconstruct(parts []*objectSDK.Object) (*objectSDK.Object, error) {
|
||||
res, err := c.ReconstructHeader(parts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
c.fillPayload(parts)
|
||||
|
||||
payload, err := reconstructExact(c.enc, int(res.PayloadSize()), c.payloadShards)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrMalformedSlice, err)
|
||||
}
|
||||
|
||||
res.SetPayload(payload)
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// ReconstructHeader returns object header reconstructed from parts.
|
||||
// All non-nil objects in parts must have EC header with the same `total` field equal to len(parts).
|
||||
// The slice must contain at least one non nil object.
|
||||
// Index of the objects in parts must be equal to it's index field in the EC header.
|
||||
// The parts slice isn't changed and can be used concurrently for reading.
|
||||
func (c *Constructor) ReconstructHeader(parts []*objectSDK.Object) (*objectSDK.Object, error) {
|
||||
c.clear()
|
||||
|
||||
if err := c.fillHeader(parts); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
obj, err := c.reconstructHeader()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrMalformedSlice, err)
|
||||
}
|
||||
return obj, nil
|
||||
}
|
||||
|
||||
// ReconstructParts reconstructs specific EC parts without reconstructing full object.
|
||||
// All non-nil objects in parts must have EC header with the same `total` field equal to len(parts).
|
||||
// The slice must contain at least one non nil object.
|
||||
// Index of the objects in parts must be equal to it's index field in the EC header.
|
||||
// Those parts for which corresponding element in required is true must be nil and will be overwritten.
|
||||
// Because partial reconstruction only makes sense for full objects, all parts must have non-empty payload.
|
||||
// If key is not nil, all reconstructed parts are signed with this key.
|
||||
func (c *Constructor) ReconstructParts(parts []*objectSDK.Object, required []bool, key *ecdsa.PrivateKey) error {
|
||||
if len(required) != len(parts) {
|
||||
return fmt.Errorf("len(parts) != len(required): %d != %d", len(parts), len(required))
|
||||
}
|
||||
|
||||
c.clear()
|
||||
|
||||
if err := c.fillHeader(parts); err != nil {
|
||||
return err
|
||||
}
|
||||
c.fillPayload(parts)
|
||||
|
||||
if err := c.enc.ReconstructSome(c.payloadShards, required); err != nil {
|
||||
return fmt.Errorf("%w: %w", ErrMalformedSlice, err)
|
||||
}
|
||||
if err := c.enc.ReconstructSome(c.headerShards, required); err != nil {
|
||||
return fmt.Errorf("%w: %w", ErrMalformedSlice, err)
|
||||
}
|
||||
|
||||
nonNilPart := 0
|
||||
for i := range parts {
|
||||
if parts[i] != nil {
|
||||
nonNilPart = i
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
ec := parts[nonNilPart].GetECHeader()
|
||||
parent := ec.Parent()
|
||||
total := ec.Total()
|
||||
|
||||
for i := range required {
|
||||
if parts[i] != nil || !required[i] {
|
||||
continue
|
||||
}
|
||||
|
||||
part := objectSDK.New()
|
||||
copyRequiredFields(part, parts[nonNilPart])
|
||||
part.SetPayload(c.payloadShards[i])
|
||||
part.SetPayloadSize(uint64(len(c.payloadShards[i])))
|
||||
part.SetECHeader(objectSDK.NewECHeader(parent, uint32(i), total,
|
||||
c.headerShards[i], c.headerLength))
|
||||
|
||||
if err := setIDWithSignature(part, key); err != nil {
|
||||
return err
|
||||
}
|
||||
parts[i] = part
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Constructor) reconstructHeader() (*objectSDK.Object, error) {
|
||||
data, err := reconstructExact(c.enc, int(c.headerLength), c.headerShards)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var obj objectSDK.Object
|
||||
return &obj, obj.Unmarshal(data)
|
||||
}
|
||||
|
||||
func reconstructExact(enc reedsolomon.Encoder, size int, shards [][]byte) ([]byte, error) {
|
||||
if err := enc.ReconstructData(shards); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Technically, this error will be returned from enc.Join().
|
||||
// However, allocating based on unvalidated user data is an easy attack vector.
|
||||
// Preallocating seems to have enough benefits to justify a slight increase in code complexity.
|
||||
maxSize := 0
|
||||
for i := range shards {
|
||||
maxSize += len(shards[i])
|
||||
}
|
||||
if size > maxSize {
|
||||
return nil, reedsolomon.ErrShortData
|
||||
}
|
||||
|
||||
buf := bytes.NewBuffer(make([]byte, 0, size))
|
||||
if err := enc.Join(buf, shards, size); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
284
object/erasurecode/reconstruct_test.go
Normal file
284
object/erasurecode/reconstruct_test.go
Normal file
|
|
@ -0,0 +1,284 @@
|
|||
package erasurecode_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"math"
|
||||
"testing"
|
||||
|
||||
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
|
||||
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/erasurecode"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version"
|
||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestErasureCodeReconstruct(t *testing.T) {
|
||||
const payloadSize = 99
|
||||
const dataCount = 3
|
||||
const parityCount = 2
|
||||
|
||||
// We would also like to test padding behaviour,
|
||||
// so ensure padding is done.
|
||||
require.NotZero(t, payloadSize%(dataCount+parityCount))
|
||||
|
||||
pk, err := keys.NewPrivateKey()
|
||||
require.NoError(t, err)
|
||||
|
||||
original := newObject(t, payloadSize, pk)
|
||||
|
||||
c, err := erasurecode.NewConstructor(dataCount, parityCount)
|
||||
require.NoError(t, err)
|
||||
|
||||
parts, err := c.Split(original, &pk.PrivateKey)
|
||||
require.NoError(t, err)
|
||||
|
||||
t.Run("reconstruct header", func(t *testing.T) {
|
||||
original := original.CutPayload()
|
||||
parts := cloneSlice(parts)
|
||||
for i := range parts {
|
||||
parts[i] = parts[i].CutPayload()
|
||||
}
|
||||
t.Run("from data", func(t *testing.T) {
|
||||
parts := cloneSlice(parts)
|
||||
for i := dataCount; i < dataCount+parityCount; i++ {
|
||||
parts[i] = nil
|
||||
}
|
||||
reconstructed, err := c.ReconstructHeader(parts)
|
||||
require.NoError(t, err)
|
||||
verifyReconstruction(t, original, reconstructed)
|
||||
})
|
||||
t.Run("from parity", func(t *testing.T) {
|
||||
parts := cloneSlice(parts)
|
||||
for i := 0; i < parityCount; i++ {
|
||||
parts[i] = nil
|
||||
}
|
||||
reconstructed, err := c.ReconstructHeader(parts)
|
||||
require.NoError(t, err)
|
||||
verifyReconstruction(t, original, reconstructed)
|
||||
|
||||
t.Run("not enough shards", func(t *testing.T) {
|
||||
parts[parityCount] = nil
|
||||
_, err := c.ReconstructHeader(parts)
|
||||
require.ErrorIs(t, err, erasurecode.ErrMalformedSlice)
|
||||
})
|
||||
})
|
||||
t.Run("only nil parts", func(t *testing.T) {
|
||||
parts := make([]*objectSDK.Object, len(parts))
|
||||
_, err := c.ReconstructHeader(parts)
|
||||
require.ErrorIs(t, err, erasurecode.ErrMalformedSlice)
|
||||
})
|
||||
t.Run("missing EC header", func(t *testing.T) {
|
||||
parts := cloneSlice(parts)
|
||||
parts[0] = deepCopy(t, parts[0])
|
||||
parts[0].SetECHeader(nil)
|
||||
|
||||
_, err := c.ReconstructHeader(parts)
|
||||
require.ErrorIs(t, err, erasurecode.ErrMalformedSlice)
|
||||
})
|
||||
t.Run("invalid index", func(t *testing.T) {
|
||||
parts := cloneSlice(parts)
|
||||
parts[0] = deepCopy(t, parts[0])
|
||||
|
||||
ec := parts[0].GetECHeader()
|
||||
ec.SetIndex(1)
|
||||
parts[0].SetECHeader(ec)
|
||||
|
||||
_, err := c.ReconstructHeader(parts)
|
||||
require.ErrorIs(t, err, erasurecode.ErrMalformedSlice)
|
||||
})
|
||||
t.Run("invalid total", func(t *testing.T) {
|
||||
parts := cloneSlice(parts)
|
||||
parts[0] = deepCopy(t, parts[0])
|
||||
|
||||
ec := parts[0].GetECHeader()
|
||||
ec.SetTotal(uint32(len(parts) + 1))
|
||||
parts[0].SetECHeader(ec)
|
||||
|
||||
_, err := c.ReconstructHeader(parts)
|
||||
require.ErrorIs(t, err, erasurecode.ErrMalformedSlice)
|
||||
})
|
||||
t.Run("inconsistent header length", func(t *testing.T) {
|
||||
parts := cloneSlice(parts)
|
||||
parts[0] = deepCopy(t, parts[0])
|
||||
|
||||
ec := parts[0].GetECHeader()
|
||||
ec.SetHeaderLength(ec.HeaderLength() - 1)
|
||||
parts[0].SetECHeader(ec)
|
||||
|
||||
_, err := c.ReconstructHeader(parts)
|
||||
require.ErrorIs(t, err, erasurecode.ErrMalformedSlice)
|
||||
})
|
||||
t.Run("invalid header length", func(t *testing.T) {
|
||||
parts := cloneSlice(parts)
|
||||
for i := range parts {
|
||||
parts[i] = deepCopy(t, parts[i])
|
||||
|
||||
ec := parts[0].GetECHeader()
|
||||
ec.SetHeaderLength(math.MaxUint32)
|
||||
parts[0].SetECHeader(ec)
|
||||
}
|
||||
|
||||
_, err := c.ReconstructHeader(parts)
|
||||
require.ErrorIs(t, err, erasurecode.ErrMalformedSlice)
|
||||
})
|
||||
})
|
||||
t.Run("reconstruct data", func(t *testing.T) {
|
||||
t.Run("from data", func(t *testing.T) {
|
||||
parts := cloneSlice(parts)
|
||||
for i := dataCount; i < dataCount+parityCount; i++ {
|
||||
parts[i] = nil
|
||||
}
|
||||
reconstructed, err := c.Reconstruct(parts)
|
||||
require.NoError(t, err)
|
||||
verifyReconstruction(t, original, reconstructed)
|
||||
})
|
||||
t.Run("from parity", func(t *testing.T) {
|
||||
parts := cloneSlice(parts)
|
||||
for i := 0; i < parityCount; i++ {
|
||||
parts[i] = nil
|
||||
}
|
||||
reconstructed, err := c.Reconstruct(parts)
|
||||
require.NoError(t, err)
|
||||
verifyReconstruction(t, original, reconstructed)
|
||||
|
||||
t.Run("not enough shards", func(t *testing.T) {
|
||||
parts[parityCount] = nil
|
||||
_, err := c.Reconstruct(parts)
|
||||
require.ErrorIs(t, err, erasurecode.ErrMalformedSlice)
|
||||
})
|
||||
})
|
||||
})
|
||||
t.Run("reconstruct parts", func(t *testing.T) {
|
||||
// We would like to also test that ReconstructParts doesn't perform
|
||||
// excessive work, so ensure this test makes sense.
|
||||
require.GreaterOrEqual(t, parityCount, 2)
|
||||
|
||||
t.Run("from data", func(t *testing.T) {
|
||||
oldParts := parts
|
||||
parts := cloneSlice(parts)
|
||||
for i := dataCount; i < dataCount+parityCount; i++ {
|
||||
parts[i] = nil
|
||||
}
|
||||
|
||||
required := make([]bool, len(parts))
|
||||
required[dataCount] = true
|
||||
|
||||
require.NoError(t, c.ReconstructParts(parts, required, nil))
|
||||
|
||||
old := deepCopy(t, oldParts[dataCount])
|
||||
old.SetSignature(nil)
|
||||
require.Equal(t, old, parts[dataCount])
|
||||
|
||||
for i := dataCount + 1; i < dataCount+parityCount; i++ {
|
||||
require.Nil(t, parts[i])
|
||||
}
|
||||
})
|
||||
t.Run("from parity", func(t *testing.T) {
|
||||
oldParts := parts
|
||||
parts := cloneSlice(parts)
|
||||
for i := 0; i < parityCount; i++ {
|
||||
parts[i] = nil
|
||||
}
|
||||
|
||||
required := make([]bool, len(parts))
|
||||
required[0] = true
|
||||
|
||||
require.NoError(t, c.ReconstructParts(parts, required, nil))
|
||||
|
||||
old := deepCopy(t, oldParts[0])
|
||||
old.SetSignature(nil)
|
||||
require.Equal(t, old, parts[0])
|
||||
|
||||
for i := 1; i < parityCount; i++ {
|
||||
require.Nil(t, parts[i])
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func newObject(t *testing.T, size uint64, pk *keys.PrivateKey) *objectSDK.Object {
|
||||
// Use transformer to form object to avoid potential bugs with yet another helper object creation in tests.
|
||||
tt := &testTarget{}
|
||||
p := transformer.NewPayloadSizeLimiter(transformer.Params{
|
||||
Key: &pk.PrivateKey,
|
||||
NextTargetInit: func() transformer.ObjectWriter { return tt },
|
||||
NetworkState: dummyEpochSource(123),
|
||||
MaxSize: size + 1,
|
||||
WithoutHomomorphicHash: true,
|
||||
})
|
||||
cnr := cidtest.ID()
|
||||
ver := version.Current()
|
||||
hdr := objectSDK.New()
|
||||
hdr.SetContainerID(cnr)
|
||||
hdr.SetType(objectSDK.TypeRegular)
|
||||
hdr.SetVersion(&ver)
|
||||
|
||||
var owner user.ID
|
||||
user.IDFromKey(&owner, pk.PrivateKey.PublicKey)
|
||||
hdr.SetOwnerID(owner)
|
||||
|
||||
var attr objectSDK.Attribute
|
||||
attr.SetKey("somekey")
|
||||
attr.SetValue("somevalue")
|
||||
hdr.SetAttributes(attr)
|
||||
|
||||
expectedPayload := make([]byte, size)
|
||||
_, _ = rand.Read(expectedPayload)
|
||||
writeObject(t, context.Background(), p, hdr, expectedPayload)
|
||||
require.Len(t, tt.objects, 1)
|
||||
return tt.objects[0]
|
||||
}
|
||||
|
||||
func writeObject(t *testing.T, ctx context.Context, target transformer.ChunkedObjectWriter, header *objectSDK.Object, payload []byte) *transformer.AccessIdentifiers {
|
||||
require.NoError(t, target.WriteHeader(ctx, header))
|
||||
|
||||
_, err := target.Write(ctx, payload)
|
||||
require.NoError(t, err)
|
||||
|
||||
ids, err := target.Close(ctx)
|
||||
require.NoError(t, err)
|
||||
|
||||
return ids
|
||||
}
|
||||
|
||||
func verifyReconstruction(t *testing.T, original, reconstructed *objectSDK.Object) {
|
||||
require.True(t, reconstructed.VerifyIDSignature())
|
||||
reconstructed.ToV2().SetMarshalData(nil)
|
||||
original.ToV2().SetMarshalData(nil)
|
||||
|
||||
require.Equal(t, original, reconstructed)
|
||||
}
|
||||
|
||||
func deepCopy(t *testing.T, obj *objectSDK.Object) *objectSDK.Object {
|
||||
data, err := obj.Marshal()
|
||||
require.NoError(t, err)
|
||||
|
||||
res := objectSDK.New()
|
||||
require.NoError(t, res.Unmarshal(data))
|
||||
return res
|
||||
}
|
||||
|
||||
func cloneSlice[T any](src []T) []T {
|
||||
dst := make([]T, len(src))
|
||||
copy(dst, src)
|
||||
return dst
|
||||
}
|
||||
|
||||
type dummyEpochSource uint64
|
||||
|
||||
func (s dummyEpochSource) CurrentEpoch() uint64 {
|
||||
return uint64(s)
|
||||
}
|
||||
|
||||
type testTarget struct {
|
||||
objects []*objectSDK.Object
|
||||
}
|
||||
|
||||
func (tt *testTarget) WriteObject(_ context.Context, o *objectSDK.Object) error {
|
||||
tt.objects = append(tt.objects, o)
|
||||
return nil // AccessIdentifiers should not be used.
|
||||
}
|
||||
69
object/erasurecode/split.go
Normal file
69
object/erasurecode/split.go
Normal file
|
|
@ -0,0 +1,69 @@
|
|||
package erasurecode
|
||||
|
||||
import (
|
||||
"crypto/ecdsa"
|
||||
|
||||
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
)
|
||||
|
||||
// Split splits fully formed object into multiple chunks.
|
||||
func (c *Constructor) Split(obj *objectSDK.Object, key *ecdsa.PrivateKey) ([]*objectSDK.Object, error) {
|
||||
c.clear()
|
||||
|
||||
header, err := obj.CutPayload().Marshal()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
headerShards, err := c.encodeRaw(header)
|
||||
aarifullin
commented
I've got a question: you separate header and a payload and then marshal the header. I've got a question: you separate header and a payload and then marshal the header.
I thought that the number of EC-encoded parts depends on binary size (`header_marshalled_size != payload_marshalled_size`) but here you expect the header shards number is equal to payload shards number. How come?
fyrchik
commented
Size is not equal, but the number of shards is equal. Size is not equal, but the number of shards is equal.
It follows, that the size of chunks are not equal.
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
payloadShards, err := c.encodeRaw(obj.Payload())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
parts := make([]*objectSDK.Object, len(payloadShards))
|
||||
parent, _ := obj.ID()
|
||||
for i := range parts {
|
||||
chunk := objectSDK.New()
|
||||
copyRequiredFields(chunk, obj)
|
||||
chunk.SetPayload(payloadShards[i])
|
||||
chunk.SetPayloadSize(uint64(len(payloadShards[i])))
|
||||
|
||||
ec := objectSDK.NewECHeader(parent, uint32(i), uint32(len(payloadShards)), headerShards[i], uint32(len(header)))
|
||||
chunk.SetECHeader(ec)
|
||||
if err := setIDWithSignature(chunk, key); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
parts[i] = chunk
|
||||
}
|
||||
return parts, nil
|
||||
}
|
||||
|
||||
func setIDWithSignature(obj *objectSDK.Object, key *ecdsa.PrivateKey) error {
|
||||
if err := objectSDK.CalculateAndSetID(obj); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
objectSDK.CalculateAndSetPayloadChecksum(obj)
|
||||
|
||||
if key == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
return objectSDK.CalculateAndSetSignature(*key, obj)
|
||||
}
|
||||
|
||||
func (c *Constructor) encodeRaw(data []byte) ([][]byte, error) {
|
||||
shards, err := c.enc.Split(data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := c.enc.Encode(shards); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return shards, nil
|
||||
}
|
||||
36
object/erasurecode/split_test.go
Normal file
36
object/erasurecode/split_test.go
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
package erasurecode_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/erasurecode"
|
||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// The library can behave differently for big shard counts.
|
||||
// This test checks we support the maximum number of chunks we promise.
|
||||
func TestSplitMaxShardCount(t *testing.T) {
|
||||
pk, err := keys.NewPrivateKey()
|
||||
require.NoError(t, err)
|
||||
|
||||
original := newObject(t, 1024, pk)
|
||||
|
||||
t.Run("only data", func(t *testing.T) {
|
||||
c, err := erasurecode.NewConstructor(erasurecode.MaxShardCount, 0)
|
||||
require.NoError(t, err)
|
||||
|
||||
parts, err := c.Split(original, &pk.PrivateKey)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, parts, erasurecode.MaxShardCount)
|
||||
})
|
||||
t.Run("data + parity", func(t *testing.T) {
|
||||
c, err := erasurecode.NewConstructor(1, erasurecode.MaxShardCount-1)
|
||||
require.NoError(t, err)
|
||||
|
||||
parts, err := c.Split(original, &pk.PrivateKey)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, parts, erasurecode.MaxShardCount)
|
||||
})
|
||||
|
||||
}
|
||||
44
object/erasurecode/target.go
Normal file
44
object/erasurecode/target.go
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
package erasurecode
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ecdsa"
|
||||
|
||||
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer"
|
||||
|
fyrchik
commented
Circular dependency, but only for interfaces, will fix. Circular dependency, but only for interfaces, will fix.
|
||||
)
|
||||
|
||||
// Target accepts regular objects and splits them into erasure-coded chunks.
|
||||
type Target struct {
|
||||
c *Constructor
|
||||
key *ecdsa.PrivateKey
|
||||
next transformer.ObjectWriter
|
||||
}
|
||||
|
||||
// ObjectWriter is an interface of the object writer that writes prepared object.
|
||||
type ObjectWriter interface {
|
||||
WriteObject(context.Context, *objectSDK.Object) error
|
||||
}
|
||||
|
||||
// NewTarget returns new target instance.
|
||||
func NewTarget(c *Constructor, key *ecdsa.PrivateKey, next ObjectWriter) *Target {
|
||||
return &Target{
|
||||
c: c,
|
||||
key: key,
|
||||
next: next,
|
||||
}
|
||||
}
|
||||
|
||||
// WriteObject implements the transformer.ObjectWriter interface.
|
||||
func (t *Target) WriteObject(ctx context.Context, obj *objectSDK.Object) error {
|
||||
parts, err := t.c.Split(obj, t.key)
|
||||
|
dstepanov-yadro
commented
Hm, this is already done in Hm, this is already done in `Constructor.Split`. Also this is `Constructor`'s internal method.
fyrchik
commented
true, fixed true, fixed
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for i := range parts {
|
||||
if err := t.next.WriteObject(ctx, parts[i]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
104
object/erasurecode/verify.go
Normal file
104
object/erasurecode/verify.go
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
package erasurecode
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
)
|
||||
|
||||
// Verify verifies that parts are well formed.
|
||||
// All parts are expected to be non-nil.
|
||||
// The number of parts must be equal to `total` field of the EC header
|
||||
// and parts must be sorted by index.
|
||||
func (c *Constructor) Verify(parts []*objectSDK.Object) error {
|
||||
c.clear()
|
||||
|
||||
|
dstepanov-yadro
commented
Parts need to be sorted by index according this check in It is important. Need to specify in comment. Parts need to be sorted by index according this check in `validatePart`:
```
if ec.Index() != uint32(i) {
return headerLength, fmt.Errorf("%w: index=%d, ec.index=%d", ErrMalformedSlice, i, ec.Index())
}
```
It is important. Need to specify in comment.
But better to create slice copy (it is slice of pointers), sort it and validate against copy.
fyrchik
commented
But this is exactly what But this is exactly what `Verify` is intended to catch: it accepts sorted slice of parts (actually it is more of a testing thing, but can also be used to verify reconstruction).
|
||||
var headerLength int
|
||||
for i := range parts {
|
||||
if parts[i] == nil {
|
||||
return ErrMalformedSlice
|
||||
}
|
||||
|
||||
var err error
|
||||
headerLength, err = validatePart(parts, i, headerLength)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
p0 := parts[0]
|
||||
for i := 1; i < len(parts); i++ {
|
||||
// This part must be kept in sync with copyRequiredFields().
|
||||
pi := parts[i]
|
||||
if p0.OwnerID().Equals(pi.OwnerID()) {
|
||||
return fmt.Errorf("%w: owner id mismatch: %s != %s", ErrMalformedSlice, p0.OwnerID(), pi.OwnerID())
|
||||
}
|
||||
if p0.Version() == nil && pi.Version() != nil || !p0.Version().Equal(*pi.Version()) {
|
||||
return fmt.Errorf("%w: version mismatch: %s != %s", ErrMalformedSlice, p0.Version(), pi.Version())
|
||||
}
|
||||
|
||||
cnr0, _ := p0.ContainerID()
|
||||
cnri, _ := pi.ContainerID()
|
||||
if !cnr0.Equals(cnri) {
|
||||
return fmt.Errorf("%w: container id mismatch: %s != %s", ErrMalformedSlice, cnr0, cnri)
|
||||
}
|
||||
}
|
||||
|
||||
if err := c.fillHeader(parts); err != nil {
|
||||
return err
|
||||
}
|
||||
c.fillPayload(parts)
|
||||
|
||||
ok, err := c.enc.Verify(c.headerShards)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !ok {
|
||||
return ErrMalformedSlice
|
||||
}
|
||||
|
||||
ok, err = c.enc.Verify(c.payloadShards)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !ok {
|
||||
return ErrMalformedSlice
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// copyRequiredFields sets all fields in dst which are copied from src and shared among all chunks.
|
||||
// src can be either another chunk of full object.
|
||||
// dst must be a chunk.
|
||||
func copyRequiredFields(dst *objectSDK.Object, src *objectSDK.Object) {
|
||||
dst.SetVersion(src.Version())
|
||||
dst.SetOwnerID(src.OwnerID())
|
||||
dst.SetCreationEpoch(src.CreationEpoch())
|
||||
dst.SetSessionToken(src.SessionToken())
|
||||
|
||||
cnr, _ := src.ContainerID()
|
||||
dst.SetContainerID(cnr)
|
||||
}
|
||||
|
||||
// validatePart makes i-th part is consistent with the rest.
|
||||
// If headerLength is not zero it is asserted to be equal in the ec header.
|
||||
// Otherwise, new headerLength is returned.
|
||||
func validatePart(parts []*objectSDK.Object, i int, headerLength int) (int, error) {
|
||||
ec := parts[i].GetECHeader()
|
||||
|
aarifullin marked this conversation as resolved
aarifullin
commented
What do you think about adding What do you think about adding `i >= 0 && i < len(parts)` check?
fyrchik
commented
This is a private method and is always called in a loop in 2 places, I think we are good here. This is a private method and is always called in a loop in 2 places, I think we are good here.
|
||||
if ec == nil {
|
||||
return headerLength, fmt.Errorf("%w: missing EC header", ErrMalformedSlice)
|
||||
}
|
||||
if ec.Index() != uint32(i) {
|
||||
return headerLength, fmt.Errorf("%w: index=%d, ec.index=%d", ErrMalformedSlice, i, ec.Index())
|
||||
}
|
||||
if ec.Total() != uint32(len(parts)) {
|
||||
return headerLength, fmt.Errorf("%w: len(parts)=%d, total=%d", ErrMalformedSlice, len(parts), ec.Total())
|
||||
}
|
||||
if headerLength == 0 {
|
||||
return int(ec.HeaderLength()), nil
|
||||
}
|
||||
if ec.HeaderLength() != uint32(headerLength) {
|
||||
return headerLength, fmt.Errorf("%w: header length mismatch %d != %d", ErrMalformedSlice, headerLength, ec.HeaderLength())
|
||||
}
|
||||
return headerLength, nil
|
||||
}
|
||||
|
|
@ -366,6 +366,14 @@ func (o *Object) Children() []oid.ID {
|
|||
return res
|
||||
}
|
||||
|
||||
func (o *Object) GetECHeader() *ECHeader {
|
||||
v2 := (*object.Object)(o).GetHeader().GetEC()
|
||||
|
||||
var ec ECHeader
|
||||
_ = ec.ReadFromV2(v2) // Errors is checked on unmarshal.
|
||||
return &ec
|
||||
}
|
||||
|
||||
// SetChildren sets list of the identifiers of the child objects.
|
||||
func (o *Object) SetChildren(v ...oid.ID) {
|
||||
var (
|
||||
|
|
|
|||
Loading…
Reference in a new issue
This means current statement will be skipped. Why?
We are interested only in EC and REP statements here and in their order.
Previously it was
AllRepStmt, now we get all children and filter all non-REP and non-EC statements, they will be processed below.