forked from TrueCloudLab/frostfs-sdk-go
Compare commits
49 commits
master
...
feat/searc
Author | SHA1 | Date | |
---|---|---|---|
0d053826b4 | |||
12ddefe078 | |||
20ab57bf7e | |||
3790142b10 | |||
ec0cb2169f | |||
425d48f68b | |||
6d0da3f861 | |||
1af9b6d18b | |||
bd2d350b09 | |||
e9be3e6d94 | |||
70e9e40c7f | |||
d33b54d280 | |||
6f248436a5 | |||
edd40474e8 | |||
d9ec7c1988 | |||
64b83f8220 | |||
7212f38115 | |||
8081445ff2 | |||
6a7ef9d8c3 | |||
6fe4e2541d | |||
a5fab572ff | |||
a86170f53a | |||
aa41f71dcc | |||
3a00fd51e4 | |||
65b4525b3b | |||
7efff9d53d | |||
110b7e4170 | |||
56debcfa56 | |||
157a9930e8 | |||
1c07098740 | |||
03d35dd1f3 | |||
dea8759762 | |||
3787477133 | |||
e91d40e250 | |||
ab75edd709 | |||
8999d2f080 | |||
6fbe1595cb | |||
a9237aabd2 | |||
a487033505 | |||
51c3618850 | |||
665e5807bc | |||
a02c0bfac8 | |||
20d325e307 | |||
670619d242 | |||
0d79d10482 | |||
9727beb47d | |||
84315fab6a | |||
71335489ae | |||
4c1feaf2cb |
122 changed files with 4391 additions and 3307 deletions
|
@ -16,6 +16,6 @@ jobs:
|
|||
go-version: '1.21'
|
||||
|
||||
- name: Run commit format checker
|
||||
uses: https://git.frostfs.info/TrueCloudLab/dco-go@v2
|
||||
uses: https://git.frostfs.info/TrueCloudLab/dco-go@v3
|
||||
with:
|
||||
from: 'origin/${{ github.event.pull_request.base.ref }}'
|
||||
|
|
|
@ -8,17 +8,24 @@ jobs:
|
|||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: golangci-lint
|
||||
uses: https://github.com/golangci/golangci-lint-action@v2
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v3
|
||||
with:
|
||||
version: latest
|
||||
go-version: '1.21'
|
||||
cache: true
|
||||
|
||||
- name: Install linters
|
||||
run: make lint-install
|
||||
|
||||
- name: Run linters
|
||||
run: make lint
|
||||
|
||||
tests:
|
||||
name: Tests
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
go_versions: [ '1.19', '1.20' ]
|
||||
go_versions: [ '1.20', '1.21' ]
|
||||
fail-fast: false
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
|
4
.gitignore
vendored
4
.gitignore
vendored
|
@ -27,3 +27,7 @@ antlr-*.jar
|
|||
|
||||
# tempfiles
|
||||
.cache
|
||||
|
||||
# binary
|
||||
bin/
|
||||
release/
|
||||
|
|
10
.gitlint
10
.gitlint
|
@ -1,10 +0,0 @@
|
|||
[general]
|
||||
fail-without-commits=true
|
||||
contrib=CC1
|
||||
|
||||
[title-match-regex]
|
||||
regex=^\[\#[0-9Xx]+\]\s
|
||||
|
||||
[ignore-by-title]
|
||||
regex=^Release(.*)
|
||||
ignore=title-match-regex
|
|
@ -18,13 +18,19 @@ repos:
|
|||
- id: end-of-file-fixer
|
||||
exclude: "(.key|.interp|.tokens)$"
|
||||
|
||||
- repo: https://github.com/golangci/golangci-lint
|
||||
rev: v1.51.2
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: golangci-lint
|
||||
- id: go-unit-tests
|
||||
name: go unit tests
|
||||
entry: make test GOFLAGS=''
|
||||
pass_filenames: false
|
||||
types: [go]
|
||||
language: system
|
||||
|
||||
- repo: https://github.com/jorisroovers/gitlint
|
||||
rev: v0.18.0
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: gitlint
|
||||
stages: [commit-msg]
|
||||
- id: make-lint
|
||||
name: Run Make Lint
|
||||
entry: make lint
|
||||
language: system
|
||||
pass_filenames: false
|
||||
|
|
24
Makefile
24
Makefile
|
@ -1,10 +1,16 @@
|
|||
#!/usr/bin/make -f
|
||||
|
||||
ANTLR_VERSION="4.13.0"
|
||||
TMP_DIR := .cache
|
||||
LINT_VERSION ?= 1.56.2
|
||||
TRUECLOUDLAB_LINT_VERSION ?= 0.0.2
|
||||
OUTPUT_LINT_DIR ?= $(shell pwd)/bin
|
||||
LINT_DIR = $(OUTPUT_LINT_DIR)/golangci-lint-$(LINT_VERSION)-v$(TRUECLOUDLAB_LINT_VERSION)
|
||||
|
||||
# Run tests
|
||||
test: GOFLAGS ?= "-cover -count=1"
|
||||
test:
|
||||
@go test ./... -cover
|
||||
@GOFLAGS=$(GOFLAGS) go test ./...
|
||||
|
||||
# Pull go dependencies
|
||||
dep:
|
||||
|
@ -15,9 +21,23 @@ dep:
|
|||
@CGO_ENABLED=0 \
|
||||
go mod tidy -v && echo OK
|
||||
|
||||
# Install linters
|
||||
lint-install:
|
||||
@mkdir -p $(TMP_DIR)
|
||||
@rm -rf $(TMP_DIR)/linters
|
||||
@git -c advice.detachedHead=false clone --branch v$(TRUECLOUDLAB_LINT_VERSION) https://git.frostfs.info/TrueCloudLab/linters.git $(TMP_DIR)/linters
|
||||
@@make -C $(TMP_DIR)/linters lib CGO_ENABLED=1 OUT_DIR=$(OUTPUT_LINT_DIR)
|
||||
@rm -rf $(TMP_DIR)/linters
|
||||
@rmdir $(TMP_DIR) 2>/dev/null || true
|
||||
@CGO_ENABLED=1 GOBIN=$(LINT_DIR) go install github.com/golangci/golangci-lint/cmd/golangci-lint@v$(LINT_VERSION)
|
||||
|
||||
# Run linters
|
||||
lint:
|
||||
@golangci-lint --timeout=5m run
|
||||
@if [ ! -d "$(LINT_DIR)" ]; then \
|
||||
echo "Run make lint-install"; \
|
||||
exit 1; \
|
||||
fi
|
||||
$(LINT_DIR)/golangci-lint run
|
||||
|
||||
# Run tests with race detection and produce coverage output
|
||||
cover:
|
||||
|
|
|
@ -42,7 +42,6 @@ Contains client for working with FrostFS.
|
|||
```go
|
||||
var prmInit client.PrmInit
|
||||
prmInit.SetDefaultPrivateKey(key) // private key for request signing
|
||||
prmInit.ResolveFrostFSFailures() // enable erroneous status parsing
|
||||
|
||||
var c client.Client
|
||||
c.Init(prmInit)
|
||||
|
@ -77,8 +76,7 @@ if needed and perform any desired action. In the case above we may want to repor
|
|||
these details to the user as well as retry an operation, possibly with different parameters.
|
||||
Status wire-format is extendable and each node can report any set of details it wants.
|
||||
The set of reserved status codes can be found in
|
||||
[FrostFS API](https://git.frostfs.info/TrueCloudLab/frostfs-api/src/branch/master/status/types.proto). There is also
|
||||
a `client.PrmInit.ResolveFrostFSFailures()` to seamlessly convert erroneous statuses into Go error type.
|
||||
[FrostFS API](https://git.frostfs.info/TrueCloudLab/frostfs-api/src/branch/master/status/types.proto).
|
||||
|
||||
### policy
|
||||
Contains helpers allowing conversion of placing policy from/to JSON representation
|
||||
|
|
|
@ -107,7 +107,7 @@ func TestToken_ForUser(t *testing.T) {
|
|||
require.Zero(t, m.GetBody())
|
||||
|
||||
// set value
|
||||
usr := *usertest.ID()
|
||||
usr := usertest.ID()
|
||||
|
||||
var usrV2 refs.OwnerID
|
||||
usr.WriteToV2(&usrV2)
|
||||
|
@ -243,11 +243,11 @@ func TestToken_AssertContainer(t *testing.T) {
|
|||
|
||||
func TestToken_AssertUser(t *testing.T) {
|
||||
var val bearer.Token
|
||||
usr := *usertest.ID()
|
||||
usr := usertest.ID()
|
||||
|
||||
require.True(t, val.AssertUser(usr))
|
||||
|
||||
val.ForUser(*usertest.ID())
|
||||
val.ForUser(usertest.ID())
|
||||
require.False(t, val.AssertUser(usr))
|
||||
|
||||
val.ForUser(usr)
|
||||
|
@ -332,7 +332,7 @@ func TestToken_ReadFromV2(t *testing.T) {
|
|||
val.WriteToV2(&m2)
|
||||
require.Equal(t, m, m2)
|
||||
|
||||
usr, usr2 := *usertest.ID(), *usertest.ID()
|
||||
usr, usr2 := usertest.ID(), usertest.ID()
|
||||
|
||||
require.True(t, val.AssertUser(usr))
|
||||
require.True(t, val.AssertUser(usr2))
|
||||
|
|
|
@ -13,7 +13,7 @@ func Token() (t bearer.Token) {
|
|||
t.SetExp(3)
|
||||
t.SetNbf(2)
|
||||
t.SetIat(1)
|
||||
t.ForUser(*usertest.ID())
|
||||
t.ForUser(usertest.ID())
|
||||
t.SetEACLTable(*eacltest.Table())
|
||||
|
||||
return t
|
||||
|
|
|
@ -17,26 +17,26 @@ import (
|
|||
|
||||
// PrmBalanceGet groups parameters of BalanceGet operation.
|
||||
type PrmBalanceGet struct {
|
||||
prmCommonMeta
|
||||
XHeaders []string
|
||||
|
||||
accountSet bool
|
||||
account user.ID
|
||||
Account user.ID
|
||||
}
|
||||
|
||||
// SetAccount sets identifier of the FrostFS account for which the balance is requested.
|
||||
// Required parameter.
|
||||
//
|
||||
// Deprecated: Use PrmBalanceGet.Account instead.
|
||||
func (x *PrmBalanceGet) SetAccount(id user.ID) {
|
||||
x.account = id
|
||||
x.accountSet = true
|
||||
x.Account = id
|
||||
}
|
||||
|
||||
func (x *PrmBalanceGet) buildRequest(c *Client) (*v2accounting.BalanceRequest, error) {
|
||||
if !x.accountSet {
|
||||
if x.Account.IsEmpty() {
|
||||
return nil, errorAccountNotSet
|
||||
}
|
||||
|
||||
var accountV2 refs.OwnerID
|
||||
x.account.WriteToV2(&accountV2)
|
||||
x.Account.WriteToV2(&accountV2)
|
||||
|
||||
var body v2accounting.BalanceRequestBody
|
||||
body.SetOwnerID(&accountV2)
|
||||
|
@ -64,9 +64,9 @@ func (x ResBalanceGet) Amount() accounting.Decimal {
|
|||
//
|
||||
// Exactly one return value is non-nil. By default, server status is returned in res structure.
|
||||
// Any client's internal or transport errors are returned as `error`,
|
||||
// If PrmInit.ResolveFrostFSFailures has been called, unsuccessful
|
||||
// FrostFS status codes are returned as `error`, otherwise, are included
|
||||
// in the returned result structure.
|
||||
// If PrmInit.DisableFrostFSFailuresResolution has been called, unsuccessful
|
||||
// FrostFS status codes are included in the returned result structure,
|
||||
// otherwise, are also returned as `error`.
|
||||
//
|
||||
// Returns an error if parameters are set incorrectly (see PrmBalanceGet docs).
|
||||
// Context is required and must not be nil. It is used for network communication.
|
||||
|
@ -79,7 +79,7 @@ func (c *Client) BalanceGet(ctx context.Context, prm PrmBalanceGet) (*ResBalance
|
|||
return nil, err
|
||||
}
|
||||
|
||||
if err := signature.SignServiceMessage(&c.prm.key, req); err != nil {
|
||||
if err := signature.SignServiceMessage(&c.prm.Key, req); err != nil {
|
||||
return nil, fmt.Errorf("sign request: %w", err)
|
||||
}
|
||||
|
||||
|
|
100
client/client.go
100
client/client.go
|
@ -78,31 +78,22 @@ func (c *Client) Init(prm PrmInit) {
|
|||
//
|
||||
// See also Init / Close.
|
||||
func (c *Client) Dial(ctx context.Context, prm PrmDial) error {
|
||||
if prm.endpoint == "" {
|
||||
if prm.Endpoint == "" {
|
||||
return errorServerAddrUnset
|
||||
}
|
||||
|
||||
if prm.timeoutDialSet {
|
||||
if prm.timeoutDial <= 0 {
|
||||
return errorNonPositiveTimeout
|
||||
if prm.DialTimeout <= 0 {
|
||||
prm.DialTimeout = defaultDialTimeout
|
||||
}
|
||||
} else {
|
||||
prm.timeoutDial = 5 * time.Second
|
||||
}
|
||||
|
||||
if prm.streamTimeoutSet {
|
||||
if prm.streamTimeout <= 0 {
|
||||
return errorNonPositiveTimeout
|
||||
}
|
||||
} else {
|
||||
prm.streamTimeout = 10 * time.Second
|
||||
if prm.StreamTimeout <= 0 {
|
||||
prm.StreamTimeout = defaultStreamTimeout
|
||||
}
|
||||
|
||||
c.c = *client.New(append(
|
||||
client.WithNetworkURIAddress(prm.endpoint, prm.tlsConfig),
|
||||
client.WithDialTimeout(prm.timeoutDial),
|
||||
client.WithRWTimeout(prm.streamTimeout),
|
||||
client.WithGRPCDialOptions(prm.dialOptions),
|
||||
client.WithNetworkURIAddress(prm.Endpoint, prm.TLSConfig),
|
||||
client.WithDialTimeout(prm.DialTimeout),
|
||||
client.WithRWTimeout(prm.StreamTimeout),
|
||||
client.WithGRPCDialOptions(prm.GRPCDialOptions),
|
||||
)...)
|
||||
|
||||
c.setFrostFSAPIServer((*coreServer)(&c.c))
|
||||
|
@ -144,52 +135,67 @@ func (c *Client) Close() error {
|
|||
//
|
||||
// See also Init.
|
||||
type PrmInit struct {
|
||||
resolveFrostFSErrors bool
|
||||
DisableFrostFSErrorResolution bool
|
||||
|
||||
key ecdsa.PrivateKey
|
||||
Key ecdsa.PrivateKey
|
||||
|
||||
cbRespInfo func(ResponseMetaInfo) error
|
||||
ResponseInfoCallback func(ResponseMetaInfo) error
|
||||
|
||||
netMagic uint64
|
||||
NetMagic uint64
|
||||
}
|
||||
|
||||
// SetDefaultPrivateKey sets Client private key to be used for the protocol
|
||||
// communication by default.
|
||||
//
|
||||
// Required for operations without custom key parametrization (see corresponding Prm* docs).
|
||||
//
|
||||
// Deprecated: Use PrmInit.Key instead.
|
||||
func (x *PrmInit) SetDefaultPrivateKey(key ecdsa.PrivateKey) {
|
||||
x.key = key
|
||||
x.Key = key
|
||||
}
|
||||
|
||||
// ResolveFrostFSFailures makes the Client to resolve failure statuses of the
|
||||
// FrostFS protocol into Go built-in errors. These errors are returned from
|
||||
// each protocol operation. By default, statuses aren't resolved and written
|
||||
// to the resulting structure (see corresponding Res* docs).
|
||||
// Deprecated: method is no-op. Option is default.
|
||||
func (x *PrmInit) ResolveFrostFSFailures() {
|
||||
x.resolveFrostFSErrors = true
|
||||
}
|
||||
|
||||
// DisableFrostFSFailuresResolution makes the Client to preserve failure statuses of the
|
||||
// FrostFS protocol only in resulting structure (see corresponding Res* docs).
|
||||
// These errors are returned from each protocol operation. By default, statuses
|
||||
// are resolved and returned as a Go built-in errors.
|
||||
//
|
||||
// Deprecated: Use PrmInit.DisableFrostFSErrorResolution instead.
|
||||
func (x *PrmInit) DisableFrostFSFailuresResolution() {
|
||||
x.DisableFrostFSErrorResolution = true
|
||||
}
|
||||
|
||||
// SetResponseInfoCallback makes the Client to pass ResponseMetaInfo from each
|
||||
// FrostFS server response to f. Nil (default) means ignore response meta info.
|
||||
//
|
||||
// Deprecated: Use PrmInit.ResponseInfoCallback instead.
|
||||
func (x *PrmInit) SetResponseInfoCallback(f func(ResponseMetaInfo) error) {
|
||||
x.cbRespInfo = f
|
||||
x.ResponseInfoCallback = f
|
||||
}
|
||||
|
||||
const (
|
||||
defaultDialTimeout = 5 * time.Second
|
||||
defaultStreamTimeout = 10 * time.Second
|
||||
)
|
||||
|
||||
// PrmDial groups connection parameters for the Client.
|
||||
//
|
||||
// See also Dial.
|
||||
type PrmDial struct {
|
||||
endpoint string
|
||||
Endpoint string
|
||||
|
||||
tlsConfig *tls.Config
|
||||
TLSConfig *tls.Config
|
||||
|
||||
timeoutDialSet bool
|
||||
timeoutDial time.Duration
|
||||
// If DialTimeout is non-positive, then it's set to defaultDialTimeout.
|
||||
DialTimeout time.Duration
|
||||
|
||||
streamTimeoutSet bool
|
||||
streamTimeout time.Duration
|
||||
// If StreamTimeout is non-positive, then it's set to defaultStreamTimeout.
|
||||
StreamTimeout time.Duration
|
||||
|
||||
dialOptions []grpc.DialOption
|
||||
GRPCDialOptions []grpc.DialOption
|
||||
}
|
||||
|
||||
// SetServerURI sets server URI in the FrostFS network.
|
||||
|
@ -205,33 +211,41 @@ type PrmDial struct {
|
|||
// grpcs
|
||||
//
|
||||
// See also SetTLSConfig.
|
||||
//
|
||||
// Deprecated: Use PrmDial.Endpoint instead.
|
||||
func (x *PrmDial) SetServerURI(endpoint string) {
|
||||
x.endpoint = endpoint
|
||||
x.Endpoint = endpoint
|
||||
}
|
||||
|
||||
// SetTLSConfig sets tls.Config to open TLS client connection
|
||||
// to the FrostFS server. Nil (default) means insecure connection.
|
||||
//
|
||||
// See also SetServerURI.
|
||||
//
|
||||
// Depreacted: Use PrmDial.TLSConfig instead.
|
||||
func (x *PrmDial) SetTLSConfig(tlsConfig *tls.Config) {
|
||||
x.tlsConfig = tlsConfig
|
||||
x.TLSConfig = tlsConfig
|
||||
}
|
||||
|
||||
// SetTimeout sets the timeout for connection to be established.
|
||||
// MUST BE positive. If not called, 5s timeout will be used by default.
|
||||
//
|
||||
// Deprecated: Use PrmDial.DialTimeout instead.
|
||||
func (x *PrmDial) SetTimeout(timeout time.Duration) {
|
||||
x.timeoutDialSet = true
|
||||
x.timeoutDial = timeout
|
||||
x.DialTimeout = timeout
|
||||
}
|
||||
|
||||
// SetStreamTimeout sets the timeout for individual operations in streaming RPC.
|
||||
// MUST BE positive. If not called, 10s timeout will be used by default.
|
||||
//
|
||||
// Deprecated: Use PrmDial.StreamTimeout instead.
|
||||
func (x *PrmDial) SetStreamTimeout(timeout time.Duration) {
|
||||
x.streamTimeoutSet = true
|
||||
x.streamTimeout = timeout
|
||||
x.StreamTimeout = timeout
|
||||
}
|
||||
|
||||
// SetGRPCDialOptions sets the gRPC dial options for new gRPC client connection.
|
||||
//
|
||||
// Deprecated: Use PrmDial.GRPCDialOptions instead.
|
||||
func (x *PrmDial) SetGRPCDialOptions(opts ...grpc.DialOption) {
|
||||
x.dialOptions = opts
|
||||
x.GRPCDialOptions = opts
|
||||
}
|
||||
|
|
|
@ -29,8 +29,9 @@ func assertStatusErr(tb testing.TB, res interface{ Status() apistatus.Status })
|
|||
}
|
||||
|
||||
func newClient(server frostFSAPIServer) *Client {
|
||||
var prm PrmInit
|
||||
prm.SetDefaultPrivateKey(*key)
|
||||
prm := PrmInit{
|
||||
Key: *key,
|
||||
}
|
||||
|
||||
var c Client
|
||||
c.Init(prm)
|
||||
|
@ -43,8 +44,9 @@ func TestClient_DialContext(t *testing.T) {
|
|||
var c Client
|
||||
|
||||
// try to connect to any host
|
||||
var prm PrmDial
|
||||
prm.SetServerURI("localhost:8080")
|
||||
prm := PrmDial{
|
||||
Endpoint: "localhost:8080",
|
||||
}
|
||||
|
||||
assert := func(ctx context.Context, errExpected error) {
|
||||
// expect particular context error according to Dial docs
|
||||
|
|
|
@ -24,24 +24,6 @@ func (x statusRes) Status() apistatus.Status {
|
|||
return x.st
|
||||
}
|
||||
|
||||
// groups meta parameters shared between all Client operations.
|
||||
type prmCommonMeta struct {
|
||||
// FrostFS request X-Headers
|
||||
xHeaders []string
|
||||
}
|
||||
|
||||
// WithXHeaders specifies list of extended headers (string key-value pairs)
|
||||
// to be attached to the request. Must have an even length.
|
||||
//
|
||||
// Slice must not be mutated until the operation completes.
|
||||
func (x *prmCommonMeta) WithXHeaders(hs ...string) {
|
||||
if len(hs)%2 != 0 {
|
||||
panic("slice of X-Headers with odd length")
|
||||
}
|
||||
|
||||
x.xHeaders = hs
|
||||
}
|
||||
|
||||
func writeXHeadersToMeta(xHeaders []string, h *v2session.RequestMetaHeader) {
|
||||
if len(xHeaders) == 0 {
|
||||
return
|
||||
|
@ -68,7 +50,6 @@ var (
|
|||
errorMissingObject = errors.New("missing object")
|
||||
errorAccountNotSet = errors.New("account not set")
|
||||
errorServerAddrUnset = errors.New("server address is unset or empty")
|
||||
errorNonPositiveTimeout = errors.New("non-positive timeout")
|
||||
errorEACLTableNotSet = errors.New("eACL table not set")
|
||||
errorMissingAnnouncements = errors.New("missing announcements")
|
||||
errorZeroRangeLength = errors.New("zero range length")
|
||||
|
@ -96,19 +77,19 @@ func (c *Client) prepareRequest(req request, meta *v2session.RequestMetaHeader)
|
|||
|
||||
meta.SetTTL(ttl)
|
||||
meta.SetVersion(verV2)
|
||||
meta.SetNetworkMagic(c.prm.netMagic)
|
||||
meta.SetNetworkMagic(c.prm.NetMagic)
|
||||
|
||||
req.SetMetaHeader(meta)
|
||||
}
|
||||
|
||||
// processResponse verifies response signature and converts status to an error if needed.
|
||||
func (c *Client) processResponse(resp responseV2) (apistatus.Status, error) {
|
||||
if c.prm.cbRespInfo != nil {
|
||||
if c.prm.ResponseInfoCallback != nil {
|
||||
rmi := ResponseMetaInfo{
|
||||
key: resp.GetVerificationHeader().GetBodySignature().GetKey(),
|
||||
epoch: resp.GetMetaHeader().GetEpoch(),
|
||||
}
|
||||
if err := c.prm.cbRespInfo(rmi); err != nil {
|
||||
if err := c.prm.ResponseInfoCallback(rmi); err != nil {
|
||||
return nil, fmt.Errorf("response callback error: %w", err)
|
||||
}
|
||||
}
|
||||
|
@ -119,7 +100,7 @@ func (c *Client) processResponse(resp responseV2) (apistatus.Status, error) {
|
|||
}
|
||||
|
||||
st := apistatus.FromStatusV2(resp.GetMetaHeader().GetStatus())
|
||||
if c.prm.resolveFrostFSErrors {
|
||||
if !c.prm.DisableFrostFSErrorResolution {
|
||||
return st, apistatus.ErrFromStatus(st)
|
||||
}
|
||||
return st, nil
|
||||
|
|
|
@ -52,7 +52,7 @@ func (prm *PrmContainerDelete) buildRequest(c *Client) (*v2container.DeleteReque
|
|||
|
||||
var sig frostfscrypto.Signature
|
||||
|
||||
err := sig.Calculate(frostfsecdsa.SignerRFC6979(c.prm.key), data)
|
||||
err := sig.Calculate(frostfsecdsa.SignerRFC6979(c.prm.Key), data)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("calculate signature: %w", err)
|
||||
}
|
||||
|
@ -101,9 +101,9 @@ type ResContainerDelete struct {
|
|||
//
|
||||
// Exactly one return value is non-nil. By default, server status is returned in res structure.
|
||||
// Any client's internal or transport errors are returned as `error`.
|
||||
// If PrmInit.ResolveFrostFSFailures has been called, unsuccessful
|
||||
// FrostFS status codes are returned as `error`, otherwise, are included
|
||||
// in the returned result structure.
|
||||
// If PrmInit.DisableFrostFSFailuresResolution has been called, unsuccessful
|
||||
// FrostFS status codes are included in the returned result structure,
|
||||
// otherwise, are also returned as `error`.
|
||||
//
|
||||
// Operation is asynchronous and no guaranteed even in the absence of errors.
|
||||
// The required time is also not predictable.
|
||||
|
@ -124,7 +124,7 @@ func (c *Client) ContainerDelete(ctx context.Context, prm PrmContainerDelete) (*
|
|||
return nil, err
|
||||
}
|
||||
|
||||
if err := signature.SignServiceMessage(&c.prm.key, req); err != nil {
|
||||
if err := signature.SignServiceMessage(&c.prm.Key, req); err != nil {
|
||||
return nil, fmt.Errorf("sign request: %w", err)
|
||||
}
|
||||
|
||||
|
|
|
@ -13,6 +13,7 @@ import (
|
|||
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||
)
|
||||
|
||||
// PrmContainerEACL groups parameters of ContainerEACL operation.
|
||||
|
@ -21,6 +22,8 @@ type PrmContainerEACL struct {
|
|||
XHeaders []string
|
||||
|
||||
ContainerID *cid.ID
|
||||
|
||||
Session *session.Container
|
||||
}
|
||||
|
||||
// SetContainer sets identifier of the FrostFS container to read the eACL table.
|
||||
|
@ -46,9 +49,19 @@ func (x *PrmContainerEACL) buildRequest(c *Client) (*v2container.GetExtendedACLR
|
|||
reqBody := new(v2container.GetExtendedACLRequestBody)
|
||||
reqBody.SetContainerID(&cidV2)
|
||||
|
||||
var meta v2session.RequestMetaHeader
|
||||
writeXHeadersToMeta(x.XHeaders, &meta)
|
||||
|
||||
if x.Session != nil {
|
||||
var tokv2 v2session.Token
|
||||
x.Session.WriteToV2(&tokv2)
|
||||
|
||||
meta.SetSessionToken(&tokv2)
|
||||
}
|
||||
|
||||
var req v2container.GetExtendedACLRequest
|
||||
req.SetBody(reqBody)
|
||||
c.prepareRequest(&req, new(v2session.RequestMetaHeader))
|
||||
c.prepareRequest(&req, &meta)
|
||||
return &req, nil
|
||||
}
|
||||
|
||||
|
@ -68,9 +81,9 @@ func (x ResContainerEACL) Table() eacl.Table {
|
|||
//
|
||||
// Exactly one return value is non-nil. By default, server status is returned in res structure.
|
||||
// Any client's internal or transport errors are returned as `error`.
|
||||
// If PrmInit.ResolveFrostFSFailures has been called, unsuccessful
|
||||
// FrostFS status codes are returned as `error`, otherwise, are included
|
||||
// in the returned result structure.
|
||||
// If PrmInit.DisableFrostFSFailuresResolution has been called, unsuccessful
|
||||
// FrostFS status codes are included in the returned result structure,
|
||||
// otherwise, are also returned as `error`.
|
||||
//
|
||||
// Returns an error if parameters are set incorrectly (see PrmContainerEACL docs).
|
||||
// Context is required and must not be nil. It is used for network communication.
|
||||
|
@ -85,7 +98,7 @@ func (c *Client) ContainerEACL(ctx context.Context, prm PrmContainerEACL) (*ResC
|
|||
return nil, err
|
||||
}
|
||||
|
||||
if err := signature.SignServiceMessage(&c.prm.key, req); err != nil {
|
||||
if err := signature.SignServiceMessage(&c.prm.Key, req); err != nil {
|
||||
return nil, fmt.Errorf("sign request: %w", err)
|
||||
}
|
||||
|
||||
|
|
|
@ -14,6 +14,7 @@ import (
|
|||
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||
)
|
||||
|
||||
// PrmContainerGet groups parameters of ContainerGet operation.
|
||||
|
@ -22,6 +23,8 @@ type PrmContainerGet struct {
|
|||
XHeaders []string
|
||||
|
||||
ContainerID *cid.ID
|
||||
|
||||
Session *session.Container
|
||||
}
|
||||
|
||||
// SetContainer sets identifier of the container to be read.
|
||||
|
@ -47,9 +50,19 @@ func (prm *PrmContainerGet) buildRequest(c *Client) (*v2container.GetRequest, er
|
|||
reqBody := new(v2container.GetRequestBody)
|
||||
reqBody.SetContainerID(&cidV2)
|
||||
|
||||
var meta v2session.RequestMetaHeader
|
||||
writeXHeadersToMeta(prm.XHeaders, &meta)
|
||||
|
||||
if prm.Session != nil {
|
||||
var tokv2 v2session.Token
|
||||
prm.Session.WriteToV2(&tokv2)
|
||||
|
||||
meta.SetSessionToken(&tokv2)
|
||||
}
|
||||
|
||||
var req v2container.GetRequest
|
||||
req.SetBody(reqBody)
|
||||
c.prepareRequest(&req, new(v2session.RequestMetaHeader))
|
||||
c.prepareRequest(&req, &meta)
|
||||
return &req, nil
|
||||
}
|
||||
|
||||
|
@ -71,9 +84,9 @@ func (x ResContainerGet) Container() container.Container {
|
|||
//
|
||||
// Exactly one return value is non-nil. By default, server status is returned in res structure.
|
||||
// Any client's internal or transport errors are returned as `error`.
|
||||
// If PrmInit.ResolveFrostFSFailures has been called, unsuccessful
|
||||
// FrostFS status codes are returned as `error`, otherwise, are included
|
||||
// in the returned result structure.
|
||||
// If PrmInit.DisableFrostFSFailuresResolution has been called, unsuccessful
|
||||
// FrostFS status codes are included in the returned result structure,
|
||||
// otherwise, are also returned as `error`.
|
||||
//
|
||||
// Returns an error if parameters are set incorrectly (see PrmContainerGet docs).
|
||||
// Context is required and must not be nil. It is used for network communication.
|
||||
|
@ -87,7 +100,7 @@ func (c *Client) ContainerGet(ctx context.Context, prm PrmContainerGet) (*ResCon
|
|||
return nil, err
|
||||
}
|
||||
|
||||
if err := signature.SignServiceMessage(&c.prm.key, req); err != nil {
|
||||
if err := signature.SignServiceMessage(&c.prm.Key, req); err != nil {
|
||||
return nil, fmt.Errorf("sign request: %w", err)
|
||||
}
|
||||
|
||||
|
|
|
@ -12,38 +12,51 @@ import (
|
|||
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/signature"
|
||||
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
|
||||
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
)
|
||||
|
||||
// PrmContainerList groups parameters of ContainerList operation.
|
||||
type PrmContainerList struct {
|
||||
prmCommonMeta
|
||||
XHeaders []string
|
||||
|
||||
ownerSet bool
|
||||
ownerID user.ID
|
||||
Account user.ID
|
||||
|
||||
Session *session.Container
|
||||
}
|
||||
|
||||
// SetAccount sets identifier of the FrostFS account to list the containers.
|
||||
// Required parameter.
|
||||
//
|
||||
// Deprecated: Use PrmContainerList.Account instead.
|
||||
func (x *PrmContainerList) SetAccount(id user.ID) {
|
||||
x.ownerID = id
|
||||
x.ownerSet = true
|
||||
x.Account = id
|
||||
}
|
||||
|
||||
func (x *PrmContainerList) buildRequest(c *Client) (*v2container.ListRequest, error) {
|
||||
if !x.ownerSet {
|
||||
if x.Account.IsEmpty() {
|
||||
return nil, errorAccountNotSet
|
||||
}
|
||||
|
||||
var ownerV2 refs.OwnerID
|
||||
x.ownerID.WriteToV2(&ownerV2)
|
||||
x.Account.WriteToV2(&ownerV2)
|
||||
|
||||
reqBody := new(v2container.ListRequestBody)
|
||||
reqBody.SetOwnerID(&ownerV2)
|
||||
|
||||
var meta v2session.RequestMetaHeader
|
||||
writeXHeadersToMeta(x.XHeaders, &meta)
|
||||
|
||||
if x.Session != nil {
|
||||
var tokv2 v2session.Token
|
||||
x.Session.WriteToV2(&tokv2)
|
||||
|
||||
meta.SetSessionToken(&tokv2)
|
||||
}
|
||||
|
||||
var req v2container.ListRequest
|
||||
req.SetBody(reqBody)
|
||||
c.prepareRequest(&req, new(v2session.RequestMetaHeader))
|
||||
c.prepareRequest(&req, &meta)
|
||||
return &req, nil
|
||||
}
|
||||
|
||||
|
@ -65,9 +78,9 @@ func (x ResContainerList) Containers() []cid.ID {
|
|||
//
|
||||
// Exactly one return value is non-nil. By default, server status is returned in res structure.
|
||||
// Any client's internal or transport errors are returned as `error`.
|
||||
// If PrmInit.ResolveFrostFSFailures has been called, unsuccessful
|
||||
// FrostFS status codes are returned as `error`, otherwise, are included
|
||||
// in the returned result structure.
|
||||
// If PrmInit.DisableFrostFSFailuresResolution has been called, unsuccessful
|
||||
// FrostFS status codes are included in the returned result structure,
|
||||
// otherwise, are also returned as `error`.
|
||||
//
|
||||
// Returns an error if parameters are set incorrectly (see PrmContainerList docs).
|
||||
// Context is required and must not be nil. It is used for network communication.
|
||||
|
@ -80,7 +93,7 @@ func (c *Client) ContainerList(ctx context.Context, prm PrmContainerList) (*ResC
|
|||
return nil, err
|
||||
}
|
||||
|
||||
if err := signature.SignServiceMessage(&c.prm.key, req); err != nil {
|
||||
if err := signature.SignServiceMessage(&c.prm.Key, req); err != nil {
|
||||
return nil, fmt.Errorf("sign request: %w", err)
|
||||
}
|
||||
|
||||
|
|
|
@ -64,7 +64,7 @@ func (x *PrmContainerPut) buildRequest(c *Client) (*v2container.PutRequest, erro
|
|||
|
||||
var sig frostfscrypto.Signature
|
||||
|
||||
err := container.CalculateSignature(&sig, *x.Container, c.prm.key)
|
||||
err := container.CalculateSignature(&sig, *x.Container, c.prm.Key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("calculate container signature: %w", err)
|
||||
}
|
||||
|
@ -110,9 +110,9 @@ func (x ResContainerPut) ID() cid.ID {
|
|||
//
|
||||
// Exactly one return value is non-nil. By default, server status is returned in res structure.
|
||||
// Any client's internal or transport errors are returned as `error`.
|
||||
// If PrmInit.ResolveFrostFSFailures has been called, unsuccessful
|
||||
// FrostFS status codes are returned as `error`, otherwise, are included
|
||||
// in the returned result structure.
|
||||
// If PrmInit.DisableFrostFSFailuresResolution has been called, unsuccessful
|
||||
// FrostFS status codes are included in the returned result structure,
|
||||
// otherwise, are also returned as `error`.
|
||||
//
|
||||
// Operation is asynchronous and no guaranteed even in the absence of errors.
|
||||
// The required time is also not predictable.
|
||||
|
@ -132,7 +132,7 @@ func (c *Client) ContainerPut(ctx context.Context, prm PrmContainerPut) (*ResCon
|
|||
return nil, err
|
||||
}
|
||||
|
||||
if err := signature.SignServiceMessage(&c.prm.key, req); err != nil {
|
||||
if err := signature.SignServiceMessage(&c.prm.Key, req); err != nil {
|
||||
return nil, fmt.Errorf("sign request: %w", err)
|
||||
}
|
||||
|
||||
|
|
|
@ -64,7 +64,7 @@ func (x *PrmContainerSetEACL) buildRequest(c *Client) (*v2container.SetExtendedA
|
|||
|
||||
var sig frostfscrypto.Signature
|
||||
|
||||
err := sig.Calculate(frostfsecdsa.SignerRFC6979(c.prm.key), eaclV2.StableMarshal(nil))
|
||||
err := sig.Calculate(frostfsecdsa.SignerRFC6979(c.prm.Key), eaclV2.StableMarshal(nil))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("calculate signature: %w", err)
|
||||
}
|
||||
|
@ -101,9 +101,9 @@ type ResContainerSetEACL struct {
|
|||
//
|
||||
// Exactly one return value is non-nil. By default, server status is returned in res structure.
|
||||
// Any client's internal or transport errors are returned as `error`.
|
||||
// If PrmInit.ResolveFrostFSFailures has been called, unsuccessful
|
||||
// FrostFS status codes are returned as `error`, otherwise, are included
|
||||
// in the returned result structure.
|
||||
// If PrmInit.DisableFrostFSFailuresResolution has been called, unsuccessful
|
||||
// FrostFS status codes are included in the returned result structure,
|
||||
// otherwise, are also returned as `error`.
|
||||
//
|
||||
// Operation is asynchronous and no guaranteed even in the absence of errors.
|
||||
// The required time is also not predictable.
|
||||
|
@ -121,7 +121,7 @@ func (c *Client) ContainerSetEACL(ctx context.Context, prm PrmContainerSetEACL)
|
|||
return nil, err
|
||||
}
|
||||
|
||||
if err := signature.SignServiceMessage(&c.prm.key, req); err != nil {
|
||||
if err := signature.SignServiceMessage(&c.prm.Key, req); err != nil {
|
||||
return nil, fmt.Errorf("sign request: %w", err)
|
||||
}
|
||||
|
||||
|
|
|
@ -57,9 +57,9 @@ type ResAnnounceSpace struct {
|
|||
//
|
||||
// Exactly one return value is non-nil. By default, server status is returned in res structure.
|
||||
// Any client's internal or transport errors are returned as `error`.
|
||||
// If PrmInit.ResolveFrostFSFailures has been called, unsuccessful
|
||||
// FrostFS status codes are returned as `error`, otherwise, are included
|
||||
// in the returned result structure.
|
||||
// If PrmInit.DisableFrostFSFailuresResolution has been called, unsuccessful
|
||||
// FrostFS status codes are included in the returned result structure,
|
||||
// otherwise, are also returned as `error`.
|
||||
//
|
||||
// Operation is asynchronous and no guaranteed even in the absence of errors.
|
||||
// The required time is also not predictable.
|
||||
|
@ -77,7 +77,7 @@ func (c *Client) ContainerAnnounceUsedSpace(ctx context.Context, prm PrmAnnounce
|
|||
return nil, err
|
||||
}
|
||||
|
||||
if err := signature.SignServiceMessage(&c.prm.key, req); err != nil {
|
||||
if err := signature.SignServiceMessage(&c.prm.Key, req); err != nil {
|
||||
return nil, fmt.Errorf("sign request: %w", err)
|
||||
}
|
||||
|
||||
|
|
|
@ -33,7 +33,8 @@ func TestErrors(t *testing.T) {
|
|||
{
|
||||
check: client.IsErrSessionExpired,
|
||||
err: new(apistatus.SessionTokenExpired),
|
||||
}, {
|
||||
},
|
||||
{
|
||||
check: client.IsErrSessionNotFound,
|
||||
err: new(apistatus.SessionTokenNotFound),
|
||||
},
|
||||
|
|
|
@ -53,9 +53,9 @@ func (x ResEndpointInfo) NodeInfo() netmap.NodeInfo {
|
|||
// Method can be used as a health check to see if node is alive and responds to requests.
|
||||
//
|
||||
// Any client's internal or transport errors are returned as `error`.
|
||||
// If PrmInit.ResolveFrostFSFailures has been called, unsuccessful
|
||||
// FrostFS status codes are returned as `error`, otherwise, are included
|
||||
// in the returned result structure.
|
||||
// If PrmInit.DisableFrostFSFailuresResolution has been called, unsuccessful
|
||||
// FrostFS status codes are included in the returned result structure,
|
||||
// otherwise, are also returned as `error`.
|
||||
//
|
||||
// Returns an error if parameters are set incorrectly (see PrmEndpointInfo docs).
|
||||
// Context is required and must not be nil. It is used for network communication.
|
||||
|
@ -71,7 +71,7 @@ func (c *Client) EndpointInfo(ctx context.Context, prm PrmEndpointInfo) (*ResEnd
|
|||
return nil, err
|
||||
}
|
||||
|
||||
if err := signature.SignServiceMessage(&c.prm.key, req); err != nil {
|
||||
if err := signature.SignServiceMessage(&c.prm.Key, req); err != nil {
|
||||
return nil, fmt.Errorf("sign request: %w", err)
|
||||
}
|
||||
|
||||
|
@ -140,9 +140,9 @@ func (x ResNetworkInfo) Info() netmap.NetworkInfo {
|
|||
// NetworkInfo requests information about the FrostFS network of which the remote server is a part.
|
||||
//
|
||||
// Any client's internal or transport errors are returned as `error`.
|
||||
// If PrmInit.ResolveFrostFSFailures has been called, unsuccessful
|
||||
// FrostFS status codes are returned as `error`, otherwise, are included
|
||||
// in the returned result structure.
|
||||
// If PrmInit.DisableFrostFSFailuresResolution has been called, unsuccessful
|
||||
// FrostFS status codes are included in the returned result structure,
|
||||
// otherwise, are also returned as `error`.
|
||||
//
|
||||
// Returns an error if parameters are set incorrectly (see PrmNetworkInfo docs).
|
||||
// Context is required and must not be nil. It is used for network communication.
|
||||
|
@ -158,7 +158,7 @@ func (c *Client) NetworkInfo(ctx context.Context, prm PrmNetworkInfo) (*ResNetwo
|
|||
return nil, err
|
||||
}
|
||||
|
||||
if err := signature.SignServiceMessage(&c.prm.key, req); err != nil {
|
||||
if err := signature.SignServiceMessage(&c.prm.Key, req); err != nil {
|
||||
return nil, fmt.Errorf("sign request: %w", err)
|
||||
}
|
||||
|
||||
|
@ -186,8 +186,7 @@ func (c *Client) NetworkInfo(ctx context.Context, prm PrmNetworkInfo) (*ResNetwo
|
|||
}
|
||||
|
||||
// PrmNetMapSnapshot groups parameters of NetMapSnapshot operation.
|
||||
type PrmNetMapSnapshot struct {
|
||||
}
|
||||
type PrmNetMapSnapshot struct{}
|
||||
|
||||
// ResNetMapSnapshot groups resulting values of NetMapSnapshot operation.
|
||||
type ResNetMapSnapshot struct {
|
||||
|
@ -204,9 +203,9 @@ func (x ResNetMapSnapshot) NetMap() netmap.NetMap {
|
|||
// NetMapSnapshot requests current network view of the remote server.
|
||||
//
|
||||
// Any client's internal or transport errors are returned as `error`.
|
||||
// If PrmInit.ResolveFrostFSFailures has been called, unsuccessful
|
||||
// FrostFS status codes are returned as `error`, otherwise, are included
|
||||
// in the returned result structure.
|
||||
// If PrmInit.DisableFrostFSFailuresResolution has been called, unsuccessful
|
||||
// FrostFS status codes are included in the returned result structure,
|
||||
// otherwise, are also returned as `error`.
|
||||
//
|
||||
// Returns an error if parameters are set incorrectly.
|
||||
// Context is required and MUST NOT be nil. It is used for network communication.
|
||||
|
@ -228,7 +227,7 @@ func (c *Client) NetMapSnapshot(ctx context.Context, _ PrmNetMapSnapshot) (*ResN
|
|||
req.SetBody(&body)
|
||||
c.prepareRequest(&req, &meta)
|
||||
|
||||
err := signature.SignServiceMessage(&c.prm.key, &req)
|
||||
err := signature.SignServiceMessage(&c.prm.Key, &req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("sign request: %w", err)
|
||||
}
|
||||
|
|
|
@ -67,6 +67,7 @@ func TestClient_NetMapSnapshot(t *testing.T) {
|
|||
var res *ResNetMapSnapshot
|
||||
var srv serverNetMap
|
||||
c := newClient(&srv)
|
||||
c.prm.DisableFrostFSFailuresResolution()
|
||||
ctx := context.Background()
|
||||
|
||||
// request signature
|
||||
|
|
|
@ -112,9 +112,9 @@ func (prm *PrmObjectDelete) buildRequest(c *Client) (*v2object.DeleteRequest, er
|
|||
//
|
||||
// Exactly one return value is non-nil. By default, server status is returned in res structure.
|
||||
// Any client's internal or transport errors are returned as `error`,
|
||||
// If PrmInit.ResolveFrostFSFailures has been called, unsuccessful
|
||||
// FrostFS status codes are returned as `error`, otherwise, are included
|
||||
// in the returned result structure.
|
||||
// If PrmInit.DisableFrostFSFailuresResolution has been called, unsuccessful
|
||||
// FrostFS status codes are included in the returned result structure,
|
||||
// otherwise, are also returned as `error`.
|
||||
//
|
||||
// Returns an error if parameters are set incorrectly (see PrmObjectDelete docs).
|
||||
// Context is required and must not be nil. It is used for network communication.
|
||||
|
@ -131,7 +131,7 @@ func (c *Client) ObjectDelete(ctx context.Context, prm PrmObjectDelete) (*ResObj
|
|||
return nil, err
|
||||
}
|
||||
|
||||
key := c.prm.key
|
||||
key := c.prm.Key
|
||||
if prm.Key != nil {
|
||||
key = *prm.Key
|
||||
}
|
||||
|
|
|
@ -150,6 +150,9 @@ func (x *ObjectReader) ReadHeader(dst *object.Object) bool {
|
|||
case *v2object.SplitInfo:
|
||||
x.err = object.NewSplitInfoError(object.NewSplitInfoFromV2(v))
|
||||
return false
|
||||
case *v2object.ECInfo:
|
||||
x.err = object.NewECInfoError(object.NewECInfoFromV2(v))
|
||||
return false
|
||||
case *v2object.GetObjectPartInit:
|
||||
partInit = v
|
||||
}
|
||||
|
@ -258,6 +261,7 @@ func (x *ObjectReader) close(ignoreEOF bool) (*ResObjectGet, error) {
|
|||
// Return errors:
|
||||
//
|
||||
// *object.SplitInfoError (returned on virtual objects with PrmObjectGet.MakeRaw).
|
||||
// *object.ECInfoError (returned on erasure-coded objects with PrmObjectGet.MakeRaw).
|
||||
//
|
||||
// Return statuses:
|
||||
// - global (see Client docs);
|
||||
|
@ -307,7 +311,7 @@ func (c *Client) ObjectGetInit(ctx context.Context, prm PrmObjectGet) (*ObjectRe
|
|||
|
||||
key := prm.Key
|
||||
if key == nil {
|
||||
key = &c.prm.key
|
||||
key = &c.prm.Key
|
||||
}
|
||||
|
||||
err = signature.SignServiceMessage(key, req)
|
||||
|
@ -444,9 +448,9 @@ func (prm *PrmObjectHead) buildRequest(c *Client) (*v2object.HeadRequest, error)
|
|||
//
|
||||
// Exactly one return value is non-nil. By default, server status is returned in res structure.
|
||||
// Any client's internal or transport errors are returned as `error`,
|
||||
// If PrmInit.ResolveFrostFSFailures has been called, unsuccessful
|
||||
// FrostFS status codes are returned as `error`, otherwise, are included
|
||||
// in the returned result structure.
|
||||
// If PrmInit.DisableFrostFSFailuresResolution has been called, unsuccessful
|
||||
// FrostFS status codes are included in the returned result structure,
|
||||
// otherwise, are also returned as `error`.
|
||||
//
|
||||
// Returns an error if parameters are set incorrectly (see PrmObjectHead docs).
|
||||
// Context is required and must not be nil. It is used for network communication.
|
||||
|
@ -454,6 +458,7 @@ func (prm *PrmObjectHead) buildRequest(c *Client) (*v2object.HeadRequest, error)
|
|||
// Return errors:
|
||||
//
|
||||
// *object.SplitInfoError (returned on virtual objects with PrmObjectHead.MakeRaw).
|
||||
// *object.ECInfoError (returned on erasure-coded objects with PrmObjectHead.MakeRaw).
|
||||
//
|
||||
// Return statuses:
|
||||
// - global (see Client docs);
|
||||
|
@ -468,7 +473,7 @@ func (c *Client) ObjectHead(ctx context.Context, prm PrmObjectHead) (*ResObjectH
|
|||
return nil, err
|
||||
}
|
||||
|
||||
key := c.prm.key
|
||||
key := c.prm.Key
|
||||
if prm.Key != nil {
|
||||
key = *prm.Key
|
||||
}
|
||||
|
@ -502,6 +507,8 @@ func (c *Client) ObjectHead(ctx context.Context, prm PrmObjectHead) (*ResObjectH
|
|||
return nil, fmt.Errorf("unexpected header type %T", v)
|
||||
case *v2object.SplitInfo:
|
||||
return nil, object.NewSplitInfoError(object.NewSplitInfoFromV2(v))
|
||||
case *v2object.ECInfo:
|
||||
return nil, object.NewECInfoError(object.NewECInfoFromV2(v))
|
||||
case *v2object.HeaderWithSignature:
|
||||
res.hdr = v
|
||||
}
|
||||
|
@ -665,6 +672,9 @@ func (x *ObjectRangeReader) readChunk(buf []byte) (int, bool) {
|
|||
case *v2object.SplitInfo:
|
||||
x.err = object.NewSplitInfoError(object.NewSplitInfoFromV2(v))
|
||||
return read, false
|
||||
case *v2object.ECInfo:
|
||||
x.err = object.NewECInfoError(object.NewECInfoFromV2(v))
|
||||
return read, false
|
||||
case *v2object.GetRangePartChunk:
|
||||
partChunk = v
|
||||
}
|
||||
|
@ -725,6 +735,7 @@ func (x *ObjectRangeReader) close(ignoreEOF bool) (*ResObjectRange, error) {
|
|||
// Return errors:
|
||||
//
|
||||
// *object.SplitInfoError (returned on virtual objects with PrmObjectRange.MakeRaw).
|
||||
// *object.ECInfoError (returned on erasure-coded objects with PrmObjectRange.MakeRaw).
|
||||
//
|
||||
// Return statuses:
|
||||
// - global (see Client docs);
|
||||
|
@ -776,7 +787,7 @@ func (c *Client) ObjectRangeInit(ctx context.Context, prm PrmObjectRange) (*Obje
|
|||
|
||||
key := prm.Key
|
||||
if key == nil {
|
||||
key = &c.prm.key
|
||||
key = &c.prm.Key
|
||||
}
|
||||
|
||||
err = signature.SignServiceMessage(key, req)
|
||||
|
|
|
@ -152,9 +152,9 @@ func (prm *PrmObjectHash) buildRequest(c *Client) (*v2object.GetRangeHashRequest
|
|||
//
|
||||
// Exactly one return value is non-nil. By default, server status is returned in res structure.
|
||||
// Any client's internal or transport errors are returned as `error`,
|
||||
// If PrmInit.ResolveFrostFSFailures has been called, unsuccessful
|
||||
// FrostFS status codes are returned as `error`, otherwise, are included
|
||||
// in the returned result structure.
|
||||
// If PrmInit.DisableFrostFSFailuresResolution has been called, unsuccessful
|
||||
// FrostFS status codes are included in the returned result structure,
|
||||
// otherwise, are also returned as `error`.
|
||||
//
|
||||
// Returns an error if parameters are set incorrectly (see PrmObjectHash docs).
|
||||
// Context is required and must not be nil. It is used for network communication.
|
||||
|
@ -172,7 +172,7 @@ func (c *Client) ObjectHash(ctx context.Context, prm PrmObjectHash) (*ResObjectH
|
|||
return nil, err
|
||||
}
|
||||
|
||||
key := c.prm.key
|
||||
key := c.prm.Key
|
||||
if prm.Key != nil {
|
||||
key = *prm.Key
|
||||
}
|
||||
|
|
|
@ -4,6 +4,7 @@ import (
|
|||
"context"
|
||||
"crypto/ecdsa"
|
||||
|
||||
buffPool "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/pool"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
|
@ -36,6 +37,8 @@ type PrmObjectPutInit struct {
|
|||
WithoutHomomorphHash bool
|
||||
|
||||
Key *ecdsa.PrivateKey
|
||||
|
||||
Pool *buffPool.BufferPool
|
||||
}
|
||||
|
||||
// SetCopiesNumber sets number of object copies that is enough to consider put successful.
|
||||
|
|
|
@ -28,7 +28,7 @@ func (c *Client) objectPutInitRaw(ctx context.Context, prm PrmObjectPutInit) (*o
|
|||
return nil, fmt.Errorf("open stream: %w", err)
|
||||
}
|
||||
|
||||
w.key = &c.prm.key
|
||||
w.key = &c.prm.Key
|
||||
if prm.Key != nil {
|
||||
w.key = prm.Key
|
||||
}
|
||||
|
|
|
@ -142,7 +142,7 @@ func (c *Client) ObjectPutSingle(ctx context.Context, prm PrmObjectPutSingle) (*
|
|||
return nil, err
|
||||
}
|
||||
|
||||
key := &c.prm.key
|
||||
key := &c.prm.Key
|
||||
if prm.Key != nil {
|
||||
key = prm.Key
|
||||
}
|
||||
|
|
|
@ -3,6 +3,7 @@ package client
|
|||
import (
|
||||
"context"
|
||||
|
||||
buffPool "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/pool"
|
||||
apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer"
|
||||
|
@ -16,7 +17,7 @@ func (c *Client) objectPutInitTransformer(prm PrmObjectPutInit) (*objectWriterTr
|
|||
client: c,
|
||||
prm: prm,
|
||||
}
|
||||
key := &c.prm.key
|
||||
key := &c.prm.Key
|
||||
if prm.Key != nil {
|
||||
key = prm.Key
|
||||
}
|
||||
|
@ -26,6 +27,7 @@ func (c *Client) objectPutInitTransformer(prm PrmObjectPutInit) (*objectWriterTr
|
|||
MaxSize: prm.MaxSize,
|
||||
WithoutHomomorphicHash: prm.WithoutHomomorphHash,
|
||||
NetworkState: prm.EpochSource,
|
||||
Pool: prm.Pool,
|
||||
})
|
||||
return &w, nil
|
||||
}
|
||||
|
@ -47,6 +49,10 @@ func (x *objectWriterTransformer) WritePayloadChunk(ctx context.Context, chunk [
|
|||
}
|
||||
|
||||
func (x *objectWriterTransformer) Close(ctx context.Context) (*ResObjectPut, error) {
|
||||
if x.err != nil {
|
||||
return nil, x.err
|
||||
}
|
||||
|
||||
ai, err := x.ot.Close(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
@ -83,7 +89,7 @@ func (it *internalTarget) putAsStream(ctx context.Context, o *object.Object) err
|
|||
wrt.WritePayloadChunk(ctx, o.Payload())
|
||||
}
|
||||
it.res, err = wrt.Close(ctx)
|
||||
if err == nil && !it.client.prm.resolveFrostFSErrors && !apistatus.IsSuccessful(it.res.st) {
|
||||
if err == nil && it.client.prm.DisableFrostFSErrorResolution && !apistatus.IsSuccessful(it.res.st) {
|
||||
err = apistatus.ErrFromStatus(it.res.st)
|
||||
}
|
||||
return err
|
||||
|
@ -110,15 +116,25 @@ func (it *internalTarget) tryPutSingle(ctx context.Context, o *object.Object) (b
|
|||
}
|
||||
|
||||
if err == nil {
|
||||
it.returnBuffPool(o.Payload())
|
||||
id, _ := o.ID()
|
||||
it.res = &ResObjectPut{
|
||||
statusRes: res.statusRes,
|
||||
obj: id,
|
||||
}
|
||||
if !it.client.prm.resolveFrostFSErrors && !apistatus.IsSuccessful(it.res.st) {
|
||||
if it.client.prm.DisableFrostFSErrorResolution && !apistatus.IsSuccessful(it.res.st) {
|
||||
return true, apistatus.ErrFromStatus(it.res.st)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
return true, err
|
||||
}
|
||||
|
||||
func (it *internalTarget) returnBuffPool(playback []byte) {
|
||||
if it.prm.Pool == nil {
|
||||
return
|
||||
}
|
||||
var buffer buffPool.Buffer
|
||||
buffer.Data = playback
|
||||
it.prm.Pool.Put(&buffer)
|
||||
}
|
||||
|
|
|
@ -24,19 +24,26 @@ import (
|
|||
|
||||
// PrmObjectSearch groups parameters of ObjectSearch operation.
|
||||
type PrmObjectSearch struct {
|
||||
meta v2session.RequestMetaHeader
|
||||
XHeaders []string
|
||||
|
||||
key *ecdsa.PrivateKey
|
||||
Local bool
|
||||
|
||||
cnrSet bool
|
||||
cnrID cid.ID
|
||||
BearerToken *bearer.Token
|
||||
|
||||
filters object.SearchFilters
|
||||
Session *session.Object
|
||||
|
||||
ContainerID *cid.ID
|
||||
|
||||
Key *ecdsa.PrivateKey
|
||||
|
||||
Filters object.SearchFilters
|
||||
}
|
||||
|
||||
// MarkLocal tells the server to execute the operation locally.
|
||||
//
|
||||
// Deprecated: Use PrmObjectSearch.Local instead.
|
||||
func (x *PrmObjectSearch) MarkLocal() {
|
||||
x.meta.SetTTL(1)
|
||||
x.Local = true
|
||||
}
|
||||
|
||||
// WithinSession specifies session within which the search query must be executed.
|
||||
|
@ -45,10 +52,10 @@ func (x *PrmObjectSearch) MarkLocal() {
|
|||
// This may affect the execution of an operation (e.g. access control).
|
||||
//
|
||||
// Must be signed.
|
||||
//
|
||||
// Deprecated: Use PrmObjectSearch.Session instead.
|
||||
func (x *PrmObjectSearch) WithinSession(t session.Object) {
|
||||
var tokv2 v2session.Token
|
||||
t.WriteToV2(&tokv2)
|
||||
x.meta.SetSessionToken(&tokv2)
|
||||
x.Session = &t
|
||||
}
|
||||
|
||||
// WithBearerToken attaches bearer token to be used for the operation.
|
||||
|
@ -56,37 +63,44 @@ func (x *PrmObjectSearch) WithinSession(t session.Object) {
|
|||
// If set, underlying eACL rules will be used in access control.
|
||||
//
|
||||
// Must be signed.
|
||||
//
|
||||
// Deprecated: Use PrmObjectSearch.BearerToken instead.
|
||||
func (x *PrmObjectSearch) WithBearerToken(t bearer.Token) {
|
||||
var v2token acl.BearerToken
|
||||
t.WriteToV2(&v2token)
|
||||
x.meta.SetBearerToken(&v2token)
|
||||
x.BearerToken = &t
|
||||
}
|
||||
|
||||
// WithXHeaders specifies list of extended headers (string key-value pairs)
|
||||
// to be attached to the request. Must have an even length.
|
||||
//
|
||||
// Slice must not be mutated until the operation completes.
|
||||
//
|
||||
// Deprecated: Use PrmObjectSearch.XHeaders instead.
|
||||
func (x *PrmObjectSearch) WithXHeaders(hs ...string) {
|
||||
writeXHeadersToMeta(hs, &x.meta)
|
||||
x.XHeaders = hs
|
||||
}
|
||||
|
||||
// UseKey specifies private key to sign the requests.
|
||||
// If key is not provided, then Client default key is used.
|
||||
//
|
||||
// Deprecated: Use PrmObjectSearch.Key instead.
|
||||
func (x *PrmObjectSearch) UseKey(key ecdsa.PrivateKey) {
|
||||
x.key = &key
|
||||
x.Key = &key
|
||||
}
|
||||
|
||||
// InContainer specifies the container in which to look for objects.
|
||||
// Required parameter.
|
||||
//
|
||||
// Deprecated: Use PrmObjectSearch.ContainerID instead.
|
||||
func (x *PrmObjectSearch) InContainer(id cid.ID) {
|
||||
x.cnrID = id
|
||||
x.cnrSet = true
|
||||
x.ContainerID = &id
|
||||
}
|
||||
|
||||
// SetFilters sets filters by which to select objects. All container objects
|
||||
// match unset/empty filters.
|
||||
//
|
||||
// Deprecated: Use PrmObjectSearch.Filters instead.
|
||||
func (x *PrmObjectSearch) SetFilters(filters object.SearchFilters) {
|
||||
x.filters = filters
|
||||
x.Filters = filters
|
||||
}
|
||||
|
||||
// ResObjectSearch groups the final result values of ObjectSearch operation.
|
||||
|
@ -212,6 +226,48 @@ func (x *ObjectListReader) Close() (*ResObjectSearch, error) {
|
|||
return &x.res, nil
|
||||
}
|
||||
|
||||
func (x *PrmObjectSearch) buildRequest(c *Client) (*v2object.SearchRequest, error) {
|
||||
if x.ContainerID == nil {
|
||||
return nil, errorMissingContainer
|
||||
}
|
||||
|
||||
if len(x.XHeaders)%2 != 0 {
|
||||
return nil, errorInvalidXHeaders
|
||||
}
|
||||
|
||||
meta := new(v2session.RequestMetaHeader)
|
||||
writeXHeadersToMeta(x.XHeaders, meta)
|
||||
|
||||
if x.BearerToken != nil {
|
||||
v2BearerToken := new(acl.BearerToken)
|
||||
x.BearerToken.WriteToV2(v2BearerToken)
|
||||
meta.SetBearerToken(v2BearerToken)
|
||||
}
|
||||
|
||||
if x.Session != nil {
|
||||
v2SessionToken := new(v2session.Token)
|
||||
x.Session.WriteToV2(v2SessionToken)
|
||||
meta.SetSessionToken(v2SessionToken)
|
||||
}
|
||||
|
||||
if x.Local {
|
||||
meta.SetTTL(1)
|
||||
}
|
||||
cnrV2 := new(v2refs.ContainerID)
|
||||
x.ContainerID.WriteToV2(cnrV2)
|
||||
|
||||
body := new(v2object.SearchRequestBody)
|
||||
body.SetVersion(1)
|
||||
body.SetContainerID(cnrV2)
|
||||
body.SetFilters(x.Filters.ToV2())
|
||||
|
||||
req := new(v2object.SearchRequest)
|
||||
req.SetBody(body)
|
||||
c.prepareRequest(req, meta)
|
||||
|
||||
return req, nil
|
||||
}
|
||||
|
||||
// ObjectSearchInit initiates object selection through a remote server using FrostFS API protocol.
|
||||
//
|
||||
// The call only opens the transmission channel, explicit fetching of matched objects
|
||||
|
@ -221,30 +277,17 @@ func (x *ObjectListReader) Close() (*ResObjectSearch, error) {
|
|||
// Returns an error if parameters are set incorrectly (see PrmObjectSearch docs).
|
||||
// Context is required and must not be nil. It is used for network communication.
|
||||
func (c *Client) ObjectSearchInit(ctx context.Context, prm PrmObjectSearch) (*ObjectListReader, error) {
|
||||
// check parameters
|
||||
if !prm.cnrSet {
|
||||
return nil, errorMissingContainer
|
||||
req, err := prm.buildRequest(c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var cidV2 v2refs.ContainerID
|
||||
prm.cnrID.WriteToV2(&cidV2)
|
||||
|
||||
var body v2object.SearchRequestBody
|
||||
body.SetVersion(1)
|
||||
body.SetContainerID(&cidV2)
|
||||
body.SetFilters(prm.filters.ToV2())
|
||||
|
||||
// init reader
|
||||
var req v2object.SearchRequest
|
||||
req.SetBody(&body)
|
||||
c.prepareRequest(&req, &prm.meta)
|
||||
|
||||
key := prm.key
|
||||
key := prm.Key
|
||||
if key == nil {
|
||||
key = &c.prm.key
|
||||
key = &c.prm.Key
|
||||
}
|
||||
|
||||
err := signature.SignServiceMessage(key, &req)
|
||||
err = signature.SignServiceMessage(key, req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("sign request: %w", err)
|
||||
}
|
||||
|
@ -252,7 +295,7 @@ func (c *Client) ObjectSearchInit(ctx context.Context, prm PrmObjectSearch) (*Ob
|
|||
var r ObjectListReader
|
||||
ctx, r.cancelCtxStream = context.WithCancel(ctx)
|
||||
|
||||
r.stream, err = rpcapi.SearchObjects(&c.c, &req, client.WithContext(ctx))
|
||||
r.stream, err = rpcapi.SearchObjects(&c.c, req, client.WithContext(ctx))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open stream: %w", err)
|
||||
}
|
||||
|
|
|
@ -39,7 +39,7 @@ func (x *PrmSessionCreate) UseKey(key ecdsa.PrivateKey) {
|
|||
}
|
||||
|
||||
func (x *PrmSessionCreate) buildRequest(c *Client) (*v2session.CreateRequest, error) {
|
||||
ownerKey := c.prm.key.PublicKey
|
||||
ownerKey := c.prm.Key.PublicKey
|
||||
if x.Key != nil {
|
||||
ownerKey = x.Key.PublicKey
|
||||
}
|
||||
|
@ -89,9 +89,9 @@ func (x ResSessionCreate) PublicKey() []byte {
|
|||
//
|
||||
// Exactly one return value is non-nil. By default, server status is returned in res structure.
|
||||
// Any client's internal or transport errors are returned as `error`.
|
||||
// If PrmInit.ResolveFrostFSFailures has been called, unsuccessful
|
||||
// FrostFS status codes are returned as `error`, otherwise, are included
|
||||
// in the returned result structure.
|
||||
// If PrmInit.DisableFrostFSFailuresResolution has been called, unsuccessful
|
||||
// FrostFS status codes are included in the returned result structure,
|
||||
// otherwise, are also returned as `error`.
|
||||
//
|
||||
// Returns an error if parameters are set incorrectly (see PrmSessionCreate docs).
|
||||
// Context is required and must not be nil. It is used for network communication.
|
||||
|
@ -104,7 +104,7 @@ func (c *Client) SessionCreate(ctx context.Context, prm PrmSessionCreate) (*ResS
|
|||
return nil, err
|
||||
}
|
||||
|
||||
if err := signature.SignServiceMessage(&c.prm.key, req); err != nil {
|
||||
if err := signature.SignServiceMessage(&c.prm.Key, req); err != nil {
|
||||
return nil, fmt.Errorf("sign request: %w", err)
|
||||
}
|
||||
|
||||
|
|
|
@ -355,7 +355,7 @@ func (x Container) IterateAttributes(f func(key, val string)) {
|
|||
func (x Container) IterateUserAttributes(f func(key, val string)) {
|
||||
attrs := x.v2.GetAttributes()
|
||||
for _, attr := range attrs {
|
||||
var key = attr.GetKey()
|
||||
key := attr.GetKey()
|
||||
if !strings.HasPrefix(key, container.SysAttributePrefix) &&
|
||||
!strings.HasPrefix(key, container.SysAttributePrefixNeoFS) {
|
||||
f(key, attr.GetValue())
|
||||
|
|
|
@ -78,7 +78,7 @@ func TestContainer_Owner(t *testing.T) {
|
|||
|
||||
val = containertest.Container()
|
||||
|
||||
owner := *usertest.ID()
|
||||
owner := usertest.ID()
|
||||
|
||||
val.SetOwner(owner)
|
||||
|
||||
|
|
|
@ -16,7 +16,7 @@ func Container() (x container.Container) {
|
|||
|
||||
x.Init()
|
||||
x.SetAttribute("some attribute", "value")
|
||||
x.SetOwner(*owner)
|
||||
x.SetOwner(owner)
|
||||
x.SetBasicACL(BasicACL())
|
||||
x.SetPlacementPolicy(netmaptest.PlacementPolicy())
|
||||
|
||||
|
|
|
@ -125,7 +125,7 @@ func (r *Record) AddObjectContainerIDFilter(m Match, id cid.ID) {
|
|||
}
|
||||
|
||||
// AddObjectOwnerIDFilter adds filter by object owner ID.
|
||||
func (r *Record) AddObjectOwnerIDFilter(m Match, id *user.ID) {
|
||||
func (r *Record) AddObjectOwnerIDFilter(m Match, id user.ID) {
|
||||
r.addObjectReservedFilter(m, fKeyObjOwnerID, id)
|
||||
}
|
||||
|
||||
|
|
|
@ -225,14 +225,6 @@ func TestReservedRecords(t *testing.T) {
|
|||
key: v2acl.FilterObjectType,
|
||||
value: "TOMBSTONE",
|
||||
},
|
||||
{
|
||||
f: func(r *Record) {
|
||||
require.True(t, typ.FromString("STORAGE_GROUP"))
|
||||
r.AddObjectTypeFilter(MatchStringEqual, *typ)
|
||||
},
|
||||
key: v2acl.FilterObjectType,
|
||||
value: "STORAGE_GROUP",
|
||||
},
|
||||
}
|
||||
|
||||
for n, testCase := range testSuit {
|
||||
|
|
|
@ -7,8 +7,7 @@ import (
|
|||
// Validator is a tool that calculates
|
||||
// the action on a request according
|
||||
// to the extended ACL rule table.
|
||||
type Validator struct {
|
||||
}
|
||||
type Validator struct{}
|
||||
|
||||
// NewValidator creates and initializes a new Validator using options.
|
||||
func NewValidator() *Validator {
|
||||
|
|
10
go.mod
10
go.mod
|
@ -2,8 +2,10 @@ module git.frostfs.info/TrueCloudLab/frostfs-sdk-go
|
|||
|
||||
go 1.20
|
||||
|
||||
replace git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240427200446-67c6f305b21f => git.frostfs.info/aarifullin/frostfs-api-go/v2 v2.15.1-0.20240503124453-87caff9c8df0
|
||||
|
||||
require (
|
||||
git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230802075510-964c3edb3f44
|
||||
git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240427200446-67c6f305b21f
|
||||
git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb
|
||||
git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0
|
||||
git.frostfs.info/TrueCloudLab/hrw v1.2.1
|
||||
|
@ -11,12 +13,14 @@ require (
|
|||
github.com/antlr4-go/antlr/v4 v4.13.0
|
||||
github.com/google/uuid v1.3.0
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.2
|
||||
github.com/klauspost/reedsolomon v1.12.1
|
||||
github.com/mr-tron/base58 v1.2.0
|
||||
github.com/nspcc-dev/neo-go v0.101.2-0.20230601131642-a0117042e8fc
|
||||
github.com/stretchr/testify v1.8.3
|
||||
go.uber.org/zap v1.24.0
|
||||
google.golang.org/grpc v1.55.0
|
||||
google.golang.org/protobuf v1.30.0
|
||||
google.golang.org/protobuf v1.33.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
require (
|
||||
|
@ -27,6 +31,7 @@ require (
|
|||
github.com/golang/protobuf v1.5.3 // indirect
|
||||
github.com/gorilla/websocket v1.5.0 // indirect
|
||||
github.com/hashicorp/golang-lru v0.6.0 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.6 // indirect
|
||||
github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22 // indirect
|
||||
github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20230615193820-9185820289ce // indirect
|
||||
github.com/nspcc-dev/rfc6979 v0.2.0 // indirect
|
||||
|
@ -42,5 +47,4 @@ require (
|
|||
golang.org/x/sys v0.8.0 // indirect
|
||||
golang.org/x/text v0.9.0 // indirect
|
||||
google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
|
13
go.sum
13
go.sum
|
@ -31,8 +31,6 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl
|
|||
cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
|
||||
cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
|
||||
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
|
||||
git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230802075510-964c3edb3f44 h1:v6JqBD/VzZx3QSxbaXnUwnnJ1KEYheU4LzLGr3IhsAE=
|
||||
git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230802075510-964c3edb3f44/go.mod h1:pKJJRLOChW4zDQsAt1e8k/snWKljJtpkiPfxV53ngjI=
|
||||
git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb h1:S/TrbOOu9qEXZRZ9/Ddw7crnxbBUQLo68PSzQWYrc9M=
|
||||
git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb/go.mod h1:nkR5gaGeez3Zv2SE7aceP0YwxG2FzIB5cGKpQO2vV2o=
|
||||
git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk=
|
||||
|
@ -43,6 +41,8 @@ git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9m
|
|||
git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0/go.mod h1:okpbKfVYf/BpejtfFTfhZqFP+sZ8rsHrP8Rr/jYPNRc=
|
||||
git.frostfs.info/TrueCloudLab/tzhash v1.8.0 h1:UFMnUIk0Zh17m8rjGHJMqku2hCgaXDqjqZzS4gsb4UA=
|
||||
git.frostfs.info/TrueCloudLab/tzhash v1.8.0/go.mod h1:dhY+oy274hV8wGvGL4MwwMpdL3GYvaX1a8GQZQHvlF8=
|
||||
git.frostfs.info/aarifullin/frostfs-api-go/v2 v2.15.1-0.20240503124453-87caff9c8df0 h1:20DrUE/PcqDbEbW4EIoE4qi+Dm9Il4tE0RoHitYiIB8=
|
||||
git.frostfs.info/aarifullin/frostfs-api-go/v2 v2.15.1-0.20240503124453-87caff9c8df0/go.mod h1:OBDSr+DqV1z4VDouoX3YMleNc4DPBVBWTG3WDT2PK1o=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
|
||||
github.com/CityOfZion/neo-go v0.62.1-pre.0.20191114145240-e740fbe708f8/go.mod h1:MJCkWUBhi9pn/CrYO1Q3P687y2KeahrOPS9BD9LDGb0=
|
||||
|
@ -227,6 +227,10 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8
|
|||
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
|
||||
github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc=
|
||||
github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
|
||||
github.com/klauspost/reedsolomon v1.12.1 h1:NhWgum1efX1x58daOBGCFWcxtEhOhXKKl1HAPQUp03Q=
|
||||
github.com/klauspost/reedsolomon v1.12.1/go.mod h1:nEi5Kjb6QqtbofI6s+cbG/j1da11c96IBYBSnVGtuBs=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
|
||||
|
@ -548,6 +552,7 @@ golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBc
|
|||
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
|
||||
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
|
@ -698,8 +703,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0
|
|||
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||
google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
|
||||
google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
|
||||
google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
|
||||
google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
|
||||
google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
|
||||
gopkg.in/abiosoft/ishell.v2 v2.0.0/go.mod h1:sFp+cGtH6o4s1FtpVPTMcHq2yue+c4DGOVohJCPUzwY=
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
|
|
|
@ -156,11 +156,7 @@ func (a *meanIQRAgg) Compute() float64 {
|
|||
}
|
||||
|
||||
func (r *reverseMinNorm) Normalize(w float64) float64 {
|
||||
if w == 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
return r.min / w
|
||||
return (r.min + 1) / (w + 1)
|
||||
}
|
||||
|
||||
func (r *sigmoidNorm) Normalize(w float64) float64 {
|
||||
|
|
|
@ -53,7 +53,6 @@ func BenchmarkNetmap_ContainerNodes(b *testing.B) {
|
|||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,99 +0,0 @@
|
|||
{
|
||||
"name": "default CBF is 3",
|
||||
"nodes": [
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Location",
|
||||
"value": "Europe"
|
||||
},
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "RU"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "St.Petersburg"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Location",
|
||||
"value": "Europe"
|
||||
},
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "RU"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Moscow"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Location",
|
||||
"value": "Europe"
|
||||
},
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "DE"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Berlin"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Location",
|
||||
"value": "Europe"
|
||||
},
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "FR"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Paris"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"tests": {
|
||||
"set default CBF": {
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "EU"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 0,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "EU",
|
||||
"count": 1,
|
||||
"clause": "SAME",
|
||||
"attribute": "Location",
|
||||
"filter": "*"
|
||||
}
|
||||
],
|
||||
"filters": []
|
||||
},
|
||||
"result": [
|
||||
[
|
||||
0,
|
||||
1,
|
||||
2
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
|
@ -1,99 +0,0 @@
|
|||
{
|
||||
"name": "Real node count multiplier is in range [1, specified CBF]",
|
||||
"nodes": [
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "ID",
|
||||
"value": "1"
|
||||
},
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "DE"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "ID",
|
||||
"value": "2"
|
||||
},
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "DE"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "ID",
|
||||
"value": "3"
|
||||
},
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "DE"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"tests": {
|
||||
"select 2, CBF is 2": {
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "X"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 2,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "X",
|
||||
"count": 2,
|
||||
"clause": "SAME",
|
||||
"attribute": "Country",
|
||||
"filter": "*"
|
||||
}
|
||||
],
|
||||
"filters": []
|
||||
},
|
||||
"result": [
|
||||
[
|
||||
0,
|
||||
1,
|
||||
2
|
||||
]
|
||||
]
|
||||
},
|
||||
"select 3, CBF is 2": {
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "X"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 2,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "X",
|
||||
"count": 3,
|
||||
"clause": "SAME",
|
||||
"attribute": "Country",
|
||||
"filter": "*"
|
||||
}
|
||||
],
|
||||
"filters": []
|
||||
},
|
||||
"result": [
|
||||
[
|
||||
0,
|
||||
1,
|
||||
2
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
|
@ -1,155 +0,0 @@
|
|||
{
|
||||
"name": "CBF requirements",
|
||||
"nodes": [
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "ID",
|
||||
"value": "1"
|
||||
},
|
||||
{
|
||||
"key": "Attr",
|
||||
"value": "Same"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "ID",
|
||||
"value": "2"
|
||||
},
|
||||
{
|
||||
"key": "Attr",
|
||||
"value": "Same"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "ID",
|
||||
"value": "3"
|
||||
},
|
||||
{
|
||||
"key": "Attr",
|
||||
"value": "Same"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "ID",
|
||||
"value": "4"
|
||||
},
|
||||
{
|
||||
"key": "Attr",
|
||||
"value": "Same"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"tests": {
|
||||
"default CBF, no selector": {
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 2,
|
||||
"selector": ""
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 0,
|
||||
"selectors": [],
|
||||
"filters": []
|
||||
},
|
||||
"result": [
|
||||
[
|
||||
0,
|
||||
2,
|
||||
1,
|
||||
3
|
||||
]
|
||||
]
|
||||
},
|
||||
"explicit CBF, no selector": {
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 2,
|
||||
"selector": ""
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 3,
|
||||
"selectors": [],
|
||||
"filters": []
|
||||
},
|
||||
"result": [
|
||||
[
|
||||
0,
|
||||
2,
|
||||
1,
|
||||
3
|
||||
]
|
||||
]
|
||||
},
|
||||
"select distinct, weak CBF": {
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 2,
|
||||
"selector": "X"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 3,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "X",
|
||||
"count": 2,
|
||||
"clause": "DISTINCT",
|
||||
"attribute": "",
|
||||
"filter": "*"
|
||||
}
|
||||
],
|
||||
"filters": []
|
||||
},
|
||||
"result": [
|
||||
[
|
||||
0,
|
||||
2,
|
||||
1,
|
||||
3
|
||||
]
|
||||
]
|
||||
},
|
||||
"select same, weak CBF": {
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 2,
|
||||
"selector": "X"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 3,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "X",
|
||||
"count": 2,
|
||||
"clause": "SAME",
|
||||
"attribute": "Attr",
|
||||
"filter": "*"
|
||||
}
|
||||
],
|
||||
"filters": []
|
||||
},
|
||||
"result": [
|
||||
[
|
||||
0,
|
||||
1,
|
||||
2,
|
||||
3
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
|
@ -1,383 +0,0 @@
|
|||
{
|
||||
"name": "compound filter",
|
||||
"nodes": [
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Storage",
|
||||
"value": "SSD"
|
||||
},
|
||||
{
|
||||
"key": "Rating",
|
||||
"value": "10"
|
||||
},
|
||||
{
|
||||
"key": "IntField",
|
||||
"value": "100"
|
||||
},
|
||||
{
|
||||
"key": "Param",
|
||||
"value": "Value1"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"tests": {
|
||||
"good": {
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "S"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 1,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "S",
|
||||
"count": 1,
|
||||
"clause": "DISTINCT",
|
||||
"attribute": "",
|
||||
"filter": "Main"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "StorageSSD",
|
||||
"key": "Storage",
|
||||
"op": "EQ",
|
||||
"value": "SSD",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "GoodRating",
|
||||
"key": "Rating",
|
||||
"op": "GE",
|
||||
"value": "4",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "Main",
|
||||
"key": "",
|
||||
"op": "AND",
|
||||
"value": "",
|
||||
"filters": [
|
||||
{
|
||||
"name": "StorageSSD",
|
||||
"key": "",
|
||||
"op": "OPERATION_UNSPECIFIED",
|
||||
"value": "",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "",
|
||||
"key": "IntField",
|
||||
"op": "LT",
|
||||
"value": "123",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "GoodRating",
|
||||
"key": "",
|
||||
"op": "OPERATION_UNSPECIFIED",
|
||||
"value": "",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "",
|
||||
"key": "",
|
||||
"op": "OR",
|
||||
"value": "",
|
||||
"filters": [
|
||||
{
|
||||
"name": "",
|
||||
"key": "Param",
|
||||
"op": "EQ",
|
||||
"value": "Value1",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "",
|
||||
"key": "Param",
|
||||
"op": "EQ",
|
||||
"value": "Value2",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"result": [
|
||||
[
|
||||
0
|
||||
]
|
||||
]
|
||||
},
|
||||
"bad storage type": {
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "S"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 1,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "S",
|
||||
"count": 1,
|
||||
"clause": "DISTINCT",
|
||||
"attribute": "",
|
||||
"filter": "Main"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "StorageSSD",
|
||||
"key": "Storage",
|
||||
"op": "EQ",
|
||||
"value": "HDD",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "GoodRating",
|
||||
"key": "Rating",
|
||||
"op": "GE",
|
||||
"value": "4",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "Main",
|
||||
"key": "",
|
||||
"op": "AND",
|
||||
"value": "",
|
||||
"filters": [
|
||||
{
|
||||
"name": "StorageSSD",
|
||||
"key": "",
|
||||
"op": "OPERATION_UNSPECIFIED",
|
||||
"value": "",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "",
|
||||
"key": "IntField",
|
||||
"op": "LT",
|
||||
"value": "123",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "GoodRating",
|
||||
"key": "",
|
||||
"op": "OPERATION_UNSPECIFIED",
|
||||
"value": "",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "",
|
||||
"key": "",
|
||||
"op": "OR",
|
||||
"value": "",
|
||||
"filters": [
|
||||
{
|
||||
"name": "",
|
||||
"key": "Param",
|
||||
"op": "EQ",
|
||||
"value": "Value1",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "",
|
||||
"key": "Param",
|
||||
"op": "EQ",
|
||||
"value": "Value2",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"error": "not enough nodes"
|
||||
},
|
||||
"bad rating": {
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "S"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 1,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "S",
|
||||
"count": 1,
|
||||
"clause": "DISTINCT",
|
||||
"attribute": "",
|
||||
"filter": "Main"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "StorageSSD",
|
||||
"key": "Storage",
|
||||
"op": "EQ",
|
||||
"value": "SSD",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "GoodRating",
|
||||
"key": "Rating",
|
||||
"op": "GE",
|
||||
"value": "15",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "Main",
|
||||
"key": "",
|
||||
"op": "AND",
|
||||
"value": "",
|
||||
"filters": [
|
||||
{
|
||||
"name": "StorageSSD",
|
||||
"key": "",
|
||||
"op": "OPERATION_UNSPECIFIED",
|
||||
"value": "",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "",
|
||||
"key": "IntField",
|
||||
"op": "LT",
|
||||
"value": "123",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "GoodRating",
|
||||
"key": "",
|
||||
"op": "OPERATION_UNSPECIFIED",
|
||||
"value": "",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "",
|
||||
"key": "",
|
||||
"op": "OR",
|
||||
"value": "",
|
||||
"filters": [
|
||||
{
|
||||
"name": "",
|
||||
"key": "Param",
|
||||
"op": "EQ",
|
||||
"value": "Value1",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "",
|
||||
"key": "Param",
|
||||
"op": "EQ",
|
||||
"value": "Value2",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"error": "not enough nodes"
|
||||
},
|
||||
"bad param": {
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "S"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 1,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "S",
|
||||
"count": 1,
|
||||
"clause": "DISTINCT",
|
||||
"attribute": "",
|
||||
"filter": "Main"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "StorageSSD",
|
||||
"key": "Storage",
|
||||
"op": "EQ",
|
||||
"value": "SSD",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "GoodRating",
|
||||
"key": "Rating",
|
||||
"op": "GE",
|
||||
"value": "4",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "Main",
|
||||
"key": "",
|
||||
"op": "AND",
|
||||
"value": "",
|
||||
"filters": [
|
||||
{
|
||||
"name": "StorageSSD",
|
||||
"key": "",
|
||||
"op": "OPERATION_UNSPECIFIED",
|
||||
"value": "",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "",
|
||||
"key": "IntField",
|
||||
"op": "LT",
|
||||
"value": "123",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "GoodRating",
|
||||
"key": "",
|
||||
"op": "OPERATION_UNSPECIFIED",
|
||||
"value": "",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "",
|
||||
"key": "",
|
||||
"op": "OR",
|
||||
"value": "",
|
||||
"filters": [
|
||||
{
|
||||
"name": "",
|
||||
"key": "Param",
|
||||
"op": "EQ",
|
||||
"value": "Value0",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "",
|
||||
"key": "Param",
|
||||
"op": "EQ",
|
||||
"value": "Value2",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"error": "not enough nodes"
|
||||
}
|
||||
}
|
||||
}
|
|
@ -1,83 +0,0 @@
|
|||
{
|
||||
"name": "invalid integer field",
|
||||
"nodes": [
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "IntegerField",
|
||||
"value": "true"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "IntegerField",
|
||||
"value": "str"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"tests": {
|
||||
"empty string is not casted to 0": {
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "S"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 1,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "S",
|
||||
"count": 1,
|
||||
"clause": "DISTINCT",
|
||||
"attribute": "",
|
||||
"filter": "Main"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "Main",
|
||||
"key": "IntegerField",
|
||||
"op": "LE",
|
||||
"value": "8",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
},
|
||||
"error": "not enough nodes"
|
||||
},
|
||||
"non-empty string is not casted to a number": {
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "S"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 1,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "S",
|
||||
"count": 1,
|
||||
"clause": "DISTINCT",
|
||||
"attribute": "",
|
||||
"filter": "Main"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "Main",
|
||||
"key": "IntegerField",
|
||||
"op": "GE",
|
||||
"value": "0",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
},
|
||||
"error": "not enough nodes"
|
||||
}
|
||||
}
|
||||
}
|
|
@ -1,403 +0,0 @@
|
|||
{
|
||||
"name": "single-op filters",
|
||||
"nodes": [
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Rating",
|
||||
"value": "4"
|
||||
},
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "Germany"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"tests": {
|
||||
"GE true": {
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "S"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 1,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "S",
|
||||
"count": 1,
|
||||
"clause": "DISTINCT",
|
||||
"attribute": "",
|
||||
"filter": "Main"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "Main",
|
||||
"key": "Rating",
|
||||
"op": "GE",
|
||||
"value": "4",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
},
|
||||
"result": [
|
||||
[
|
||||
0
|
||||
]
|
||||
]
|
||||
},
|
||||
"GE false": {
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "S"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 1,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "S",
|
||||
"count": 1,
|
||||
"clause": "DISTINCT",
|
||||
"attribute": "",
|
||||
"filter": "Main"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "Main",
|
||||
"key": "Rating",
|
||||
"op": "GE",
|
||||
"value": "5",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
},
|
||||
"error": "not enough nodes"
|
||||
},
|
||||
"GT true": {
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "S"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 1,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "S",
|
||||
"count": 1,
|
||||
"clause": "DISTINCT",
|
||||
"attribute": "",
|
||||
"filter": "Main"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "Main",
|
||||
"key": "Rating",
|
||||
"op": "GT",
|
||||
"value": "3",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
},
|
||||
"result": [
|
||||
[
|
||||
0
|
||||
]
|
||||
]
|
||||
},
|
||||
"GT false": {
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "S"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 1,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "S",
|
||||
"count": 1,
|
||||
"clause": "DISTINCT",
|
||||
"attribute": "",
|
||||
"filter": "Main"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "Main",
|
||||
"key": "Rating",
|
||||
"op": "GT",
|
||||
"value": "4",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
},
|
||||
"error": "not enough nodes"
|
||||
},
|
||||
"LE true": {
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "S"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 1,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "S",
|
||||
"count": 1,
|
||||
"clause": "DISTINCT",
|
||||
"attribute": "",
|
||||
"filter": "Main"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "Main",
|
||||
"key": "Rating",
|
||||
"op": "LE",
|
||||
"value": "4",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
},
|
||||
"result": [
|
||||
[
|
||||
0
|
||||
]
|
||||
]
|
||||
},
|
||||
"LE false": {
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "S"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 1,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "S",
|
||||
"count": 1,
|
||||
"clause": "DISTINCT",
|
||||
"attribute": "",
|
||||
"filter": "Main"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "Main",
|
||||
"key": "Rating",
|
||||
"op": "LE",
|
||||
"value": "3",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
},
|
||||
"error": "not enough nodes"
|
||||
},
|
||||
"LT true": {
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "S"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 1,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "S",
|
||||
"count": 1,
|
||||
"clause": "DISTINCT",
|
||||
"attribute": "",
|
||||
"filter": "Main"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "Main",
|
||||
"key": "Rating",
|
||||
"op": "LT",
|
||||
"value": "5",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
},
|
||||
"result": [
|
||||
[
|
||||
0
|
||||
]
|
||||
]
|
||||
},
|
||||
"LT false": {
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "S"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 1,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "S",
|
||||
"count": 1,
|
||||
"clause": "DISTINCT",
|
||||
"attribute": "",
|
||||
"filter": "Main"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "Main",
|
||||
"key": "Rating",
|
||||
"op": "LT",
|
||||
"value": "4",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
},
|
||||
"error": "not enough nodes"
|
||||
},
|
||||
"EQ true": {
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "S"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 1,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "S",
|
||||
"count": 1,
|
||||
"clause": "DISTINCT",
|
||||
"attribute": "",
|
||||
"filter": "Main"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "Main",
|
||||
"key": "Country",
|
||||
"op": "EQ",
|
||||
"value": "Germany",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
},
|
||||
"result": [
|
||||
[
|
||||
0
|
||||
]
|
||||
]
|
||||
},
|
||||
"EQ false": {
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "S"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 1,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "S",
|
||||
"count": 1,
|
||||
"clause": "DISTINCT",
|
||||
"attribute": "",
|
||||
"filter": "Main"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "Main",
|
||||
"key": "Country",
|
||||
"op": "EQ",
|
||||
"value": "China",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
},
|
||||
"error": "not enough nodes"
|
||||
},
|
||||
"NE true": {
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "S"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 1,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "S",
|
||||
"count": 1,
|
||||
"clause": "DISTINCT",
|
||||
"attribute": "",
|
||||
"filter": "Main"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "Main",
|
||||
"key": "Country",
|
||||
"op": "NE",
|
||||
"value": "France",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
},
|
||||
"result": [
|
||||
[
|
||||
0
|
||||
]
|
||||
]
|
||||
},
|
||||
"NE false": {
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "S"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 1,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "S",
|
||||
"count": 1,
|
||||
"clause": "DISTINCT",
|
||||
"attribute": "",
|
||||
"filter": "Main"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "Main",
|
||||
"key": "Country",
|
||||
"op": "NE",
|
||||
"value": "Germany",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
},
|
||||
"error": "not enough nodes"
|
||||
}
|
||||
}
|
||||
}
|
|
@ -1,165 +0,0 @@
|
|||
{
|
||||
"name": "HRW ordering",
|
||||
"nodes": [
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "Germany"
|
||||
},
|
||||
{
|
||||
"key": "Price",
|
||||
"value": "2"
|
||||
},
|
||||
{
|
||||
"key": "Capacity",
|
||||
"value": "10000"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "Germany"
|
||||
},
|
||||
{
|
||||
"key": "Price",
|
||||
"value": "4"
|
||||
},
|
||||
{
|
||||
"key": "Capacity",
|
||||
"value": "1"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "France"
|
||||
},
|
||||
{
|
||||
"key": "Price",
|
||||
"value": "3"
|
||||
},
|
||||
{
|
||||
"key": "Capacity",
|
||||
"value": "10"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "Russia"
|
||||
},
|
||||
{
|
||||
"key": "Price",
|
||||
"value": "2"
|
||||
},
|
||||
{
|
||||
"key": "Capacity",
|
||||
"value": "10000"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "Russia"
|
||||
},
|
||||
{
|
||||
"key": "Price",
|
||||
"value": "1"
|
||||
},
|
||||
{
|
||||
"key": "Capacity",
|
||||
"value": "10000"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "Russia"
|
||||
},
|
||||
{
|
||||
"key": "Capacity",
|
||||
"value": "10000"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "France"
|
||||
},
|
||||
{
|
||||
"key": "Price",
|
||||
"value": "100"
|
||||
},
|
||||
{
|
||||
"key": "Capacity",
|
||||
"value": "1"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "France"
|
||||
},
|
||||
{
|
||||
"key": "Price",
|
||||
"value": "7"
|
||||
},
|
||||
{
|
||||
"key": "Capacity",
|
||||
"value": "10000"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "Russia"
|
||||
},
|
||||
{
|
||||
"key": "Price",
|
||||
"value": "2"
|
||||
},
|
||||
{
|
||||
"key": "Capacity",
|
||||
"value": "1"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"tests": {
|
||||
"select 3 nodes in 3 distinct countries, same placement": {
|
||||
"policy": {"replicas":[{"count":1,"selector":"Main"}],"containerBackupFactor":1,"selectors":[{"name":"Main","count":3,"clause":"DISTINCT","attribute":"Country","filter":"*"}],"filters":[]},
|
||||
"pivot": "Y29udGFpbmVySUQ=",
|
||||
"result": [[0, 2, 3]],
|
||||
"placement": {
|
||||
"pivot": "b2JqZWN0SUQ=",
|
||||
"result": [[0, 2, 3]]
|
||||
}
|
||||
},
|
||||
"select 6 nodes in 3 distinct countries, different placement": {
|
||||
"policy": {"replicas":[{"count":1,"selector":"Main"}],"containerBackupFactor":2,"selectors":[{"name":"Main","count":3,"clause":"DISTINCT","attribute":"Country","filter":"*"}],"filters":[]},
|
||||
"pivot": "Y29udGFpbmVySUQ=",
|
||||
"result": [[0, 1, 2, 6, 3, 4]],
|
||||
"placement": {
|
||||
"pivot": "b2JqZWN0SUQ=",
|
||||
"result": [[0, 1, 2, 6, 3, 4]]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
|
@ -1,108 +0,0 @@
|
|||
{
|
||||
"name": "unnamed selector (nspcc-dev/neofs-api-go#213)",
|
||||
"nodes": [
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Location",
|
||||
"value": "Europe"
|
||||
},
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "Russia"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Moscow"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Location",
|
||||
"value": "Europe"
|
||||
},
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "Russia"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Saint-Petersburg"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Location",
|
||||
"value": "Europe"
|
||||
},
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "Sweden"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Stockholm"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Location",
|
||||
"value": "Europe"
|
||||
},
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "Finalnd"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Helsinki"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"tests": {
|
||||
"test": {
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 4,
|
||||
"selector": ""
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 1,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "",
|
||||
"count": 4,
|
||||
"clause": "DISTINCT",
|
||||
"attribute": "",
|
||||
"filter": "LOC_EU"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "LOC_EU",
|
||||
"key": "Location",
|
||||
"op": "EQ",
|
||||
"value": "Europe",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
},
|
||||
"result": [
|
||||
[
|
||||
0,
|
||||
1,
|
||||
2,
|
||||
3
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
|
@ -1,192 +0,0 @@
|
|||
{
|
||||
"name": "single-op filters",
|
||||
"nodes": [
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "Russia"
|
||||
},
|
||||
{
|
||||
"key": "Rating",
|
||||
"value": "1"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "SPB"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "Germany"
|
||||
},
|
||||
{
|
||||
"key": "Rating",
|
||||
"value": "5"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Berlin"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "Russia"
|
||||
},
|
||||
{
|
||||
"key": "Rating",
|
||||
"value": "6"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Moscow"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "France"
|
||||
},
|
||||
{
|
||||
"key": "Rating",
|
||||
"value": "4"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Paris"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "France"
|
||||
},
|
||||
{
|
||||
"key": "Rating",
|
||||
"value": "1"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Lyon"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "Russia"
|
||||
},
|
||||
{
|
||||
"key": "Rating",
|
||||
"value": "5"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "SPB"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "Russia"
|
||||
},
|
||||
{
|
||||
"key": "Rating",
|
||||
"value": "7"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Moscow"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "Germany"
|
||||
},
|
||||
{
|
||||
"key": "Rating",
|
||||
"value": "3"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Darmstadt"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "Germany"
|
||||
},
|
||||
{
|
||||
"key": "Rating",
|
||||
"value": "7"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Frankfurt"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "Russia"
|
||||
},
|
||||
{
|
||||
"key": "Rating",
|
||||
"value": "9"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "SPB"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "Russia"
|
||||
},
|
||||
{
|
||||
"key": "Rating",
|
||||
"value": "9"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "SPB"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"tests": {
|
||||
"Select": {
|
||||
"policy": {"replicas":[{"count":1,"selector":"SameRU"},{"count":1,"selector":"DistinctRU"},{"count":1,"selector":"Good"},{"count":1,"selector":"Main"}],"containerBackupFactor":2,"selectors":[{"name":"SameRU","count":2,"clause":"SAME","attribute":"City","filter":"FromRU"},{"name":"DistinctRU","count":2,"clause":"DISTINCT","attribute":"City","filter":"FromRU"},{"name":"Good","count":2,"clause":"DISTINCT","attribute":"Country","filter":"Good"},{"name":"Main","count":3,"clause":"DISTINCT","attribute":"Country","filter":"*"}],"filters":[{"name":"FromRU","key":"Country","op":"EQ","value":"Russia"},{"name":"Good","key":"Rating","op":"GE","value":"4"}]},
|
||||
"result": [
|
||||
[0, 5, 9, 10],
|
||||
[2, 6, 0, 5],
|
||||
[1, 8, 2, 5],
|
||||
[3, 4, 1, 7, 0, 2]
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
|
@ -1,94 +0,0 @@
|
|||
{
|
||||
"name": "multiple replicas (#215)",
|
||||
"nodes": [
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Saint-Petersburg"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Moscow"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Berlin"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Paris"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"tests": {
|
||||
"test": {
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "LOC_SPB_PLACE"
|
||||
},
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "LOC_MSK_PLACE"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 1,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "LOC_SPB_PLACE",
|
||||
"count": 1,
|
||||
"clause": "CLAUSE_UNSPECIFIED",
|
||||
"attribute": "",
|
||||
"filter": "LOC_SPB"
|
||||
},
|
||||
{
|
||||
"name": "LOC_MSK_PLACE",
|
||||
"count": 1,
|
||||
"clause": "CLAUSE_UNSPECIFIED",
|
||||
"attribute": "",
|
||||
"filter": "LOC_MSK"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "LOC_SPB",
|
||||
"key": "City",
|
||||
"op": "EQ",
|
||||
"value": "Saint-Petersburg",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "LOC_MSK",
|
||||
"key": "City",
|
||||
"op": "EQ",
|
||||
"value": "Moscow",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
},
|
||||
"result": [
|
||||
[
|
||||
0
|
||||
],
|
||||
[
|
||||
1
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
|
@ -1,331 +0,0 @@
|
|||
{
|
||||
"name": "multiple REP, asymmetric",
|
||||
"nodes": [
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "ID",
|
||||
"value": "1"
|
||||
},
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "RU"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "St.Petersburg"
|
||||
},
|
||||
{
|
||||
"key": "SSD",
|
||||
"value": "0"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "ID",
|
||||
"value": "2"
|
||||
},
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "RU"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "St.Petersburg"
|
||||
},
|
||||
{
|
||||
"key": "SSD",
|
||||
"value": "1"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "ID",
|
||||
"value": "3"
|
||||
},
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "RU"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Moscow"
|
||||
},
|
||||
{
|
||||
"key": "SSD",
|
||||
"value": "1"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "ID",
|
||||
"value": "4"
|
||||
},
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "RU"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Moscow"
|
||||
},
|
||||
{
|
||||
"key": "SSD",
|
||||
"value": "1"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "ID",
|
||||
"value": "5"
|
||||
},
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "RU"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "St.Petersburg"
|
||||
},
|
||||
{
|
||||
"key": "SSD",
|
||||
"value": "1"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "ID",
|
||||
"value": "6"
|
||||
},
|
||||
{
|
||||
"key": "Continent",
|
||||
"value": "NA"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "NewYork"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "ID",
|
||||
"value": "7"
|
||||
},
|
||||
{
|
||||
"key": "Continent",
|
||||
"value": "AF"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Cairo"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "ID",
|
||||
"value": "8"
|
||||
},
|
||||
{
|
||||
"key": "Continent",
|
||||
"value": "AF"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Cairo"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "ID",
|
||||
"value": "9"
|
||||
},
|
||||
{
|
||||
"key": "Continent",
|
||||
"value": "SA"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Lima"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "ID",
|
||||
"value": "10"
|
||||
},
|
||||
{
|
||||
"key": "Continent",
|
||||
"value": "AF"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Cairo"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "ID",
|
||||
"value": "11"
|
||||
},
|
||||
{
|
||||
"key": "Continent",
|
||||
"value": "NA"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "NewYork"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "ID",
|
||||
"value": "12"
|
||||
},
|
||||
{
|
||||
"key": "Continent",
|
||||
"value": "NA"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "LosAngeles"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "ID",
|
||||
"value": "13"
|
||||
},
|
||||
{
|
||||
"key": "Continent",
|
||||
"value": "SA"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Lima"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"tests": {
|
||||
"test": {
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "SPB"
|
||||
},
|
||||
{
|
||||
"count": 2,
|
||||
"selector": "Americas"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 2,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "SPB",
|
||||
"count": 1,
|
||||
"clause": "SAME",
|
||||
"attribute": "City",
|
||||
"filter": "SPBSSD"
|
||||
},
|
||||
{
|
||||
"name": "Americas",
|
||||
"count": 2,
|
||||
"clause": "DISTINCT",
|
||||
"attribute": "City",
|
||||
"filter": "Americas"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "SPBSSD",
|
||||
"key": "",
|
||||
"op": "AND",
|
||||
"value": "",
|
||||
"filters": [
|
||||
{
|
||||
"name": "",
|
||||
"key": "Country",
|
||||
"op": "EQ",
|
||||
"value": "RU",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "",
|
||||
"key": "City",
|
||||
"op": "EQ",
|
||||
"value": "St.Petersburg",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "",
|
||||
"key": "SSD",
|
||||
"op": "EQ",
|
||||
"value": "1",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Americas",
|
||||
"key": "",
|
||||
"op": "OR",
|
||||
"value": "",
|
||||
"filters": [
|
||||
{
|
||||
"name": "",
|
||||
"key": "Continent",
|
||||
"op": "EQ",
|
||||
"value": "NA",
|
||||
"filters": []
|
||||
},
|
||||
{
|
||||
"name": "",
|
||||
"key": "Continent",
|
||||
"op": "EQ",
|
||||
"value": "SA",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"result": [
|
||||
[
|
||||
1,
|
||||
4
|
||||
],
|
||||
[
|
||||
8,
|
||||
12,
|
||||
5,
|
||||
10
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
|
@ -1,95 +0,0 @@
|
|||
{
|
||||
"name": "non-strict selections",
|
||||
"comment": "These test specify loose selection behaviour, to allow fetching already PUT objects even when there is not enough nodes to select from.",
|
||||
"nodes": [
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "Russia"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "Germany"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [ ]
|
||||
}
|
||||
],
|
||||
"tests": {
|
||||
"not enough nodes (backup factor)": {
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "MyStore"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 2,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "MyStore",
|
||||
"count": 2,
|
||||
"clause": "DISTINCT",
|
||||
"attribute": "Country",
|
||||
"filter": "FromRU"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "FromRU",
|
||||
"key": "Country",
|
||||
"op": "EQ",
|
||||
"value": "Russia",
|
||||
"filters": [ ]
|
||||
}
|
||||
]
|
||||
},
|
||||
"result": [
|
||||
[
|
||||
0
|
||||
]
|
||||
]
|
||||
},
|
||||
"not enough nodes (buckets)": {
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "MyStore"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 1,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "MyStore",
|
||||
"count": 2,
|
||||
"clause": "DISTINCT",
|
||||
"attribute": "Country",
|
||||
"filter": "FromRU"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "FromRU",
|
||||
"key": "Country",
|
||||
"op": "EQ",
|
||||
"value": "Russia",
|
||||
"filters": [ ]
|
||||
}
|
||||
]
|
||||
},
|
||||
"result": [
|
||||
[
|
||||
0
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
|
@ -1,117 +0,0 @@
|
|||
{
|
||||
"name": "REP X",
|
||||
"nodes": [
|
||||
{
|
||||
"publicKey": "",
|
||||
"addresses": [],
|
||||
"attributes": [
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Saint-Petersburg",
|
||||
"parents": []
|
||||
}
|
||||
],
|
||||
"state": "UNSPECIFIED"
|
||||
},
|
||||
{
|
||||
"publicKey": "",
|
||||
"addresses": [],
|
||||
"attributes": [
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Moscow",
|
||||
"parents": []
|
||||
}
|
||||
],
|
||||
"state": "UNSPECIFIED"
|
||||
},
|
||||
{
|
||||
"publicKey": "",
|
||||
"addresses": [],
|
||||
"attributes": [
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Berlin",
|
||||
"parents": []
|
||||
}
|
||||
],
|
||||
"state": "UNSPECIFIED"
|
||||
},
|
||||
{
|
||||
"publicKey": "",
|
||||
"addresses": [],
|
||||
"attributes": [
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Paris",
|
||||
"parents": []
|
||||
}
|
||||
],
|
||||
"state": "UNSPECIFIED"
|
||||
}
|
||||
],
|
||||
"tests": {
|
||||
"REP 1": {
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": ""
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 0,
|
||||
"selectors": [],
|
||||
"filters": []
|
||||
},
|
||||
"result": [
|
||||
[
|
||||
0,
|
||||
1,
|
||||
2
|
||||
]
|
||||
]
|
||||
},
|
||||
"REP 3": {
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 3,
|
||||
"selector": ""
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 0,
|
||||
"selectors": [],
|
||||
"filters": []
|
||||
},
|
||||
"result": [
|
||||
[
|
||||
0,
|
||||
3,
|
||||
1,
|
||||
2
|
||||
]
|
||||
]
|
||||
},
|
||||
"REP 5": {
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 5,
|
||||
"selector": ""
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 0,
|
||||
"selectors": [],
|
||||
"filters": []
|
||||
},
|
||||
"result": [
|
||||
[
|
||||
0,
|
||||
1,
|
||||
2,
|
||||
3
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
|
@ -1,116 +0,0 @@
|
|||
{
|
||||
"name": "select with unspecified attribute",
|
||||
"nodes": [
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "ID",
|
||||
"value": "1"
|
||||
},
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "RU"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "St.Petersburg"
|
||||
},
|
||||
{
|
||||
"key": "SSD",
|
||||
"value": "0"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "ID",
|
||||
"value": "2"
|
||||
},
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "RU"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "St.Petersburg"
|
||||
},
|
||||
{
|
||||
"key": "SSD",
|
||||
"value": "1"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "ID",
|
||||
"value": "3"
|
||||
},
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "RU"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Moscow"
|
||||
},
|
||||
{
|
||||
"key": "SSD",
|
||||
"value": "1"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "ID",
|
||||
"value": "4"
|
||||
},
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "RU"
|
||||
},
|
||||
{
|
||||
"key": "City",
|
||||
"value": "Moscow"
|
||||
},
|
||||
{
|
||||
"key": "SSD",
|
||||
"value": "1"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"tests": {
|
||||
"test": {
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "X"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 1,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "X",
|
||||
"count": 4,
|
||||
"clause": "DISTINCT",
|
||||
"attribute": "",
|
||||
"filter": "*"
|
||||
}
|
||||
],
|
||||
"filters": []
|
||||
},
|
||||
"result": [
|
||||
[
|
||||
0,
|
||||
1,
|
||||
2,
|
||||
3
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
|
@ -1,86 +0,0 @@
|
|||
{
|
||||
"name": "invalid selections",
|
||||
"nodes": [
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "Russia"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": [
|
||||
{
|
||||
"key": "Country",
|
||||
"value": "Germany"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attributes": []
|
||||
}
|
||||
],
|
||||
"tests": {
|
||||
"missing filter": {
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "MyStore"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 1,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "MyStore",
|
||||
"count": 1,
|
||||
"clause": "DISTINCT",
|
||||
"attribute": "Country",
|
||||
"filter": "FromNL"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "FromRU",
|
||||
"key": "Country",
|
||||
"op": "EQ",
|
||||
"value": "Russia",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
},
|
||||
"error": "filter not found"
|
||||
},
|
||||
"not enough nodes (filter results in empty set)": {
|
||||
"policy": {
|
||||
"replicas": [
|
||||
{
|
||||
"count": 1,
|
||||
"selector": "MyStore"
|
||||
}
|
||||
],
|
||||
"containerBackupFactor": 2,
|
||||
"selectors": [
|
||||
{
|
||||
"name": "MyStore",
|
||||
"count": 2,
|
||||
"clause": "DISTINCT",
|
||||
"attribute": "Country",
|
||||
"filter": "FromMoon"
|
||||
}
|
||||
],
|
||||
"filters": [
|
||||
{
|
||||
"name": "FromMoon",
|
||||
"key": "Country",
|
||||
"op": "EQ",
|
||||
"value": "Moon",
|
||||
"filters": []
|
||||
}
|
||||
]
|
||||
},
|
||||
"error": "not enough nodes"
|
||||
}
|
||||
}
|
||||
}
|
|
@ -209,6 +209,25 @@ func (m NetMap) SelectFilterNodes(expr *SelectFilterExpr) ([][]NodeInfo, error)
|
|||
return ret, nil
|
||||
}
|
||||
|
||||
func countNodes(r netmap.Replica) uint32 {
|
||||
if r.GetCount() != 0 {
|
||||
return r.GetCount()
|
||||
}
|
||||
return r.GetECDataCount() + r.GetECParityCount()
|
||||
}
|
||||
|
||||
func (p PlacementPolicy) isUnique() bool {
|
||||
if p.unique {
|
||||
return true
|
||||
}
|
||||
for _, r := range p.replicas {
|
||||
if r.GetECDataCount() != 0 || r.GetECParityCount() != 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ContainerNodes returns two-dimensional list of nodes as a result of applying
|
||||
// given PlacementPolicy to the NetMap. Each line of the list corresponds to a
|
||||
// replica descriptor. Line order corresponds to order of ReplicaDescriptor list
|
||||
|
@ -230,6 +249,7 @@ func (m NetMap) ContainerNodes(p PlacementPolicy, pivot []byte) ([][]NodeInfo, e
|
|||
return nil, err
|
||||
}
|
||||
|
||||
unique := p.isUnique()
|
||||
result := make([][]NodeInfo, len(p.replicas))
|
||||
|
||||
// Note that the cached selectors are not used when the policy contains the UNIQUE flag.
|
||||
|
@ -240,7 +260,7 @@ func (m NetMap) ContainerNodes(p PlacementPolicy, pivot []byte) ([][]NodeInfo, e
|
|||
sName := p.replicas[i].GetSelector()
|
||||
if sName == "" && !(len(p.replicas) == 1 && len(p.selectors) == 1) {
|
||||
var s netmap.Selector
|
||||
s.SetCount(p.replicas[i].GetCount())
|
||||
s.SetCount(countNodes(p.replicas[i]))
|
||||
s.SetFilter(mainFilterName)
|
||||
|
||||
nodes, err := c.getSelection(s)
|
||||
|
@ -250,14 +270,14 @@ func (m NetMap) ContainerNodes(p PlacementPolicy, pivot []byte) ([][]NodeInfo, e
|
|||
|
||||
result[i] = append(result[i], flattenNodes(nodes)...)
|
||||
|
||||
if p.unique {
|
||||
if unique {
|
||||
c.addUsedNodes(result[i]...)
|
||||
}
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
if p.unique {
|
||||
if unique {
|
||||
if c.processedSelectors[sName] == nil {
|
||||
return nil, fmt.Errorf("selector not found: '%s'", sName)
|
||||
}
|
||||
|
|
|
@ -62,6 +62,8 @@ func (x *NetworkInfo) readFromV2(m netmap.NetworkInfo, checkFieldPresence bool)
|
|||
configEpochDuration,
|
||||
configIRCandidateFee,
|
||||
configMaxObjSize,
|
||||
configMaxECDataCount,
|
||||
configMaxECParityCount,
|
||||
configWithdrawalFee:
|
||||
_, err = decodeConfigValueUint64(prm.GetValue())
|
||||
case configHomomorphicHashingDisabled,
|
||||
|
@ -234,6 +236,8 @@ func (x *NetworkInfo) IterateRawNetworkParameters(f func(name string, value []by
|
|||
configEpochDuration,
|
||||
configIRCandidateFee,
|
||||
configMaxObjSize,
|
||||
configMaxECDataCount,
|
||||
configMaxECParityCount,
|
||||
configWithdrawalFee,
|
||||
configHomomorphicHashingDisabled,
|
||||
configMaintenanceModeAllowed:
|
||||
|
@ -432,6 +436,34 @@ func (x NetworkInfo) MaxObjectSize() uint64 {
|
|||
return x.configUint64(configMaxObjSize)
|
||||
}
|
||||
|
||||
const configMaxECDataCount = "MaxECDataCount"
|
||||
|
||||
// SetMaxECDataCount sets maximum number of data shards for erasure codes.
|
||||
//
|
||||
// Zero means no restrictions.
|
||||
func (x *NetworkInfo) SetMaxECDataCount(dataCount uint64) {
|
||||
x.setConfigUint64(configMaxECDataCount, dataCount)
|
||||
}
|
||||
|
||||
// MaxECDataCount returns maximum number of data shards for erasure codes.
|
||||
func (x NetworkInfo) MaxECDataCount() uint64 {
|
||||
return x.configUint64(configMaxECDataCount)
|
||||
}
|
||||
|
||||
const configMaxECParityCount = "MaxECParityCount"
|
||||
|
||||
// SetMaxECParityCount sets maximum number of parity shards for erasure codes.
|
||||
//
|
||||
// Zero means no restrictions.
|
||||
func (x *NetworkInfo) SetMaxECParityCount(parityCount uint64) {
|
||||
x.setConfigUint64(configMaxECParityCount, parityCount)
|
||||
}
|
||||
|
||||
// MaxECParityCount returns maximum number of parity shards for erasure codes.
|
||||
func (x NetworkInfo) MaxECParityCount() uint64 {
|
||||
return x.configUint64(configMaxECParityCount)
|
||||
}
|
||||
|
||||
const configWithdrawalFee = "WithdrawFee"
|
||||
|
||||
// SetWithdrawalFee sets fee for withdrawals from the FrostFS accounts that
|
||||
|
|
|
@ -173,6 +173,32 @@ func TestNetworkInfo_MaxObjectSize(t *testing.T) {
|
|||
)
|
||||
}
|
||||
|
||||
func TestNetworkInfo_MaxECDataCount(t *testing.T) {
|
||||
testConfigValue(t,
|
||||
func(x NetworkInfo) any { return x.MaxECDataCount() },
|
||||
func(info *NetworkInfo, val any) { info.SetMaxECDataCount(val.(uint64)) },
|
||||
uint64(1), uint64(2),
|
||||
"MaxECDataCount", func(val any) []byte {
|
||||
data := make([]byte, 8)
|
||||
binary.LittleEndian.PutUint64(data, val.(uint64))
|
||||
return data
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func TestNetworkInfo_MaxECParityCount(t *testing.T) {
|
||||
testConfigValue(t,
|
||||
func(x NetworkInfo) any { return x.MaxECParityCount() },
|
||||
func(info *NetworkInfo, val any) { info.SetMaxECParityCount(val.(uint64)) },
|
||||
uint64(1), uint64(2),
|
||||
"MaxECParityCount", func(val any) []byte {
|
||||
data := make([]byte, 8)
|
||||
binary.LittleEndian.PutUint64(data, val.(uint64))
|
||||
return data
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func TestNetworkInfo_WithdrawalFee(t *testing.T) {
|
||||
testConfigValue(t,
|
||||
func(x NetworkInfo) any { return x.WithdrawalFee() },
|
||||
|
|
|
@ -4,10 +4,14 @@ options {
|
|||
tokenVocab = QueryLexer;
|
||||
}
|
||||
|
||||
policy: UNIQUE? repStmt+ cbfStmt? selectStmt* filterStmt* EOF;
|
||||
policy: UNIQUE? (repStmt | ecStmt)+ cbfStmt? selectStmt* filterStmt* EOF;
|
||||
|
||||
selectFilterExpr: cbfStmt? selectStmt? filterStmt* EOF;
|
||||
|
||||
ecStmt:
|
||||
EC Data = NUMBER1 DOT Parity = NUMBER1 // erasure code configuration
|
||||
(IN Selector = ident)?; // optional selector name
|
||||
|
||||
repStmt:
|
||||
REP Count = NUMBER1 // number of object replicas
|
||||
(IN Selector = ident)?; // optional selector name
|
||||
|
|
Binary file not shown.
Binary file not shown.
|
@ -7,6 +7,7 @@ SIMPLE_OP : 'EQ' | 'NE' | 'GE' | 'GT' | 'LT' | 'LE';
|
|||
|
||||
UNIQUE : 'UNIQUE';
|
||||
REP : 'REP';
|
||||
EC : 'EC';
|
||||
IN : 'IN';
|
||||
AS : 'AS';
|
||||
CBF : 'CBF';
|
||||
|
@ -14,6 +15,7 @@ SELECT : 'SELECT';
|
|||
FROM : 'FROM';
|
||||
FILTER : 'FILTER';
|
||||
WILDCARD : '*';
|
||||
DOT : '.';
|
||||
|
||||
CLAUSE_SAME : 'SAME';
|
||||
CLAUSE_DISTINCT : 'DISTINCT';
|
||||
|
|
Binary file not shown.
Binary file not shown.
|
@ -1,4 +1,4 @@
|
|||
// Code generated from Query.g4 by ANTLR 4.13.0. DO NOT EDIT.
|
||||
// Code generated from /repo/frostfs/sdk-go/netmap/parser/Query.g4 by ANTLR 4.13.0. DO NOT EDIT.
|
||||
|
||||
package parser // Query
|
||||
|
||||
|
@ -16,6 +16,10 @@ func (v *BaseQueryVisitor) VisitSelectFilterExpr(ctx *SelectFilterExprContext) i
|
|||
return v.VisitChildren(ctx)
|
||||
}
|
||||
|
||||
func (v *BaseQueryVisitor) VisitEcStmt(ctx *EcStmtContext) interface{} {
|
||||
return v.VisitChildren(ctx)
|
||||
}
|
||||
|
||||
func (v *BaseQueryVisitor) VisitRepStmt(ctx *RepStmtContext) interface{} {
|
||||
return v.VisitChildren(ctx)
|
||||
}
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
// Code generated from QueryLexer.g4 by ANTLR 4.13.0. DO NOT EDIT.
|
||||
// Code generated from /repo/frostfs/sdk-go/netmap/parser/QueryLexer.g4 by ANTLR 4.13.0. DO NOT EDIT.
|
||||
|
||||
package parser
|
||||
|
||||
|
@ -43,119 +43,123 @@ func querylexerLexerInit() {
|
|||
"DEFAULT_MODE",
|
||||
}
|
||||
staticData.LiteralNames = []string{
|
||||
"", "'NOT'", "'AND'", "'OR'", "", "'UNIQUE'", "'REP'", "'IN'", "'AS'",
|
||||
"'CBF'", "'SELECT'", "'FROM'", "'FILTER'", "'*'", "'SAME'", "'DISTINCT'",
|
||||
"'('", "')'", "'@'", "", "", "'0'",
|
||||
"", "'NOT'", "'AND'", "'OR'", "", "'UNIQUE'", "'REP'", "'EC'", "'IN'",
|
||||
"'AS'", "'CBF'", "'SELECT'", "'FROM'", "'FILTER'", "'*'", "'.'", "'SAME'",
|
||||
"'DISTINCT'", "'('", "')'", "'@'", "", "", "'0'",
|
||||
}
|
||||
staticData.SymbolicNames = []string{
|
||||
"", "NOT_OP", "AND_OP", "OR_OP", "SIMPLE_OP", "UNIQUE", "REP", "IN",
|
||||
"AS", "CBF", "SELECT", "FROM", "FILTER", "WILDCARD", "CLAUSE_SAME",
|
||||
"", "NOT_OP", "AND_OP", "OR_OP", "SIMPLE_OP", "UNIQUE", "REP", "EC",
|
||||
"IN", "AS", "CBF", "SELECT", "FROM", "FILTER", "WILDCARD", "DOT", "CLAUSE_SAME",
|
||||
"CLAUSE_DISTINCT", "L_PAREN", "R_PAREN", "AT", "IDENT", "NUMBER1", "ZERO",
|
||||
"STRING", "WS",
|
||||
}
|
||||
staticData.RuleNames = []string{
|
||||
"NOT_OP", "AND_OP", "OR_OP", "SIMPLE_OP", "UNIQUE", "REP", "IN", "AS",
|
||||
"CBF", "SELECT", "FROM", "FILTER", "WILDCARD", "CLAUSE_SAME", "CLAUSE_DISTINCT",
|
||||
"L_PAREN", "R_PAREN", "AT", "IDENT", "Digit", "Nondigit", "NUMBER1",
|
||||
"ZERO", "STRING", "ESC", "UNICODE", "HEX", "SAFECODEPOINTSINGLE", "SAFECODEPOINTDOUBLE",
|
||||
"WS",
|
||||
"NOT_OP", "AND_OP", "OR_OP", "SIMPLE_OP", "UNIQUE", "REP", "EC", "IN",
|
||||
"AS", "CBF", "SELECT", "FROM", "FILTER", "WILDCARD", "DOT", "CLAUSE_SAME",
|
||||
"CLAUSE_DISTINCT", "L_PAREN", "R_PAREN", "AT", "IDENT", "Digit", "Nondigit",
|
||||
"NUMBER1", "ZERO", "STRING", "ESC", "UNICODE", "HEX", "SAFECODEPOINTSINGLE",
|
||||
"SAFECODEPOINTDOUBLE", "WS",
|
||||
}
|
||||
staticData.PredictionContextCache = antlr.NewPredictionContextCache()
|
||||
staticData.serializedATN = []int32{
|
||||
4, 0, 23, 213, 6, -1, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2,
|
||||
4, 0, 25, 222, 6, -1, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2,
|
||||
4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2,
|
||||
10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15,
|
||||
7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7,
|
||||
20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25,
|
||||
2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 1, 0, 1, 0, 1,
|
||||
0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 2, 1, 3, 1, 3, 1, 3, 1,
|
||||
3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 3, 3, 85, 8, 3, 1, 4,
|
||||
1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 6, 1, 6,
|
||||
1, 6, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 9, 1, 9, 1, 9, 1, 9,
|
||||
1, 9, 1, 9, 1, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 11, 1, 11, 1, 11,
|
||||
1, 11, 1, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 1, 13, 1,
|
||||
13, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 15,
|
||||
1, 15, 1, 16, 1, 16, 1, 17, 1, 17, 1, 18, 1, 18, 1, 18, 5, 18, 152, 8,
|
||||
18, 10, 18, 12, 18, 155, 9, 18, 1, 19, 1, 19, 1, 20, 1, 20, 1, 21, 1, 21,
|
||||
5, 21, 163, 8, 21, 10, 21, 12, 21, 166, 9, 21, 1, 22, 1, 22, 1, 23, 1,
|
||||
23, 1, 23, 5, 23, 173, 8, 23, 10, 23, 12, 23, 176, 9, 23, 1, 23, 1, 23,
|
||||
1, 23, 1, 23, 5, 23, 182, 8, 23, 10, 23, 12, 23, 185, 9, 23, 1, 23, 3,
|
||||
23, 188, 8, 23, 1, 24, 1, 24, 1, 24, 3, 24, 193, 8, 24, 1, 25, 1, 25, 1,
|
||||
25, 1, 25, 1, 25, 1, 25, 1, 26, 1, 26, 1, 27, 1, 27, 1, 28, 1, 28, 1, 29,
|
||||
4, 29, 208, 8, 29, 11, 29, 12, 29, 209, 1, 29, 1, 29, 0, 0, 30, 1, 1, 3,
|
||||
2, 5, 3, 7, 4, 9, 5, 11, 6, 13, 7, 15, 8, 17, 9, 19, 10, 21, 11, 23, 12,
|
||||
25, 13, 27, 14, 29, 15, 31, 16, 33, 17, 35, 18, 37, 19, 39, 0, 41, 0, 43,
|
||||
20, 45, 21, 47, 22, 49, 0, 51, 0, 53, 0, 55, 0, 57, 0, 59, 23, 1, 0, 8,
|
||||
1, 0, 48, 57, 3, 0, 65, 90, 95, 95, 97, 122, 1, 0, 49, 57, 9, 0, 34, 34,
|
||||
39, 39, 47, 47, 92, 92, 98, 98, 102, 102, 110, 110, 114, 114, 116, 116,
|
||||
3, 0, 48, 57, 65, 70, 97, 102, 3, 0, 0, 31, 39, 39, 92, 92, 3, 0, 0, 31,
|
||||
34, 34, 92, 92, 3, 0, 9, 10, 13, 13, 32, 32, 220, 0, 1, 1, 0, 0, 0, 0,
|
||||
3, 1, 0, 0, 0, 0, 5, 1, 0, 0, 0, 0, 7, 1, 0, 0, 0, 0, 9, 1, 0, 0, 0, 0,
|
||||
11, 1, 0, 0, 0, 0, 13, 1, 0, 0, 0, 0, 15, 1, 0, 0, 0, 0, 17, 1, 0, 0, 0,
|
||||
0, 19, 1, 0, 0, 0, 0, 21, 1, 0, 0, 0, 0, 23, 1, 0, 0, 0, 0, 25, 1, 0, 0,
|
||||
0, 0, 27, 1, 0, 0, 0, 0, 29, 1, 0, 0, 0, 0, 31, 1, 0, 0, 0, 0, 33, 1, 0,
|
||||
0, 0, 0, 35, 1, 0, 0, 0, 0, 37, 1, 0, 0, 0, 0, 43, 1, 0, 0, 0, 0, 45, 1,
|
||||
0, 0, 0, 0, 47, 1, 0, 0, 0, 0, 59, 1, 0, 0, 0, 1, 61, 1, 0, 0, 0, 3, 65,
|
||||
1, 0, 0, 0, 5, 69, 1, 0, 0, 0, 7, 84, 1, 0, 0, 0, 9, 86, 1, 0, 0, 0, 11,
|
||||
93, 1, 0, 0, 0, 13, 97, 1, 0, 0, 0, 15, 100, 1, 0, 0, 0, 17, 103, 1, 0,
|
||||
0, 0, 19, 107, 1, 0, 0, 0, 21, 114, 1, 0, 0, 0, 23, 119, 1, 0, 0, 0, 25,
|
||||
126, 1, 0, 0, 0, 27, 128, 1, 0, 0, 0, 29, 133, 1, 0, 0, 0, 31, 142, 1,
|
||||
0, 0, 0, 33, 144, 1, 0, 0, 0, 35, 146, 1, 0, 0, 0, 37, 148, 1, 0, 0, 0,
|
||||
39, 156, 1, 0, 0, 0, 41, 158, 1, 0, 0, 0, 43, 160, 1, 0, 0, 0, 45, 167,
|
||||
1, 0, 0, 0, 47, 187, 1, 0, 0, 0, 49, 189, 1, 0, 0, 0, 51, 194, 1, 0, 0,
|
||||
0, 53, 200, 1, 0, 0, 0, 55, 202, 1, 0, 0, 0, 57, 204, 1, 0, 0, 0, 59, 207,
|
||||
1, 0, 0, 0, 61, 62, 5, 78, 0, 0, 62, 63, 5, 79, 0, 0, 63, 64, 5, 84, 0,
|
||||
0, 64, 2, 1, 0, 0, 0, 65, 66, 5, 65, 0, 0, 66, 67, 5, 78, 0, 0, 67, 68,
|
||||
5, 68, 0, 0, 68, 4, 1, 0, 0, 0, 69, 70, 5, 79, 0, 0, 70, 71, 5, 82, 0,
|
||||
0, 71, 6, 1, 0, 0, 0, 72, 73, 5, 69, 0, 0, 73, 85, 5, 81, 0, 0, 74, 75,
|
||||
5, 78, 0, 0, 75, 85, 5, 69, 0, 0, 76, 77, 5, 71, 0, 0, 77, 85, 5, 69, 0,
|
||||
0, 78, 79, 5, 71, 0, 0, 79, 85, 5, 84, 0, 0, 80, 81, 5, 76, 0, 0, 81, 85,
|
||||
5, 84, 0, 0, 82, 83, 5, 76, 0, 0, 83, 85, 5, 69, 0, 0, 84, 72, 1, 0, 0,
|
||||
0, 84, 74, 1, 0, 0, 0, 84, 76, 1, 0, 0, 0, 84, 78, 1, 0, 0, 0, 84, 80,
|
||||
1, 0, 0, 0, 84, 82, 1, 0, 0, 0, 85, 8, 1, 0, 0, 0, 86, 87, 5, 85, 0, 0,
|
||||
87, 88, 5, 78, 0, 0, 88, 89, 5, 73, 0, 0, 89, 90, 5, 81, 0, 0, 90, 91,
|
||||
5, 85, 0, 0, 91, 92, 5, 69, 0, 0, 92, 10, 1, 0, 0, 0, 93, 94, 5, 82, 0,
|
||||
0, 94, 95, 5, 69, 0, 0, 95, 96, 5, 80, 0, 0, 96, 12, 1, 0, 0, 0, 97, 98,
|
||||
5, 73, 0, 0, 98, 99, 5, 78, 0, 0, 99, 14, 1, 0, 0, 0, 100, 101, 5, 65,
|
||||
0, 0, 101, 102, 5, 83, 0, 0, 102, 16, 1, 0, 0, 0, 103, 104, 5, 67, 0, 0,
|
||||
104, 105, 5, 66, 0, 0, 105, 106, 5, 70, 0, 0, 106, 18, 1, 0, 0, 0, 107,
|
||||
108, 5, 83, 0, 0, 108, 109, 5, 69, 0, 0, 109, 110, 5, 76, 0, 0, 110, 111,
|
||||
5, 69, 0, 0, 111, 112, 5, 67, 0, 0, 112, 113, 5, 84, 0, 0, 113, 20, 1,
|
||||
0, 0, 0, 114, 115, 5, 70, 0, 0, 115, 116, 5, 82, 0, 0, 116, 117, 5, 79,
|
||||
0, 0, 117, 118, 5, 77, 0, 0, 118, 22, 1, 0, 0, 0, 119, 120, 5, 70, 0, 0,
|
||||
120, 121, 5, 73, 0, 0, 121, 122, 5, 76, 0, 0, 122, 123, 5, 84, 0, 0, 123,
|
||||
124, 5, 69, 0, 0, 124, 125, 5, 82, 0, 0, 125, 24, 1, 0, 0, 0, 126, 127,
|
||||
5, 42, 0, 0, 127, 26, 1, 0, 0, 0, 128, 129, 5, 83, 0, 0, 129, 130, 5, 65,
|
||||
0, 0, 130, 131, 5, 77, 0, 0, 131, 132, 5, 69, 0, 0, 132, 28, 1, 0, 0, 0,
|
||||
133, 134, 5, 68, 0, 0, 134, 135, 5, 73, 0, 0, 135, 136, 5, 83, 0, 0, 136,
|
||||
137, 5, 84, 0, 0, 137, 138, 5, 73, 0, 0, 138, 139, 5, 78, 0, 0, 139, 140,
|
||||
5, 67, 0, 0, 140, 141, 5, 84, 0, 0, 141, 30, 1, 0, 0, 0, 142, 143, 5, 40,
|
||||
0, 0, 143, 32, 1, 0, 0, 0, 144, 145, 5, 41, 0, 0, 145, 34, 1, 0, 0, 0,
|
||||
146, 147, 5, 64, 0, 0, 147, 36, 1, 0, 0, 0, 148, 153, 3, 41, 20, 0, 149,
|
||||
152, 3, 39, 19, 0, 150, 152, 3, 41, 20, 0, 151, 149, 1, 0, 0, 0, 151, 150,
|
||||
1, 0, 0, 0, 152, 155, 1, 0, 0, 0, 153, 151, 1, 0, 0, 0, 153, 154, 1, 0,
|
||||
0, 0, 154, 38, 1, 0, 0, 0, 155, 153, 1, 0, 0, 0, 156, 157, 7, 0, 0, 0,
|
||||
157, 40, 1, 0, 0, 0, 158, 159, 7, 1, 0, 0, 159, 42, 1, 0, 0, 0, 160, 164,
|
||||
7, 2, 0, 0, 161, 163, 3, 39, 19, 0, 162, 161, 1, 0, 0, 0, 163, 166, 1,
|
||||
0, 0, 0, 164, 162, 1, 0, 0, 0, 164, 165, 1, 0, 0, 0, 165, 44, 1, 0, 0,
|
||||
0, 166, 164, 1, 0, 0, 0, 167, 168, 5, 48, 0, 0, 168, 46, 1, 0, 0, 0, 169,
|
||||
174, 5, 34, 0, 0, 170, 173, 3, 49, 24, 0, 171, 173, 3, 57, 28, 0, 172,
|
||||
170, 1, 0, 0, 0, 172, 171, 1, 0, 0, 0, 173, 176, 1, 0, 0, 0, 174, 172,
|
||||
1, 0, 0, 0, 174, 175, 1, 0, 0, 0, 175, 177, 1, 0, 0, 0, 176, 174, 1, 0,
|
||||
0, 0, 177, 188, 5, 34, 0, 0, 178, 183, 5, 39, 0, 0, 179, 182, 3, 49, 24,
|
||||
0, 180, 182, 3, 55, 27, 0, 181, 179, 1, 0, 0, 0, 181, 180, 1, 0, 0, 0,
|
||||
182, 185, 1, 0, 0, 0, 183, 181, 1, 0, 0, 0, 183, 184, 1, 0, 0, 0, 184,
|
||||
186, 1, 0, 0, 0, 185, 183, 1, 0, 0, 0, 186, 188, 5, 39, 0, 0, 187, 169,
|
||||
1, 0, 0, 0, 187, 178, 1, 0, 0, 0, 188, 48, 1, 0, 0, 0, 189, 192, 5, 92,
|
||||
0, 0, 190, 193, 7, 3, 0, 0, 191, 193, 3, 51, 25, 0, 192, 190, 1, 0, 0,
|
||||
0, 192, 191, 1, 0, 0, 0, 193, 50, 1, 0, 0, 0, 194, 195, 5, 117, 0, 0, 195,
|
||||
196, 3, 53, 26, 0, 196, 197, 3, 53, 26, 0, 197, 198, 3, 53, 26, 0, 198,
|
||||
199, 3, 53, 26, 0, 199, 52, 1, 0, 0, 0, 200, 201, 7, 4, 0, 0, 201, 54,
|
||||
1, 0, 0, 0, 202, 203, 8, 5, 0, 0, 203, 56, 1, 0, 0, 0, 204, 205, 8, 6,
|
||||
0, 0, 205, 58, 1, 0, 0, 0, 206, 208, 7, 7, 0, 0, 207, 206, 1, 0, 0, 0,
|
||||
208, 209, 1, 0, 0, 0, 209, 207, 1, 0, 0, 0, 209, 210, 1, 0, 0, 0, 210,
|
||||
211, 1, 0, 0, 0, 211, 212, 6, 29, 0, 0, 212, 60, 1, 0, 0, 0, 12, 0, 84,
|
||||
151, 153, 164, 172, 174, 181, 183, 187, 192, 209, 1, 6, 0, 0,
|
||||
2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2,
|
||||
31, 7, 31, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2,
|
||||
1, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3,
|
||||
1, 3, 3, 3, 89, 8, 3, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 5, 1,
|
||||
5, 1, 5, 1, 5, 1, 6, 1, 6, 1, 6, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1,
|
||||
9, 1, 9, 1, 9, 1, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1,
|
||||
11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12,
|
||||
1, 12, 1, 13, 1, 13, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1,
|
||||
16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 17, 1, 17,
|
||||
1, 18, 1, 18, 1, 19, 1, 19, 1, 20, 1, 20, 1, 20, 5, 20, 161, 8, 20, 10,
|
||||
20, 12, 20, 164, 9, 20, 1, 21, 1, 21, 1, 22, 1, 22, 1, 23, 1, 23, 5, 23,
|
||||
172, 8, 23, 10, 23, 12, 23, 175, 9, 23, 1, 24, 1, 24, 1, 25, 1, 25, 1,
|
||||
25, 5, 25, 182, 8, 25, 10, 25, 12, 25, 185, 9, 25, 1, 25, 1, 25, 1, 25,
|
||||
1, 25, 5, 25, 191, 8, 25, 10, 25, 12, 25, 194, 9, 25, 1, 25, 3, 25, 197,
|
||||
8, 25, 1, 26, 1, 26, 1, 26, 3, 26, 202, 8, 26, 1, 27, 1, 27, 1, 27, 1,
|
||||
27, 1, 27, 1, 27, 1, 28, 1, 28, 1, 29, 1, 29, 1, 30, 1, 30, 1, 31, 4, 31,
|
||||
217, 8, 31, 11, 31, 12, 31, 218, 1, 31, 1, 31, 0, 0, 32, 1, 1, 3, 2, 5,
|
||||
3, 7, 4, 9, 5, 11, 6, 13, 7, 15, 8, 17, 9, 19, 10, 21, 11, 23, 12, 25,
|
||||
13, 27, 14, 29, 15, 31, 16, 33, 17, 35, 18, 37, 19, 39, 20, 41, 21, 43,
|
||||
0, 45, 0, 47, 22, 49, 23, 51, 24, 53, 0, 55, 0, 57, 0, 59, 0, 61, 0, 63,
|
||||
25, 1, 0, 8, 1, 0, 48, 57, 3, 0, 65, 90, 95, 95, 97, 122, 1, 0, 49, 57,
|
||||
9, 0, 34, 34, 39, 39, 47, 47, 92, 92, 98, 98, 102, 102, 110, 110, 114,
|
||||
114, 116, 116, 3, 0, 48, 57, 65, 70, 97, 102, 3, 0, 0, 31, 39, 39, 92,
|
||||
92, 3, 0, 0, 31, 34, 34, 92, 92, 3, 0, 9, 10, 13, 13, 32, 32, 229, 0, 1,
|
||||
1, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 5, 1, 0, 0, 0, 0, 7, 1, 0, 0, 0, 0, 9,
|
||||
1, 0, 0, 0, 0, 11, 1, 0, 0, 0, 0, 13, 1, 0, 0, 0, 0, 15, 1, 0, 0, 0, 0,
|
||||
17, 1, 0, 0, 0, 0, 19, 1, 0, 0, 0, 0, 21, 1, 0, 0, 0, 0, 23, 1, 0, 0, 0,
|
||||
0, 25, 1, 0, 0, 0, 0, 27, 1, 0, 0, 0, 0, 29, 1, 0, 0, 0, 0, 31, 1, 0, 0,
|
||||
0, 0, 33, 1, 0, 0, 0, 0, 35, 1, 0, 0, 0, 0, 37, 1, 0, 0, 0, 0, 39, 1, 0,
|
||||
0, 0, 0, 41, 1, 0, 0, 0, 0, 47, 1, 0, 0, 0, 0, 49, 1, 0, 0, 0, 0, 51, 1,
|
||||
0, 0, 0, 0, 63, 1, 0, 0, 0, 1, 65, 1, 0, 0, 0, 3, 69, 1, 0, 0, 0, 5, 73,
|
||||
1, 0, 0, 0, 7, 88, 1, 0, 0, 0, 9, 90, 1, 0, 0, 0, 11, 97, 1, 0, 0, 0, 13,
|
||||
101, 1, 0, 0, 0, 15, 104, 1, 0, 0, 0, 17, 107, 1, 0, 0, 0, 19, 110, 1,
|
||||
0, 0, 0, 21, 114, 1, 0, 0, 0, 23, 121, 1, 0, 0, 0, 25, 126, 1, 0, 0, 0,
|
||||
27, 133, 1, 0, 0, 0, 29, 135, 1, 0, 0, 0, 31, 137, 1, 0, 0, 0, 33, 142,
|
||||
1, 0, 0, 0, 35, 151, 1, 0, 0, 0, 37, 153, 1, 0, 0, 0, 39, 155, 1, 0, 0,
|
||||
0, 41, 157, 1, 0, 0, 0, 43, 165, 1, 0, 0, 0, 45, 167, 1, 0, 0, 0, 47, 169,
|
||||
1, 0, 0, 0, 49, 176, 1, 0, 0, 0, 51, 196, 1, 0, 0, 0, 53, 198, 1, 0, 0,
|
||||
0, 55, 203, 1, 0, 0, 0, 57, 209, 1, 0, 0, 0, 59, 211, 1, 0, 0, 0, 61, 213,
|
||||
1, 0, 0, 0, 63, 216, 1, 0, 0, 0, 65, 66, 5, 78, 0, 0, 66, 67, 5, 79, 0,
|
||||
0, 67, 68, 5, 84, 0, 0, 68, 2, 1, 0, 0, 0, 69, 70, 5, 65, 0, 0, 70, 71,
|
||||
5, 78, 0, 0, 71, 72, 5, 68, 0, 0, 72, 4, 1, 0, 0, 0, 73, 74, 5, 79, 0,
|
||||
0, 74, 75, 5, 82, 0, 0, 75, 6, 1, 0, 0, 0, 76, 77, 5, 69, 0, 0, 77, 89,
|
||||
5, 81, 0, 0, 78, 79, 5, 78, 0, 0, 79, 89, 5, 69, 0, 0, 80, 81, 5, 71, 0,
|
||||
0, 81, 89, 5, 69, 0, 0, 82, 83, 5, 71, 0, 0, 83, 89, 5, 84, 0, 0, 84, 85,
|
||||
5, 76, 0, 0, 85, 89, 5, 84, 0, 0, 86, 87, 5, 76, 0, 0, 87, 89, 5, 69, 0,
|
||||
0, 88, 76, 1, 0, 0, 0, 88, 78, 1, 0, 0, 0, 88, 80, 1, 0, 0, 0, 88, 82,
|
||||
1, 0, 0, 0, 88, 84, 1, 0, 0, 0, 88, 86, 1, 0, 0, 0, 89, 8, 1, 0, 0, 0,
|
||||
90, 91, 5, 85, 0, 0, 91, 92, 5, 78, 0, 0, 92, 93, 5, 73, 0, 0, 93, 94,
|
||||
5, 81, 0, 0, 94, 95, 5, 85, 0, 0, 95, 96, 5, 69, 0, 0, 96, 10, 1, 0, 0,
|
||||
0, 97, 98, 5, 82, 0, 0, 98, 99, 5, 69, 0, 0, 99, 100, 5, 80, 0, 0, 100,
|
||||
12, 1, 0, 0, 0, 101, 102, 5, 69, 0, 0, 102, 103, 5, 67, 0, 0, 103, 14,
|
||||
1, 0, 0, 0, 104, 105, 5, 73, 0, 0, 105, 106, 5, 78, 0, 0, 106, 16, 1, 0,
|
||||
0, 0, 107, 108, 5, 65, 0, 0, 108, 109, 5, 83, 0, 0, 109, 18, 1, 0, 0, 0,
|
||||
110, 111, 5, 67, 0, 0, 111, 112, 5, 66, 0, 0, 112, 113, 5, 70, 0, 0, 113,
|
||||
20, 1, 0, 0, 0, 114, 115, 5, 83, 0, 0, 115, 116, 5, 69, 0, 0, 116, 117,
|
||||
5, 76, 0, 0, 117, 118, 5, 69, 0, 0, 118, 119, 5, 67, 0, 0, 119, 120, 5,
|
||||
84, 0, 0, 120, 22, 1, 0, 0, 0, 121, 122, 5, 70, 0, 0, 122, 123, 5, 82,
|
||||
0, 0, 123, 124, 5, 79, 0, 0, 124, 125, 5, 77, 0, 0, 125, 24, 1, 0, 0, 0,
|
||||
126, 127, 5, 70, 0, 0, 127, 128, 5, 73, 0, 0, 128, 129, 5, 76, 0, 0, 129,
|
||||
130, 5, 84, 0, 0, 130, 131, 5, 69, 0, 0, 131, 132, 5, 82, 0, 0, 132, 26,
|
||||
1, 0, 0, 0, 133, 134, 5, 42, 0, 0, 134, 28, 1, 0, 0, 0, 135, 136, 5, 46,
|
||||
0, 0, 136, 30, 1, 0, 0, 0, 137, 138, 5, 83, 0, 0, 138, 139, 5, 65, 0, 0,
|
||||
139, 140, 5, 77, 0, 0, 140, 141, 5, 69, 0, 0, 141, 32, 1, 0, 0, 0, 142,
|
||||
143, 5, 68, 0, 0, 143, 144, 5, 73, 0, 0, 144, 145, 5, 83, 0, 0, 145, 146,
|
||||
5, 84, 0, 0, 146, 147, 5, 73, 0, 0, 147, 148, 5, 78, 0, 0, 148, 149, 5,
|
||||
67, 0, 0, 149, 150, 5, 84, 0, 0, 150, 34, 1, 0, 0, 0, 151, 152, 5, 40,
|
||||
0, 0, 152, 36, 1, 0, 0, 0, 153, 154, 5, 41, 0, 0, 154, 38, 1, 0, 0, 0,
|
||||
155, 156, 5, 64, 0, 0, 156, 40, 1, 0, 0, 0, 157, 162, 3, 45, 22, 0, 158,
|
||||
161, 3, 43, 21, 0, 159, 161, 3, 45, 22, 0, 160, 158, 1, 0, 0, 0, 160, 159,
|
||||
1, 0, 0, 0, 161, 164, 1, 0, 0, 0, 162, 160, 1, 0, 0, 0, 162, 163, 1, 0,
|
||||
0, 0, 163, 42, 1, 0, 0, 0, 164, 162, 1, 0, 0, 0, 165, 166, 7, 0, 0, 0,
|
||||
166, 44, 1, 0, 0, 0, 167, 168, 7, 1, 0, 0, 168, 46, 1, 0, 0, 0, 169, 173,
|
||||
7, 2, 0, 0, 170, 172, 3, 43, 21, 0, 171, 170, 1, 0, 0, 0, 172, 175, 1,
|
||||
0, 0, 0, 173, 171, 1, 0, 0, 0, 173, 174, 1, 0, 0, 0, 174, 48, 1, 0, 0,
|
||||
0, 175, 173, 1, 0, 0, 0, 176, 177, 5, 48, 0, 0, 177, 50, 1, 0, 0, 0, 178,
|
||||
183, 5, 34, 0, 0, 179, 182, 3, 53, 26, 0, 180, 182, 3, 61, 30, 0, 181,
|
||||
179, 1, 0, 0, 0, 181, 180, 1, 0, 0, 0, 182, 185, 1, 0, 0, 0, 183, 181,
|
||||
1, 0, 0, 0, 183, 184, 1, 0, 0, 0, 184, 186, 1, 0, 0, 0, 185, 183, 1, 0,
|
||||
0, 0, 186, 197, 5, 34, 0, 0, 187, 192, 5, 39, 0, 0, 188, 191, 3, 53, 26,
|
||||
0, 189, 191, 3, 59, 29, 0, 190, 188, 1, 0, 0, 0, 190, 189, 1, 0, 0, 0,
|
||||
191, 194, 1, 0, 0, 0, 192, 190, 1, 0, 0, 0, 192, 193, 1, 0, 0, 0, 193,
|
||||
195, 1, 0, 0, 0, 194, 192, 1, 0, 0, 0, 195, 197, 5, 39, 0, 0, 196, 178,
|
||||
1, 0, 0, 0, 196, 187, 1, 0, 0, 0, 197, 52, 1, 0, 0, 0, 198, 201, 5, 92,
|
||||
0, 0, 199, 202, 7, 3, 0, 0, 200, 202, 3, 55, 27, 0, 201, 199, 1, 0, 0,
|
||||
0, 201, 200, 1, 0, 0, 0, 202, 54, 1, 0, 0, 0, 203, 204, 5, 117, 0, 0, 204,
|
||||
205, 3, 57, 28, 0, 205, 206, 3, 57, 28, 0, 206, 207, 3, 57, 28, 0, 207,
|
||||
208, 3, 57, 28, 0, 208, 56, 1, 0, 0, 0, 209, 210, 7, 4, 0, 0, 210, 58,
|
||||
1, 0, 0, 0, 211, 212, 8, 5, 0, 0, 212, 60, 1, 0, 0, 0, 213, 214, 8, 6,
|
||||
0, 0, 214, 62, 1, 0, 0, 0, 215, 217, 7, 7, 0, 0, 216, 215, 1, 0, 0, 0,
|
||||
217, 218, 1, 0, 0, 0, 218, 216, 1, 0, 0, 0, 218, 219, 1, 0, 0, 0, 219,
|
||||
220, 1, 0, 0, 0, 220, 221, 6, 31, 0, 0, 221, 64, 1, 0, 0, 0, 12, 0, 88,
|
||||
160, 162, 173, 181, 183, 190, 192, 196, 201, 218, 1, 6, 0, 0,
|
||||
}
|
||||
deserializer := antlr.NewATNDeserializer(nil)
|
||||
staticData.atn = deserializer.Deserialize(staticData.serializedATN)
|
||||
|
@ -202,21 +206,23 @@ const (
|
|||
QueryLexerSIMPLE_OP = 4
|
||||
QueryLexerUNIQUE = 5
|
||||
QueryLexerREP = 6
|
||||
QueryLexerIN = 7
|
||||
QueryLexerAS = 8
|
||||
QueryLexerCBF = 9
|
||||
QueryLexerSELECT = 10
|
||||
QueryLexerFROM = 11
|
||||
QueryLexerFILTER = 12
|
||||
QueryLexerWILDCARD = 13
|
||||
QueryLexerCLAUSE_SAME = 14
|
||||
QueryLexerCLAUSE_DISTINCT = 15
|
||||
QueryLexerL_PAREN = 16
|
||||
QueryLexerR_PAREN = 17
|
||||
QueryLexerAT = 18
|
||||
QueryLexerIDENT = 19
|
||||
QueryLexerNUMBER1 = 20
|
||||
QueryLexerZERO = 21
|
||||
QueryLexerSTRING = 22
|
||||
QueryLexerWS = 23
|
||||
QueryLexerEC = 7
|
||||
QueryLexerIN = 8
|
||||
QueryLexerAS = 9
|
||||
QueryLexerCBF = 10
|
||||
QueryLexerSELECT = 11
|
||||
QueryLexerFROM = 12
|
||||
QueryLexerFILTER = 13
|
||||
QueryLexerWILDCARD = 14
|
||||
QueryLexerDOT = 15
|
||||
QueryLexerCLAUSE_SAME = 16
|
||||
QueryLexerCLAUSE_DISTINCT = 17
|
||||
QueryLexerL_PAREN = 18
|
||||
QueryLexerR_PAREN = 19
|
||||
QueryLexerAT = 20
|
||||
QueryLexerIDENT = 21
|
||||
QueryLexerNUMBER1 = 22
|
||||
QueryLexerZERO = 23
|
||||
QueryLexerSTRING = 24
|
||||
QueryLexerWS = 25
|
||||
)
|
||||
|
|
File diff suppressed because it is too large
Load diff
|
@ -1,4 +1,4 @@
|
|||
// Code generated from Query.g4 by ANTLR 4.13.0. DO NOT EDIT.
|
||||
// Code generated from /repo/frostfs/sdk-go/netmap/parser/Query.g4 by ANTLR 4.13.0. DO NOT EDIT.
|
||||
|
||||
package parser // Query
|
||||
|
||||
|
@ -14,6 +14,9 @@ type QueryVisitor interface {
|
|||
// Visit a parse tree produced by Query#selectFilterExpr.
|
||||
VisitSelectFilterExpr(ctx *SelectFilterExprContext) interface{}
|
||||
|
||||
// Visit a parse tree produced by Query#ecStmt.
|
||||
VisitEcStmt(ctx *EcStmtContext) interface{}
|
||||
|
||||
// Visit a parse tree produced by Query#repStmt.
|
||||
VisitRepStmt(ctx *RepStmtContext) interface{}
|
||||
|
||||
|
|
136
netmap/policy.go
136
netmap/policy.go
|
@ -34,9 +34,18 @@ type PlacementPolicy struct {
|
|||
|
||||
func (p *PlacementPolicy) readFromV2(m netmap.PlacementPolicy, checkFieldPresence bool) error {
|
||||
p.replicas = m.GetReplicas()
|
||||
if checkFieldPresence && len(p.replicas) == 0 {
|
||||
if checkFieldPresence {
|
||||
if len(p.replicas) == 0 {
|
||||
return errors.New("missing replicas")
|
||||
}
|
||||
if len(p.replicas) != 1 {
|
||||
for i := range p.replicas {
|
||||
if p.replicas[i].GetECDataCount() != 0 || p.replicas[i].GetECParityCount() != 0 {
|
||||
return errors.New("erasure code group must be used exclusively")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
p.backupFactor = m.GetContainerBackupFactor()
|
||||
p.selectors = m.GetSelectors()
|
||||
|
@ -130,6 +139,14 @@ func (r *ReplicaDescriptor) SetNumberOfObjects(c uint32) {
|
|||
r.m.SetCount(c)
|
||||
}
|
||||
|
||||
func (r ReplicaDescriptor) SetECDataCount(v uint32) {
|
||||
r.m.SetECDataCount(v)
|
||||
}
|
||||
|
||||
func (r ReplicaDescriptor) SetECParityCount(v uint32) {
|
||||
r.m.SetECParityCount(v)
|
||||
}
|
||||
|
||||
// NumberOfObjects returns number set using SetNumberOfObjects.
|
||||
//
|
||||
// Zero ReplicaDescriptor has zero number of objects.
|
||||
|
@ -137,6 +154,19 @@ func (r ReplicaDescriptor) NumberOfObjects() uint32 {
|
|||
return r.m.GetCount()
|
||||
}
|
||||
|
||||
func (r ReplicaDescriptor) GetECDataCount() uint32 {
|
||||
return r.m.GetECDataCount()
|
||||
}
|
||||
|
||||
func (r ReplicaDescriptor) GetECParityCount() uint32 {
|
||||
return r.m.GetECParityCount()
|
||||
}
|
||||
|
||||
// TotalECPartCount returns total sum of ECDataCount and ECParityCount.
|
||||
func (r ReplicaDescriptor) TotalECPartCount() uint32 {
|
||||
return r.m.GetECDataCount() + r.m.GetECParityCount()
|
||||
}
|
||||
|
||||
// SetSelectorName sets name of the related Selector.
|
||||
//
|
||||
// Zero ReplicaDescriptor references to the root bucket's selector: it contains
|
||||
|
@ -170,10 +200,19 @@ func (p PlacementPolicy) NumberOfReplicas() int {
|
|||
// descriptor. Index MUST be in range [0; NumberOfReplicas()).
|
||||
//
|
||||
// Zero PlacementPolicy has no replicas.
|
||||
//
|
||||
// Deprecated: Use PlacementPolicy.ReplicaDescriptor(int).NumberOfObjects() instead.
|
||||
func (p PlacementPolicy) ReplicaNumberByIndex(i int) uint32 {
|
||||
return p.replicas[i].GetCount()
|
||||
}
|
||||
|
||||
// ReplicaDescriptor returns i-th replica descriptor. Index MUST be in range [0; NumberOfReplicas()).
|
||||
func (p PlacementPolicy) ReplicaDescriptor(i int) ReplicaDescriptor {
|
||||
return ReplicaDescriptor{
|
||||
m: p.replicas[i],
|
||||
}
|
||||
}
|
||||
|
||||
// SetContainerBackupFactor sets container backup factor: it controls how deep
|
||||
// FrostFS will search for nodes alternatives to include into container's nodes subset.
|
||||
//
|
||||
|
@ -217,6 +256,11 @@ func (s *Selector) SelectByBucketAttribute(bucket string) {
|
|||
s.m.SetAttribute(bucket)
|
||||
}
|
||||
|
||||
// SetClause sets the clause for the Selector.
|
||||
func (s *Selector) SetClause(clause netmap.Clause) {
|
||||
s.m.SetClause(clause)
|
||||
}
|
||||
|
||||
// SelectSame makes selection algorithm to select only nodes having the same values
|
||||
// of the bucket attribute.
|
||||
//
|
||||
|
@ -388,10 +432,14 @@ func (p PlacementPolicy) WriteStringTo(w io.StringWriter) (err error) {
|
|||
c := p.replicas[i].GetCount()
|
||||
s := p.replicas[i].GetSelector()
|
||||
|
||||
if s != "" {
|
||||
_, err = w.WriteString(fmt.Sprintf("%sREP %d IN %s", delim, c, s))
|
||||
} else {
|
||||
if c != 0 {
|
||||
_, err = w.WriteString(fmt.Sprintf("%sREP %d", delim, c))
|
||||
} else {
|
||||
ecx, ecy := p.replicas[i].GetECDataCount(), p.replicas[i].GetECParityCount()
|
||||
_, err = w.WriteString(fmt.Sprintf("%sEC %d.%d", delim, ecx, ecy))
|
||||
}
|
||||
if s != "" {
|
||||
_, err = w.WriteString(fmt.Sprintf(" IN %s", s))
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
|
@ -455,7 +503,7 @@ func (p PlacementPolicy) WriteStringTo(w io.StringWriter) (err error) {
|
|||
return err
|
||||
}
|
||||
|
||||
err = writeFilterStringTo(w, p.filters[i])
|
||||
err = writeFilterStringTo(w, p.filters[i], false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
@ -464,7 +512,7 @@ func (p PlacementPolicy) WriteStringTo(w io.StringWriter) (err error) {
|
|||
return nil
|
||||
}
|
||||
|
||||
func writeFilterStringTo(w io.StringWriter, f netmap.Filter) error {
|
||||
func writeFilterStringTo(w io.StringWriter, f netmap.Filter, mayNeedOuterBrackets bool) error {
|
||||
var err error
|
||||
var s string
|
||||
op := f.GetOp()
|
||||
|
@ -489,7 +537,7 @@ func writeFilterStringTo(w io.StringWriter, f netmap.Filter) error {
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = writeFilterStringTo(w, inner[0])
|
||||
err = writeFilterStringTo(w, inner[0], false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
@ -498,6 +546,13 @@ func writeFilterStringTo(w io.StringWriter, f netmap.Filter) error {
|
|||
return err
|
||||
}
|
||||
} else {
|
||||
useBrackets := mayNeedOuterBrackets && op == netmap.OR && len(inner) > 1
|
||||
if useBrackets {
|
||||
_, err = w.WriteString("(")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for i := range inner {
|
||||
if i != 0 {
|
||||
_, err = w.WriteString(" " + op.String() + " ")
|
||||
|
@ -505,7 +560,13 @@ func writeFilterStringTo(w io.StringWriter, f netmap.Filter) error {
|
|||
return err
|
||||
}
|
||||
}
|
||||
err = writeFilterStringTo(w, inner[i])
|
||||
err = writeFilterStringTo(w, inner[i], true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if useBrackets {
|
||||
_, err = w.WriteString(")")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
@ -612,6 +673,8 @@ var (
|
|||
"make sure to pair REP and SELECT clauses: \"REP .. IN X\" + \"SELECT ... AS X\"")
|
||||
// errRedundantSelector is returned for errors found by filters policy validator.
|
||||
errRedundantFilter = errors.New("policy: found redundant filter")
|
||||
// errECFewSelectors is returned when EC keyword is used without UNIQUE keyword.
|
||||
errECFewSelectors = errors.New("policy: too few nodes to select")
|
||||
)
|
||||
|
||||
type policyVisitor struct {
|
||||
|
@ -639,11 +702,18 @@ func (p *policyVisitor) VisitPolicy(ctx *parser.PolicyContext) any {
|
|||
|
||||
pl.unique = ctx.UNIQUE() != nil
|
||||
|
||||
repStmts := ctx.AllRepStmt()
|
||||
pl.replicas = make([]netmap.Replica, 0, len(repStmts))
|
||||
|
||||
for _, r := range repStmts {
|
||||
res, ok := r.Accept(p).(*netmap.Replica)
|
||||
stmts := ctx.GetChildren()
|
||||
for _, r := range stmts {
|
||||
var res *netmap.Replica
|
||||
var ok bool
|
||||
switch r := r.(type) {
|
||||
case parser.IRepStmtContext:
|
||||
res, ok = r.Accept(p).(*netmap.Replica)
|
||||
case parser.IEcStmtContext:
|
||||
res, ok = r.Accept(p).(*netmap.Replica)
|
||||
default:
|
||||
continue
|
||||
}
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
@ -740,6 +810,28 @@ func (p *policyVisitor) VisitRepStmt(ctx *parser.RepStmtContext) any {
|
|||
return rs
|
||||
}
|
||||
|
||||
// VisitRepStmt implements parser.QueryVisitor interface.
|
||||
func (p *policyVisitor) VisitEcStmt(ctx *parser.EcStmtContext) any {
|
||||
dataCount, err := strconv.ParseUint(ctx.GetData().GetText(), 10, 32)
|
||||
if err != nil {
|
||||
return p.reportError(errInvalidNumber)
|
||||
}
|
||||
parityCount, err := strconv.ParseUint(ctx.GetParity().GetText(), 10, 32)
|
||||
if err != nil {
|
||||
return p.reportError(errInvalidNumber)
|
||||
}
|
||||
|
||||
rs := new(netmap.Replica)
|
||||
rs.SetECDataCount(uint32(dataCount))
|
||||
rs.SetECParityCount(uint32(parityCount))
|
||||
|
||||
if sel := ctx.GetSelector(); sel != nil {
|
||||
rs.SetSelector(sel.GetText())
|
||||
}
|
||||
|
||||
return rs
|
||||
}
|
||||
|
||||
// VisitSelectStmt implements parser.QueryVisitor interface.
|
||||
func (p *policyVisitor) VisitSelectStmt(ctx *parser.SelectStmtContext) any {
|
||||
res, err := strconv.ParseUint(ctx.GetCount().GetText(), 10, 32)
|
||||
|
@ -892,6 +984,14 @@ func validatePolicy(p PlacementPolicy) error {
|
|||
if seenSelectors[selName] == nil {
|
||||
return fmt.Errorf("%w: '%s'", errUnknownSelector, selName)
|
||||
}
|
||||
|
||||
dataCount := p.replicas[i].GetECDataCount()
|
||||
parityCount := p.replicas[i].GetECParityCount()
|
||||
if dataCount != 0 || parityCount != 0 {
|
||||
if c := seenSelectors[selName].GetCount(); c < dataCount+parityCount {
|
||||
return fmt.Errorf("%w: %d < %d + %d", errECFewSelectors, c, dataCount, parityCount)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -922,10 +1022,14 @@ func operationFromString(s string) (op netmap.Operation) {
|
|||
return
|
||||
}
|
||||
|
||||
// escapeString returns single quote wrapped string if it contains special
|
||||
// characters '-' and whitespace.
|
||||
// escapeString returns single quote wrapped string.
|
||||
// Wrapping rules must be kept in sync with QueryLexer.g4.
|
||||
// Currently only ASCII letters, digits and underscore can be parsed without quotes.
|
||||
func escapeString(s string) string {
|
||||
if strings.ContainsAny(s, " -\t") {
|
||||
for _, r := range s {
|
||||
if 'a' <= r && r <= 'z' || 'A' <= r && r <= 'Z' || '0' <= r && r <= '9' || r == '_' {
|
||||
continue
|
||||
}
|
||||
return "'" + s + "'"
|
||||
}
|
||||
return s
|
||||
|
|
|
@ -36,9 +36,16 @@ FILTER City EQ SPB AND SSD EQ true OR City EQ SPB AND Rating GE 5 AS SPBSSD`,
|
|||
SELECT 1 IN City FROM SPBSSD AS SPB
|
||||
FILTER NOT (NOT (City EQ SPB) AND SSD EQ true OR City EQ SPB AND Rating GE 5) AS SPBSSD`,
|
||||
|
||||
`REP 1 IN FNODE
|
||||
CBF 1
|
||||
SELECT 1 FROM F AS FNODE
|
||||
FILTER Node EQ '10.78.8.11' AS F`,
|
||||
|
||||
`UNIQUE
|
||||
REP 1
|
||||
REP 1`,
|
||||
`EC 1.2 IN X
|
||||
SELECT 3 IN City FROM * AS X`,
|
||||
}
|
||||
|
||||
var p PlacementPolicy
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
package netmap_test
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
. "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
|
||||
|
@ -29,6 +30,79 @@ func TestPlacementPolicyEncoding(t *testing.T) {
|
|||
})
|
||||
}
|
||||
|
||||
func TestPlacementPolicyWriteString(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
input string
|
||||
output string // If the output is empty, make it equal to input.
|
||||
}{
|
||||
{
|
||||
name: "no compound operators",
|
||||
input: `REP 1
|
||||
CBF 1
|
||||
SELECT 1 FROM Color
|
||||
FILTER Color EQ Red AS Color`,
|
||||
},
|
||||
{
|
||||
name: "no brackets in single level same-operator chain",
|
||||
input: `REP 1
|
||||
CBF 1
|
||||
SELECT 1 FROM Color
|
||||
FILTER Color EQ Red OR Color EQ Blue OR Color EQ Green AS Color`,
|
||||
},
|
||||
{
|
||||
name: "no brackets aroung higher precedence op",
|
||||
input: `REP 1
|
||||
CBF 1
|
||||
SELECT 1 FROM Color
|
||||
FILTER Color EQ Red OR Color EQ Blue AND Color NE Green AS Color`,
|
||||
},
|
||||
{
|
||||
name: "no brackets aroung higher precedence op, even if present in the input",
|
||||
input: `REP 1
|
||||
CBF 1
|
||||
SELECT 1 FROM Color
|
||||
FILTER Color EQ Red OR (Color EQ Blue AND Color NE Green) AS Color`,
|
||||
output: `REP 1
|
||||
CBF 1
|
||||
SELECT 1 FROM Color
|
||||
FILTER Color EQ Red OR Color EQ Blue AND Color NE Green AS Color`,
|
||||
},
|
||||
{
|
||||
name: "brackets aroung lower precedence op",
|
||||
input: `REP 1
|
||||
CBF 1
|
||||
SELECT 1 FROM Color
|
||||
FILTER (Color EQ Red OR Color EQ Blue) AND Color NE Green AS Color`,
|
||||
},
|
||||
{
|
||||
name: "no extra brackets for bracketed same-operator chain",
|
||||
input: `REP 1
|
||||
CBF 1
|
||||
SELECT 1 FROM Color
|
||||
FILTER (Color EQ Red OR Color EQ Blue OR Color EQ Yellow) AND Color NE Green AS Color`,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
var p PlacementPolicy
|
||||
require.NoError(t, p.DecodeString(tc.input))
|
||||
|
||||
var sb strings.Builder
|
||||
require.NoError(t, p.WriteStringTo(&sb))
|
||||
|
||||
if tc.output == "" {
|
||||
require.Equal(t, tc.input, sb.String())
|
||||
} else {
|
||||
require.Equal(t, tc.output, sb.String())
|
||||
|
||||
var p1 PlacementPolicy
|
||||
require.NoError(t, p1.DecodeString(tc.output))
|
||||
require.Equal(t, p, p1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeSelectFilterExpr(t *testing.T) {
|
||||
for _, s := range []string{
|
||||
"SELECT 1 FROM *",
|
||||
|
|
|
@ -99,10 +99,12 @@ func BenchmarkPolicyHRWType(b *testing.B) {
|
|||
p := newPlacementPolicy(1,
|
||||
[]ReplicaDescriptor{
|
||||
newReplica(1, "loc1"),
|
||||
newReplica(1, "loc2")},
|
||||
newReplica(1, "loc2"),
|
||||
},
|
||||
[]Selector{
|
||||
newSelector("loc1", "Location", 1, "loc1", (*Selector).SelectSame),
|
||||
newSelector("loc2", "Location", 1, "loc2", (*Selector).SelectSame)},
|
||||
newSelector("loc2", "Location", 1, "loc2", (*Selector).SelectSame),
|
||||
},
|
||||
[]Filter{
|
||||
newFilter("loc1", "Location", "Shanghai", netmap.EQ),
|
||||
newFilter("loc2", "Location", "Shanghai", netmap.NE),
|
||||
|
@ -144,10 +146,12 @@ func TestPlacementPolicy_DeterministicOrder(t *testing.T) {
|
|||
p := newPlacementPolicy(1,
|
||||
[]ReplicaDescriptor{
|
||||
newReplica(1, "loc1"),
|
||||
newReplica(1, "loc2")},
|
||||
newReplica(1, "loc2"),
|
||||
},
|
||||
[]Selector{
|
||||
newSelector("loc1", "Location", 1, "loc1", (*Selector).SelectSame),
|
||||
newSelector("loc2", "Location", 1, "loc2", (*Selector).SelectSame)},
|
||||
newSelector("loc2", "Location", 1, "loc2", (*Selector).SelectSame),
|
||||
},
|
||||
[]Filter{
|
||||
newFilter("loc1", "Location", "Shanghai", netmap.EQ),
|
||||
newFilter("loc2", "Location", "Shanghai", netmap.NE),
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
package netmap
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
|
@ -8,24 +9,41 @@ import (
|
|||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// TestCase represents collection of placement policy tests for a single node set.
|
||||
type TestCase struct {
|
||||
Name string `json:"name"`
|
||||
Nodes []NodeInfo `json:"nodes"`
|
||||
Name string `json:"name" yaml:"name"`
|
||||
Nodes []NodeInfo `json:"nodes" yaml:"nodes"`
|
||||
Tests map[string]struct {
|
||||
Policy PlacementPolicy `json:"policy"`
|
||||
Pivot []byte `json:"pivot,omitempty"`
|
||||
Result [][]int `json:"result,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Policy PlacementPolicy
|
||||
Pivot Base64
|
||||
Result [][]int `json:"result,omitempty" yaml:"result,omitempty"`
|
||||
Error string `json:"error,omitempty" yaml:"error,omitempty"`
|
||||
Placement struct {
|
||||
Pivot []byte
|
||||
Result [][]int
|
||||
} `json:"placement,omitempty"`
|
||||
Pivot Base64 `json:"pivot" yaml:"pivot"`
|
||||
Result [][]int `json:"result,omitempty" yaml:"result,omitempty"`
|
||||
} `json:"placement,omitempty" yaml:"placement,omitempty"`
|
||||
}
|
||||
}
|
||||
|
||||
// Base64 is a type that will hold the decoded Base64 data.
|
||||
type Base64 []byte
|
||||
|
||||
func (b *Base64) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
||||
var base64Str string
|
||||
if err := unmarshal(&base64Str); err != nil {
|
||||
return err
|
||||
}
|
||||
decodedBytes, err := base64.StdEncoding.DecodeString(base64Str)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*b = decodedBytes
|
||||
return nil
|
||||
}
|
||||
|
||||
var _, _ json.Unmarshaler = new(NodeInfo), new(PlacementPolicy)
|
||||
|
||||
func compareNodes(t testing.TB, expected [][]int, nodes nodes, actual [][]NodeInfo) {
|
||||
|
@ -39,7 +57,7 @@ func compareNodes(t testing.TB, expected [][]int, nodes nodes, actual [][]NodeIn
|
|||
}
|
||||
|
||||
func TestPlacementPolicy_Interopability(t *testing.T) {
|
||||
const testsDir = "./json_tests"
|
||||
const testsDir = "./yml_tests"
|
||||
|
||||
f, err := os.Open(testsDir)
|
||||
require.NoError(t, err)
|
||||
|
@ -53,7 +71,7 @@ func TestPlacementPolicy_Interopability(t *testing.T) {
|
|||
require.NoError(t, err)
|
||||
|
||||
var tc TestCase
|
||||
require.NoError(t, json.Unmarshal(bs, &tc), "cannot unmarshal %s", ds[i].Name())
|
||||
require.NoError(t, yaml.Unmarshal(bs, &tc), "cannot unmarshal %s", ds[i].Name())
|
||||
|
||||
srcNodes := make([]NodeInfo, len(tc.Nodes))
|
||||
copy(srcNodes, tc.Nodes)
|
||||
|
@ -88,7 +106,7 @@ func TestPlacementPolicy_Interopability(t *testing.T) {
|
|||
}
|
||||
|
||||
func BenchmarkPlacementPolicyInteropability(b *testing.B) {
|
||||
const testsDir = "./json_tests"
|
||||
const testsDir = "./yml_tests"
|
||||
|
||||
f, err := os.Open(testsDir)
|
||||
require.NoError(b, err)
|
||||
|
@ -101,7 +119,7 @@ func BenchmarkPlacementPolicyInteropability(b *testing.B) {
|
|||
require.NoError(b, err)
|
||||
|
||||
var tc TestCase
|
||||
require.NoError(b, json.Unmarshal(bs, &tc), "cannot unmarshal %s", ds[i].Name())
|
||||
require.NoError(b, yaml.Unmarshal(bs, &tc), "cannot unmarshal %s", ds[i].Name())
|
||||
|
||||
b.Run(tc.Name, func(b *testing.B) {
|
||||
var nm NetMap
|
||||
|
@ -140,12 +158,12 @@ func BenchmarkPlacementPolicyInteropability(b *testing.B) {
|
|||
}
|
||||
|
||||
func BenchmarkManySelects(b *testing.B) {
|
||||
testsFile := filepath.Join("json_tests", "many_selects.json")
|
||||
testsFile := filepath.Join("yml_tests", "many_selects.yml")
|
||||
bs, err := os.ReadFile(testsFile)
|
||||
require.NoError(b, err)
|
||||
|
||||
var tc TestCase
|
||||
require.NoError(b, json.Unmarshal(bs, &tc))
|
||||
require.NoError(b, yaml.Unmarshal(bs, &tc))
|
||||
tt, ok := tc.Tests["Select"]
|
||||
require.True(b, ok)
|
||||
|
48
netmap/yml_tests/cbf_default.yml
Normal file
48
netmap/yml_tests/cbf_default.yml
Normal file
|
@ -0,0 +1,48 @@
|
|||
name: default CBF is 3
|
||||
nodes:
|
||||
- attributes:
|
||||
- key: Location
|
||||
value: Europe
|
||||
- key: Country
|
||||
value: RU
|
||||
- key: City
|
||||
value: St.Petersburg
|
||||
- attributes:
|
||||
- key: Location
|
||||
value: Europe
|
||||
- key: Country
|
||||
value: RU
|
||||
- key: City
|
||||
value: Moscow
|
||||
- attributes:
|
||||
- key: Location
|
||||
value: Europe
|
||||
- key: Country
|
||||
value: DE
|
||||
- key: City
|
||||
value: Berlin
|
||||
- attributes:
|
||||
- key: Location
|
||||
value: Europe
|
||||
- key: Country
|
||||
value: FR
|
||||
- key: City
|
||||
value: Paris
|
||||
tests:
|
||||
set default CBF:
|
||||
policy:
|
||||
replicas:
|
||||
- count: 1
|
||||
selector: EU
|
||||
containerBackupFactor: 0
|
||||
selectors:
|
||||
- name: EU
|
||||
count: 1
|
||||
clause: SAME
|
||||
attribute: Location
|
||||
filter: '*'
|
||||
filters: []
|
||||
result:
|
||||
- - 0
|
||||
- 1
|
||||
- 2
|
52
netmap/yml_tests/cbf_minimal.yml
Normal file
52
netmap/yml_tests/cbf_minimal.yml
Normal file
|
@ -0,0 +1,52 @@
|
|||
name: Real node count multiplier is in range [1, specified CBF]
|
||||
nodes:
|
||||
- attributes:
|
||||
- key: ID
|
||||
value: '1'
|
||||
- key: Country
|
||||
value: DE
|
||||
- attributes:
|
||||
- key: ID
|
||||
value: '2'
|
||||
- key: Country
|
||||
value: DE
|
||||
- attributes:
|
||||
- key: ID
|
||||
value: '3'
|
||||
- key: Country
|
||||
value: DE
|
||||
tests:
|
||||
select 2, CBF is 2:
|
||||
policy:
|
||||
replicas:
|
||||
- count: 1
|
||||
selector: X
|
||||
containerBackupFactor: 2
|
||||
selectors:
|
||||
- name: X
|
||||
count: 2
|
||||
clause: SAME
|
||||
attribute: Country
|
||||
filter: '*'
|
||||
filters: []
|
||||
result:
|
||||
- - 0
|
||||
- 1
|
||||
- 2
|
||||
select 3, CBF is 2:
|
||||
policy:
|
||||
replicas:
|
||||
- count: 1
|
||||
selector: X
|
||||
containerBackupFactor: 2
|
||||
selectors:
|
||||
- name: X
|
||||
count: 3
|
||||
clause: SAME
|
||||
attribute: Country
|
||||
filter: '*'
|
||||
filters: []
|
||||
result:
|
||||
- - 0
|
||||
- 1
|
||||
- 2
|
82
netmap/yml_tests/cbf_requirements.yml
Normal file
82
netmap/yml_tests/cbf_requirements.yml
Normal file
|
@ -0,0 +1,82 @@
|
|||
name: CBF requirements
|
||||
nodes:
|
||||
- attributes:
|
||||
- key: ID
|
||||
value: '1'
|
||||
- key: Attr
|
||||
value: Same
|
||||
- attributes:
|
||||
- key: ID
|
||||
value: '2'
|
||||
- key: Attr
|
||||
value: Same
|
||||
- attributes:
|
||||
- key: ID
|
||||
value: '3'
|
||||
- key: Attr
|
||||
value: Same
|
||||
- attributes:
|
||||
- key: ID
|
||||
value: '4'
|
||||
- key: Attr
|
||||
value: Same
|
||||
tests:
|
||||
default CBF, no selector:
|
||||
policy:
|
||||
replicas:
|
||||
- count: 2
|
||||
containerBackupFactor: 0
|
||||
selectors: []
|
||||
filters: []
|
||||
result:
|
||||
- - 0
|
||||
- 2
|
||||
- 1
|
||||
- 3
|
||||
explicit CBF, no selector:
|
||||
policy:
|
||||
replicas:
|
||||
- count: 2
|
||||
containerBackupFactor: 3
|
||||
selectors: []
|
||||
filters: []
|
||||
result:
|
||||
- - 0
|
||||
- 2
|
||||
- 1
|
||||
- 3
|
||||
select distinct, weak CBF:
|
||||
policy:
|
||||
replicas:
|
||||
- count: 2
|
||||
selector: X
|
||||
containerBackupFactor: 3
|
||||
selectors:
|
||||
- name: X
|
||||
count: 2
|
||||
clause: DISTINCT
|
||||
filter: '*'
|
||||
filters: []
|
||||
result:
|
||||
- - 0
|
||||
- 2
|
||||
- 1
|
||||
- 3
|
||||
select same, weak CBF:
|
||||
policy:
|
||||
replicas:
|
||||
- count: 2
|
||||
selector: X
|
||||
containerBackupFactor: 3
|
||||
selectors:
|
||||
- name: X
|
||||
count: 2
|
||||
clause: SAME
|
||||
attribute: Attr
|
||||
filter: '*'
|
||||
filters: []
|
||||
result:
|
||||
- - 0
|
||||
- 1
|
||||
- 2
|
||||
- 3
|
210
netmap/yml_tests/filter_complex.yml
Normal file
210
netmap/yml_tests/filter_complex.yml
Normal file
|
@ -0,0 +1,210 @@
|
|||
name: compound filter
|
||||
nodes:
|
||||
- attributes:
|
||||
- key: Storage
|
||||
value: SSD
|
||||
- key: Rating
|
||||
value: '10'
|
||||
- key: IntField
|
||||
value: '100'
|
||||
- key: Param
|
||||
value: Value1
|
||||
tests:
|
||||
good:
|
||||
policy:
|
||||
replicas:
|
||||
- count: 1
|
||||
selector: S
|
||||
containerBackupFactor: 1
|
||||
selectors:
|
||||
- name: S
|
||||
count: 1
|
||||
clause: DISTINCT
|
||||
filter: Main
|
||||
filters:
|
||||
- name: StorageSSD
|
||||
key: Storage
|
||||
op: EQ
|
||||
value: SSD
|
||||
filters: []
|
||||
- name: GoodRating
|
||||
key: Rating
|
||||
op: GE
|
||||
value: '4'
|
||||
filters: []
|
||||
- name: Main
|
||||
op: AND
|
||||
filters:
|
||||
- name: StorageSSD
|
||||
op: OPERATION_UNSPECIFIED
|
||||
filters: []
|
||||
- name: ''
|
||||
key: IntField
|
||||
op: LT
|
||||
value: '123'
|
||||
filters: []
|
||||
- name: GoodRating
|
||||
op: OPERATION_UNSPECIFIED
|
||||
filters: []
|
||||
- op: OR
|
||||
filters:
|
||||
- key: Param
|
||||
op: EQ
|
||||
value: Value1
|
||||
filters: []
|
||||
- key: Param
|
||||
op: EQ
|
||||
value: Value2
|
||||
filters: []
|
||||
result:
|
||||
- - 0
|
||||
bad storage type:
|
||||
policy:
|
||||
replicas:
|
||||
- count: 1
|
||||
selector: S
|
||||
containerBackupFactor: 1
|
||||
selectors:
|
||||
- name: S
|
||||
count: 1
|
||||
clause: DISTINCT
|
||||
filter: Main
|
||||
filters:
|
||||
- name: StorageSSD
|
||||
key: Storage
|
||||
op: EQ
|
||||
value: HDD
|
||||
filters: []
|
||||
- name: GoodRating
|
||||
key: Rating
|
||||
op: GE
|
||||
value: '4'
|
||||
filters: []
|
||||
- name: Main
|
||||
op: AND
|
||||
filters:
|
||||
- name: StorageSSD
|
||||
op: OPERATION_UNSPECIFIED
|
||||
filters: []
|
||||
- name: ''
|
||||
key: IntField
|
||||
op: LT
|
||||
value: '123'
|
||||
filters: []
|
||||
- name: GoodRating
|
||||
op: OPERATION_UNSPECIFIED
|
||||
filters: []
|
||||
- name: ''
|
||||
op: OR
|
||||
filters:
|
||||
- name: ''
|
||||
key: Param
|
||||
op: EQ
|
||||
value: Value1
|
||||
filters: []
|
||||
- name: ''
|
||||
key: Param
|
||||
op: EQ
|
||||
value: Value2
|
||||
filters: []
|
||||
error: not enough nodes
|
||||
bad rating:
|
||||
policy:
|
||||
replicas:
|
||||
- count: 1
|
||||
selector: S
|
||||
containerBackupFactor: 1
|
||||
selectors:
|
||||
- name: S
|
||||
count: 1
|
||||
clause: DISTINCT
|
||||
filter: Main
|
||||
filters:
|
||||
- name: StorageSSD
|
||||
key: Storage
|
||||
op: EQ
|
||||
value: SSD
|
||||
filters: []
|
||||
- name: GoodRating
|
||||
key: Rating
|
||||
op: GE
|
||||
value: '15'
|
||||
filters: []
|
||||
- name: Main
|
||||
op: AND
|
||||
filters:
|
||||
- name: StorageSSD
|
||||
op: OPERATION_UNSPECIFIED
|
||||
filters: []
|
||||
- name: ''
|
||||
key: IntField
|
||||
op: LT
|
||||
value: '123'
|
||||
filters: []
|
||||
- name: GoodRating
|
||||
op: OPERATION_UNSPECIFIED
|
||||
filters: []
|
||||
- name: ''
|
||||
op: OR
|
||||
filters:
|
||||
- name: ''
|
||||
key: Param
|
||||
op: EQ
|
||||
value: Value1
|
||||
filters: []
|
||||
- name: ''
|
||||
key: Param
|
||||
op: EQ
|
||||
value: Value2
|
||||
filters: []
|
||||
error: not enough nodes
|
||||
bad param:
|
||||
policy:
|
||||
replicas:
|
||||
- count: 1
|
||||
selector: S
|
||||
containerBackupFactor: 1
|
||||
selectors:
|
||||
- name: S
|
||||
count: 1
|
||||
clause: DISTINCT
|
||||
filter: Main
|
||||
filters:
|
||||
- name: StorageSSD
|
||||
key: Storage
|
||||
op: EQ
|
||||
value: SSD
|
||||
filters: []
|
||||
- name: GoodRating
|
||||
key: Rating
|
||||
op: GE
|
||||
value: '4'
|
||||
filters: []
|
||||
- name: Main
|
||||
op: AND
|
||||
filters:
|
||||
- name: StorageSSD
|
||||
op: OPERATION_UNSPECIFIED
|
||||
filters: []
|
||||
- name: ''
|
||||
key: IntField
|
||||
op: LT
|
||||
value: '123'
|
||||
filters: []
|
||||
- name: GoodRating
|
||||
op: OPERATION_UNSPECIFIED
|
||||
filters: []
|
||||
- name: ''
|
||||
op: OR
|
||||
filters:
|
||||
- name: ''
|
||||
key: Param
|
||||
op: EQ
|
||||
value: Value0
|
||||
filters: []
|
||||
- name: ''
|
||||
key: Param
|
||||
op: EQ
|
||||
value: Value2
|
||||
filters: []
|
||||
error: not enough nodes
|
45
netmap/yml_tests/filter_invalid_integer.yml
Normal file
45
netmap/yml_tests/filter_invalid_integer.yml
Normal file
|
@ -0,0 +1,45 @@
|
|||
name: invalid integer field
|
||||
nodes:
|
||||
- attributes:
|
||||
- key: IntegerField
|
||||
value: 'true'
|
||||
- attributes:
|
||||
- key: IntegerField
|
||||
value: str
|
||||
tests:
|
||||
empty string is not casted to 0:
|
||||
policy:
|
||||
replicas:
|
||||
- count: 1
|
||||
selector: S
|
||||
containerBackupFactor: 1
|
||||
selectors:
|
||||
- name: S
|
||||
count: 1
|
||||
clause: DISTINCT
|
||||
filter: Main
|
||||
filters:
|
||||
- name: Main
|
||||
key: IntegerField
|
||||
op: LE
|
||||
value: '8'
|
||||
filters: []
|
||||
error: not enough nodes
|
||||
non-empty string is not casted to a number:
|
||||
policy:
|
||||
replicas:
|
||||
- count: 1
|
||||
selector: S
|
||||
containerBackupFactor: 1
|
||||
selectors:
|
||||
- name: S
|
||||
count: 1
|
||||
clause: DISTINCT
|
||||
filter: Main
|
||||
filters:
|
||||
- name: Main
|
||||
key: IntegerField
|
||||
op: GE
|
||||
value: '0'
|
||||
filters: []
|
||||
error: not enough nodes
|
230
netmap/yml_tests/filter_simple.yml
Normal file
230
netmap/yml_tests/filter_simple.yml
Normal file
|
@ -0,0 +1,230 @@
|
|||
name: single-op filters
|
||||
nodes:
|
||||
- attributes:
|
||||
- key: Rating
|
||||
value: '4'
|
||||
- key: Country
|
||||
value: Germany
|
||||
tests:
|
||||
GE true:
|
||||
policy:
|
||||
replicas:
|
||||
- count: 1
|
||||
selector: S
|
||||
containerBackupFactor: 1
|
||||
selectors:
|
||||
- name: S
|
||||
count: 1
|
||||
clause: DISTINCT
|
||||
filter: Main
|
||||
filters:
|
||||
- name: Main
|
||||
key: Rating
|
||||
op: GE
|
||||
value: '4'
|
||||
filters: []
|
||||
result:
|
||||
- - 0
|
||||
GE false:
|
||||
policy:
|
||||
replicas:
|
||||
- count: 1
|
||||
selector: S
|
||||
containerBackupFactor: 1
|
||||
selectors:
|
||||
- name: S
|
||||
count: 1
|
||||
clause: DISTINCT
|
||||
filter: Main
|
||||
filters:
|
||||
- name: Main
|
||||
key: Rating
|
||||
op: GE
|
||||
value: '5'
|
||||
filters: []
|
||||
error: not enough nodes
|
||||
GT true:
|
||||
policy:
|
||||
replicas:
|
||||
- count: 1
|
||||
selector: S
|
||||
containerBackupFactor: 1
|
||||
selectors:
|
||||
- name: S
|
||||
count: 1
|
||||
clause: DISTINCT
|
||||
filter: Main
|
||||
filters:
|
||||
- name: Main
|
||||
key: Rating
|
||||
op: GT
|
||||
value: '3'
|
||||
filters: []
|
||||
result:
|
||||
- - 0
|
||||
GT false:
|
||||
policy:
|
||||
replicas:
|
||||
- count: 1
|
||||
selector: S
|
||||
containerBackupFactor: 1
|
||||
selectors:
|
||||
- name: S
|
||||
count: 1
|
||||
clause: DISTINCT
|
||||
filter: Main
|
||||
filters:
|
||||
- name: Main
|
||||
key: Rating
|
||||
op: GT
|
||||
value: '4'
|
||||
filters: []
|
||||
error: not enough nodes
|
||||
LE true:
|
||||
policy:
|
||||
replicas:
|
||||
- count: 1
|
||||
selector: S
|
||||
containerBackupFactor: 1
|
||||
selectors:
|
||||
- name: S
|
||||
count: 1
|
||||
clause: DISTINCT
|
||||
filter: Main
|
||||
filters:
|
||||
- name: Main
|
||||
key: Rating
|
||||
op: LE
|
||||
value: '4'
|
||||
filters: []
|
||||
result:
|
||||
- - 0
|
||||
LE false:
|
||||
policy:
|
||||
replicas:
|
||||
- count: 1
|
||||
selector: S
|
||||
containerBackupFactor: 1
|
||||
selectors:
|
||||
- name: S
|
||||
count: 1
|
||||
clause: DISTINCT
|
||||
filter: Main
|
||||
filters:
|
||||
- name: Main
|
||||
key: Rating
|
||||
op: LE
|
||||
value: '3'
|
||||
filters: []
|
||||
error: not enough nodes
|
||||
LT true:
|
||||
policy:
|
||||
replicas:
|
||||
- count: 1
|
||||
selector: S
|
||||
containerBackupFactor: 1
|
||||
selectors:
|
||||
- name: S
|
||||
count: 1
|
||||
clause: DISTINCT
|
||||
filter: Main
|
||||
filters:
|
||||
- name: Main
|
||||
key: Rating
|
||||
op: LT
|
||||
value: '5'
|
||||
filters: []
|
||||
result:
|
||||
- - 0
|
||||
LT false:
|
||||
policy:
|
||||
replicas:
|
||||
- count: 1
|
||||
selector: S
|
||||
containerBackupFactor: 1
|
||||
selectors:
|
||||
- name: S
|
||||
count: 1
|
||||
clause: DISTINCT
|
||||
filter: Main
|
||||
filters:
|
||||
- name: Main
|
||||
key: Rating
|
||||
op: LT
|
||||
value: '4'
|
||||
filters: []
|
||||
error: not enough nodes
|
||||
EQ true:
|
||||
policy:
|
||||
replicas:
|
||||
- count: 1
|
||||
selector: S
|
||||
containerBackupFactor: 1
|
||||
selectors:
|
||||
- name: S
|
||||
count: 1
|
||||
clause: DISTINCT
|
||||
filter: Main
|
||||
filters:
|
||||
- name: Main
|
||||
key: Country
|
||||
op: EQ
|
||||
value: Germany
|
||||
filters: []
|
||||
result:
|
||||
- - 0
|
||||
EQ false:
|
||||
policy:
|
||||
replicas:
|
||||
- count: 1
|
||||
selector: S
|
||||
containerBackupFactor: 1
|
||||
selectors:
|
||||
- name: S
|
||||
count: 1
|
||||
clause: DISTINCT
|
||||
filter: Main
|
||||
filters:
|
||||
- name: Main
|
||||
key: Country
|
||||
op: EQ
|
||||
value: China
|
||||
filters: []
|
||||
error: not enough nodes
|
||||
NE true:
|
||||
policy:
|
||||
replicas:
|
||||
- count: 1
|
||||
selector: S
|
||||
containerBackupFactor: 1
|
||||
selectors:
|
||||
- name: S
|
||||
count: 1
|
||||
clause: DISTINCT
|
||||
filter: Main
|
||||
filters:
|
||||
- name: Main
|
||||
key: Country
|
||||
op: NE
|
||||
value: France
|
||||
filters: []
|
||||
result:
|
||||
- - 0
|
||||
NE false:
|
||||
policy:
|
||||
replicas:
|
||||
- count: 1
|
||||
selector: S
|
||||
containerBackupFactor: 1
|
||||
selectors:
|
||||
- name: S
|
||||
count: 1
|
||||
clause: DISTINCT
|
||||
filter: Main
|
||||
filters:
|
||||
- name: Main
|
||||
key: Country
|
||||
op: NE
|
||||
value: Germany
|
||||
filters: []
|
||||
error: not enough nodes
|
118
netmap/yml_tests/hrw_sort.yml
Normal file
118
netmap/yml_tests/hrw_sort.yml
Normal file
|
@ -0,0 +1,118 @@
|
|||
name: HRW ordering
|
||||
nodes:
|
||||
- attributes:
|
||||
- key: Country
|
||||
value: Germany
|
||||
- key: Price
|
||||
value: '2'
|
||||
- key: Capacity
|
||||
value: '10000'
|
||||
- attributes:
|
||||
- key: Country
|
||||
value: Germany
|
||||
- key: Price
|
||||
value: '4'
|
||||
- key: Capacity
|
||||
value: '1'
|
||||
- attributes:
|
||||
- key: Country
|
||||
value: France
|
||||
- key: Price
|
||||
value: '3'
|
||||
- key: Capacity
|
||||
value: '10'
|
||||
- attributes:
|
||||
- key: Country
|
||||
value: Russia
|
||||
- key: Price
|
||||
value: '2'
|
||||
- key: Capacity
|
||||
value: '10000'
|
||||
- attributes:
|
||||
- key: Country
|
||||
value: Russia
|
||||
- key: Price
|
||||
value: '1'
|
||||
- key: Capacity
|
||||
value: '10000'
|
||||
- attributes:
|
||||
- key: Country
|
||||
value: Russia
|
||||
- key: Capacity
|
||||
value: '10000'
|
||||
- attributes:
|
||||
- key: Country
|
||||
value: France
|
||||
- key: Price
|
||||
value: '100'
|
||||
- key: Capacity
|
||||
value: '1'
|
||||
- attributes:
|
||||
- key: Country
|
||||
value: France
|
||||
- key: Price
|
||||
value: '7'
|
||||
- key: Capacity
|
||||
value: '10000'
|
||||
- attributes:
|
||||
- key: Country
|
||||
value: Russia
|
||||
- key: Price
|
||||
value: '2'
|
||||
- key: Capacity
|
||||
value: '1'
|
||||
tests:
|
||||
select 3 nodes in 3 distinct countries, same placement:
|
||||
policy:
|
||||
replicas:
|
||||
- count: 1
|
||||
selector: Main
|
||||
containerBackupFactor: 1
|
||||
selectors:
|
||||
- name: Main
|
||||
count: 3
|
||||
clause: DISTINCT
|
||||
attribute: Country
|
||||
filter: '*'
|
||||
filters: []
|
||||
pivot: Y29udGFpbmVySUQ=
|
||||
result:
|
||||
- - 5
|
||||
- 0
|
||||
- 7
|
||||
placement:
|
||||
pivot: b2JqZWN0SUQ=
|
||||
result:
|
||||
- - 5
|
||||
- 0
|
||||
- 7
|
||||
select 6 nodes in 3 distinct countries, different placement:
|
||||
policy:
|
||||
replicas:
|
||||
- count: 1
|
||||
selector: Main
|
||||
containerBackupFactor: 2
|
||||
selectors:
|
||||
- name: Main
|
||||
count: 3
|
||||
clause: DISTINCT
|
||||
attribute: Country
|
||||
filter: '*'
|
||||
filters: []
|
||||
pivot: Y29udGFpbmVySUQ=
|
||||
result:
|
||||
- - 5
|
||||
- 4
|
||||
- 0
|
||||
- 1
|
||||
- 7
|
||||
- 2
|
||||
placement:
|
||||
pivot: b2JqZWN0SUQ=
|
||||
result:
|
||||
- - 5
|
||||
- 4
|
||||
- 0
|
||||
- 7
|
||||
- 2
|
||||
- 1
|
52
netmap/yml_tests/issue213.yml
Normal file
52
netmap/yml_tests/issue213.yml
Normal file
|
@ -0,0 +1,52 @@
|
|||
name: unnamed selector (nspcc-dev/neofs-api-go#213)
|
||||
nodes:
|
||||
- attributes:
|
||||
- key: Location
|
||||
value: Europe
|
||||
- key: Country
|
||||
value: Russia
|
||||
- key: City
|
||||
value: Moscow
|
||||
- attributes:
|
||||
- key: Location
|
||||
value: Europe
|
||||
- key: Country
|
||||
value: Russia
|
||||
- key: City
|
||||
value: Saint-Petersburg
|
||||
- attributes:
|
||||
- key: Location
|
||||
value: Europe
|
||||
- key: Country
|
||||
value: Sweden
|
||||
- key: City
|
||||
value: Stockholm
|
||||
- attributes:
|
||||
- key: Location
|
||||
value: Europe
|
||||
- key: Country
|
||||
value: Finalnd
|
||||
- key: City
|
||||
value: Helsinki
|
||||
tests:
|
||||
test:
|
||||
policy:
|
||||
replicas:
|
||||
- count: 4
|
||||
containerBackupFactor: 1
|
||||
selectors:
|
||||
- name: ''
|
||||
count: 4
|
||||
clause: DISTINCT
|
||||
filter: LOC_EU
|
||||
filters:
|
||||
- name: LOC_EU
|
||||
key: Location
|
||||
op: EQ
|
||||
value: Europe
|
||||
filters: []
|
||||
result:
|
||||
- - 0
|
||||
- 1
|
||||
- 2
|
||||
- 3
|
141
netmap/yml_tests/many_selects.yml
Normal file
141
netmap/yml_tests/many_selects.yml
Normal file
|
@ -0,0 +1,141 @@
|
|||
name: single-op filters
|
||||
nodes:
|
||||
- attributes:
|
||||
- key: Country
|
||||
value: Russia
|
||||
- key: Rating
|
||||
value: '1'
|
||||
- key: City
|
||||
value: SPB
|
||||
- attributes:
|
||||
- key: Country
|
||||
value: Germany
|
||||
- key: Rating
|
||||
value: '5'
|
||||
- key: City
|
||||
value: Berlin
|
||||
- attributes:
|
||||
- key: Country
|
||||
value: Russia
|
||||
- key: Rating
|
||||
value: '6'
|
||||
- key: City
|
||||
value: Moscow
|
||||
- attributes:
|
||||
- key: Country
|
||||
value: France
|
||||
- key: Rating
|
||||
value: '4'
|
||||
- key: City
|
||||
value: Paris
|
||||
- attributes:
|
||||
- key: Country
|
||||
value: France
|
||||
- key: Rating
|
||||
value: '1'
|
||||
- key: City
|
||||
value: Lyon
|
||||
- attributes:
|
||||
- key: Country
|
||||
value: Russia
|
||||
- key: Rating
|
||||
value: '5'
|
||||
- key: City
|
||||
value: SPB
|
||||
- attributes:
|
||||
- key: Country
|
||||
value: Russia
|
||||
- key: Rating
|
||||
value: '7'
|
||||
- key: City
|
||||
value: Moscow
|
||||
- attributes:
|
||||
- key: Country
|
||||
value: Germany
|
||||
- key: Rating
|
||||
value: '3'
|
||||
- key: City
|
||||
value: Darmstadt
|
||||
- attributes:
|
||||
- key: Country
|
||||
value: Germany
|
||||
- key: Rating
|
||||
value: '7'
|
||||
- key: City
|
||||
value: Frankfurt
|
||||
- attributes:
|
||||
- key: Country
|
||||
value: Russia
|
||||
- key: Rating
|
||||
value: '9'
|
||||
- key: City
|
||||
value: SPB
|
||||
- attributes:
|
||||
- key: Country
|
||||
value: Russia
|
||||
- key: Rating
|
||||
value: '9'
|
||||
- key: City
|
||||
value: SPB
|
||||
tests:
|
||||
Select:
|
||||
policy:
|
||||
replicas:
|
||||
- count: 1
|
||||
selector: SameRU
|
||||
- count: 1
|
||||
selector: DistinctRU
|
||||
- count: 1
|
||||
selector: Good
|
||||
- count: 1
|
||||
selector: Main
|
||||
containerBackupFactor: 2
|
||||
selectors:
|
||||
- name: SameRU
|
||||
count: 2
|
||||
clause: SAME
|
||||
attribute: City
|
||||
filter: FromRU
|
||||
- name: DistinctRU
|
||||
count: 2
|
||||
clause: DISTINCT
|
||||
attribute: City
|
||||
filter: FromRU
|
||||
- name: Good
|
||||
count: 2
|
||||
clause: DISTINCT
|
||||
attribute: Country
|
||||
filter: Good
|
||||
- name: Main
|
||||
count: 3
|
||||
clause: DISTINCT
|
||||
attribute: Country
|
||||
filter: '*'
|
||||
filters:
|
||||
- name: FromRU
|
||||
key: Country
|
||||
op: EQ
|
||||
value: Russia
|
||||
- name: Good
|
||||
key: Rating
|
||||
op: GE
|
||||
value: '4'
|
||||
result:
|
||||
- - 0
|
||||
- 5
|
||||
- 9
|
||||
- 10
|
||||
- - 2
|
||||
- 6
|
||||
- 0
|
||||
- 5
|
||||
- - 1
|
||||
- 8
|
||||
- 2
|
||||
- 5
|
||||
- - 3
|
||||
- 4
|
||||
- 1
|
||||
- 7
|
||||
- 0
|
||||
- 2
|
46
netmap/yml_tests/multiple_rep.yml
Normal file
46
netmap/yml_tests/multiple_rep.yml
Normal file
|
@ -0,0 +1,46 @@
|
|||
name: multiple replicas (#215)
|
||||
nodes:
|
||||
- attributes:
|
||||
- key: City
|
||||
value: Saint-Petersburg
|
||||
- attributes:
|
||||
- key: City
|
||||
value: Moscow
|
||||
- attributes:
|
||||
- key: City
|
||||
value: Berlin
|
||||
- attributes:
|
||||
- key: City
|
||||
value: Paris
|
||||
tests:
|
||||
test:
|
||||
policy:
|
||||
replicas:
|
||||
- count: 1
|
||||
selector: LOC_SPB_PLACE
|
||||
- count: 1
|
||||
selector: LOC_MSK_PLACE
|
||||
containerBackupFactor: 1
|
||||
selectors:
|
||||
- name: LOC_SPB_PLACE
|
||||
count: 1
|
||||
clause: CLAUSE_UNSPECIFIED
|
||||
filter: LOC_SPB
|
||||
- name: LOC_MSK_PLACE
|
||||
count: 1
|
||||
clause: CLAUSE_UNSPECIFIED
|
||||
filter: LOC_MSK
|
||||
filters:
|
||||
- name: LOC_SPB
|
||||
key: City
|
||||
op: EQ
|
||||
value: Saint-Petersburg
|
||||
filters: []
|
||||
- name: LOC_MSK
|
||||
key: City
|
||||
op: EQ
|
||||
value: Moscow
|
||||
filters: []
|
||||
result:
|
||||
- - 0
|
||||
- - 1
|
162
netmap/yml_tests/multiple_rep_asymmetric.yml
Normal file
162
netmap/yml_tests/multiple_rep_asymmetric.yml
Normal file
|
@ -0,0 +1,162 @@
|
|||
name: multiple REP, asymmetric
|
||||
nodes:
|
||||
- attributes:
|
||||
- key: ID
|
||||
value: '1'
|
||||
- key: Country
|
||||
value: RU
|
||||
- key: City
|
||||
value: St.Petersburg
|
||||
- key: SSD
|
||||
value: '0'
|
||||
- attributes:
|
||||
- key: ID
|
||||
value: '2'
|
||||
- key: Country
|
||||
value: RU
|
||||
- key: City
|
||||
value: St.Petersburg
|
||||
- key: SSD
|
||||
value: '1'
|
||||
- attributes:
|
||||
- key: ID
|
||||
value: '3'
|
||||
- key: Country
|
||||
value: RU
|
||||
- key: City
|
||||
value: Moscow
|
||||
- key: SSD
|
||||
value: '1'
|
||||
- attributes:
|
||||
- key: ID
|
||||
value: '4'
|
||||
- key: Country
|
||||
value: RU
|
||||
- key: City
|
||||
value: Moscow
|
||||
- key: SSD
|
||||
value: '1'
|
||||
- attributes:
|
||||
- key: ID
|
||||
value: '5'
|
||||
- key: Country
|
||||
value: RU
|
||||
- key: City
|
||||
value: St.Petersburg
|
||||
- key: SSD
|
||||
value: '1'
|
||||
- attributes:
|
||||
- key: ID
|
||||
value: '6'
|
||||
- key: Continent
|
||||
value: NA
|
||||
- key: City
|
||||
value: NewYork
|
||||
- attributes:
|
||||
- key: ID
|
||||
value: '7'
|
||||
- key: Continent
|
||||
value: AF
|
||||
- key: City
|
||||
value: Cairo
|
||||
- attributes:
|
||||
- key: ID
|
||||
value: '8'
|
||||
- key: Continent
|
||||
value: AF
|
||||
- key: City
|
||||
value: Cairo
|
||||
- attributes:
|
||||
- key: ID
|
||||
value: '9'
|
||||
- key: Continent
|
||||
value: SA
|
||||
- key: City
|
||||
value: Lima
|
||||
- attributes:
|
||||
- key: ID
|
||||
value: '10'
|
||||
- key: Continent
|
||||
value: AF
|
||||
- key: City
|
||||
value: Cairo
|
||||
- attributes:
|
||||
- key: ID
|
||||
value: '11'
|
||||
- key: Continent
|
||||
value: NA
|
||||
- key: City
|
||||
value: NewYork
|
||||
- attributes:
|
||||
- key: ID
|
||||
value: '12'
|
||||
- key: Continent
|
||||
value: NA
|
||||
- key: City
|
||||
value: LosAngeles
|
||||
- attributes:
|
||||
- key: ID
|
||||
value: '13'
|
||||
- key: Continent
|
||||
value: SA
|
||||
- key: City
|
||||
value: Lima
|
||||
tests:
|
||||
test:
|
||||
policy:
|
||||
replicas:
|
||||
- count: 1
|
||||
selector: SPB
|
||||
- count: 2
|
||||
selector: Americas
|
||||
containerBackupFactor: 2
|
||||
selectors:
|
||||
- name: SPB
|
||||
count: 1
|
||||
clause: SAME
|
||||
attribute: City
|
||||
filter: SPBSSD
|
||||
- name: Americas
|
||||
count: 2
|
||||
clause: DISTINCT
|
||||
attribute: City
|
||||
filter: Americas
|
||||
filters:
|
||||
- name: SPBSSD
|
||||
op: AND
|
||||
filters:
|
||||
- name: ''
|
||||
key: Country
|
||||
op: EQ
|
||||
value: RU
|
||||
filters: []
|
||||
- name: ''
|
||||
key: City
|
||||
op: EQ
|
||||
value: St.Petersburg
|
||||
filters: []
|
||||
- name: ''
|
||||
key: SSD
|
||||
op: EQ
|
||||
value: '1'
|
||||
filters: []
|
||||
- name: Americas
|
||||
op: OR
|
||||
filters:
|
||||
- name: ''
|
||||
key: Continent
|
||||
op: EQ
|
||||
value: NA
|
||||
filters: []
|
||||
- name: ''
|
||||
key: Continent
|
||||
op: EQ
|
||||
value: SA
|
||||
filters: []
|
||||
result:
|
||||
- - 1
|
||||
- 4
|
||||
- - 8
|
||||
- 12
|
||||
- 5
|
||||
- 10
|
52
netmap/yml_tests/non_strict.yml
Normal file
52
netmap/yml_tests/non_strict.yml
Normal file
|
@ -0,0 +1,52 @@
|
|||
name: non-strict selections
|
||||
comment: These test specify loose selection behaviour, to allow fetching already PUT
|
||||
objects even when there is not enough nodes to select from.
|
||||
nodes:
|
||||
- attributes:
|
||||
- key: Country
|
||||
value: Russia
|
||||
- attributes:
|
||||
- key: Country
|
||||
value: Germany
|
||||
- attributes: []
|
||||
tests:
|
||||
not enough nodes (backup factor):
|
||||
policy:
|
||||
replicas:
|
||||
- count: 1
|
||||
selector: MyStore
|
||||
containerBackupFactor: 2
|
||||
selectors:
|
||||
- name: MyStore
|
||||
count: 2
|
||||
clause: DISTINCT
|
||||
attribute: Country
|
||||
filter: FromRU
|
||||
filters:
|
||||
- name: FromRU
|
||||
key: Country
|
||||
op: EQ
|
||||
value: Russia
|
||||
filters: []
|
||||
result:
|
||||
- - 0
|
||||
not enough nodes (buckets):
|
||||
policy:
|
||||
replicas:
|
||||
- count: 1
|
||||
selector: MyStore
|
||||
containerBackupFactor: 1
|
||||
selectors:
|
||||
- name: MyStore
|
||||
count: 2
|
||||
clause: DISTINCT
|
||||
attribute: Country
|
||||
filter: FromRU
|
||||
filters:
|
||||
- name: FromRU
|
||||
key: Country
|
||||
op: EQ
|
||||
value: Russia
|
||||
filters: []
|
||||
result:
|
||||
- - 0
|
66
netmap/yml_tests/rep_only.yml
Normal file
66
netmap/yml_tests/rep_only.yml
Normal file
|
@ -0,0 +1,66 @@
|
|||
name: REP X
|
||||
nodes:
|
||||
- publicKey: ''
|
||||
addresses: []
|
||||
attributes:
|
||||
- key: City
|
||||
value: Saint-Petersburg
|
||||
parents: []
|
||||
state: UNSPECIFIED
|
||||
- publicKey: ''
|
||||
addresses: []
|
||||
attributes:
|
||||
- key: City
|
||||
value: Moscow
|
||||
parents: []
|
||||
state: UNSPECIFIED
|
||||
- publicKey: ''
|
||||
addresses: []
|
||||
attributes:
|
||||
- key: City
|
||||
value: Berlin
|
||||
parents: []
|
||||
state: UNSPECIFIED
|
||||
- publicKey: ''
|
||||
addresses: []
|
||||
attributes:
|
||||
- key: City
|
||||
value: Paris
|
||||
parents: []
|
||||
state: UNSPECIFIED
|
||||
tests:
|
||||
REP 1:
|
||||
policy:
|
||||
replicas:
|
||||
- count: 1
|
||||
containerBackupFactor: 0
|
||||
selectors: []
|
||||
filters: []
|
||||
result:
|
||||
- - 0
|
||||
- 1
|
||||
- 2
|
||||
REP 3:
|
||||
policy:
|
||||
replicas:
|
||||
- count: 3
|
||||
containerBackupFactor: 0
|
||||
selectors: []
|
||||
filters: []
|
||||
result:
|
||||
- - 0
|
||||
- 3
|
||||
- 1
|
||||
- 2
|
||||
REP 5:
|
||||
policy:
|
||||
replicas:
|
||||
- count: 5
|
||||
containerBackupFactor: 0
|
||||
selectors: []
|
||||
filters: []
|
||||
result:
|
||||
- - 0
|
||||
- 1
|
||||
- 2
|
||||
- 3
|
56
netmap/yml_tests/select_no_attribute.yml
Normal file
56
netmap/yml_tests/select_no_attribute.yml
Normal file
|
@ -0,0 +1,56 @@
|
|||
name: select with unspecified attribute
|
||||
nodes:
|
||||
- attributes:
|
||||
- key: ID
|
||||
value: '1'
|
||||
- key: Country
|
||||
value: RU
|
||||
- key: City
|
||||
value: St.Petersburg
|
||||
- key: SSD
|
||||
value: '0'
|
||||
- attributes:
|
||||
- key: ID
|
||||
value: '2'
|
||||
- key: Country
|
||||
value: RU
|
||||
- key: City
|
||||
value: St.Petersburg
|
||||
- key: SSD
|
||||
value: '1'
|
||||
- attributes:
|
||||
- key: ID
|
||||
value: '3'
|
||||
- key: Country
|
||||
value: RU
|
||||
- key: City
|
||||
value: Moscow
|
||||
- key: SSD
|
||||
value: '1'
|
||||
- attributes:
|
||||
- key: ID
|
||||
value: '4'
|
||||
- key: Country
|
||||
value: RU
|
||||
- key: City
|
||||
value: Moscow
|
||||
- key: SSD
|
||||
value: '1'
|
||||
tests:
|
||||
test:
|
||||
policy:
|
||||
replicas:
|
||||
- count: 1
|
||||
selector: X
|
||||
containerBackupFactor: 1
|
||||
selectors:
|
||||
- name: X
|
||||
count: 4
|
||||
clause: DISTINCT
|
||||
filter: '*'
|
||||
filters: []
|
||||
result:
|
||||
- - 0
|
||||
- 1
|
||||
- 2
|
||||
- 3
|
48
netmap/yml_tests/selector_invalid.yml
Normal file
48
netmap/yml_tests/selector_invalid.yml
Normal file
|
@ -0,0 +1,48 @@
|
|||
name: invalid selections
|
||||
nodes:
|
||||
- attributes:
|
||||
- key: Country
|
||||
value: Russia
|
||||
- attributes:
|
||||
- key: Country
|
||||
value: Germany
|
||||
- attributes: []
|
||||
tests:
|
||||
missing filter:
|
||||
policy:
|
||||
replicas:
|
||||
- count: 1
|
||||
selector: MyStore
|
||||
containerBackupFactor: 1
|
||||
selectors:
|
||||
- name: MyStore
|
||||
count: 1
|
||||
clause: DISTINCT
|
||||
attribute: Country
|
||||
filter: FromNL
|
||||
filters:
|
||||
- name: FromRU
|
||||
key: Country
|
||||
op: EQ
|
||||
value: Russia
|
||||
filters: []
|
||||
error: filter not found
|
||||
not enough nodes (filter results in empty set):
|
||||
policy:
|
||||
replicas:
|
||||
- count: 1
|
||||
selector: MyStore
|
||||
containerBackupFactor: 2
|
||||
selectors:
|
||||
- name: MyStore
|
||||
count: 2
|
||||
clause: DISTINCT
|
||||
attribute: Country
|
||||
filter: FromMoon
|
||||
filters:
|
||||
- name: FromMoon
|
||||
key: Country
|
||||
op: EQ
|
||||
value: Moon
|
||||
filters: []
|
||||
error: not enough nodes
|
142
netmap/yml_unmarshal.go
Normal file
142
netmap/yml_unmarshal.go
Normal file
|
@ -0,0 +1,142 @@
|
|||
package netmap
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap"
|
||||
)
|
||||
|
||||
type tempPlacementPolicy struct {
|
||||
BackupFactor uint32 `yaml:"containerBackupFactor"`
|
||||
Filters []tempFilter `yaml:"filters"`
|
||||
Selectors []tempSelector `yaml:"selectors"`
|
||||
Replicas []tempReplica `yaml:"replicas"`
|
||||
Unique bool `yaml:"unique"`
|
||||
}
|
||||
|
||||
type tempFilter struct {
|
||||
Name string `yaml:"name"`
|
||||
Key string `yaml:"key"`
|
||||
Op string `yaml:"op"`
|
||||
Value string `yaml:"value"`
|
||||
Filters []tempFilter `yaml:"filters"`
|
||||
}
|
||||
|
||||
type tempSelector struct {
|
||||
Name string `yaml:"name"`
|
||||
Count uint32 `yaml:"count"`
|
||||
Clause string `yaml:"clause"`
|
||||
Attribute string `yaml:"attribute"`
|
||||
Filter string `yaml:"filter"`
|
||||
}
|
||||
|
||||
type tempReplica struct {
|
||||
Count uint32 `yaml:"count"`
|
||||
Selector string `yaml:"selector"`
|
||||
}
|
||||
|
||||
func convertNFilters(temp []tempFilter) []netmap.Filter {
|
||||
var filters []netmap.Filter
|
||||
for _, tf := range temp {
|
||||
filters = append(filters, convertNFilter(tf))
|
||||
}
|
||||
return filters
|
||||
}
|
||||
|
||||
var stringToOperationMap = map[string]netmap.Operation{
|
||||
"EQ": netmap.EQ,
|
||||
"NE": netmap.NE,
|
||||
"GT": netmap.GT,
|
||||
"GE": netmap.GE,
|
||||
"LT": netmap.LT,
|
||||
"LE": netmap.LE,
|
||||
"OR": netmap.OR,
|
||||
"AND": netmap.AND,
|
||||
"NOT": netmap.NOT,
|
||||
}
|
||||
|
||||
func convertStringToOperation(opStr string) netmap.Operation {
|
||||
opStr = strings.ToUpper(opStr)
|
||||
if op, exists := stringToOperationMap[opStr]; exists {
|
||||
return op
|
||||
}
|
||||
return netmap.UnspecifiedOperation
|
||||
}
|
||||
|
||||
func convertStringToClause(clauseStr string) netmap.Clause {
|
||||
switch strings.ToUpper(clauseStr) {
|
||||
case "DISTINCT":
|
||||
return netmap.Distinct
|
||||
default:
|
||||
return netmap.Same
|
||||
}
|
||||
}
|
||||
|
||||
func convertNFilter(temp tempFilter) netmap.Filter {
|
||||
filter := netmap.Filter{}
|
||||
filter.SetKey(temp.Key)
|
||||
filter.SetName(temp.Name)
|
||||
filter.SetValue(temp.Value)
|
||||
filter.SetOp(convertStringToOperation(temp.Op))
|
||||
|
||||
if temp.Filters != nil {
|
||||
filter.SetFilters(convertNFilters(temp.Filters))
|
||||
}
|
||||
return filter
|
||||
}
|
||||
|
||||
func (p *PlacementPolicy) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
||||
var temp tempPlacementPolicy
|
||||
if err := unmarshal(&temp); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, ts := range temp.Filters {
|
||||
netmapFilters := convertNFilter(ts)
|
||||
p.AddFilters(Filter{m: netmapFilters})
|
||||
}
|
||||
|
||||
for _, ts := range temp.Selectors {
|
||||
selector := Selector{}
|
||||
selector.SetName(ts.Name)
|
||||
selector.SetNumberOfNodes(ts.Count)
|
||||
selector.SetClause(convertStringToClause(ts.Clause))
|
||||
selector.SelectByBucketAttribute(ts.Attribute)
|
||||
selector.SetFilterName(ts.Filter)
|
||||
p.AddSelectors(selector)
|
||||
}
|
||||
|
||||
for _, tr := range temp.Replicas {
|
||||
replica := ReplicaDescriptor{}
|
||||
replica.SetSelectorName(tr.Selector)
|
||||
replica.m.SetCount(tr.Count)
|
||||
p.AddReplicas(replica)
|
||||
}
|
||||
|
||||
p.SetContainerBackupFactor(temp.BackupFactor)
|
||||
p.SetUnique(temp.Unique)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type Attribute struct {
|
||||
Key string `yaml:"key"`
|
||||
Value string `yaml:"value"`
|
||||
}
|
||||
|
||||
type tempNode struct {
|
||||
Attributes []Attribute `yaml:"attributes"`
|
||||
}
|
||||
|
||||
func (x *NodeInfo) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
||||
var temp tempNode
|
||||
if err := unmarshal(&temp); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, atr := range temp.Attributes {
|
||||
x.SetAttribute(atr.Key, atr.Value)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
69
object/ecinfo.go
Normal file
69
object/ecinfo.go
Normal file
|
@ -0,0 +1,69 @@
|
|||
package object
|
||||
|
||||
import (
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
)
|
||||
|
||||
type ECChunk object.ECChunk
|
||||
|
||||
func (c *ECChunk) SetID(id oid.ID) {
|
||||
objV2 := new(refs.ObjectID)
|
||||
id.WriteToV2(objV2)
|
||||
c.ID = *objV2
|
||||
}
|
||||
|
||||
// ToV2 converts ECChunk to v2 ECChunk message.
|
||||
//
|
||||
// Nil ECChunk converts to nil.
|
||||
func (c *ECChunk) ToV2() *object.ECChunk {
|
||||
return (*object.ECChunk)(c)
|
||||
}
|
||||
|
||||
func NewECChunkFromV2(v2 *object.ECChunk) *ECChunk {
|
||||
return (*ECChunk)(v2)
|
||||
}
|
||||
|
||||
type ECInfo object.ECInfo
|
||||
|
||||
// NewECInfoFromV2 wraps v2 ECInfo message to ECInfo.
|
||||
//
|
||||
// Nil object.ECInfo converts to nil.
|
||||
func NewECInfoFromV2(v2 *object.ECInfo) *ECInfo {
|
||||
return (*ECInfo)(v2)
|
||||
}
|
||||
|
||||
// NewECInfo creates and initializes blank ECInfo.
|
||||
func NewECInfo() *ECInfo {
|
||||
return NewECInfoFromV2(new(object.ECInfo))
|
||||
}
|
||||
|
||||
// ToV2 converts ECInfo to v2 ECInfo message.
|
||||
//
|
||||
// Nil ECInfo converts to nil.
|
||||
func (s *ECInfo) ToV2() *object.ECInfo {
|
||||
return (*object.ECInfo)(s)
|
||||
}
|
||||
|
||||
func (s *ECInfo) Marshal() ([]byte, error) {
|
||||
return (*object.ECInfo)(s).StableMarshal(nil), nil
|
||||
}
|
||||
|
||||
func (s *ECInfo) Unmarshal(data []byte) error {
|
||||
return (*object.ECInfo)(s).Unmarshal(data)
|
||||
}
|
||||
|
||||
// MarshalJSON implements json.Marshaler.
|
||||
func (s *ECInfo) MarshalJSON() ([]byte, error) {
|
||||
return (*object.ECInfo)(s).MarshalJSON()
|
||||
}
|
||||
|
||||
// UnmarshalJSON implements json.Unmarshaler.
|
||||
func (s *ECInfo) UnmarshalJSON(data []byte) error {
|
||||
return (*object.ECInfo)(s).UnmarshalJSON(data)
|
||||
}
|
||||
|
||||
func (s *ECInfo) AddChunk(chunk ECChunk) {
|
||||
s.Chunks = append(s.Chunks, *chunk.ToV2())
|
||||
}
|
121
object/erasure_code.go
Normal file
121
object/erasure_code.go
Normal file
|
@ -0,0 +1,121 @@
|
|||
package object
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object"
|
||||
refs "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs"
|
||||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
)
|
||||
|
||||
// ECHeader represents erasure coding header.
|
||||
type ECHeader struct {
|
||||
parent oid.ID
|
||||
index uint32
|
||||
total uint32
|
||||
header []byte
|
||||
headerLength uint32
|
||||
}
|
||||
|
||||
// NewECHeader constructs new erasure coding header.
|
||||
func NewECHeader(parent oid.ID, index, total uint32, header []byte, headerLength uint32) *ECHeader {
|
||||
return &ECHeader{
|
||||
parent: parent,
|
||||
index: index,
|
||||
total: total,
|
||||
header: header,
|
||||
headerLength: headerLength,
|
||||
}
|
||||
}
|
||||
|
||||
// WriteToV2 converts SDK structure to v2-api one.
|
||||
func (e *ECHeader) WriteToV2(h *object.ECHeader) {
|
||||
var parent refs.ObjectID
|
||||
e.parent.WriteToV2(&parent)
|
||||
h.Parent = &parent
|
||||
h.Index = e.index
|
||||
h.Total = e.total
|
||||
h.Header = e.header
|
||||
h.HeaderLength = e.headerLength
|
||||
}
|
||||
|
||||
// ReadFromV2 converts v2-api structure to SDK one.
|
||||
func (e *ECHeader) ReadFromV2(h *object.ECHeader) error {
|
||||
if h == nil {
|
||||
return nil
|
||||
}
|
||||
if h.Parent == nil {
|
||||
return errors.New("empty parent")
|
||||
}
|
||||
|
||||
_ = e.parent.ReadFromV2(*h.Parent)
|
||||
e.index = h.Index
|
||||
e.total = h.Total
|
||||
e.header = h.Header
|
||||
e.headerLength = h.HeaderLength
|
||||
return nil
|
||||
}
|
||||
|
||||
func (o *Object) ECHeader() *ECHeader {
|
||||
ec := (*object.Object)(o).GetHeader().GetEC()
|
||||
if ec == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
h := new(ECHeader)
|
||||
_ = h.ReadFromV2(ec)
|
||||
return h
|
||||
}
|
||||
|
||||
func (o *Object) SetECHeader(ec *ECHeader) {
|
||||
o.setHeaderField(func(h *object.Header) {
|
||||
if ec == nil {
|
||||
h.SetEC(nil)
|
||||
return
|
||||
}
|
||||
|
||||
v2 := new(object.ECHeader)
|
||||
ec.WriteToV2(v2)
|
||||
h.SetEC(v2)
|
||||
})
|
||||
}
|
||||
|
||||
func (e *ECHeader) Parent() oid.ID {
|
||||
return e.parent
|
||||
}
|
||||
|
||||
func (e *ECHeader) SetParent(id oid.ID) {
|
||||
e.parent = id
|
||||
}
|
||||
|
||||
func (e *ECHeader) Index() uint32 {
|
||||
return e.index
|
||||
}
|
||||
|
||||
func (e *ECHeader) SetIndex(i uint32) {
|
||||
e.index = i
|
||||
}
|
||||
|
||||
func (e *ECHeader) Total() uint32 {
|
||||
return e.total
|
||||
}
|
||||
|
||||
func (e *ECHeader) SetTotal(i uint32) {
|
||||
e.total = i
|
||||
}
|
||||
|
||||
func (e *ECHeader) Header() []byte {
|
||||
return e.header
|
||||
}
|
||||
|
||||
func (e *ECHeader) SetHeader(header []byte) {
|
||||
e.header = header
|
||||
}
|
||||
|
||||
func (e *ECHeader) HeaderLength() uint32 {
|
||||
return e.headerLength
|
||||
}
|
||||
|
||||
func (e *ECHeader) SetHeaderLength(l uint32) {
|
||||
e.headerLength = l
|
||||
}
|
87
object/erasurecode/constructor.go
Normal file
87
object/erasurecode/constructor.go
Normal file
|
@ -0,0 +1,87 @@
|
|||
package erasurecode
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
"github.com/klauspost/reedsolomon"
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrMalformedSlice is returned when a slice of EC chunks is inconsistent.
|
||||
ErrMalformedSlice = errors.New("inconsistent EC headers")
|
||||
// ErrInvShardNum is returned from NewConstructor when the number of shards is invalid.
|
||||
ErrInvShardNum = reedsolomon.ErrInvShardNum
|
||||
// ErrMaxShardNum is returned from NewConstructor when the number of shards is too big.
|
||||
ErrMaxShardNum = reedsolomon.ErrMaxShardNum
|
||||
)
|
||||
|
||||
// MaxShardCount is the maximum number of shards.
|
||||
const MaxShardCount = 256
|
||||
|
||||
// Constructor is a wrapper around encoder allowing to reconstruct objects.
|
||||
// It's methods are not thread-safe.
|
||||
type Constructor struct {
|
||||
enc reedsolomon.Encoder
|
||||
headerLength uint32
|
||||
payloadShards [][]byte
|
||||
headerShards [][]byte
|
||||
}
|
||||
|
||||
// NewConstructor returns new constructor instance.
|
||||
func NewConstructor(dataCount int, parityCount int) (*Constructor, error) {
|
||||
// The library supports up to 65536 shards with some restrictions.
|
||||
// This can easily result in OOM or panic, thus SDK declares it's own restriction.
|
||||
if dataCount+parityCount > MaxShardCount {
|
||||
return nil, ErrMaxShardNum
|
||||
}
|
||||
|
||||
enc, err := reedsolomon.New(dataCount, parityCount)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Constructor{enc: enc}, nil
|
||||
}
|
||||
|
||||
// clear clears internal state of the constructor, so it can be reused.
|
||||
func (c *Constructor) clear() {
|
||||
c.headerLength = 0
|
||||
c.payloadShards = nil
|
||||
c.headerShards = nil
|
||||
}
|
||||
|
||||
func (c *Constructor) fillHeader(parts []*objectSDK.Object) error {
|
||||
shards := make([][]byte, len(parts))
|
||||
headerLength := 0
|
||||
for i := range parts {
|
||||
if parts[i] == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
var err error
|
||||
headerLength, err = validatePart(parts, i, headerLength)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
shards[i] = parts[i].GetECHeader().Header()
|
||||
}
|
||||
|
||||
c.headerLength = uint32(headerLength)
|
||||
c.headerShards = shards
|
||||
return nil
|
||||
}
|
||||
|
||||
// fillPayload fills the payload shards.
|
||||
// Currently there is no case when it can be called without reconstructing header,
|
||||
// thus fillHeader() must be called before and this function performs no validation.
|
||||
func (c *Constructor) fillPayload(parts []*objectSDK.Object) {
|
||||
shards := make([][]byte, len(parts))
|
||||
for i := range parts {
|
||||
if parts[i] == nil {
|
||||
continue
|
||||
}
|
||||
shards[i] = parts[i].Payload()
|
||||
}
|
||||
c.payloadShards = shards
|
||||
}
|
31
object/erasurecode/constructor_test.go
Normal file
31
object/erasurecode/constructor_test.go
Normal file
|
@ -0,0 +1,31 @@
|
|||
package erasurecode_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/erasurecode"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestErasureConstruct(t *testing.T) {
|
||||
t.Run("negative, no panic", func(t *testing.T) {
|
||||
_, err := erasurecode.NewConstructor(-1, 2)
|
||||
require.ErrorIs(t, err, erasurecode.ErrInvShardNum)
|
||||
})
|
||||
t.Run("negative, no panic", func(t *testing.T) {
|
||||
_, err := erasurecode.NewConstructor(2, -1)
|
||||
require.ErrorIs(t, err, erasurecode.ErrInvShardNum)
|
||||
})
|
||||
t.Run("zero parity", func(t *testing.T) {
|
||||
_, err := erasurecode.NewConstructor(1, 0)
|
||||
require.NoError(t, err)
|
||||
})
|
||||
t.Run("max shard num", func(t *testing.T) {
|
||||
_, err := erasurecode.NewConstructor(erasurecode.MaxShardCount, 0)
|
||||
require.NoError(t, err)
|
||||
})
|
||||
t.Run("max+1 shard num", func(t *testing.T) {
|
||||
_, err := erasurecode.NewConstructor(erasurecode.MaxShardCount+1, 0)
|
||||
require.ErrorIs(t, err, erasurecode.ErrMaxShardNum)
|
||||
})
|
||||
}
|
142
object/erasurecode/reconstruct.go
Normal file
142
object/erasurecode/reconstruct.go
Normal file
|
@ -0,0 +1,142 @@
|
|||
package erasurecode
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/ecdsa"
|
||||
"fmt"
|
||||
|
||||
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
"github.com/klauspost/reedsolomon"
|
||||
)
|
||||
|
||||
// Reconstruct returns full object reconstructed from parts.
|
||||
// All non-nil objects in parts must have EC header with the same `total` field equal to len(parts).
|
||||
// The slice must contain at least one non nil object.
|
||||
// Index of the objects in parts must be equal to it's index field in the EC header.
|
||||
// The parts slice isn't changed and can be used concurrently for reading.
|
||||
func (c *Constructor) Reconstruct(parts []*objectSDK.Object) (*objectSDK.Object, error) {
|
||||
res, err := c.ReconstructHeader(parts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
c.fillPayload(parts)
|
||||
|
||||
payload, err := reconstructExact(c.enc, int(res.PayloadSize()), c.payloadShards)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrMalformedSlice, err)
|
||||
}
|
||||
|
||||
res.SetPayload(payload)
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// ReconstructHeader returns object header reconstructed from parts.
|
||||
// All non-nil objects in parts must have EC header with the same `total` field equal to len(parts).
|
||||
// The slice must contain at least one non nil object.
|
||||
// Index of the objects in parts must be equal to it's index field in the EC header.
|
||||
// The parts slice isn't changed and can be used concurrently for reading.
|
||||
func (c *Constructor) ReconstructHeader(parts []*objectSDK.Object) (*objectSDK.Object, error) {
|
||||
c.clear()
|
||||
|
||||
if err := c.fillHeader(parts); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
obj, err := c.reconstructHeader()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrMalformedSlice, err)
|
||||
}
|
||||
return obj, nil
|
||||
}
|
||||
|
||||
// ReconstructParts reconstructs specific EC parts without reconstructing full object.
|
||||
// All non-nil objects in parts must have EC header with the same `total` field equal to len(parts).
|
||||
// The slice must contain at least one non nil object.
|
||||
// Index of the objects in parts must be equal to it's index field in the EC header.
|
||||
// Those parts for which corresponding element in required is true must be nil and will be overwritten.
|
||||
// Because partial reconstruction only makes sense for full objects, all parts must have non-empty payload.
|
||||
// If key is not nil, all reconstructed parts are signed with this key.
|
||||
func (c *Constructor) ReconstructParts(parts []*objectSDK.Object, required []bool, key *ecdsa.PrivateKey) error {
|
||||
if len(required) != len(parts) {
|
||||
return fmt.Errorf("len(parts) != len(required): %d != %d", len(parts), len(required))
|
||||
}
|
||||
|
||||
c.clear()
|
||||
|
||||
if err := c.fillHeader(parts); err != nil {
|
||||
return err
|
||||
}
|
||||
c.fillPayload(parts)
|
||||
|
||||
if err := c.enc.ReconstructSome(c.payloadShards, required); err != nil {
|
||||
return fmt.Errorf("%w: %w", ErrMalformedSlice, err)
|
||||
}
|
||||
if err := c.enc.ReconstructSome(c.headerShards, required); err != nil {
|
||||
return fmt.Errorf("%w: %w", ErrMalformedSlice, err)
|
||||
}
|
||||
|
||||
nonNilPart := 0
|
||||
for i := range parts {
|
||||
if parts[i] != nil {
|
||||
nonNilPart = i
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
ec := parts[nonNilPart].GetECHeader()
|
||||
parent := ec.Parent()
|
||||
total := ec.Total()
|
||||
|
||||
for i := range required {
|
||||
if parts[i] != nil || !required[i] {
|
||||
continue
|
||||
}
|
||||
|
||||
part := objectSDK.New()
|
||||
copyRequiredFields(part, parts[nonNilPart])
|
||||
part.SetPayload(c.payloadShards[i])
|
||||
part.SetPayloadSize(uint64(len(c.payloadShards[i])))
|
||||
part.SetECHeader(objectSDK.NewECHeader(parent, uint32(i), total,
|
||||
c.headerShards[i], c.headerLength))
|
||||
|
||||
if err := setIDWithSignature(part, key); err != nil {
|
||||
return err
|
||||
}
|
||||
parts[i] = part
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Constructor) reconstructHeader() (*objectSDK.Object, error) {
|
||||
data, err := reconstructExact(c.enc, int(c.headerLength), c.headerShards)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var obj objectSDK.Object
|
||||
return &obj, obj.Unmarshal(data)
|
||||
}
|
||||
|
||||
func reconstructExact(enc reedsolomon.Encoder, size int, shards [][]byte) ([]byte, error) {
|
||||
if err := enc.ReconstructData(shards); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Technically, this error will be returned from enc.Join().
|
||||
// However, allocating based on unvalidated user data is an easy attack vector.
|
||||
// Preallocating seems to have enough benefits to justify a slight increase in code complexity.
|
||||
maxSize := 0
|
||||
for i := range shards {
|
||||
maxSize += len(shards[i])
|
||||
}
|
||||
if size > maxSize {
|
||||
return nil, reedsolomon.ErrShortData
|
||||
}
|
||||
|
||||
buf := bytes.NewBuffer(make([]byte, 0, size))
|
||||
if err := enc.Join(buf, shards, size); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return buf.Bytes(), nil
|
||||
}
|
284
object/erasurecode/reconstruct_test.go
Normal file
284
object/erasurecode/reconstruct_test.go
Normal file
|
@ -0,0 +1,284 @@
|
|||
package erasurecode_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"math"
|
||||
"testing"
|
||||
|
||||
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
|
||||
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/erasurecode"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version"
|
||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestErasureCodeReconstruct(t *testing.T) {
|
||||
const payloadSize = 99
|
||||
const dataCount = 3
|
||||
const parityCount = 2
|
||||
|
||||
// We would also like to test padding behaviour,
|
||||
// so ensure padding is done.
|
||||
require.NotZero(t, payloadSize%(dataCount+parityCount))
|
||||
|
||||
pk, err := keys.NewPrivateKey()
|
||||
require.NoError(t, err)
|
||||
|
||||
original := newObject(t, payloadSize, pk)
|
||||
|
||||
c, err := erasurecode.NewConstructor(dataCount, parityCount)
|
||||
require.NoError(t, err)
|
||||
|
||||
parts, err := c.Split(original, &pk.PrivateKey)
|
||||
require.NoError(t, err)
|
||||
|
||||
t.Run("reconstruct header", func(t *testing.T) {
|
||||
original := original.CutPayload()
|
||||
parts := cloneSlice(parts)
|
||||
for i := range parts {
|
||||
parts[i] = parts[i].CutPayload()
|
||||
}
|
||||
t.Run("from data", func(t *testing.T) {
|
||||
parts := cloneSlice(parts)
|
||||
for i := dataCount; i < dataCount+parityCount; i++ {
|
||||
parts[i] = nil
|
||||
}
|
||||
reconstructed, err := c.ReconstructHeader(parts)
|
||||
require.NoError(t, err)
|
||||
verifyReconstruction(t, original, reconstructed)
|
||||
})
|
||||
t.Run("from parity", func(t *testing.T) {
|
||||
parts := cloneSlice(parts)
|
||||
for i := 0; i < parityCount; i++ {
|
||||
parts[i] = nil
|
||||
}
|
||||
reconstructed, err := c.ReconstructHeader(parts)
|
||||
require.NoError(t, err)
|
||||
verifyReconstruction(t, original, reconstructed)
|
||||
|
||||
t.Run("not enough shards", func(t *testing.T) {
|
||||
parts[parityCount] = nil
|
||||
_, err := c.ReconstructHeader(parts)
|
||||
require.ErrorIs(t, err, erasurecode.ErrMalformedSlice)
|
||||
})
|
||||
})
|
||||
t.Run("only nil parts", func(t *testing.T) {
|
||||
parts := make([]*objectSDK.Object, len(parts))
|
||||
_, err := c.ReconstructHeader(parts)
|
||||
require.ErrorIs(t, err, erasurecode.ErrMalformedSlice)
|
||||
})
|
||||
t.Run("missing EC header", func(t *testing.T) {
|
||||
parts := cloneSlice(parts)
|
||||
parts[0] = deepCopy(t, parts[0])
|
||||
parts[0].SetECHeader(nil)
|
||||
|
||||
_, err := c.ReconstructHeader(parts)
|
||||
require.ErrorIs(t, err, erasurecode.ErrMalformedSlice)
|
||||
})
|
||||
t.Run("invalid index", func(t *testing.T) {
|
||||
parts := cloneSlice(parts)
|
||||
parts[0] = deepCopy(t, parts[0])
|
||||
|
||||
ec := parts[0].GetECHeader()
|
||||
ec.SetIndex(1)
|
||||
parts[0].SetECHeader(ec)
|
||||
|
||||
_, err := c.ReconstructHeader(parts)
|
||||
require.ErrorIs(t, err, erasurecode.ErrMalformedSlice)
|
||||
})
|
||||
t.Run("invalid total", func(t *testing.T) {
|
||||
parts := cloneSlice(parts)
|
||||
parts[0] = deepCopy(t, parts[0])
|
||||
|
||||
ec := parts[0].GetECHeader()
|
||||
ec.SetTotal(uint32(len(parts) + 1))
|
||||
parts[0].SetECHeader(ec)
|
||||
|
||||
_, err := c.ReconstructHeader(parts)
|
||||
require.ErrorIs(t, err, erasurecode.ErrMalformedSlice)
|
||||
})
|
||||
t.Run("inconsistent header length", func(t *testing.T) {
|
||||
parts := cloneSlice(parts)
|
||||
parts[0] = deepCopy(t, parts[0])
|
||||
|
||||
ec := parts[0].GetECHeader()
|
||||
ec.SetHeaderLength(ec.HeaderLength() - 1)
|
||||
parts[0].SetECHeader(ec)
|
||||
|
||||
_, err := c.ReconstructHeader(parts)
|
||||
require.ErrorIs(t, err, erasurecode.ErrMalformedSlice)
|
||||
})
|
||||
t.Run("invalid header length", func(t *testing.T) {
|
||||
parts := cloneSlice(parts)
|
||||
for i := range parts {
|
||||
parts[i] = deepCopy(t, parts[i])
|
||||
|
||||
ec := parts[0].GetECHeader()
|
||||
ec.SetHeaderLength(math.MaxUint32)
|
||||
parts[0].SetECHeader(ec)
|
||||
}
|
||||
|
||||
_, err := c.ReconstructHeader(parts)
|
||||
require.ErrorIs(t, err, erasurecode.ErrMalformedSlice)
|
||||
})
|
||||
})
|
||||
t.Run("reconstruct data", func(t *testing.T) {
|
||||
t.Run("from data", func(t *testing.T) {
|
||||
parts := cloneSlice(parts)
|
||||
for i := dataCount; i < dataCount+parityCount; i++ {
|
||||
parts[i] = nil
|
||||
}
|
||||
reconstructed, err := c.Reconstruct(parts)
|
||||
require.NoError(t, err)
|
||||
verifyReconstruction(t, original, reconstructed)
|
||||
})
|
||||
t.Run("from parity", func(t *testing.T) {
|
||||
parts := cloneSlice(parts)
|
||||
for i := 0; i < parityCount; i++ {
|
||||
parts[i] = nil
|
||||
}
|
||||
reconstructed, err := c.Reconstruct(parts)
|
||||
require.NoError(t, err)
|
||||
verifyReconstruction(t, original, reconstructed)
|
||||
|
||||
t.Run("not enough shards", func(t *testing.T) {
|
||||
parts[parityCount] = nil
|
||||
_, err := c.Reconstruct(parts)
|
||||
require.ErrorIs(t, err, erasurecode.ErrMalformedSlice)
|
||||
})
|
||||
})
|
||||
})
|
||||
t.Run("reconstruct parts", func(t *testing.T) {
|
||||
// We would like to also test that ReconstructParts doesn't perform
|
||||
// excessive work, so ensure this test makes sense.
|
||||
require.GreaterOrEqual(t, parityCount, 2)
|
||||
|
||||
t.Run("from data", func(t *testing.T) {
|
||||
oldParts := parts
|
||||
parts := cloneSlice(parts)
|
||||
for i := dataCount; i < dataCount+parityCount; i++ {
|
||||
parts[i] = nil
|
||||
}
|
||||
|
||||
required := make([]bool, len(parts))
|
||||
required[dataCount] = true
|
||||
|
||||
require.NoError(t, c.ReconstructParts(parts, required, nil))
|
||||
|
||||
old := deepCopy(t, oldParts[dataCount])
|
||||
old.SetSignature(nil)
|
||||
require.Equal(t, old, parts[dataCount])
|
||||
|
||||
for i := dataCount + 1; i < dataCount+parityCount; i++ {
|
||||
require.Nil(t, parts[i])
|
||||
}
|
||||
})
|
||||
t.Run("from parity", func(t *testing.T) {
|
||||
oldParts := parts
|
||||
parts := cloneSlice(parts)
|
||||
for i := 0; i < parityCount; i++ {
|
||||
parts[i] = nil
|
||||
}
|
||||
|
||||
required := make([]bool, len(parts))
|
||||
required[0] = true
|
||||
|
||||
require.NoError(t, c.ReconstructParts(parts, required, nil))
|
||||
|
||||
old := deepCopy(t, oldParts[0])
|
||||
old.SetSignature(nil)
|
||||
require.Equal(t, old, parts[0])
|
||||
|
||||
for i := 1; i < parityCount; i++ {
|
||||
require.Nil(t, parts[i])
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func newObject(t *testing.T, size uint64, pk *keys.PrivateKey) *objectSDK.Object {
|
||||
// Use transformer to form object to avoid potential bugs with yet another helper object creation in tests.
|
||||
tt := &testTarget{}
|
||||
p := transformer.NewPayloadSizeLimiter(transformer.Params{
|
||||
Key: &pk.PrivateKey,
|
||||
NextTargetInit: func() transformer.ObjectWriter { return tt },
|
||||
NetworkState: dummyEpochSource(123),
|
||||
MaxSize: size + 1,
|
||||
WithoutHomomorphicHash: true,
|
||||
})
|
||||
cnr := cidtest.ID()
|
||||
ver := version.Current()
|
||||
hdr := objectSDK.New()
|
||||
hdr.SetContainerID(cnr)
|
||||
hdr.SetType(objectSDK.TypeRegular)
|
||||
hdr.SetVersion(&ver)
|
||||
|
||||
var owner user.ID
|
||||
user.IDFromKey(&owner, pk.PrivateKey.PublicKey)
|
||||
hdr.SetOwnerID(owner)
|
||||
|
||||
var attr objectSDK.Attribute
|
||||
attr.SetKey("somekey")
|
||||
attr.SetValue("somevalue")
|
||||
hdr.SetAttributes(attr)
|
||||
|
||||
expectedPayload := make([]byte, size)
|
||||
_, _ = rand.Read(expectedPayload)
|
||||
writeObject(t, context.Background(), p, hdr, expectedPayload)
|
||||
require.Len(t, tt.objects, 1)
|
||||
return tt.objects[0]
|
||||
}
|
||||
|
||||
func writeObject(t *testing.T, ctx context.Context, target transformer.ChunkedObjectWriter, header *objectSDK.Object, payload []byte) *transformer.AccessIdentifiers {
|
||||
require.NoError(t, target.WriteHeader(ctx, header))
|
||||
|
||||
_, err := target.Write(ctx, payload)
|
||||
require.NoError(t, err)
|
||||
|
||||
ids, err := target.Close(ctx)
|
||||
require.NoError(t, err)
|
||||
|
||||
return ids
|
||||
}
|
||||
|
||||
func verifyReconstruction(t *testing.T, original, reconstructed *objectSDK.Object) {
|
||||
require.True(t, reconstructed.VerifyIDSignature())
|
||||
reconstructed.ToV2().SetMarshalData(nil)
|
||||
original.ToV2().SetMarshalData(nil)
|
||||
|
||||
require.Equal(t, original, reconstructed)
|
||||
}
|
||||
|
||||
func deepCopy(t *testing.T, obj *objectSDK.Object) *objectSDK.Object {
|
||||
data, err := obj.Marshal()
|
||||
require.NoError(t, err)
|
||||
|
||||
res := objectSDK.New()
|
||||
require.NoError(t, res.Unmarshal(data))
|
||||
return res
|
||||
}
|
||||
|
||||
func cloneSlice[T any](src []T) []T {
|
||||
dst := make([]T, len(src))
|
||||
copy(dst, src)
|
||||
return dst
|
||||
}
|
||||
|
||||
type dummyEpochSource uint64
|
||||
|
||||
func (s dummyEpochSource) CurrentEpoch() uint64 {
|
||||
return uint64(s)
|
||||
}
|
||||
|
||||
type testTarget struct {
|
||||
objects []*objectSDK.Object
|
||||
}
|
||||
|
||||
func (tt *testTarget) WriteObject(_ context.Context, o *objectSDK.Object) error {
|
||||
tt.objects = append(tt.objects, o)
|
||||
return nil // AccessIdentifiers should not be used.
|
||||
}
|
76
object/erasurecode/split.go
Normal file
76
object/erasurecode/split.go
Normal file
|
@ -0,0 +1,76 @@
|
|||
package erasurecode
|
||||
|
||||
import (
|
||||
"crypto/ecdsa"
|
||||
|
||||
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
)
|
||||
|
||||
// Split splits fully formed object into multiple chunks.
|
||||
func (c *Constructor) Split(obj *objectSDK.Object, key *ecdsa.PrivateKey) ([]*objectSDK.Object, error) {
|
||||
c.clear()
|
||||
|
||||
header, err := obj.CutPayload().Marshal()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
headerShards, err := c.encodeRaw(header)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
payloadShards, err := c.encodeRaw(obj.Payload())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
parts := make([]*objectSDK.Object, len(payloadShards))
|
||||
parent, _ := obj.ID()
|
||||
for i := range parts {
|
||||
chunk := objectSDK.New()
|
||||
copyRequiredFields(chunk, obj)
|
||||
chunk.SetPayload(payloadShards[i])
|
||||
chunk.SetPayloadSize(uint64(len(payloadShards[i])))
|
||||
|
||||
if obj.SplitID() != nil {
|
||||
var attr objectSDK.Attribute
|
||||
attr.SetKey("split")
|
||||
attr.SetValue("true")
|
||||
chunk.SetAttributes(attr)
|
||||
}
|
||||
|
||||
ec := objectSDK.NewECHeader(parent, uint32(i), uint32(len(payloadShards)), headerShards[i], uint32(len(header)))
|
||||
chunk.SetECHeader(ec)
|
||||
if err := setIDWithSignature(chunk, key); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
parts[i] = chunk
|
||||
}
|
||||
return parts, nil
|
||||
}
|
||||
|
||||
func setIDWithSignature(obj *objectSDK.Object, key *ecdsa.PrivateKey) error {
|
||||
objectSDK.CalculateAndSetPayloadChecksum(obj)
|
||||
|
||||
if err := objectSDK.CalculateAndSetID(obj); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if key == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
return objectSDK.CalculateAndSetSignature(*key, obj)
|
||||
}
|
||||
|
||||
func (c *Constructor) encodeRaw(data []byte) ([][]byte, error) {
|
||||
shards, err := c.enc.Split(data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := c.enc.Encode(shards); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return shards, nil
|
||||
}
|
45
object/erasurecode/split_test.go
Normal file
45
object/erasurecode/split_test.go
Normal file
|
@ -0,0 +1,45 @@
|
|||
package erasurecode_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/erasurecode"
|
||||
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// The library can behave differently for big shard counts.
|
||||
// This test checks we support the maximum number of chunks we promise.
|
||||
func TestSplitMaxShardCount(t *testing.T) {
|
||||
pk, err := keys.NewPrivateKey()
|
||||
require.NoError(t, err)
|
||||
|
||||
original := newObject(t, 1024, pk)
|
||||
|
||||
t.Run("only data", func(t *testing.T) {
|
||||
c, err := erasurecode.NewConstructor(erasurecode.MaxShardCount, 0)
|
||||
require.NoError(t, err)
|
||||
|
||||
parts, err := c.Split(original, &pk.PrivateKey)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, parts, erasurecode.MaxShardCount)
|
||||
|
||||
for _, part := range parts {
|
||||
require.NoError(t, objectSDK.CheckHeaderVerificationFields(part))
|
||||
}
|
||||
})
|
||||
t.Run("data + parity", func(t *testing.T) {
|
||||
c, err := erasurecode.NewConstructor(1, erasurecode.MaxShardCount-1)
|
||||
require.NoError(t, err)
|
||||
|
||||
parts, err := c.Split(original, &pk.PrivateKey)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, parts, erasurecode.MaxShardCount)
|
||||
|
||||
for _, part := range parts {
|
||||
require.NoError(t, objectSDK.CheckHeaderVerificationFields(part))
|
||||
}
|
||||
})
|
||||
|
||||
}
|
44
object/erasurecode/target.go
Normal file
44
object/erasurecode/target.go
Normal file
|
@ -0,0 +1,44 @@
|
|||
package erasurecode
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ecdsa"
|
||||
|
||||
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer"
|
||||
)
|
||||
|
||||
// Target accepts regular objects and splits them into erasure-coded chunks.
|
||||
type Target struct {
|
||||
c *Constructor
|
||||
key *ecdsa.PrivateKey
|
||||
next transformer.ObjectWriter
|
||||
}
|
||||
|
||||
// ObjectWriter is an interface of the object writer that writes prepared object.
|
||||
type ObjectWriter interface {
|
||||
WriteObject(context.Context, *objectSDK.Object) error
|
||||
}
|
||||
|
||||
// NewTarget returns new target instance.
|
||||
func NewTarget(c *Constructor, key *ecdsa.PrivateKey, next ObjectWriter) *Target {
|
||||
return &Target{
|
||||
c: c,
|
||||
key: key,
|
||||
next: next,
|
||||
}
|
||||
}
|
||||
|
||||
// WriteObject implements the transformer.ObjectWriter interface.
|
||||
func (t *Target) WriteObject(ctx context.Context, obj *objectSDK.Object) error {
|
||||
parts, err := t.c.Split(obj, t.key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for i := range parts {
|
||||
if err := t.next.WriteObject(ctx, parts[i]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue