Compare commits

...
Sign in to create a new pull request.

13 commits

Author SHA1 Message Date
6fbe1595cb [#121] pool: Refactor PrmObjectSearch usage
Signed-off-by: Airat Arifullin <a.arifullin@yadro.com>
2023-11-01 17:45:15 +03:00
a9237aabd2 [#121] client: Make PrmObjectSearch fields public
Signed-off-by: Airat Arifullin <a.arifullin@yadro.com>
2023-11-01 17:45:06 +03:00
a487033505 [#121] client: Nuke out unused prmCommonMeta
Signed-off-by: Airat Arifullin a.arifullin@yadro.com
2023-10-31 22:55:25 +03:00
51c3618850 [#121] client: Make PrmContainerList fields public
Signed-off-by: Airat Arifullin a.arifullin@yadro.com
2023-10-31 22:55:25 +03:00
665e5807bc [#188] transformer: Allow to provide size hint
For big objects with known size we can optimize allocation patterns
by providing size hint. As with any hint, it does not affect transformer
functionality: slices with capacity > MaxSize are never allocated.

```
goos: linux
goarch: amd64
pkg: git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer
cpu: 11th Gen Intel(R) Core(TM) i5-1135G7 @ 2.40GHz
                                                │     out     │
                                                │   sec/op    │
Transformer/small/no_size_hint-8                  65.44µ ± 3%
Transformer/small/no_size_hint,_with_buffer-8     64.24µ ± 5%
Transformer/small/with_size_hint,_with_buffer-8   58.70µ ± 5%
Transformer/big/no_size_hint-8                    367.8m ± 3%
Transformer/big/no_size_hint,_with_buffer-8       562.7m ± 0%
Transformer/big/with_size_hint,_with_buffer-8     385.6m ± 7%
geomean                                           5.197m

                                                │     out      │
                                                │     B/op     │
Transformer/small/no_size_hint-8                  13.40Ki ± 0%
Transformer/small/no_size_hint,_with_buffer-8     13.40Ki ± 0%
Transformer/small/with_size_hint,_with_buffer-8   13.39Ki ± 0%
Transformer/big/no_size_hint-8                    288.0Mi ± 0%
Transformer/big/no_size_hint,_with_buffer-8       1.390Gi ± 0%
Transformer/big/with_size_hint,_with_buffer-8     288.0Mi ± 0%
geomean                                           2.533Mi

                                                │    out     │
                                                │ allocs/op  │
Transformer/small/no_size_hint-8                  92.00 ± 0%
Transformer/small/no_size_hint,_with_buffer-8     92.00 ± 0%
Transformer/small/with_size_hint,_with_buffer-8   92.00 ± 0%
Transformer/big/no_size_hint-8                    546.5 ± 0%
Transformer/big/no_size_hint,_with_buffer-8       607.5 ± 0%
Transformer/big/with_size_hint,_with_buffer-8     545.5 ± 0%
geomean                                           228.1
```

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2023-10-30 12:13:04 +00:00
a02c0bfac8 [#186] netmap: Marshal policy with brackets
Brackets can be semantically important and must not be omitted,
otherwise the output is plain wrong.
We do not take the responsibility to preserve every bracket, though,
because parser does some optimizations related to grouping long chains
of filters combined with the same operation.

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2023-10-27 10:54:45 +03:00
20d325e307 [#167] netmap: Fix reverse min agregator
The higher the price, the lower reverse min weight should be.
Previously nodes with 0 price had 0 weight which is a bit misleading.

Introduced in d71a0e0755.

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
2023-10-27 07:53:19 +00:00
670619d242 [#131] client: keep backwards-compatibility, update README.md, fix chore
Signed-off-by: Egor Olefirenko <egor.olefirenko892@gmail.com>
2023-10-26 14:35:49 +00:00
0d79d10482 [#131] client: rename option consistently and fix test
Signed-off-by: Egor Olefirenko <egor.olefirenko892@gmail.com>
2023-10-26 14:35:49 +00:00
9727beb47d [#131] client: Switch ResolveFrostFSFailures to DontResolveFrostFSFailures option
Signed-off-by: Egor Olefirenko <egor.olefirenko892@gmail.com>
2023-10-26 14:35:49 +00:00
84315fab6a [#121] client: Make PrmBalanceGet fields public
Signed-off-by: Airat Arifullin <a.arifullin@yadro.com>
2023-10-23 18:53:14 +03:00
71335489ae [#183] forgejo: Make linter great again
Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
2023-10-23 18:13:40 +03:00
4c1feaf2cb [#182] pool: Fix linter error about deprecated methods
Signed-off-by: Airat Arifullin <a.arifullin@yadro.com>
2023-10-23 17:40:00 +03:00
32 changed files with 441 additions and 192 deletions

View file

@ -8,17 +8,24 @@ jobs:
steps:
- uses: actions/checkout@v3
- name: golangci-lint
uses: https://github.com/golangci/golangci-lint-action@v2
- name: Set up Go
uses: actions/setup-go@v3
with:
version: latest
go-version: '1.21'
cache: true
- name: Install linters
run: make lint-install
- name: Run linters
run: make lint
tests:
name: Tests
runs-on: ubuntu-latest
strategy:
matrix:
go_versions: [ '1.19', '1.20' ]
go_versions: [ '1.20', '1.21' ]
fail-fast: false
steps:
- uses: actions/checkout@v3

4
.gitignore vendored
View file

@ -27,3 +27,7 @@ antlr-*.jar
# tempfiles
.cache
# binary
bin/
release/

View file

@ -1,6 +1,11 @@
#!/usr/bin/make -f
ANTLR_VERSION="4.13.0"
TMP_DIR := .cache
LINT_VERSION ?= 1.55.0
TRUECLOUDLAB_LINT_VERSION ?= 0.0.2
OUTPUT_LINT_DIR ?= $(shell pwd)/bin
LINT_DIR = $(OUTPUT_LINT_DIR)/golangci-lint-$(LINT_VERSION)-v$(TRUECLOUDLAB_LINT_VERSION)
# Run tests
test:
@ -15,9 +20,23 @@ dep:
@CGO_ENABLED=0 \
go mod tidy -v && echo OK
# Install linters
lint-install:
@mkdir -p $(TMP_DIR)
@rm -rf $(TMP_DIR)/linters
@git -c advice.detachedHead=false clone --branch v$(TRUECLOUDLAB_LINT_VERSION) https://git.frostfs.info/TrueCloudLab/linters.git $(TMP_DIR)/linters
@@make -C $(TMP_DIR)/linters lib CGO_ENABLED=1 OUT_DIR=$(OUTPUT_LINT_DIR)
@rm -rf $(TMP_DIR)/linters
@rmdir $(TMP_DIR) 2>/dev/null || true
@CGO_ENABLED=1 GOBIN=$(LINT_DIR) go install github.com/golangci/golangci-lint/cmd/golangci-lint@v$(LINT_VERSION)
# Run linters
lint:
@golangci-lint --timeout=5m run
@if [ ! -d "$(LINT_DIR)" ]; then \
echo "Run make lint-install"; \
exit 1; \
fi
$(LINT_DIR)/golangci-lint run
# Run tests with race detection and produce coverage output
cover:

View file

@ -42,7 +42,6 @@ Contains client for working with FrostFS.
```go
var prmInit client.PrmInit
prmInit.SetDefaultPrivateKey(key) // private key for request signing
prmInit.ResolveFrostFSFailures() // enable erroneous status parsing
var c client.Client
c.Init(prmInit)
@ -77,8 +76,7 @@ if needed and perform any desired action. In the case above we may want to repor
these details to the user as well as retry an operation, possibly with different parameters.
Status wire-format is extendable and each node can report any set of details it wants.
The set of reserved status codes can be found in
[FrostFS API](https://git.frostfs.info/TrueCloudLab/frostfs-api/src/branch/master/status/types.proto). There is also
a `client.PrmInit.ResolveFrostFSFailures()` to seamlessly convert erroneous statuses into Go error type.
[FrostFS API](https://git.frostfs.info/TrueCloudLab/frostfs-api/src/branch/master/status/types.proto).
### policy
Contains helpers allowing conversion of placing policy from/to JSON representation

View file

@ -17,26 +17,26 @@ import (
// PrmBalanceGet groups parameters of BalanceGet operation.
type PrmBalanceGet struct {
prmCommonMeta
XHeaders []string
accountSet bool
account user.ID
Account *user.ID
}
// SetAccount sets identifier of the FrostFS account for which the balance is requested.
// Required parameter.
//
// Deprecated: Use PrmBalanceGet.Account instead.
func (x *PrmBalanceGet) SetAccount(id user.ID) {
x.account = id
x.accountSet = true
x.Account = &id
}
func (x *PrmBalanceGet) buildRequest(c *Client) (*v2accounting.BalanceRequest, error) {
if !x.accountSet {
if x.Account == nil {
return nil, errorAccountNotSet
}
var accountV2 refs.OwnerID
x.account.WriteToV2(&accountV2)
x.Account.WriteToV2(&accountV2)
var body v2accounting.BalanceRequestBody
body.SetOwnerID(&accountV2)
@ -64,9 +64,9 @@ func (x ResBalanceGet) Amount() accounting.Decimal {
//
// Exactly one return value is non-nil. By default, server status is returned in res structure.
// Any client's internal or transport errors are returned as `error`,
// If PrmInit.ResolveFrostFSFailures has been called, unsuccessful
// FrostFS status codes are returned as `error`, otherwise, are included
// in the returned result structure.
// If PrmInit.DisableFrostFSFailuresResolution has been called, unsuccessful
// FrostFS status codes are included in the returned result structure,
// otherwise, are also returned as `error`.
//
// Returns an error if parameters are set incorrectly (see PrmBalanceGet docs).
// Context is required and must not be nil. It is used for network communication.

View file

@ -144,7 +144,7 @@ func (c *Client) Close() error {
//
// See also Init.
type PrmInit struct {
resolveFrostFSErrors bool
disableFrostFSErrorResolution bool
key ecdsa.PrivateKey
@ -161,12 +161,16 @@ func (x *PrmInit) SetDefaultPrivateKey(key ecdsa.PrivateKey) {
x.key = key
}
// ResolveFrostFSFailures makes the Client to resolve failure statuses of the
// FrostFS protocol into Go built-in errors. These errors are returned from
// each protocol operation. By default, statuses aren't resolved and written
// to the resulting structure (see corresponding Res* docs).
// Deprecated: method is no-op. Option is default.
func (x *PrmInit) ResolveFrostFSFailures() {
x.resolveFrostFSErrors = true
}
// DisableFrostFSFailuresResolution makes the Client to preserve failure statuses of the
// FrostFS protocol only in resulting structure (see corresponding Res* docs).
// These errors are returned from each protocol operation. By default, statuses
// are resolved and returned as a Go built-in errors.
func (x *PrmInit) DisableFrostFSFailuresResolution() {
x.disableFrostFSErrorResolution = true
}
// SetResponseInfoCallback makes the Client to pass ResponseMetaInfo from each

View file

@ -24,24 +24,6 @@ func (x statusRes) Status() apistatus.Status {
return x.st
}
// groups meta parameters shared between all Client operations.
type prmCommonMeta struct {
// FrostFS request X-Headers
xHeaders []string
}
// WithXHeaders specifies list of extended headers (string key-value pairs)
// to be attached to the request. Must have an even length.
//
// Slice must not be mutated until the operation completes.
func (x *prmCommonMeta) WithXHeaders(hs ...string) {
if len(hs)%2 != 0 {
panic("slice of X-Headers with odd length")
}
x.xHeaders = hs
}
func writeXHeadersToMeta(xHeaders []string, h *v2session.RequestMetaHeader) {
if len(xHeaders) == 0 {
return
@ -119,7 +101,7 @@ func (c *Client) processResponse(resp responseV2) (apistatus.Status, error) {
}
st := apistatus.FromStatusV2(resp.GetMetaHeader().GetStatus())
if c.prm.resolveFrostFSErrors {
if !c.prm.disableFrostFSErrorResolution {
return st, apistatus.ErrFromStatus(st)
}
return st, nil

View file

@ -101,9 +101,9 @@ type ResContainerDelete struct {
//
// Exactly one return value is non-nil. By default, server status is returned in res structure.
// Any client's internal or transport errors are returned as `error`.
// If PrmInit.ResolveFrostFSFailures has been called, unsuccessful
// FrostFS status codes are returned as `error`, otherwise, are included
// in the returned result structure.
// If PrmInit.DisableFrostFSFailuresResolution has been called, unsuccessful
// FrostFS status codes are included in the returned result structure,
// otherwise, are also returned as `error`.
//
// Operation is asynchronous and no guaranteed even in the absence of errors.
// The required time is also not predictable.

View file

@ -68,9 +68,9 @@ func (x ResContainerEACL) Table() eacl.Table {
//
// Exactly one return value is non-nil. By default, server status is returned in res structure.
// Any client's internal or transport errors are returned as `error`.
// If PrmInit.ResolveFrostFSFailures has been called, unsuccessful
// FrostFS status codes are returned as `error`, otherwise, are included
// in the returned result structure.
// If PrmInit.DisableFrostFSFailuresResolution has been called, unsuccessful
// FrostFS status codes are included in the returned result structure,
// otherwise, are also returned as `error`.
//
// Returns an error if parameters are set incorrectly (see PrmContainerEACL docs).
// Context is required and must not be nil. It is used for network communication.

View file

@ -71,9 +71,9 @@ func (x ResContainerGet) Container() container.Container {
//
// Exactly one return value is non-nil. By default, server status is returned in res structure.
// Any client's internal or transport errors are returned as `error`.
// If PrmInit.ResolveFrostFSFailures has been called, unsuccessful
// FrostFS status codes are returned as `error`, otherwise, are included
// in the returned result structure.
// If PrmInit.DisableFrostFSFailuresResolution has been called, unsuccessful
// FrostFS status codes are included in the returned result structure,
// otherwise, are also returned as `error`.
//
// Returns an error if parameters are set incorrectly (see PrmContainerGet docs).
// Context is required and must not be nil. It is used for network communication.

View file

@ -17,26 +17,26 @@ import (
// PrmContainerList groups parameters of ContainerList operation.
type PrmContainerList struct {
prmCommonMeta
XHeaders []string
ownerSet bool
ownerID user.ID
Account *user.ID
}
// SetAccount sets identifier of the FrostFS account to list the containers.
// Required parameter.
//
// Deprecated: Use PrmContainerList.Account instead.
func (x *PrmContainerList) SetAccount(id user.ID) {
x.ownerID = id
x.ownerSet = true
x.Account = &id
}
func (x *PrmContainerList) buildRequest(c *Client) (*v2container.ListRequest, error) {
if !x.ownerSet {
if x.Account == nil {
return nil, errorAccountNotSet
}
var ownerV2 refs.OwnerID
x.ownerID.WriteToV2(&ownerV2)
x.Account.WriteToV2(&ownerV2)
reqBody := new(v2container.ListRequestBody)
reqBody.SetOwnerID(&ownerV2)
@ -65,9 +65,9 @@ func (x ResContainerList) Containers() []cid.ID {
//
// Exactly one return value is non-nil. By default, server status is returned in res structure.
// Any client's internal or transport errors are returned as `error`.
// If PrmInit.ResolveFrostFSFailures has been called, unsuccessful
// FrostFS status codes are returned as `error`, otherwise, are included
// in the returned result structure.
// If PrmInit.DisableFrostFSFailuresResolution has been called, unsuccessful
// FrostFS status codes are included in the returned result structure,
// otherwise, are also returned as `error`.
//
// Returns an error if parameters are set incorrectly (see PrmContainerList docs).
// Context is required and must not be nil. It is used for network communication.

View file

@ -110,9 +110,9 @@ func (x ResContainerPut) ID() cid.ID {
//
// Exactly one return value is non-nil. By default, server status is returned in res structure.
// Any client's internal or transport errors are returned as `error`.
// If PrmInit.ResolveFrostFSFailures has been called, unsuccessful
// FrostFS status codes are returned as `error`, otherwise, are included
// in the returned result structure.
// If PrmInit.DisableFrostFSFailuresResolution has been called, unsuccessful
// FrostFS status codes are included in the returned result structure,
// otherwise, are also returned as `error`.
//
// Operation is asynchronous and no guaranteed even in the absence of errors.
// The required time is also not predictable.

View file

@ -101,9 +101,9 @@ type ResContainerSetEACL struct {
//
// Exactly one return value is non-nil. By default, server status is returned in res structure.
// Any client's internal or transport errors are returned as `error`.
// If PrmInit.ResolveFrostFSFailures has been called, unsuccessful
// FrostFS status codes are returned as `error`, otherwise, are included
// in the returned result structure.
// If PrmInit.DisableFrostFSFailuresResolution has been called, unsuccessful
// FrostFS status codes are included in the returned result structure,
// otherwise, are also returned as `error`.
//
// Operation is asynchronous and no guaranteed even in the absence of errors.
// The required time is also not predictable.

View file

@ -57,9 +57,9 @@ type ResAnnounceSpace struct {
//
// Exactly one return value is non-nil. By default, server status is returned in res structure.
// Any client's internal or transport errors are returned as `error`.
// If PrmInit.ResolveFrostFSFailures has been called, unsuccessful
// FrostFS status codes are returned as `error`, otherwise, are included
// in the returned result structure.
// If PrmInit.DisableFrostFSFailuresResolution has been called, unsuccessful
// FrostFS status codes are included in the returned result structure,
// otherwise, are also returned as `error`.
//
// Operation is asynchronous and no guaranteed even in the absence of errors.
// The required time is also not predictable.

View file

@ -53,9 +53,9 @@ func (x ResEndpointInfo) NodeInfo() netmap.NodeInfo {
// Method can be used as a health check to see if node is alive and responds to requests.
//
// Any client's internal or transport errors are returned as `error`.
// If PrmInit.ResolveFrostFSFailures has been called, unsuccessful
// FrostFS status codes are returned as `error`, otherwise, are included
// in the returned result structure.
// If PrmInit.DisableFrostFSFailuresResolution has been called, unsuccessful
// FrostFS status codes are included in the returned result structure,
// otherwise, are also returned as `error`.
//
// Returns an error if parameters are set incorrectly (see PrmEndpointInfo docs).
// Context is required and must not be nil. It is used for network communication.
@ -140,9 +140,9 @@ func (x ResNetworkInfo) Info() netmap.NetworkInfo {
// NetworkInfo requests information about the FrostFS network of which the remote server is a part.
//
// Any client's internal or transport errors are returned as `error`.
// If PrmInit.ResolveFrostFSFailures has been called, unsuccessful
// FrostFS status codes are returned as `error`, otherwise, are included
// in the returned result structure.
// If PrmInit.DisableFrostFSFailuresResolution has been called, unsuccessful
// FrostFS status codes are included in the returned result structure,
// otherwise, are also returned as `error`.
//
// Returns an error if parameters are set incorrectly (see PrmNetworkInfo docs).
// Context is required and must not be nil. It is used for network communication.
@ -204,9 +204,9 @@ func (x ResNetMapSnapshot) NetMap() netmap.NetMap {
// NetMapSnapshot requests current network view of the remote server.
//
// Any client's internal or transport errors are returned as `error`.
// If PrmInit.ResolveFrostFSFailures has been called, unsuccessful
// FrostFS status codes are returned as `error`, otherwise, are included
// in the returned result structure.
// If PrmInit.DisableFrostFSFailuresResolution has been called, unsuccessful
// FrostFS status codes are included in the returned result structure,
// otherwise, are also returned as `error`.
//
// Returns an error if parameters are set incorrectly.
// Context is required and MUST NOT be nil. It is used for network communication.

View file

@ -67,6 +67,7 @@ func TestClient_NetMapSnapshot(t *testing.T) {
var res *ResNetMapSnapshot
var srv serverNetMap
c := newClient(&srv)
c.prm.DisableFrostFSFailuresResolution()
ctx := context.Background()
// request signature

View file

@ -112,9 +112,9 @@ func (prm *PrmObjectDelete) buildRequest(c *Client) (*v2object.DeleteRequest, er
//
// Exactly one return value is non-nil. By default, server status is returned in res structure.
// Any client's internal or transport errors are returned as `error`,
// If PrmInit.ResolveFrostFSFailures has been called, unsuccessful
// FrostFS status codes are returned as `error`, otherwise, are included
// in the returned result structure.
// If PrmInit.DisableFrostFSFailuresResolution has been called, unsuccessful
// FrostFS status codes are included in the returned result structure,
// otherwise, are also returned as `error`.
//
// Returns an error if parameters are set incorrectly (see PrmObjectDelete docs).
// Context is required and must not be nil. It is used for network communication.

View file

@ -444,9 +444,9 @@ func (prm *PrmObjectHead) buildRequest(c *Client) (*v2object.HeadRequest, error)
//
// Exactly one return value is non-nil. By default, server status is returned in res structure.
// Any client's internal or transport errors are returned as `error`,
// If PrmInit.ResolveFrostFSFailures has been called, unsuccessful
// FrostFS status codes are returned as `error`, otherwise, are included
// in the returned result structure.
// If PrmInit.DisableFrostFSFailuresResolution has been called, unsuccessful
// FrostFS status codes are included in the returned result structure,
// otherwise, are also returned as `error`.
//
// Returns an error if parameters are set incorrectly (see PrmObjectHead docs).
// Context is required and must not be nil. It is used for network communication.

View file

@ -152,9 +152,9 @@ func (prm *PrmObjectHash) buildRequest(c *Client) (*v2object.GetRangeHashRequest
//
// Exactly one return value is non-nil. By default, server status is returned in res structure.
// Any client's internal or transport errors are returned as `error`,
// If PrmInit.ResolveFrostFSFailures has been called, unsuccessful
// FrostFS status codes are returned as `error`, otherwise, are included
// in the returned result structure.
// If PrmInit.DisableFrostFSFailuresResolution has been called, unsuccessful
// FrostFS status codes are included in the returned result structure,
// otherwise, are also returned as `error`.
//
// Returns an error if parameters are set incorrectly (see PrmObjectHash docs).
// Context is required and must not be nil. It is used for network communication.

View file

@ -83,7 +83,7 @@ func (it *internalTarget) putAsStream(ctx context.Context, o *object.Object) err
wrt.WritePayloadChunk(ctx, o.Payload())
}
it.res, err = wrt.Close(ctx)
if err == nil && !it.client.prm.resolveFrostFSErrors && !apistatus.IsSuccessful(it.res.st) {
if err == nil && it.client.prm.disableFrostFSErrorResolution && !apistatus.IsSuccessful(it.res.st) {
err = apistatus.ErrFromStatus(it.res.st)
}
return err
@ -115,7 +115,7 @@ func (it *internalTarget) tryPutSingle(ctx context.Context, o *object.Object) (b
statusRes: res.statusRes,
obj: id,
}
if !it.client.prm.resolveFrostFSErrors && !apistatus.IsSuccessful(it.res.st) {
if it.client.prm.disableFrostFSErrorResolution && !apistatus.IsSuccessful(it.res.st) {
return true, apistatus.ErrFromStatus(it.res.st)
}
return true, nil

View file

@ -24,19 +24,26 @@ import (
// PrmObjectSearch groups parameters of ObjectSearch operation.
type PrmObjectSearch struct {
meta v2session.RequestMetaHeader
XHeaders []string
key *ecdsa.PrivateKey
Local bool
cnrSet bool
cnrID cid.ID
BearerToken *bearer.Token
filters object.SearchFilters
Session *session.Object
ContainerID *cid.ID
Key *ecdsa.PrivateKey
Filters object.SearchFilters
}
// MarkLocal tells the server to execute the operation locally.
//
// Deprecated: Use PrmObjectSearch.Local instead.
func (x *PrmObjectSearch) MarkLocal() {
x.meta.SetTTL(1)
x.Local = true
}
// WithinSession specifies session within which the search query must be executed.
@ -45,10 +52,10 @@ func (x *PrmObjectSearch) MarkLocal() {
// This may affect the execution of an operation (e.g. access control).
//
// Must be signed.
//
// Deprecated: Use PrmObjectSearch.Session instead.
func (x *PrmObjectSearch) WithinSession(t session.Object) {
var tokv2 v2session.Token
t.WriteToV2(&tokv2)
x.meta.SetSessionToken(&tokv2)
x.Session = &t
}
// WithBearerToken attaches bearer token to be used for the operation.
@ -56,37 +63,44 @@ func (x *PrmObjectSearch) WithinSession(t session.Object) {
// If set, underlying eACL rules will be used in access control.
//
// Must be signed.
//
// Deprecated: Use PrmObjectSearch.BearerToken instead.
func (x *PrmObjectSearch) WithBearerToken(t bearer.Token) {
var v2token acl.BearerToken
t.WriteToV2(&v2token)
x.meta.SetBearerToken(&v2token)
x.BearerToken = &t
}
// WithXHeaders specifies list of extended headers (string key-value pairs)
// to be attached to the request. Must have an even length.
//
// Slice must not be mutated until the operation completes.
//
// Deprecated: Use PrmObjectSearch.XHeaders instead.
func (x *PrmObjectSearch) WithXHeaders(hs ...string) {
writeXHeadersToMeta(hs, &x.meta)
x.XHeaders = hs
}
// UseKey specifies private key to sign the requests.
// If key is not provided, then Client default key is used.
//
// Deprecated: Use PrmObjectSearch.Key instead.
func (x *PrmObjectSearch) UseKey(key ecdsa.PrivateKey) {
x.key = &key
x.Key = &key
}
// InContainer specifies the container in which to look for objects.
// Required parameter.
//
// Deprecated: Use PrmObjectSearch.ContainerID instead.
func (x *PrmObjectSearch) InContainer(id cid.ID) {
x.cnrID = id
x.cnrSet = true
x.ContainerID = &id
}
// SetFilters sets filters by which to select objects. All container objects
// match unset/empty filters.
//
// Deprecated: Use PrmObjectSearch.Filters instead.
func (x *PrmObjectSearch) SetFilters(filters object.SearchFilters) {
x.filters = filters
x.Filters = filters
}
// ResObjectSearch groups the final result values of ObjectSearch operation.
@ -212,6 +226,48 @@ func (x *ObjectListReader) Close() (*ResObjectSearch, error) {
return &x.res, nil
}
func (x *PrmObjectSearch) buildRequest(c *Client) (*v2object.SearchRequest, error) {
if x.ContainerID == nil {
return nil, errorMissingContainer
}
if len(x.XHeaders)%2 != 0 {
return nil, errorInvalidXHeaders
}
meta := new(v2session.RequestMetaHeader)
writeXHeadersToMeta(x.XHeaders, meta)
if x.BearerToken != nil {
v2BearerToken := new(acl.BearerToken)
x.BearerToken.WriteToV2(v2BearerToken)
meta.SetBearerToken(v2BearerToken)
}
if x.Session != nil {
v2SessionToken := new(v2session.Token)
x.Session.WriteToV2(v2SessionToken)
meta.SetSessionToken(v2SessionToken)
}
if x.Local {
meta.SetTTL(1)
}
cnrV2 := new(v2refs.ContainerID)
x.ContainerID.WriteToV2(cnrV2)
body := new(v2object.SearchRequestBody)
body.SetVersion(1)
body.SetContainerID(cnrV2)
body.SetFilters(x.Filters.ToV2())
req := new(v2object.SearchRequest)
req.SetBody(body)
c.prepareRequest(req, meta)
return req, nil
}
// ObjectSearchInit initiates object selection through a remote server using FrostFS API protocol.
//
// The call only opens the transmission channel, explicit fetching of matched objects
@ -221,30 +277,17 @@ func (x *ObjectListReader) Close() (*ResObjectSearch, error) {
// Returns an error if parameters are set incorrectly (see PrmObjectSearch docs).
// Context is required and must not be nil. It is used for network communication.
func (c *Client) ObjectSearchInit(ctx context.Context, prm PrmObjectSearch) (*ObjectListReader, error) {
// check parameters
if !prm.cnrSet {
return nil, errorMissingContainer
req, err := prm.buildRequest(c)
if err != nil {
return nil, err
}
var cidV2 v2refs.ContainerID
prm.cnrID.WriteToV2(&cidV2)
var body v2object.SearchRequestBody
body.SetVersion(1)
body.SetContainerID(&cidV2)
body.SetFilters(prm.filters.ToV2())
// init reader
var req v2object.SearchRequest
req.SetBody(&body)
c.prepareRequest(&req, &prm.meta)
key := prm.key
key := prm.Key
if key == nil {
key = &c.prm.key
}
err := signature.SignServiceMessage(key, &req)
err = signature.SignServiceMessage(key, req)
if err != nil {
return nil, fmt.Errorf("sign request: %w", err)
}
@ -252,7 +295,7 @@ func (c *Client) ObjectSearchInit(ctx context.Context, prm PrmObjectSearch) (*Ob
var r ObjectListReader
ctx, r.cancelCtxStream = context.WithCancel(ctx)
r.stream, err = rpcapi.SearchObjects(&c.c, &req, client.WithContext(ctx))
r.stream, err = rpcapi.SearchObjects(&c.c, req, client.WithContext(ctx))
if err != nil {
return nil, fmt.Errorf("open stream: %w", err)
}

View file

@ -89,9 +89,9 @@ func (x ResSessionCreate) PublicKey() []byte {
//
// Exactly one return value is non-nil. By default, server status is returned in res structure.
// Any client's internal or transport errors are returned as `error`.
// If PrmInit.ResolveFrostFSFailures has been called, unsuccessful
// FrostFS status codes are returned as `error`, otherwise, are included
// in the returned result structure.
// If PrmInit.DisableFrostFSFailuresResolution has been called, unsuccessful
// FrostFS status codes are included in the returned result structure,
// otherwise, are also returned as `error`.
//
// Returns an error if parameters are set incorrectly (see PrmSessionCreate docs).
// Context is required and must not be nil. It is used for network communication.

View file

@ -156,11 +156,7 @@ func (a *meanIQRAgg) Compute() float64 {
}
func (r *reverseMinNorm) Normalize(w float64) float64 {
if w == 0 {
return 0
}
return r.min / w
return (r.min + 1) / (w + 1)
}
func (r *sigmoidNorm) Normalize(w float64) float64 {

View file

@ -146,19 +146,19 @@
"select 3 nodes in 3 distinct countries, same placement": {
"policy": {"replicas":[{"count":1,"selector":"Main"}],"containerBackupFactor":1,"selectors":[{"name":"Main","count":3,"clause":"DISTINCT","attribute":"Country","filter":"*"}],"filters":[]},
"pivot": "Y29udGFpbmVySUQ=",
"result": [[0, 2, 3]],
"result": [ [ 5, 0, 7 ] ],
"placement": {
"pivot": "b2JqZWN0SUQ=",
"result": [[0, 2, 3]]
"result": [ [ 5, 0, 7 ] ]
}
},
"select 6 nodes in 3 distinct countries, different placement": {
"policy": {"replicas":[{"count":1,"selector":"Main"}],"containerBackupFactor":2,"selectors":[{"name":"Main","count":3,"clause":"DISTINCT","attribute":"Country","filter":"*"}],"filters":[]},
"pivot": "Y29udGFpbmVySUQ=",
"result": [[0, 1, 2, 6, 3, 4]],
"result": [ [ 5, 4, 0, 1, 7, 2 ] ],
"placement": {
"pivot": "b2JqZWN0SUQ=",
"result": [[0, 1, 2, 6, 3, 4]]
"result": [ [ 5, 4, 0, 7, 2, 1 ] ]
}
}
}

View file

@ -455,7 +455,7 @@ func (p PlacementPolicy) WriteStringTo(w io.StringWriter) (err error) {
return err
}
err = writeFilterStringTo(w, p.filters[i])
err = writeFilterStringTo(w, p.filters[i], false)
if err != nil {
return err
}
@ -464,7 +464,7 @@ func (p PlacementPolicy) WriteStringTo(w io.StringWriter) (err error) {
return nil
}
func writeFilterStringTo(w io.StringWriter, f netmap.Filter) error {
func writeFilterStringTo(w io.StringWriter, f netmap.Filter, mayNeedOuterBrackets bool) error {
var err error
var s string
op := f.GetOp()
@ -489,7 +489,7 @@ func writeFilterStringTo(w io.StringWriter, f netmap.Filter) error {
if err != nil {
return err
}
err = writeFilterStringTo(w, inner[0])
err = writeFilterStringTo(w, inner[0], false)
if err != nil {
return err
}
@ -498,6 +498,13 @@ func writeFilterStringTo(w io.StringWriter, f netmap.Filter) error {
return err
}
} else {
useBrackets := mayNeedOuterBrackets && op == netmap.OR && len(inner) > 1
if useBrackets {
_, err = w.WriteString("(")
if err != nil {
return err
}
}
for i := range inner {
if i != 0 {
_, err = w.WriteString(" " + op.String() + " ")
@ -505,7 +512,13 @@ func writeFilterStringTo(w io.StringWriter, f netmap.Filter) error {
return err
}
}
err = writeFilterStringTo(w, inner[i])
err = writeFilterStringTo(w, inner[i], true)
if err != nil {
return err
}
}
if useBrackets {
_, err = w.WriteString(")")
if err != nil {
return err
}

View file

@ -1,6 +1,7 @@
package netmap_test
import (
"strings"
"testing"
. "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
@ -29,6 +30,79 @@ func TestPlacementPolicyEncoding(t *testing.T) {
})
}
func TestPlacementPolicyWriteString(t *testing.T) {
var testCases = []struct {
name string
input string
output string // If the output is empty, make it equal to input.
}{
{
name: "no compound operators",
input: `REP 1
CBF 1
SELECT 1 FROM Color
FILTER Color EQ Red AS Color`,
},
{
name: "no brackets in single level same-operator chain",
input: `REP 1
CBF 1
SELECT 1 FROM Color
FILTER Color EQ Red OR Color EQ Blue OR Color EQ Green AS Color`,
},
{
name: "no brackets aroung higher precedence op",
input: `REP 1
CBF 1
SELECT 1 FROM Color
FILTER Color EQ Red OR Color EQ Blue AND Color NE Green AS Color`,
},
{
name: "no brackets aroung higher precedence op, even if present in the input",
input: `REP 1
CBF 1
SELECT 1 FROM Color
FILTER Color EQ Red OR (Color EQ Blue AND Color NE Green) AS Color`,
output: `REP 1
CBF 1
SELECT 1 FROM Color
FILTER Color EQ Red OR Color EQ Blue AND Color NE Green AS Color`,
},
{
name: "brackets aroung lower precedence op",
input: `REP 1
CBF 1
SELECT 1 FROM Color
FILTER (Color EQ Red OR Color EQ Blue) AND Color NE Green AS Color`,
},
{
name: "no extra brackets for bracketed same-operator chain",
input: `REP 1
CBF 1
SELECT 1 FROM Color
FILTER (Color EQ Red OR Color EQ Blue OR Color EQ Yellow) AND Color NE Green AS Color`,
},
}
for _, tc := range testCases {
var p PlacementPolicy
require.NoError(t, p.DecodeString(tc.input))
var sb strings.Builder
require.NoError(t, p.WriteStringTo(&sb))
if tc.output == "" {
require.Equal(t, tc.input, sb.String())
} else {
require.Equal(t, tc.output, sb.String())
var p1 PlacementPolicy
require.NoError(t, p1.DecodeString(tc.output))
require.Equal(t, p, p1)
}
}
}
func TestDecodeSelectFilterExpr(t *testing.T) {
for _, s := range []string{
"SELECT 1 FROM *",

View file

@ -18,8 +18,8 @@ func TestChannelTarget(t *testing.T) {
tt := new(testTarget)
ct := NewChannelTarget(ch)
chTarget, _ := newPayloadSizeLimiter(maxSize, func() ObjectWriter { return ct })
testTarget, _ := newPayloadSizeLimiter(maxSize, func() ObjectWriter { return tt })
chTarget, _ := newPayloadSizeLimiter(maxSize, 0, func() ObjectWriter { return ct })
testTarget, _ := newPayloadSizeLimiter(maxSize, 0, func() ObjectWriter { return tt })
ver := version.Current()
cnr := cidtest.ID()

View file

@ -0,0 +1,72 @@
package transformer
import (
"context"
"crypto/rand"
"math"
"testing"
cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"github.com/stretchr/testify/require"
)
func TestTransformerSizeHintCorrectness(t *testing.T) {
const (
maxSize = 100
payloadSize = maxSize*2 + maxSize/2
)
pk, err := keys.NewPrivateKey()
require.NoError(t, err)
p := Params{
Key: &pk.PrivateKey,
NetworkState: dummyEpochSource(123),
MaxSize: maxSize,
WithoutHomomorphicHash: true,
}
cnr := cidtest.ID()
hdr := newObject(cnr)
var owner user.ID
user.IDFromKey(&owner, pk.PrivateKey.PublicKey)
hdr.SetOwnerID(&owner)
expected := make([]byte, payloadSize)
_, _ = rand.Read(expected)
t.Run("default", func(t *testing.T) {
p.SizeHint = 0
testPayloadEqual(t, p, hdr, expected)
})
t.Run("size hint is perfect", func(t *testing.T) {
p.SizeHint = payloadSize
testPayloadEqual(t, p, hdr, expected)
})
t.Run("size hint < payload size", func(t *testing.T) {
p.SizeHint = payloadSize / 2
testPayloadEqual(t, p, hdr, expected)
})
t.Run("size hint > payload size", func(t *testing.T) {
p.SizeHint = math.MaxUint64
testPayloadEqual(t, p, hdr, expected)
})
}
func testPayloadEqual(t *testing.T, p Params, hdr *objectSDK.Object, expected []byte) {
tt := new(testTarget)
p.NextTargetInit = func() ObjectWriter { return tt }
target := NewPayloadSizeLimiter(p)
writeObject(t, context.Background(), target, hdr, expected)
var actual []byte
for i := range tt.objects {
actual = append(actual, tt.objects[i].Payload()...)
}
require.Equal(t, expected, actual)
}

View file

@ -40,6 +40,11 @@ type Params struct {
NetworkState EpochSource
MaxSize uint64
WithoutHomomorphicHash bool
// SizeHint is a hint for the total payload size to be processed.
// It is used primarily to optimize allocations and doesn't affect
// functionality. Primary usecases are providing file size when putting an object
// with the frostfs-cli or using Content-Length header in gateways.
SizeHint uint64
}
// NewPayloadSizeLimiter returns ObjectTarget instance that restricts payload length
@ -121,7 +126,18 @@ func (s *payloadSizeLimiter) initializeCurrent() {
s.nextTarget = s.NextTargetInit()
s.writtenCurrent = 0
s.initPayloadHashers()
s.payload = make([]byte, 0)
var payloadSize uint64
// Check whether SizeHint is valid.
if remaining := s.SizeHint - s.written; remaining <= s.SizeHint {
if remaining >= s.MaxSize {
payloadSize = s.MaxSize
} else {
payloadSize = remaining % s.MaxSize
}
}
s.payload = make([]byte, 0, payloadSize)
}
func (s *payloadSizeLimiter) initPayloadHashers() {

View file

@ -20,7 +20,7 @@ func TestTransformer(t *testing.T) {
tt := new(testTarget)
target, pk := newPayloadSizeLimiter(maxSize, func() ObjectWriter { return tt })
target, pk := newPayloadSizeLimiter(maxSize, 0, func() ObjectWriter { return tt })
cnr := cidtest.ID()
hdr := newObject(cnr)
@ -114,15 +114,37 @@ func writeObject(t *testing.T, ctx context.Context, target ChunkedObjectWriter,
func BenchmarkTransformer(b *testing.B) {
hdr := newObject(cidtest.ID())
const (
// bufferSize is taken from https://git.frostfs.info/TrueCloudLab/frostfs-sdk-go/src/commit/670619d2426fee233a37efe21a0471989b16a4fc/pool/pool.go#L1825
bufferSize = 3 * 1024 * 1024
smallSize = 8 * 1024
bigSize = 64 * 1024 * 1024 * 9 / 2 // 4.5 parts
)
b.Run("small", func(b *testing.B) {
benchmarkTransformer(b, hdr, 8*1024)
b.Run("no size hint", func(b *testing.B) {
benchmarkTransformer(b, hdr, smallSize, 0, 0)
})
b.Run("no size hint, with buffer", func(b *testing.B) {
benchmarkTransformer(b, hdr, smallSize, 0, bufferSize)
})
b.Run("with size hint, with buffer", func(b *testing.B) {
benchmarkTransformer(b, hdr, smallSize, smallSize, bufferSize)
})
})
b.Run("big", func(b *testing.B) {
benchmarkTransformer(b, hdr, 64*1024*1024*9/2)
b.Run("no size hint", func(b *testing.B) {
benchmarkTransformer(b, hdr, bigSize, 0, 0)
})
b.Run("no size hint, with buffer", func(b *testing.B) {
benchmarkTransformer(b, hdr, bigSize, 0, bufferSize)
})
b.Run("with size hint, with buffer", func(b *testing.B) {
benchmarkTransformer(b, hdr, bigSize, bigSize, bufferSize)
})
})
}
func benchmarkTransformer(b *testing.B, header *objectSDK.Object, payloadSize int) {
func benchmarkTransformer(b *testing.B, header *objectSDK.Object, payloadSize, sizeHint, bufferSize int) {
const maxSize = 64 * 1024 * 1024
payload := make([]byte, payloadSize)
@ -131,12 +153,24 @@ func benchmarkTransformer(b *testing.B, header *objectSDK.Object, payloadSize in
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
f, _ := newPayloadSizeLimiter(maxSize, func() ObjectWriter { return benchTarget{} })
f, _ := newPayloadSizeLimiter(maxSize, uint64(sizeHint), func() ObjectWriter { return benchTarget{} })
if err := f.WriteHeader(ctx, header); err != nil {
b.Fatalf("write header: %v", err)
}
if _, err := f.Write(ctx, payload); err != nil {
b.Fatalf("write: %v", err)
if bufferSize == 0 {
if _, err := f.Write(ctx, payload); err != nil {
b.Fatalf("write: %v", err)
}
} else {
j := 0
for ; j+bufferSize < payloadSize; j += bufferSize {
if _, err := f.Write(ctx, payload[j:j+bufferSize]); err != nil {
b.Fatalf("write: %v", err)
}
}
if _, err := f.Write(ctx, payload[j:payloadSize]); err != nil {
b.Fatalf("write: %v", err)
}
}
if _, err := f.Close(ctx); err != nil {
b.Fatalf("close: %v", err)
@ -144,7 +178,7 @@ func benchmarkTransformer(b *testing.B, header *objectSDK.Object, payloadSize in
}
}
func newPayloadSizeLimiter(maxSize uint64, nextTarget TargetInitializer) (ChunkedObjectWriter, *keys.PrivateKey) {
func newPayloadSizeLimiter(maxSize uint64, sizeHint uint64, nextTarget TargetInitializer) (ChunkedObjectWriter, *keys.PrivateKey) {
p, err := keys.NewPrivateKey()
if err != nil {
panic(err)
@ -155,6 +189,7 @@ func newPayloadSizeLimiter(maxSize uint64, nextTarget TargetInitializer) (Chunke
NextTargetInit: nextTarget,
NetworkState: dummyEpochSource(123),
MaxSize: maxSize,
SizeHint: sizeHint,
WithoutHomomorphicHash: true,
}), p
}

View file

@ -110,16 +110,11 @@ func (it *internalTarget) WriteObject(ctx context.Context, o *object.Object) err
}
func (it *internalTarget) putAsStream(ctx context.Context, o *object.Object) error {
var cliPrm sdkClient.PrmObjectPutInit
cliPrm.SetCopiesNumberByVectors(it.prm.copiesNumber)
if it.prm.stoken != nil {
cliPrm.WithinSession(*it.prm.stoken)
}
if it.prm.key != nil {
cliPrm.UseKey(*it.prm.key)
}
if it.prm.btoken != nil {
cliPrm.WithBearerToken(*it.prm.btoken)
cliPrm := sdkClient.PrmObjectPutInit{
CopiesNumber: it.prm.copiesNumber,
Session: it.prm.stoken,
Key: it.prm.key,
BearerToken: it.prm.btoken,
}
wrt, err := it.client.ObjectPutInit(ctx, cliPrm)
@ -141,16 +136,13 @@ func (it *internalTarget) tryPutSingle(ctx context.Context, o *object.Object) (b
if it.useStream {
return false, nil
}
var cliPrm sdkClient.PrmObjectPutSingle
cliPrm.SetCopiesNumber(it.prm.copiesNumber)
cliPrm.UseKey(it.prm.key)
if it.prm.stoken != nil {
cliPrm.WithinSession(*it.prm.stoken)
cliPrm := sdkClient.PrmObjectPutSingle{
CopiesNumber: it.prm.copiesNumber,
Key: it.prm.key,
Session: it.prm.stoken,
BearerToken: it.prm.btoken,
Object: o,
}
if it.prm.btoken != nil {
cliPrm.WithBearerToken(*it.prm.btoken)
}
cliPrm.SetObject(o.ToV2())
res, err := it.client.ObjectPutSingle(ctx, cliPrm)
if err != nil && status.Code(err) == codes.Unimplemented {

View file

@ -417,8 +417,9 @@ func (c *clientWrapper) balanceGet(ctx context.Context, prm PrmBalanceGet) (acco
return accounting.Decimal{}, err
}
var cliPrm sdkClient.PrmBalanceGet
cliPrm.SetAccount(prm.account)
cliPrm := sdkClient.PrmBalanceGet{
Account: &prm.account,
}
start := time.Now()
res, err := cl.BalanceGet(ctx, cliPrm)
@ -502,8 +503,9 @@ func (c *clientWrapper) containerList(ctx context.Context, prm PrmContainerList)
return nil, err
}
var cliPrm sdkClient.PrmContainerList
cliPrm.SetAccount(prm.ownerID)
cliPrm := sdkClient.PrmContainerList{
Account: &prm.ownerID,
}
start := time.Now()
res, err := cl.ContainerList(ctx, cliPrm)
@ -998,21 +1000,12 @@ func (c *clientWrapper) objectSearch(ctx context.Context, prm PrmObjectSearch) (
return ResObjectSearch{}, err
}
var cliPrm sdkClient.PrmObjectSearch
cliPrm.InContainer(prm.cnrID)
cliPrm.SetFilters(prm.filters)
if prm.stoken != nil {
cliPrm.WithinSession(*prm.stoken)
}
if prm.btoken != nil {
cliPrm.WithBearerToken(*prm.btoken)
}
if prm.key != nil {
cliPrm.UseKey(*prm.key)
cliPrm := sdkClient.PrmObjectSearch{
ContainerID: &prm.cnrID,
Filters: prm.filters,
Session: prm.stoken,
BearerToken: prm.btoken,
Key: prm.key,
}
res, err := cl.ObjectSearchInit(ctx, cliPrm)