aarifullin/frostfs-sdk-go
1
0
Fork 0
forked from TrueCloudLab/frostfs-sdk-go
Code Pull requests Activity

[#227] netmap: Refactor and document package functionality

Browse source 
Signed-off-by: Leonard Lyubich <leonard@nspcc.ru>
This commit is contained in:
Leonard Lyubich 2022-06-07 11:25:34 +03:00 committed by LeL
parent 5bfdb64251
commit ca523f1ff1
51 changed files with 2460 additions and 3819 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
client/netmap.go
View file

@ -2,6 +2,8 @@ package client
import (
"context"
"errors"
"fmt"
v2netmap "github.com/nspcc-dev/neofs-api-go/v2/netmap"
rpcapi "github.com/nspcc-dev/neofs-api-go/v2/rpc"
@ -96,7 +98,22 @@ func (c *Client) EndpointInfo(ctx context.Context, prm PrmEndpointInfo) (*ResEnd
ver.ReadFromV2(*v2ver)
}
res.setLatestVersion(&ver)
res.setNodeInfo(netmap.NewNodeInfoFromV2(body.GetNodeInfo()))
nodeV2 := body.GetNodeInfo()
if nodeV2 == nil {
cc.err = errors.New("missing node info in response body")
return
}
var node netmap.NodeInfo
cc.err = node.ReadFromV2(*nodeV2)
if cc.err != nil {
cc.err = fmt.Errorf("invalid node info: %w", cc.err)
return
}
res.setNodeInfo(&node)
}
// process call
@ -171,7 +188,21 @@ func (c *Client) NetworkInfo(ctx context.Context, prm PrmNetworkInfo) (*ResNetwo
cc.result = func(r responseV2) {
resp := r.(*v2netmap.NetworkInfoResponse)
res.setInfo(netmap.NewNetworkInfoFromV2(resp.GetBody().GetNetworkInfo()))
netInfoV2 := resp.GetBody().GetNetworkInfo()
if netInfoV2 == nil {
cc.err = errors.New("missing network info in response body")
return
}
var netInfo netmap.NetworkInfo
cc.err = netInfo.ReadFromV2(*netInfoV2)
if cc.err != nil {
cc.err = fmt.Errorf("invalid network info: %w", cc.err)
return
}
res.setInfo(&netInfo)
}
// process call

24
container/container.go
View file

@ -5,6 +5,7 @@ import (
"github.com/google/uuid"
"github.com/nspcc-dev/neofs-api-go/v2/container"
v2netmap "github.com/nspcc-dev/neofs-api-go/v2/netmap"
"github.com/nspcc-dev/neofs-api-go/v2/refs"
"github.com/nspcc-dev/neofs-sdk-go/acl"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
@ -158,11 +159,30 @@ func (c *Container) SetAttributes(v Attributes) {
}
func (c *Container) PlacementPolicy() *netmap.PlacementPolicy {
return netmap.NewPlacementPolicyFromV2(c.v2.GetPlacementPolicy())
m := c.v2.GetPlacementPolicy()
if m == nil {
return nil
}
var p netmap.PlacementPolicy
// FIXME(@cthulhu-rider): #225 handle error
err := p.ReadFromV2(*m)
if err != nil {
panic(err)
}
return &p
}
func (c *Container) SetPlacementPolicy(v *netmap.PlacementPolicy) {
c.v2.SetPlacementPolicy(v.ToV2())
var m *v2netmap.PlacementPolicy
if v != nil {
m = new(v2netmap.PlacementPolicy)
v.WriteToV2(m)
}
c.v2.SetPlacementPolicy(m)
}
// SessionToken returns token of the session within

4
container/container_test.go
View file

@ -29,7 +29,7 @@ func TestNewContainer(t *testing.T) {
attrs := containertest.Attributes()
c.SetAttributes(attrs)
c.SetPlacementPolicy(policy)
c.SetPlacementPolicy(&policy)
c.SetNonceUUID(nonce)
c.SetOwnerID(ownerID)
@ -39,7 +39,7 @@ func TestNewContainer(t *testing.T) {
v2 := c.ToV2()
newContainer := container.NewContainerFromV2(v2)
require.EqualValues(t, newContainer.PlacementPolicy(), policy)
require.EqualValues(t, newContainer.PlacementPolicy(), &policy)
require.EqualValues(t, newContainer.Attributes(), attrs)
require.EqualValues(t, newContainer.BasicACL(), acl.PublicBasicRule)

3
container/test/generate.go
View file

@ -32,7 +32,8 @@ func Container() *container.Container {
x.SetAttributes(Attributes())
x.SetOwnerID(usertest.ID())
x.SetBasicACL(123)
x.SetPlacementPolicy(netmaptest.PlacementPolicy())
p := netmaptest.PlacementPolicy()
x.SetPlacementPolicy(&p)
return x
}

2
netmap/aggregator.go
View file

@ -56,7 +56,7 @@ var (
// capacity and price.
func newWeightFunc(capNorm, priceNorm normalizer) weightFunc {
return func(n NodeInfo) float64 {
return capNorm.Normalize(float64(n.capacity())) * priceNorm.Normalize(float64(n.price()))
return capNorm.Normalize(float64(n.capacity())) * priceNorm.Normalize(float64(n.Price()))
}
}

69
netmap/clause.go
View file

@ -1,69 +0,0 @@
package netmap
import (
"github.com/nspcc-dev/neofs-api-go/v2/netmap"
)
// Clause is an enumeration of selector modifiers
// that shows how the node set will be formed.
type Clause uint32
const (
ClauseUnspecified Clause = iota
// ClauseSame is a selector modifier to select only nodes having the same value of bucket attribute.
ClauseSame
// ClauseDistinct is a selector modifier to select nodes having different values of bucket attribute.
ClauseDistinct
)
// ClauseFromV2 converts v2 Clause to Clause.
func ClauseFromV2(c netmap.Clause) Clause {
switch c {
default:
return ClauseUnspecified
case netmap.Same:
return ClauseSame
case netmap.Distinct:
return ClauseDistinct
}
}
// ToV2 converts Clause to v2 Clause.
func (c Clause) ToV2() netmap.Clause {
switch c {
default:
return netmap.UnspecifiedClause
case ClauseDistinct:
return netmap.Distinct
case ClauseSame:
return netmap.Same
}
}
// String returns string representation of Clause.
//
// String mapping:
// * ClauseDistinct: DISTINCT;
// * ClauseSame: SAME;
// * ClauseUnspecified, default: CLAUSE_UNSPECIFIED.
func (c Clause) String() string {
return c.ToV2().String()
}
// FromString parses Clause from a string representation.
// It is a reverse action to String().
//
// Returns true if s was parsed successfully.
func (c *Clause) FromString(s string) bool {
var g netmap.Clause
ok := g.FromString(s)
if ok {
*c = ClauseFromV2(g)
}
return ok
}

43
netmap/clause_test.go
View file

@ -1,43 +0,0 @@
package netmap
import (
"testing"
"github.com/nspcc-dev/neofs-api-go/v2/netmap"
"github.com/stretchr/testify/require"
)
func TestClauseFromV2(t *testing.T) {
for _, item := range []struct {
c Clause
cV2 netmap.Clause
}{
{
c: ClauseUnspecified,
cV2: netmap.UnspecifiedClause,
},
{
c: ClauseSame,
cV2: netmap.Same,
},
{
c: ClauseDistinct,
cV2: netmap.Distinct,
},
} {
require.Equal(t, item.c, ClauseFromV2(item.cV2))
require.Equal(t, item.cV2, item.c.ToV2())
}
}
func TestClause_String(t *testing.T) {
toPtr := func(v Clause) *Clause {
return &v
}
testEnumStrings(t, new(Clause), []enumStringItem{
{val: toPtr(ClauseDistinct), str: "DISTINCT"},
{val: toPtr(ClauseSame), str: "SAME"},
{val: toPtr(ClauseUnspecified), str: "CLAUSE_UNSPECIFIED"},
})
}

85
netmap/context.go
View file

@ -4,88 +4,85 @@ import (
"errors"
"github.com/nspcc-dev/hrw"
"github.com/nspcc-dev/neofs-api-go/v2/netmap"
)
// context contains references to named filters and cached numeric values.
// context of a placement build process.
type context struct {
// Netmap is a netmap structure to operate on.
Netmap *Netmap
// Filters stores processed filters.
Filters map[string]*Filter
// Selectors stores processed selectors.
Selectors map[string]*Selector
// Selections stores result of selector processing.
Selections map[string][]nodes
// network map to operate on
netMap NetMap
// numCache stores parsed numeric values.
// cache of processed filters
processedFilters map[string]*netmap.Filter
// cache of processed selectors
processedSelectors map[string]*netmap.Selector
// stores results of selector processing
selections map[string][]nodes
// cache of parsed numeric values
numCache map[string]uint64
// pivot is a seed for HRW.
pivot []byte
// pivotHash is a saved HRW hash of pivot
pivotHash uint64
// aggregator is returns aggregator determining bucket weight.
// By default it returns mean value from IQR interval.
aggregator func() aggregator
// weightFunc is a weighting function for determining node priority.
// By default in combines favours low price and high capacity.
hrwSeed []byte
// hrw.Hash of hrwSeed
hrwSeedHash uint64
// weightFunc is a weighting function for determining node priority
// which combines low price and high performance
weightFunc weightFunc
// container backup factor is a factor for selector counters that expand
// amount of chosen nodes.
// container backup factor
cbf uint32
}
// Various validation errors.
var (
ErrMissingField = errors.New("netmap: nil field")
ErrInvalidFilterName = errors.New("netmap: filter name is invalid")
ErrInvalidNumber = errors.New("netmap: number value expected")
ErrInvalidFilterOp = errors.New("netmap: invalid filter operation")
ErrFilterNotFound = errors.New("netmap: filter not found")
ErrNonEmptyFilters = errors.New("netmap: simple filter must no contain sub-filters")
ErrNotEnoughNodes = errors.New("netmap: not enough nodes to SELECT from")
ErrSelectorNotFound = errors.New("netmap: selector not found")
ErrUnnamedTopFilter = errors.New("netmap: all filters on top level must be named")
errInvalidFilterName = errors.New("filter name is invalid")
errInvalidNumber = errors.New("invalid number")
errInvalidFilterOp = errors.New("invalid filter operation")
errFilterNotFound = errors.New("filter not found")
errNonEmptyFilters = errors.New("simple filter contains sub-filters")
errNotEnoughNodes = errors.New("not enough nodes to SELECT from")
errUnnamedTopFilter = errors.New("unnamed top-level filter")
)
// newContext creates new context. It contains various caches.
// In future it may create hierarchical netmap structure to work with.
func newContext(nm *Netmap) *context {
// newContext returns initialized context.
func newContext(nm NetMap) *context {
return &context{
Netmap: nm,
Filters: make(map[string]*Filter),
Selectors: make(map[string]*Selector),
Selections: make(map[string][]nodes),
netMap: nm,
processedFilters: make(map[string]*netmap.Filter),
processedSelectors: make(map[string]*netmap.Selector),
selections: make(map[string][]nodes),
numCache: make(map[string]uint64),
aggregator: newMeanIQRAgg,
weightFunc: defaultWeightFunc(nm.nodes),
cbf: defaultCBF,
}
}
func (c *context) setPivot(pivot []byte) {
if len(pivot) != 0 {
c.pivot = pivot
c.pivotHash = hrw.Hash(pivot)
c.hrwSeed = pivot
c.hrwSeedHash = hrw.Hash(pivot)
}
}
func (c *context) setCBF(cbf uint32) {
if cbf == 0 {
c.cbf = defaultCBF
c.cbf = 3
} else {
c.cbf = cbf
}
}
// defaultWeightFunc returns default weighting function.
func defaultWeightFunc(ns nodes) weightFunc {
mean := newMeanAgg()
min := newMinAgg()
for i := range ns {
mean.Add(float64(ns[i].capacity()))
min.Add(float64(ns[i].price()))
min.Add(float64(ns[i].Price()))
}
return newWeightFunc(

45
netmap/doc.go
View file

@ -1,11 +1,42 @@
/*
Package netmap provides routines for working with netmap and placement policy.
Work is done in 4 steps:
1. Create context containing results shared between steps.
2. Processing filters.
3. Processing selectors.
4. Processing replicas.
Package netmap provides functionality for working with information about the
NeoFS network, primarily a layer of storage nodes.
The package concentrates all the characteristics of NeoFS networks.
NetMap represents NeoFS network map - one of the main technologies used to
store data in the system. It is composed of information about all storage nodes
(NodeInfo type) in a particular network. NetMap methods allow you to impose
container storage policies (PlacementPolicy type) on a fixed composition of
nodes for selecting nodes corresponding to the placement rules chosen by the
container creator.
NetworkInfo type is dedicated to descriptive characterization of network state
and settings.
Instances can be also used to process NeoFS API V2 protocol messages
(see neo.fs.v2.netmap package in https://github.com/nspcc-dev/neofs-api).
On client side:
import "github.com/nspcc-dev/neofs-api-go/v2/netmap"
var msg netmap.NodeInfo
msg.WriteToV2(&msg)
// send msg
On server side:
// recv msg
var info netmap.NodeInfo
err := info.ReadFromV2(msg)
// ...
// process dec
Using package types in an application is recommended to potentially work with
different protocol versions with which these types are compatible.
Each step depends only on previous ones.
*/
package netmap

257
netmap/filter.go
View file

@ -7,77 +7,68 @@ import (
"github.com/nspcc-dev/neofs-api-go/v2/netmap"
)
// Filter represents v2-compatible netmap filter.
type Filter netmap.Filter
// MainFilterName is a name of the filter
// mainFilterName is a name of the filter
// which points to the whole netmap.
const MainFilterName = "*"
// applyFilter applies named filter to b.
func (c *context) applyFilter(name string, b NodeInfo) bool {
return name == MainFilterName || c.match(c.Filters[name], b)
}
const mainFilterName = "*"
// processFilters processes filters and returns error is any of them is invalid.
func (c *context) processFilters(p *PlacementPolicy) error {
filters := p.Filters()
for i := range filters {
if err := c.processFilter(&filters[i], true); err != nil {
return err
func (c *context) processFilters(p PlacementPolicy) error {
for i := range p.filters {
if err := c.processFilter(p.filters[i], true); err != nil {
return fmt.Errorf("process filter #%d (%s): %w", i, p.filters[i].GetName(), err)
}
}
return nil
}
func (c *context) processFilter(f *Filter, top bool) error {
if f == nil {
return fmt.Errorf("%w: FILTER", ErrMissingField)
func (c *context) processFilter(f netmap.Filter, top bool) error {
fName := f.GetName()
if fName == mainFilterName {
return fmt.Errorf("%w: '%s' is reserved", errInvalidFilterName, mainFilterName)
}
if f.Name() == MainFilterName {
return fmt.Errorf("%w: '*' is reserved", ErrInvalidFilterName)
if top && fName == "" {
return errUnnamedTopFilter
}
if top && f.Name() == "" {
return ErrUnnamedTopFilter
if !top && fName != "" && c.processedFilters[fName] == nil {
return errFilterNotFound
}
if !top && f.Name() != "" && c.Filters[f.Name()] == nil {
return fmt.Errorf("%w: '%s'", ErrFilterNotFound, f.Name())
}
inner := f.GetFilters()
switch f.Operation() {
case OpAND, OpOR:
for _, flt := range f.InnerFilters() {
if err := c.processFilter(&flt, false); err != nil {
return err
switch op := f.GetOp(); op {
case netmap.AND, netmap.OR:
for i := range inner {
if err := c.processFilter(inner[i], false); err != nil {
return fmt.Errorf("process inner filter #%d: %w", i, err)
}
}
default:
if len(f.InnerFilters()) != 0 {
return ErrNonEmptyFilters
} else if !top && f.Name() != "" { // named reference
if len(inner) != 0 {
return errNonEmptyFilters
} else if !top && fName != "" { // named reference
return nil
}
switch f.Operation() {
case OpEQ, OpNE:
case OpGT, OpGE, OpLT, OpLE:
n, err := strconv.ParseUint(f.Value(), 10, 64)
switch op {
case netmap.EQ, netmap.NE:
case netmap.GT, netmap.GE, netmap.LT, netmap.LE:
val := f.GetValue()
n, err := strconv.ParseUint(val, 10, 64)
if err != nil {
return fmt.Errorf("%w: '%s'", ErrInvalidNumber, f.Value())
return fmt.Errorf("%w: '%s'", errInvalidNumber, f.GetValue())
}
c.numCache[f.Value()] = n
c.numCache[val] = n
default:
return fmt.Errorf("%w: %s", ErrInvalidFilterOp, f.Operation())
return fmt.Errorf("%w: %s", errInvalidFilterOp, op)
}
}
if top {
c.Filters[f.Name()] = f
c.processedFilters[fName] = &f
}
return nil
@ -86,44 +77,46 @@ func (c *context) processFilter(f *Filter, top bool) error {
// match matches f against b. It returns no errors because
// filter should have been parsed during context creation
// and missing node properties are considered as a regular fail.
func (c *context) match(f *Filter, b NodeInfo) bool {
switch f.Operation() {
case OpAND, OpOR:
for _, lf := range f.InnerFilters() {
if lf.Name() != "" {
lf = *c.Filters[lf.Name()]
func (c *context) match(f *netmap.Filter, b NodeInfo) bool {
switch f.GetOp() {
case netmap.AND, netmap.OR:
inner := f.GetFilters()
for i := range inner {
fSub := &inner[i]
if name := inner[i].GetName(); name != "" {
fSub = c.processedFilters[name]
}
ok := c.match(&lf, b)
if ok == (f.Operation() == OpOR) {
ok := c.match(fSub, b)
if ok == (f.GetOp() == netmap.OR) {
return ok
}
}
return f.Operation() == OpAND
return f.GetOp() == netmap.AND
default:
return c.matchKeyValue(f, b)
}
}
func (c *context) matchKeyValue(f *Filter, b NodeInfo) bool {
switch f.Operation() {
case OpEQ:
return b.attribute(f.Key()) == f.Value()
case OpNE:
return b.attribute(f.Key()) != f.Value()
func (c *context) matchKeyValue(f *netmap.Filter, b NodeInfo) bool {
switch op := f.GetOp(); op {
case netmap.EQ:
return b.Attribute(f.GetKey()) == f.GetValue()
case netmap.NE:
return b.Attribute(f.GetKey()) != f.GetValue()
default:
var attr uint64
switch f.Key() {
case AttrPrice:
attr = b.price()
case AttrCapacity:
switch f.GetKey() {
case attrPrice:
attr = b.Price()
case attrCapacity:
attr = b.capacity()
default:
var err error
attr, err = strconv.ParseUint(b.attribute(f.Key()), 10, 64)
attr, err = strconv.ParseUint(b.Attribute(f.GetKey()), 10, 64)
if err != nil {
// Note: because filters are somewhat independent from nodes attributes,
// We don't report an error here, and fail filter instead.
@ -131,15 +124,15 @@ func (c *context) matchKeyValue(f *Filter, b NodeInfo) bool {
}
}
switch f.Operation() {
case OpGT:
return attr > c.numCache[f.Value()]
case OpGE:
return attr >= c.numCache[f.Value()]
case OpLT:
return attr < c.numCache[f.Value()]
case OpLE:
return attr <= c.numCache[f.Value()]
switch op {
case netmap.GT:
return attr > c.numCache[f.GetValue()]
case netmap.GE:
return attr >= c.numCache[f.GetValue()]
case netmap.LT:
return attr < c.numCache[f.GetValue()]
case netmap.LE:
return attr <= c.numCache[f.GetValue()]
default:
// do nothing and return false
}
@ -147,127 +140,3 @@ func (c *context) matchKeyValue(f *Filter, b NodeInfo) bool {
// will not happen if context was created from f (maybe panic?)
return false
}
// NewFilter creates and returns new Filter instance.
//
// Defaults:
// - name: "";
// - key: "";
// - value: "";
// - operation: 0;
// - filters: nil.
func NewFilter() *Filter {
return NewFilterFromV2(new(netmap.Filter))
}
// NewFilterFromV2 converts v2 Filter to Filter.
//
// Nil netmap.Filter converts to nil.
func NewFilterFromV2(f *netmap.Filter) *Filter {
return (*Filter)(f)
}
// ToV2 converts Filter to v2 Filter.
//
// Nil Filter converts to nil.
func (f *Filter) ToV2() *netmap.Filter {
return (*netmap.Filter)(f)
}
// Key returns key to filter.
func (f *Filter) Key() string {
return (*netmap.Filter)(f).GetKey()
}
// SetKey sets key to filter.
func (f *Filter) SetKey(key string) {
(*netmap.Filter)(f).SetKey(key)
}
// Value returns value to match.
func (f *Filter) Value() string {
return (*netmap.Filter)(f).GetValue()
}
// SetValue sets value to match.
func (f *Filter) SetValue(val string) {
(*netmap.Filter)(f).SetValue(val)
}
// Name returns filter name.
func (f *Filter) Name() string {
return (*netmap.Filter)(f).GetName()
}
// SetName sets filter name.
func (f *Filter) SetName(name string) {
(*netmap.Filter)(f).SetName(name)
}
// Operation returns filtering operation.
func (f *Filter) Operation() Operation {
return OperationFromV2(
(*netmap.Filter)(f).GetOp())
}
// SetOperation sets filtering operation.
func (f *Filter) SetOperation(op Operation) {
(*netmap.Filter)(f).SetOp(op.ToV2())
}
func filtersFromV2(fs []netmap.Filter) []Filter {
if fs == nil {
return nil
}
res := make([]Filter, len(fs))
for i := range fs {
res[i] = *NewFilterFromV2(&fs[i])
}
return res
}
// InnerFilters returns list of inner filters.
func (f *Filter) InnerFilters() []Filter {
return filtersFromV2((*netmap.Filter)(f).GetFilters())
}
func filtersToV2(fs []Filter) (fsV2 []netmap.Filter) {
if fs != nil {
fsV2 = make([]netmap.Filter, len(fs))
for i := range fs {
fsV2[i] = *fs[i].ToV2()
}
}
return
}
// SetInnerFilters sets list of inner filters.
func (f *Filter) SetInnerFilters(fs ...Filter) {
(*netmap.Filter)(f).
SetFilters(filtersToV2(fs))
}
// Marshal marshals Filter into a protobuf binary form.
func (f *Filter) Marshal() ([]byte, error) {
return (*netmap.Filter)(f).StableMarshal(nil), nil
}
// Unmarshal unmarshals protobuf binary representation of Filter.
func (f *Filter) Unmarshal(data []byte) error {
return (*netmap.Filter)(f).Unmarshal(data)
}
// MarshalJSON encodes Filter to protobuf JSON format.
func (f *Filter) MarshalJSON() ([]byte, error) {
return (*netmap.Filter)(f).MarshalJSON()
}
// UnmarshalJSON decodes Filter from protobuf JSON format.
func (f *Filter) UnmarshalJSON(data []byte) error {
return (*netmap.Filter)(f).UnmarshalJSON(data)
}

189
netmap/filter_test.go
View file

@ -10,24 +10,24 @@ import (
func TestContext_ProcessFilters(t *testing.T) {
fs := []Filter{
newFilter("StorageSSD", "Storage", "SSD", OpEQ),
newFilter("GoodRating", "Rating", "4", OpGE),
newFilter("Main", "", "", OpAND,
newFilter("StorageSSD", "Storage", "SSD", netmap.EQ),
newFilter("GoodRating", "Rating", "4", netmap.GE),
newFilter("Main", "", "", netmap.AND,
newFilter("StorageSSD", "", "", 0),
newFilter("", "IntField", "123", OpLT),
newFilter("", "IntField", "123", netmap.LT),
newFilter("GoodRating", "", "", 0)),
}
c := newContext(new(Netmap))
c := newContext(NetMap{})
p := newPlacementPolicy(1, nil, nil, fs)
require.NoError(t, c.processFilters(p))
require.Equal(t, 3, len(c.Filters))
require.Equal(t, 3, len(c.processedFilters))
for _, f := range fs {
require.Equal(t, f, *c.Filters[f.Name()])
require.Equal(t, f.m, *c.processedFilters[f.m.GetName()])
}
require.Equal(t, uint64(4), c.numCache[fs[1].Value()])
require.Equal(t, uint64(123), c.numCache[fs[2].InnerFilters()[1].Value()])
require.Equal(t, uint64(4), c.numCache[fs[1].m.GetValue()])
require.Equal(t, uint64(123), c.numCache[fs[2].m.GetFilters()[1].GetValue()])
}
func TestContext_ProcessFiltersInvalid(t *testing.T) {
@ -38,40 +38,40 @@ func TestContext_ProcessFiltersInvalid(t *testing.T) {
}{
{
"UnnamedTop",
newFilter("", "Storage", "SSD", OpEQ),
ErrUnnamedTopFilter,
newFilter("", "Storage", "SSD", netmap.EQ),
errUnnamedTopFilter,
},
{
"InvalidReference",
newFilter("Main", "", "", OpAND,
newFilter("Main", "", "", netmap.AND,
newFilter("StorageSSD", "", "", 0)),
ErrFilterNotFound,
errFilterNotFound,
},
{
"NonEmptyKeyed",
newFilter("Main", "Storage", "SSD", OpEQ,
newFilter("Main", "Storage", "SSD", netmap.EQ,
newFilter("StorageSSD", "", "", 0)),
ErrNonEmptyFilters,
errNonEmptyFilters,
},
{
"InvalidNumber",
newFilter("Main", "Rating", "three", OpGE),
ErrInvalidNumber,
newFilter("Main", "Rating", "three", netmap.GE),
errInvalidNumber,
},
{
"InvalidOp",
newFilter("Main", "Rating", "3", 0),
ErrInvalidFilterOp,
errInvalidFilterOp,
},
{
"InvalidName",
newFilter("*", "Rating", "3", OpGE),
ErrInvalidFilterName,
newFilter("*", "Rating", "3", netmap.GE),
errInvalidFilterName,
},
}
for _, tc := range errTestCases {
t.Run(tc.name, func(t *testing.T) {
c := newContext(new(Netmap))
c := newContext(NetMap{})
p := newPlacementPolicy(1, nil, nil, []Filter{tc.filter})
err := c.processFilters(p)
require.True(t, errors.Is(err, tc.err), "got: %v", err)
@ -80,151 +80,16 @@ func TestContext_ProcessFiltersInvalid(t *testing.T) {
}
func TestFilter_MatchSimple_InvalidOp(t *testing.T) {
var aRating NodeAttribute
aRating.SetKey("Rating")
aRating.SetValue("4")
var aCountry NodeAttribute
aRating.SetKey("Country")
aRating.SetValue("Germany")
var b NodeInfo
b.SetAttributes(aRating, aCountry)
b.SetAttribute("Rating", "4")
b.SetAttribute("Country", "Germany")
f := newFilter("Main", "Rating", "5", OpEQ)
c := newContext(new(Netmap))
f := newFilter("Main", "Rating", "5", netmap.EQ)
c := newContext(NetMap{})
p := newPlacementPolicy(1, nil, nil, []Filter{f})
require.NoError(t, c.processFilters(p))
// just for the coverage
f.SetOperation(0)
require.False(t, c.match(&f, b))
}
func testFilter() *Filter {
f := NewFilter()
f.SetOperation(OpGE)
f.SetName("name")
f.SetKey("key")
f.SetValue("value")
return f
}
func TestFilterFromV2(t *testing.T) {
t.Run("nil from V2", func(t *testing.T) {
var x *netmap.Filter
require.Nil(t, NewFilterFromV2(x))
})
t.Run("nil to V2", func(t *testing.T) {
var x *Filter
require.Nil(t, x.ToV2())
})
fV2 := new(netmap.Filter)
fV2.SetOp(netmap.GE)
fV2.SetName("name")
fV2.SetKey("key")
fV2.SetValue("value")
f := NewFilterFromV2(fV2)
require.Equal(t, fV2, f.ToV2())
}
func TestFilter_Key(t *testing.T) {
f := NewFilter()
key := "some key"
f.SetKey(key)
require.Equal(t, key, f.Key())
}
func TestFilter_Value(t *testing.T) {
f := NewFilter()
val := "some value"
f.SetValue(val)
require.Equal(t, val, f.Value())
}
func TestFilter_Name(t *testing.T) {
f := NewFilter()
name := "some name"
f.SetName(name)
require.Equal(t, name, f.Name())
}
func TestFilter_Operation(t *testing.T) {
f := NewFilter()
op := OpGE
f.SetOperation(op)
require.Equal(t, op, f.Operation())
}
func TestFilter_InnerFilters(t *testing.T) {
f := NewFilter()
f1, f2 := *testFilter(), *testFilter()
f.SetInnerFilters(f1, f2)
require.Equal(t, []Filter{f1, f2}, f.InnerFilters())
}
func TestFilterEncoding(t *testing.T) {
f := newFilter("name", "key", "value", OpEQ,
newFilter("name2", "key2", "value", OpOR),
)
t.Run("binary", func(t *testing.T) {
data, err := f.Marshal()
require.NoError(t, err)
f2 := *NewFilter()
require.NoError(t, f2.Unmarshal(data))
require.Equal(t, f, f2)
})
t.Run("json", func(t *testing.T) {
data, err := f.MarshalJSON()
require.NoError(t, err)
f2 := *NewFilter()
require.NoError(t, f2.UnmarshalJSON(data))
require.Equal(t, f, f2)
})
}
func TestNewFilter(t *testing.T) {
t.Run("default values", func(t *testing.T) {
filter := NewFilter()
// check initial values
require.Empty(t, filter.Name())
require.Empty(t, filter.Key())
require.Empty(t, filter.Value())
require.Zero(t, filter.Operation())
require.Nil(t, filter.InnerFilters())
// convert to v2 message
filterV2 := filter.ToV2()
require.Empty(t, filterV2.GetName())
require.Empty(t, filterV2.GetKey())
require.Empty(t, filterV2.GetValue())
require.Equal(t, netmap.UnspecifiedOperation, filterV2.GetOp())
require.Nil(t, filterV2.GetFilters())
})
f.m.SetOp(0)
require.False(t, c.match(&f.m, b))
}

84
netmap/helper_test.go
View file

@ -1,81 +1,49 @@
package netmap
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/nspcc-dev/neofs-api-go/v2/netmap"
)
func newFilter(name string, k, v string, op Operation, fs ...Filter) (f Filter) {
func newFilter(name string, k, v string, op netmap.Operation, fs ...Filter) (f Filter) {
f.SetName(name)
f.SetKey(k)
f.SetOperation(op)
f.SetValue(v)
f.SetInnerFilters(fs...)
f.m.SetKey(k)
f.m.SetOp(op)
f.m.SetValue(v)
inner := make([]netmap.Filter, len(fs))
for i := range fs {
inner[i] = fs[i].m
}
f.m.SetFilters(inner)
return f
}
func newSelector(name string, attr string, c Clause, count uint32, filter string) (s Selector) {
func newSelector(name string, attr string, count uint32, filter string, clause func(*Selector)) (s Selector) {
s.SetName(name)
s.SetAttribute(attr)
s.SetCount(count)
s.SetClause(c)
s.SetFilter(filter)
s.SelectByBucketAttribute(attr)
s.SetNodeAmount(count)
clause(&s)
s.SetFilterName(filter)
return s
}
func newPlacementPolicy(bf uint32, rs []Replica, ss []Selector, fs []Filter) *PlacementPolicy {
p := NewPlacementPolicy()
func newPlacementPolicy(bf uint32, rs []ReplicaDescriptor, ss []Selector, fs []Filter) (p PlacementPolicy) {
p.SetContainerBackupFactor(bf)
p.SetReplicas(rs...)
p.SetSelectors(ss...)
p.SetFilters(fs...)
p.AddReplicas(rs...)
p.AddSelectors(ss...)
p.AddFilters(fs...)
return p
}
func newReplica(c uint32, s string) (r Replica) {
r.SetCount(c)
r.SetSelector(s)
func newReplica(c uint32, s string) (r ReplicaDescriptor) {
r.SetAmount(c)
r.SetSelectorName(s)
return r
}
func nodeInfoFromAttributes(props ...string) NodeInfo {
attrs := make([]NodeAttribute, len(props)/2)
for i := range attrs {
attrs[i].SetKey(props[i*2])
attrs[i].SetValue(props[i*2+1])
}
n := NewNodeInfo()
n.SetAttributes(attrs...)
return *n
}
type enumIface interface {
FromString(string) bool
String() string
}
type enumStringItem struct {
val enumIface
str string
}
func testEnumStrings(t *testing.T, e enumIface, items []enumStringItem) {
for _, item := range items {
require.Equal(t, item.str, item.val.String())
s := item.val.String()
require.True(t, e.FromString(s), s)
require.EqualValues(t, item.val, e, item.val)
func nodeInfoFromAttributes(props ...string) (n NodeInfo) {
for i := 0; i < len(props); i += 2 {
n.SetAttribute(props[i], props[i+1])
}
// incorrect strings
for _, str := range []string{
"some string",
"undefined",
} {
require.False(t, e.FromString(str))
}
return
}

18
netmap/json_test.go
View file

@ -26,6 +26,8 @@ type TestCase struct {
}
}
var _, _ json.Unmarshaler = new(NodeInfo), new(PlacementPolicy)
func compareNodes(t testing.TB, expected [][]int, nodes nodes, actual [][]NodeInfo) {
require.Equal(t, len(expected), len(actual))
for i := range expected {
@ -56,12 +58,12 @@ func TestPlacementPolicy_Interopability(t *testing.T) {
copy(srcNodes, tc.Nodes)
t.Run(tc.Name, func(t *testing.T) {
var nm Netmap
var nm NetMap
nm.SetNodes(tc.Nodes)
for name, tt := range tc.Tests {
t.Run(name, func(t *testing.T) {
v, err := nm.GetContainerNodes(&tt.Policy, tt.Pivot)
v, err := nm.ContainerNodes(tt.Policy, tt.Pivot)
if tt.Result == nil {
require.Error(t, err)
require.Contains(t, err.Error(), tt.Error)
@ -72,7 +74,7 @@ func TestPlacementPolicy_Interopability(t *testing.T) {
compareNodes(t, tt.Result, tc.Nodes, v)
if tt.Placement.Result != nil {
res, err := nm.GetPlacementVectors(v, tt.Placement.Pivot)
res, err := nm.PlacementVectors(v, tt.Placement.Pivot)
require.NoError(t, err)
compareNodes(t, tt.Placement.Result, tc.Nodes, res)
require.Equal(t, srcNodes, tc.Nodes)
@ -101,7 +103,7 @@ func BenchmarkPlacementPolicyInteropability(b *testing.B) {
require.NoError(b, json.Unmarshal(bs, &tc), "cannot unmarshal %s", ds[i].Name())
b.Run(tc.Name, func(b *testing.B) {
var nm Netmap
var nm NetMap
nm.SetNodes(tc.Nodes)
require.NoError(b, err)
@ -111,7 +113,7 @@ func BenchmarkPlacementPolicyInteropability(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
b.StartTimer()
v, err := nm.GetContainerNodes(&tt.Policy, tt.Pivot)
v, err := nm.ContainerNodes(tt.Policy, tt.Pivot)
b.StopTimer()
if tt.Result == nil {
require.Error(b, err)
@ -123,7 +125,7 @@ func BenchmarkPlacementPolicyInteropability(b *testing.B) {
if tt.Placement.Result != nil {
b.StartTimer()
res, err := nm.GetPlacementVectors(v, tt.Placement.Pivot)
res, err := nm.PlacementVectors(v, tt.Placement.Pivot)
b.StopTimer()
require.NoError(b, err)
compareNodes(b, tt.Placement.Result, tc.Nodes, res)
@ -146,14 +148,14 @@ func BenchmarkManySelects(b *testing.B) {
tt, ok := tc.Tests["Select"]
require.True(b, ok)
var nm Netmap
var nm NetMap
nm.SetNodes(tc.Nodes)
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
_, err = nm.GetContainerNodes(&tt.Policy, tt.Pivot)
_, err = nm.ContainerNodes(tt.Policy, tt.Pivot)
if err != nil {
b.FailNow()
}

2
netmap/json_tests/filter_invalid_integer.json
View file

@ -5,7 +5,7 @@
"attributes": [
{
"key": "IntegerField",
"value": ""
"value": "true"
}
]
},

158
netmap/netmap.go
View file

@ -1,112 +1,40 @@
package netmap
import (
"bytes"
"fmt"
"strconv"
"github.com/nspcc-dev/hrw"
"github.com/nspcc-dev/neofs-api-go/v2/netmap"
)
const defaultCBF = 3
var _, _ hrw.Hasher = NodeInfo{}, nodes{}
// Hash implements hrw.Hasher interface.
//
// Hash is needed to support weighted HRW therefore sort function sorts nodes
// based on their public key.
func (i NodeInfo) Hash() uint64 {
return hrw.Hash(i.m.GetPublicKey())
}
func (i NodeInfo) less(i2 NodeInfo) bool {
return bytes.Compare(i.PublicKey(), i2.PublicKey()) < 0
}
// attribute returns value of the node attribute by the given key. Returns empty
// string if attribute is missing.
//
// Method is needed to internal placement needs.
func (i NodeInfo) attribute(key string) string {
as := i.m.GetAttributes()
for j := range as {
if as[j].GetKey() == key {
return as[j].GetValue()
}
}
return ""
}
func (i *NodeInfo) syncAttributes() {
as := i.m.GetAttributes()
for j := range as {
switch as[j].GetKey() {
case AttrPrice:
i.priceAttr, _ = strconv.ParseUint(as[j].GetValue(), 10, 64)
case AttrCapacity:
i.capAttr, _ = strconv.ParseUint(as[j].GetValue(), 10, 64)
}
}
}
func (i *NodeInfo) setPrice(price uint64) {
i.priceAttr = price
as := i.m.GetAttributes()
for j := range as {
if as[j].GetKey() == AttrPrice {
as[j].SetValue(strconv.FormatUint(i.capAttr, 10))
return
}
}
as = append(as, netmap.Attribute{})
as[len(as)-1].SetKey(AttrPrice)
as[len(as)-1].SetValue(strconv.FormatUint(i.capAttr, 10))
i.m.SetAttributes(as)
}
func (i *NodeInfo) price() uint64 {
return i.priceAttr
}
func (i *NodeInfo) setCapacity(capacity uint64) {
i.capAttr = capacity
as := i.m.GetAttributes()
for j := range as {
if as[j].GetKey() == AttrCapacity {
as[j].SetValue(strconv.FormatUint(i.capAttr, 10))
return
}
}
as = append(as, netmap.Attribute{})
as[len(as)-1].SetKey(AttrCapacity)
as[len(as)-1].SetValue(strconv.FormatUint(i.capAttr, 10))
i.m.SetAttributes(as)
}
func (i NodeInfo) capacity() uint64 {
return i.capAttr
}
// Netmap represents netmap which contains preprocessed nodes.
type Netmap struct {
// NetMap represents NeoFS network map. It includes information about all
// storage nodes registered in NeoFS the network.
type NetMap struct {
nodes []NodeInfo
}
func (m *Netmap) SetNodes(nodes []NodeInfo) {
// SetNodes sets information list about all storage nodes from the NeoFS network.
//
// Argument MUST NOT be mutated, make a copy first.
//
// See also Nodes.
func (m *NetMap) SetNodes(nodes []NodeInfo) {
m.nodes = nodes
}
// Nodes returns nodes set using SetNodes.
//
// Return value MUST not be mutated, make a copy first.
func (m NetMap) Nodes() []NodeInfo {
return m.nodes
}
// nodes is a slice of NodeInfo instances needed for HRW sorting.
type nodes []NodeInfo
// assert nodes type provides hrw.Hasher required for HRW sorting.
var _ hrw.Hasher = nodes{}
// Hash is a function from hrw.Hasher interface. It is implemented
// to support weighted hrw sorting of buckets. Each bucket is already sorted by hrw,
// thus giving us needed "randomness".
@ -143,8 +71,12 @@ func flattenNodes(ns []nodes) nodes {
return result
}
// GetPlacementVectors returns placement vectors for an object given containerNodes cnt.
func (m *Netmap) GetPlacementVectors(vectors [][]NodeInfo, pivot []byte) ([][]NodeInfo, error) {
// PlacementVectors sorts container nodes returned by ContainerNodes method
// and returns placement vectors for the entity identified by the given pivot.
// For example,in order to build node list to store the object, binary-encoded
// object identifier can be used as pivot. Result is deterministic for
// the fixed NetMap and parameters.
func (m NetMap) PlacementVectors(vectors [][]NodeInfo, pivot []byte) ([][]NodeInfo, error) {
h := hrw.Hash(pivot)
wf := defaultWeightFunc(m.nodes)
result := make([][]NodeInfo, len(vectors))
@ -158,13 +90,18 @@ func (m *Netmap) GetPlacementVectors(vectors [][]NodeInfo, pivot []byte) ([][]No
return result, nil
}
// GetContainerNodes returns nodes corresponding to each replica.
// Order of returned nodes corresponds to order of replicas in p.
// pivot is a seed for HRW sorting.
func (m *Netmap) GetContainerNodes(p *PlacementPolicy, pivot []byte) ([][]NodeInfo, error) {
// ContainerNodes returns two-dimensional list of nodes as a result of applying
// given PlacementPolicy to the NetMap. Each line of the list corresponds to a
// replica descriptor. Line order corresponds to order of ReplicaDescriptor list
// in the policy. Nodes are pre-filtered according to the Filter list from
// the policy, and then selected by Selector list. Result is deterministic for
// the fixed NetMap and parameters.
//
// Result can be used in PlacementVectors.
func (m NetMap) ContainerNodes(p PlacementPolicy, pivot []byte) ([][]NodeInfo, error) {
c := newContext(m)
c.setPivot(pivot)
c.setCBF(p.ContainerBackupFactor())
c.setCBF(p.backupFactor)
if err := c.processFilters(p); err != nil {
return nil, err
@ -174,14 +111,15 @@ func (m *Netmap) GetContainerNodes(p *PlacementPolicy, pivot []byte) ([][]NodeIn
return nil, err
}
result := make([][]NodeInfo, len(p.Replicas()))
result := make([][]NodeInfo, len(p.replicas))
for i, r := range p.Replicas() {
if r.Selector() == "" {
if len(p.Selectors()) == 0 {
s := new(Selector)
s.SetCount(r.Count())
s.SetFilter(MainFilterName)
for i := range p.replicas {
sName := p.replicas[i].GetSelector()
if sName == "" {
if len(p.selectors) == 0 {
var s netmap.Selector
s.SetCount(p.replicas[i].GetCount())
s.SetFilter(mainFilterName)
nodes, err := c.getSelection(p, s)
if err != nil {
@ -191,16 +129,16 @@ func (m *Netmap) GetContainerNodes(p *PlacementPolicy, pivot []byte) ([][]NodeIn
result[i] = flattenNodes(nodes)
}
for _, s := range p.Selectors() {
result[i] = append(result[i], flattenNodes(c.Selections[s.Name()])...)
for i := range p.selectors {
result[i] = append(result[i], flattenNodes(c.selections[p.selectors[i].GetName()])...)
}
continue
}
nodes, ok := c.Selections[r.Selector()]
nodes, ok := c.selections[sName]
if !ok {
return nil, fmt.Errorf("%w: REPLICA '%s'", ErrSelectorNotFound, r.Selector())
return nil, fmt.Errorf("selector not found: REPLICA '%s'", sName)
}
result[i] = append(result[i], flattenNodes(nodes)...)

580
netmap/network_info.go
View file

@ -1,204 +1,462 @@
package netmap
import (
"bytes"
"encoding/binary"
"errors"
"fmt"
"math"
"github.com/nspcc-dev/neofs-api-go/v2/netmap"
)
// NetworkInfo represents v2-compatible structure
// with information about NeoFS network.
type NetworkInfo netmap.NetworkInfo
// NewNetworkInfoFromV2 wraps v2 NetworkInfo message to NetworkInfo.
// NetworkInfo groups information about the NeoFS network state. Mainly used to
// describe the current state of the network.
//
// Nil netmap.NetworkInfo converts to nil.
func NewNetworkInfoFromV2(iV2 *netmap.NetworkInfo) *NetworkInfo {
return (*NetworkInfo)(iV2)
// NetworkInfo is mutually compatible with github.com/nspcc-dev/neofs-api-go/v2/netmap.NetworkInfo
// message. See ReadFromV2 / WriteToV2 methods.
//
// Instances can be created using built-in var declaration.
type NetworkInfo struct {
m netmap.NetworkInfo
}
// NewNetworkInfo creates and initializes blank NetworkInfo.
//
// Defaults:
// - curEpoch: 0;
// - magicNum: 0;
// - msPerBlock: 0;
// - network config: nil.
func NewNetworkInfo() *NetworkInfo {
return NewNetworkInfoFromV2(new(netmap.NetworkInfo))
// reads NetworkInfo from netmap.NetworkInfo message. If checkFieldPresence is set,
// returns an error on absence of any protocol-required field. Verifies format of any
// presented field according to NeoFS API V2 protocol.
func (x *NetworkInfo) readFromV2(m netmap.NetworkInfo, checkFieldPresence bool) error {
c := m.GetNetworkConfig()
if checkFieldPresence && c == nil {
return errors.New("missing network config")
}
if checkFieldPresence && c.NumberOfParameters() <= 0 {
return fmt.Errorf("missing network parameters")
}
var err error
mNames := make(map[string]struct{}, c.NumberOfParameters())
c.IterateParameters(func(prm *netmap.NetworkParameter) bool {
name := string(prm.GetKey())
_, was := mNames[name]
if was {
err = fmt.Errorf("duplicated parameter name: %s", name)
return true
}
mNames[name] = struct{}{}
switch name {
default:
if len(prm.GetValue()) == 0 {
err = fmt.Errorf("empty attribute value %s", name)
return true
}
case configEigenTrustAlpha:
var num uint64
num, err = decodeConfigValueUint64(prm.GetValue())
if err == nil {
if alpha := math.Float64frombits(num); alpha < 0 && alpha > 1 {
err = fmt.Errorf("EigenTrust alpha value %0.2f is out of range [0, 1]", alpha)
}
}
case
configAuditFee,
configStoragePrice,
configContainerFee,
configNamedContainerFee,
configEigenTrustIterationsAmount,
configEpochDuration,
configIRCandidateFee,
configMaxObjSize,
configWithdrawalFee:
_, err = decodeConfigValueUint64(prm.GetValue())
}
if err != nil {
err = fmt.Errorf("invalid %s parameter: %w", name, err)
}
return err != nil
})
if err != nil {
return err
}
x.m = m
return nil
}
// ToV2 converts NetworkInfo to v2 NetworkInfo.
// ReadFromV2 reads NetworkInfo from the netmap.NetworkInfo message. Checks if the
// message conforms to NeoFS API V2 protocol.
//
// Nil NetworkInfo converts to nil.
func (i *NetworkInfo) ToV2() *netmap.NetworkInfo {
return (*netmap.NetworkInfo)(i)
// See also WriteToV2.
func (x *NetworkInfo) ReadFromV2(m netmap.NetworkInfo) error {
return x.readFromV2(m, true)
}
// CurrentEpoch returns current epoch of the NeoFS network.
func (i *NetworkInfo) CurrentEpoch() uint64 {
return (*netmap.NetworkInfo)(i).GetCurrentEpoch()
// WriteToV2 writes NetworkInfo to the netmap.NetworkInfo message. The message
// MUST NOT be nil.
//
// See also ReadFromV2.
func (x NetworkInfo) WriteToV2(m *netmap.NetworkInfo) {
*m = x.m
}
// CurrentEpoch returns epoch set using SetCurrentEpoch.
//
// Zero NetworkInfo has zero current epoch.
func (x NetworkInfo) CurrentEpoch() uint64 {
return x.m.GetCurrentEpoch()
}
// SetCurrentEpoch sets current epoch of the NeoFS network.
func (i *NetworkInfo) SetCurrentEpoch(epoch uint64) {
(*netmap.NetworkInfo)(i).SetCurrentEpoch(epoch)
func (x *NetworkInfo) SetCurrentEpoch(epoch uint64) {
x.m.SetCurrentEpoch(epoch)
}
// MagicNumber returns magic number of the sidechain.
func (i *NetworkInfo) MagicNumber() uint64 {
return (*netmap.NetworkInfo)(i).GetMagicNumber()
}
// SetMagicNumber sets magic number of the sidechain.
func (i *NetworkInfo) SetMagicNumber(epoch uint64) {
(*netmap.NetworkInfo)(i).SetMagicNumber(epoch)
}
// MsPerBlock returns MillisecondsPerBlock network parameter.
func (i *NetworkInfo) MsPerBlock() int64 {
return (*netmap.NetworkInfo)(i).
GetMsPerBlock()
}
// SetMsPerBlock sets MillisecondsPerBlock network parameter.
func (i *NetworkInfo) SetMsPerBlock(v int64) {
(*netmap.NetworkInfo)(i).
SetMsPerBlock(v)
}
// NetworkConfig returns NeoFS network configuration.
func (i *NetworkInfo) NetworkConfig() *NetworkConfig {
return NewNetworkConfigFromV2(
(*netmap.NetworkInfo)(i).
GetNetworkConfig(),
)
}
// SetNetworkConfig sets NeoFS network configuration.
func (i *NetworkInfo) SetNetworkConfig(v *NetworkConfig) {
(*netmap.NetworkInfo)(i).
SetNetworkConfig(v.ToV2())
}
// Marshal marshals NetworkInfo into a protobuf binary form.
func (i *NetworkInfo) Marshal() ([]byte, error) {
return (*netmap.NetworkInfo)(i).StableMarshal(nil), nil
}
// Unmarshal unmarshals protobuf binary representation of NetworkInfo.
func (i *NetworkInfo) Unmarshal(data []byte) error {
return (*netmap.NetworkInfo)(i).Unmarshal(data)
}
// MarshalJSON encodes NetworkInfo to protobuf JSON format.
func (i *NetworkInfo) MarshalJSON() ([]byte, error) {
return (*netmap.NetworkInfo)(i).MarshalJSON()
}
// UnmarshalJSON decodes NetworkInfo from protobuf JSON format.
func (i *NetworkInfo) UnmarshalJSON(data []byte) error {
return (*netmap.NetworkInfo)(i).UnmarshalJSON(data)
}
// NetworkParameter represents v2-compatible NeoFS network parameter.
type NetworkParameter netmap.NetworkParameter
// NewNetworkParameterFromV2 wraps v2 NetworkParameter message to NetworkParameter.
// MagicNumber returns magic number set using SetMagicNumber.
//
// Nil netmap.NetworkParameter converts to nil.
func NewNetworkParameterFromV2(pv2 *netmap.NetworkParameter) *NetworkParameter {
return (*NetworkParameter)(pv2)
// Zero NetworkInfo has zero magic.
func (x NetworkInfo) MagicNumber() uint64 {
return x.m.GetMagicNumber()
}
// NewNetworkParameter creates and initializes blank NetworkParameter.
// SetMagicNumber sets magic number of the NeoFS Sidechain.
//
// Defaults:
// - key: nil;
// - value: nil.
func NewNetworkParameter() *NetworkParameter {
return NewNetworkParameterFromV2(new(netmap.NetworkParameter))
// See also MagicNumber.
func (x *NetworkInfo) SetMagicNumber(epoch uint64) {
x.m.SetMagicNumber(epoch)
}
// ToV2 converts NetworkParameter to v2 NetworkParameter.
// MsPerBlock returns network parameter set using SetMsPerBlock.
func (x NetworkInfo) MsPerBlock() int64 {
return x.m.GetMsPerBlock()
}
// SetMsPerBlock sets MillisecondsPerBlock network parameter of the NeoFS Sidechain.
//
// Nil NetworkParameter converts to nil.
func (x *NetworkParameter) ToV2() *netmap.NetworkParameter {
return (*netmap.NetworkParameter)(x)
// See also MsPerBlock.
func (x *NetworkInfo) SetMsPerBlock(v int64) {
x.m.SetMsPerBlock(v)
}
// Key returns key to network parameter.
func (x *NetworkParameter) Key() []byte {
return (*netmap.NetworkParameter)(x).GetKey()
func (x *NetworkInfo) setConfig(name string, val []byte) {
c := x.m.GetNetworkConfig()
if c == nil {
c = new(netmap.NetworkConfig)
var prm netmap.NetworkParameter
prm.SetKey([]byte(name))
prm.SetValue(val)
c.SetParameters(prm)
x.m.SetNetworkConfig(c)
return
}
found := false
prms := make([]netmap.NetworkParameter, 0, c.NumberOfParameters())
c.IterateParameters(func(prm *netmap.NetworkParameter) bool {
found = bytes.Equal(prm.GetKey(), []byte(name))
if found {
prm.SetValue(val)
} else {
prms = append(prms, *prm)
}
return found
})
if !found {
prms = append(prms, netmap.NetworkParameter{})
prms[len(prms)-1].SetKey([]byte(name))
prms[len(prms)-1].SetValue(val)
c.SetParameters(prms...)
}
}
// SetKey sets key to the network parameter.
func (x *NetworkParameter) SetKey(key []byte) {
(*netmap.NetworkParameter)(x).SetKey(key)
func (x NetworkInfo) configValue(name string) (res []byte) {
x.m.GetNetworkConfig().IterateParameters(func(prm *netmap.NetworkParameter) bool {
if string(prm.GetKey()) == name {
res = prm.GetValue()
return true
}
return false
})
return
}
// Value returns value of the network parameter.
func (x *NetworkParameter) Value() []byte {
return (*netmap.NetworkParameter)(x).GetValue()
}
// SetValue sets value of the network parameter.
func (x *NetworkParameter) SetValue(val []byte) {
(*netmap.NetworkParameter)(x).SetValue(val)
}
// NetworkConfig represents v2-compatible NeoFS network configuration.
type NetworkConfig netmap.NetworkConfig
// NewNetworkConfigFromV2 wraps v2 NetworkConfig message to NetworkConfig.
// SetRawNetworkParameter sets named NeoFS network parameter whose value is
// transmitted but not interpreted by the NeoFS API protocol.
//
// Nil netmap.NetworkConfig converts to nil.
func NewNetworkConfigFromV2(cv2 *netmap.NetworkConfig) *NetworkConfig {
return (*NetworkConfig)(cv2)
}
// NewNetworkConfig creates and initializes blank NetworkConfig.
// Argument MUST NOT be mutated, make a copy first.
//
// Defaults:
// - parameters num: 0.
func NewNetworkConfig() *NetworkConfig {
return NewNetworkConfigFromV2(new(netmap.NetworkConfig))
// See also RawNetworkParameter, IterateRawNetworkParameters.
func (x *NetworkInfo) SetRawNetworkParameter(name string, value []byte) {
x.setConfig(name, value)
}
// ToV2 converts NetworkConfig to v2 NetworkConfig.
// RawNetworkParameter reads raw network parameter set using SetRawNetworkParameter
// by its name. Returns nil if value is missing.
//
// Nil NetworkConfig converts to nil.
func (x *NetworkConfig) ToV2() *netmap.NetworkConfig {
return (*netmap.NetworkConfig)(x)
}
// NumberOfParameters returns number of network parameters.
func (x *NetworkConfig) NumberOfParameters() int {
return (*netmap.NetworkConfig)(x).NumberOfParameters()
}
// IterateAddresses iterates over network parameters.
// Breaks iteration on f's true return.
// Return value MUST NOT be mutated, make a copy first.
//
// Handler should not be nil.
func (x *NetworkConfig) IterateParameters(f func(*NetworkParameter) bool) {
(*netmap.NetworkConfig)(x).
IterateParameters(func(p *netmap.NetworkParameter) bool {
return f(NewNetworkParameterFromV2(p))
// Zero NetworkInfo has no network parameters.
func (x *NetworkInfo) RawNetworkParameter(name string) []byte {
return x.configValue(name)
}
// IterateRawNetworkParameters iterates over all raw networks parameters set
// using SetRawNetworkParameter and passes them into f.
//
// Handler MUST NOT be nil. Handler MUST NOT mutate value parameter.
//
// Zero NetworkInfo has no network parameters.
func (x *NetworkInfo) IterateRawNetworkParameters(f func(name string, value []byte)) {
c := x.m.GetNetworkConfig()
c.IterateParameters(func(prm *netmap.NetworkParameter) bool {
name := string(prm.GetKey())
switch name {
default:
f(name, prm.GetValue())
case
configEigenTrustAlpha,
configAuditFee,
configStoragePrice,
configContainerFee,
configNamedContainerFee,
configEigenTrustIterationsAmount,
configEpochDuration,
configIRCandidateFee,
configMaxObjSize,
configWithdrawalFee:
}
return false
})
}
// Value returns value of the network parameter.
func (x *NetworkConfig) SetParameters(ps ...NetworkParameter) {
var psV2 []netmap.NetworkParameter
func (x *NetworkInfo) setConfigUint64(name string, num uint64) {
val := make([]byte, 8)
binary.LittleEndian.PutUint64(val, num)
if ps != nil {
ln := len(ps)
psV2 = make([]netmap.NetworkParameter, ln)
for i := 0; i < ln; i++ {
psV2[i] = *ps[i].ToV2()
}
}
(*netmap.NetworkConfig)(x).SetParameters(psV2...)
x.setConfig(name, val)
}
func decodeConfigValueUint64(val []byte) (uint64, error) {
if ln := len(val); ln != 8 {
return 0, fmt.Errorf("invalid uint64 parameter length %d", ln)
}
return binary.LittleEndian.Uint64(val), nil
}
func (x NetworkInfo) configUint64(name string) uint64 {
val := x.configValue(name)
if val == nil {
return 0
}
res, err := decodeConfigValueUint64(val)
if err != nil {
// potential panic is OK since value MUST be correct since it is
// verified in ReadFromV2 or set by provided method.
panic(err)
}
return res
}
const configAuditFee = "AuditFee"
// SetAuditFee sets the configuration value of the audit fee for the Inner Ring.
//
// See also AuditFee.
func (x *NetworkInfo) SetAuditFee(fee uint64) {
x.setConfigUint64(configAuditFee, fee)
}
// AuditFee returns audit fee set using SetAuditFee.
//
// Zero NetworkInfo has zero audit fee.
func (x NetworkInfo) AuditFee() uint64 {
return x.configUint64(configAuditFee)
}
const configStoragePrice = "BasicIncomeRate"
// SetStoragePrice sets the price per gigabyte of data storage that data owners
// pay to storage nodes.
//
// See also StoragePrice.
func (x *NetworkInfo) SetStoragePrice(price uint64) {
x.setConfigUint64(configStoragePrice, price)
}
// StoragePrice returns storage price set using SetStoragePrice.
//
// Zero NetworkInfo has zero storage price.
func (x NetworkInfo) StoragePrice() uint64 {
return x.configUint64(configStoragePrice)
}
const configContainerFee = "ContainerFee"
// SetContainerFee sets fee for the container creation that creator pays to
// each Alphabet node.
//
// See also ContainerFee.
func (x *NetworkInfo) SetContainerFee(fee uint64) {
x.setConfigUint64(configContainerFee, fee)
}
// ContainerFee returns container fee set using SetContainerFee.
//
// Zero NetworkInfo has zero container fee.
func (x NetworkInfo) ContainerFee() uint64 {
return x.configUint64(configContainerFee)
}
const configNamedContainerFee = "ContainerAliasFee"
// SetNamedContainerFee sets fee for creation of the named container creation
// that creator pays to each Alphabet node.
//
// See also NamedContainerFee.
func (x *NetworkInfo) SetNamedContainerFee(fee uint64) {
x.setConfigUint64(configNamedContainerFee, fee)
}
// NamedContainerFee returns container fee set using SetNamedContainerFee.
//
// Zero NetworkInfo has zero container fee.
func (x NetworkInfo) NamedContainerFee() uint64 {
return x.configUint64(configNamedContainerFee)
}
const configEigenTrustAlpha = "EigenTrustAlpha"
// SetEigenTrustAlpha sets alpha parameter for EigenTrust algorithm used in
// reputation system of the storage nodes. Value MUST be in range [0, 1].
//
// See also EigenTrustAlpha.
func (x *NetworkInfo) SetEigenTrustAlpha(alpha float64) {
if alpha < 0 || alpha > 1 {
panic(fmt.Sprintf("EigenTrust alpha parameter MUST be in range [0, 1], got %.2f", alpha))
}
x.setConfigUint64(configEigenTrustAlpha, math.Float64bits(alpha))
}
// EigenTrustAlpha returns EigenTrust parameter set using SetEigenTrustAlpha.
//
// Zero NetworkInfo has zero alpha parameter.
func (x NetworkInfo) EigenTrustAlpha() float64 {
alpha := math.Float64frombits(x.configUint64(configEigenTrustAlpha))
if alpha < 0 || alpha > 1 {
panic(fmt.Sprintf("unexpected invalid %s parameter value %.2f", configEigenTrustAlpha, alpha))
}
return alpha
}
const configEigenTrustIterationsAmount = "EigenTrustIterations"
// SetEigenTrustIterationAmount sets number of iterations of the EigenTrust
// algorithm to perform. The algorithm is used by the storage nodes for
// calculating the reputation values.
//
// See also EigenTrustIterationAmount.
func (x *NetworkInfo) SetEigenTrustIterationAmount(amount uint64) {
x.setConfigUint64(configEigenTrustIterationsAmount, amount)
}
// EigenTrustIterationAmount returns EigenTrust iteration amount set using
// SetEigenTrustIterationAmount.
//
// Zero NetworkInfo has zero iteration number.
func (x NetworkInfo) EigenTrustIterationAmount() uint64 {
return x.configUint64(configEigenTrustIterationsAmount)
}
const configEpochDuration = "EpochDuration"
// SetEpochDuration sets NeoFS epoch duration measured in block amount of the
// NeoFS Sidechain.
//
// See also EpochDuration.
func (x *NetworkInfo) SetEpochDuration(blocks uint64) {
x.setConfigUint64(configEpochDuration, blocks)
}
// EpochDuration returns epoch duration set using SetEpochDuration.
//
// Zero NetworkInfo has zero iteration number.
func (x NetworkInfo) EpochDuration() uint64 {
return x.configUint64(configEpochDuration)
}
const configIRCandidateFee = "InnerRingCandidateFee"
// SetIRCandidateFee sets fee for Inner Ring entrance paid by a new member.
//
// See also IRCandidateFee.
func (x *NetworkInfo) SetIRCandidateFee(fee uint64) {
x.setConfigUint64(configIRCandidateFee, fee)
}
// IRCandidateFee returns IR entrance fee set using SetIRCandidateFee.
//
// Zero NetworkInfo has zero fee.
func (x NetworkInfo) IRCandidateFee() uint64 {
return x.configUint64(configIRCandidateFee)
}
const configMaxObjSize = "MaxObjectSize"
// SetMaxObjectSize sets maximum size of the object stored locally on the
// storage nodes (physical objects). Binary representation of any physically
// stored object MUST NOT overflow the limit.
//
// See also MaxObjectSize.
func (x *NetworkInfo) SetMaxObjectSize(sz uint64) {
x.setConfigUint64(configMaxObjSize, sz)
}
// MaxObjectSize returns maximum object size set using SetMaxObjectSize.
//
// Zero NetworkInfo has zero maximum size.
func (x NetworkInfo) MaxObjectSize() uint64 {
return x.configUint64(configMaxObjSize)
}
const configWithdrawalFee = "WithdrawFee"
// SetWithdrawalFee sets fee for withdrawals from the NeoFS accounts that
// account owners pay to each Alphabet node.
//
// See also WithdrawalFee.
func (x *NetworkInfo) SetWithdrawalFee(sz uint64) {
x.setConfigUint64(configWithdrawalFee, sz)
}
// WithdrawalFee returns withdrawal fee set using SetWithdrawalFee.
//
// Zero NetworkInfo has zero fee.
func (x NetworkInfo) WithdrawalFee() uint64 {
return x.configUint64(configWithdrawalFee)
}

350
netmap/network_info_test.go
View file

@ -1,214 +1,214 @@
package netmap_test
import (
"encoding/binary"
"math"
"testing"
"github.com/nspcc-dev/neofs-api-go/v2/netmap"
. "github.com/nspcc-dev/neofs-sdk-go/netmap"
netmaptest "github.com/nspcc-dev/neofs-sdk-go/netmap/test"
"github.com/stretchr/testify/require"
)
func TestNetworkParameter_Key(t *testing.T) {
i := NewNetworkParameter()
k := []byte("key")
i.SetKey(k)
require.Equal(t, k, i.Key())
require.Equal(t, k, i.ToV2().GetKey())
}
func TestNetworkParameter_Value(t *testing.T) {
i := NewNetworkParameter()
v := []byte("value")
i.SetValue(v)
require.Equal(t, v, i.Value())
require.Equal(t, v, i.ToV2().GetValue())
}
func TestNewNetworkParameterFromV2(t *testing.T) {
t.Run("nil", func(t *testing.T) {
require.Nil(t, NewNetworkParameterFromV2(nil))
})
}
func TestNetworkParameter_ToV2(t *testing.T) {
t.Run("nil", func(t *testing.T) {
var x *NetworkParameter
require.Nil(t, x.ToV2())
})
}
func TestNewNetworkParameter(t *testing.T) {
x := NewNetworkParameter()
// check initial values
require.Nil(t, x.Key())
require.Nil(t, x.Value())
// convert to v2 message
xV2 := x.ToV2()
require.Nil(t, xV2.GetKey())
require.Nil(t, xV2.GetValue())
}
func TestNetworkConfig_SetParameters(t *testing.T) {
x := NewNetworkConfig()
require.Zero(t, x.NumberOfParameters())
called := 0
x.IterateParameters(func(p *NetworkParameter) bool {
called++
return false
})
require.Zero(t, called)
pps := []NetworkParameter{
*netmaptest.NetworkParameter(),
*netmaptest.NetworkParameter(),
}
x.SetParameters(pps...)
require.EqualValues(t, len(pps), x.NumberOfParameters())
var dst []NetworkParameter
x.IterateParameters(func(p *NetworkParameter) bool {
dst = append(dst, *p)
called++
return false
})
require.Equal(t, pps, dst)
require.Equal(t, len(pps), called)
}
func TestNewNetworkConfigFromV2(t *testing.T) {
t.Run("nil", func(t *testing.T) {
require.Nil(t, NewNetworkConfigFromV2(nil))
})
}
func TestNetworkConfig_ToV2(t *testing.T) {
t.Run("nil", func(t *testing.T) {
var x *NetworkConfig
require.Nil(t, x.ToV2())
})
}
func TestNewNetworkConfig(t *testing.T) {
x := NewNetworkConfig()
// check initial values
require.Zero(t, x.NumberOfParameters())
// convert to v2 message
xV2 := x.ToV2()
require.Zero(t, xV2.NumberOfParameters())
}
func TestNetworkInfo_CurrentEpoch(t *testing.T) {
i := NewNetworkInfo()
e := uint64(13)
var x NetworkInfo
i.SetCurrentEpoch(e)
require.Zero(t, x.CurrentEpoch())
require.Equal(t, e, i.CurrentEpoch())
require.Equal(t, e, i.ToV2().GetCurrentEpoch())
const e = 13
x.SetCurrentEpoch(e)
require.EqualValues(t, e, x.CurrentEpoch())
var m netmap.NetworkInfo
x.WriteToV2(&m)
require.EqualValues(t, e, m.GetCurrentEpoch())
}
func TestNetworkInfo_MagicNumber(t *testing.T) {
i := NewNetworkInfo()
m := uint64(666)
var x NetworkInfo
i.SetMagicNumber(m)
require.Zero(t, x.MagicNumber())
require.Equal(t, m, i.MagicNumber())
require.Equal(t, m, i.ToV2().GetMagicNumber())
const magic = 321
x.SetMagicNumber(magic)
require.EqualValues(t, magic, x.MagicNumber())
var m netmap.NetworkInfo
x.WriteToV2(&m)
require.EqualValues(t, magic, m.GetMagicNumber())
}
func TestNetworkInfo_MsPerBlock(t *testing.T) {
i := NewNetworkInfo()
var x NetworkInfo
const ms = 987
require.Zero(t, x.MsPerBlock())
i.SetMsPerBlock(ms)
const ms = 789
require.EqualValues(t, ms, i.MsPerBlock())
require.EqualValues(t, ms, i.ToV2().GetMsPerBlock())
x.SetMsPerBlock(ms)
require.EqualValues(t, ms, x.MsPerBlock())
var m netmap.NetworkInfo
x.WriteToV2(&m)
require.EqualValues(t, ms, m.GetMsPerBlock())
}
func TestNetworkInfo_Config(t *testing.T) {
i := NewNetworkInfo()
func testConfigValue(t *testing.T,
getter func(x NetworkInfo) interface{},
setter func(x *NetworkInfo, val interface{}),
val1, val2 interface{},
v2Key string, v2Val func(val interface{}) []byte,
) {
var x NetworkInfo
c := netmaptest.NetworkConfig()
require.Zero(t, getter(x))
i.SetNetworkConfig(c)
checkVal := func(exp interface{}) {
require.EqualValues(t, exp, getter(x))
require.Equal(t, c, i.NetworkConfig())
}
var m netmap.NetworkInfo
x.WriteToV2(&m)
func TestNetworkInfoEncoding(t *testing.T) {
i := netmaptest.NetworkInfo()
t.Run("binary", func(t *testing.T) {
data, err := i.Marshal()
require.NoError(t, err)
i2 := NewNetworkInfo()
require.NoError(t, i2.Unmarshal(data))
require.Equal(t, i, i2)
require.EqualValues(t, 1, m.GetNetworkConfig().NumberOfParameters())
found := false
m.GetNetworkConfig().IterateParameters(func(prm *netmap.NetworkParameter) bool {
require.False(t, found)
require.Equal(t, []byte(v2Key), prm.GetKey())
require.Equal(t, v2Val(exp), prm.GetValue())
found = true
return false
})
require.True(t, found)
}
t.Run("json", func(t *testing.T) {
data, err := i.MarshalJSON()
require.NoError(t, err)
setter(&x, val1)
checkVal(val1)
i2 := NewNetworkInfo()
require.NoError(t, i2.UnmarshalJSON(data))
require.Equal(t, i, i2)
})
setter(&x, val2)
checkVal(val2)
}
func TestNewNetworkInfoFromV2(t *testing.T) {
t.Run("nil", func(t *testing.T) {
require.Nil(t, NewNetworkInfoFromV2(nil))
})
func TestNetworkInfo_AuditFee(t *testing.T) {
testConfigValue(t,
func(x NetworkInfo) interface{} { return x.AuditFee() },
func(info *NetworkInfo, val interface{}) { info.SetAuditFee(val.(uint64)) },
uint64(1), uint64(2),
"AuditFee", func(val interface{}) []byte {
data := make([]byte, 8)
binary.LittleEndian.PutUint64(data, val.(uint64))
return data
},
)
}
func TestNetworkInfo_ToV2(t *testing.T) {
t.Run("nil", func(t *testing.T) {
var x *NetworkInfo
require.Nil(t, x.ToV2())
})
func TestNetworkInfo_StoragePrice(t *testing.T) {
testConfigValue(t,
func(x NetworkInfo) interface{} { return x.StoragePrice() },
func(info *NetworkInfo, val interface{}) { info.SetStoragePrice(val.(uint64)) },
uint64(1), uint64(2),
"BasicIncomeRate", func(val interface{}) []byte {
data := make([]byte, 8)
binary.LittleEndian.PutUint64(data, val.(uint64))
return data
},
)
}
func TestNewNetworkInfo(t *testing.T) {
ni := NewNetworkInfo()
// check initial values
require.Zero(t, ni.CurrentEpoch())
require.Zero(t, ni.MagicNumber())
require.Zero(t, ni.MsPerBlock())
// convert to v2 message
niV2 := ni.ToV2()
require.Zero(t, niV2.GetCurrentEpoch())
require.Zero(t, niV2.GetMagicNumber())
require.Zero(t, niV2.GetMsPerBlock())
func TestNetworkInfo_ContainerFee(t *testing.T) {
testConfigValue(t,
func(x NetworkInfo) interface{} { return x.ContainerFee() },
func(info *NetworkInfo, val interface{}) { info.SetContainerFee(val.(uint64)) },
uint64(1), uint64(2),
"ContainerFee", func(val interface{}) []byte {
data := make([]byte, 8)
binary.LittleEndian.PutUint64(data, val.(uint64))
return data
},
)
}
func TestNetworkInfo_NamedContainerFee(t *testing.T) {
testConfigValue(t,
func(x NetworkInfo) interface{} { return x.NamedContainerFee() },
func(info *NetworkInfo, val interface{}) { info.SetNamedContainerFee(val.(uint64)) },
uint64(1), uint64(2),
"ContainerAliasFee", func(val interface{}) []byte {
data := make([]byte, 8)
binary.LittleEndian.PutUint64(data, val.(uint64))
return data
},
)
}
func TestNetworkInfo_EigenTrustAlpha(t *testing.T) {
testConfigValue(t,
func(x NetworkInfo) interface{} { return x.EigenTrustAlpha() },
func(info *NetworkInfo, val interface{}) { info.SetEigenTrustAlpha(val.(float64)) },
0.1, 0.2,
"EigenTrustAlpha", func(val interface{}) []byte {
data := make([]byte, 8)
binary.LittleEndian.PutUint64(data, math.Float64bits(val.(float64)))
return data
},
)
}
func TestNetworkInfo_EigenTrustIterationAmount(t *testing.T) {
testConfigValue(t,
func(x NetworkInfo) interface{} { return x.EigenTrustIterationAmount() },
func(info *NetworkInfo, val interface{}) { info.SetEigenTrustIterationAmount(val.(uint64)) },
uint64(1), uint64(2),
"EigenTrustIterations", func(val interface{}) []byte {
data := make([]byte, 8)
binary.LittleEndian.PutUint64(data, val.(uint64))
return data
},
)
}
func TestNetworkInfo_IRCandidateFee(t *testing.T) {
testConfigValue(t,
func(x NetworkInfo) interface{} { return x.IRCandidateFee() },
func(info *NetworkInfo, val interface{}) { info.SetIRCandidateFee(val.(uint64)) },
uint64(1), uint64(2),
"InnerRingCandidateFee", func(val interface{}) []byte {
data := make([]byte, 8)
binary.LittleEndian.PutUint64(data, val.(uint64))
return data
},
)
}
func TestNetworkInfo_MaxObjectSize(t *testing.T) {
testConfigValue(t,
func(x NetworkInfo) interface{} { return x.MaxObjectSize() },
func(info *NetworkInfo, val interface{}) { info.SetMaxObjectSize(val.(uint64)) },
uint64(1), uint64(2),
"MaxObjectSize", func(val interface{}) []byte {
data := make([]byte, 8)
binary.LittleEndian.PutUint64(data, val.(uint64))
return data
},
)
}
func TestNetworkInfo_WithdrawalFee(t *testing.T) {
testConfigValue(t,
func(x NetworkInfo) interface{} { return x.WithdrawalFee() },
func(info *NetworkInfo, val interface{}) { info.SetWithdrawalFee(val.(uint64)) },
uint64(1), uint64(2),
"WithdrawFee", func(val interface{}) []byte {
data := make([]byte, 8)
binary.LittleEndian.PutUint64(data, val.(uint64))
return data
},
)
}

849
netmap/node_info.go
View file

@ -1,377 +1,560 @@
package netmap
import (
"errors"
"fmt"
"sort"
"strconv"
"strings"
"github.com/nspcc-dev/hrw"
"github.com/nspcc-dev/neofs-api-go/v2/netmap"
"github.com/nspcc-dev/neofs-api-go/v2/refs"
subnetid "github.com/nspcc-dev/neofs-sdk-go/subnet/id"
)
// NodeState is an enumeration of various states of the NeoFS node.
type NodeState uint32
// NodeAttribute represents v2 compatible attribute of the NeoFS Storage Node.
type NodeAttribute netmap.Attribute
// NodeInfo represents v2 compatible descriptor of the NeoFS node.
// NodeInfo groups information about NeoFS storage node which is reflected
// in the NeoFS network map. Storage nodes advertise this information when
// registering with the NeoFS network. After successful registration, information
// about the nodes is available to all network participants to work with the network
// map (mainly to comply with container storage policies).
//
// NodeInfo is mutually compatible with github.com/nspcc-dev/neofs-api-go/v2/netmap.NodeInfo
// message. See ReadFromV2 / WriteToV2 methods.
//
// Instances can be created using built-in var declaration.
type NodeInfo struct {
priceAttr uint64
capAttr uint64
m *netmap.NodeInfo
m netmap.NodeInfo
}
const (
_ NodeState = iota
// reads NodeInfo from netmap.NodeInfo message. If checkFieldPresence is set,
// returns an error on absence of any protocol-required field. Verifies format of any
// presented field according to NeoFS API V2 protocol.
func (x *NodeInfo) readFromV2(m netmap.NodeInfo, checkFieldPresence bool) error {
var err error
// NodeStateOffline is network unavailable state.
NodeStateOffline
// NodeStateOnline is an active state in the network.
NodeStateOnline
)
// Enumeration of well-known attributes.
const (
// AttrPrice is a key to the node attribute that indicates the
// price in GAS tokens for storing one GB of data during one Epoch.
AttrPrice = "Price"
// AttrCapacity is a key to the node attribute that indicates the
// total available disk space in Gigabytes.
AttrCapacity = "Capacity"
// AttrSubnet is a key to the node attribute that indicates the
// string ID of node's storage subnet.
AttrSubnet = "Subnet"
// AttrUNLOCODE is a key to the node attribute that indicates the
// node's geographic location in UN/LOCODE format.
AttrUNLOCODE = "UN-LOCODE"
// AttrCountryCode is a key to the node attribute that indicates the
// Country code in ISO 3166-1_alpha-2 format.
AttrCountryCode = "CountryCode"
// AttrCountry is a key to the node attribute that indicates the
// country short name in English, as defined in ISO-3166.
AttrCountry = "Country"
// AttrLocation is a key to the node attribute that indicates the
// place name of the node location.
AttrLocation = "Location"
// AttrSubDivCode is a key to the node attribute that indicates the
// country's administrative subdivision where node is located
// in ISO 3166-2 format.
AttrSubDivCode = "SubDivCode"
// AttrSubDiv is a key to the node attribute that indicates the
// country's administrative subdivision name, as defined in
// ISO 3166-2.
AttrSubDiv = "SubDiv"
// AttrContinent is a key to the node attribute that indicates the
// node's continent name according to the Seven-Continent model.
AttrContinent = "Continent"
)
// calcBucketWeight computes weight for a Bucket.
func calcBucketWeight(ns nodes, a aggregator, wf weightFunc) float64 {
for i := range ns {
a.Add(wf(ns[i]))
binPublicKey := m.GetPublicKey()
if checkFieldPresence && len(binPublicKey) == 0 {
return errors.New("missing public key")
}
return a.Compute()
}
if checkFieldPresence && m.NumberOfAddresses() <= 0 {
return errors.New("missing network endpoints")
}
// NodeStateFromV2 converts v2 NodeState to NodeState.
func NodeStateFromV2(s netmap.NodeState) NodeState {
switch s {
attributes := m.GetAttributes()
mAttr := make(map[string]struct{}, len(attributes))
for i := range attributes {
key := attributes[i].GetKey()
if key == "" {
return fmt.Errorf("empty key of the attribute #%d", i)
} else if _, ok := mAttr[key]; ok {
return fmt.Errorf("duplicated attbiuted %s", key)
}
const subnetPrefix = "__NEOFS__SUBNET_"
switch {
case key == attrCapacity:
_, err = strconv.ParseUint(attributes[i].GetValue(), 10, 64)
if err != nil {
return fmt.Errorf("invalid %s attribute: %w", attrCapacity, err)
}
case key == attrPrice:
var err error
_, err = strconv.ParseUint(attributes[i].GetValue(), 10, 64)
if err != nil {
return fmt.Errorf("invalid %s attribute: %w", attrPrice, err)
}
case strings.HasPrefix(key, subnetPrefix):
var id subnetid.ID
err = id.DecodeString(strings.TrimPrefix(key, subnetPrefix))
if err != nil {
return fmt.Errorf("invalid key to the subnet attribute %s: %w", key, err)
}
if val := attributes[i].GetValue(); val != "True" && val != "False" {
return fmt.Errorf("invalid value of the subnet attribute %s: %w", val, err)
}
default:
return 0
case netmap.Online:
return NodeStateOnline
case netmap.Offline:
return NodeStateOffline
if attributes[i].GetValue() == "" {
return fmt.Errorf("empty value of the attribute #%d", i)
}
}
// ToV2 converts NodeState to v2 NodeState.
func (s NodeState) ToV2() netmap.NodeState {
switch s {
default:
return netmap.UnspecifiedState
case NodeStateOffline:
return netmap.Offline
case NodeStateOnline:
return netmap.Online
}
}
// String returns string representation of NodeState.
//
// String mapping:
// * NodeStateOnline: ONLINE;
// * NodeStateOffline: OFFLINE;
// * default: UNSPECIFIED.
func (s NodeState) String() string {
return s.ToV2().String()
}
// FromString parses NodeState from a string representation.
// It is a reverse action to String().
//
// Returns true if s was parsed successfully.
func (s *NodeState) FromString(str string) bool {
var g netmap.NodeState
ok := g.FromString(str)
if ok {
*s = NodeStateFromV2(g)
}
return ok
}
x.m = m
// NewNodeAttribute creates and returns new NodeAttribute instance.
//
// Defaults:
// - key: "";
// - value: "";
// - parents: nil.
func NewNodeAttribute() *NodeAttribute {
return NewNodeAttributeFromV2(new(netmap.Attribute))
}
// NodeAttributeFromV2 converts v2 node Attribute to NodeAttribute.
//
// Nil netmap.Attribute converts to nil.
func NewNodeAttributeFromV2(a *netmap.Attribute) *NodeAttribute {
return (*NodeAttribute)(a)
}
// ToV2 converts NodeAttribute to v2 node Attribute.
//
// Nil NodeAttribute converts to nil.
func (a *NodeAttribute) ToV2() *netmap.Attribute {
return (*netmap.Attribute)(a)
}
// Key returns key to the node attribute.
func (a *NodeAttribute) Key() string {
return (*netmap.Attribute)(a).
GetKey()
}
// SetKey sets key to the node attribute.
func (a *NodeAttribute) SetKey(key string) {
(*netmap.Attribute)(a).
SetKey(key)
}
// Value returns value of the node attribute.
func (a *NodeAttribute) Value() string {
return (*netmap.Attribute)(a).
GetValue()
}
// SetValue sets value of the node attribute.
func (a *NodeAttribute) SetValue(val string) {
(*netmap.Attribute)(a).
SetValue(val)
}
// ParentKeys returns list of parent keys.
func (a *NodeAttribute) ParentKeys() []string {
return (*netmap.Attribute)(a).
GetParents()
}
// SetParentKeys sets list of parent keys.
func (a *NodeAttribute) SetParentKeys(keys ...string) {
(*netmap.Attribute)(a).
SetParents(keys)
}
// Marshal marshals NodeAttribute into a protobuf binary form.
func (a *NodeAttribute) Marshal() ([]byte, error) {
return (*netmap.Attribute)(a).StableMarshal(nil), nil
}
// Unmarshal unmarshals protobuf binary representation of NodeAttribute.
func (a *NodeAttribute) Unmarshal(data []byte) error {
return (*netmap.Attribute)(a).Unmarshal(data)
}
// MarshalJSON encodes NodeAttribute to protobuf JSON format.
func (a *NodeAttribute) MarshalJSON() ([]byte, error) {
return (*netmap.Attribute)(a).MarshalJSON()
}
// UnmarshalJSON decodes NodeAttribute from protobuf JSON format.
func (a *NodeAttribute) UnmarshalJSON(data []byte) error {
return (*netmap.Attribute)(a).UnmarshalJSON(data)
}
// NewNodeInfo creates and returns new NodeInfo instance.
//
// Defaults:
// - publicKey: nil;
// - address: "";
// - attributes nil;
// - state: 0.
func NewNodeInfo() *NodeInfo {
return NewNodeInfoFromV2(new(netmap.NodeInfo))
}
// NewNodeInfoFromV2 converts v2 NodeInfo to NodeInfo.
//
// Nil netmap.NodeInfo converts to nil.
func NewNodeInfoFromV2(i *netmap.NodeInfo) *NodeInfo {
var res NodeInfo
res.m = i
res.syncAttributes()
return &res
}
// ToV2 converts NodeInfo to v2 NodeInfo.
//
// Nil NodeInfo converts to nil.
func (i *NodeInfo) ToV2() *netmap.NodeInfo {
if i == nil {
return nil
}
return i.m
}
// PublicKey returns public key of the node in a binary format.
func (i *NodeInfo) PublicKey() []byte {
return i.m.GetPublicKey()
}
// SetPublicKey sets public key of the node in a binary format.
func (i *NodeInfo) SetPublicKey(key []byte) {
if i.m == nil {
i.m = new(netmap.NodeInfo)
}
i.m.SetPublicKey(key)
}
// NumberOfAddresses returns number of network addresses of the node.
func (i *NodeInfo) NumberOfAddresses() int {
return i.m.NumberOfAddresses()
}
// IterateAddresses iterates over network addresses of the node.
// Breaks iteration on f's true return.
// ReadFromV2 reads NodeInfo from the netmap.NodeInfo message. Checks if the
// message conforms to NeoFS API V2 protocol.
//
// Handler should not be nil.
func (i *NodeInfo) IterateAddresses(f func(string) bool) {
i.m.IterateAddresses(f)
// See also WriteToV2.
func (x *NodeInfo) ReadFromV2(m netmap.NodeInfo) error {
return x.readFromV2(m, true)
}
// IterateAllAddresses is a helper function to unconditionally
// iterate over all node addresses.
func IterateAllAddresses(i *NodeInfo, f func(string)) {
i.IterateAddresses(func(addr string) bool {
// WriteToV2 writes NodeInfo to the netmap.NodeInfo message. The message MUST NOT
// be nil.
//
// See also ReadFromV2.
func (x NodeInfo) WriteToV2(m *netmap.NodeInfo) {
*m = x.m
}
// Marshal encodes NodeInfo into a binary format of the NeoFS API protocol
// (Protocol Buffers with direct field order).
//
// See also Unmarshal.
func (x NodeInfo) Marshal() []byte {
var m netmap.NodeInfo
x.WriteToV2(&m)
return m.StableMarshal(nil)
}
// Unmarshal decodes NeoFS API protocol binary format into the NodeInfo
// (Protocol Buffers with direct field order). Returns an error describing
// a format violation.
//
// See also Marshal.
func (x *NodeInfo) Unmarshal(data []byte) error {
var m netmap.NodeInfo
err := m.Unmarshal(data)
if err != nil {
return err
}
return x.readFromV2(m, false)
}
// MarshalJSON encodes NodeInfo into a JSON format of the NeoFS API protocol
// (Protocol Buffers JSON).
//
// See also UnmarshalJSON.
func (x NodeInfo) MarshalJSON() ([]byte, error) {
var m netmap.NodeInfo
x.WriteToV2(&m)
return m.MarshalJSON()
}
// UnmarshalJSON decodes NeoFS API protocol JSON format into the NodeInfo
// (Protocol Buffers JSON). Returns an error describing a format violation.
//
// See also MarshalJSON.
func (x *NodeInfo) UnmarshalJSON(data []byte) error {
var m netmap.NodeInfo
err := m.UnmarshalJSON(data)
if err != nil {
return err
}
return x.readFromV2(m, false)
}
// SetPublicKey sets binary-encoded public key bound to the node. The key
// authenticates the storage node, so it MUST be unique within the network.
//
// Argument MUST NOT be mutated, make a copy first.
//
// See also PublicKey.
func (x *NodeInfo) SetPublicKey(key []byte) {
x.m.SetPublicKey(key)
}
// PublicKey returns value set using SetPublicKey.
//
// Zero NodeInfo has no public key, which is incorrect according to
// NeoFS system requirements.
//
// Return value MUST not be mutated, make a copy first.
func (x NodeInfo) PublicKey() []byte {
return x.m.GetPublicKey()
}
// SetNetworkEndpoints sets list to the announced node's network endpoints.
// Node MUSt have at least one announced endpoint. List MUST be unique.
// Endpoints are used for communication with the storage node within NeoFS
// network. It is expected that node serves storage node services on these
// endpoints (it also adds a wait on their network availability).
//
// Argument MUST NOT be mutated, make a copy first.
//
// See also IterateNetworkEndpoints.
func (x *NodeInfo) SetNetworkEndpoints(v ...string) {
x.m.SetAddresses(v...)
}
// NumberOfNetworkEndpoints returns number of network endpoints announced by the node.
//
// See also SetNetworkEndpoints.
func (x NodeInfo) NumberOfNetworkEndpoints() int {
return x.m.NumberOfAddresses()
}
// IterateNetworkEndpoints iterates over network endpoints announced by the
// node and pass them into f. Breaks iteration on f's true return. Handler
// MUST NOT be nil.
//
// Zero NodeInfo contains no endpoints which is incorrect according to
// NeoFS system requirements.
//
// See also SetNetworkEndpoints.
func (x NodeInfo) IterateNetworkEndpoints(f func(string) bool) {
x.m.IterateAddresses(f)
}
// IterateNetworkEndpoints is an extra-sugared function over IterateNetworkEndpoints
// method which allows to unconditionally iterate over all node's network endpoints.
func IterateNetworkEndpoints(node NodeInfo, f func(string)) {
node.IterateNetworkEndpoints(func(addr string) bool {
f(addr)
return false
})
}
// SetAddresses sets list of network addresses of the node.
func (i *NodeInfo) SetAddresses(v ...string) {
if i.m == nil {
i.m = new(netmap.NodeInfo)
}
// assert NodeInfo type provides hrw.Hasher required for HRW sorting.
var _ hrw.Hasher = NodeInfo{}
i.m.SetAddresses(v...)
// Hash implements hrw.Hasher interface.
//
// Hash is needed to support weighted HRW therefore sort function sorts nodes
// based on their public key. Hash isn't expected to be used directly.
func (x NodeInfo) Hash() uint64 {
return hrw.Hash(x.m.GetPublicKey())
}
// Attributes returns list of the node attributes.
func (i *NodeInfo) Attributes() []NodeAttribute {
if i == nil {
return nil
}
as := i.m.GetAttributes()
if as == nil {
return nil
}
res := make([]NodeAttribute, len(as))
for ind := range as {
res[ind] = *NewNodeAttributeFromV2(&as[ind])
}
return res
// less declares "less than" comparison between two NodeInfo instances:
// x1 is less than x2 if it has less Hash().
//
// Method is needed for internal placement needs.
func less(x1, x2 NodeInfo) bool {
return x1.Hash() < x2.Hash()
}
// SetAttributes sets list of the node attributes.
func (i *NodeInfo) SetAttributes(as ...NodeAttribute) {
asV2 := make([]netmap.Attribute, len(as))
func (x *NodeInfo) setNumericAttribute(key string, num uint64) {
x.SetAttribute(key, strconv.FormatUint(num, 10))
}
for ind := range as {
asV2[ind] = *as[ind].ToV2()
// SetPrice sets the storage cost declared by the node. By default, zero
// price is announced.
func (x *NodeInfo) SetPrice(price uint64) {
x.setNumericAttribute(attrPrice, price)
}
// Price returns price set using SetPrice.
//
// Zero NodeInfo has zero price.
func (x NodeInfo) Price() uint64 {
val := x.Attribute(attrPrice)
if val == "" {
return 0
}
if i.m == nil {
i.m = new(netmap.NodeInfo)
}
i.m.SetAttributes(asV2)
}
// State returns node state.
func (i *NodeInfo) State() NodeState {
return NodeStateFromV2(i.m.GetState())
}
// SetState sets node state.
func (i *NodeInfo) SetState(s NodeState) {
if i.m == nil {
i.m = new(netmap.NodeInfo)
}
i.m.SetState(s.ToV2())
}
// Marshal marshals NodeInfo into a protobuf binary form.
func (i *NodeInfo) Marshal() ([]byte, error) {
return i.m.StableMarshal(nil), nil
}
// Unmarshal unmarshals protobuf binary representation of NodeInfo.
func (i *NodeInfo) Unmarshal(data []byte) error {
if i.m == nil {
i.m = new(netmap.NodeInfo)
}
return i.m.Unmarshal(data)
}
// MarshalJSON encodes NodeInfo to protobuf JSON format.
func (i *NodeInfo) MarshalJSON() ([]byte, error) {
return i.m.MarshalJSON()
}
// UnmarshalJSON decodes NodeInfo from protobuf JSON format.
func (i *NodeInfo) UnmarshalJSON(data []byte) error {
if i.m == nil {
i.m = new(netmap.NodeInfo)
}
err := i.m.UnmarshalJSON(data)
price, err := strconv.ParseUint(val, 10, 64)
if err != nil {
return err
panic(fmt.Sprintf("unexpected price parsing error %s: %v", val, err))
}
i.syncAttributes()
return price
}
// SetCapacity sets the storage capacity declared by the node. By default, zero
// capacity is announced.
func (x *NodeInfo) SetCapacity(capacity uint64) {
x.setNumericAttribute(attrCapacity, capacity)
}
// capacity returns capacity set using SetCapacity.
//
// Zero NodeInfo has zero capacity.
func (x NodeInfo) capacity() uint64 {
val := x.Attribute(attrCapacity)
if val == "" {
return 0
}
capacity, err := strconv.ParseUint(val, 10, 64)
if err != nil {
panic(fmt.Sprintf("unexpected capacity parsing error %s: %v", val, err))
}
return capacity
}
const attrUNLOCODE = "UN-LOCODE"
// SetLOCODE specifies node's geographic location in UN/LOCODE format. Each
// storage node MUST declare it for entrance to the NeoFS network. Node MAY
// declare the code of the nearest location as needed, for example, when it is
// impossible to unambiguously attribute the node to any location from UN/LOCODE
// database.
//
// See also LOCODE.
func (x *NodeInfo) SetLOCODE(locode string) {
x.SetAttribute(attrUNLOCODE, locode)
}
// LOCODE returns node's location code set using SetLOCODE.
//
// Zero NodeInfo has empty location code which is invalid according to
// NeoFS API system requirement.
func (x NodeInfo) LOCODE() string {
return x.Attribute(attrUNLOCODE)
}
// SetCountryCode sets code of the country in ISO 3166-1_alpha-2 to which
// storage node belongs (or the closest one).
//
// SetCountryCode is intended only for processing the network registration
// request by the Inner Ring. Other parties SHOULD NOT use it.
func (x *NodeInfo) SetCountryCode(countryCode string) {
x.SetAttribute("CountryCode", countryCode)
}
// SetCountryName sets short name of the country in ISO-3166 format to which
// storage node belongs (or the closest one).
//
// SetCountryName is intended only for processing the network registration
// request by the Inner Ring. Other parties SHOULD NOT use it.
func (x *NodeInfo) SetCountryName(country string) {
x.SetAttribute("Country", country)
}
// SetLocationName sets storage node's location name from "NameWoDiacritics"
// column in the UN/LOCODE record corresponding to the specified LOCODE.
//
// SetLocationName is intended only for processing the network registration
// request by the Inner Ring. Other parties SHOULD NOT use it.
func (x *NodeInfo) SetLocationName(location string) {
x.SetAttribute("Location", location)
}
// SetSubdivisionCode sets storage node's subdivision code from "SubDiv" column in
// the UN/LOCODE record corresponding to the specified LOCODE.
//
// SetSubdivisionCode is intended only for processing the network registration
// request by the Inner Ring. Other parties SHOULD NOT use it.
func (x *NodeInfo) SetSubdivisionCode(subDiv string) {
x.SetAttribute("SubDivCode", subDiv)
}
// SetSubdivisionName sets storage node's subdivision name in ISO 3166-2 format.
//
// SetSubdivisionName is intended only for processing the network registration
// request by the Inner Ring. Other parties SHOULD NOT use it.
func (x *NodeInfo) SetSubdivisionName(subDiv string) {
x.SetAttribute("SubDiv", subDiv)
}
// SetContinentName sets name of the storage node's continent from
// Seven-Continent model.
//
// SetContinentName is intended only for processing the network registration
// request by the Inner Ring. Other parties SHOULD NOT use it.
func (x *NodeInfo) SetContinentName(continent string) {
x.SetAttribute("Continent", continent)
}
// Enumeration of well-known attributes.
const (
// attrPrice is a key to the node attribute that indicates the
// price in GAS tokens for storing one GB of data during one Epoch.
attrPrice = "Price"
// attrCapacity is a key to the node attribute that indicates the
// total available disk space in Gigabytes.
attrCapacity = "Capacity"
)
// NumberOfAttributes returns number of attributes announced by the node.
//
// See also SetAttribute.
func (x NodeInfo) NumberOfAttributes() int {
return len(x.m.GetAttributes())
}
// IterateAttributes iterates over all node attributes and passes the into f.
// Handler MUST NOT be nil.
func (x NodeInfo) IterateAttributes(f func(key, value string)) {
a := x.m.GetAttributes()
for i := range a {
f(a[i].GetKey(), a[i].GetValue())
}
}
// SetAttribute sets value of the node attribute value by the given key.
// Both key and value MUST NOT be empty.
func (x *NodeInfo) SetAttribute(key, value string) {
if key == "" {
panic("empty key in SetAttribute")
} else if value == "" {
panic("empty value in SetAttribute")
}
a := x.m.GetAttributes()
for i := range a {
if a[i].GetKey() == key {
a[i].SetValue(value)
return
}
}
a = append(a, netmap.Attribute{})
a[len(a)-1].SetKey(key)
a[len(a)-1].SetValue(value)
x.m.SetAttributes(a)
}
// Attribute returns value of the node attribute set using SetAttribute by the
// given key. Returns empty string if attribute is missing.
func (x NodeInfo) Attribute(key string) string {
a := x.m.GetAttributes()
for i := range a {
if a[i].GetKey() == key {
return a[i].GetValue()
}
}
return ""
}
// SortAttributes sorts node attributes set using SetAttribute lexicographically.
// The method is only needed to make NodeInfo consistent, e.g. for signing.
func (x *NodeInfo) SortAttributes() {
as := x.m.GetAttributes()
if len(as) == 0 {
return
}
sort.Slice(as, func(i, j int) bool {
switch strings.Compare(as[i].GetKey(), as[j].GetKey()) {
case -1:
return true
case 1:
return false
default:
return as[i].GetValue() < as[j].GetValue()
}
})
x.m.SetAttributes(as)
}
// EnterSubnet writes storage node's intention to enter the given subnet.
//
// Zero NodeInfo belongs to zero subnet.
func (x *NodeInfo) EnterSubnet(id subnetid.ID) {
x.changeSubnet(id, true)
}
// ExitSubnet writes storage node's intention to exit the given subnet.
func (x *NodeInfo) ExitSubnet(id subnetid.ID) {
x.changeSubnet(id, false)
}
func (x *NodeInfo) changeSubnet(id subnetid.ID, isMember bool) {
var (
idv2 refs.SubnetID
info netmap.NodeSubnetInfo
)
id.WriteToV2(&idv2)
info.SetID(&idv2)
info.SetEntryFlag(isMember)
netmap.WriteSubnetInfo(&x.m, info)
}
// ErrRemoveSubnet is returned when a node needs to leave the subnet.
var ErrRemoveSubnet = netmap.ErrRemoveSubnet
// IterateSubnets iterates over all subnets the node belongs to and passes the IDs to f.
// Handler MUST NOT be nil.
//
// If f returns ErrRemoveSubnet, then removes subnet entry. Note that this leads to an
// instant mutation of NodeInfo. Breaks on any other non-nil error and returns it.
//
// Returns an error if subnet incorrectly enabled/disabled.
// Returns an error if the node is not included to any subnet by the end of the loop.
//
// See also EnterSubnet, ExitSubnet.
func (x NodeInfo) IterateSubnets(f func(subnetid.ID) error) error {
var id subnetid.ID
return netmap.IterateSubnets(&x.m, func(idv2 refs.SubnetID) error {
err := id.ReadFromV2(idv2)
if err != nil {
return fmt.Errorf("invalid subnet: %w", err)
}
err = f(id)
if errors.Is(err, ErrRemoveSubnet) {
return netmap.ErrRemoveSubnet
}
return err
})
}
var errAbortSubnetIter = errors.New("abort subnet iterator")
// BelongsToSubnet is a helper function over the IterateSubnets method which
// checks whether a node belongs to a subnet.
//
// Zero NodeInfo belongs to zero subnet only.
func BelongsToSubnet(node NodeInfo, id subnetid.ID) bool {
err := node.IterateSubnets(func(id_ subnetid.ID) error {
if id.Equals(id_) {
return errAbortSubnetIter
}
return nil
})
return errors.Is(err, errAbortSubnetIter)
}
// SetOffline sets the state of the node to "offline". When a node updates
// information about itself in the network map, this action is interpreted as
// an intention to leave the network.
func (x *NodeInfo) SetOffline() {
x.m.SetState(netmap.Offline)
}
// IsOffline checks if the node is in the "offline" state.
//
// Zero NodeInfo has undefined state which is not offline (note that it does not
// mean online).
//
// See also SetOffline.
func (x NodeInfo) IsOffline() bool {
return x.m.GetState() == netmap.Offline
}
// SetOnline sets the state of the node to "online". When a node updates
// information about itself in the network map, this
// action is interpreted as an intention to enter the network.
//
// See also IsOnline.
func (x *NodeInfo) SetOnline() {
x.m.SetState(netmap.Online)
}
// IsOnline checks if the node is in the "online" state.
//
// Zero NodeInfo has undefined state which is not online (note that it does not
// mean offline).
//
// See also SetOnline.
func (x NodeInfo) IsOnline() bool {
return x.m.GetState() == netmap.Online
}

261
netmap/node_info_test.go
View file

@ -3,261 +3,22 @@ package netmap
import (
"testing"
"github.com/nspcc-dev/neofs-api-go/v2/netmap"
testv2 "github.com/nspcc-dev/neofs-api-go/v2/netmap/test"
"github.com/stretchr/testify/require"
)
func TestNodeStateFromV2(t *testing.T) {
for _, item := range []struct {
s NodeState
sV2 netmap.NodeState
}{
{
s: 0,
sV2: netmap.UnspecifiedState,
},
{
s: NodeStateOnline,
sV2: netmap.Online,
},
{
s: NodeStateOffline,
sV2: netmap.Offline,
},
} {
require.Equal(t, item.s, NodeStateFromV2(item.sV2))
require.Equal(t, item.sV2, item.s.ToV2())
}
}
func TestNodeInfo_SetAttribute(t *testing.T) {
var n NodeInfo
func TestNodeAttributeFromV2(t *testing.T) {
t.Run("from nil", func(t *testing.T) {
var x *netmap.Attribute
require.Nil(t, NewNodeAttributeFromV2(x))
})
t.Run("from non-nil", func(t *testing.T) {
aV2 := testv2.GenerateAttribute(false)
a := NewNodeAttributeFromV2(aV2)
require.Equal(t, aV2, a.ToV2())
})
}
func TestNodeAttribute_ToV2(t *testing.T) {
t.Run("nil", func(t *testing.T) {
var x *NodeAttribute
require.Nil(t, x.ToV2())
})
}
func TestNodeAttribute_Key(t *testing.T) {
a := NewNodeAttribute()
key := "some key"
a.SetKey(key)
require.Equal(t, key, a.Key())
}
func TestNodeAttribute_Value(t *testing.T) {
a := NewNodeAttribute()
const key = "some key"
val := "some value"
a.SetValue(val)
require.Zero(t, n.Attribute(val))
require.Equal(t, val, a.Value())
}
func TestNodeAttribute_ParentKeys(t *testing.T) {
a := NewNodeAttribute()
keys := []string{"par1", "par2"}
a.SetParentKeys(keys...)
require.Equal(t, keys, a.ParentKeys())
}
func testNodeAttribute() *NodeAttribute {
a := new(NodeAttribute)
a.SetKey("key")
a.SetValue("value")
a.SetParentKeys("par1", "par2")
return a
}
func TestNodeInfoFromV2(t *testing.T) {
t.Run("from nil", func(t *testing.T) {
var x *netmap.NodeInfo
require.Nil(t, NewNodeInfoFromV2(x).m)
})
t.Run("from non-nil", func(t *testing.T) {
iV2 := testv2.GenerateNodeInfo(false)
i := NewNodeInfoFromV2(iV2)
require.Equal(t, iV2, i.ToV2())
})
}
func TestNodeInfo_ToV2(t *testing.T) {
t.Run("nil", func(t *testing.T) {
var x *NodeInfo
require.Nil(t, x.ToV2())
})
}
func TestNodeInfo_PublicKey(t *testing.T) {
i := new(NodeInfo)
key := []byte{1, 2, 3}
i.SetPublicKey(key)
require.Equal(t, key, i.PublicKey())
}
func TestNodeInfo_IterateAddresses(t *testing.T) {
i := new(NodeInfo)
as := []string{"127.0.0.1:8080", "127.0.0.1:8081"}
i.SetAddresses(as...)
as2 := make([]string, 0, i.NumberOfAddresses())
IterateAllAddresses(i, func(addr string) {
as2 = append(as2, addr)
})
require.Equal(t, as, as2)
require.EqualValues(t, len(as), i.NumberOfAddresses())
}
func TestNodeInfo_State(t *testing.T) {
i := new(NodeInfo)
s := NodeStateOnline
i.SetState(s)
require.Equal(t, s, i.State())
}
func TestNodeInfo_Attributes(t *testing.T) {
i := new(NodeInfo)
as := []NodeAttribute{*testNodeAttribute(), *testNodeAttribute()}
i.SetAttributes(as...)
require.Equal(t, as, i.Attributes())
}
func TestNodeAttributeEncoding(t *testing.T) {
a := testNodeAttribute()
t.Run("binary", func(t *testing.T) {
data, err := a.Marshal()
require.NoError(t, err)
a2 := NewNodeAttribute()
require.NoError(t, a2.Unmarshal(data))
require.Equal(t, a, a2)
})
t.Run("json", func(t *testing.T) {
data, err := a.MarshalJSON()
require.NoError(t, err)
a2 := NewNodeAttribute()
require.NoError(t, a2.UnmarshalJSON(data))
require.Equal(t, a, a2)
})
}
func TestNodeInfoEncoding(t *testing.T) {
i := NewNodeInfo()
i.SetPublicKey([]byte{1, 2, 3})
i.SetAddresses("192.168.0.1", "192.168.0.2")
i.SetState(NodeStateOnline)
i.SetAttributes(*testNodeAttribute())
t.Run("binary", func(t *testing.T) {
data, err := i.Marshal()
require.NoError(t, err)
i2 := NewNodeInfo()
require.NoError(t, i2.Unmarshal(data))
require.Equal(t, i, i2)
})
t.Run("json", func(t *testing.T) {
data, err := i.MarshalJSON()
require.NoError(t, err)
i2 := NewNodeInfo()
require.NoError(t, i2.UnmarshalJSON(data))
require.Equal(t, i, i2)
})
}
func TestNewNodeAttribute(t *testing.T) {
t.Run("default values", func(t *testing.T) {
attr := NewNodeAttribute()
// check initial values
require.Empty(t, attr.Key())
require.Empty(t, attr.Value())
require.Nil(t, attr.ParentKeys())
// convert to v2 message
attrV2 := attr.ToV2()
require.Empty(t, attrV2.GetKey())
require.Empty(t, attrV2.GetValue())
require.Nil(t, attrV2.GetParents())
})
}
func TestNewNodeInfo(t *testing.T) {
t.Run("default values", func(t *testing.T) {
ni := NewNodeInfo()
// check initial values
require.Nil(t, ni.PublicKey())
require.Zero(t, ni.NumberOfAddresses())
require.Nil(t, ni.Attributes())
require.Zero(t, ni.State())
// convert to v2 message
niV2 := ni.ToV2()
require.Nil(t, niV2.GetPublicKey())
require.Zero(t, niV2.NumberOfAddresses())
require.Nil(t, niV2.GetAttributes())
require.EqualValues(t, netmap.UnspecifiedState, niV2.GetState())
})
}
func TestNodeState_String(t *testing.T) {
toPtr := func(v NodeState) *NodeState {
return &v
}
testEnumStrings(t, new(NodeState), []enumStringItem{
{val: toPtr(NodeStateOnline), str: "ONLINE"},
{val: toPtr(NodeStateOffline), str: "OFFLINE"},
{val: toPtr(0), str: "UNSPECIFIED"},
})
n.SetAttribute(key, val)
require.Equal(t, val, n.Attribute(key))
val = "some other value"
n.SetAttribute(key, val)
require.Equal(t, val, n.Attribute(key))
}

116
netmap/operation.go
View file

@ -1,116 +0,0 @@
package netmap
import (
"github.com/nspcc-dev/neofs-api-go/v2/netmap"
)
// Operation is an enumeration of v2-compatible filtering operations.
type Operation uint32
const (
_ Operation = iota
// OpEQ is an "Equal" operation.
OpEQ
// OpNE is a "Not equal" operation.
OpNE
// OpGT is a "Greater than" operation.
OpGT
// OpGE is a "Greater than or equal to" operation.
OpGE
// OpLT is a "Less than" operation.
OpLT
// OpLE is a "Less than or equal to" operation.
OpLE
// OpOR is an "OR" operation.
OpOR
// OpAND is an "AND" operation.
OpAND
)
// OperationFromV2 converts v2 Operation to Operation.
func OperationFromV2(op netmap.Operation) Operation {
switch op {
default:
return 0
case netmap.OR:
return OpOR
case netmap.AND:
return OpAND
case netmap.GE:
return OpGE
case netmap.GT:
return OpGT
case netmap.LE:
return OpLE
case netmap.LT:
return OpLT
case netmap.EQ:
return OpEQ
case netmap.NE:
return OpNE
}
}
// ToV2 converts Operation to v2 Operation.
func (op Operation) ToV2() netmap.Operation {
switch op {
default:
return netmap.UnspecifiedOperation
case OpOR:
return netmap.OR
case OpAND:
return netmap.AND
case OpGE:
return netmap.GE
case OpGT:
return netmap.GT
case OpLE:
return netmap.LE
case OpLT:
return netmap.LT
case OpEQ:
return netmap.EQ
case OpNE:
return netmap.NE
}
}
// String returns string representation of Operation.
//
// String mapping:
// * OpNE: NE;
// * OpEQ: EQ;
// * OpLT: LT;
// * OpLE: LE;
// * OpGT: GT;
// * OpGE: GE;
// * OpAND: AND;
// * OpOR: OR;
// * default: OPERATION_UNSPECIFIED.
func (op Operation) String() string {
return op.ToV2().String()
}
// FromString parses Operation from a string representation.
// It is a reverse action to String().
//
// Returns true if s was parsed successfully.
func (op *Operation) FromString(s string) bool {
var g netmap.Operation
ok := g.FromString(s)
if ok {
*op = OperationFromV2(g)
}
return ok
}

73
netmap/operation_test.go
View file

@ -1,73 +0,0 @@
package netmap
import (
"testing"
"github.com/nspcc-dev/neofs-api-go/v2/netmap"
"github.com/stretchr/testify/require"
)
func TestOperationFromV2(t *testing.T) {
for _, item := range []struct {
op Operation
opV2 netmap.Operation
}{
{
op: 0,
opV2: netmap.UnspecifiedOperation,
},
{
op: OpEQ,
opV2: netmap.EQ,
},
{
op: OpNE,
opV2: netmap.NE,
},
{
op: OpOR,
opV2: netmap.OR,
},
{
op: OpAND,
opV2: netmap.AND,
},
{
op: OpLE,
opV2: netmap.LE,
},
{
op: OpLT,
opV2: netmap.LT,
},
{
op: OpGT,
opV2: netmap.GT,
},
{
op: OpGE,
opV2: netmap.GE,
},
} {
require.Equal(t, item.op, OperationFromV2(item.opV2))
require.Equal(t, item.opV2, item.op.ToV2())
}
}
func TestOperation_String(t *testing.T) {
toPtr := func(v Operation) *Operation {
return &v
}
testEnumStrings(t, new(Operation), []enumStringItem{
{val: toPtr(OpEQ), str: "EQ"},
{val: toPtr(OpNE), str: "NE"},
{val: toPtr(OpGT), str: "GT"},
{val: toPtr(OpGE), str: "GE"},
{val: toPtr(OpLT), str: "LT"},
{val: toPtr(OpLE), str: "LE"},
{val: toPtr(OpAND), str: "AND"},
{val: toPtr(OpOR), str: "OR"},
{val: toPtr(0), str: "OPERATION_UNSPECIFIED"},
})
}

0
policy/parser/Query.g4 → netmap/parser/Query.g4
View file

0
policy/parser/Query.interp → netmap/parser/Query.interp
View file

0
policy/parser/Query.tokens → netmap/parser/Query.tokens
View file

0
policy/parser/QueryLexer.g4 → netmap/parser/QueryLexer.g4
View file

0
policy/parser/QueryLexer.interp → netmap/parser/QueryLexer.interp
View file

0
policy/parser/QueryLexer.tokens → netmap/parser/QueryLexer.tokens
View file

0
policy/parser/generate.go → netmap/parser/generate.go
View file

0
policy/parser/query_base_listener.go → netmap/parser/query_base_listener.go
View file

0
policy/parser/query_base_visitor.go → netmap/parser/query_base_visitor.go
View file

0
policy/parser/query_lexer.go → netmap/parser/query_lexer.go
View file

0
policy/parser/query_listener.go → netmap/parser/query_listener.go
View file

0
policy/parser/query_parser.go → netmap/parser/query_parser.go
View file

0
policy/parser/query_visitor.go → netmap/parser/query_visitor.go
View file

921
netmap/policy.go
View file

@ -1,176 +1,787 @@
package netmap
import (
"errors"
"fmt"
"io"
"strconv"
"strings"
"github.com/antlr/antlr4/runtime/Go/antlr"
"github.com/nspcc-dev/neofs-api-go/v2/netmap"
"github.com/nspcc-dev/neofs-api-go/v2/refs"
"github.com/nspcc-dev/neofs-sdk-go/netmap/parser"
subnetid "github.com/nspcc-dev/neofs-sdk-go/subnet/id"
)
// PlacementPolicy represents v2-compatible placement policy.
type PlacementPolicy netmap.PlacementPolicy
// NewPlacementPolicy creates and returns new PlacementPolicy instance.
// PlacementPolicy declares policy to store objects in the NeoFS container.
// Within itself, PlacementPolicy represents a set of rules to select a subset
// of nodes from NeoFS network map - node-candidates for object storage.
//
// Defaults:
// - backupFactor: 0;
// - replicas nil;
// - selectors nil;
// - filters nil.
func NewPlacementPolicy() *PlacementPolicy {
return NewPlacementPolicyFromV2(new(netmap.PlacementPolicy))
// PlacementPolicy is mutually compatible with github.com/nspcc-dev/neofs-api-go/v2/netmap.PlacementPolicy
// message. See ReadFromV2 / WriteToV2 methods.
//
// Instances can be created using built-in var declaration.
type PlacementPolicy struct {
backupFactor uint32
subnet subnetid.ID
filters []netmap.Filter
selectors []netmap.Selector
replicas []netmap.Replica
}
// NewPlacementPolicyFromV2 converts v2 PlacementPolicy to PlacementPolicy.
//
// Nil netmap.PlacementPolicy converts to nil.
func NewPlacementPolicyFromV2(f *netmap.PlacementPolicy) *PlacementPolicy {
return (*PlacementPolicy)(f)
}
func (p *PlacementPolicy) readFromV2(m netmap.PlacementPolicy) error {
p.backupFactor = m.GetContainerBackupFactor()
p.filters = m.GetFilters()
p.selectors = m.GetSelectors()
p.replicas = m.GetReplicas()
// ToV2 converts PlacementPolicy to v2 PlacementPolicy.
//
// Nil PlacementPolicy converts to nil.
func (p *PlacementPolicy) ToV2() *netmap.PlacementPolicy {
return (*netmap.PlacementPolicy)(p)
}
// SubnetID returns subnet to select nodes from.
func (p *PlacementPolicy) SubnetID() *subnetid.ID {
idv2 := (*netmap.PlacementPolicy)(p).GetSubnetID()
if idv2 == nil {
return nil
}
var id subnetid.ID
err := id.ReadFromV2(*idv2)
subnetV2 := m.GetSubnetID()
if subnetV2 != nil {
err := p.subnet.ReadFromV2(*subnetV2)
if err != nil {
panic(err) // will disappear after netmap package refactor
return fmt.Errorf("invalid subnet: %w", err)
}
} else {
p.subnet = subnetid.ID{}
}
return &id
}
// SetSubnetID sets subnet to select nodes from.
func (p *PlacementPolicy) SetSubnetID(subnet *subnetid.ID) {
var idv2 *refs.SubnetID
if subnet != nil {
idv2 = new(refs.SubnetID)
subnet.WriteToV2(idv2)
}
(*netmap.PlacementPolicy)(p).SetSubnetID(idv2)
}
// Replicas returns list of object replica descriptors.
func (p *PlacementPolicy) Replicas() []Replica {
rs := (*netmap.PlacementPolicy)(p).
GetReplicas()
if rs == nil {
return nil
}
res := make([]Replica, len(rs))
for i := range rs {
res[i] = *NewReplicaFromV2(&rs[i])
}
return res
}
// SetReplicas sets list of object replica descriptors.
func (p *PlacementPolicy) SetReplicas(rs ...Replica) {
var rsV2 []netmap.Replica
if rs != nil {
rsV2 = make([]netmap.Replica, len(rs))
for i := range rs {
rsV2[i] = *rs[i].ToV2()
}
}
(*netmap.PlacementPolicy)(p).SetReplicas(rsV2)
}
// ContainerBackupFactor returns container backup factor.
func (p *PlacementPolicy) ContainerBackupFactor() uint32 {
return (*netmap.PlacementPolicy)(p).
GetContainerBackupFactor()
}
// SetContainerBackupFactor sets container backup factor.
func (p *PlacementPolicy) SetContainerBackupFactor(f uint32) {
(*netmap.PlacementPolicy)(p).
SetContainerBackupFactor(f)
}
// Selector returns set of selectors to form the container's nodes subset.
func (p *PlacementPolicy) Selectors() []Selector {
rs := (*netmap.PlacementPolicy)(p).
GetSelectors()
if rs == nil {
return nil
}
res := make([]Selector, len(rs))
for i := range rs {
res[i] = *NewSelectorFromV2(&rs[i])
}
return res
}
// SetSelectors sets set of selectors to form the container's nodes subset.
func (p *PlacementPolicy) SetSelectors(ss ...Selector) {
var ssV2 []netmap.Selector
if ss != nil {
ssV2 = make([]netmap.Selector, len(ss))
for i := range ss {
ssV2[i] = *ss[i].ToV2()
}
}
(*netmap.PlacementPolicy)(p).SetSelectors(ssV2)
}
// Filters returns list of named filters to reference in selectors.
func (p *PlacementPolicy) Filters() []Filter {
return filtersFromV2(
(*netmap.PlacementPolicy)(p).
GetFilters(),
)
}
// SetFilters sets list of named filters to reference in selectors.
func (p *PlacementPolicy) SetFilters(fs ...Filter) {
(*netmap.PlacementPolicy)(p).
SetFilters(filtersToV2(fs))
}
// Marshal marshals PlacementPolicy into a protobuf binary form.
func (p *PlacementPolicy) Marshal() ([]byte, error) {
return (*netmap.PlacementPolicy)(p).StableMarshal(nil), nil
}
// Unmarshal unmarshals protobuf binary representation of PlacementPolicy.
func (p *PlacementPolicy) Unmarshal(data []byte) error {
return (*netmap.PlacementPolicy)(p).Unmarshal(data)
}
// MarshalJSON encodes PlacementPolicy to protobuf JSON format.
func (p *PlacementPolicy) MarshalJSON() ([]byte, error) {
return (*netmap.PlacementPolicy)(p).MarshalJSON()
}
// UnmarshalJSON decodes PlacementPolicy from protobuf JSON format.
func (p *PlacementPolicy) UnmarshalJSON(data []byte) error {
return (*netmap.PlacementPolicy)(p).UnmarshalJSON(data)
var m netmap.PlacementPolicy
err := m.UnmarshalJSON(data)
if err != nil {
return err
}
return p.readFromV2(m)
}
// ReadFromV2 reads PlacementPolicy from the netmap.PlacementPolicy message.
// Checks if the message conforms to NeoFS API V2 protocol.
//
// See also WriteToV2.
func (p *PlacementPolicy) ReadFromV2(m netmap.PlacementPolicy) error {
return p.readFromV2(m)
}
// WriteToV2 writes PlacementPolicy to the session.Token message.
// The message must not be nil.
//
// See also ReadFromV2.
func (p PlacementPolicy) WriteToV2(m *netmap.PlacementPolicy) {
var subnetV2 refs.SubnetID
p.subnet.WriteToV2(&subnetV2)
m.SetContainerBackupFactor(p.backupFactor)
m.SetSubnetID(&subnetV2)
m.SetFilters(p.filters)
m.SetSelectors(p.selectors)
m.SetReplicas(p.replicas)
}
// RestrictSubnet sets a rule to select nodes from the given subnet only.
// By default, nodes from zero subnet are selected (whole network map).
func (p *PlacementPolicy) RestrictSubnet(subnet subnetid.ID) {
p.subnet = subnet
}
// Subnet returns subnet set using RestrictSubnet.
//
// Zero PlacementPolicy returns zero subnet meaning unlimited.
func (p PlacementPolicy) Subnet() subnetid.ID {
return p.subnet
}
// ReplicaDescriptor replica descriptor characterizes replicas of objects from
// the subset selected by a particular Selector.
type ReplicaDescriptor struct {
m netmap.Replica
}
// SetAmount sets number of object replicas.
func (r *ReplicaDescriptor) SetAmount(c uint32) {
r.m.SetCount(c)
}
// Amount returns number set using SetAmount.
//
// Zero ReplicaDescriptor has zero object amount.
func (r ReplicaDescriptor) Amount() uint32 {
return r.m.GetCount()
}
// SetSelectorName sets name of the related Selector.
//
// Zero ReplicaDescriptor references to the root bucket's selector: it contains
// all possible nodes to store the object.
func (r *ReplicaDescriptor) SetSelectorName(s string) {
r.m.SetSelector(s)
}
// AddReplicas adds a bunch object replica's characteristics.
//
// Zero PlacementPolicy does not declare replicas ???.
//
// See also IterateReplicas.
func (p *PlacementPolicy) AddReplicas(rs ...ReplicaDescriptor) {
off := len(p.replicas)
p.replicas = append(p.replicas, make([]netmap.Replica, len(rs))...)
for i := range rs {
p.replicas[off+i] = rs[i].m
}
}
// NumberOfReplicas returns amount of replica descriptors set using AddReplicas.
//
// Zero PlacementPolicy has no replicas.
func (p PlacementPolicy) NumberOfReplicas() int {
return len(p.replicas)
}
// ReplicaAmountByIndex returns amount of object replicas from the i-th replica
// descriptor. Index MUST be in range [0; NumberOfReplicas()).
//
// Zero PlacementPolicy has no replicas.
func (p PlacementPolicy) ReplicaAmountByIndex(i int) uint32 {
return p.replicas[i].GetCount()
}
// SetContainerBackupFactor sets container backup factor: it controls how deep
// NeoFS will search for nodes alternatives to include into container's nodes subset.
//
// Zero PlacementPolicy has zero container backup factor.
func (p *PlacementPolicy) SetContainerBackupFactor(f uint32) {
p.backupFactor = f
}
// Selector describes the bucket selection operator: choose a number of nodes
// from the bucket taking the nearest nodes to the related container by hash distance.
type Selector struct {
m netmap.Selector
}
// SetName sets name with which the Selector can be referenced.
//
// Zero Selector is unnamed.
func (s *Selector) SetName(name string) {
s.m.SetName(name)
}
// SetNodeAmount sets number of nodes to select from the bucket.
//
// Zero Selector selects nothing.
func (s *Selector) SetNodeAmount(amount uint32) {
s.m.SetCount(amount)
}
// SelectByBucketAttribute sets attribute of the bucket to select nodes from.
//
// Zero Selector ???.
func (s *Selector) SelectByBucketAttribute(bucket string) {
s.m.SetAttribute(bucket)
}
// SelectSame makes selection algorithm to select only nodes having the same values
// of the bucket attribute.
//
// Zero Selector doesn't specify selection modifier so nodes are selected randomly.
//
// See also SelectByBucketAttribute.
func (s *Selector) SelectSame() {
s.m.SetClause(netmap.Same)
}
// SelectDistinct makes selection algorithm to select only nodes having the different values
// of the bucket attribute.
//
// Zero Selector doesn't specify selection modifier so nodes are selected randomly.
//
// See also SelectByBucketAttribute.
func (s *Selector) SelectDistinct() {
s.m.SetClause(netmap.Distinct)
}
// SetFilterName sets name of filter to reference to select from ???.
//
// Zero Selector ???.
func (s *Selector) SetFilterName(f string) {
s.m.SetFilter(f)
}
// AddSelectors adds a Selector bunch to form the subset of the nodes
// to store container objects.
//
// Zero PlacementPolicy does not declare replicas ???.
func (p *PlacementPolicy) AddSelectors(ss ...Selector) {
off := len(p.selectors)
p.selectors = append(p.selectors, make([]netmap.Selector, len(ss))...)
for i := range ss {
p.selectors[off+i] = ss[i].m
}
}
// Filter contains rules for filtering the node sets.
type Filter struct {
m netmap.Filter
}
// SetName sets name with which the Filter can be referenced or, for inner filters,
// to which the Filter references. Top-level filters MUST have be named. The name
// MUST NOT be '*'.
//
// Zero Filter is unnamed.
func (x *Filter) SetName(name string) {
x.m.SetName(name)
}
func (x *Filter) setAttribute(key string, op netmap.Operation, val string) {
x.m.SetKey(key)
x.m.SetOp(op)
x.m.SetValue(val)
}
// Equal applies the rule to accept only nodes with the same attribute value.
//
// Method SHOULD NOT be called along with other similar methods.
func (x *Filter) Equal(key, value string) {
x.setAttribute(key, netmap.EQ, value)
}
// NotEqual applies the rule to accept only nodes with the distinct attribute value.
//
// Method SHOULD NOT be called along with other similar methods.
func (x *Filter) NotEqual(key, value string) {
x.setAttribute(key, netmap.NE, value)
}
// NumericGT applies the rule to accept only nodes with the numeric attribute
// greater than given number.
//
// Method SHOULD NOT be called along with other similar methods.
func (x *Filter) NumericGT(key string, num int64) {
x.setAttribute(key, netmap.GT, strconv.FormatInt(num, 10))
}
// NumericGE applies the rule to accept only nodes with the numeric attribute
// greater than or equal to given number.
//
// Method SHOULD NOT be called along with other similar methods.
func (x *Filter) NumericGE(key string, num int64) {
x.setAttribute(key, netmap.GE, strconv.FormatInt(num, 10))
}
// NumericLT applies the rule to accept only nodes with the numeric attribute
// less than given number.
//
// Method SHOULD NOT be called along with other similar methods.
func (x *Filter) NumericLT(key string, num int64) {
x.setAttribute(key, netmap.LT, strconv.FormatInt(num, 10))
}
// NumericLE applies the rule to accept only nodes with the numeric attribute
// less than or equal to given number.
//
// Method SHOULD NOT be called along with other similar methods.
func (x *Filter) NumericLE(key string, num int64) {
x.setAttribute(key, netmap.LE, strconv.FormatInt(num, 10))
}
func (x *Filter) setInnerFilters(op netmap.Operation, filters []Filter) {
x.setAttribute("", op, "")
inner := x.m.GetFilters()
if rem := len(filters) - len(inner); rem > 0 {
inner = append(inner, make([]netmap.Filter, rem)...)
}
for i := range filters {
inner[i] = filters[i].m
}
x.m.SetFilters(inner)
}
// LogicalOR applies the rule to accept only nodes which satisfy at least one
// of the given filters.
//
// Method SHOULD NOT be called along with other similar methods.
func (x *Filter) LogicalOR(filters ...Filter) {
x.setInnerFilters(netmap.OR, filters)
}
// LogicalAND applies the rule to accept only nodes which satisfy all the given
// filters.
//
// Method SHOULD NOT be called along with other similar methods.
func (x *Filter) LogicalAND(filters ...Filter) {
x.setInnerFilters(netmap.AND, filters)
}
// AddFilters adds a Filter bunch that will be applied when selecting nodes.
//
// Zero PlacementPolicy has no filters ???.
func (p *PlacementPolicy) AddFilters(fs ...Filter) {
off := len(p.filters)
p.filters = append(p.filters, make([]netmap.Filter, len(fs))...)
for i := range fs {
p.filters[off+i] = fs[i].m
}
}
// WriteStringTo encodes PlacementPolicy into human-readably query and writes
// the result into w. Returns w's errors directly.
//
// See also DecodeString.
func (p PlacementPolicy) WriteStringTo(w io.StringWriter) (err error) {
writtenSmth := false
writeLnIfNeeded := func() error {
if writtenSmth {
_, err = w.WriteString("\n")
return err
}
writtenSmth = true
return nil
}
for i := range p.replicas {
err = writeLnIfNeeded()
if err != nil {
return err
}
c := p.replicas[i].GetCount()
s := p.replicas[i].GetSelector()
if s != "" {
_, err = w.WriteString(fmt.Sprintf("REP %d IN %s", c, s))
} else {
_, err = w.WriteString(fmt.Sprintf("REP %d", c))
}
if err != nil {
return err
}
}
if p.backupFactor > 0 {
err = writeLnIfNeeded()
if err != nil {
return err
}
_, err = w.WriteString(fmt.Sprintf("CBF %d", p.backupFactor))
if err != nil {
return err
}
}
var s string
for i := range p.selectors {
err = writeLnIfNeeded()
if err != nil {
return err
}
_, err = w.WriteString(fmt.Sprintf("SELECT %d", p.selectors[i].GetCount()))
if err != nil {
return err
}
if s = p.selectors[i].GetAttribute(); s != "" {
var clause string
switch p.selectors[i].GetClause() {
case netmap.Same:
clause = "SAME "
case netmap.Distinct:
clause = "DISTINCT "
default:
clause = ""
}
_, err = w.WriteString(fmt.Sprintf(" IN %s%s", clause, s))
if err != nil {
return err
}
}
if s = p.selectors[i].GetFilter(); s != "" {
_, err = w.WriteString(" FROM " + s)
if err != nil {
return err
}
}
if s = p.selectors[i].GetName(); s != "" {
_, err = w.WriteString(" AS " + s)
if err != nil {
return err
}
}
}
for i := range p.filters {
err = writeLnIfNeeded()
if err != nil {
return err
}
_, err = w.WriteString("FILTER ")
if err != nil {
return err
}
err = writeFilterStringTo(w, p.filters[i])
if err != nil {
return err
}
}
return nil
}
func writeFilterStringTo(w io.StringWriter, f netmap.Filter) error {
var err error
var s string
op := f.GetOp()
unspecified := op == 0
if s = f.GetKey(); s != "" {
_, err = w.WriteString(fmt.Sprintf("%s %s %s", s, op, f.GetValue()))
if err != nil {
return err
}
} else if s = f.GetName(); unspecified && s != "" {
_, err = w.WriteString(fmt.Sprintf("@%s", s))
if err != nil {
return err
}
}
inner := f.GetFilters()
for i := range inner {
if i != 0 {
_, err = w.WriteString(" " + op.String() + " ")
if err != nil {
return err
}
}
err = writeFilterStringTo(w, inner[i])
if err != nil {
return err
}
}
if s = f.GetName(); s != "" && !unspecified {
_, err = w.WriteString(" AS " + s)
if err != nil {
return err
}
}
return nil
}
// DecodeString decodes PlacementPolicy from the string composed using
// WriteStringTo. Returns error if s is malformed.
func (p *PlacementPolicy) DecodeString(s string) error {
input := antlr.NewInputStream(s)
lexer := parser.NewQueryLexer(input)
stream := antlr.NewCommonTokenStream(lexer, 0)
pp := parser.NewQuery(stream)
pp.BuildParseTrees = true
var v policyVisitor
pp.RemoveErrorListeners()
pp.AddErrorListener(&v)
pl := pp.Policy().Accept(&v)
if len(v.errors) != 0 {
return v.errors[0]
}
parsed, ok := pl.(*PlacementPolicy)
if !ok {
return fmt.Errorf("unexpected parsed instance type %T", pl)
} else if parsed == nil {
return errors.New("parsed nil value")
}
if err := validatePolicy(*p); err != nil {
return fmt.Errorf("invalid policy: %w", err)
}
*p = *parsed
return nil
}
var (
// errUnknownFilter is returned when a value of FROM in a query is unknown.
errUnknownFilter = errors.New("filter not found")
// errUnknownSelector is returned when a value of IN is unknown.
errUnknownSelector = errors.New("policy: selector not found")
// errSyntaxError is returned for errors found by ANTLR parser.
errSyntaxError = errors.New("policy: syntax error")
)
type policyVisitor struct {
errors []error
parser.BaseQueryVisitor
antlr.DefaultErrorListener
}
func (p *policyVisitor) SyntaxError(_ antlr.Recognizer, _ interface{}, line, column int, msg string, _ antlr.RecognitionException) {
p.reportError(fmt.Errorf("%w: line %d:%d %s", errSyntaxError, line, column, msg))
}
func (p *policyVisitor) reportError(err error) interface{} {
p.errors = append(p.errors, err)
return nil
}
// VisitPolicy implements parser.QueryVisitor interface.
func (p *policyVisitor) VisitPolicy(ctx *parser.PolicyContext) interface{} {
if len(p.errors) != 0 {
return nil
}
pl := new(PlacementPolicy)
repStmts := ctx.AllRepStmt()
pl.replicas = make([]netmap.Replica, 0, len(repStmts))
for _, r := range repStmts {
res, ok := r.Accept(p).(*netmap.Replica)
if !ok {
return nil
}
pl.replicas = append(pl.replicas, *res)
}
if cbfStmt := ctx.CbfStmt(); cbfStmt != nil {
cbf, ok := cbfStmt.(*parser.CbfStmtContext).Accept(p).(uint32)
if !ok {
return nil
}
pl.SetContainerBackupFactor(cbf)
}
selStmts := ctx.AllSelectStmt()
pl.selectors = make([]netmap.Selector, 0, len(selStmts))
for _, s := range selStmts {
res, ok := s.Accept(p).(*netmap.Selector)
if !ok {
return nil
}
pl.selectors = append(pl.selectors, *res)
}
filtStmts := ctx.AllFilterStmt()
pl.filters = make([]netmap.Filter, 0, len(filtStmts))
for _, f := range filtStmts {
pl.filters = append(pl.filters, *f.Accept(p).(*netmap.Filter))
}
return pl
}
func (p *policyVisitor) VisitCbfStmt(ctx *parser.CbfStmtContext) interface{} {
cbf, err := strconv.ParseUint(ctx.GetBackupFactor().GetText(), 10, 32)
if err != nil {
return p.reportError(errInvalidNumber)
}
return uint32(cbf)
}
// VisitRepStmt implements parser.QueryVisitor interface.
func (p *policyVisitor) VisitRepStmt(ctx *parser.RepStmtContext) interface{} {
num, err := strconv.ParseUint(ctx.GetCount().GetText(), 10, 32)
if err != nil {
return p.reportError(errInvalidNumber)
}
rs := new(netmap.Replica)
rs.SetCount(uint32(num))
if sel := ctx.GetSelector(); sel != nil {
rs.SetSelector(sel.GetText())
}
return rs
}
// VisitSelectStmt implements parser.QueryVisitor interface.
func (p *policyVisitor) VisitSelectStmt(ctx *parser.SelectStmtContext) interface{} {
res, err := strconv.ParseUint(ctx.GetCount().GetText(), 10, 32)
if err != nil {
return p.reportError(errInvalidNumber)
}
s := new(netmap.Selector)
s.SetCount(uint32(res))
if clStmt := ctx.Clause(); clStmt != nil {
s.SetClause(clauseFromString(clStmt.GetText()))
}
if bStmt := ctx.GetBucket(); bStmt != nil {
s.SetAttribute(ctx.GetBucket().GetText())
}
s.SetFilter(ctx.GetFilter().GetText()) // either ident or wildcard
if ctx.AS() != nil {
s.SetName(ctx.GetName().GetText())
}
return s
}
// VisitFilterStmt implements parser.QueryVisitor interface.
func (p *policyVisitor) VisitFilterStmt(ctx *parser.FilterStmtContext) interface{} {
f := p.VisitFilterExpr(ctx.GetExpr().(*parser.FilterExprContext)).(*netmap.Filter)
f.SetName(ctx.GetName().GetText())
return f
}
func (p *policyVisitor) VisitFilterExpr(ctx *parser.FilterExprContext) interface{} {
if eCtx := ctx.Expr(); eCtx != nil {
return eCtx.Accept(p)
}
if inner := ctx.GetInner(); inner != nil {
return inner.Accept(p)
}
f := new(netmap.Filter)
op := operationFromString(ctx.GetOp().GetText())
f.SetOp(op)
f1 := *ctx.GetF1().Accept(p).(*netmap.Filter)
f2 := *ctx.GetF2().Accept(p).(*netmap.Filter)
// Consider f1=(.. AND ..) AND f2. This can be merged because our AND operation
// is of arbitrary arity. ANTLR generates left-associative parse-tree by default.
if f1.GetOp() == op {
f.SetFilters(append(f1.GetFilters(), f2))
return f
}
f.SetFilters([]netmap.Filter{f1, f2})
return f
}
// VisitFilterKey implements parser.QueryVisitor interface.
func (p *policyVisitor) VisitFilterKey(ctx *parser.FilterKeyContext) interface{} {
if id := ctx.Ident(); id != nil {
return id.GetText()
}
str := ctx.STRING().GetText()
return str[1 : len(str)-1]
}
func (p *policyVisitor) VisitFilterValue(ctx *parser.FilterValueContext) interface{} {
if id := ctx.Ident(); id != nil {
return id.GetText()
}
if num := ctx.Number(); num != nil {
return num.GetText()
}
str := ctx.STRING().GetText()
return str[1 : len(str)-1]
}
// VisitExpr implements parser.QueryVisitor interface.
func (p *policyVisitor) VisitExpr(ctx *parser.ExprContext) interface{} {
f := new(netmap.Filter)
if flt := ctx.GetFilter(); flt != nil {
f.SetName(flt.GetText())
return f
}
key := ctx.GetKey().Accept(p)
opStr := ctx.SIMPLE_OP().GetText()
value := ctx.GetValue().Accept(p)
f.SetKey(key.(string))
f.SetOp(operationFromString(opStr))
f.SetValue(value.(string))
return f
}
// validatePolicy checks high-level constraints such as filter link in SELECT
// being actually defined in FILTER section.
func validatePolicy(p PlacementPolicy) error {
seenFilters := map[string]bool{}
for i := range p.filters {
seenFilters[p.filters[i].GetName()] = true
}
seenSelectors := map[string]bool{}
for i := range p.selectors {
if flt := p.selectors[i].GetFilter(); flt != mainFilterName && !seenFilters[flt] {
return fmt.Errorf("%w: '%s'", errUnknownFilter, flt)
}
seenSelectors[p.selectors[i].GetName()] = true
}
for i := range p.replicas {
if sel := p.replicas[i].GetSelector(); sel != "" && !seenSelectors[sel] {
return fmt.Errorf("%w: '%s'", errUnknownSelector, sel)
}
}
return nil
}
func clauseFromString(s string) (c netmap.Clause) {
if !c.FromString(strings.ToUpper(s)) {
// Such errors should be handled by ANTLR code thus this panic.
panic(fmt.Errorf("BUG: invalid clause: %s", c))
}
return
}
func operationFromString(s string) (op netmap.Operation) {
if !op.FromString(strings.ToUpper(s)) {
// Such errors should be handled by ANTLR code thus this panic.
panic(fmt.Errorf("BUG: invalid operation: %s", op))
}
return
}

180
netmap/policy_test.go
View file

@ -1,127 +1,123 @@
package netmap
import (
"math/rand"
"strings"
"sync"
"testing"
"github.com/nspcc-dev/neofs-api-go/v2/netmap"
"github.com/stretchr/testify/require"
)
func TestPlacementPolicyFromV2(t *testing.T) {
pV2 := new(netmap.PlacementPolicy)
func TestEncode(t *testing.T) {
testCases := []string{
`REP 1 IN X
CBF 1
SELECT 2 IN SAME Location FROM * AS X`,
pV2.SetReplicas([]netmap.Replica{
*testReplica().ToV2(),
*testReplica().ToV2(),
})
`REP 1
SELECT 2 IN City FROM Good
FILTER Country EQ RU AS FromRU
FILTER @FromRU AND Rating GT 7 AS Good`,
pV2.SetContainerBackupFactor(3)
`REP 7 IN SPB
SELECT 1 IN City FROM SPBSSD AS SPB
FILTER City EQ SPB AND SSD EQ true OR City EQ SPB AND Rating GE 5 AS SPBSSD`,
}
pV2.SetSelectors([]netmap.Selector{
*testSelector().ToV2(),
*testSelector().ToV2(),
})
for _, testCase := range testCases {
var p PlacementPolicy
pV2.SetFilters([]netmap.Filter{
*testFilter().ToV2(),
*testFilter().ToV2(),
})
require.NoError(t, p.DecodeString(testCase))
p := NewPlacementPolicyFromV2(pV2)
var b strings.Builder
require.NoError(t, p.WriteStringTo(&b))
require.Equal(t, pV2, p.ToV2())
require.Equal(t, testCase, b.String())
}
}
func TestPlacementPolicy_Replicas(t *testing.T) {
p := NewPlacementPolicy()
rs := []Replica{*testReplica(), *testReplica()}
type cache struct {
mtx sync.RWMutex
p.SetReplicas(rs...)
require.Equal(t, rs, p.Replicas())
item map[string]struct{}
}
func TestPlacementPolicy_ContainerBackupFactor(t *testing.T) {
p := NewPlacementPolicy()
f := uint32(3)
p.SetContainerBackupFactor(f)
require.Equal(t, f, p.ContainerBackupFactor())
func (x *cache) add(key string) {
x.mtx.Lock()
x.item[key] = struct{}{}
x.mtx.Unlock()
}
func TestPlacementPolicy_Selectors(t *testing.T) {
p := NewPlacementPolicy()
ss := []Selector{*testSelector(), *testSelector()}
func (x *cache) has(key string) bool {
x.mtx.RLock()
_, ok := x.item[key]
x.mtx.RUnlock()
p.SetSelectors(ss...)
require.Equal(t, ss, p.Selectors())
return ok
}
func TestPlacementPolicy_Filters(t *testing.T) {
p := NewPlacementPolicy()
fs := []Filter{*testFilter(), *testFilter()}
func BenchmarkCache(b *testing.B) {
c := cache{
item: make(map[string]struct{}),
}
p.SetFilters(fs...)
var key string
buf := make([]byte, 32)
require.Equal(t, fs, p.Filters())
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
b.StopTimer()
rand.Read(buf)
key = string(buf)
b.StartTimer()
c.add(key)
c.has(key)
}
}
func TestPlacementPolicyEncoding(t *testing.T) {
p := newPlacementPolicy(3, nil, nil, nil)
type cacheP struct {
mtx *sync.RWMutex
t.Run("binary", func(t *testing.T) {
data, err := p.Marshal()
require.NoError(t, err)
p2 := NewPlacementPolicy()
require.NoError(t, p2.Unmarshal(data))
require.Equal(t, p, p2)
})
t.Run("json", func(t *testing.T) {
data, err := p.MarshalJSON()
require.NoError(t, err)
p2 := NewPlacementPolicy()
require.NoError(t, p2.UnmarshalJSON(data))
require.Equal(t, p, p2)
})
item map[string]struct{}
}
func TestNewPlacementPolicy(t *testing.T) {
t.Run("nil", func(t *testing.T) {
var x *PlacementPolicy
require.Nil(t, x.ToV2())
})
t.Run("default values", func(t *testing.T) {
pp := NewPlacementPolicy()
// check initial values
require.Nil(t, pp.Replicas())
require.Nil(t, pp.Filters())
require.Nil(t, pp.Selectors())
require.Zero(t, pp.ContainerBackupFactor())
// convert to v2 message
ppV2 := pp.ToV2()
require.Nil(t, ppV2.GetReplicas())
require.Nil(t, ppV2.GetFilters())
require.Nil(t, ppV2.GetSelectors())
require.Zero(t, ppV2.GetContainerBackupFactor())
})
func (x cacheP) add(key string) {
x.mtx.Lock()
x.item[key] = struct{}{}
x.mtx.Unlock()
}
func TestNewPlacementPolicyFromV2(t *testing.T) {
t.Run("from nil", func(t *testing.T) {
var x *netmap.PlacementPolicy
func (x cacheP) has(key string) bool {
x.mtx.RLock()
_, ok := x.item[key]
x.mtx.RUnlock()
require.Nil(t, NewPlacementPolicyFromV2(x))
})
return ok
}
func BenchmarkCacheP(b *testing.B) {
c := cacheP{
mtx: &sync.RWMutex{},
item: make(map[string]struct{}),
}
var key string
buf := make([]byte, 32)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
b.StopTimer()
rand.Read(buf)
key = string(buf)
b.StartTimer()
c.add(key)
c.has(key)
}
}

71
netmap/replica.go
View file

@ -1,71 +0,0 @@
package netmap
import (
"github.com/nspcc-dev/neofs-api-go/v2/netmap"
)
// Replica represents v2-compatible object replica descriptor.
type Replica netmap.Replica
// NewReplica creates and returns new Replica instance.
//
// Defaults:
// - count: 0;
// - selector: "".
func NewReplica() *Replica {
return NewReplicaFromV2(new(netmap.Replica))
}
// NewReplicaFromV2 converts v2 Replica to Replica.
//
// Nil netmap.Replica converts to nil.
func NewReplicaFromV2(f *netmap.Replica) *Replica {
return (*Replica)(f)
}
// ToV2 converts Replica to v2 Replica.
//
// Nil Replica converts to nil.
func (r *Replica) ToV2() *netmap.Replica {
return (*netmap.Replica)(r)
}
// Count returns number of object replicas.
func (r *Replica) Count() uint32 {
return (*netmap.Replica)(r).GetCount()
}
// SetCount sets number of object replicas.
func (r *Replica) SetCount(c uint32) {
(*netmap.Replica)(r).SetCount(c)
}
// Selector returns name of selector bucket to put replicas.
func (r *Replica) Selector() string {
return (*netmap.Replica)(r).GetSelector()
}
// SetSelector sets name of selector bucket to put replicas.
func (r *Replica) SetSelector(s string) {
(*netmap.Replica)(r).SetSelector(s)
}
// Marshal marshals Replica into a protobuf binary form.
func (r *Replica) Marshal() ([]byte, error) {
return (*netmap.Replica)(r).StableMarshal(nil), nil
}
// Unmarshal unmarshals protobuf binary representation of Replica.
func (r *Replica) Unmarshal(data []byte) error {
return (*netmap.Replica)(r).Unmarshal(data)
}
// MarshalJSON encodes Replica to protobuf JSON format.
func (r *Replica) MarshalJSON() ([]byte, error) {
return (*netmap.Replica)(r).MarshalJSON()
}
// UnmarshalJSON decodes Replica from protobuf JSON format.
func (r *Replica) UnmarshalJSON(data []byte) error {
return (*netmap.Replica)(r).UnmarshalJSON(data)
}

99
netmap/replica_test.go
View file

@ -1,99 +0,0 @@
package netmap
import (
"testing"
"github.com/nspcc-dev/neofs-api-go/v2/netmap"
testv2 "github.com/nspcc-dev/neofs-api-go/v2/netmap/test"
"github.com/stretchr/testify/require"
)
func testReplica() *Replica {
r := new(Replica)
r.SetCount(3)
r.SetSelector("selector")
return r
}
func TestReplicaFromV2(t *testing.T) {
t.Run("from nil", func(t *testing.T) {
var x *netmap.Replica
require.Nil(t, NewReplicaFromV2(x))
})
t.Run("from non-nil", func(t *testing.T) {
rV2 := testv2.GenerateReplica(false)
r := NewReplicaFromV2(rV2)
require.Equal(t, rV2, r.ToV2())
})
}
func TestReplica_Count(t *testing.T) {
r := NewReplica()
c := uint32(3)
r.SetCount(c)
require.Equal(t, c, r.Count())
}
func TestReplica_Selector(t *testing.T) {
r := NewReplica()
s := "some selector"
r.SetSelector(s)
require.Equal(t, s, r.Selector())
}
func TestReplicaEncoding(t *testing.T) {
r := newReplica(3, "selector")
t.Run("binary", func(t *testing.T) {
data, err := r.Marshal()
require.NoError(t, err)
r2 := *NewReplica()
require.NoError(t, r2.Unmarshal(data))
require.Equal(t, r, r2)
})
t.Run("json", func(t *testing.T) {
data, err := r.MarshalJSON()
require.NoError(t, err)
r2 := *NewReplica()
require.NoError(t, r2.UnmarshalJSON(data))
require.Equal(t, r, r2)
})
}
func TestReplica_ToV2(t *testing.T) {
t.Run("nil", func(t *testing.T) {
var x *Replica
require.Nil(t, x.ToV2())
})
}
func TestNewReplica(t *testing.T) {
t.Run("default values", func(t *testing.T) {
r := NewReplica()
// check initial values
require.Zero(t, r.Count())
require.Empty(t, r.Selector())
// convert to v2 message
rV2 := r.ToV2()
require.Zero(t, rV2.GetCount())
require.Empty(t, rV2.GetSelector())
})
}

205
netmap/selector.go
View file

@ -9,62 +9,70 @@ import (
subnetid "github.com/nspcc-dev/neofs-sdk-go/subnet/id"
)
// Selector represents v2-compatible netmap selector.
type Selector netmap.Selector
// processSelectors processes selectors and returns error is any of them is invalid.
func (c *context) processSelectors(p *PlacementPolicy) error {
selectors := p.Selectors()
for i, s := range selectors {
if s.Filter() != MainFilterName {
_, ok := c.Filters[s.Filter()]
func (c *context) processSelectors(p PlacementPolicy) error {
for i := range p.selectors {
fName := p.selectors[i].GetFilter()
if fName != mainFilterName {
_, ok := c.processedFilters[p.selectors[i].GetFilter()]
if !ok {
return fmt.Errorf("%w: SELECT FROM '%s'", ErrFilterNotFound, s.Filter())
return fmt.Errorf("%w: SELECT FROM '%s'", errFilterNotFound, fName)
}
}
c.Selectors[s.Name()] = &selectors[i]
sName := p.selectors[i].GetName()
result, err := c.getSelection(p, &s)
c.processedSelectors[sName] = &p.selectors[i]
result, err := c.getSelection(p, p.selectors[i])
if err != nil {
return err
}
c.Selections[s.Name()] = result
c.selections[sName] = result
}
return nil
}
// GetNodesCount returns amount of buckets and minimum number of nodes in every bucket
// calcNodesCount returns amount of buckets and minimum number of nodes in every bucket
// for the given selector.
func GetNodesCount(_ *PlacementPolicy, s *Selector) (int, int) {
switch s.Clause() {
case ClauseSame:
return 1, int(s.Count())
func calcNodesCount(s netmap.Selector) (int, int) {
switch s.GetClause() {
case netmap.Same:
return 1, int(s.GetCount())
default:
return int(s.Count()), 1
return int(s.GetCount()), 1
}
}
// calcBucketWeight computes weight for a node bucket.
func calcBucketWeight(ns nodes, a aggregator, wf weightFunc) float64 {
for i := range ns {
a.Add(wf(ns[i]))
}
return a.Compute()
}
// getSelection returns nodes grouped by s.attribute.
// Last argument specifies if more buckets can be used to fulfill CBF.
func (c *context) getSelection(p *PlacementPolicy, s *Selector) ([]nodes, error) {
bucketCount, nodesInBucket := GetNodesCount(p, s)
buckets := c.getSelectionBase(p.SubnetID(), s)
func (c *context) getSelection(p PlacementPolicy, s netmap.Selector) ([]nodes, error) {
bucketCount, nodesInBucket := calcNodesCount(s)
buckets := c.getSelectionBase(p.subnet, s)
if len(buckets) < bucketCount {
return nil, fmt.Errorf("%w: '%s'", ErrNotEnoughNodes, s.Name())
return nil, fmt.Errorf("%w: '%s'", errNotEnoughNodes, s.GetName())
}
// We need deterministic output in case there is no pivot.
// If pivot is set, buckets are sorted by HRW.
// However, because initial order influences HRW order for buckets with equal weights,
// we also need to have deterministic input to HRW sorting routine.
if len(c.pivot) == 0 {
if s.Attribute() == "" {
if len(c.hrwSeed) == 0 {
if s.GetAttribute() == "" {
sort.Slice(buckets, func(i, j int) bool {
return buckets[i].nodes[0].less(buckets[j].nodes[0])
return less(buckets[i].nodes[0], buckets[j].nodes[0])
})
} else {
sort.Slice(buckets, func(i, j int) bool {
@ -90,20 +98,20 @@ func (c *context) getSelection(p *PlacementPolicy, s *Selector) ([]nodes, error)
// Fallback to using minimum allowed backup factor (1).
res = append(res, fallback...)
if len(res) < bucketCount {
return nil, fmt.Errorf("%w: '%s'", ErrNotEnoughNodes, s.Name())
return nil, fmt.Errorf("%w: '%s'", errNotEnoughNodes, s.GetName())
}
}
if len(c.pivot) != 0 {
if len(c.hrwSeed) != 0 {
weights := make([]float64, len(res))
for i := range res {
weights[i] = calcBucketWeight(res[i], c.aggregator(), c.weightFunc)
weights[i] = calcBucketWeight(res[i], newMeanIQRAgg(), c.weightFunc)
}
hrw.SortSliceByWeightValue(res, weights, c.pivotHash)
hrw.SortSliceByWeightValue(res, weights, c.hrwSeedHash)
}
if s.Attribute() == "" {
if s.GetAttribute() == "" {
res, fallback = res[:bucketCount], res[bucketCount:]
for i := range fallback {
index := i % bucketCount
@ -124,29 +132,26 @@ type nodeAttrPair struct {
// getSelectionBase returns nodes grouped by selector attribute.
// It it guaranteed that each pair will contain at least one node.
func (c *context) getSelectionBase(subnetID *subnetid.ID, s *Selector) []nodeAttrPair {
f := c.Filters[s.Filter()]
isMain := s.Filter() == MainFilterName
func (c *context) getSelectionBase(subnetID subnetid.ID, s netmap.Selector) []nodeAttrPair {
fName := s.GetFilter()
f := c.processedFilters[fName]
isMain := fName == mainFilterName
result := []nodeAttrPair{}
nodeMap := map[string][]NodeInfo{}
attr := s.Attribute()
attr := s.GetAttribute()
for i := range c.Netmap.nodes {
var sid subnetid.ID
if subnetID != nil {
sid = *subnetID
}
for i := range c.netMap.nodes {
// TODO(fyrchik): make `BelongsToSubnet` to accept pointer
if !BelongsToSubnet(&c.Netmap.nodes[i], sid) {
if !BelongsToSubnet(c.netMap.nodes[i], subnetID) {
continue
}
if isMain || c.match(f, c.Netmap.nodes[i]) {
if isMain || c.match(f, c.netMap.nodes[i]) {
if attr == "" {
// Default attribute is transparent identifier which is different for every node.
result = append(result, nodeAttrPair{attr: "", nodes: nodes{c.Netmap.nodes[i]}})
result = append(result, nodeAttrPair{attr: "", nodes: nodes{c.netMap.nodes[i]}})
} else {
v := c.Netmap.nodes[i].attribute(attr)
nodeMap[v] = append(nodeMap[v], c.Netmap.nodes[i])
v := c.netMap.nodes[i].Attribute(attr)
nodeMap[v] = append(nodeMap[v], c.netMap.nodes[i])
}
}
}
@ -157,119 +162,11 @@ func (c *context) getSelectionBase(subnetID *subnetid.ID, s *Selector) []nodeAtt
}
}
if len(c.pivot) != 0 {
if len(c.hrwSeed) != 0 {
for i := range result {
hrw.SortSliceByWeightValue(result[i].nodes, result[i].nodes.weights(c.weightFunc), c.pivotHash)
hrw.SortSliceByWeightValue(result[i].nodes, result[i].nodes.weights(c.weightFunc), c.hrwSeedHash)
}
}
return result
}
// NewSelector creates and returns new Selector instance.
//
// Defaults:
// - name: "";
// - attribute: "";
// - filter: "";
// - clause: ClauseUnspecified;
// - count: 0.
func NewSelector() *Selector {
return NewSelectorFromV2(new(netmap.Selector))
}
// NewSelectorFromV2 converts v2 Selector to Selector.
//
// Nil netmap.Selector converts to nil.
func NewSelectorFromV2(f *netmap.Selector) *Selector {
return (*Selector)(f)
}
// ToV2 converts Selector to v2 Selector.
//
// Nil Selector converts to nil.
func (s *Selector) ToV2() *netmap.Selector {
return (*netmap.Selector)(s)
}
// Name returns selector name.
func (s *Selector) Name() string {
return (*netmap.Selector)(s).
GetName()
}
// SetName sets selector name.
func (s *Selector) SetName(name string) {
(*netmap.Selector)(s).
SetName(name)
}
// Count returns count of nodes to select from bucket.
func (s *Selector) Count() uint32 {
return (*netmap.Selector)(s).
GetCount()
}
// SetCount sets count of nodes to select from bucket.
func (s *Selector) SetCount(c uint32) {
(*netmap.Selector)(s).
SetCount(c)
}
// Clause returns modifier showing how to form a bucket.
func (s *Selector) Clause() Clause {
return ClauseFromV2(
(*netmap.Selector)(s).
GetClause(),
)
}
// SetClause sets modifier showing how to form a bucket.
func (s *Selector) SetClause(c Clause) {
(*netmap.Selector)(s).
SetClause(c.ToV2())
}
// Attribute returns attribute bucket to select from.
func (s *Selector) Attribute() string {
return (*netmap.Selector)(s).
GetAttribute()
}
// SetAttribute sets attribute bucket to select from.
func (s *Selector) SetAttribute(a string) {
(*netmap.Selector)(s).
SetAttribute(a)
}
// Filter returns filter reference to select from.
func (s *Selector) Filter() string {
return (*netmap.Selector)(s).
GetFilter()
}
// SetFilter sets filter reference to select from.
func (s *Selector) SetFilter(f string) {
(*netmap.Selector)(s).
SetFilter(f)
}
// Marshal marshals Selector into a protobuf binary form.
func (s *Selector) Marshal() ([]byte, error) {
return (*netmap.Selector)(s).StableMarshal(nil), nil
}
// Unmarshal unmarshals protobuf binary representation of Selector.
func (s *Selector) Unmarshal(data []byte) error {
return (*netmap.Selector)(s).Unmarshal(data)
}
// MarshalJSON encodes Selector to protobuf JSON format.
func (s *Selector) MarshalJSON() ([]byte, error) {
return (*netmap.Selector)(s).MarshalJSON()
}
// UnmarshalJSON decodes Selector from protobuf JSON format.
func (s *Selector) UnmarshalJSON(data []byte) error {
return (*netmap.Selector)(s).UnmarshalJSON(data)
}

203
netmap/selector_test.go
View file

@ -9,7 +9,6 @@ import (
"github.com/nspcc-dev/hrw"
"github.com/nspcc-dev/neofs-api-go/v2/netmap"
testv2 "github.com/nspcc-dev/neofs-api-go/v2/netmap/test"
"github.com/stretchr/testify/require"
)
@ -23,8 +22,8 @@ func BenchmarkHRWSort(b *testing.B) {
rand.Read(key)
var node NodeInfo
node.setPrice(1)
node.setCapacity(100)
node.SetPrice(1)
node.SetCapacity(100)
node.SetPublicKey(key)
vectors[i] = nodes{node}
@ -85,7 +84,7 @@ func BenchmarkHRWSort(b *testing.B) {
b.StartTimer()
sort.Slice(vectors, func(i, j int) bool {
return vectors[i][0].less(vectors[j][0])
return less(vectors[i][0], vectors[j][0])
})
hrw.SortSliceByWeightIndex(realNodes, weights, pivot)
}
@ -96,15 +95,15 @@ func BenchmarkPolicyHRWType(b *testing.B) {
const netmapSize = 100
p := newPlacementPolicy(1,
[]Replica{
[]ReplicaDescriptor{
newReplica(1, "loc1"),
newReplica(1, "loc2")},
[]Selector{
newSelector("loc1", "Location", ClauseSame, 1, "loc1"),
newSelector("loc2", "Location", ClauseSame, 1, "loc2")},
newSelector("loc1", "Location", 1, "loc1", (*Selector).SelectSame),
newSelector("loc2", "Location", 1, "loc2", (*Selector).SelectSame)},
[]Filter{
newFilter("loc1", "Location", "Shanghai", OpEQ),
newFilter("loc2", "Location", "Shanghai", OpNE),
newFilter("loc1", "Location", "Shanghai", netmap.EQ),
newFilter("loc2", "Location", "Shanghai", netmap.NE),
})
nodes := make([]NodeInfo, netmapSize)
@ -125,12 +124,12 @@ func BenchmarkPolicyHRWType(b *testing.B) {
nodes[i].SetPublicKey(pub)
}
var nm Netmap
var nm NetMap
nm.SetNodes(nodes)
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, err := nm.GetContainerNodes(p, []byte{1})
_, err := nm.ContainerNodes(p, []byte{1})
if err != nil {
b.Fatal()
}
@ -141,15 +140,15 @@ func TestPlacementPolicy_DeterministicOrder(t *testing.T) {
const netmapSize = 100
p := newPlacementPolicy(1,
[]Replica{
[]ReplicaDescriptor{
newReplica(1, "loc1"),
newReplica(1, "loc2")},
[]Selector{
newSelector("loc1", "Location", ClauseSame, 1, "loc1"),
newSelector("loc2", "Location", ClauseSame, 1, "loc2")},
newSelector("loc1", "Location", 1, "loc1", (*Selector).SelectSame),
newSelector("loc2", "Location", 1, "loc2", (*Selector).SelectSame)},
[]Filter{
newFilter("loc1", "Location", "Shanghai", OpEQ),
newFilter("loc2", "Location", "Shanghai", OpNE),
newFilter("loc1", "Location", "Shanghai", netmap.EQ),
newFilter("loc2", "Location", "Shanghai", netmap.NE),
})
nodeList := make([]NodeInfo, netmapSize)
@ -170,11 +169,11 @@ func TestPlacementPolicy_DeterministicOrder(t *testing.T) {
nodeList[i].SetPublicKey(pub)
}
var nm Netmap
var nm NetMap
nm.SetNodes(nodeList)
getIndices := func(t *testing.T) (uint64, uint64) {
v, err := nm.GetContainerNodes(p, []byte{1})
v, err := nm.ContainerNodes(p, []byte{1})
require.NoError(t, err)
nss := make([]nodes, len(v))
@ -198,14 +197,14 @@ func TestPlacementPolicy_DeterministicOrder(t *testing.T) {
func TestPlacementPolicy_ProcessSelectors(t *testing.T) {
p := newPlacementPolicy(2, nil,
[]Selector{
newSelector("SameRU", "City", ClauseSame, 2, "FromRU"),
newSelector("DistinctRU", "City", ClauseDistinct, 2, "FromRU"),
newSelector("Good", "Country", ClauseDistinct, 2, "Good"),
newSelector("Main", "Country", ClauseDistinct, 3, "*"),
newSelector("SameRU", "City", 2, "FromRU", (*Selector).SelectSame),
newSelector("DistinctRU", "City", 2, "FromRU", (*Selector).SelectDistinct),
newSelector("Good", "Country", 2, "Good", (*Selector).SelectDistinct),
newSelector("Main", "Country", 3, "*", (*Selector).SelectDistinct),
},
[]Filter{
newFilter("FromRU", "Country", "Russia", OpEQ),
newFilter("Good", "Rating", "4", OpGE),
newFilter("FromRU", "Country", "Russia", netmap.EQ),
newFilter("Good", "Rating", "4", netmap.GE),
})
nodes := []NodeInfo{
nodeInfoFromAttributes("Country", "Russia", "Rating", "1", "City", "SPB"),
@ -221,151 +220,79 @@ func TestPlacementPolicy_ProcessSelectors(t *testing.T) {
nodeInfoFromAttributes("Country", "Russia", "Rating", "9", "City", "SPB"),
}
var nm Netmap
var nm NetMap
nm.SetNodes(nodes)
c := newContext(&nm)
c.setCBF(p.ContainerBackupFactor())
c := newContext(nm)
c.setCBF(p.backupFactor)
require.NoError(t, c.processFilters(p))
require.NoError(t, c.processSelectors(p))
for _, s := range p.Selectors() {
sel := c.Selections[s.Name()]
s := c.Selectors[s.Name()]
bucketCount, nodesInBucket := GetNodesCount(p, s)
for _, s := range p.selectors {
sel := c.selections[s.GetName()]
s := c.processedSelectors[s.GetName()]
bucketCount, nodesInBucket := calcNodesCount(*s)
nodesInBucket *= int(c.cbf)
targ := fmt.Sprintf("selector '%s'", s.Name())
targ := fmt.Sprintf("selector '%s'", s.GetName())
require.Equal(t, bucketCount, len(sel), targ)
fName := s.GetFilter()
for _, res := range sel {
require.Equal(t, nodesInBucket, len(res), targ)
for j := range res {
require.True(t, c.applyFilter(s.Filter(), res[j]), targ)
require.True(t, fName == mainFilterName || c.match(c.processedFilters[fName], res[j]), targ)
}
}
}
}
func testSelector() *Selector {
s := new(Selector)
s.SetName("name")
s.SetCount(3)
s.SetFilter("filter")
s.SetAttribute("attribute")
s.SetClause(ClauseDistinct)
func TestSelector_SetName(t *testing.T) {
const name = "some name"
var s Selector
return s
}
func TestSelector_Name(t *testing.T) {
s := NewSelector()
name := "some name"
require.Zero(t, s.m.GetName())
s.SetName(name)
require.Equal(t, name, s.Name())
require.Equal(t, name, s.m.GetName())
}
func TestSelector_Count(t *testing.T) {
s := NewSelector()
c := uint32(3)
func TestSelector_SetNodeAmount(t *testing.T) {
const amount = 3
var s Selector
s.SetCount(c)
require.Zero(t, s.m.GetCount())
require.Equal(t, c, s.Count())
s.SetNodeAmount(amount)
require.EqualValues(t, amount, s.m.GetCount())
}
func TestSelector_Clause(t *testing.T) {
s := NewSelector()
c := ClauseSame
func TestSelectorClauses(t *testing.T) {
var s Selector
s.SetClause(c)
require.Equal(t, netmap.UnspecifiedClause, s.m.GetClause())
require.Equal(t, c, s.Clause())
s.SelectDistinct()
require.Equal(t, netmap.Distinct, s.m.GetClause())
s.SelectSame()
require.Equal(t, netmap.Same, s.m.GetClause())
}
func TestSelector_Attribute(t *testing.T) {
s := NewSelector()
a := "some attribute"
func TestSelector_SelectByBucketAttribute(t *testing.T) {
const attr = "some attribute"
var s Selector
s.SetAttribute(a)
require.Zero(t, s.m.GetAttribute())
require.Equal(t, a, s.Attribute())
s.SelectByBucketAttribute(attr)
require.Equal(t, attr, s.m.GetAttribute())
}
func TestSelector_Filter(t *testing.T) {
s := NewSelector()
f := "some filter"
func TestSelector_SetFilterName(t *testing.T) {
const fName = "some filter"
var s Selector
s.SetFilter(f)
require.Zero(t, s.m.GetFilter())
require.Equal(t, f, s.Filter())
}
func TestSelectorEncoding(t *testing.T) {
s := newSelector("name", "atte", ClauseSame, 1, "filter")
t.Run("binary", func(t *testing.T) {
data, err := s.Marshal()
require.NoError(t, err)
s2 := *NewSelector()
require.NoError(t, s2.Unmarshal(data))
require.Equal(t, s, s2)
})
t.Run("json", func(t *testing.T) {
data, err := s.MarshalJSON()
require.NoError(t, err)
s2 := *NewSelector()
require.NoError(t, s2.UnmarshalJSON(data))
require.Equal(t, s, s2)
})
}
func TestSelector_ToV2(t *testing.T) {
t.Run("nil", func(t *testing.T) {
var x *Selector
require.Nil(t, x.ToV2())
})
}
func TestNewSelectorFromV2(t *testing.T) {
t.Run("from nil", func(t *testing.T) {
var x *netmap.Selector
require.Nil(t, NewSelectorFromV2(x))
})
t.Run("from non-nil", func(t *testing.T) {
sV2 := testv2.GenerateSelector(false)
s := NewSelectorFromV2(sV2)
require.Equal(t, sV2, s.ToV2())
})
}
func TestNewSelector(t *testing.T) {
t.Run("default values", func(t *testing.T) {
s := NewSelector()
// check initial values
require.Zero(t, s.Count())
require.Equal(t, ClauseUnspecified, s.Clause())
require.Empty(t, s.Attribute())
require.Empty(t, s.Name())
require.Empty(t, s.Filter())
// convert to v2 message
sV2 := s.ToV2()
require.Zero(t, sV2.GetCount())
require.Equal(t, netmap.UnspecifiedClause, sV2.GetClause())
require.Empty(t, sV2.GetAttribute())
require.Empty(t, sV2.GetName())
require.Empty(t, sV2.GetFilter())
})
s.SetFilterName(fName)
require.Equal(t, fName, s.m.GetFilter())
}

85
netmap/subnet.go
View file

@ -1,85 +0,0 @@
package netmap
import (
"errors"
"fmt"
"github.com/nspcc-dev/neofs-api-go/v2/netmap"
"github.com/nspcc-dev/neofs-api-go/v2/refs"
subnetid "github.com/nspcc-dev/neofs-sdk-go/subnet/id"
)
// EnterSubnet writes to NodeInfo the intention to enter the subnet. Must not be called on nil.
// Zero NodeInfo belongs to zero subnet.
func (i *NodeInfo) EnterSubnet(id subnetid.ID) {
i.changeSubnet(id, true)
}
// ExitSubnet writes to NodeInfo the intention to exit subnet. Must not be called on nil.
func (i *NodeInfo) ExitSubnet(id subnetid.ID) {
i.changeSubnet(id, false)
}
func (i *NodeInfo) changeSubnet(id subnetid.ID, isMember bool) {
var (
idv2 refs.SubnetID
info netmap.NodeSubnetInfo
)
id.WriteToV2(&idv2)
info.SetID(&idv2)
info.SetEntryFlag(isMember)
if i.m == nil {
i.m = new(netmap.NodeInfo)
}
netmap.WriteSubnetInfo(i.m, info)
}
// ErrRemoveSubnet is returned when a node needs to leave the subnet.
var ErrRemoveSubnet = netmap.ErrRemoveSubnet
// IterateSubnets iterates over all subnets the node belongs to and passes the IDs to f.
// Must not be called on nil. Handler must not be nil.
//
// If f returns ErrRemoveSubnet, then removes subnet entry. Note that this leads to an instant mutation of NodeInfo.
// Breaks on any other non-nil error and returns it.
//
// Returns an error if subnet incorrectly enabled/disabled.
// Returns an error if the node is not included in any subnet by the end of the loop.
func (i *NodeInfo) IterateSubnets(f func(subnetid.ID) error) error {
var id subnetid.ID
return netmap.IterateSubnets(i.m, func(idv2 refs.SubnetID) error {
err := id.ReadFromV2(idv2)
if err != nil {
return fmt.Errorf("invalid subnet: %w", err)
}
err = f(id)
if errors.Is(err, ErrRemoveSubnet) {
return netmap.ErrRemoveSubnet
}
return err
})
}
var errAbortSubnetIter = errors.New("abort subnet iterator")
// BelongsToSubnet checks if node belongs to subnet by ID.
//
// Function is NPE-safe: nil NodeInfo always belongs to zero subnet only.
func BelongsToSubnet(node *NodeInfo, id subnetid.ID) bool {
err := node.IterateSubnets(func(id_ subnetid.ID) error {
if id.Equals(id_) {
return errAbortSubnetIter
}
return nil
})
return errors.Is(err, errAbortSubnetIter)
}

20
netmap/subnet_test.go
View file

@ -92,22 +92,22 @@ func TestEnterSubnet(t *testing.T) {
node netmap.NodeInfo
)
require.True(t, netmap.BelongsToSubnet(&node, id))
require.True(t, netmap.BelongsToSubnet(node, id))
node.EnterSubnet(id)
require.True(t, netmap.BelongsToSubnet(&node, id))
require.True(t, netmap.BelongsToSubnet(node, id))
node.ExitSubnet(id)
require.False(t, netmap.BelongsToSubnet(&node, id))
require.False(t, netmap.BelongsToSubnet(node, id))
id.SetNumeric(10)
node.EnterSubnet(id)
require.True(t, netmap.BelongsToSubnet(&node, id))
require.False(t, netmap.BelongsToSubnet(&node, subnetid.ID{}))
require.True(t, netmap.BelongsToSubnet(node, id))
require.False(t, netmap.BelongsToSubnet(node, subnetid.ID{}))
node.ExitSubnet(id)
require.False(t, netmap.BelongsToSubnet(&node, id))
require.False(t, netmap.BelongsToSubnet(&node, subnetid.ID{}))
require.False(t, netmap.BelongsToSubnet(node, id))
require.False(t, netmap.BelongsToSubnet(node, subnetid.ID{}))
}
func TestBelongsToSubnet(t *testing.T) {
@ -120,7 +120,7 @@ func TestBelongsToSubnet(t *testing.T) {
node.EnterSubnet(id)
require.True(t, netmap.BelongsToSubnet(&node, idZero))
require.True(t, netmap.BelongsToSubnet(&node, id))
require.False(t, netmap.BelongsToSubnet(&node, idMiss))
require.True(t, netmap.BelongsToSubnet(node, idZero))
require.True(t, netmap.BelongsToSubnet(node, id))
require.False(t, netmap.BelongsToSubnet(node, idMiss))
}

98
netmap/test/generate.go
View file

@ -1,92 +1,70 @@
package netmaptest
import "github.com/nspcc-dev/neofs-sdk-go/netmap"
func filter(withInner bool) *netmap.Filter {
x := netmap.NewFilter()
import (
"github.com/nspcc-dev/neofs-sdk-go/netmap"
subnetidtest "github.com/nspcc-dev/neofs-sdk-go/subnet/id/test"
)
func filter(withInner bool) (x netmap.Filter) {
x.SetName("name")
x.SetKey("key")
x.SetValue("value")
x.SetOperation(netmap.OpAND)
if withInner {
x.SetInnerFilters(*filter(false), *filter(false))
x.LogicalOR(filter(false), filter(false))
} else {
x.NumericGE("epoch", 13)
}
return x
}
// Filter returns random netmap.Filter.
func Filter() *netmap.Filter {
func Filter() netmap.Filter {
return filter(true)
}
// Replica returns random netmap.Replica.
func Replica() *netmap.Replica {
x := netmap.NewReplica()
// Replica returns random netmap.ReplicaDescriptor.
func Replica() (x netmap.ReplicaDescriptor) {
x.SetAmount(666)
x.SetSelectorName("selector")
x.SetCount(666)
x.SetSelector("selector")
return x
return
}
// Selector returns random netmap.Selector.
func Selector() *netmap.Selector {
x := netmap.NewSelector()
x.SetCount(11)
func Selector() (x netmap.Selector) {
x.SetNodeAmount(11)
x.SetName("name")
x.SetFilter("filter")
x.SetAttribute("attribute")
x.SetClause(netmap.ClauseDistinct)
x.SetFilterName("filter")
x.SelectByBucketAttribute("attribute")
x.SelectDistinct()
return x
return
}
// PlacementPolicy returns random netmap.PlacementPolicy.
func PlacementPolicy() *netmap.PlacementPolicy {
x := netmap.NewPlacementPolicy()
func PlacementPolicy() (p netmap.PlacementPolicy) {
p.SetContainerBackupFactor(9)
p.AddFilters(Filter(), Filter())
p.AddReplicas(Replica(), Replica())
p.AddSelectors(Selector(), Selector())
p.RestrictSubnet(subnetidtest.ID())
x.SetContainerBackupFactor(9)
x.SetFilters(*Filter(), *Filter())
x.SetReplicas(*Replica(), *Replica())
x.SetSelectors(*Selector(), *Selector())
return x
}
// NetworkParameter returns random netmap.NetworkParameter.
func NetworkParameter() *netmap.NetworkParameter {
x := netmap.NewNetworkParameter()
x.SetKey([]byte("key"))
x.SetValue([]byte("value"))
return x
}
// NetworkConfig returns random netmap.NetworkConfig.
func NetworkConfig() *netmap.NetworkConfig {
x := netmap.NewNetworkConfig()
x.SetParameters(
*NetworkParameter(),
*NetworkParameter(),
)
return x
return
}
// NetworkInfo returns random netmap.NetworkInfo.
func NetworkInfo() *netmap.NetworkInfo {
x := netmap.NewNetworkInfo()
func NetworkInfo() (x netmap.NetworkInfo) {
x.SetCurrentEpoch(21)
x.SetMagicNumber(32)
x.SetMsPerBlock(43)
x.SetNetworkConfig(NetworkConfig())
x.SetAuditFee(1)
x.SetStoragePrice(2)
x.SetContainerFee(3)
x.SetEigenTrustAlpha(0.4)
x.SetEigenTrustIterationAmount(5)
x.SetEpochDuration(6)
x.SetIRCandidateFee(7)
x.SetMaxObjectSize(8)
x.SetWithdrawalFee(9)
return x
return
}

19
policy/doc.go
View file

@ -1,19 +0,0 @@
// Package policy provides facilities for creating policy from SQL-like language.
// ANTLRv4 grammar is provided in `parser/Query.g4` and `parser/QueryLexer.g4`.
//
// Current limitations:
// 1. Filters must be defined before they are used.
// This requirement may be relaxed in future.
// 2. Keywords are key-sensitive. This can be changed if necessary
// https://github.com/antlr/antlr4/blob/master/doc/case-insensitive-lexing.md .
//
// Example query:
// REP 1 IN SPB
// REP 2 IN Americas
// CBF 4
// SELECT 1 IN City FROM SPBSSD AS SPB
// SELECT 2 IN SAME City FROM Americas AS Americas
// FILTER SSD EQ true AS IsSSD
// FILTER @IsSSD AND Country EQ "RU" AND City EQ "St.Petersburg" AS SPBSSD
// FILTER 'Continent' EQ 'North America' OR Continent EQ 'South America' AS Americas
package policy

141
policy/encode.go
View file

@ -1,141 +0,0 @@
package policy
import (
"fmt"
"strconv"
"strings"
"github.com/nspcc-dev/neofs-sdk-go/netmap"
)
// Encode parses data of PlacementPolicy to a string.
func Encode(p *netmap.PlacementPolicy) []string {
if p == nil {
return nil
}
var (
replicas = p.Replicas()
selectors = p.Selectors()
filters = p.Filters()
)
// 1 for container backup factor
result := make([]string, 0, len(replicas)+len(selectors)+len(filters)+1)
// first print replicas
encodeReplicas(replicas, &result)
// then backup factor
if backupFactor := p.ContainerBackupFactor(); backupFactor != 0 {
result = append(result, fmt.Sprintf("CBF %d", backupFactor))
}
// then selectors
encodeSelectors(selectors, &result)
// then filters
encodeFilters(filters, &result)
return result
}
func encodeReplicas(replicas []netmap.Replica, dst *[]string) {
builder := new(strings.Builder)
for _, replica := range replicas {
builder.WriteString("REP ")
builder.WriteString(strconv.FormatUint(uint64(replica.Count()), 10))
if s := replica.Selector(); s != "" {
builder.WriteString(" IN ")
builder.WriteString(s)
}
*dst = append(*dst, builder.String())
builder.Reset()
}
}
func encodeSelectors(selectors []netmap.Selector, dst *[]string) {
builder := new(strings.Builder)
for _, selector := range selectors {
builder.WriteString("SELECT ")
builder.WriteString(strconv.FormatUint(uint64(selector.Count()), 10))
if a := selector.Attribute(); a != "" {
builder.WriteString(" IN")
switch selector.Clause() {
case netmap.ClauseSame:
builder.WriteString(" SAME ")
case netmap.ClauseDistinct:
builder.WriteString(" DISTINCT ")
default:
builder.WriteString(" ")
}
builder.WriteString(a)
}
if f := selector.Filter(); f != "" {
builder.WriteString(" FROM ")
builder.WriteString(f)
}
if n := selector.Name(); n != "" {
builder.WriteString(" AS ")
builder.WriteString(n)
}
*dst = append(*dst, builder.String())
builder.Reset()
}
}
func encodeFilters(filters []netmap.Filter, dst *[]string) {
builder := new(strings.Builder)
for _, filter := range filters {
builder.WriteString("FILTER ")
builder.WriteString(encodeFilter(&filter))
*dst = append(*dst, builder.String())
builder.Reset()
}
}
func encodeFilter(filter *netmap.Filter) string {
builder := new(strings.Builder)
unspecified := filter.Operation() == 0
if k := filter.Key(); k != "" {
builder.WriteString(k)
builder.WriteString(" ")
builder.WriteString(filter.Operation().String())
builder.WriteString(" ")
builder.WriteString(filter.Value())
} else if n := filter.Name(); unspecified && n != "" {
builder.WriteString("@")
builder.WriteString(n)
}
for i, subfilter := range filter.InnerFilters() {
if i != 0 {
builder.WriteString(" ")
builder.WriteString(filter.Operation().String())
builder.WriteString(" ")
}
builder.WriteString(encodeFilter(&subfilter))
}
if n := filter.Name(); n != "" && !unspecified {
builder.WriteString(" AS ")
builder.WriteString(n)
}
return builder.String()
}

34
policy/encode_test.go
View file

@ -1,34 +0,0 @@
package policy_test
import (
"strings"
"testing"
"github.com/nspcc-dev/neofs-sdk-go/policy"
"github.com/stretchr/testify/require"
)
func TestEncode(t *testing.T) {
testCases := []string{
`REP 1 IN X
CBF 1
SELECT 2 IN SAME Location FROM * AS X`,
`REP 1
SELECT 2 IN City FROM Good
FILTER Country EQ RU AS FromRU
FILTER @FromRU AND Rating GT 7 AS Good`,
`REP 7 IN SPB
SELECT 1 IN City FROM SPBSSD AS SPB
FILTER City EQ SPB AND SSD EQ true OR City EQ SPB AND Rating GE 5 AS SPBSSD`,
}
for _, testCase := range testCases {
q, err := policy.Parse(testCase)
require.NoError(t, err)
got := policy.Encode(q)
require.Equal(t, testCase, strings.Join(got, "\n"))
}
}

225
policy/json.go
View file

@ -1,225 +0,0 @@
package policy
import (
"encoding/json"
"fmt"
"strings"
"github.com/nspcc-dev/neofs-sdk-go/netmap"
)
type (
filter struct {
Name string `json:"name,omitempty"`
Key string `json:"key,omitempty"`
Op string `json:"op,omitempty"`
Value string `json:"value,omitempty"`
Filters []filter `json:"filters,omitempty"`
}
replica struct {
Count uint32 `json:"count"`
Selector string `json:"selector,omitempty"`
}
selector struct {
Count uint32 `json:"count"`
Attribute string `json:"attribute"`
Filter string `json:"filter,omitempty"`
Name string `json:"name,omitempty"`
Clause string `json:"clause,omitempty"`
}
placement struct {
Replicas []replica `json:"replicas"`
CBF uint32 `json:"container_backup_factor,omitempty"`
Selectors []selector `json:"selectors,omitempty"`
Filters []filter `json:"filters,omitempty"`
}
)
// ToJSON converts placement policy to JSON.
func ToJSON(np *netmap.PlacementPolicy) ([]byte, error) {
p := new(placement)
p.CBF = np.ContainerBackupFactor()
p.Filters = make([]filter, len(np.Filters()))
for i, f := range np.Filters() {
p.Filters[i].fromNetmap(&f)
}
p.Selectors = make([]selector, len(np.Selectors()))
for i, s := range np.Selectors() {
p.Selectors[i].fromNetmap(&s)
}
p.Replicas = make([]replica, len(np.Replicas()))
for i, r := range np.Replicas() {
p.Replicas[i].fromNetmap(&r)
}
return json.Marshal(p)
}
// FromJSON creates placement policy from JSON.
func FromJSON(data []byte) (*netmap.PlacementPolicy, error) {
p := new(placement)
if err := json.Unmarshal(data, p); err != nil {
return nil, err
}
rs := make([]netmap.Replica, len(p.Replicas))
for i := range p.Replicas {
rs[i] = *p.Replicas[i].toNetmap()
}
var fs []netmap.Filter
if len(p.Filters) != 0 {
fs = make([]netmap.Filter, len(p.Filters))
for i := range p.Filters {
f, err := p.Filters[i].toNetmap()
if err != nil {
return nil, err
}
fs[i] = *f
}
}
var ss []netmap.Selector
if len(p.Selectors) != 0 {
ss = make([]netmap.Selector, len(p.Selectors))
for i := range p.Selectors {
s, err := p.Selectors[i].toNetmap()
if err != nil {
return nil, err
}
ss[i] = *s
}
}
pp := new(netmap.PlacementPolicy)
pp.SetReplicas(rs...)
pp.SetContainerBackupFactor(p.CBF)
pp.SetFilters(fs...)
pp.SetSelectors(ss...)
return pp, nil
}
func (r *replica) toNetmap() *netmap.Replica {
nr := new(netmap.Replica)
nr.SetCount(r.Count)
nr.SetSelector(r.Selector)
return nr
}
func (r *replica) fromNetmap(nr *netmap.Replica) {
r.Count = nr.Count()
r.Selector = nr.Selector()
}
func (f *filter) toNetmap() (*netmap.Filter, error) {
var op netmap.Operation
switch strings.ToUpper(f.Op) {
case "EQ":
op = netmap.OpEQ
case "NE":
op = netmap.OpNE
case "GT":
op = netmap.OpGT
case "GE":
op = netmap.OpGE
case "LT":
op = netmap.OpLT
case "LE":
op = netmap.OpLE
case "AND":
op = netmap.OpAND
case "OR":
op = netmap.OpOR
case "":
default:
return nil, fmt.Errorf("%w: '%s'", ErrUnknownOp, f.Op)
}
var fs []netmap.Filter
if len(f.Filters) != 0 {
fs = make([]netmap.Filter, len(f.Filters))
for i := range f.Filters {
var err error
fsp, err := f.Filters[i].toNetmap()
if err != nil {
return nil, err
}
fs[i] = *fsp
}
}
nf := new(netmap.Filter)
nf.SetInnerFilters(fs...)
nf.SetOperation(op)
nf.SetName(f.Name)
nf.SetValue(f.Value)
nf.SetKey(f.Key)
return nf, nil
}
func (f *filter) fromNetmap(nf *netmap.Filter) {
f.Name = nf.Name()
f.Key = nf.Key()
f.Value = nf.Value()
switch nf.Operation() {
case netmap.OpEQ:
f.Op = "EQ"
case netmap.OpNE:
f.Op = "NE"
case netmap.OpGT:
f.Op = "GT"
case netmap.OpGE:
f.Op = "GE"
case netmap.OpLT:
f.Op = "LT"
case netmap.OpLE:
f.Op = "LE"
case netmap.OpAND:
f.Op = "AND"
case netmap.OpOR:
f.Op = "OR"
default:
// do nothing
}
if nf.InnerFilters() != nil {
f.Filters = make([]filter, len(nf.InnerFilters()))
for i, sf := range nf.InnerFilters() {
f.Filters[i].fromNetmap(&sf)
}
}
}
func (s *selector) toNetmap() (*netmap.Selector, error) {
var c netmap.Clause
switch strings.ToUpper(s.Clause) {
case "SAME":
c = netmap.ClauseSame
case "DISTINCT":
c = netmap.ClauseDistinct
case "":
default:
return nil, fmt.Errorf("%w: '%s'", ErrUnknownClause, s.Clause)
}
ns := new(netmap.Selector)
ns.SetName(s.Name)
ns.SetAttribute(s.Attribute)
ns.SetCount(s.Count)
ns.SetClause(c)
ns.SetFilter(s.Filter)
return ns, nil
}
func (s *selector) fromNetmap(ns *netmap.Selector) {
s.Name = ns.Name()
s.Filter = ns.Filter()
s.Count = ns.Count()
s.Attribute = ns.Attribute()
switch ns.Clause() {
case netmap.ClauseSame:
s.Clause = "same"
case netmap.ClauseDistinct:
s.Clause = "distinct"
default:
// do nothing
}
s.Name = ns.Name()
}

82
policy/json_test.go
View file

@ -1,82 +0,0 @@
package policy
import (
"testing"
"github.com/nspcc-dev/neofs-sdk-go/netmap"
"github.com/stretchr/testify/require"
)
func TestToJSON(t *testing.T) {
check := func(t *testing.T, p *netmap.PlacementPolicy, json string) {
data, err := ToJSON(p)
require.NoError(t, err)
require.JSONEq(t, json, string(data))
np, err := FromJSON(data)
require.NoError(t, err)
require.Equal(t, p, np)
}
t.Run("SimpleREP", func(t *testing.T) {
p := new(netmap.PlacementPolicy)
p.SetReplicas(newReplica("", 3))
check(t, p, `{"replicas":[{"count":3}]}`)
})
t.Run("REPWithCBF", func(t *testing.T) {
p := new(netmap.PlacementPolicy)
p.SetReplicas(newReplica("", 3))
p.SetContainerBackupFactor(3)
check(t, p, `{"replicas":[{"count":3}],"container_backup_factor":3}`)
})
t.Run("REPFromSelector", func(t *testing.T) {
p := new(netmap.PlacementPolicy)
p.SetReplicas(newReplica("Nodes", 3))
p.SetContainerBackupFactor(3)
p.SetSelectors(
newSelector(1, netmap.ClauseDistinct, "City", "", "Nodes"))
check(t, p, `{
"replicas":[{"count":3,"selector":"Nodes"}],
"container_backup_factor":3,
"selectors": [{
"name":"Nodes",
"attribute":"City",
"clause":"distinct",
"count":1
}]}`)
})
t.Run("FilterOps", func(t *testing.T) {
p := new(netmap.PlacementPolicy)
p.SetReplicas(newReplica("Nodes", 3))
p.SetContainerBackupFactor(3)
p.SetSelectors(
newSelector(1, netmap.ClauseSame, "City", "Good", "Nodes"))
p.SetFilters(
newFilter("GoodRating", "Rating", "5", netmap.OpGE),
newFilter("Good", "", "", netmap.OpOR,
newFilter("GoodRating", "", "", 0),
newFilter("", "Attr1", "Val1", netmap.OpEQ),
newFilter("", "Attr2", "Val2", netmap.OpNE),
newFilter("", "", "", netmap.OpAND,
newFilter("", "Attr4", "2", netmap.OpLT),
newFilter("", "Attr5", "3", netmap.OpLE)),
newFilter("", "Attr3", "1", netmap.OpGT)),
)
check(t, p, `{
"replicas":[{"count":3,"selector":"Nodes"}],
"container_backup_factor":3,
"selectors": [{"name":"Nodes","attribute":"City","clause":"same","count":1,"filter":"Good"}],
"filters": [
{"name":"GoodRating","key":"Rating","op":"GE","value":"5"},
{"name":"Good","op":"OR","filters":[
{"name":"GoodRating"},
{"key":"Attr1","op":"EQ","value":"Val1"},
{"key":"Attr2","op":"NE","value":"Val2"},
{"op":"AND","filters":[
{"key":"Attr4","op":"LT","value":"2"},
{"key":"Attr5","op":"LE","value":"3"}
]},
{"key":"Attr3","op":"GT","value":"1"}
]}
]}`)
})
}

311
policy/query.go
View file

@ -1,311 +0,0 @@
package policy
import (
"errors"
"fmt"
"strconv"
"strings"
"github.com/antlr/antlr4/runtime/Go/antlr"
"github.com/nspcc-dev/neofs-sdk-go/netmap"
"github.com/nspcc-dev/neofs-sdk-go/policy/parser"
)
var (
// ErrInvalidNumber is returned when a value of SELECT is 0.
ErrInvalidNumber = errors.New("policy: expected positive integer")
// ErrUnknownClause is returned when a statement(clause) in a query is unknown.
ErrUnknownClause = errors.New("policy: unknown clause")
// ErrUnknownOp is returned when an operation in a query is unknown.
ErrUnknownOp = errors.New("policy: unknown operation")
// ErrUnknownFilter is returned when a value of FROM in a query is unknown.
ErrUnknownFilter = errors.New("policy: filter not found")
// ErrUnknownSelector is returned when a value of IN is unknown.
ErrUnknownSelector = errors.New("policy: selector not found")
// ErrSyntaxError is returned for errors found by ANTLR parser.
ErrSyntaxError = errors.New("policy: syntax error")
)
type policyVisitor struct {
errors []error
parser.BaseQueryVisitor
antlr.DefaultErrorListener
}
// Parse parses s into a placement policy.
func Parse(s string) (*netmap.PlacementPolicy, error) {
return parse(s)
}
func newPolicyVisitor() *policyVisitor {
return &policyVisitor{}
}
func parse(s string) (*netmap.PlacementPolicy, error) {
input := antlr.NewInputStream(s)
lexer := parser.NewQueryLexer(input)
stream := antlr.NewCommonTokenStream(lexer, 0)
p := parser.NewQuery(stream)
p.BuildParseTrees = true
v := newPolicyVisitor()
p.RemoveErrorListeners()
p.AddErrorListener(v)
pl := p.Policy().Accept(v)
if len(v.errors) != 0 {
return nil, v.errors[0]
}
if err := validatePolicy(pl.(*netmap.PlacementPolicy)); err != nil {
return nil, err
}
return pl.(*netmap.PlacementPolicy), nil
}
func (p *policyVisitor) SyntaxError(recognizer antlr.Recognizer, offendingSymbol interface{}, line, column int, msg string, e antlr.RecognitionException) {
p.reportError(fmt.Errorf("%w: line %d:%d %s", ErrSyntaxError, line, column, msg))
}
func (p *policyVisitor) reportError(err error) interface{} {
p.errors = append(p.errors, err)
return nil
}
// VisitPolicy implements parser.QueryVisitor interface.
func (p *policyVisitor) VisitPolicy(ctx *parser.PolicyContext) interface{} {
if len(p.errors) != 0 {
return nil
}
pl := new(netmap.PlacementPolicy)
repStmts := ctx.AllRepStmt()
rs := make([]netmap.Replica, 0, len(repStmts))
for _, r := range repStmts {
res, ok := r.Accept(p).(*netmap.Replica)
if !ok {
return nil
}
rs = append(rs, *res)
}
pl.SetReplicas(rs...)
if cbfStmt := ctx.CbfStmt(); cbfStmt != nil {
cbf, ok := cbfStmt.(*parser.CbfStmtContext).Accept(p).(uint32)
if !ok {
return nil
}
pl.SetContainerBackupFactor(cbf)
}
selStmts := ctx.AllSelectStmt()
ss := make([]netmap.Selector, 0, len(selStmts))
for _, s := range selStmts {
res, ok := s.Accept(p).(*netmap.Selector)
if !ok {
return nil
}
ss = append(ss, *res)
}
pl.SetSelectors(ss...)
filtStmts := ctx.AllFilterStmt()
fs := make([]netmap.Filter, 0, len(filtStmts))
for _, f := range filtStmts {
fs = append(fs, *f.Accept(p).(*netmap.Filter))
}
pl.SetFilters(fs...)
return pl
}
func (p *policyVisitor) VisitCbfStmt(ctx *parser.CbfStmtContext) interface{} {
cbf, err := strconv.ParseUint(ctx.GetBackupFactor().GetText(), 10, 32)
if err != nil {
return p.reportError(ErrInvalidNumber)
}
return uint32(cbf)
}
// VisitRepStmt implements parser.QueryVisitor interface.
func (p *policyVisitor) VisitRepStmt(ctx *parser.RepStmtContext) interface{} {
num, err := strconv.ParseUint(ctx.GetCount().GetText(), 10, 32)
if err != nil {
return p.reportError(ErrInvalidNumber)
}
rs := new(netmap.Replica)
rs.SetCount(uint32(num))
if sel := ctx.GetSelector(); sel != nil {
rs.SetSelector(sel.GetText())
}
return rs
}
// VisitSelectStmt implements parser.QueryVisitor interface.
func (p *policyVisitor) VisitSelectStmt(ctx *parser.SelectStmtContext) interface{} {
res, err := strconv.ParseUint(ctx.GetCount().GetText(), 10, 32)
if err != nil {
return p.reportError(ErrInvalidNumber)
}
s := new(netmap.Selector)
s.SetCount(uint32(res))
if clStmt := ctx.Clause(); clStmt != nil {
s.SetClause(clauseFromString(clStmt.GetText()))
}
if bStmt := ctx.GetBucket(); bStmt != nil {
s.SetAttribute(ctx.GetBucket().GetText())
}
s.SetFilter(ctx.GetFilter().GetText()) // either ident or wildcard
if ctx.AS() != nil {
s.SetName(ctx.GetName().GetText())
}
return s
}
// VisitFilterStmt implements parser.QueryVisitor interface.
func (p *policyVisitor) VisitFilterStmt(ctx *parser.FilterStmtContext) interface{} {
f := p.VisitFilterExpr(ctx.GetExpr().(*parser.FilterExprContext)).(*netmap.Filter)
f.SetName(ctx.GetName().GetText())
return f
}
func (p *policyVisitor) VisitFilterExpr(ctx *parser.FilterExprContext) interface{} {
if eCtx := ctx.Expr(); eCtx != nil {
return eCtx.Accept(p)
}
if inner := ctx.GetInner(); inner != nil {
return inner.Accept(p)
}
f := new(netmap.Filter)
op := operationFromString(ctx.GetOp().GetText())
f.SetOperation(op)
f1 := *ctx.GetF1().Accept(p).(*netmap.Filter)
f2 := *ctx.GetF2().Accept(p).(*netmap.Filter)
// Consider f1=(.. AND ..) AND f2. This can be merged because our AND operation
// is of arbitrary arity. ANTLR generates left-associative parse-tree by default.
if f1.Operation() == op {
f.SetInnerFilters(append(f1.InnerFilters(), f2)...)
return f
}
f.SetInnerFilters(f1, f2)
return f
}
// VisitFilterKey implements parser.QueryVisitor interface.
func (p *policyVisitor) VisitFilterKey(ctx *parser.FilterKeyContext) interface{} {
if id := ctx.Ident(); id != nil {
return id.GetText()
}
str := ctx.STRING().GetText()
return str[1 : len(str)-1]
}
func (p *policyVisitor) VisitFilterValue(ctx *parser.FilterValueContext) interface{} {
if id := ctx.Ident(); id != nil {
return id.GetText()
}
if num := ctx.Number(); num != nil {
return num.GetText()
}
str := ctx.STRING().GetText()
return str[1 : len(str)-1]
}
// VisitExpr implements parser.QueryVisitor interface.
func (p *policyVisitor) VisitExpr(ctx *parser.ExprContext) interface{} {
f := new(netmap.Filter)
if flt := ctx.GetFilter(); flt != nil {
f.SetName(flt.GetText())
return f
}
key := ctx.GetKey().Accept(p)
opStr := ctx.SIMPLE_OP().GetText()
value := ctx.GetValue().Accept(p)
f.SetKey(key.(string))
f.SetOperation(operationFromString(opStr))
f.SetValue(value.(string))
return f
}
// validatePolicy checks high-level constraints such as filter link in SELECT
// being actually defined in FILTER section.
func validatePolicy(p *netmap.PlacementPolicy) error {
seenFilters := map[string]bool{}
for _, f := range p.Filters() {
seenFilters[f.Name()] = true
}
seenSelectors := map[string]bool{}
for _, s := range p.Selectors() {
if flt := s.Filter(); flt != netmap.MainFilterName && !seenFilters[flt] {
return fmt.Errorf("%w: '%s'", ErrUnknownFilter, flt)
}
seenSelectors[s.Name()] = true
}
for _, r := range p.Replicas() {
if sel := r.Selector(); sel != "" && !seenSelectors[sel] {
return fmt.Errorf("%w: '%s'", ErrUnknownSelector, sel)
}
}
return nil
}
func clauseFromString(s string) netmap.Clause {
switch strings.ToUpper(s) {
case "SAME":
return netmap.ClauseSame
case "DISTINCT":
return netmap.ClauseDistinct
default:
// Such errors should be handled by ANTLR code thus this panic.
panic(fmt.Errorf("BUG: invalid clause: %s", s))
}
}
func operationFromString(op string) netmap.Operation {
switch strings.ToUpper(op) {
case "AND":
return netmap.OpAND
case "OR":
return netmap.OpOR
case "EQ":
return netmap.OpEQ
case "NE":
return netmap.OpNE
case "GE":
return netmap.OpGE
case "GT":
return netmap.OpGT
case "LE":
return netmap.OpLE
case "LT":
return netmap.OpLT
default:
// Such errors should be handled by ANTLR code thus this panic.
panic(fmt.Errorf("BUG: invalid operation: %s", op))
}
}

324
policy/query_test.go
View file

@ -1,324 +0,0 @@
package policy
import (
"errors"
"fmt"
"math"
"testing"
"github.com/nspcc-dev/neofs-sdk-go/netmap"
"github.com/stretchr/testify/require"
)
func TestSimple(t *testing.T) {
q := `REP 3`
expected := new(netmap.PlacementPolicy)
expected.SetFilters([]netmap.Filter{}...)
expected.SetSelectors([]netmap.Selector{}...)
expected.SetReplicas(newReplica("", 3))
r, err := Parse(q)
require.NoError(t, err)
require.EqualValues(t, expected, r)
}
func TestSimpleWithHRWB(t *testing.T) {
q := `REP 3 CBF 4`
expected := new(netmap.PlacementPolicy)
expected.SetFilters([]netmap.Filter{}...)
expected.SetSelectors([]netmap.Selector{}...)
expected.SetReplicas(newReplica("", 3))
expected.SetContainerBackupFactor(4)
r, err := Parse(q)
require.NoError(t, err)
require.EqualValues(t, expected, r)
}
func TestFromSelect(t *testing.T) {
q := `REP 1 IN SPB
SELECT 1 IN City FROM * AS SPB`
expected := new(netmap.PlacementPolicy)
expected.SetFilters([]netmap.Filter{}...)
expected.SetSelectors(newSelector(1, netmap.ClauseUnspecified, "City", "*", "SPB"))
expected.SetReplicas(newReplica("SPB", 1))
r, err := Parse(q)
require.NoError(t, err)
require.EqualValues(t, expected, r)
}
// https://github.com/nspcc-dev/neofs-node/issues/46
func TestFromSelectNoAttribute(t *testing.T) {
t.Run("Simple", func(t *testing.T) {
q := `REP 2
SELECT 6 FROM *`
expected := new(netmap.PlacementPolicy)
expected.SetFilters([]netmap.Filter{}...)
expected.SetSelectors(newSelector(6, netmap.ClauseUnspecified, "", "*", ""))
expected.SetReplicas(newReplica("", 2))
r, err := Parse(q)
require.NoError(t, err)
require.EqualValues(t, expected, r)
})
t.Run("with filter", func(t *testing.T) {
q := `REP 2
SELECT 6 FROM F
FILTER StorageType EQ SSD AS F`
expected := new(netmap.PlacementPolicy)
expected.SetFilters(newFilter("F", "StorageType", "SSD", netmap.OpEQ))
expected.SetSelectors(newSelector(6, netmap.ClauseUnspecified, "", "F", ""))
expected.SetReplicas(newReplica("", 2))
r, err := Parse(q)
require.NoError(t, err)
require.EqualValues(t, expected, r)
})
}
func TestString(t *testing.T) {
qTemplate := `REP 1
SELECT 1 IN City FROM Filt
FILTER Property EQ %s AND Something NE 7 AS Filt`
testCases := []string{
`"double-quoted"`,
`"with ' single"`,
`'single-quoted'`,
`'with " double'`,
}
for _, s := range testCases {
t.Run(s, func(t *testing.T) {
q := fmt.Sprintf(qTemplate, s)
r, err := Parse(q)
require.NoError(t, err)
expected := newFilter("Filt", "", "", netmap.OpAND,
newFilter("", "Property", s[1:len(s)-1], netmap.OpEQ),
newFilter("", "Something", "7", netmap.OpNE))
require.EqualValues(t, []netmap.Filter{expected}, r.Filters())
})
}
}
func TestFromSelectClause(t *testing.T) {
q := `REP 4
SELECT 3 IN Country FROM *
SELECT 2 IN SAME City FROM *
SELECT 1 IN DISTINCT Continent FROM *`
expected := new(netmap.PlacementPolicy)
expected.SetFilters([]netmap.Filter{}...)
expected.SetSelectors(
newSelector(3, netmap.ClauseUnspecified, "Country", "*", ""),
newSelector(2, netmap.ClauseSame, "City", "*", ""),
newSelector(1, netmap.ClauseDistinct, "Continent", "*", ""))
expected.SetReplicas(newReplica("", 4))
r, err := Parse(q)
require.NoError(t, err)
require.EqualValues(t, expected, r)
}
func TestSimpleFilter(t *testing.T) {
q := `REP 1
SELECT 1 IN City FROM Good
FILTER Rating GT 7 AS Good`
expected := new(netmap.PlacementPolicy)
expected.SetReplicas(newReplica("", 1))
expected.SetSelectors(
newSelector(1, netmap.ClauseUnspecified, "City", "Good", ""))
expected.SetFilters(newFilter("Good", "Rating", "7", netmap.OpGT))
r, err := Parse(q)
require.NoError(t, err)
require.EqualValues(t, expected, r)
}
func TestFilterReference(t *testing.T) {
q := `REP 1
SELECT 2 IN City FROM Good
FILTER Country EQ "RU" AS FromRU
FILTER @FromRU AND Rating GT 7 AS Good`
expected := new(netmap.PlacementPolicy)
expected.SetReplicas(newReplica("", 1))
expected.SetSelectors(
newSelector(2, netmap.ClauseUnspecified, "City", "Good", ""))
expected.SetFilters(
newFilter("FromRU", "Country", "RU", netmap.OpEQ),
newFilter("Good", "", "", netmap.OpAND,
newFilter("FromRU", "", "", 0),
newFilter("", "Rating", "7", netmap.OpGT)),
)
r, err := Parse(q)
require.NoError(t, err)
require.EqualValues(t, expected, r)
}
func TestFilterOps(t *testing.T) {
q := `REP 1
SELECT 2 IN City FROM Good
FILTER A GT 1 AND B GE 2 AND C LT 3 AND D LE 4
AND E EQ 5 AND F NE 6 AS Good`
expected := new(netmap.PlacementPolicy)
expected.SetReplicas(newReplica("", 1))
expected.SetSelectors(
newSelector(2, netmap.ClauseUnspecified, "City", "Good", ""))
expected.SetFilters(newFilter("Good", "", "", netmap.OpAND,
newFilter("", "A", "1", netmap.OpGT),
newFilter("", "B", "2", netmap.OpGE),
newFilter("", "C", "3", netmap.OpLT),
newFilter("", "D", "4", netmap.OpLE),
newFilter("", "E", "5", netmap.OpEQ),
newFilter("", "F", "6", netmap.OpNE)))
r, err := Parse(q)
require.NoError(t, err)
require.EqualValues(t, expected, r)
}
func TestWithFilterPrecedence(t *testing.T) {
q := `REP 7 IN SPB
SELECT 1 IN City FROM SPBSSD AS SPB
FILTER City EQ "SPB" AND SSD EQ true OR City EQ "SPB" AND Rating GE 5 AS SPBSSD`
expected := new(netmap.PlacementPolicy)
expected.SetReplicas(newReplica("SPB", 7))
expected.SetSelectors(
newSelector(1, netmap.ClauseUnspecified, "City", "SPBSSD", "SPB"))
expected.SetFilters(
newFilter("SPBSSD", "", "", netmap.OpOR,
newFilter("", "", "", netmap.OpAND,
newFilter("", "City", "SPB", netmap.OpEQ),
newFilter("", "SSD", "true", netmap.OpEQ)),
newFilter("", "", "", netmap.OpAND,
newFilter("", "City", "SPB", netmap.OpEQ),
newFilter("", "Rating", "5", netmap.OpGE))))
r, err := Parse(q)
require.NoError(t, err)
require.EqualValues(t, expected, r)
}
func TestBrackets(t *testing.T) {
q := `REP 7 IN SPB
SELECT 1 IN City FROM SPBSSD AS SPB
FILTER ( City EQ "SPB" OR SSD EQ true ) AND (City EQ "SPB" OR Rating GE 5) AS SPBSSD`
expected := new(netmap.PlacementPolicy)
expected.SetReplicas(newReplica("SPB", 7))
expected.SetSelectors(
newSelector(1, netmap.ClauseUnspecified, "City", "SPBSSD", "SPB"))
expected.SetFilters(
newFilter("SPBSSD", "", "", netmap.OpAND,
newFilter("", "", "", netmap.OpOR,
newFilter("", "City", "SPB", netmap.OpEQ),
newFilter("", "SSD", "true", netmap.OpEQ)),
newFilter("", "", "", netmap.OpOR,
newFilter("", "City", "SPB", netmap.OpEQ),
newFilter("", "Rating", "5", netmap.OpGE))))
r, err := Parse(q)
require.NoError(t, err)
require.EqualValues(t, expected, r)
}
func TestValidation(t *testing.T) {
t.Run("MissingSelector", func(t *testing.T) {
q := `REP 3 IN RU`
_, err := Parse(q)
require.True(t, errors.Is(err, ErrUnknownSelector), "got: %v", err)
})
t.Run("MissingFilter", func(t *testing.T) {
q := `REP 3
SELECT 1 IN City FROM MissingFilter`
_, err := Parse(q)
require.True(t, errors.Is(err, ErrUnknownFilter), "got: %v", err)
})
t.Run("UnknownOp", func(t *testing.T) {
q := `REP 3
SELECT 1 IN City FROM F
FILTER Country KEK RU AS F`
_, err := Parse(q)
require.True(t, errors.Is(err, ErrSyntaxError), "got: %v", err)
})
t.Run("TypoInREP", func(t *testing.T) {
q := `REK 3`
_, err := Parse(q)
require.True(t, errors.Is(err, ErrSyntaxError))
})
t.Run("InvalidFilterName", func(t *testing.T) {
q := `REP 3
SELECT 1 IN City FROM F
FILTER Good AND Country EQ RU AS F
FILTER Rating EQ 5 AS Good`
_, err := Parse(q)
require.Error(t, err)
})
}
// Checks that an error is returned in cases when positive 32-bit integer is expected.
func TestInvalidNumbers(t *testing.T) {
tmpls := []string{
"REP %d",
"REP 1 CBF %d",
"REP 1 SELECT %d FROM *",
}
for i := range tmpls {
zero := fmt.Sprintf(tmpls[i], 0)
t.Run(zero, func(t *testing.T) {
_, err := Parse(zero)
require.Error(t, err)
})
big := fmt.Sprintf(tmpls[i], int64(math.MaxUint32)+1)
t.Run(big, func(t *testing.T) {
_, err := Parse(big)
require.Error(t, err)
})
}
}
func TestFilterStringSymbols(t *testing.T) {
q := `REP 1 IN S
SELECT 1 FROM F AS S
FILTER "UN-LOCODE" EQ "RU LED" AS F`
expected := new(netmap.PlacementPolicy)
expected.SetReplicas(newReplica("S", 1))
expected.SetSelectors(
newSelector(1, netmap.ClauseUnspecified, "", "F", "S"))
expected.SetFilters(newFilter("F", "UN-LOCODE", "RU LED", netmap.OpEQ))
r, err := Parse(q)
require.NoError(t, err)
require.EqualValues(t, expected, r)
}
func newFilter(name, key, value string, op netmap.Operation, sub ...netmap.Filter) (f netmap.Filter) {
f.SetName(name)
f.SetKey(key)
f.SetValue(value)
f.SetOperation(op)
f.SetInnerFilters(sub...)
return f
}
func newReplica(s string, c uint32) (r netmap.Replica) {
r.SetSelector(s)
r.SetCount(c)
return r
}
func newSelector(count uint32, c netmap.Clause, attr, f, name string) (s netmap.Selector) {
s.SetCount(count)
s.SetClause(c)
s.SetAttribute(attr)
s.SetFilter(f)
s.SetName(name)
return s
}

2
pool/pool_test.go
View file

@ -45,7 +45,7 @@ func TestBuildPoolCreateSessionFailed(t *testing.T) {
ctrl := gomock.NewController(t)
ni := &netmap.NodeInfo{}
ni.SetAddresses("addr1", "addr2")
ni.SetNetworkEndpoints("addr1", "addr2")
clientBuilder := func(_ string) (client, error) {
mockClient := NewMockClient(ctrl)