From ca523f1ff10294a82a14f9f3dcb8023d43190562 Mon Sep 17 00:00:00 2001 From: Leonard Lyubich Date: Tue, 7 Jun 2022 11:25:34 +0300 Subject: [PATCH] [#227] netmap: Refactor and document package functionality Signed-off-by: Leonard Lyubich --- client/netmap.go | 35 +- container/container.go | 24 +- container/container_test.go | 4 +- container/test/generate.go | 3 +- netmap/aggregator.go | 2 +- netmap/clause.go | 69 -- netmap/clause_test.go | 43 - netmap/context.go | 85 +- netmap/doc.go | 45 +- netmap/filter.go | 257 ++--- netmap/filter_test.go | 189 +--- netmap/helper_test.go | 84 +- netmap/json_test.go | 18 +- netmap/json_tests/filter_invalid_integer.json | 2 +- netmap/netmap.go | 158 +-- netmap/network_info.go | 596 ++++++++---- netmap/network_info_test.go | 354 +++---- netmap/node_info.go | 841 +++++++++------- netmap/node_info_test.go | 261 +---- netmap/operation.go | 116 --- netmap/operation_test.go | 73 -- {policy => netmap}/parser/Query.g4 | 0 {policy => netmap}/parser/Query.interp | Bin {policy => netmap}/parser/Query.tokens | Bin {policy => netmap}/parser/QueryLexer.g4 | 0 {policy => netmap}/parser/QueryLexer.interp | Bin {policy => netmap}/parser/QueryLexer.tokens | Bin {policy => netmap}/parser/generate.go | 0 .../parser/query_base_listener.go | 0 .../parser/query_base_visitor.go | 0 {policy => netmap}/parser/query_lexer.go | 0 {policy => netmap}/parser/query_listener.go | 0 {policy => netmap}/parser/query_parser.go | 0 {policy => netmap}/parser/query_visitor.go | 0 netmap/policy.go | 921 +++++++++++++++--- netmap/policy_test.go | 180 ++-- netmap/replica.go | 71 -- netmap/replica_test.go | 99 -- netmap/selector.go | 205 +--- netmap/selector_test.go | 203 ++-- netmap/subnet.go | 85 -- netmap/subnet_test.go | 20 +- netmap/test/generate.go | 98 +- policy/doc.go | 19 - policy/encode.go | 141 --- policy/encode_test.go | 34 - policy/json.go | 225 ----- policy/json_test.go | 82 -- policy/query.go | 311 ------ policy/query_test.go | 324 ------ pool/pool_test.go | 2 +- 51 files changed, 2460 insertions(+), 3819 deletions(-) delete mode 100644 netmap/clause.go delete mode 100644 netmap/clause_test.go delete mode 100644 netmap/operation.go delete mode 100644 netmap/operation_test.go rename {policy => netmap}/parser/Query.g4 (100%) rename {policy => netmap}/parser/Query.interp (100%) rename {policy => netmap}/parser/Query.tokens (100%) rename {policy => netmap}/parser/QueryLexer.g4 (100%) rename {policy => netmap}/parser/QueryLexer.interp (100%) rename {policy => netmap}/parser/QueryLexer.tokens (100%) rename {policy => netmap}/parser/generate.go (100%) rename {policy => netmap}/parser/query_base_listener.go (100%) rename {policy => netmap}/parser/query_base_visitor.go (100%) rename {policy => netmap}/parser/query_lexer.go (100%) rename {policy => netmap}/parser/query_listener.go (100%) rename {policy => netmap}/parser/query_parser.go (100%) rename {policy => netmap}/parser/query_visitor.go (100%) delete mode 100644 netmap/replica.go delete mode 100644 netmap/replica_test.go delete mode 100644 netmap/subnet.go delete mode 100644 policy/doc.go delete mode 100644 policy/encode.go delete mode 100644 policy/encode_test.go delete mode 100644 policy/json.go delete mode 100644 policy/json_test.go delete mode 100644 policy/query.go delete mode 100644 policy/query_test.go diff --git a/client/netmap.go b/client/netmap.go index aeb16b2..ca45b66 100644 --- a/client/netmap.go +++ b/client/netmap.go @@ -2,6 +2,8 @@ package client import ( "context" + "errors" + "fmt" v2netmap "github.com/nspcc-dev/neofs-api-go/v2/netmap" rpcapi "github.com/nspcc-dev/neofs-api-go/v2/rpc" @@ -96,7 +98,22 @@ func (c *Client) EndpointInfo(ctx context.Context, prm PrmEndpointInfo) (*ResEnd ver.ReadFromV2(*v2ver) } res.setLatestVersion(&ver) - res.setNodeInfo(netmap.NewNodeInfoFromV2(body.GetNodeInfo())) + + nodeV2 := body.GetNodeInfo() + if nodeV2 == nil { + cc.err = errors.New("missing node info in response body") + return + } + + var node netmap.NodeInfo + + cc.err = node.ReadFromV2(*nodeV2) + if cc.err != nil { + cc.err = fmt.Errorf("invalid node info: %w", cc.err) + return + } + + res.setNodeInfo(&node) } // process call @@ -171,7 +188,21 @@ func (c *Client) NetworkInfo(ctx context.Context, prm PrmNetworkInfo) (*ResNetwo cc.result = func(r responseV2) { resp := r.(*v2netmap.NetworkInfoResponse) - res.setInfo(netmap.NewNetworkInfoFromV2(resp.GetBody().GetNetworkInfo())) + netInfoV2 := resp.GetBody().GetNetworkInfo() + if netInfoV2 == nil { + cc.err = errors.New("missing network info in response body") + return + } + + var netInfo netmap.NetworkInfo + + cc.err = netInfo.ReadFromV2(*netInfoV2) + if cc.err != nil { + cc.err = fmt.Errorf("invalid network info: %w", cc.err) + return + } + + res.setInfo(&netInfo) } // process call diff --git a/container/container.go b/container/container.go index dd65c24..d51d60d 100644 --- a/container/container.go +++ b/container/container.go @@ -5,6 +5,7 @@ import ( "github.com/google/uuid" "github.com/nspcc-dev/neofs-api-go/v2/container" + v2netmap "github.com/nspcc-dev/neofs-api-go/v2/netmap" "github.com/nspcc-dev/neofs-api-go/v2/refs" "github.com/nspcc-dev/neofs-sdk-go/acl" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" @@ -158,11 +159,30 @@ func (c *Container) SetAttributes(v Attributes) { } func (c *Container) PlacementPolicy() *netmap.PlacementPolicy { - return netmap.NewPlacementPolicyFromV2(c.v2.GetPlacementPolicy()) + m := c.v2.GetPlacementPolicy() + if m == nil { + return nil + } + + var p netmap.PlacementPolicy + // FIXME(@cthulhu-rider): #225 handle error + err := p.ReadFromV2(*m) + if err != nil { + panic(err) + } + + return &p } func (c *Container) SetPlacementPolicy(v *netmap.PlacementPolicy) { - c.v2.SetPlacementPolicy(v.ToV2()) + var m *v2netmap.PlacementPolicy + + if v != nil { + m = new(v2netmap.PlacementPolicy) + v.WriteToV2(m) + } + + c.v2.SetPlacementPolicy(m) } // SessionToken returns token of the session within diff --git a/container/container_test.go b/container/container_test.go index a88e329..41a52fc 100644 --- a/container/container_test.go +++ b/container/container_test.go @@ -29,7 +29,7 @@ func TestNewContainer(t *testing.T) { attrs := containertest.Attributes() c.SetAttributes(attrs) - c.SetPlacementPolicy(policy) + c.SetPlacementPolicy(&policy) c.SetNonceUUID(nonce) c.SetOwnerID(ownerID) @@ -39,7 +39,7 @@ func TestNewContainer(t *testing.T) { v2 := c.ToV2() newContainer := container.NewContainerFromV2(v2) - require.EqualValues(t, newContainer.PlacementPolicy(), policy) + require.EqualValues(t, newContainer.PlacementPolicy(), &policy) require.EqualValues(t, newContainer.Attributes(), attrs) require.EqualValues(t, newContainer.BasicACL(), acl.PublicBasicRule) diff --git a/container/test/generate.go b/container/test/generate.go index 6b26a3e..4ee76d4 100644 --- a/container/test/generate.go +++ b/container/test/generate.go @@ -32,7 +32,8 @@ func Container() *container.Container { x.SetAttributes(Attributes()) x.SetOwnerID(usertest.ID()) x.SetBasicACL(123) - x.SetPlacementPolicy(netmaptest.PlacementPolicy()) + p := netmaptest.PlacementPolicy() + x.SetPlacementPolicy(&p) return x } diff --git a/netmap/aggregator.go b/netmap/aggregator.go index e709511..c018e6b 100644 --- a/netmap/aggregator.go +++ b/netmap/aggregator.go @@ -56,7 +56,7 @@ var ( // capacity and price. func newWeightFunc(capNorm, priceNorm normalizer) weightFunc { return func(n NodeInfo) float64 { - return capNorm.Normalize(float64(n.capacity())) * priceNorm.Normalize(float64(n.price())) + return capNorm.Normalize(float64(n.capacity())) * priceNorm.Normalize(float64(n.Price())) } } diff --git a/netmap/clause.go b/netmap/clause.go deleted file mode 100644 index d2b5c40..0000000 --- a/netmap/clause.go +++ /dev/null @@ -1,69 +0,0 @@ -package netmap - -import ( - "github.com/nspcc-dev/neofs-api-go/v2/netmap" -) - -// Clause is an enumeration of selector modifiers -// that shows how the node set will be formed. -type Clause uint32 - -const ( - ClauseUnspecified Clause = iota - - // ClauseSame is a selector modifier to select only nodes having the same value of bucket attribute. - ClauseSame - - // ClauseDistinct is a selector modifier to select nodes having different values of bucket attribute. - ClauseDistinct -) - -// ClauseFromV2 converts v2 Clause to Clause. -func ClauseFromV2(c netmap.Clause) Clause { - switch c { - default: - return ClauseUnspecified - case netmap.Same: - return ClauseSame - case netmap.Distinct: - return ClauseDistinct - } -} - -// ToV2 converts Clause to v2 Clause. -func (c Clause) ToV2() netmap.Clause { - switch c { - default: - return netmap.UnspecifiedClause - case ClauseDistinct: - return netmap.Distinct - case ClauseSame: - return netmap.Same - } -} - -// String returns string representation of Clause. -// -// String mapping: -// * ClauseDistinct: DISTINCT; -// * ClauseSame: SAME; -// * ClauseUnspecified, default: CLAUSE_UNSPECIFIED. -func (c Clause) String() string { - return c.ToV2().String() -} - -// FromString parses Clause from a string representation. -// It is a reverse action to String(). -// -// Returns true if s was parsed successfully. -func (c *Clause) FromString(s string) bool { - var g netmap.Clause - - ok := g.FromString(s) - - if ok { - *c = ClauseFromV2(g) - } - - return ok -} diff --git a/netmap/clause_test.go b/netmap/clause_test.go deleted file mode 100644 index 6ce5e82..0000000 --- a/netmap/clause_test.go +++ /dev/null @@ -1,43 +0,0 @@ -package netmap - -import ( - "testing" - - "github.com/nspcc-dev/neofs-api-go/v2/netmap" - "github.com/stretchr/testify/require" -) - -func TestClauseFromV2(t *testing.T) { - for _, item := range []struct { - c Clause - cV2 netmap.Clause - }{ - { - c: ClauseUnspecified, - cV2: netmap.UnspecifiedClause, - }, - { - c: ClauseSame, - cV2: netmap.Same, - }, - { - c: ClauseDistinct, - cV2: netmap.Distinct, - }, - } { - require.Equal(t, item.c, ClauseFromV2(item.cV2)) - require.Equal(t, item.cV2, item.c.ToV2()) - } -} - -func TestClause_String(t *testing.T) { - toPtr := func(v Clause) *Clause { - return &v - } - - testEnumStrings(t, new(Clause), []enumStringItem{ - {val: toPtr(ClauseDistinct), str: "DISTINCT"}, - {val: toPtr(ClauseSame), str: "SAME"}, - {val: toPtr(ClauseUnspecified), str: "CLAUSE_UNSPECIFIED"}, - }) -} diff --git a/netmap/context.go b/netmap/context.go index e467482..fc3b91c 100644 --- a/netmap/context.go +++ b/netmap/context.go @@ -4,88 +4,85 @@ import ( "errors" "github.com/nspcc-dev/hrw" + "github.com/nspcc-dev/neofs-api-go/v2/netmap" ) -// context contains references to named filters and cached numeric values. +// context of a placement build process. type context struct { - // Netmap is a netmap structure to operate on. - Netmap *Netmap - // Filters stores processed filters. - Filters map[string]*Filter - // Selectors stores processed selectors. - Selectors map[string]*Selector - // Selections stores result of selector processing. - Selections map[string][]nodes + // network map to operate on + netMap NetMap - // numCache stores parsed numeric values. + // cache of processed filters + processedFilters map[string]*netmap.Filter + + // cache of processed selectors + processedSelectors map[string]*netmap.Selector + + // stores results of selector processing + selections map[string][]nodes + + // cache of parsed numeric values numCache map[string]uint64 - // pivot is a seed for HRW. - pivot []byte - // pivotHash is a saved HRW hash of pivot - pivotHash uint64 - // aggregator is returns aggregator determining bucket weight. - // By default it returns mean value from IQR interval. - aggregator func() aggregator - // weightFunc is a weighting function for determining node priority. - // By default in combines favours low price and high capacity. + + hrwSeed []byte + + // hrw.Hash of hrwSeed + hrwSeedHash uint64 + + // weightFunc is a weighting function for determining node priority + // which combines low price and high performance weightFunc weightFunc - // container backup factor is a factor for selector counters that expand - // amount of chosen nodes. + + // container backup factor cbf uint32 } // Various validation errors. var ( - ErrMissingField = errors.New("netmap: nil field") - ErrInvalidFilterName = errors.New("netmap: filter name is invalid") - ErrInvalidNumber = errors.New("netmap: number value expected") - ErrInvalidFilterOp = errors.New("netmap: invalid filter operation") - ErrFilterNotFound = errors.New("netmap: filter not found") - ErrNonEmptyFilters = errors.New("netmap: simple filter must no contain sub-filters") - ErrNotEnoughNodes = errors.New("netmap: not enough nodes to SELECT from") - ErrSelectorNotFound = errors.New("netmap: selector not found") - ErrUnnamedTopFilter = errors.New("netmap: all filters on top level must be named") + errInvalidFilterName = errors.New("filter name is invalid") + errInvalidNumber = errors.New("invalid number") + errInvalidFilterOp = errors.New("invalid filter operation") + errFilterNotFound = errors.New("filter not found") + errNonEmptyFilters = errors.New("simple filter contains sub-filters") + errNotEnoughNodes = errors.New("not enough nodes to SELECT from") + errUnnamedTopFilter = errors.New("unnamed top-level filter") ) -// newContext creates new context. It contains various caches. -// In future it may create hierarchical netmap structure to work with. -func newContext(nm *Netmap) *context { +// newContext returns initialized context. +func newContext(nm NetMap) *context { return &context{ - Netmap: nm, - Filters: make(map[string]*Filter), - Selectors: make(map[string]*Selector), - Selections: make(map[string][]nodes), + netMap: nm, + processedFilters: make(map[string]*netmap.Filter), + processedSelectors: make(map[string]*netmap.Selector), + selections: make(map[string][]nodes), numCache: make(map[string]uint64), - aggregator: newMeanIQRAgg, weightFunc: defaultWeightFunc(nm.nodes), - cbf: defaultCBF, } } func (c *context) setPivot(pivot []byte) { if len(pivot) != 0 { - c.pivot = pivot - c.pivotHash = hrw.Hash(pivot) + c.hrwSeed = pivot + c.hrwSeedHash = hrw.Hash(pivot) } } func (c *context) setCBF(cbf uint32) { if cbf == 0 { - c.cbf = defaultCBF + c.cbf = 3 } else { c.cbf = cbf } } -// defaultWeightFunc returns default weighting function. func defaultWeightFunc(ns nodes) weightFunc { mean := newMeanAgg() min := newMinAgg() for i := range ns { mean.Add(float64(ns[i].capacity())) - min.Add(float64(ns[i].price())) + min.Add(float64(ns[i].Price())) } return newWeightFunc( diff --git a/netmap/doc.go b/netmap/doc.go index a38e985..e03a2b5 100644 --- a/netmap/doc.go +++ b/netmap/doc.go @@ -1,11 +1,42 @@ /* -Package netmap provides routines for working with netmap and placement policy. -Work is done in 4 steps: -1. Create context containing results shared between steps. -2. Processing filters. -3. Processing selectors. -4. Processing replicas. +Package netmap provides functionality for working with information about the +NeoFS network, primarily a layer of storage nodes. + +The package concentrates all the characteristics of NeoFS networks. + +NetMap represents NeoFS network map - one of the main technologies used to +store data in the system. It is composed of information about all storage nodes +(NodeInfo type) in a particular network. NetMap methods allow you to impose +container storage policies (PlacementPolicy type) on a fixed composition of +nodes for selecting nodes corresponding to the placement rules chosen by the +container creator. + +NetworkInfo type is dedicated to descriptive characterization of network state +and settings. + +Instances can be also used to process NeoFS API V2 protocol messages +(see neo.fs.v2.netmap package in https://github.com/nspcc-dev/neofs-api). + +On client side: + import "github.com/nspcc-dev/neofs-api-go/v2/netmap" + + var msg netmap.NodeInfo + msg.WriteToV2(&msg) + + // send msg + +On server side: + // recv msg + + var info netmap.NodeInfo + + err := info.ReadFromV2(msg) + // ... + + // process dec + +Using package types in an application is recommended to potentially work with +different protocol versions with which these types are compatible. -Each step depends only on previous ones. */ package netmap diff --git a/netmap/filter.go b/netmap/filter.go index bd37abd..bd229ca 100644 --- a/netmap/filter.go +++ b/netmap/filter.go @@ -7,77 +7,68 @@ import ( "github.com/nspcc-dev/neofs-api-go/v2/netmap" ) -// Filter represents v2-compatible netmap filter. -type Filter netmap.Filter - -// MainFilterName is a name of the filter +// mainFilterName is a name of the filter // which points to the whole netmap. -const MainFilterName = "*" - -// applyFilter applies named filter to b. -func (c *context) applyFilter(name string, b NodeInfo) bool { - return name == MainFilterName || c.match(c.Filters[name], b) -} +const mainFilterName = "*" // processFilters processes filters and returns error is any of them is invalid. -func (c *context) processFilters(p *PlacementPolicy) error { - filters := p.Filters() - for i := range filters { - if err := c.processFilter(&filters[i], true); err != nil { - return err +func (c *context) processFilters(p PlacementPolicy) error { + for i := range p.filters { + if err := c.processFilter(p.filters[i], true); err != nil { + return fmt.Errorf("process filter #%d (%s): %w", i, p.filters[i].GetName(), err) } } return nil } -func (c *context) processFilter(f *Filter, top bool) error { - if f == nil { - return fmt.Errorf("%w: FILTER", ErrMissingField) +func (c *context) processFilter(f netmap.Filter, top bool) error { + fName := f.GetName() + if fName == mainFilterName { + return fmt.Errorf("%w: '%s' is reserved", errInvalidFilterName, mainFilterName) } - if f.Name() == MainFilterName { - return fmt.Errorf("%w: '*' is reserved", ErrInvalidFilterName) + if top && fName == "" { + return errUnnamedTopFilter } - if top && f.Name() == "" { - return ErrUnnamedTopFilter + if !top && fName != "" && c.processedFilters[fName] == nil { + return errFilterNotFound } - if !top && f.Name() != "" && c.Filters[f.Name()] == nil { - return fmt.Errorf("%w: '%s'", ErrFilterNotFound, f.Name()) - } + inner := f.GetFilters() - switch f.Operation() { - case OpAND, OpOR: - for _, flt := range f.InnerFilters() { - if err := c.processFilter(&flt, false); err != nil { - return err + switch op := f.GetOp(); op { + case netmap.AND, netmap.OR: + for i := range inner { + if err := c.processFilter(inner[i], false); err != nil { + return fmt.Errorf("process inner filter #%d: %w", i, err) } } default: - if len(f.InnerFilters()) != 0 { - return ErrNonEmptyFilters - } else if !top && f.Name() != "" { // named reference + if len(inner) != 0 { + return errNonEmptyFilters + } else if !top && fName != "" { // named reference return nil } - switch f.Operation() { - case OpEQ, OpNE: - case OpGT, OpGE, OpLT, OpLE: - n, err := strconv.ParseUint(f.Value(), 10, 64) + switch op { + case netmap.EQ, netmap.NE: + case netmap.GT, netmap.GE, netmap.LT, netmap.LE: + val := f.GetValue() + n, err := strconv.ParseUint(val, 10, 64) if err != nil { - return fmt.Errorf("%w: '%s'", ErrInvalidNumber, f.Value()) + return fmt.Errorf("%w: '%s'", errInvalidNumber, f.GetValue()) } - c.numCache[f.Value()] = n + c.numCache[val] = n default: - return fmt.Errorf("%w: %s", ErrInvalidFilterOp, f.Operation()) + return fmt.Errorf("%w: %s", errInvalidFilterOp, op) } } if top { - c.Filters[f.Name()] = f + c.processedFilters[fName] = &f } return nil @@ -86,44 +77,46 @@ func (c *context) processFilter(f *Filter, top bool) error { // match matches f against b. It returns no errors because // filter should have been parsed during context creation // and missing node properties are considered as a regular fail. -func (c *context) match(f *Filter, b NodeInfo) bool { - switch f.Operation() { - case OpAND, OpOR: - for _, lf := range f.InnerFilters() { - if lf.Name() != "" { - lf = *c.Filters[lf.Name()] +func (c *context) match(f *netmap.Filter, b NodeInfo) bool { + switch f.GetOp() { + case netmap.AND, netmap.OR: + inner := f.GetFilters() + for i := range inner { + fSub := &inner[i] + if name := inner[i].GetName(); name != "" { + fSub = c.processedFilters[name] } - ok := c.match(&lf, b) - if ok == (f.Operation() == OpOR) { + ok := c.match(fSub, b) + if ok == (f.GetOp() == netmap.OR) { return ok } } - return f.Operation() == OpAND + return f.GetOp() == netmap.AND default: return c.matchKeyValue(f, b) } } -func (c *context) matchKeyValue(f *Filter, b NodeInfo) bool { - switch f.Operation() { - case OpEQ: - return b.attribute(f.Key()) == f.Value() - case OpNE: - return b.attribute(f.Key()) != f.Value() +func (c *context) matchKeyValue(f *netmap.Filter, b NodeInfo) bool { + switch op := f.GetOp(); op { + case netmap.EQ: + return b.Attribute(f.GetKey()) == f.GetValue() + case netmap.NE: + return b.Attribute(f.GetKey()) != f.GetValue() default: var attr uint64 - switch f.Key() { - case AttrPrice: - attr = b.price() - case AttrCapacity: + switch f.GetKey() { + case attrPrice: + attr = b.Price() + case attrCapacity: attr = b.capacity() default: var err error - attr, err = strconv.ParseUint(b.attribute(f.Key()), 10, 64) + attr, err = strconv.ParseUint(b.Attribute(f.GetKey()), 10, 64) if err != nil { // Note: because filters are somewhat independent from nodes attributes, // We don't report an error here, and fail filter instead. @@ -131,15 +124,15 @@ func (c *context) matchKeyValue(f *Filter, b NodeInfo) bool { } } - switch f.Operation() { - case OpGT: - return attr > c.numCache[f.Value()] - case OpGE: - return attr >= c.numCache[f.Value()] - case OpLT: - return attr < c.numCache[f.Value()] - case OpLE: - return attr <= c.numCache[f.Value()] + switch op { + case netmap.GT: + return attr > c.numCache[f.GetValue()] + case netmap.GE: + return attr >= c.numCache[f.GetValue()] + case netmap.LT: + return attr < c.numCache[f.GetValue()] + case netmap.LE: + return attr <= c.numCache[f.GetValue()] default: // do nothing and return false } @@ -147,127 +140,3 @@ func (c *context) matchKeyValue(f *Filter, b NodeInfo) bool { // will not happen if context was created from f (maybe panic?) return false } - -// NewFilter creates and returns new Filter instance. -// -// Defaults: -// - name: ""; -// - key: ""; -// - value: ""; -// - operation: 0; -// - filters: nil. -func NewFilter() *Filter { - return NewFilterFromV2(new(netmap.Filter)) -} - -// NewFilterFromV2 converts v2 Filter to Filter. -// -// Nil netmap.Filter converts to nil. -func NewFilterFromV2(f *netmap.Filter) *Filter { - return (*Filter)(f) -} - -// ToV2 converts Filter to v2 Filter. -// -// Nil Filter converts to nil. -func (f *Filter) ToV2() *netmap.Filter { - return (*netmap.Filter)(f) -} - -// Key returns key to filter. -func (f *Filter) Key() string { - return (*netmap.Filter)(f).GetKey() -} - -// SetKey sets key to filter. -func (f *Filter) SetKey(key string) { - (*netmap.Filter)(f).SetKey(key) -} - -// Value returns value to match. -func (f *Filter) Value() string { - return (*netmap.Filter)(f).GetValue() -} - -// SetValue sets value to match. -func (f *Filter) SetValue(val string) { - (*netmap.Filter)(f).SetValue(val) -} - -// Name returns filter name. -func (f *Filter) Name() string { - return (*netmap.Filter)(f).GetName() -} - -// SetName sets filter name. -func (f *Filter) SetName(name string) { - (*netmap.Filter)(f).SetName(name) -} - -// Operation returns filtering operation. -func (f *Filter) Operation() Operation { - return OperationFromV2( - (*netmap.Filter)(f).GetOp()) -} - -// SetOperation sets filtering operation. -func (f *Filter) SetOperation(op Operation) { - (*netmap.Filter)(f).SetOp(op.ToV2()) -} - -func filtersFromV2(fs []netmap.Filter) []Filter { - if fs == nil { - return nil - } - - res := make([]Filter, len(fs)) - - for i := range fs { - res[i] = *NewFilterFromV2(&fs[i]) - } - - return res -} - -// InnerFilters returns list of inner filters. -func (f *Filter) InnerFilters() []Filter { - return filtersFromV2((*netmap.Filter)(f).GetFilters()) -} - -func filtersToV2(fs []Filter) (fsV2 []netmap.Filter) { - if fs != nil { - fsV2 = make([]netmap.Filter, len(fs)) - - for i := range fs { - fsV2[i] = *fs[i].ToV2() - } - } - - return -} - -// SetInnerFilters sets list of inner filters. -func (f *Filter) SetInnerFilters(fs ...Filter) { - (*netmap.Filter)(f). - SetFilters(filtersToV2(fs)) -} - -// Marshal marshals Filter into a protobuf binary form. -func (f *Filter) Marshal() ([]byte, error) { - return (*netmap.Filter)(f).StableMarshal(nil), nil -} - -// Unmarshal unmarshals protobuf binary representation of Filter. -func (f *Filter) Unmarshal(data []byte) error { - return (*netmap.Filter)(f).Unmarshal(data) -} - -// MarshalJSON encodes Filter to protobuf JSON format. -func (f *Filter) MarshalJSON() ([]byte, error) { - return (*netmap.Filter)(f).MarshalJSON() -} - -// UnmarshalJSON decodes Filter from protobuf JSON format. -func (f *Filter) UnmarshalJSON(data []byte) error { - return (*netmap.Filter)(f).UnmarshalJSON(data) -} diff --git a/netmap/filter_test.go b/netmap/filter_test.go index ea3383f..fc30073 100644 --- a/netmap/filter_test.go +++ b/netmap/filter_test.go @@ -10,24 +10,24 @@ import ( func TestContext_ProcessFilters(t *testing.T) { fs := []Filter{ - newFilter("StorageSSD", "Storage", "SSD", OpEQ), - newFilter("GoodRating", "Rating", "4", OpGE), - newFilter("Main", "", "", OpAND, + newFilter("StorageSSD", "Storage", "SSD", netmap.EQ), + newFilter("GoodRating", "Rating", "4", netmap.GE), + newFilter("Main", "", "", netmap.AND, newFilter("StorageSSD", "", "", 0), - newFilter("", "IntField", "123", OpLT), + newFilter("", "IntField", "123", netmap.LT), newFilter("GoodRating", "", "", 0)), } - c := newContext(new(Netmap)) + c := newContext(NetMap{}) p := newPlacementPolicy(1, nil, nil, fs) require.NoError(t, c.processFilters(p)) - require.Equal(t, 3, len(c.Filters)) + require.Equal(t, 3, len(c.processedFilters)) for _, f := range fs { - require.Equal(t, f, *c.Filters[f.Name()]) + require.Equal(t, f.m, *c.processedFilters[f.m.GetName()]) } - require.Equal(t, uint64(4), c.numCache[fs[1].Value()]) - require.Equal(t, uint64(123), c.numCache[fs[2].InnerFilters()[1].Value()]) + require.Equal(t, uint64(4), c.numCache[fs[1].m.GetValue()]) + require.Equal(t, uint64(123), c.numCache[fs[2].m.GetFilters()[1].GetValue()]) } func TestContext_ProcessFiltersInvalid(t *testing.T) { @@ -38,40 +38,40 @@ func TestContext_ProcessFiltersInvalid(t *testing.T) { }{ { "UnnamedTop", - newFilter("", "Storage", "SSD", OpEQ), - ErrUnnamedTopFilter, + newFilter("", "Storage", "SSD", netmap.EQ), + errUnnamedTopFilter, }, { "InvalidReference", - newFilter("Main", "", "", OpAND, + newFilter("Main", "", "", netmap.AND, newFilter("StorageSSD", "", "", 0)), - ErrFilterNotFound, + errFilterNotFound, }, { "NonEmptyKeyed", - newFilter("Main", "Storage", "SSD", OpEQ, + newFilter("Main", "Storage", "SSD", netmap.EQ, newFilter("StorageSSD", "", "", 0)), - ErrNonEmptyFilters, + errNonEmptyFilters, }, { "InvalidNumber", - newFilter("Main", "Rating", "three", OpGE), - ErrInvalidNumber, + newFilter("Main", "Rating", "three", netmap.GE), + errInvalidNumber, }, { "InvalidOp", newFilter("Main", "Rating", "3", 0), - ErrInvalidFilterOp, + errInvalidFilterOp, }, { "InvalidName", - newFilter("*", "Rating", "3", OpGE), - ErrInvalidFilterName, + newFilter("*", "Rating", "3", netmap.GE), + errInvalidFilterName, }, } for _, tc := range errTestCases { t.Run(tc.name, func(t *testing.T) { - c := newContext(new(Netmap)) + c := newContext(NetMap{}) p := newPlacementPolicy(1, nil, nil, []Filter{tc.filter}) err := c.processFilters(p) require.True(t, errors.Is(err, tc.err), "got: %v", err) @@ -80,151 +80,16 @@ func TestContext_ProcessFiltersInvalid(t *testing.T) { } func TestFilter_MatchSimple_InvalidOp(t *testing.T) { - var aRating NodeAttribute - aRating.SetKey("Rating") - aRating.SetValue("4") - - var aCountry NodeAttribute - aRating.SetKey("Country") - aRating.SetValue("Germany") - var b NodeInfo - b.SetAttributes(aRating, aCountry) + b.SetAttribute("Rating", "4") + b.SetAttribute("Country", "Germany") - f := newFilter("Main", "Rating", "5", OpEQ) - c := newContext(new(Netmap)) + f := newFilter("Main", "Rating", "5", netmap.EQ) + c := newContext(NetMap{}) p := newPlacementPolicy(1, nil, nil, []Filter{f}) require.NoError(t, c.processFilters(p)) // just for the coverage - f.SetOperation(0) - require.False(t, c.match(&f, b)) -} - -func testFilter() *Filter { - f := NewFilter() - f.SetOperation(OpGE) - f.SetName("name") - f.SetKey("key") - f.SetValue("value") - - return f -} - -func TestFilterFromV2(t *testing.T) { - t.Run("nil from V2", func(t *testing.T) { - var x *netmap.Filter - - require.Nil(t, NewFilterFromV2(x)) - }) - - t.Run("nil to V2", func(t *testing.T) { - var x *Filter - - require.Nil(t, x.ToV2()) - }) - - fV2 := new(netmap.Filter) - fV2.SetOp(netmap.GE) - fV2.SetName("name") - fV2.SetKey("key") - fV2.SetValue("value") - - f := NewFilterFromV2(fV2) - - require.Equal(t, fV2, f.ToV2()) -} - -func TestFilter_Key(t *testing.T) { - f := NewFilter() - key := "some key" - - f.SetKey(key) - - require.Equal(t, key, f.Key()) -} - -func TestFilter_Value(t *testing.T) { - f := NewFilter() - val := "some value" - - f.SetValue(val) - - require.Equal(t, val, f.Value()) -} - -func TestFilter_Name(t *testing.T) { - f := NewFilter() - name := "some name" - - f.SetName(name) - - require.Equal(t, name, f.Name()) -} - -func TestFilter_Operation(t *testing.T) { - f := NewFilter() - op := OpGE - - f.SetOperation(op) - - require.Equal(t, op, f.Operation()) -} - -func TestFilter_InnerFilters(t *testing.T) { - f := NewFilter() - - f1, f2 := *testFilter(), *testFilter() - - f.SetInnerFilters(f1, f2) - - require.Equal(t, []Filter{f1, f2}, f.InnerFilters()) -} - -func TestFilterEncoding(t *testing.T) { - f := newFilter("name", "key", "value", OpEQ, - newFilter("name2", "key2", "value", OpOR), - ) - - t.Run("binary", func(t *testing.T) { - data, err := f.Marshal() - require.NoError(t, err) - - f2 := *NewFilter() - require.NoError(t, f2.Unmarshal(data)) - - require.Equal(t, f, f2) - }) - - t.Run("json", func(t *testing.T) { - data, err := f.MarshalJSON() - require.NoError(t, err) - - f2 := *NewFilter() - require.NoError(t, f2.UnmarshalJSON(data)) - - require.Equal(t, f, f2) - }) -} - -func TestNewFilter(t *testing.T) { - t.Run("default values", func(t *testing.T) { - filter := NewFilter() - - // check initial values - require.Empty(t, filter.Name()) - require.Empty(t, filter.Key()) - require.Empty(t, filter.Value()) - require.Zero(t, filter.Operation()) - require.Nil(t, filter.InnerFilters()) - - // convert to v2 message - filterV2 := filter.ToV2() - - require.Empty(t, filterV2.GetName()) - require.Empty(t, filterV2.GetKey()) - require.Empty(t, filterV2.GetValue()) - require.Equal(t, netmap.UnspecifiedOperation, filterV2.GetOp()) - require.Nil(t, filterV2.GetFilters()) - }) + f.m.SetOp(0) + require.False(t, c.match(&f.m, b)) } diff --git a/netmap/helper_test.go b/netmap/helper_test.go index f9ca738..f5cc831 100644 --- a/netmap/helper_test.go +++ b/netmap/helper_test.go @@ -1,81 +1,49 @@ package netmap import ( - "testing" - - "github.com/stretchr/testify/require" + "github.com/nspcc-dev/neofs-api-go/v2/netmap" ) -func newFilter(name string, k, v string, op Operation, fs ...Filter) (f Filter) { +func newFilter(name string, k, v string, op netmap.Operation, fs ...Filter) (f Filter) { f.SetName(name) - f.SetKey(k) - f.SetOperation(op) - f.SetValue(v) - f.SetInnerFilters(fs...) + f.m.SetKey(k) + f.m.SetOp(op) + f.m.SetValue(v) + inner := make([]netmap.Filter, len(fs)) + for i := range fs { + inner[i] = fs[i].m + } + f.m.SetFilters(inner) return f } -func newSelector(name string, attr string, c Clause, count uint32, filter string) (s Selector) { +func newSelector(name string, attr string, count uint32, filter string, clause func(*Selector)) (s Selector) { s.SetName(name) - s.SetAttribute(attr) - s.SetCount(count) - s.SetClause(c) - s.SetFilter(filter) + s.SelectByBucketAttribute(attr) + s.SetNodeAmount(count) + clause(&s) + s.SetFilterName(filter) return s } -func newPlacementPolicy(bf uint32, rs []Replica, ss []Selector, fs []Filter) *PlacementPolicy { - p := NewPlacementPolicy() +func newPlacementPolicy(bf uint32, rs []ReplicaDescriptor, ss []Selector, fs []Filter) (p PlacementPolicy) { p.SetContainerBackupFactor(bf) - p.SetReplicas(rs...) - p.SetSelectors(ss...) - p.SetFilters(fs...) + p.AddReplicas(rs...) + p.AddSelectors(ss...) + p.AddFilters(fs...) return p } -func newReplica(c uint32, s string) (r Replica) { - r.SetCount(c) - r.SetSelector(s) +func newReplica(c uint32, s string) (r ReplicaDescriptor) { + r.SetAmount(c) + r.SetSelectorName(s) return r } -func nodeInfoFromAttributes(props ...string) NodeInfo { - attrs := make([]NodeAttribute, len(props)/2) - for i := range attrs { - attrs[i].SetKey(props[i*2]) - attrs[i].SetValue(props[i*2+1]) - } - n := NewNodeInfo() - n.SetAttributes(attrs...) - return *n -} - -type enumIface interface { - FromString(string) bool - String() string -} - -type enumStringItem struct { - val enumIface - str string -} - -func testEnumStrings(t *testing.T, e enumIface, items []enumStringItem) { - for _, item := range items { - require.Equal(t, item.str, item.val.String()) - - s := item.val.String() - - require.True(t, e.FromString(s), s) - - require.EqualValues(t, item.val, e, item.val) +func nodeInfoFromAttributes(props ...string) (n NodeInfo) { + for i := 0; i < len(props); i += 2 { + n.SetAttribute(props[i], props[i+1]) } - // incorrect strings - for _, str := range []string{ - "some string", - "undefined", - } { - require.False(t, e.FromString(str)) - } + return } diff --git a/netmap/json_test.go b/netmap/json_test.go index 2ceb2ef..3e89e56 100644 --- a/netmap/json_test.go +++ b/netmap/json_test.go @@ -26,6 +26,8 @@ type TestCase struct { } } +var _, _ json.Unmarshaler = new(NodeInfo), new(PlacementPolicy) + func compareNodes(t testing.TB, expected [][]int, nodes nodes, actual [][]NodeInfo) { require.Equal(t, len(expected), len(actual)) for i := range expected { @@ -56,12 +58,12 @@ func TestPlacementPolicy_Interopability(t *testing.T) { copy(srcNodes, tc.Nodes) t.Run(tc.Name, func(t *testing.T) { - var nm Netmap + var nm NetMap nm.SetNodes(tc.Nodes) for name, tt := range tc.Tests { t.Run(name, func(t *testing.T) { - v, err := nm.GetContainerNodes(&tt.Policy, tt.Pivot) + v, err := nm.ContainerNodes(tt.Policy, tt.Pivot) if tt.Result == nil { require.Error(t, err) require.Contains(t, err.Error(), tt.Error) @@ -72,7 +74,7 @@ func TestPlacementPolicy_Interopability(t *testing.T) { compareNodes(t, tt.Result, tc.Nodes, v) if tt.Placement.Result != nil { - res, err := nm.GetPlacementVectors(v, tt.Placement.Pivot) + res, err := nm.PlacementVectors(v, tt.Placement.Pivot) require.NoError(t, err) compareNodes(t, tt.Placement.Result, tc.Nodes, res) require.Equal(t, srcNodes, tc.Nodes) @@ -101,7 +103,7 @@ func BenchmarkPlacementPolicyInteropability(b *testing.B) { require.NoError(b, json.Unmarshal(bs, &tc), "cannot unmarshal %s", ds[i].Name()) b.Run(tc.Name, func(b *testing.B) { - var nm Netmap + var nm NetMap nm.SetNodes(tc.Nodes) require.NoError(b, err) @@ -111,7 +113,7 @@ func BenchmarkPlacementPolicyInteropability(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { b.StartTimer() - v, err := nm.GetContainerNodes(&tt.Policy, tt.Pivot) + v, err := nm.ContainerNodes(tt.Policy, tt.Pivot) b.StopTimer() if tt.Result == nil { require.Error(b, err) @@ -123,7 +125,7 @@ func BenchmarkPlacementPolicyInteropability(b *testing.B) { if tt.Placement.Result != nil { b.StartTimer() - res, err := nm.GetPlacementVectors(v, tt.Placement.Pivot) + res, err := nm.PlacementVectors(v, tt.Placement.Pivot) b.StopTimer() require.NoError(b, err) compareNodes(b, tt.Placement.Result, tc.Nodes, res) @@ -146,14 +148,14 @@ func BenchmarkManySelects(b *testing.B) { tt, ok := tc.Tests["Select"] require.True(b, ok) - var nm Netmap + var nm NetMap nm.SetNodes(tc.Nodes) b.ResetTimer() b.ReportAllocs() for i := 0; i < b.N; i++ { - _, err = nm.GetContainerNodes(&tt.Policy, tt.Pivot) + _, err = nm.ContainerNodes(tt.Policy, tt.Pivot) if err != nil { b.FailNow() } diff --git a/netmap/json_tests/filter_invalid_integer.json b/netmap/json_tests/filter_invalid_integer.json index e04cbb3..933f939 100644 --- a/netmap/json_tests/filter_invalid_integer.json +++ b/netmap/json_tests/filter_invalid_integer.json @@ -5,7 +5,7 @@ "attributes": [ { "key": "IntegerField", - "value": "" + "value": "true" } ] }, diff --git a/netmap/netmap.go b/netmap/netmap.go index 0572cc0..a5b2933 100644 --- a/netmap/netmap.go +++ b/netmap/netmap.go @@ -1,112 +1,40 @@ package netmap import ( - "bytes" "fmt" - "strconv" "github.com/nspcc-dev/hrw" "github.com/nspcc-dev/neofs-api-go/v2/netmap" ) -const defaultCBF = 3 - -var _, _ hrw.Hasher = NodeInfo{}, nodes{} - -// Hash implements hrw.Hasher interface. -// -// Hash is needed to support weighted HRW therefore sort function sorts nodes -// based on their public key. -func (i NodeInfo) Hash() uint64 { - return hrw.Hash(i.m.GetPublicKey()) -} - -func (i NodeInfo) less(i2 NodeInfo) bool { - return bytes.Compare(i.PublicKey(), i2.PublicKey()) < 0 -} - -// attribute returns value of the node attribute by the given key. Returns empty -// string if attribute is missing. -// -// Method is needed to internal placement needs. -func (i NodeInfo) attribute(key string) string { - as := i.m.GetAttributes() - for j := range as { - if as[j].GetKey() == key { - return as[j].GetValue() - } - } - - return "" -} - -func (i *NodeInfo) syncAttributes() { - as := i.m.GetAttributes() - for j := range as { - switch as[j].GetKey() { - case AttrPrice: - i.priceAttr, _ = strconv.ParseUint(as[j].GetValue(), 10, 64) - case AttrCapacity: - i.capAttr, _ = strconv.ParseUint(as[j].GetValue(), 10, 64) - } - } -} - -func (i *NodeInfo) setPrice(price uint64) { - i.priceAttr = price - - as := i.m.GetAttributes() - for j := range as { - if as[j].GetKey() == AttrPrice { - as[j].SetValue(strconv.FormatUint(i.capAttr, 10)) - return - } - } - - as = append(as, netmap.Attribute{}) - as[len(as)-1].SetKey(AttrPrice) - as[len(as)-1].SetValue(strconv.FormatUint(i.capAttr, 10)) - - i.m.SetAttributes(as) -} - -func (i *NodeInfo) price() uint64 { - return i.priceAttr -} - -func (i *NodeInfo) setCapacity(capacity uint64) { - i.capAttr = capacity - - as := i.m.GetAttributes() - for j := range as { - if as[j].GetKey() == AttrCapacity { - as[j].SetValue(strconv.FormatUint(i.capAttr, 10)) - return - } - } - - as = append(as, netmap.Attribute{}) - as[len(as)-1].SetKey(AttrCapacity) - as[len(as)-1].SetValue(strconv.FormatUint(i.capAttr, 10)) - - i.m.SetAttributes(as) -} - -func (i NodeInfo) capacity() uint64 { - return i.capAttr -} - -// Netmap represents netmap which contains preprocessed nodes. -type Netmap struct { +// NetMap represents NeoFS network map. It includes information about all +// storage nodes registered in NeoFS the network. +type NetMap struct { nodes []NodeInfo } -func (m *Netmap) SetNodes(nodes []NodeInfo) { +// SetNodes sets information list about all storage nodes from the NeoFS network. +// +// Argument MUST NOT be mutated, make a copy first. +// +// See also Nodes. +func (m *NetMap) SetNodes(nodes []NodeInfo) { m.nodes = nodes } +// Nodes returns nodes set using SetNodes. +// +// Return value MUST not be mutated, make a copy first. +func (m NetMap) Nodes() []NodeInfo { + return m.nodes +} + +// nodes is a slice of NodeInfo instances needed for HRW sorting. type nodes []NodeInfo +// assert nodes type provides hrw.Hasher required for HRW sorting. +var _ hrw.Hasher = nodes{} + // Hash is a function from hrw.Hasher interface. It is implemented // to support weighted hrw sorting of buckets. Each bucket is already sorted by hrw, // thus giving us needed "randomness". @@ -143,8 +71,12 @@ func flattenNodes(ns []nodes) nodes { return result } -// GetPlacementVectors returns placement vectors for an object given containerNodes cnt. -func (m *Netmap) GetPlacementVectors(vectors [][]NodeInfo, pivot []byte) ([][]NodeInfo, error) { +// PlacementVectors sorts container nodes returned by ContainerNodes method +// and returns placement vectors for the entity identified by the given pivot. +// For example,in order to build node list to store the object, binary-encoded +// object identifier can be used as pivot. Result is deterministic for +// the fixed NetMap and parameters. +func (m NetMap) PlacementVectors(vectors [][]NodeInfo, pivot []byte) ([][]NodeInfo, error) { h := hrw.Hash(pivot) wf := defaultWeightFunc(m.nodes) result := make([][]NodeInfo, len(vectors)) @@ -158,13 +90,18 @@ func (m *Netmap) GetPlacementVectors(vectors [][]NodeInfo, pivot []byte) ([][]No return result, nil } -// GetContainerNodes returns nodes corresponding to each replica. -// Order of returned nodes corresponds to order of replicas in p. -// pivot is a seed for HRW sorting. -func (m *Netmap) GetContainerNodes(p *PlacementPolicy, pivot []byte) ([][]NodeInfo, error) { +// ContainerNodes returns two-dimensional list of nodes as a result of applying +// given PlacementPolicy to the NetMap. Each line of the list corresponds to a +// replica descriptor. Line order corresponds to order of ReplicaDescriptor list +// in the policy. Nodes are pre-filtered according to the Filter list from +// the policy, and then selected by Selector list. Result is deterministic for +// the fixed NetMap and parameters. +// +// Result can be used in PlacementVectors. +func (m NetMap) ContainerNodes(p PlacementPolicy, pivot []byte) ([][]NodeInfo, error) { c := newContext(m) c.setPivot(pivot) - c.setCBF(p.ContainerBackupFactor()) + c.setCBF(p.backupFactor) if err := c.processFilters(p); err != nil { return nil, err @@ -174,14 +111,15 @@ func (m *Netmap) GetContainerNodes(p *PlacementPolicy, pivot []byte) ([][]NodeIn return nil, err } - result := make([][]NodeInfo, len(p.Replicas())) + result := make([][]NodeInfo, len(p.replicas)) - for i, r := range p.Replicas() { - if r.Selector() == "" { - if len(p.Selectors()) == 0 { - s := new(Selector) - s.SetCount(r.Count()) - s.SetFilter(MainFilterName) + for i := range p.replicas { + sName := p.replicas[i].GetSelector() + if sName == "" { + if len(p.selectors) == 0 { + var s netmap.Selector + s.SetCount(p.replicas[i].GetCount()) + s.SetFilter(mainFilterName) nodes, err := c.getSelection(p, s) if err != nil { @@ -191,16 +129,16 @@ func (m *Netmap) GetContainerNodes(p *PlacementPolicy, pivot []byte) ([][]NodeIn result[i] = flattenNodes(nodes) } - for _, s := range p.Selectors() { - result[i] = append(result[i], flattenNodes(c.Selections[s.Name()])...) + for i := range p.selectors { + result[i] = append(result[i], flattenNodes(c.selections[p.selectors[i].GetName()])...) } continue } - nodes, ok := c.Selections[r.Selector()] + nodes, ok := c.selections[sName] if !ok { - return nil, fmt.Errorf("%w: REPLICA '%s'", ErrSelectorNotFound, r.Selector()) + return nil, fmt.Errorf("selector not found: REPLICA '%s'", sName) } result[i] = append(result[i], flattenNodes(nodes)...) diff --git a/netmap/network_info.go b/netmap/network_info.go index 7af240d..002ec80 100644 --- a/netmap/network_info.go +++ b/netmap/network_info.go @@ -1,204 +1,462 @@ package netmap import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "math" + "github.com/nspcc-dev/neofs-api-go/v2/netmap" ) -// NetworkInfo represents v2-compatible structure -// with information about NeoFS network. -type NetworkInfo netmap.NetworkInfo - -// NewNetworkInfoFromV2 wraps v2 NetworkInfo message to NetworkInfo. +// NetworkInfo groups information about the NeoFS network state. Mainly used to +// describe the current state of the network. // -// Nil netmap.NetworkInfo converts to nil. -func NewNetworkInfoFromV2(iV2 *netmap.NetworkInfo) *NetworkInfo { - return (*NetworkInfo)(iV2) +// NetworkInfo is mutually compatible with github.com/nspcc-dev/neofs-api-go/v2/netmap.NetworkInfo +// message. See ReadFromV2 / WriteToV2 methods. +// +// Instances can be created using built-in var declaration. +type NetworkInfo struct { + m netmap.NetworkInfo } -// NewNetworkInfo creates and initializes blank NetworkInfo. -// -// Defaults: -// - curEpoch: 0; -// - magicNum: 0; -// - msPerBlock: 0; -// - network config: nil. -func NewNetworkInfo() *NetworkInfo { - return NewNetworkInfoFromV2(new(netmap.NetworkInfo)) +// reads NetworkInfo from netmap.NetworkInfo message. If checkFieldPresence is set, +// returns an error on absence of any protocol-required field. Verifies format of any +// presented field according to NeoFS API V2 protocol. +func (x *NetworkInfo) readFromV2(m netmap.NetworkInfo, checkFieldPresence bool) error { + c := m.GetNetworkConfig() + if checkFieldPresence && c == nil { + return errors.New("missing network config") + } + + if checkFieldPresence && c.NumberOfParameters() <= 0 { + return fmt.Errorf("missing network parameters") + } + + var err error + mNames := make(map[string]struct{}, c.NumberOfParameters()) + + c.IterateParameters(func(prm *netmap.NetworkParameter) bool { + name := string(prm.GetKey()) + + _, was := mNames[name] + if was { + err = fmt.Errorf("duplicated parameter name: %s", name) + return true + } + + mNames[name] = struct{}{} + + switch name { + default: + if len(prm.GetValue()) == 0 { + err = fmt.Errorf("empty attribute value %s", name) + return true + } + case configEigenTrustAlpha: + var num uint64 + + num, err = decodeConfigValueUint64(prm.GetValue()) + if err == nil { + if alpha := math.Float64frombits(num); alpha < 0 && alpha > 1 { + err = fmt.Errorf("EigenTrust alpha value %0.2f is out of range [0, 1]", alpha) + } + } + case + configAuditFee, + configStoragePrice, + configContainerFee, + configNamedContainerFee, + configEigenTrustIterationsAmount, + configEpochDuration, + configIRCandidateFee, + configMaxObjSize, + configWithdrawalFee: + _, err = decodeConfigValueUint64(prm.GetValue()) + } + + if err != nil { + err = fmt.Errorf("invalid %s parameter: %w", name, err) + } + + return err != nil + }) + + if err != nil { + return err + } + + x.m = m + + return nil } -// ToV2 converts NetworkInfo to v2 NetworkInfo. +// ReadFromV2 reads NetworkInfo from the netmap.NetworkInfo message. Checks if the +// message conforms to NeoFS API V2 protocol. // -// Nil NetworkInfo converts to nil. -func (i *NetworkInfo) ToV2() *netmap.NetworkInfo { - return (*netmap.NetworkInfo)(i) +// See also WriteToV2. +func (x *NetworkInfo) ReadFromV2(m netmap.NetworkInfo) error { + return x.readFromV2(m, true) } -// CurrentEpoch returns current epoch of the NeoFS network. -func (i *NetworkInfo) CurrentEpoch() uint64 { - return (*netmap.NetworkInfo)(i).GetCurrentEpoch() +// WriteToV2 writes NetworkInfo to the netmap.NetworkInfo message. The message +// MUST NOT be nil. +// +// See also ReadFromV2. +func (x NetworkInfo) WriteToV2(m *netmap.NetworkInfo) { + *m = x.m +} + +// CurrentEpoch returns epoch set using SetCurrentEpoch. +// +// Zero NetworkInfo has zero current epoch. +func (x NetworkInfo) CurrentEpoch() uint64 { + return x.m.GetCurrentEpoch() } // SetCurrentEpoch sets current epoch of the NeoFS network. -func (i *NetworkInfo) SetCurrentEpoch(epoch uint64) { - (*netmap.NetworkInfo)(i).SetCurrentEpoch(epoch) +func (x *NetworkInfo) SetCurrentEpoch(epoch uint64) { + x.m.SetCurrentEpoch(epoch) } -// MagicNumber returns magic number of the sidechain. -func (i *NetworkInfo) MagicNumber() uint64 { - return (*netmap.NetworkInfo)(i).GetMagicNumber() -} - -// SetMagicNumber sets magic number of the sidechain. -func (i *NetworkInfo) SetMagicNumber(epoch uint64) { - (*netmap.NetworkInfo)(i).SetMagicNumber(epoch) -} - -// MsPerBlock returns MillisecondsPerBlock network parameter. -func (i *NetworkInfo) MsPerBlock() int64 { - return (*netmap.NetworkInfo)(i). - GetMsPerBlock() -} - -// SetMsPerBlock sets MillisecondsPerBlock network parameter. -func (i *NetworkInfo) SetMsPerBlock(v int64) { - (*netmap.NetworkInfo)(i). - SetMsPerBlock(v) -} - -// NetworkConfig returns NeoFS network configuration. -func (i *NetworkInfo) NetworkConfig() *NetworkConfig { - return NewNetworkConfigFromV2( - (*netmap.NetworkInfo)(i). - GetNetworkConfig(), - ) -} - -// SetNetworkConfig sets NeoFS network configuration. -func (i *NetworkInfo) SetNetworkConfig(v *NetworkConfig) { - (*netmap.NetworkInfo)(i). - SetNetworkConfig(v.ToV2()) -} - -// Marshal marshals NetworkInfo into a protobuf binary form. -func (i *NetworkInfo) Marshal() ([]byte, error) { - return (*netmap.NetworkInfo)(i).StableMarshal(nil), nil -} - -// Unmarshal unmarshals protobuf binary representation of NetworkInfo. -func (i *NetworkInfo) Unmarshal(data []byte) error { - return (*netmap.NetworkInfo)(i).Unmarshal(data) -} - -// MarshalJSON encodes NetworkInfo to protobuf JSON format. -func (i *NetworkInfo) MarshalJSON() ([]byte, error) { - return (*netmap.NetworkInfo)(i).MarshalJSON() -} - -// UnmarshalJSON decodes NetworkInfo from protobuf JSON format. -func (i *NetworkInfo) UnmarshalJSON(data []byte) error { - return (*netmap.NetworkInfo)(i).UnmarshalJSON(data) -} - -// NetworkParameter represents v2-compatible NeoFS network parameter. -type NetworkParameter netmap.NetworkParameter - -// NewNetworkParameterFromV2 wraps v2 NetworkParameter message to NetworkParameter. +// MagicNumber returns magic number set using SetMagicNumber. // -// Nil netmap.NetworkParameter converts to nil. -func NewNetworkParameterFromV2(pv2 *netmap.NetworkParameter) *NetworkParameter { - return (*NetworkParameter)(pv2) +// Zero NetworkInfo has zero magic. +func (x NetworkInfo) MagicNumber() uint64 { + return x.m.GetMagicNumber() } -// NewNetworkParameter creates and initializes blank NetworkParameter. +// SetMagicNumber sets magic number of the NeoFS Sidechain. // -// Defaults: -// - key: nil; -// - value: nil. -func NewNetworkParameter() *NetworkParameter { - return NewNetworkParameterFromV2(new(netmap.NetworkParameter)) +// See also MagicNumber. +func (x *NetworkInfo) SetMagicNumber(epoch uint64) { + x.m.SetMagicNumber(epoch) } -// ToV2 converts NetworkParameter to v2 NetworkParameter. +// MsPerBlock returns network parameter set using SetMsPerBlock. +func (x NetworkInfo) MsPerBlock() int64 { + return x.m.GetMsPerBlock() +} + +// SetMsPerBlock sets MillisecondsPerBlock network parameter of the NeoFS Sidechain. // -// Nil NetworkParameter converts to nil. -func (x *NetworkParameter) ToV2() *netmap.NetworkParameter { - return (*netmap.NetworkParameter)(x) +// See also MsPerBlock. +func (x *NetworkInfo) SetMsPerBlock(v int64) { + x.m.SetMsPerBlock(v) } -// Key returns key to network parameter. -func (x *NetworkParameter) Key() []byte { - return (*netmap.NetworkParameter)(x).GetKey() -} +func (x *NetworkInfo) setConfig(name string, val []byte) { + c := x.m.GetNetworkConfig() + if c == nil { + c = new(netmap.NetworkConfig) -// SetKey sets key to the network parameter. -func (x *NetworkParameter) SetKey(key []byte) { - (*netmap.NetworkParameter)(x).SetKey(key) -} + var prm netmap.NetworkParameter + prm.SetKey([]byte(name)) + prm.SetValue(val) -// Value returns value of the network parameter. -func (x *NetworkParameter) Value() []byte { - return (*netmap.NetworkParameter)(x).GetValue() -} + c.SetParameters(prm) -// SetValue sets value of the network parameter. -func (x *NetworkParameter) SetValue(val []byte) { - (*netmap.NetworkParameter)(x).SetValue(val) -} + x.m.SetNetworkConfig(c) -// NetworkConfig represents v2-compatible NeoFS network configuration. -type NetworkConfig netmap.NetworkConfig - -// NewNetworkConfigFromV2 wraps v2 NetworkConfig message to NetworkConfig. -// -// Nil netmap.NetworkConfig converts to nil. -func NewNetworkConfigFromV2(cv2 *netmap.NetworkConfig) *NetworkConfig { - return (*NetworkConfig)(cv2) -} - -// NewNetworkConfig creates and initializes blank NetworkConfig. -// -// Defaults: -// - parameters num: 0. -func NewNetworkConfig() *NetworkConfig { - return NewNetworkConfigFromV2(new(netmap.NetworkConfig)) -} - -// ToV2 converts NetworkConfig to v2 NetworkConfig. -// -// Nil NetworkConfig converts to nil. -func (x *NetworkConfig) ToV2() *netmap.NetworkConfig { - return (*netmap.NetworkConfig)(x) -} - -// NumberOfParameters returns number of network parameters. -func (x *NetworkConfig) NumberOfParameters() int { - return (*netmap.NetworkConfig)(x).NumberOfParameters() -} - -// IterateAddresses iterates over network parameters. -// Breaks iteration on f's true return. -// -// Handler should not be nil. -func (x *NetworkConfig) IterateParameters(f func(*NetworkParameter) bool) { - (*netmap.NetworkConfig)(x). - IterateParameters(func(p *netmap.NetworkParameter) bool { - return f(NewNetworkParameterFromV2(p)) - }) -} - -// Value returns value of the network parameter. -func (x *NetworkConfig) SetParameters(ps ...NetworkParameter) { - var psV2 []netmap.NetworkParameter - - if ps != nil { - ln := len(ps) - - psV2 = make([]netmap.NetworkParameter, ln) - - for i := 0; i < ln; i++ { - psV2[i] = *ps[i].ToV2() - } + return } - (*netmap.NetworkConfig)(x).SetParameters(psV2...) + found := false + prms := make([]netmap.NetworkParameter, 0, c.NumberOfParameters()) + + c.IterateParameters(func(prm *netmap.NetworkParameter) bool { + found = bytes.Equal(prm.GetKey(), []byte(name)) + if found { + prm.SetValue(val) + } else { + prms = append(prms, *prm) + } + + return found + }) + + if !found { + prms = append(prms, netmap.NetworkParameter{}) + prms[len(prms)-1].SetKey([]byte(name)) + prms[len(prms)-1].SetValue(val) + + c.SetParameters(prms...) + } +} + +func (x NetworkInfo) configValue(name string) (res []byte) { + x.m.GetNetworkConfig().IterateParameters(func(prm *netmap.NetworkParameter) bool { + if string(prm.GetKey()) == name { + res = prm.GetValue() + + return true + } + + return false + }) + + return +} + +// SetRawNetworkParameter sets named NeoFS network parameter whose value is +// transmitted but not interpreted by the NeoFS API protocol. +// +// Argument MUST NOT be mutated, make a copy first. +// +// See also RawNetworkParameter, IterateRawNetworkParameters. +func (x *NetworkInfo) SetRawNetworkParameter(name string, value []byte) { + x.setConfig(name, value) +} + +// RawNetworkParameter reads raw network parameter set using SetRawNetworkParameter +// by its name. Returns nil if value is missing. +// +// Return value MUST NOT be mutated, make a copy first. +// +// Zero NetworkInfo has no network parameters. +func (x *NetworkInfo) RawNetworkParameter(name string) []byte { + return x.configValue(name) +} + +// IterateRawNetworkParameters iterates over all raw networks parameters set +// using SetRawNetworkParameter and passes them into f. +// +// Handler MUST NOT be nil. Handler MUST NOT mutate value parameter. +// +// Zero NetworkInfo has no network parameters. +func (x *NetworkInfo) IterateRawNetworkParameters(f func(name string, value []byte)) { + c := x.m.GetNetworkConfig() + + c.IterateParameters(func(prm *netmap.NetworkParameter) bool { + name := string(prm.GetKey()) + switch name { + default: + f(name, prm.GetValue()) + case + configEigenTrustAlpha, + configAuditFee, + configStoragePrice, + configContainerFee, + configNamedContainerFee, + configEigenTrustIterationsAmount, + configEpochDuration, + configIRCandidateFee, + configMaxObjSize, + configWithdrawalFee: + } + + return false + }) +} + +func (x *NetworkInfo) setConfigUint64(name string, num uint64) { + val := make([]byte, 8) + binary.LittleEndian.PutUint64(val, num) + + x.setConfig(name, val) +} + +func decodeConfigValueUint64(val []byte) (uint64, error) { + if ln := len(val); ln != 8 { + return 0, fmt.Errorf("invalid uint64 parameter length %d", ln) + } + + return binary.LittleEndian.Uint64(val), nil +} + +func (x NetworkInfo) configUint64(name string) uint64 { + val := x.configValue(name) + if val == nil { + return 0 + } + + res, err := decodeConfigValueUint64(val) + if err != nil { + // potential panic is OK since value MUST be correct since it is + // verified in ReadFromV2 or set by provided method. + panic(err) + } + + return res +} + +const configAuditFee = "AuditFee" + +// SetAuditFee sets the configuration value of the audit fee for the Inner Ring. +// +// See also AuditFee. +func (x *NetworkInfo) SetAuditFee(fee uint64) { + x.setConfigUint64(configAuditFee, fee) +} + +// AuditFee returns audit fee set using SetAuditFee. +// +// Zero NetworkInfo has zero audit fee. +func (x NetworkInfo) AuditFee() uint64 { + return x.configUint64(configAuditFee) +} + +const configStoragePrice = "BasicIncomeRate" + +// SetStoragePrice sets the price per gigabyte of data storage that data owners +// pay to storage nodes. +// +// See also StoragePrice. +func (x *NetworkInfo) SetStoragePrice(price uint64) { + x.setConfigUint64(configStoragePrice, price) +} + +// StoragePrice returns storage price set using SetStoragePrice. +// +// Zero NetworkInfo has zero storage price. +func (x NetworkInfo) StoragePrice() uint64 { + return x.configUint64(configStoragePrice) +} + +const configContainerFee = "ContainerFee" + +// SetContainerFee sets fee for the container creation that creator pays to +// each Alphabet node. +// +// See also ContainerFee. +func (x *NetworkInfo) SetContainerFee(fee uint64) { + x.setConfigUint64(configContainerFee, fee) +} + +// ContainerFee returns container fee set using SetContainerFee. +// +// Zero NetworkInfo has zero container fee. +func (x NetworkInfo) ContainerFee() uint64 { + return x.configUint64(configContainerFee) +} + +const configNamedContainerFee = "ContainerAliasFee" + +// SetNamedContainerFee sets fee for creation of the named container creation +// that creator pays to each Alphabet node. +// +// See also NamedContainerFee. +func (x *NetworkInfo) SetNamedContainerFee(fee uint64) { + x.setConfigUint64(configNamedContainerFee, fee) +} + +// NamedContainerFee returns container fee set using SetNamedContainerFee. +// +// Zero NetworkInfo has zero container fee. +func (x NetworkInfo) NamedContainerFee() uint64 { + return x.configUint64(configNamedContainerFee) +} + +const configEigenTrustAlpha = "EigenTrustAlpha" + +// SetEigenTrustAlpha sets alpha parameter for EigenTrust algorithm used in +// reputation system of the storage nodes. Value MUST be in range [0, 1]. +// +// See also EigenTrustAlpha. +func (x *NetworkInfo) SetEigenTrustAlpha(alpha float64) { + if alpha < 0 || alpha > 1 { + panic(fmt.Sprintf("EigenTrust alpha parameter MUST be in range [0, 1], got %.2f", alpha)) + } + + x.setConfigUint64(configEigenTrustAlpha, math.Float64bits(alpha)) +} + +// EigenTrustAlpha returns EigenTrust parameter set using SetEigenTrustAlpha. +// +// Zero NetworkInfo has zero alpha parameter. +func (x NetworkInfo) EigenTrustAlpha() float64 { + alpha := math.Float64frombits(x.configUint64(configEigenTrustAlpha)) + if alpha < 0 || alpha > 1 { + panic(fmt.Sprintf("unexpected invalid %s parameter value %.2f", configEigenTrustAlpha, alpha)) + } + + return alpha +} + +const configEigenTrustIterationsAmount = "EigenTrustIterations" + +// SetEigenTrustIterationAmount sets number of iterations of the EigenTrust +// algorithm to perform. The algorithm is used by the storage nodes for +// calculating the reputation values. +// +// See also EigenTrustIterationAmount. +func (x *NetworkInfo) SetEigenTrustIterationAmount(amount uint64) { + x.setConfigUint64(configEigenTrustIterationsAmount, amount) +} + +// EigenTrustIterationAmount returns EigenTrust iteration amount set using +// SetEigenTrustIterationAmount. +// +// Zero NetworkInfo has zero iteration number. +func (x NetworkInfo) EigenTrustIterationAmount() uint64 { + return x.configUint64(configEigenTrustIterationsAmount) +} + +const configEpochDuration = "EpochDuration" + +// SetEpochDuration sets NeoFS epoch duration measured in block amount of the +// NeoFS Sidechain. +// +// See also EpochDuration. +func (x *NetworkInfo) SetEpochDuration(blocks uint64) { + x.setConfigUint64(configEpochDuration, blocks) +} + +// EpochDuration returns epoch duration set using SetEpochDuration. +// +// Zero NetworkInfo has zero iteration number. +func (x NetworkInfo) EpochDuration() uint64 { + return x.configUint64(configEpochDuration) +} + +const configIRCandidateFee = "InnerRingCandidateFee" + +// SetIRCandidateFee sets fee for Inner Ring entrance paid by a new member. +// +// See also IRCandidateFee. +func (x *NetworkInfo) SetIRCandidateFee(fee uint64) { + x.setConfigUint64(configIRCandidateFee, fee) +} + +// IRCandidateFee returns IR entrance fee set using SetIRCandidateFee. +// +// Zero NetworkInfo has zero fee. +func (x NetworkInfo) IRCandidateFee() uint64 { + return x.configUint64(configIRCandidateFee) +} + +const configMaxObjSize = "MaxObjectSize" + +// SetMaxObjectSize sets maximum size of the object stored locally on the +// storage nodes (physical objects). Binary representation of any physically +// stored object MUST NOT overflow the limit. +// +// See also MaxObjectSize. +func (x *NetworkInfo) SetMaxObjectSize(sz uint64) { + x.setConfigUint64(configMaxObjSize, sz) +} + +// MaxObjectSize returns maximum object size set using SetMaxObjectSize. +// +// Zero NetworkInfo has zero maximum size. +func (x NetworkInfo) MaxObjectSize() uint64 { + return x.configUint64(configMaxObjSize) +} + +const configWithdrawalFee = "WithdrawFee" + +// SetWithdrawalFee sets fee for withdrawals from the NeoFS accounts that +// account owners pay to each Alphabet node. +// +// See also WithdrawalFee. +func (x *NetworkInfo) SetWithdrawalFee(sz uint64) { + x.setConfigUint64(configWithdrawalFee, sz) +} + +// WithdrawalFee returns withdrawal fee set using SetWithdrawalFee. +// +// Zero NetworkInfo has zero fee. +func (x NetworkInfo) WithdrawalFee() uint64 { + return x.configUint64(configWithdrawalFee) } diff --git a/netmap/network_info_test.go b/netmap/network_info_test.go index 56843a4..cc096f5 100644 --- a/netmap/network_info_test.go +++ b/netmap/network_info_test.go @@ -1,214 +1,214 @@ package netmap_test import ( + "encoding/binary" + "math" "testing" + "github.com/nspcc-dev/neofs-api-go/v2/netmap" . "github.com/nspcc-dev/neofs-sdk-go/netmap" - netmaptest "github.com/nspcc-dev/neofs-sdk-go/netmap/test" "github.com/stretchr/testify/require" ) -func TestNetworkParameter_Key(t *testing.T) { - i := NewNetworkParameter() - - k := []byte("key") - - i.SetKey(k) - - require.Equal(t, k, i.Key()) - require.Equal(t, k, i.ToV2().GetKey()) -} - -func TestNetworkParameter_Value(t *testing.T) { - i := NewNetworkParameter() - - v := []byte("value") - - i.SetValue(v) - - require.Equal(t, v, i.Value()) - require.Equal(t, v, i.ToV2().GetValue()) -} - -func TestNewNetworkParameterFromV2(t *testing.T) { - t.Run("nil", func(t *testing.T) { - require.Nil(t, NewNetworkParameterFromV2(nil)) - }) -} - -func TestNetworkParameter_ToV2(t *testing.T) { - t.Run("nil", func(t *testing.T) { - var x *NetworkParameter - - require.Nil(t, x.ToV2()) - }) -} - -func TestNewNetworkParameter(t *testing.T) { - x := NewNetworkParameter() - - // check initial values - require.Nil(t, x.Key()) - require.Nil(t, x.Value()) - - // convert to v2 message - xV2 := x.ToV2() - - require.Nil(t, xV2.GetKey()) - require.Nil(t, xV2.GetValue()) -} - -func TestNetworkConfig_SetParameters(t *testing.T) { - x := NewNetworkConfig() - - require.Zero(t, x.NumberOfParameters()) - - called := 0 - - x.IterateParameters(func(p *NetworkParameter) bool { - called++ - return false - }) - - require.Zero(t, called) - - pps := []NetworkParameter{ - *netmaptest.NetworkParameter(), - *netmaptest.NetworkParameter(), - } - - x.SetParameters(pps...) - - require.EqualValues(t, len(pps), x.NumberOfParameters()) - - var dst []NetworkParameter - - x.IterateParameters(func(p *NetworkParameter) bool { - dst = append(dst, *p) - called++ - return false - }) - - require.Equal(t, pps, dst) - require.Equal(t, len(pps), called) -} - -func TestNewNetworkConfigFromV2(t *testing.T) { - t.Run("nil", func(t *testing.T) { - require.Nil(t, NewNetworkConfigFromV2(nil)) - }) -} - -func TestNetworkConfig_ToV2(t *testing.T) { - t.Run("nil", func(t *testing.T) { - var x *NetworkConfig - require.Nil(t, x.ToV2()) - }) -} - -func TestNewNetworkConfig(t *testing.T) { - x := NewNetworkConfig() - - // check initial values - require.Zero(t, x.NumberOfParameters()) - - // convert to v2 message - xV2 := x.ToV2() - - require.Zero(t, xV2.NumberOfParameters()) -} - func TestNetworkInfo_CurrentEpoch(t *testing.T) { - i := NewNetworkInfo() - e := uint64(13) + var x NetworkInfo - i.SetCurrentEpoch(e) + require.Zero(t, x.CurrentEpoch()) - require.Equal(t, e, i.CurrentEpoch()) - require.Equal(t, e, i.ToV2().GetCurrentEpoch()) + const e = 13 + + x.SetCurrentEpoch(e) + + require.EqualValues(t, e, x.CurrentEpoch()) + + var m netmap.NetworkInfo + x.WriteToV2(&m) + + require.EqualValues(t, e, m.GetCurrentEpoch()) } func TestNetworkInfo_MagicNumber(t *testing.T) { - i := NewNetworkInfo() - m := uint64(666) + var x NetworkInfo - i.SetMagicNumber(m) + require.Zero(t, x.MagicNumber()) - require.Equal(t, m, i.MagicNumber()) - require.Equal(t, m, i.ToV2().GetMagicNumber()) + const magic = 321 + + x.SetMagicNumber(magic) + + require.EqualValues(t, magic, x.MagicNumber()) + + var m netmap.NetworkInfo + x.WriteToV2(&m) + + require.EqualValues(t, magic, m.GetMagicNumber()) } func TestNetworkInfo_MsPerBlock(t *testing.T) { - i := NewNetworkInfo() + var x NetworkInfo - const ms = 987 + require.Zero(t, x.MsPerBlock()) - i.SetMsPerBlock(ms) + const ms = 789 - require.EqualValues(t, ms, i.MsPerBlock()) - require.EqualValues(t, ms, i.ToV2().GetMsPerBlock()) + x.SetMsPerBlock(ms) + + require.EqualValues(t, ms, x.MsPerBlock()) + + var m netmap.NetworkInfo + x.WriteToV2(&m) + + require.EqualValues(t, ms, m.GetMsPerBlock()) } -func TestNetworkInfo_Config(t *testing.T) { - i := NewNetworkInfo() +func testConfigValue(t *testing.T, + getter func(x NetworkInfo) interface{}, + setter func(x *NetworkInfo, val interface{}), + val1, val2 interface{}, + v2Key string, v2Val func(val interface{}) []byte, +) { + var x NetworkInfo - c := netmaptest.NetworkConfig() + require.Zero(t, getter(x)) - i.SetNetworkConfig(c) + checkVal := func(exp interface{}) { + require.EqualValues(t, exp, getter(x)) - require.Equal(t, c, i.NetworkConfig()) + var m netmap.NetworkInfo + x.WriteToV2(&m) + + require.EqualValues(t, 1, m.GetNetworkConfig().NumberOfParameters()) + found := false + m.GetNetworkConfig().IterateParameters(func(prm *netmap.NetworkParameter) bool { + require.False(t, found) + require.Equal(t, []byte(v2Key), prm.GetKey()) + require.Equal(t, v2Val(exp), prm.GetValue()) + found = true + return false + }) + require.True(t, found) + } + + setter(&x, val1) + checkVal(val1) + + setter(&x, val2) + checkVal(val2) } -func TestNetworkInfoEncoding(t *testing.T) { - i := netmaptest.NetworkInfo() - - t.Run("binary", func(t *testing.T) { - data, err := i.Marshal() - require.NoError(t, err) - - i2 := NewNetworkInfo() - require.NoError(t, i2.Unmarshal(data)) - - require.Equal(t, i, i2) - }) - - t.Run("json", func(t *testing.T) { - data, err := i.MarshalJSON() - require.NoError(t, err) - - i2 := NewNetworkInfo() - require.NoError(t, i2.UnmarshalJSON(data)) - - require.Equal(t, i, i2) - }) +func TestNetworkInfo_AuditFee(t *testing.T) { + testConfigValue(t, + func(x NetworkInfo) interface{} { return x.AuditFee() }, + func(info *NetworkInfo, val interface{}) { info.SetAuditFee(val.(uint64)) }, + uint64(1), uint64(2), + "AuditFee", func(val interface{}) []byte { + data := make([]byte, 8) + binary.LittleEndian.PutUint64(data, val.(uint64)) + return data + }, + ) } -func TestNewNetworkInfoFromV2(t *testing.T) { - t.Run("nil", func(t *testing.T) { - require.Nil(t, NewNetworkInfoFromV2(nil)) - }) +func TestNetworkInfo_StoragePrice(t *testing.T) { + testConfigValue(t, + func(x NetworkInfo) interface{} { return x.StoragePrice() }, + func(info *NetworkInfo, val interface{}) { info.SetStoragePrice(val.(uint64)) }, + uint64(1), uint64(2), + "BasicIncomeRate", func(val interface{}) []byte { + data := make([]byte, 8) + binary.LittleEndian.PutUint64(data, val.(uint64)) + return data + }, + ) } -func TestNetworkInfo_ToV2(t *testing.T) { - t.Run("nil", func(t *testing.T) { - var x *NetworkInfo - - require.Nil(t, x.ToV2()) - }) +func TestNetworkInfo_ContainerFee(t *testing.T) { + testConfigValue(t, + func(x NetworkInfo) interface{} { return x.ContainerFee() }, + func(info *NetworkInfo, val interface{}) { info.SetContainerFee(val.(uint64)) }, + uint64(1), uint64(2), + "ContainerFee", func(val interface{}) []byte { + data := make([]byte, 8) + binary.LittleEndian.PutUint64(data, val.(uint64)) + return data + }, + ) } -func TestNewNetworkInfo(t *testing.T) { - ni := NewNetworkInfo() - - // check initial values - require.Zero(t, ni.CurrentEpoch()) - require.Zero(t, ni.MagicNumber()) - require.Zero(t, ni.MsPerBlock()) - - // convert to v2 message - niV2 := ni.ToV2() - - require.Zero(t, niV2.GetCurrentEpoch()) - require.Zero(t, niV2.GetMagicNumber()) - require.Zero(t, niV2.GetMsPerBlock()) +func TestNetworkInfo_NamedContainerFee(t *testing.T) { + testConfigValue(t, + func(x NetworkInfo) interface{} { return x.NamedContainerFee() }, + func(info *NetworkInfo, val interface{}) { info.SetNamedContainerFee(val.(uint64)) }, + uint64(1), uint64(2), + "ContainerAliasFee", func(val interface{}) []byte { + data := make([]byte, 8) + binary.LittleEndian.PutUint64(data, val.(uint64)) + return data + }, + ) +} + +func TestNetworkInfo_EigenTrustAlpha(t *testing.T) { + testConfigValue(t, + func(x NetworkInfo) interface{} { return x.EigenTrustAlpha() }, + func(info *NetworkInfo, val interface{}) { info.SetEigenTrustAlpha(val.(float64)) }, + 0.1, 0.2, + "EigenTrustAlpha", func(val interface{}) []byte { + data := make([]byte, 8) + binary.LittleEndian.PutUint64(data, math.Float64bits(val.(float64))) + return data + }, + ) +} + +func TestNetworkInfo_EigenTrustIterationAmount(t *testing.T) { + testConfigValue(t, + func(x NetworkInfo) interface{} { return x.EigenTrustIterationAmount() }, + func(info *NetworkInfo, val interface{}) { info.SetEigenTrustIterationAmount(val.(uint64)) }, + uint64(1), uint64(2), + "EigenTrustIterations", func(val interface{}) []byte { + data := make([]byte, 8) + binary.LittleEndian.PutUint64(data, val.(uint64)) + return data + }, + ) +} + +func TestNetworkInfo_IRCandidateFee(t *testing.T) { + testConfigValue(t, + func(x NetworkInfo) interface{} { return x.IRCandidateFee() }, + func(info *NetworkInfo, val interface{}) { info.SetIRCandidateFee(val.(uint64)) }, + uint64(1), uint64(2), + "InnerRingCandidateFee", func(val interface{}) []byte { + data := make([]byte, 8) + binary.LittleEndian.PutUint64(data, val.(uint64)) + return data + }, + ) +} + +func TestNetworkInfo_MaxObjectSize(t *testing.T) { + testConfigValue(t, + func(x NetworkInfo) interface{} { return x.MaxObjectSize() }, + func(info *NetworkInfo, val interface{}) { info.SetMaxObjectSize(val.(uint64)) }, + uint64(1), uint64(2), + "MaxObjectSize", func(val interface{}) []byte { + data := make([]byte, 8) + binary.LittleEndian.PutUint64(data, val.(uint64)) + return data + }, + ) +} + +func TestNetworkInfo_WithdrawalFee(t *testing.T) { + testConfigValue(t, + func(x NetworkInfo) interface{} { return x.WithdrawalFee() }, + func(info *NetworkInfo, val interface{}) { info.SetWithdrawalFee(val.(uint64)) }, + uint64(1), uint64(2), + "WithdrawFee", func(val interface{}) []byte { + data := make([]byte, 8) + binary.LittleEndian.PutUint64(data, val.(uint64)) + return data + }, + ) } diff --git a/netmap/node_info.go b/netmap/node_info.go index a0cef19..b1dcdfb 100644 --- a/netmap/node_info.go +++ b/netmap/node_info.go @@ -1,377 +1,560 @@ package netmap import ( + "errors" + "fmt" + "sort" + "strconv" + "strings" + + "github.com/nspcc-dev/hrw" "github.com/nspcc-dev/neofs-api-go/v2/netmap" + "github.com/nspcc-dev/neofs-api-go/v2/refs" + subnetid "github.com/nspcc-dev/neofs-sdk-go/subnet/id" ) -// NodeState is an enumeration of various states of the NeoFS node. -type NodeState uint32 - -// NodeAttribute represents v2 compatible attribute of the NeoFS Storage Node. -type NodeAttribute netmap.Attribute - -// NodeInfo represents v2 compatible descriptor of the NeoFS node. +// NodeInfo groups information about NeoFS storage node which is reflected +// in the NeoFS network map. Storage nodes advertise this information when +// registering with the NeoFS network. After successful registration, information +// about the nodes is available to all network participants to work with the network +// map (mainly to comply with container storage policies). +// +// NodeInfo is mutually compatible with github.com/nspcc-dev/neofs-api-go/v2/netmap.NodeInfo +// message. See ReadFromV2 / WriteToV2 methods. +// +// Instances can be created using built-in var declaration. type NodeInfo struct { - priceAttr uint64 - - capAttr uint64 - - m *netmap.NodeInfo + m netmap.NodeInfo } -const ( - _ NodeState = iota +// reads NodeInfo from netmap.NodeInfo message. If checkFieldPresence is set, +// returns an error on absence of any protocol-required field. Verifies format of any +// presented field according to NeoFS API V2 protocol. +func (x *NodeInfo) readFromV2(m netmap.NodeInfo, checkFieldPresence bool) error { + var err error - // NodeStateOffline is network unavailable state. - NodeStateOffline - - // NodeStateOnline is an active state in the network. - NodeStateOnline -) - -// Enumeration of well-known attributes. -const ( - // AttrPrice is a key to the node attribute that indicates the - // price in GAS tokens for storing one GB of data during one Epoch. - AttrPrice = "Price" - - // AttrCapacity is a key to the node attribute that indicates the - // total available disk space in Gigabytes. - AttrCapacity = "Capacity" - - // AttrSubnet is a key to the node attribute that indicates the - // string ID of node's storage subnet. - AttrSubnet = "Subnet" - - // AttrUNLOCODE is a key to the node attribute that indicates the - // node's geographic location in UN/LOCODE format. - AttrUNLOCODE = "UN-LOCODE" - - // AttrCountryCode is a key to the node attribute that indicates the - // Country code in ISO 3166-1_alpha-2 format. - AttrCountryCode = "CountryCode" - - // AttrCountry is a key to the node attribute that indicates the - // country short name in English, as defined in ISO-3166. - AttrCountry = "Country" - - // AttrLocation is a key to the node attribute that indicates the - // place name of the node location. - AttrLocation = "Location" - - // AttrSubDivCode is a key to the node attribute that indicates the - // country's administrative subdivision where node is located - // in ISO 3166-2 format. - AttrSubDivCode = "SubDivCode" - - // AttrSubDiv is a key to the node attribute that indicates the - // country's administrative subdivision name, as defined in - // ISO 3166-2. - AttrSubDiv = "SubDiv" - - // AttrContinent is a key to the node attribute that indicates the - // node's continent name according to the Seven-Continent model. - AttrContinent = "Continent" -) - -// calcBucketWeight computes weight for a Bucket. -func calcBucketWeight(ns nodes, a aggregator, wf weightFunc) float64 { - for i := range ns { - a.Add(wf(ns[i])) + binPublicKey := m.GetPublicKey() + if checkFieldPresence && len(binPublicKey) == 0 { + return errors.New("missing public key") } - return a.Compute() -} - -// NodeStateFromV2 converts v2 NodeState to NodeState. -func NodeStateFromV2(s netmap.NodeState) NodeState { - switch s { - default: - return 0 - case netmap.Online: - return NodeStateOnline - case netmap.Offline: - return NodeStateOffline - } -} - -// ToV2 converts NodeState to v2 NodeState. -func (s NodeState) ToV2() netmap.NodeState { - switch s { - default: - return netmap.UnspecifiedState - case NodeStateOffline: - return netmap.Offline - case NodeStateOnline: - return netmap.Online - } -} - -// String returns string representation of NodeState. -// -// String mapping: -// * NodeStateOnline: ONLINE; -// * NodeStateOffline: OFFLINE; -// * default: UNSPECIFIED. -func (s NodeState) String() string { - return s.ToV2().String() -} - -// FromString parses NodeState from a string representation. -// It is a reverse action to String(). -// -// Returns true if s was parsed successfully. -func (s *NodeState) FromString(str string) bool { - var g netmap.NodeState - - ok := g.FromString(str) - - if ok { - *s = NodeStateFromV2(g) + if checkFieldPresence && m.NumberOfAddresses() <= 0 { + return errors.New("missing network endpoints") } - return ok -} + attributes := m.GetAttributes() + mAttr := make(map[string]struct{}, len(attributes)) + for i := range attributes { + key := attributes[i].GetKey() + if key == "" { + return fmt.Errorf("empty key of the attribute #%d", i) + } else if _, ok := mAttr[key]; ok { + return fmt.Errorf("duplicated attbiuted %s", key) + } -// NewNodeAttribute creates and returns new NodeAttribute instance. -// -// Defaults: -// - key: ""; -// - value: ""; -// - parents: nil. -func NewNodeAttribute() *NodeAttribute { - return NewNodeAttributeFromV2(new(netmap.Attribute)) -} + const subnetPrefix = "__NEOFS__SUBNET_" -// NodeAttributeFromV2 converts v2 node Attribute to NodeAttribute. -// -// Nil netmap.Attribute converts to nil. -func NewNodeAttributeFromV2(a *netmap.Attribute) *NodeAttribute { - return (*NodeAttribute)(a) -} + switch { + case key == attrCapacity: + _, err = strconv.ParseUint(attributes[i].GetValue(), 10, 64) + if err != nil { + return fmt.Errorf("invalid %s attribute: %w", attrCapacity, err) + } + case key == attrPrice: + var err error + _, err = strconv.ParseUint(attributes[i].GetValue(), 10, 64) + if err != nil { + return fmt.Errorf("invalid %s attribute: %w", attrPrice, err) + } + case strings.HasPrefix(key, subnetPrefix): + var id subnetid.ID -// ToV2 converts NodeAttribute to v2 node Attribute. -// -// Nil NodeAttribute converts to nil. -func (a *NodeAttribute) ToV2() *netmap.Attribute { - return (*netmap.Attribute)(a) -} + err = id.DecodeString(strings.TrimPrefix(key, subnetPrefix)) + if err != nil { + return fmt.Errorf("invalid key to the subnet attribute %s: %w", key, err) + } -// Key returns key to the node attribute. -func (a *NodeAttribute) Key() string { - return (*netmap.Attribute)(a). - GetKey() -} - -// SetKey sets key to the node attribute. -func (a *NodeAttribute) SetKey(key string) { - (*netmap.Attribute)(a). - SetKey(key) -} - -// Value returns value of the node attribute. -func (a *NodeAttribute) Value() string { - return (*netmap.Attribute)(a). - GetValue() -} - -// SetValue sets value of the node attribute. -func (a *NodeAttribute) SetValue(val string) { - (*netmap.Attribute)(a). - SetValue(val) -} - -// ParentKeys returns list of parent keys. -func (a *NodeAttribute) ParentKeys() []string { - return (*netmap.Attribute)(a). - GetParents() -} - -// SetParentKeys sets list of parent keys. -func (a *NodeAttribute) SetParentKeys(keys ...string) { - (*netmap.Attribute)(a). - SetParents(keys) -} - -// Marshal marshals NodeAttribute into a protobuf binary form. -func (a *NodeAttribute) Marshal() ([]byte, error) { - return (*netmap.Attribute)(a).StableMarshal(nil), nil -} - -// Unmarshal unmarshals protobuf binary representation of NodeAttribute. -func (a *NodeAttribute) Unmarshal(data []byte) error { - return (*netmap.Attribute)(a).Unmarshal(data) -} - -// MarshalJSON encodes NodeAttribute to protobuf JSON format. -func (a *NodeAttribute) MarshalJSON() ([]byte, error) { - return (*netmap.Attribute)(a).MarshalJSON() -} - -// UnmarshalJSON decodes NodeAttribute from protobuf JSON format. -func (a *NodeAttribute) UnmarshalJSON(data []byte) error { - return (*netmap.Attribute)(a).UnmarshalJSON(data) -} - -// NewNodeInfo creates and returns new NodeInfo instance. -// -// Defaults: -// - publicKey: nil; -// - address: ""; -// - attributes nil; -// - state: 0. -func NewNodeInfo() *NodeInfo { - return NewNodeInfoFromV2(new(netmap.NodeInfo)) -} - -// NewNodeInfoFromV2 converts v2 NodeInfo to NodeInfo. -// -// Nil netmap.NodeInfo converts to nil. -func NewNodeInfoFromV2(i *netmap.NodeInfo) *NodeInfo { - var res NodeInfo - res.m = i - res.syncAttributes() - - return &res -} - -// ToV2 converts NodeInfo to v2 NodeInfo. -// -// Nil NodeInfo converts to nil. -func (i *NodeInfo) ToV2() *netmap.NodeInfo { - if i == nil { - return nil + if val := attributes[i].GetValue(); val != "True" && val != "False" { + return fmt.Errorf("invalid value of the subnet attribute %s: %w", val, err) + } + default: + if attributes[i].GetValue() == "" { + return fmt.Errorf("empty value of the attribute #%d", i) + } + } } - return i.m + x.m = m + + return nil } -// PublicKey returns public key of the node in a binary format. -func (i *NodeInfo) PublicKey() []byte { - return i.m.GetPublicKey() +// ReadFromV2 reads NodeInfo from the netmap.NodeInfo message. Checks if the +// message conforms to NeoFS API V2 protocol. +// +// See also WriteToV2. +func (x *NodeInfo) ReadFromV2(m netmap.NodeInfo) error { + return x.readFromV2(m, true) } -// SetPublicKey sets public key of the node in a binary format. -func (i *NodeInfo) SetPublicKey(key []byte) { - if i.m == nil { - i.m = new(netmap.NodeInfo) +// WriteToV2 writes NodeInfo to the netmap.NodeInfo message. The message MUST NOT +// be nil. +// +// See also ReadFromV2. +func (x NodeInfo) WriteToV2(m *netmap.NodeInfo) { + *m = x.m +} + +// Marshal encodes NodeInfo into a binary format of the NeoFS API protocol +// (Protocol Buffers with direct field order). +// +// See also Unmarshal. +func (x NodeInfo) Marshal() []byte { + var m netmap.NodeInfo + x.WriteToV2(&m) + + return m.StableMarshal(nil) +} + +// Unmarshal decodes NeoFS API protocol binary format into the NodeInfo +// (Protocol Buffers with direct field order). Returns an error describing +// a format violation. +// +// See also Marshal. +func (x *NodeInfo) Unmarshal(data []byte) error { + var m netmap.NodeInfo + + err := m.Unmarshal(data) + if err != nil { + return err } - i.m.SetPublicKey(key) + return x.readFromV2(m, false) } -// NumberOfAddresses returns number of network addresses of the node. -func (i *NodeInfo) NumberOfAddresses() int { - return i.m.NumberOfAddresses() -} - -// IterateAddresses iterates over network addresses of the node. -// Breaks iteration on f's true return. +// MarshalJSON encodes NodeInfo into a JSON format of the NeoFS API protocol +// (Protocol Buffers JSON). // -// Handler should not be nil. -func (i *NodeInfo) IterateAddresses(f func(string) bool) { - i.m.IterateAddresses(f) +// See also UnmarshalJSON. +func (x NodeInfo) MarshalJSON() ([]byte, error) { + var m netmap.NodeInfo + x.WriteToV2(&m) + + return m.MarshalJSON() } -// IterateAllAddresses is a helper function to unconditionally -// iterate over all node addresses. -func IterateAllAddresses(i *NodeInfo, f func(string)) { - i.IterateAddresses(func(addr string) bool { +// UnmarshalJSON decodes NeoFS API protocol JSON format into the NodeInfo +// (Protocol Buffers JSON). Returns an error describing a format violation. +// +// See also MarshalJSON. +func (x *NodeInfo) UnmarshalJSON(data []byte) error { + var m netmap.NodeInfo + + err := m.UnmarshalJSON(data) + if err != nil { + return err + } + + return x.readFromV2(m, false) +} + +// SetPublicKey sets binary-encoded public key bound to the node. The key +// authenticates the storage node, so it MUST be unique within the network. +// +// Argument MUST NOT be mutated, make a copy first. +// +// See also PublicKey. +func (x *NodeInfo) SetPublicKey(key []byte) { + x.m.SetPublicKey(key) +} + +// PublicKey returns value set using SetPublicKey. +// +// Zero NodeInfo has no public key, which is incorrect according to +// NeoFS system requirements. +// +// Return value MUST not be mutated, make a copy first. +func (x NodeInfo) PublicKey() []byte { + return x.m.GetPublicKey() +} + +// SetNetworkEndpoints sets list to the announced node's network endpoints. +// Node MUSt have at least one announced endpoint. List MUST be unique. +// Endpoints are used for communication with the storage node within NeoFS +// network. It is expected that node serves storage node services on these +// endpoints (it also adds a wait on their network availability). +// +// Argument MUST NOT be mutated, make a copy first. +// +// See also IterateNetworkEndpoints. +func (x *NodeInfo) SetNetworkEndpoints(v ...string) { + x.m.SetAddresses(v...) +} + +// NumberOfNetworkEndpoints returns number of network endpoints announced by the node. +// +// See also SetNetworkEndpoints. +func (x NodeInfo) NumberOfNetworkEndpoints() int { + return x.m.NumberOfAddresses() +} + +// IterateNetworkEndpoints iterates over network endpoints announced by the +// node and pass them into f. Breaks iteration on f's true return. Handler +// MUST NOT be nil. +// +// Zero NodeInfo contains no endpoints which is incorrect according to +// NeoFS system requirements. +// +// See also SetNetworkEndpoints. +func (x NodeInfo) IterateNetworkEndpoints(f func(string) bool) { + x.m.IterateAddresses(f) +} + +// IterateNetworkEndpoints is an extra-sugared function over IterateNetworkEndpoints +// method which allows to unconditionally iterate over all node's network endpoints. +func IterateNetworkEndpoints(node NodeInfo, f func(string)) { + node.IterateNetworkEndpoints(func(addr string) bool { f(addr) return false }) } -// SetAddresses sets list of network addresses of the node. -func (i *NodeInfo) SetAddresses(v ...string) { - if i.m == nil { - i.m = new(netmap.NodeInfo) - } +// assert NodeInfo type provides hrw.Hasher required for HRW sorting. +var _ hrw.Hasher = NodeInfo{} - i.m.SetAddresses(v...) +// Hash implements hrw.Hasher interface. +// +// Hash is needed to support weighted HRW therefore sort function sorts nodes +// based on their public key. Hash isn't expected to be used directly. +func (x NodeInfo) Hash() uint64 { + return hrw.Hash(x.m.GetPublicKey()) } -// Attributes returns list of the node attributes. -func (i *NodeInfo) Attributes() []NodeAttribute { - if i == nil { - return nil - } - - as := i.m.GetAttributes() - - if as == nil { - return nil - } - - res := make([]NodeAttribute, len(as)) - - for ind := range as { - res[ind] = *NewNodeAttributeFromV2(&as[ind]) - } - - return res +// less declares "less than" comparison between two NodeInfo instances: +// x1 is less than x2 if it has less Hash(). +// +// Method is needed for internal placement needs. +func less(x1, x2 NodeInfo) bool { + return x1.Hash() < x2.Hash() } -// SetAttributes sets list of the node attributes. -func (i *NodeInfo) SetAttributes(as ...NodeAttribute) { - asV2 := make([]netmap.Attribute, len(as)) +func (x *NodeInfo) setNumericAttribute(key string, num uint64) { + x.SetAttribute(key, strconv.FormatUint(num, 10)) +} - for ind := range as { - asV2[ind] = *as[ind].ToV2() +// SetPrice sets the storage cost declared by the node. By default, zero +// price is announced. +func (x *NodeInfo) SetPrice(price uint64) { + x.setNumericAttribute(attrPrice, price) +} + +// Price returns price set using SetPrice. +// +// Zero NodeInfo has zero price. +func (x NodeInfo) Price() uint64 { + val := x.Attribute(attrPrice) + if val == "" { + return 0 } - if i.m == nil { - i.m = new(netmap.NodeInfo) - } - - i.m.SetAttributes(asV2) -} - -// State returns node state. -func (i *NodeInfo) State() NodeState { - return NodeStateFromV2(i.m.GetState()) -} - -// SetState sets node state. -func (i *NodeInfo) SetState(s NodeState) { - if i.m == nil { - i.m = new(netmap.NodeInfo) - } - - i.m.SetState(s.ToV2()) -} - -// Marshal marshals NodeInfo into a protobuf binary form. -func (i *NodeInfo) Marshal() ([]byte, error) { - return i.m.StableMarshal(nil), nil -} - -// Unmarshal unmarshals protobuf binary representation of NodeInfo. -func (i *NodeInfo) Unmarshal(data []byte) error { - if i.m == nil { - i.m = new(netmap.NodeInfo) - } - - return i.m.Unmarshal(data) -} - -// MarshalJSON encodes NodeInfo to protobuf JSON format. -func (i *NodeInfo) MarshalJSON() ([]byte, error) { - return i.m.MarshalJSON() -} - -// UnmarshalJSON decodes NodeInfo from protobuf JSON format. -func (i *NodeInfo) UnmarshalJSON(data []byte) error { - if i.m == nil { - i.m = new(netmap.NodeInfo) - } - - err := i.m.UnmarshalJSON(data) + price, err := strconv.ParseUint(val, 10, 64) if err != nil { - return err + panic(fmt.Sprintf("unexpected price parsing error %s: %v", val, err)) } - i.syncAttributes() - - return nil + return price +} + +// SetCapacity sets the storage capacity declared by the node. By default, zero +// capacity is announced. +func (x *NodeInfo) SetCapacity(capacity uint64) { + x.setNumericAttribute(attrCapacity, capacity) +} + +// capacity returns capacity set using SetCapacity. +// +// Zero NodeInfo has zero capacity. +func (x NodeInfo) capacity() uint64 { + val := x.Attribute(attrCapacity) + if val == "" { + return 0 + } + + capacity, err := strconv.ParseUint(val, 10, 64) + if err != nil { + panic(fmt.Sprintf("unexpected capacity parsing error %s: %v", val, err)) + } + + return capacity +} + +const attrUNLOCODE = "UN-LOCODE" + +// SetLOCODE specifies node's geographic location in UN/LOCODE format. Each +// storage node MUST declare it for entrance to the NeoFS network. Node MAY +// declare the code of the nearest location as needed, for example, when it is +// impossible to unambiguously attribute the node to any location from UN/LOCODE +// database. +// +// See also LOCODE. +func (x *NodeInfo) SetLOCODE(locode string) { + x.SetAttribute(attrUNLOCODE, locode) +} + +// LOCODE returns node's location code set using SetLOCODE. +// +// Zero NodeInfo has empty location code which is invalid according to +// NeoFS API system requirement. +func (x NodeInfo) LOCODE() string { + return x.Attribute(attrUNLOCODE) +} + +// SetCountryCode sets code of the country in ISO 3166-1_alpha-2 to which +// storage node belongs (or the closest one). +// +// SetCountryCode is intended only for processing the network registration +// request by the Inner Ring. Other parties SHOULD NOT use it. +func (x *NodeInfo) SetCountryCode(countryCode string) { + x.SetAttribute("CountryCode", countryCode) +} + +// SetCountryName sets short name of the country in ISO-3166 format to which +// storage node belongs (or the closest one). +// +// SetCountryName is intended only for processing the network registration +// request by the Inner Ring. Other parties SHOULD NOT use it. +func (x *NodeInfo) SetCountryName(country string) { + x.SetAttribute("Country", country) +} + +// SetLocationName sets storage node's location name from "NameWoDiacritics" +// column in the UN/LOCODE record corresponding to the specified LOCODE. +// +// SetLocationName is intended only for processing the network registration +// request by the Inner Ring. Other parties SHOULD NOT use it. +func (x *NodeInfo) SetLocationName(location string) { + x.SetAttribute("Location", location) +} + +// SetSubdivisionCode sets storage node's subdivision code from "SubDiv" column in +// the UN/LOCODE record corresponding to the specified LOCODE. +// +// SetSubdivisionCode is intended only for processing the network registration +// request by the Inner Ring. Other parties SHOULD NOT use it. +func (x *NodeInfo) SetSubdivisionCode(subDiv string) { + x.SetAttribute("SubDivCode", subDiv) +} + +// SetSubdivisionName sets storage node's subdivision name in ISO 3166-2 format. +// +// SetSubdivisionName is intended only for processing the network registration +// request by the Inner Ring. Other parties SHOULD NOT use it. +func (x *NodeInfo) SetSubdivisionName(subDiv string) { + x.SetAttribute("SubDiv", subDiv) +} + +// SetContinentName sets name of the storage node's continent from +// Seven-Continent model. +// +// SetContinentName is intended only for processing the network registration +// request by the Inner Ring. Other parties SHOULD NOT use it. +func (x *NodeInfo) SetContinentName(continent string) { + x.SetAttribute("Continent", continent) +} + +// Enumeration of well-known attributes. +const ( + // attrPrice is a key to the node attribute that indicates the + // price in GAS tokens for storing one GB of data during one Epoch. + attrPrice = "Price" + + // attrCapacity is a key to the node attribute that indicates the + // total available disk space in Gigabytes. + attrCapacity = "Capacity" +) + +// NumberOfAttributes returns number of attributes announced by the node. +// +// See also SetAttribute. +func (x NodeInfo) NumberOfAttributes() int { + return len(x.m.GetAttributes()) +} + +// IterateAttributes iterates over all node attributes and passes the into f. +// Handler MUST NOT be nil. +func (x NodeInfo) IterateAttributes(f func(key, value string)) { + a := x.m.GetAttributes() + for i := range a { + f(a[i].GetKey(), a[i].GetValue()) + } +} + +// SetAttribute sets value of the node attribute value by the given key. +// Both key and value MUST NOT be empty. +func (x *NodeInfo) SetAttribute(key, value string) { + if key == "" { + panic("empty key in SetAttribute") + } else if value == "" { + panic("empty value in SetAttribute") + } + + a := x.m.GetAttributes() + for i := range a { + if a[i].GetKey() == key { + a[i].SetValue(value) + return + } + } + + a = append(a, netmap.Attribute{}) + a[len(a)-1].SetKey(key) + a[len(a)-1].SetValue(value) + + x.m.SetAttributes(a) +} + +// Attribute returns value of the node attribute set using SetAttribute by the +// given key. Returns empty string if attribute is missing. +func (x NodeInfo) Attribute(key string) string { + a := x.m.GetAttributes() + for i := range a { + if a[i].GetKey() == key { + return a[i].GetValue() + } + } + + return "" +} + +// SortAttributes sorts node attributes set using SetAttribute lexicographically. +// The method is only needed to make NodeInfo consistent, e.g. for signing. +func (x *NodeInfo) SortAttributes() { + as := x.m.GetAttributes() + if len(as) == 0 { + return + } + + sort.Slice(as, func(i, j int) bool { + switch strings.Compare(as[i].GetKey(), as[j].GetKey()) { + case -1: + return true + case 1: + return false + default: + return as[i].GetValue() < as[j].GetValue() + } + }) + + x.m.SetAttributes(as) +} + +// EnterSubnet writes storage node's intention to enter the given subnet. +// +// Zero NodeInfo belongs to zero subnet. +func (x *NodeInfo) EnterSubnet(id subnetid.ID) { + x.changeSubnet(id, true) +} + +// ExitSubnet writes storage node's intention to exit the given subnet. +func (x *NodeInfo) ExitSubnet(id subnetid.ID) { + x.changeSubnet(id, false) +} + +func (x *NodeInfo) changeSubnet(id subnetid.ID, isMember bool) { + var ( + idv2 refs.SubnetID + info netmap.NodeSubnetInfo + ) + + id.WriteToV2(&idv2) + + info.SetID(&idv2) + info.SetEntryFlag(isMember) + + netmap.WriteSubnetInfo(&x.m, info) +} + +// ErrRemoveSubnet is returned when a node needs to leave the subnet. +var ErrRemoveSubnet = netmap.ErrRemoveSubnet + +// IterateSubnets iterates over all subnets the node belongs to and passes the IDs to f. +// Handler MUST NOT be nil. +// +// If f returns ErrRemoveSubnet, then removes subnet entry. Note that this leads to an +// instant mutation of NodeInfo. Breaks on any other non-nil error and returns it. +// +// Returns an error if subnet incorrectly enabled/disabled. +// Returns an error if the node is not included to any subnet by the end of the loop. +// +// See also EnterSubnet, ExitSubnet. +func (x NodeInfo) IterateSubnets(f func(subnetid.ID) error) error { + var id subnetid.ID + + return netmap.IterateSubnets(&x.m, func(idv2 refs.SubnetID) error { + err := id.ReadFromV2(idv2) + if err != nil { + return fmt.Errorf("invalid subnet: %w", err) + } + + err = f(id) + if errors.Is(err, ErrRemoveSubnet) { + return netmap.ErrRemoveSubnet + } + + return err + }) +} + +var errAbortSubnetIter = errors.New("abort subnet iterator") + +// BelongsToSubnet is a helper function over the IterateSubnets method which +// checks whether a node belongs to a subnet. +// +// Zero NodeInfo belongs to zero subnet only. +func BelongsToSubnet(node NodeInfo, id subnetid.ID) bool { + err := node.IterateSubnets(func(id_ subnetid.ID) error { + if id.Equals(id_) { + return errAbortSubnetIter + } + + return nil + }) + + return errors.Is(err, errAbortSubnetIter) +} + +// SetOffline sets the state of the node to "offline". When a node updates +// information about itself in the network map, this action is interpreted as +// an intention to leave the network. +func (x *NodeInfo) SetOffline() { + x.m.SetState(netmap.Offline) +} + +// IsOffline checks if the node is in the "offline" state. +// +// Zero NodeInfo has undefined state which is not offline (note that it does not +// mean online). +// +// See also SetOffline. +func (x NodeInfo) IsOffline() bool { + return x.m.GetState() == netmap.Offline +} + +// SetOnline sets the state of the node to "online". When a node updates +// information about itself in the network map, this +// action is interpreted as an intention to enter the network. +// +// See also IsOnline. +func (x *NodeInfo) SetOnline() { + x.m.SetState(netmap.Online) +} + +// IsOnline checks if the node is in the "online" state. +// +// Zero NodeInfo has undefined state which is not online (note that it does not +// mean offline). +// +// See also SetOnline. +func (x NodeInfo) IsOnline() bool { + return x.m.GetState() == netmap.Online } diff --git a/netmap/node_info_test.go b/netmap/node_info_test.go index 4b5bd3d..76194b7 100644 --- a/netmap/node_info_test.go +++ b/netmap/node_info_test.go @@ -3,261 +3,22 @@ package netmap import ( "testing" - "github.com/nspcc-dev/neofs-api-go/v2/netmap" - testv2 "github.com/nspcc-dev/neofs-api-go/v2/netmap/test" "github.com/stretchr/testify/require" ) -func TestNodeStateFromV2(t *testing.T) { - for _, item := range []struct { - s NodeState - sV2 netmap.NodeState - }{ - { - s: 0, - sV2: netmap.UnspecifiedState, - }, - { - s: NodeStateOnline, - sV2: netmap.Online, - }, - { - s: NodeStateOffline, - sV2: netmap.Offline, - }, - } { - require.Equal(t, item.s, NodeStateFromV2(item.sV2)) - require.Equal(t, item.sV2, item.s.ToV2()) - } -} +func TestNodeInfo_SetAttribute(t *testing.T) { + var n NodeInfo -func TestNodeAttributeFromV2(t *testing.T) { - t.Run("from nil", func(t *testing.T) { - var x *netmap.Attribute - - require.Nil(t, NewNodeAttributeFromV2(x)) - }) - - t.Run("from non-nil", func(t *testing.T) { - aV2 := testv2.GenerateAttribute(false) - - a := NewNodeAttributeFromV2(aV2) - - require.Equal(t, aV2, a.ToV2()) - }) -} - -func TestNodeAttribute_ToV2(t *testing.T) { - t.Run("nil", func(t *testing.T) { - var x *NodeAttribute - - require.Nil(t, x.ToV2()) - }) -} - -func TestNodeAttribute_Key(t *testing.T) { - a := NewNodeAttribute() - key := "some key" - - a.SetKey(key) - - require.Equal(t, key, a.Key()) -} - -func TestNodeAttribute_Value(t *testing.T) { - a := NewNodeAttribute() + const key = "some key" val := "some value" - a.SetValue(val) + require.Zero(t, n.Attribute(val)) - require.Equal(t, val, a.Value()) -} - -func TestNodeAttribute_ParentKeys(t *testing.T) { - a := NewNodeAttribute() - keys := []string{"par1", "par2"} - - a.SetParentKeys(keys...) - - require.Equal(t, keys, a.ParentKeys()) -} - -func testNodeAttribute() *NodeAttribute { - a := new(NodeAttribute) - a.SetKey("key") - a.SetValue("value") - a.SetParentKeys("par1", "par2") - - return a -} - -func TestNodeInfoFromV2(t *testing.T) { - t.Run("from nil", func(t *testing.T) { - var x *netmap.NodeInfo - - require.Nil(t, NewNodeInfoFromV2(x).m) - }) - - t.Run("from non-nil", func(t *testing.T) { - iV2 := testv2.GenerateNodeInfo(false) - - i := NewNodeInfoFromV2(iV2) - - require.Equal(t, iV2, i.ToV2()) - }) -} - -func TestNodeInfo_ToV2(t *testing.T) { - t.Run("nil", func(t *testing.T) { - var x *NodeInfo - - require.Nil(t, x.ToV2()) - }) -} - -func TestNodeInfo_PublicKey(t *testing.T) { - i := new(NodeInfo) - key := []byte{1, 2, 3} - - i.SetPublicKey(key) - - require.Equal(t, key, i.PublicKey()) -} - -func TestNodeInfo_IterateAddresses(t *testing.T) { - i := new(NodeInfo) - - as := []string{"127.0.0.1:8080", "127.0.0.1:8081"} - - i.SetAddresses(as...) - - as2 := make([]string, 0, i.NumberOfAddresses()) - - IterateAllAddresses(i, func(addr string) { - as2 = append(as2, addr) - }) - - require.Equal(t, as, as2) - require.EqualValues(t, len(as), i.NumberOfAddresses()) -} - -func TestNodeInfo_State(t *testing.T) { - i := new(NodeInfo) - s := NodeStateOnline - - i.SetState(s) - - require.Equal(t, s, i.State()) -} - -func TestNodeInfo_Attributes(t *testing.T) { - i := new(NodeInfo) - as := []NodeAttribute{*testNodeAttribute(), *testNodeAttribute()} - - i.SetAttributes(as...) - - require.Equal(t, as, i.Attributes()) -} - -func TestNodeAttributeEncoding(t *testing.T) { - a := testNodeAttribute() - - t.Run("binary", func(t *testing.T) { - data, err := a.Marshal() - require.NoError(t, err) - - a2 := NewNodeAttribute() - require.NoError(t, a2.Unmarshal(data)) - - require.Equal(t, a, a2) - }) - - t.Run("json", func(t *testing.T) { - data, err := a.MarshalJSON() - require.NoError(t, err) - - a2 := NewNodeAttribute() - require.NoError(t, a2.UnmarshalJSON(data)) - - require.Equal(t, a, a2) - }) -} - -func TestNodeInfoEncoding(t *testing.T) { - i := NewNodeInfo() - i.SetPublicKey([]byte{1, 2, 3}) - i.SetAddresses("192.168.0.1", "192.168.0.2") - i.SetState(NodeStateOnline) - i.SetAttributes(*testNodeAttribute()) - - t.Run("binary", func(t *testing.T) { - data, err := i.Marshal() - require.NoError(t, err) - - i2 := NewNodeInfo() - require.NoError(t, i2.Unmarshal(data)) - - require.Equal(t, i, i2) - }) - - t.Run("json", func(t *testing.T) { - data, err := i.MarshalJSON() - require.NoError(t, err) - - i2 := NewNodeInfo() - require.NoError(t, i2.UnmarshalJSON(data)) - - require.Equal(t, i, i2) - }) -} - -func TestNewNodeAttribute(t *testing.T) { - t.Run("default values", func(t *testing.T) { - attr := NewNodeAttribute() - - // check initial values - require.Empty(t, attr.Key()) - require.Empty(t, attr.Value()) - require.Nil(t, attr.ParentKeys()) - - // convert to v2 message - attrV2 := attr.ToV2() - - require.Empty(t, attrV2.GetKey()) - require.Empty(t, attrV2.GetValue()) - require.Nil(t, attrV2.GetParents()) - }) -} - -func TestNewNodeInfo(t *testing.T) { - t.Run("default values", func(t *testing.T) { - ni := NewNodeInfo() - - // check initial values - require.Nil(t, ni.PublicKey()) - - require.Zero(t, ni.NumberOfAddresses()) - require.Nil(t, ni.Attributes()) - require.Zero(t, ni.State()) - - // convert to v2 message - niV2 := ni.ToV2() - - require.Nil(t, niV2.GetPublicKey()) - require.Zero(t, niV2.NumberOfAddresses()) - require.Nil(t, niV2.GetAttributes()) - require.EqualValues(t, netmap.UnspecifiedState, niV2.GetState()) - }) -} - -func TestNodeState_String(t *testing.T) { - toPtr := func(v NodeState) *NodeState { - return &v - } - - testEnumStrings(t, new(NodeState), []enumStringItem{ - {val: toPtr(NodeStateOnline), str: "ONLINE"}, - {val: toPtr(NodeStateOffline), str: "OFFLINE"}, - {val: toPtr(0), str: "UNSPECIFIED"}, - }) + n.SetAttribute(key, val) + require.Equal(t, val, n.Attribute(key)) + + val = "some other value" + + n.SetAttribute(key, val) + require.Equal(t, val, n.Attribute(key)) } diff --git a/netmap/operation.go b/netmap/operation.go deleted file mode 100644 index af3e042..0000000 --- a/netmap/operation.go +++ /dev/null @@ -1,116 +0,0 @@ -package netmap - -import ( - "github.com/nspcc-dev/neofs-api-go/v2/netmap" -) - -// Operation is an enumeration of v2-compatible filtering operations. -type Operation uint32 - -const ( - _ Operation = iota - - // OpEQ is an "Equal" operation. - OpEQ - - // OpNE is a "Not equal" operation. - OpNE - - // OpGT is a "Greater than" operation. - OpGT - - // OpGE is a "Greater than or equal to" operation. - OpGE - - // OpLT is a "Less than" operation. - OpLT - - // OpLE is a "Less than or equal to" operation. - OpLE - - // OpOR is an "OR" operation. - OpOR - - // OpAND is an "AND" operation. - OpAND -) - -// OperationFromV2 converts v2 Operation to Operation. -func OperationFromV2(op netmap.Operation) Operation { - switch op { - default: - return 0 - case netmap.OR: - return OpOR - case netmap.AND: - return OpAND - case netmap.GE: - return OpGE - case netmap.GT: - return OpGT - case netmap.LE: - return OpLE - case netmap.LT: - return OpLT - case netmap.EQ: - return OpEQ - case netmap.NE: - return OpNE - } -} - -// ToV2 converts Operation to v2 Operation. -func (op Operation) ToV2() netmap.Operation { - switch op { - default: - return netmap.UnspecifiedOperation - case OpOR: - return netmap.OR - case OpAND: - return netmap.AND - case OpGE: - return netmap.GE - case OpGT: - return netmap.GT - case OpLE: - return netmap.LE - case OpLT: - return netmap.LT - case OpEQ: - return netmap.EQ - case OpNE: - return netmap.NE - } -} - -// String returns string representation of Operation. -// -// String mapping: -// * OpNE: NE; -// * OpEQ: EQ; -// * OpLT: LT; -// * OpLE: LE; -// * OpGT: GT; -// * OpGE: GE; -// * OpAND: AND; -// * OpOR: OR; -// * default: OPERATION_UNSPECIFIED. -func (op Operation) String() string { - return op.ToV2().String() -} - -// FromString parses Operation from a string representation. -// It is a reverse action to String(). -// -// Returns true if s was parsed successfully. -func (op *Operation) FromString(s string) bool { - var g netmap.Operation - - ok := g.FromString(s) - - if ok { - *op = OperationFromV2(g) - } - - return ok -} diff --git a/netmap/operation_test.go b/netmap/operation_test.go deleted file mode 100644 index e8b74e3..0000000 --- a/netmap/operation_test.go +++ /dev/null @@ -1,73 +0,0 @@ -package netmap - -import ( - "testing" - - "github.com/nspcc-dev/neofs-api-go/v2/netmap" - "github.com/stretchr/testify/require" -) - -func TestOperationFromV2(t *testing.T) { - for _, item := range []struct { - op Operation - opV2 netmap.Operation - }{ - { - op: 0, - opV2: netmap.UnspecifiedOperation, - }, - { - op: OpEQ, - opV2: netmap.EQ, - }, - { - op: OpNE, - opV2: netmap.NE, - }, - { - op: OpOR, - opV2: netmap.OR, - }, - { - op: OpAND, - opV2: netmap.AND, - }, - { - op: OpLE, - opV2: netmap.LE, - }, - { - op: OpLT, - opV2: netmap.LT, - }, - { - op: OpGT, - opV2: netmap.GT, - }, - { - op: OpGE, - opV2: netmap.GE, - }, - } { - require.Equal(t, item.op, OperationFromV2(item.opV2)) - require.Equal(t, item.opV2, item.op.ToV2()) - } -} - -func TestOperation_String(t *testing.T) { - toPtr := func(v Operation) *Operation { - return &v - } - - testEnumStrings(t, new(Operation), []enumStringItem{ - {val: toPtr(OpEQ), str: "EQ"}, - {val: toPtr(OpNE), str: "NE"}, - {val: toPtr(OpGT), str: "GT"}, - {val: toPtr(OpGE), str: "GE"}, - {val: toPtr(OpLT), str: "LT"}, - {val: toPtr(OpLE), str: "LE"}, - {val: toPtr(OpAND), str: "AND"}, - {val: toPtr(OpOR), str: "OR"}, - {val: toPtr(0), str: "OPERATION_UNSPECIFIED"}, - }) -} diff --git a/policy/parser/Query.g4 b/netmap/parser/Query.g4 similarity index 100% rename from policy/parser/Query.g4 rename to netmap/parser/Query.g4 diff --git a/policy/parser/Query.interp b/netmap/parser/Query.interp similarity index 100% rename from policy/parser/Query.interp rename to netmap/parser/Query.interp diff --git a/policy/parser/Query.tokens b/netmap/parser/Query.tokens similarity index 100% rename from policy/parser/Query.tokens rename to netmap/parser/Query.tokens diff --git a/policy/parser/QueryLexer.g4 b/netmap/parser/QueryLexer.g4 similarity index 100% rename from policy/parser/QueryLexer.g4 rename to netmap/parser/QueryLexer.g4 diff --git a/policy/parser/QueryLexer.interp b/netmap/parser/QueryLexer.interp similarity index 100% rename from policy/parser/QueryLexer.interp rename to netmap/parser/QueryLexer.interp diff --git a/policy/parser/QueryLexer.tokens b/netmap/parser/QueryLexer.tokens similarity index 100% rename from policy/parser/QueryLexer.tokens rename to netmap/parser/QueryLexer.tokens diff --git a/policy/parser/generate.go b/netmap/parser/generate.go similarity index 100% rename from policy/parser/generate.go rename to netmap/parser/generate.go diff --git a/policy/parser/query_base_listener.go b/netmap/parser/query_base_listener.go similarity index 100% rename from policy/parser/query_base_listener.go rename to netmap/parser/query_base_listener.go diff --git a/policy/parser/query_base_visitor.go b/netmap/parser/query_base_visitor.go similarity index 100% rename from policy/parser/query_base_visitor.go rename to netmap/parser/query_base_visitor.go diff --git a/policy/parser/query_lexer.go b/netmap/parser/query_lexer.go similarity index 100% rename from policy/parser/query_lexer.go rename to netmap/parser/query_lexer.go diff --git a/policy/parser/query_listener.go b/netmap/parser/query_listener.go similarity index 100% rename from policy/parser/query_listener.go rename to netmap/parser/query_listener.go diff --git a/policy/parser/query_parser.go b/netmap/parser/query_parser.go similarity index 100% rename from policy/parser/query_parser.go rename to netmap/parser/query_parser.go diff --git a/policy/parser/query_visitor.go b/netmap/parser/query_visitor.go similarity index 100% rename from policy/parser/query_visitor.go rename to netmap/parser/query_visitor.go diff --git a/netmap/policy.go b/netmap/policy.go index 5e23a9e..5219bf8 100644 --- a/netmap/policy.go +++ b/netmap/policy.go @@ -1,176 +1,787 @@ package netmap import ( + "errors" + "fmt" + "io" + "strconv" + "strings" + + "github.com/antlr/antlr4/runtime/Go/antlr" "github.com/nspcc-dev/neofs-api-go/v2/netmap" "github.com/nspcc-dev/neofs-api-go/v2/refs" + "github.com/nspcc-dev/neofs-sdk-go/netmap/parser" subnetid "github.com/nspcc-dev/neofs-sdk-go/subnet/id" ) -// PlacementPolicy represents v2-compatible placement policy. -type PlacementPolicy netmap.PlacementPolicy - -// NewPlacementPolicy creates and returns new PlacementPolicy instance. +// PlacementPolicy declares policy to store objects in the NeoFS container. +// Within itself, PlacementPolicy represents a set of rules to select a subset +// of nodes from NeoFS network map - node-candidates for object storage. // -// Defaults: -// - backupFactor: 0; -// - replicas nil; -// - selectors nil; -// - filters nil. -func NewPlacementPolicy() *PlacementPolicy { - return NewPlacementPolicyFromV2(new(netmap.PlacementPolicy)) -} - -// NewPlacementPolicyFromV2 converts v2 PlacementPolicy to PlacementPolicy. +// PlacementPolicy is mutually compatible with github.com/nspcc-dev/neofs-api-go/v2/netmap.PlacementPolicy +// message. See ReadFromV2 / WriteToV2 methods. // -// Nil netmap.PlacementPolicy converts to nil. -func NewPlacementPolicyFromV2(f *netmap.PlacementPolicy) *PlacementPolicy { - return (*PlacementPolicy)(f) +// Instances can be created using built-in var declaration. +type PlacementPolicy struct { + backupFactor uint32 + + subnet subnetid.ID + + filters []netmap.Filter + + selectors []netmap.Selector + + replicas []netmap.Replica } -// ToV2 converts PlacementPolicy to v2 PlacementPolicy. -// -// Nil PlacementPolicy converts to nil. -func (p *PlacementPolicy) ToV2() *netmap.PlacementPolicy { - return (*netmap.PlacementPolicy)(p) -} +func (p *PlacementPolicy) readFromV2(m netmap.PlacementPolicy) error { + p.backupFactor = m.GetContainerBackupFactor() + p.filters = m.GetFilters() + p.selectors = m.GetSelectors() + p.replicas = m.GetReplicas() -// SubnetID returns subnet to select nodes from. -func (p *PlacementPolicy) SubnetID() *subnetid.ID { - idv2 := (*netmap.PlacementPolicy)(p).GetSubnetID() - if idv2 == nil { - return nil - } - - var id subnetid.ID - - err := id.ReadFromV2(*idv2) - if err != nil { - panic(err) // will disappear after netmap package refactor - } - - return &id -} - -// SetSubnetID sets subnet to select nodes from. -func (p *PlacementPolicy) SetSubnetID(subnet *subnetid.ID) { - var idv2 *refs.SubnetID - - if subnet != nil { - idv2 = new(refs.SubnetID) - subnet.WriteToV2(idv2) - } - - (*netmap.PlacementPolicy)(p).SetSubnetID(idv2) -} - -// Replicas returns list of object replica descriptors. -func (p *PlacementPolicy) Replicas() []Replica { - rs := (*netmap.PlacementPolicy)(p). - GetReplicas() - - if rs == nil { - return nil - } - - res := make([]Replica, len(rs)) - - for i := range rs { - res[i] = *NewReplicaFromV2(&rs[i]) - } - - return res -} - -// SetReplicas sets list of object replica descriptors. -func (p *PlacementPolicy) SetReplicas(rs ...Replica) { - var rsV2 []netmap.Replica - - if rs != nil { - rsV2 = make([]netmap.Replica, len(rs)) - - for i := range rs { - rsV2[i] = *rs[i].ToV2() + subnetV2 := m.GetSubnetID() + if subnetV2 != nil { + err := p.subnet.ReadFromV2(*subnetV2) + if err != nil { + return fmt.Errorf("invalid subnet: %w", err) } + } else { + p.subnet = subnetid.ID{} } - (*netmap.PlacementPolicy)(p).SetReplicas(rsV2) -} - -// ContainerBackupFactor returns container backup factor. -func (p *PlacementPolicy) ContainerBackupFactor() uint32 { - return (*netmap.PlacementPolicy)(p). - GetContainerBackupFactor() -} - -// SetContainerBackupFactor sets container backup factor. -func (p *PlacementPolicy) SetContainerBackupFactor(f uint32) { - (*netmap.PlacementPolicy)(p). - SetContainerBackupFactor(f) -} - -// Selector returns set of selectors to form the container's nodes subset. -func (p *PlacementPolicy) Selectors() []Selector { - rs := (*netmap.PlacementPolicy)(p). - GetSelectors() - - if rs == nil { - return nil - } - - res := make([]Selector, len(rs)) - - for i := range rs { - res[i] = *NewSelectorFromV2(&rs[i]) - } - - return res -} - -// SetSelectors sets set of selectors to form the container's nodes subset. -func (p *PlacementPolicy) SetSelectors(ss ...Selector) { - var ssV2 []netmap.Selector - - if ss != nil { - ssV2 = make([]netmap.Selector, len(ss)) - - for i := range ss { - ssV2[i] = *ss[i].ToV2() - } - } - - (*netmap.PlacementPolicy)(p).SetSelectors(ssV2) -} - -// Filters returns list of named filters to reference in selectors. -func (p *PlacementPolicy) Filters() []Filter { - return filtersFromV2( - (*netmap.PlacementPolicy)(p). - GetFilters(), - ) -} - -// SetFilters sets list of named filters to reference in selectors. -func (p *PlacementPolicy) SetFilters(fs ...Filter) { - (*netmap.PlacementPolicy)(p). - SetFilters(filtersToV2(fs)) -} - -// Marshal marshals PlacementPolicy into a protobuf binary form. -func (p *PlacementPolicy) Marshal() ([]byte, error) { - return (*netmap.PlacementPolicy)(p).StableMarshal(nil), nil -} - -// Unmarshal unmarshals protobuf binary representation of PlacementPolicy. -func (p *PlacementPolicy) Unmarshal(data []byte) error { - return (*netmap.PlacementPolicy)(p).Unmarshal(data) -} - -// MarshalJSON encodes PlacementPolicy to protobuf JSON format. -func (p *PlacementPolicy) MarshalJSON() ([]byte, error) { - return (*netmap.PlacementPolicy)(p).MarshalJSON() + return nil } // UnmarshalJSON decodes PlacementPolicy from protobuf JSON format. func (p *PlacementPolicy) UnmarshalJSON(data []byte) error { - return (*netmap.PlacementPolicy)(p).UnmarshalJSON(data) + var m netmap.PlacementPolicy + + err := m.UnmarshalJSON(data) + if err != nil { + return err + } + + return p.readFromV2(m) +} + +// ReadFromV2 reads PlacementPolicy from the netmap.PlacementPolicy message. +// Checks if the message conforms to NeoFS API V2 protocol. +// +// See also WriteToV2. +func (p *PlacementPolicy) ReadFromV2(m netmap.PlacementPolicy) error { + return p.readFromV2(m) +} + +// WriteToV2 writes PlacementPolicy to the session.Token message. +// The message must not be nil. +// +// See also ReadFromV2. +func (p PlacementPolicy) WriteToV2(m *netmap.PlacementPolicy) { + var subnetV2 refs.SubnetID + p.subnet.WriteToV2(&subnetV2) + + m.SetContainerBackupFactor(p.backupFactor) + m.SetSubnetID(&subnetV2) + m.SetFilters(p.filters) + m.SetSelectors(p.selectors) + m.SetReplicas(p.replicas) +} + +// RestrictSubnet sets a rule to select nodes from the given subnet only. +// By default, nodes from zero subnet are selected (whole network map). +func (p *PlacementPolicy) RestrictSubnet(subnet subnetid.ID) { + p.subnet = subnet +} + +// Subnet returns subnet set using RestrictSubnet. +// +// Zero PlacementPolicy returns zero subnet meaning unlimited. +func (p PlacementPolicy) Subnet() subnetid.ID { + return p.subnet +} + +// ReplicaDescriptor replica descriptor characterizes replicas of objects from +// the subset selected by a particular Selector. +type ReplicaDescriptor struct { + m netmap.Replica +} + +// SetAmount sets number of object replicas. +func (r *ReplicaDescriptor) SetAmount(c uint32) { + r.m.SetCount(c) +} + +// Amount returns number set using SetAmount. +// +// Zero ReplicaDescriptor has zero object amount. +func (r ReplicaDescriptor) Amount() uint32 { + return r.m.GetCount() +} + +// SetSelectorName sets name of the related Selector. +// +// Zero ReplicaDescriptor references to the root bucket's selector: it contains +// all possible nodes to store the object. +func (r *ReplicaDescriptor) SetSelectorName(s string) { + r.m.SetSelector(s) +} + +// AddReplicas adds a bunch object replica's characteristics. +// +// Zero PlacementPolicy does not declare replicas ???. +// +// See also IterateReplicas. +func (p *PlacementPolicy) AddReplicas(rs ...ReplicaDescriptor) { + off := len(p.replicas) + + p.replicas = append(p.replicas, make([]netmap.Replica, len(rs))...) + + for i := range rs { + p.replicas[off+i] = rs[i].m + } +} + +// NumberOfReplicas returns amount of replica descriptors set using AddReplicas. +// +// Zero PlacementPolicy has no replicas. +func (p PlacementPolicy) NumberOfReplicas() int { + return len(p.replicas) +} + +// ReplicaAmountByIndex returns amount of object replicas from the i-th replica +// descriptor. Index MUST be in range [0; NumberOfReplicas()). +// +// Zero PlacementPolicy has no replicas. +func (p PlacementPolicy) ReplicaAmountByIndex(i int) uint32 { + return p.replicas[i].GetCount() +} + +// SetContainerBackupFactor sets container backup factor: it controls how deep +// NeoFS will search for nodes alternatives to include into container's nodes subset. +// +// Zero PlacementPolicy has zero container backup factor. +func (p *PlacementPolicy) SetContainerBackupFactor(f uint32) { + p.backupFactor = f +} + +// Selector describes the bucket selection operator: choose a number of nodes +// from the bucket taking the nearest nodes to the related container by hash distance. +type Selector struct { + m netmap.Selector +} + +// SetName sets name with which the Selector can be referenced. +// +// Zero Selector is unnamed. +func (s *Selector) SetName(name string) { + s.m.SetName(name) +} + +// SetNodeAmount sets number of nodes to select from the bucket. +// +// Zero Selector selects nothing. +func (s *Selector) SetNodeAmount(amount uint32) { + s.m.SetCount(amount) +} + +// SelectByBucketAttribute sets attribute of the bucket to select nodes from. +// +// Zero Selector ???. +func (s *Selector) SelectByBucketAttribute(bucket string) { + s.m.SetAttribute(bucket) +} + +// SelectSame makes selection algorithm to select only nodes having the same values +// of the bucket attribute. +// +// Zero Selector doesn't specify selection modifier so nodes are selected randomly. +// +// See also SelectByBucketAttribute. +func (s *Selector) SelectSame() { + s.m.SetClause(netmap.Same) +} + +// SelectDistinct makes selection algorithm to select only nodes having the different values +// of the bucket attribute. +// +// Zero Selector doesn't specify selection modifier so nodes are selected randomly. +// +// See also SelectByBucketAttribute. +func (s *Selector) SelectDistinct() { + s.m.SetClause(netmap.Distinct) +} + +// SetFilterName sets name of filter to reference to select from ???. +// +// Zero Selector ???. +func (s *Selector) SetFilterName(f string) { + s.m.SetFilter(f) +} + +// AddSelectors adds a Selector bunch to form the subset of the nodes +// to store container objects. +// +// Zero PlacementPolicy does not declare replicas ???. +func (p *PlacementPolicy) AddSelectors(ss ...Selector) { + off := len(p.selectors) + + p.selectors = append(p.selectors, make([]netmap.Selector, len(ss))...) + + for i := range ss { + p.selectors[off+i] = ss[i].m + } +} + +// Filter contains rules for filtering the node sets. +type Filter struct { + m netmap.Filter +} + +// SetName sets name with which the Filter can be referenced or, for inner filters, +// to which the Filter references. Top-level filters MUST have be named. The name +// MUST NOT be '*'. +// +// Zero Filter is unnamed. +func (x *Filter) SetName(name string) { + x.m.SetName(name) +} + +func (x *Filter) setAttribute(key string, op netmap.Operation, val string) { + x.m.SetKey(key) + x.m.SetOp(op) + x.m.SetValue(val) +} + +// Equal applies the rule to accept only nodes with the same attribute value. +// +// Method SHOULD NOT be called along with other similar methods. +func (x *Filter) Equal(key, value string) { + x.setAttribute(key, netmap.EQ, value) +} + +// NotEqual applies the rule to accept only nodes with the distinct attribute value. +// +// Method SHOULD NOT be called along with other similar methods. +func (x *Filter) NotEqual(key, value string) { + x.setAttribute(key, netmap.NE, value) +} + +// NumericGT applies the rule to accept only nodes with the numeric attribute +// greater than given number. +// +// Method SHOULD NOT be called along with other similar methods. +func (x *Filter) NumericGT(key string, num int64) { + x.setAttribute(key, netmap.GT, strconv.FormatInt(num, 10)) +} + +// NumericGE applies the rule to accept only nodes with the numeric attribute +// greater than or equal to given number. +// +// Method SHOULD NOT be called along with other similar methods. +func (x *Filter) NumericGE(key string, num int64) { + x.setAttribute(key, netmap.GE, strconv.FormatInt(num, 10)) +} + +// NumericLT applies the rule to accept only nodes with the numeric attribute +// less than given number. +// +// Method SHOULD NOT be called along with other similar methods. +func (x *Filter) NumericLT(key string, num int64) { + x.setAttribute(key, netmap.LT, strconv.FormatInt(num, 10)) +} + +// NumericLE applies the rule to accept only nodes with the numeric attribute +// less than or equal to given number. +// +// Method SHOULD NOT be called along with other similar methods. +func (x *Filter) NumericLE(key string, num int64) { + x.setAttribute(key, netmap.LE, strconv.FormatInt(num, 10)) +} + +func (x *Filter) setInnerFilters(op netmap.Operation, filters []Filter) { + x.setAttribute("", op, "") + + inner := x.m.GetFilters() + if rem := len(filters) - len(inner); rem > 0 { + inner = append(inner, make([]netmap.Filter, rem)...) + } + + for i := range filters { + inner[i] = filters[i].m + } + + x.m.SetFilters(inner) +} + +// LogicalOR applies the rule to accept only nodes which satisfy at least one +// of the given filters. +// +// Method SHOULD NOT be called along with other similar methods. +func (x *Filter) LogicalOR(filters ...Filter) { + x.setInnerFilters(netmap.OR, filters) +} + +// LogicalAND applies the rule to accept only nodes which satisfy all the given +// filters. +// +// Method SHOULD NOT be called along with other similar methods. +func (x *Filter) LogicalAND(filters ...Filter) { + x.setInnerFilters(netmap.AND, filters) +} + +// AddFilters adds a Filter bunch that will be applied when selecting nodes. +// +// Zero PlacementPolicy has no filters ???. +func (p *PlacementPolicy) AddFilters(fs ...Filter) { + off := len(p.filters) + + p.filters = append(p.filters, make([]netmap.Filter, len(fs))...) + + for i := range fs { + p.filters[off+i] = fs[i].m + } +} + +// WriteStringTo encodes PlacementPolicy into human-readably query and writes +// the result into w. Returns w's errors directly. +// +// See also DecodeString. +func (p PlacementPolicy) WriteStringTo(w io.StringWriter) (err error) { + writtenSmth := false + + writeLnIfNeeded := func() error { + if writtenSmth { + _, err = w.WriteString("\n") + return err + } + + writtenSmth = true + + return nil + } + + for i := range p.replicas { + err = writeLnIfNeeded() + if err != nil { + return err + } + + c := p.replicas[i].GetCount() + s := p.replicas[i].GetSelector() + + if s != "" { + _, err = w.WriteString(fmt.Sprintf("REP %d IN %s", c, s)) + } else { + _, err = w.WriteString(fmt.Sprintf("REP %d", c)) + } + + if err != nil { + return err + } + } + + if p.backupFactor > 0 { + err = writeLnIfNeeded() + if err != nil { + return err + } + + _, err = w.WriteString(fmt.Sprintf("CBF %d", p.backupFactor)) + if err != nil { + return err + } + } + + var s string + + for i := range p.selectors { + err = writeLnIfNeeded() + if err != nil { + return err + } + + _, err = w.WriteString(fmt.Sprintf("SELECT %d", p.selectors[i].GetCount())) + if err != nil { + return err + } + + if s = p.selectors[i].GetAttribute(); s != "" { + var clause string + + switch p.selectors[i].GetClause() { + case netmap.Same: + clause = "SAME " + case netmap.Distinct: + clause = "DISTINCT " + default: + clause = "" + } + + _, err = w.WriteString(fmt.Sprintf(" IN %s%s", clause, s)) + if err != nil { + return err + } + } + + if s = p.selectors[i].GetFilter(); s != "" { + _, err = w.WriteString(" FROM " + s) + if err != nil { + return err + } + } + + if s = p.selectors[i].GetName(); s != "" { + _, err = w.WriteString(" AS " + s) + if err != nil { + return err + } + } + } + + for i := range p.filters { + err = writeLnIfNeeded() + if err != nil { + return err + } + + _, err = w.WriteString("FILTER ") + if err != nil { + return err + } + + err = writeFilterStringTo(w, p.filters[i]) + if err != nil { + return err + } + } + + return nil +} + +func writeFilterStringTo(w io.StringWriter, f netmap.Filter) error { + var err error + var s string + op := f.GetOp() + unspecified := op == 0 + + if s = f.GetKey(); s != "" { + _, err = w.WriteString(fmt.Sprintf("%s %s %s", s, op, f.GetValue())) + if err != nil { + return err + } + } else if s = f.GetName(); unspecified && s != "" { + _, err = w.WriteString(fmt.Sprintf("@%s", s)) + if err != nil { + return err + } + } + + inner := f.GetFilters() + for i := range inner { + if i != 0 { + _, err = w.WriteString(" " + op.String() + " ") + if err != nil { + return err + } + } + + err = writeFilterStringTo(w, inner[i]) + if err != nil { + return err + } + } + + if s = f.GetName(); s != "" && !unspecified { + _, err = w.WriteString(" AS " + s) + if err != nil { + return err + } + } + + return nil +} + +// DecodeString decodes PlacementPolicy from the string composed using +// WriteStringTo. Returns error if s is malformed. +func (p *PlacementPolicy) DecodeString(s string) error { + input := antlr.NewInputStream(s) + lexer := parser.NewQueryLexer(input) + stream := antlr.NewCommonTokenStream(lexer, 0) + + pp := parser.NewQuery(stream) + pp.BuildParseTrees = true + + var v policyVisitor + + pp.RemoveErrorListeners() + pp.AddErrorListener(&v) + pl := pp.Policy().Accept(&v) + + if len(v.errors) != 0 { + return v.errors[0] + } + + parsed, ok := pl.(*PlacementPolicy) + if !ok { + return fmt.Errorf("unexpected parsed instance type %T", pl) + } else if parsed == nil { + return errors.New("parsed nil value") + } + + if err := validatePolicy(*p); err != nil { + return fmt.Errorf("invalid policy: %w", err) + } + + *p = *parsed + + return nil +} + +var ( + // errUnknownFilter is returned when a value of FROM in a query is unknown. + errUnknownFilter = errors.New("filter not found") + // errUnknownSelector is returned when a value of IN is unknown. + errUnknownSelector = errors.New("policy: selector not found") + // errSyntaxError is returned for errors found by ANTLR parser. + errSyntaxError = errors.New("policy: syntax error") +) + +type policyVisitor struct { + errors []error + parser.BaseQueryVisitor + antlr.DefaultErrorListener +} + +func (p *policyVisitor) SyntaxError(_ antlr.Recognizer, _ interface{}, line, column int, msg string, _ antlr.RecognitionException) { + p.reportError(fmt.Errorf("%w: line %d:%d %s", errSyntaxError, line, column, msg)) +} + +func (p *policyVisitor) reportError(err error) interface{} { + p.errors = append(p.errors, err) + return nil +} + +// VisitPolicy implements parser.QueryVisitor interface. +func (p *policyVisitor) VisitPolicy(ctx *parser.PolicyContext) interface{} { + if len(p.errors) != 0 { + return nil + } + + pl := new(PlacementPolicy) + repStmts := ctx.AllRepStmt() + pl.replicas = make([]netmap.Replica, 0, len(repStmts)) + + for _, r := range repStmts { + res, ok := r.Accept(p).(*netmap.Replica) + if !ok { + return nil + } + + pl.replicas = append(pl.replicas, *res) + } + + if cbfStmt := ctx.CbfStmt(); cbfStmt != nil { + cbf, ok := cbfStmt.(*parser.CbfStmtContext).Accept(p).(uint32) + if !ok { + return nil + } + pl.SetContainerBackupFactor(cbf) + } + + selStmts := ctx.AllSelectStmt() + pl.selectors = make([]netmap.Selector, 0, len(selStmts)) + + for _, s := range selStmts { + res, ok := s.Accept(p).(*netmap.Selector) + if !ok { + return nil + } + + pl.selectors = append(pl.selectors, *res) + } + + filtStmts := ctx.AllFilterStmt() + pl.filters = make([]netmap.Filter, 0, len(filtStmts)) + + for _, f := range filtStmts { + pl.filters = append(pl.filters, *f.Accept(p).(*netmap.Filter)) + } + + return pl +} + +func (p *policyVisitor) VisitCbfStmt(ctx *parser.CbfStmtContext) interface{} { + cbf, err := strconv.ParseUint(ctx.GetBackupFactor().GetText(), 10, 32) + if err != nil { + return p.reportError(errInvalidNumber) + } + + return uint32(cbf) +} + +// VisitRepStmt implements parser.QueryVisitor interface. +func (p *policyVisitor) VisitRepStmt(ctx *parser.RepStmtContext) interface{} { + num, err := strconv.ParseUint(ctx.GetCount().GetText(), 10, 32) + if err != nil { + return p.reportError(errInvalidNumber) + } + + rs := new(netmap.Replica) + rs.SetCount(uint32(num)) + + if sel := ctx.GetSelector(); sel != nil { + rs.SetSelector(sel.GetText()) + } + + return rs +} + +// VisitSelectStmt implements parser.QueryVisitor interface. +func (p *policyVisitor) VisitSelectStmt(ctx *parser.SelectStmtContext) interface{} { + res, err := strconv.ParseUint(ctx.GetCount().GetText(), 10, 32) + if err != nil { + return p.reportError(errInvalidNumber) + } + + s := new(netmap.Selector) + s.SetCount(uint32(res)) + + if clStmt := ctx.Clause(); clStmt != nil { + s.SetClause(clauseFromString(clStmt.GetText())) + } + + if bStmt := ctx.GetBucket(); bStmt != nil { + s.SetAttribute(ctx.GetBucket().GetText()) + } + + s.SetFilter(ctx.GetFilter().GetText()) // either ident or wildcard + + if ctx.AS() != nil { + s.SetName(ctx.GetName().GetText()) + } + return s +} + +// VisitFilterStmt implements parser.QueryVisitor interface. +func (p *policyVisitor) VisitFilterStmt(ctx *parser.FilterStmtContext) interface{} { + f := p.VisitFilterExpr(ctx.GetExpr().(*parser.FilterExprContext)).(*netmap.Filter) + f.SetName(ctx.GetName().GetText()) + return f +} + +func (p *policyVisitor) VisitFilterExpr(ctx *parser.FilterExprContext) interface{} { + if eCtx := ctx.Expr(); eCtx != nil { + return eCtx.Accept(p) + } + + if inner := ctx.GetInner(); inner != nil { + return inner.Accept(p) + } + + f := new(netmap.Filter) + op := operationFromString(ctx.GetOp().GetText()) + f.SetOp(op) + + f1 := *ctx.GetF1().Accept(p).(*netmap.Filter) + f2 := *ctx.GetF2().Accept(p).(*netmap.Filter) + + // Consider f1=(.. AND ..) AND f2. This can be merged because our AND operation + // is of arbitrary arity. ANTLR generates left-associative parse-tree by default. + if f1.GetOp() == op { + f.SetFilters(append(f1.GetFilters(), f2)) + return f + } + + f.SetFilters([]netmap.Filter{f1, f2}) + + return f +} + +// VisitFilterKey implements parser.QueryVisitor interface. +func (p *policyVisitor) VisitFilterKey(ctx *parser.FilterKeyContext) interface{} { + if id := ctx.Ident(); id != nil { + return id.GetText() + } + + str := ctx.STRING().GetText() + return str[1 : len(str)-1] +} + +func (p *policyVisitor) VisitFilterValue(ctx *parser.FilterValueContext) interface{} { + if id := ctx.Ident(); id != nil { + return id.GetText() + } + + if num := ctx.Number(); num != nil { + return num.GetText() + } + + str := ctx.STRING().GetText() + return str[1 : len(str)-1] +} + +// VisitExpr implements parser.QueryVisitor interface. +func (p *policyVisitor) VisitExpr(ctx *parser.ExprContext) interface{} { + f := new(netmap.Filter) + if flt := ctx.GetFilter(); flt != nil { + f.SetName(flt.GetText()) + return f + } + + key := ctx.GetKey().Accept(p) + opStr := ctx.SIMPLE_OP().GetText() + value := ctx.GetValue().Accept(p) + + f.SetKey(key.(string)) + f.SetOp(operationFromString(opStr)) + f.SetValue(value.(string)) + + return f +} + +// validatePolicy checks high-level constraints such as filter link in SELECT +// being actually defined in FILTER section. +func validatePolicy(p PlacementPolicy) error { + seenFilters := map[string]bool{} + + for i := range p.filters { + seenFilters[p.filters[i].GetName()] = true + } + + seenSelectors := map[string]bool{} + + for i := range p.selectors { + if flt := p.selectors[i].GetFilter(); flt != mainFilterName && !seenFilters[flt] { + return fmt.Errorf("%w: '%s'", errUnknownFilter, flt) + } + + seenSelectors[p.selectors[i].GetName()] = true + } + + for i := range p.replicas { + if sel := p.replicas[i].GetSelector(); sel != "" && !seenSelectors[sel] { + return fmt.Errorf("%w: '%s'", errUnknownSelector, sel) + } + } + + return nil +} + +func clauseFromString(s string) (c netmap.Clause) { + if !c.FromString(strings.ToUpper(s)) { + // Such errors should be handled by ANTLR code thus this panic. + panic(fmt.Errorf("BUG: invalid clause: %s", c)) + } + + return +} + +func operationFromString(s string) (op netmap.Operation) { + if !op.FromString(strings.ToUpper(s)) { + // Such errors should be handled by ANTLR code thus this panic. + panic(fmt.Errorf("BUG: invalid operation: %s", op)) + } + + return } diff --git a/netmap/policy_test.go b/netmap/policy_test.go index 737b180..1b5e385 100644 --- a/netmap/policy_test.go +++ b/netmap/policy_test.go @@ -1,127 +1,123 @@ package netmap import ( + "math/rand" + "strings" + "sync" "testing" - "github.com/nspcc-dev/neofs-api-go/v2/netmap" "github.com/stretchr/testify/require" ) -func TestPlacementPolicyFromV2(t *testing.T) { - pV2 := new(netmap.PlacementPolicy) +func TestEncode(t *testing.T) { + testCases := []string{ + `REP 1 IN X +CBF 1 +SELECT 2 IN SAME Location FROM * AS X`, - pV2.SetReplicas([]netmap.Replica{ - *testReplica().ToV2(), - *testReplica().ToV2(), - }) + `REP 1 +SELECT 2 IN City FROM Good +FILTER Country EQ RU AS FromRU +FILTER @FromRU AND Rating GT 7 AS Good`, - pV2.SetContainerBackupFactor(3) + `REP 7 IN SPB +SELECT 1 IN City FROM SPBSSD AS SPB +FILTER City EQ SPB AND SSD EQ true OR City EQ SPB AND Rating GE 5 AS SPBSSD`, + } - pV2.SetSelectors([]netmap.Selector{ - *testSelector().ToV2(), - *testSelector().ToV2(), - }) + for _, testCase := range testCases { + var p PlacementPolicy - pV2.SetFilters([]netmap.Filter{ - *testFilter().ToV2(), - *testFilter().ToV2(), - }) + require.NoError(t, p.DecodeString(testCase)) - p := NewPlacementPolicyFromV2(pV2) + var b strings.Builder + require.NoError(t, p.WriteStringTo(&b)) - require.Equal(t, pV2, p.ToV2()) + require.Equal(t, testCase, b.String()) + } } -func TestPlacementPolicy_Replicas(t *testing.T) { - p := NewPlacementPolicy() - rs := []Replica{*testReplica(), *testReplica()} +type cache struct { + mtx sync.RWMutex - p.SetReplicas(rs...) - - require.Equal(t, rs, p.Replicas()) + item map[string]struct{} } -func TestPlacementPolicy_ContainerBackupFactor(t *testing.T) { - p := NewPlacementPolicy() - f := uint32(3) - - p.SetContainerBackupFactor(f) - - require.Equal(t, f, p.ContainerBackupFactor()) +func (x *cache) add(key string) { + x.mtx.Lock() + x.item[key] = struct{}{} + x.mtx.Unlock() } -func TestPlacementPolicy_Selectors(t *testing.T) { - p := NewPlacementPolicy() - ss := []Selector{*testSelector(), *testSelector()} +func (x *cache) has(key string) bool { + x.mtx.RLock() + _, ok := x.item[key] + x.mtx.RUnlock() - p.SetSelectors(ss...) - - require.Equal(t, ss, p.Selectors()) + return ok } -func TestPlacementPolicy_Filters(t *testing.T) { - p := NewPlacementPolicy() - fs := []Filter{*testFilter(), *testFilter()} +func BenchmarkCache(b *testing.B) { + c := cache{ + item: make(map[string]struct{}), + } - p.SetFilters(fs...) + var key string + buf := make([]byte, 32) - require.Equal(t, fs, p.Filters()) + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + b.StopTimer() + rand.Read(buf) + key = string(buf) + b.StartTimer() + + c.add(key) + c.has(key) + } } -func TestPlacementPolicyEncoding(t *testing.T) { - p := newPlacementPolicy(3, nil, nil, nil) +type cacheP struct { + mtx *sync.RWMutex - t.Run("binary", func(t *testing.T) { - data, err := p.Marshal() - require.NoError(t, err) - - p2 := NewPlacementPolicy() - require.NoError(t, p2.Unmarshal(data)) - - require.Equal(t, p, p2) - }) - - t.Run("json", func(t *testing.T) { - data, err := p.MarshalJSON() - require.NoError(t, err) - - p2 := NewPlacementPolicy() - require.NoError(t, p2.UnmarshalJSON(data)) - - require.Equal(t, p, p2) - }) + item map[string]struct{} } -func TestNewPlacementPolicy(t *testing.T) { - t.Run("nil", func(t *testing.T) { - var x *PlacementPolicy - - require.Nil(t, x.ToV2()) - }) - - t.Run("default values", func(t *testing.T) { - pp := NewPlacementPolicy() - - // check initial values - require.Nil(t, pp.Replicas()) - require.Nil(t, pp.Filters()) - require.Nil(t, pp.Selectors()) - require.Zero(t, pp.ContainerBackupFactor()) - - // convert to v2 message - ppV2 := pp.ToV2() - - require.Nil(t, ppV2.GetReplicas()) - require.Nil(t, ppV2.GetFilters()) - require.Nil(t, ppV2.GetSelectors()) - require.Zero(t, ppV2.GetContainerBackupFactor()) - }) +func (x cacheP) add(key string) { + x.mtx.Lock() + x.item[key] = struct{}{} + x.mtx.Unlock() } -func TestNewPlacementPolicyFromV2(t *testing.T) { - t.Run("from nil", func(t *testing.T) { - var x *netmap.PlacementPolicy +func (x cacheP) has(key string) bool { + x.mtx.RLock() + _, ok := x.item[key] + x.mtx.RUnlock() - require.Nil(t, NewPlacementPolicyFromV2(x)) - }) + return ok +} + +func BenchmarkCacheP(b *testing.B) { + c := cacheP{ + mtx: &sync.RWMutex{}, + item: make(map[string]struct{}), + } + + var key string + buf := make([]byte, 32) + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + b.StopTimer() + rand.Read(buf) + key = string(buf) + b.StartTimer() + + c.add(key) + c.has(key) + } } diff --git a/netmap/replica.go b/netmap/replica.go deleted file mode 100644 index ddc0994..0000000 --- a/netmap/replica.go +++ /dev/null @@ -1,71 +0,0 @@ -package netmap - -import ( - "github.com/nspcc-dev/neofs-api-go/v2/netmap" -) - -// Replica represents v2-compatible object replica descriptor. -type Replica netmap.Replica - -// NewReplica creates and returns new Replica instance. -// -// Defaults: -// - count: 0; -// - selector: "". -func NewReplica() *Replica { - return NewReplicaFromV2(new(netmap.Replica)) -} - -// NewReplicaFromV2 converts v2 Replica to Replica. -// -// Nil netmap.Replica converts to nil. -func NewReplicaFromV2(f *netmap.Replica) *Replica { - return (*Replica)(f) -} - -// ToV2 converts Replica to v2 Replica. -// -// Nil Replica converts to nil. -func (r *Replica) ToV2() *netmap.Replica { - return (*netmap.Replica)(r) -} - -// Count returns number of object replicas. -func (r *Replica) Count() uint32 { - return (*netmap.Replica)(r).GetCount() -} - -// SetCount sets number of object replicas. -func (r *Replica) SetCount(c uint32) { - (*netmap.Replica)(r).SetCount(c) -} - -// Selector returns name of selector bucket to put replicas. -func (r *Replica) Selector() string { - return (*netmap.Replica)(r).GetSelector() -} - -// SetSelector sets name of selector bucket to put replicas. -func (r *Replica) SetSelector(s string) { - (*netmap.Replica)(r).SetSelector(s) -} - -// Marshal marshals Replica into a protobuf binary form. -func (r *Replica) Marshal() ([]byte, error) { - return (*netmap.Replica)(r).StableMarshal(nil), nil -} - -// Unmarshal unmarshals protobuf binary representation of Replica. -func (r *Replica) Unmarshal(data []byte) error { - return (*netmap.Replica)(r).Unmarshal(data) -} - -// MarshalJSON encodes Replica to protobuf JSON format. -func (r *Replica) MarshalJSON() ([]byte, error) { - return (*netmap.Replica)(r).MarshalJSON() -} - -// UnmarshalJSON decodes Replica from protobuf JSON format. -func (r *Replica) UnmarshalJSON(data []byte) error { - return (*netmap.Replica)(r).UnmarshalJSON(data) -} diff --git a/netmap/replica_test.go b/netmap/replica_test.go deleted file mode 100644 index d7d3f44..0000000 --- a/netmap/replica_test.go +++ /dev/null @@ -1,99 +0,0 @@ -package netmap - -import ( - "testing" - - "github.com/nspcc-dev/neofs-api-go/v2/netmap" - testv2 "github.com/nspcc-dev/neofs-api-go/v2/netmap/test" - "github.com/stretchr/testify/require" -) - -func testReplica() *Replica { - r := new(Replica) - r.SetCount(3) - r.SetSelector("selector") - - return r -} - -func TestReplicaFromV2(t *testing.T) { - t.Run("from nil", func(t *testing.T) { - var x *netmap.Replica - - require.Nil(t, NewReplicaFromV2(x)) - }) - - t.Run("from non-nil", func(t *testing.T) { - rV2 := testv2.GenerateReplica(false) - - r := NewReplicaFromV2(rV2) - - require.Equal(t, rV2, r.ToV2()) - }) -} - -func TestReplica_Count(t *testing.T) { - r := NewReplica() - c := uint32(3) - - r.SetCount(c) - - require.Equal(t, c, r.Count()) -} - -func TestReplica_Selector(t *testing.T) { - r := NewReplica() - s := "some selector" - - r.SetSelector(s) - - require.Equal(t, s, r.Selector()) -} - -func TestReplicaEncoding(t *testing.T) { - r := newReplica(3, "selector") - - t.Run("binary", func(t *testing.T) { - data, err := r.Marshal() - require.NoError(t, err) - - r2 := *NewReplica() - require.NoError(t, r2.Unmarshal(data)) - - require.Equal(t, r, r2) - }) - - t.Run("json", func(t *testing.T) { - data, err := r.MarshalJSON() - require.NoError(t, err) - - r2 := *NewReplica() - require.NoError(t, r2.UnmarshalJSON(data)) - - require.Equal(t, r, r2) - }) -} - -func TestReplica_ToV2(t *testing.T) { - t.Run("nil", func(t *testing.T) { - var x *Replica - - require.Nil(t, x.ToV2()) - }) -} - -func TestNewReplica(t *testing.T) { - t.Run("default values", func(t *testing.T) { - r := NewReplica() - - // check initial values - require.Zero(t, r.Count()) - require.Empty(t, r.Selector()) - - // convert to v2 message - rV2 := r.ToV2() - - require.Zero(t, rV2.GetCount()) - require.Empty(t, rV2.GetSelector()) - }) -} diff --git a/netmap/selector.go b/netmap/selector.go index b723d19..03beb59 100644 --- a/netmap/selector.go +++ b/netmap/selector.go @@ -9,62 +9,70 @@ import ( subnetid "github.com/nspcc-dev/neofs-sdk-go/subnet/id" ) -// Selector represents v2-compatible netmap selector. -type Selector netmap.Selector - // processSelectors processes selectors and returns error is any of them is invalid. -func (c *context) processSelectors(p *PlacementPolicy) error { - selectors := p.Selectors() - for i, s := range selectors { - if s.Filter() != MainFilterName { - _, ok := c.Filters[s.Filter()] +func (c *context) processSelectors(p PlacementPolicy) error { + for i := range p.selectors { + fName := p.selectors[i].GetFilter() + if fName != mainFilterName { + _, ok := c.processedFilters[p.selectors[i].GetFilter()] if !ok { - return fmt.Errorf("%w: SELECT FROM '%s'", ErrFilterNotFound, s.Filter()) + return fmt.Errorf("%w: SELECT FROM '%s'", errFilterNotFound, fName) } } - c.Selectors[s.Name()] = &selectors[i] + sName := p.selectors[i].GetName() - result, err := c.getSelection(p, &s) + c.processedSelectors[sName] = &p.selectors[i] + + result, err := c.getSelection(p, p.selectors[i]) if err != nil { return err } - c.Selections[s.Name()] = result + c.selections[sName] = result } return nil } -// GetNodesCount returns amount of buckets and minimum number of nodes in every bucket +// calcNodesCount returns amount of buckets and minimum number of nodes in every bucket // for the given selector. -func GetNodesCount(_ *PlacementPolicy, s *Selector) (int, int) { - switch s.Clause() { - case ClauseSame: - return 1, int(s.Count()) +func calcNodesCount(s netmap.Selector) (int, int) { + switch s.GetClause() { + case netmap.Same: + return 1, int(s.GetCount()) default: - return int(s.Count()), 1 + return int(s.GetCount()), 1 } } +// calcBucketWeight computes weight for a node bucket. +func calcBucketWeight(ns nodes, a aggregator, wf weightFunc) float64 { + for i := range ns { + a.Add(wf(ns[i])) + } + + return a.Compute() +} + // getSelection returns nodes grouped by s.attribute. // Last argument specifies if more buckets can be used to fulfill CBF. -func (c *context) getSelection(p *PlacementPolicy, s *Selector) ([]nodes, error) { - bucketCount, nodesInBucket := GetNodesCount(p, s) - buckets := c.getSelectionBase(p.SubnetID(), s) +func (c *context) getSelection(p PlacementPolicy, s netmap.Selector) ([]nodes, error) { + bucketCount, nodesInBucket := calcNodesCount(s) + buckets := c.getSelectionBase(p.subnet, s) if len(buckets) < bucketCount { - return nil, fmt.Errorf("%w: '%s'", ErrNotEnoughNodes, s.Name()) + return nil, fmt.Errorf("%w: '%s'", errNotEnoughNodes, s.GetName()) } // We need deterministic output in case there is no pivot. // If pivot is set, buckets are sorted by HRW. // However, because initial order influences HRW order for buckets with equal weights, // we also need to have deterministic input to HRW sorting routine. - if len(c.pivot) == 0 { - if s.Attribute() == "" { + if len(c.hrwSeed) == 0 { + if s.GetAttribute() == "" { sort.Slice(buckets, func(i, j int) bool { - return buckets[i].nodes[0].less(buckets[j].nodes[0]) + return less(buckets[i].nodes[0], buckets[j].nodes[0]) }) } else { sort.Slice(buckets, func(i, j int) bool { @@ -90,20 +98,20 @@ func (c *context) getSelection(p *PlacementPolicy, s *Selector) ([]nodes, error) // Fallback to using minimum allowed backup factor (1). res = append(res, fallback...) if len(res) < bucketCount { - return nil, fmt.Errorf("%w: '%s'", ErrNotEnoughNodes, s.Name()) + return nil, fmt.Errorf("%w: '%s'", errNotEnoughNodes, s.GetName()) } } - if len(c.pivot) != 0 { + if len(c.hrwSeed) != 0 { weights := make([]float64, len(res)) for i := range res { - weights[i] = calcBucketWeight(res[i], c.aggregator(), c.weightFunc) + weights[i] = calcBucketWeight(res[i], newMeanIQRAgg(), c.weightFunc) } - hrw.SortSliceByWeightValue(res, weights, c.pivotHash) + hrw.SortSliceByWeightValue(res, weights, c.hrwSeedHash) } - if s.Attribute() == "" { + if s.GetAttribute() == "" { res, fallback = res[:bucketCount], res[bucketCount:] for i := range fallback { index := i % bucketCount @@ -124,29 +132,26 @@ type nodeAttrPair struct { // getSelectionBase returns nodes grouped by selector attribute. // It it guaranteed that each pair will contain at least one node. -func (c *context) getSelectionBase(subnetID *subnetid.ID, s *Selector) []nodeAttrPair { - f := c.Filters[s.Filter()] - isMain := s.Filter() == MainFilterName +func (c *context) getSelectionBase(subnetID subnetid.ID, s netmap.Selector) []nodeAttrPair { + fName := s.GetFilter() + f := c.processedFilters[fName] + isMain := fName == mainFilterName result := []nodeAttrPair{} nodeMap := map[string][]NodeInfo{} - attr := s.Attribute() + attr := s.GetAttribute() - for i := range c.Netmap.nodes { - var sid subnetid.ID - if subnetID != nil { - sid = *subnetID - } + for i := range c.netMap.nodes { // TODO(fyrchik): make `BelongsToSubnet` to accept pointer - if !BelongsToSubnet(&c.Netmap.nodes[i], sid) { + if !BelongsToSubnet(c.netMap.nodes[i], subnetID) { continue } - if isMain || c.match(f, c.Netmap.nodes[i]) { + if isMain || c.match(f, c.netMap.nodes[i]) { if attr == "" { // Default attribute is transparent identifier which is different for every node. - result = append(result, nodeAttrPair{attr: "", nodes: nodes{c.Netmap.nodes[i]}}) + result = append(result, nodeAttrPair{attr: "", nodes: nodes{c.netMap.nodes[i]}}) } else { - v := c.Netmap.nodes[i].attribute(attr) - nodeMap[v] = append(nodeMap[v], c.Netmap.nodes[i]) + v := c.netMap.nodes[i].Attribute(attr) + nodeMap[v] = append(nodeMap[v], c.netMap.nodes[i]) } } } @@ -157,119 +162,11 @@ func (c *context) getSelectionBase(subnetID *subnetid.ID, s *Selector) []nodeAtt } } - if len(c.pivot) != 0 { + if len(c.hrwSeed) != 0 { for i := range result { - hrw.SortSliceByWeightValue(result[i].nodes, result[i].nodes.weights(c.weightFunc), c.pivotHash) + hrw.SortSliceByWeightValue(result[i].nodes, result[i].nodes.weights(c.weightFunc), c.hrwSeedHash) } } return result } - -// NewSelector creates and returns new Selector instance. -// -// Defaults: -// - name: ""; -// - attribute: ""; -// - filter: ""; -// - clause: ClauseUnspecified; -// - count: 0. -func NewSelector() *Selector { - return NewSelectorFromV2(new(netmap.Selector)) -} - -// NewSelectorFromV2 converts v2 Selector to Selector. -// -// Nil netmap.Selector converts to nil. -func NewSelectorFromV2(f *netmap.Selector) *Selector { - return (*Selector)(f) -} - -// ToV2 converts Selector to v2 Selector. -// -// Nil Selector converts to nil. -func (s *Selector) ToV2() *netmap.Selector { - return (*netmap.Selector)(s) -} - -// Name returns selector name. -func (s *Selector) Name() string { - return (*netmap.Selector)(s). - GetName() -} - -// SetName sets selector name. -func (s *Selector) SetName(name string) { - (*netmap.Selector)(s). - SetName(name) -} - -// Count returns count of nodes to select from bucket. -func (s *Selector) Count() uint32 { - return (*netmap.Selector)(s). - GetCount() -} - -// SetCount sets count of nodes to select from bucket. -func (s *Selector) SetCount(c uint32) { - (*netmap.Selector)(s). - SetCount(c) -} - -// Clause returns modifier showing how to form a bucket. -func (s *Selector) Clause() Clause { - return ClauseFromV2( - (*netmap.Selector)(s). - GetClause(), - ) -} - -// SetClause sets modifier showing how to form a bucket. -func (s *Selector) SetClause(c Clause) { - (*netmap.Selector)(s). - SetClause(c.ToV2()) -} - -// Attribute returns attribute bucket to select from. -func (s *Selector) Attribute() string { - return (*netmap.Selector)(s). - GetAttribute() -} - -// SetAttribute sets attribute bucket to select from. -func (s *Selector) SetAttribute(a string) { - (*netmap.Selector)(s). - SetAttribute(a) -} - -// Filter returns filter reference to select from. -func (s *Selector) Filter() string { - return (*netmap.Selector)(s). - GetFilter() -} - -// SetFilter sets filter reference to select from. -func (s *Selector) SetFilter(f string) { - (*netmap.Selector)(s). - SetFilter(f) -} - -// Marshal marshals Selector into a protobuf binary form. -func (s *Selector) Marshal() ([]byte, error) { - return (*netmap.Selector)(s).StableMarshal(nil), nil -} - -// Unmarshal unmarshals protobuf binary representation of Selector. -func (s *Selector) Unmarshal(data []byte) error { - return (*netmap.Selector)(s).Unmarshal(data) -} - -// MarshalJSON encodes Selector to protobuf JSON format. -func (s *Selector) MarshalJSON() ([]byte, error) { - return (*netmap.Selector)(s).MarshalJSON() -} - -// UnmarshalJSON decodes Selector from protobuf JSON format. -func (s *Selector) UnmarshalJSON(data []byte) error { - return (*netmap.Selector)(s).UnmarshalJSON(data) -} diff --git a/netmap/selector_test.go b/netmap/selector_test.go index 7944a08..6de1546 100644 --- a/netmap/selector_test.go +++ b/netmap/selector_test.go @@ -9,7 +9,6 @@ import ( "github.com/nspcc-dev/hrw" "github.com/nspcc-dev/neofs-api-go/v2/netmap" - testv2 "github.com/nspcc-dev/neofs-api-go/v2/netmap/test" "github.com/stretchr/testify/require" ) @@ -23,8 +22,8 @@ func BenchmarkHRWSort(b *testing.B) { rand.Read(key) var node NodeInfo - node.setPrice(1) - node.setCapacity(100) + node.SetPrice(1) + node.SetCapacity(100) node.SetPublicKey(key) vectors[i] = nodes{node} @@ -85,7 +84,7 @@ func BenchmarkHRWSort(b *testing.B) { b.StartTimer() sort.Slice(vectors, func(i, j int) bool { - return vectors[i][0].less(vectors[j][0]) + return less(vectors[i][0], vectors[j][0]) }) hrw.SortSliceByWeightIndex(realNodes, weights, pivot) } @@ -96,15 +95,15 @@ func BenchmarkPolicyHRWType(b *testing.B) { const netmapSize = 100 p := newPlacementPolicy(1, - []Replica{ + []ReplicaDescriptor{ newReplica(1, "loc1"), newReplica(1, "loc2")}, []Selector{ - newSelector("loc1", "Location", ClauseSame, 1, "loc1"), - newSelector("loc2", "Location", ClauseSame, 1, "loc2")}, + newSelector("loc1", "Location", 1, "loc1", (*Selector).SelectSame), + newSelector("loc2", "Location", 1, "loc2", (*Selector).SelectSame)}, []Filter{ - newFilter("loc1", "Location", "Shanghai", OpEQ), - newFilter("loc2", "Location", "Shanghai", OpNE), + newFilter("loc1", "Location", "Shanghai", netmap.EQ), + newFilter("loc2", "Location", "Shanghai", netmap.NE), }) nodes := make([]NodeInfo, netmapSize) @@ -125,12 +124,12 @@ func BenchmarkPolicyHRWType(b *testing.B) { nodes[i].SetPublicKey(pub) } - var nm Netmap + var nm NetMap nm.SetNodes(nodes) b.ResetTimer() for i := 0; i < b.N; i++ { - _, err := nm.GetContainerNodes(p, []byte{1}) + _, err := nm.ContainerNodes(p, []byte{1}) if err != nil { b.Fatal() } @@ -141,15 +140,15 @@ func TestPlacementPolicy_DeterministicOrder(t *testing.T) { const netmapSize = 100 p := newPlacementPolicy(1, - []Replica{ + []ReplicaDescriptor{ newReplica(1, "loc1"), newReplica(1, "loc2")}, []Selector{ - newSelector("loc1", "Location", ClauseSame, 1, "loc1"), - newSelector("loc2", "Location", ClauseSame, 1, "loc2")}, + newSelector("loc1", "Location", 1, "loc1", (*Selector).SelectSame), + newSelector("loc2", "Location", 1, "loc2", (*Selector).SelectSame)}, []Filter{ - newFilter("loc1", "Location", "Shanghai", OpEQ), - newFilter("loc2", "Location", "Shanghai", OpNE), + newFilter("loc1", "Location", "Shanghai", netmap.EQ), + newFilter("loc2", "Location", "Shanghai", netmap.NE), }) nodeList := make([]NodeInfo, netmapSize) @@ -170,11 +169,11 @@ func TestPlacementPolicy_DeterministicOrder(t *testing.T) { nodeList[i].SetPublicKey(pub) } - var nm Netmap + var nm NetMap nm.SetNodes(nodeList) getIndices := func(t *testing.T) (uint64, uint64) { - v, err := nm.GetContainerNodes(p, []byte{1}) + v, err := nm.ContainerNodes(p, []byte{1}) require.NoError(t, err) nss := make([]nodes, len(v)) @@ -198,14 +197,14 @@ func TestPlacementPolicy_DeterministicOrder(t *testing.T) { func TestPlacementPolicy_ProcessSelectors(t *testing.T) { p := newPlacementPolicy(2, nil, []Selector{ - newSelector("SameRU", "City", ClauseSame, 2, "FromRU"), - newSelector("DistinctRU", "City", ClauseDistinct, 2, "FromRU"), - newSelector("Good", "Country", ClauseDistinct, 2, "Good"), - newSelector("Main", "Country", ClauseDistinct, 3, "*"), + newSelector("SameRU", "City", 2, "FromRU", (*Selector).SelectSame), + newSelector("DistinctRU", "City", 2, "FromRU", (*Selector).SelectDistinct), + newSelector("Good", "Country", 2, "Good", (*Selector).SelectDistinct), + newSelector("Main", "Country", 3, "*", (*Selector).SelectDistinct), }, []Filter{ - newFilter("FromRU", "Country", "Russia", OpEQ), - newFilter("Good", "Rating", "4", OpGE), + newFilter("FromRU", "Country", "Russia", netmap.EQ), + newFilter("Good", "Rating", "4", netmap.GE), }) nodes := []NodeInfo{ nodeInfoFromAttributes("Country", "Russia", "Rating", "1", "City", "SPB"), @@ -221,151 +220,79 @@ func TestPlacementPolicy_ProcessSelectors(t *testing.T) { nodeInfoFromAttributes("Country", "Russia", "Rating", "9", "City", "SPB"), } - var nm Netmap + var nm NetMap nm.SetNodes(nodes) - c := newContext(&nm) - c.setCBF(p.ContainerBackupFactor()) + c := newContext(nm) + c.setCBF(p.backupFactor) require.NoError(t, c.processFilters(p)) require.NoError(t, c.processSelectors(p)) - for _, s := range p.Selectors() { - sel := c.Selections[s.Name()] - s := c.Selectors[s.Name()] - bucketCount, nodesInBucket := GetNodesCount(p, s) + for _, s := range p.selectors { + sel := c.selections[s.GetName()] + s := c.processedSelectors[s.GetName()] + bucketCount, nodesInBucket := calcNodesCount(*s) nodesInBucket *= int(c.cbf) - targ := fmt.Sprintf("selector '%s'", s.Name()) + targ := fmt.Sprintf("selector '%s'", s.GetName()) require.Equal(t, bucketCount, len(sel), targ) + fName := s.GetFilter() for _, res := range sel { require.Equal(t, nodesInBucket, len(res), targ) for j := range res { - require.True(t, c.applyFilter(s.Filter(), res[j]), targ) + require.True(t, fName == mainFilterName || c.match(c.processedFilters[fName], res[j]), targ) } } } } -func testSelector() *Selector { - s := new(Selector) - s.SetName("name") - s.SetCount(3) - s.SetFilter("filter") - s.SetAttribute("attribute") - s.SetClause(ClauseDistinct) +func TestSelector_SetName(t *testing.T) { + const name = "some name" + var s Selector - return s -} - -func TestSelector_Name(t *testing.T) { - s := NewSelector() - name := "some name" + require.Zero(t, s.m.GetName()) s.SetName(name) - - require.Equal(t, name, s.Name()) + require.Equal(t, name, s.m.GetName()) } -func TestSelector_Count(t *testing.T) { - s := NewSelector() - c := uint32(3) +func TestSelector_SetNodeAmount(t *testing.T) { + const amount = 3 + var s Selector - s.SetCount(c) + require.Zero(t, s.m.GetCount()) - require.Equal(t, c, s.Count()) + s.SetNodeAmount(amount) + + require.EqualValues(t, amount, s.m.GetCount()) } -func TestSelector_Clause(t *testing.T) { - s := NewSelector() - c := ClauseSame +func TestSelectorClauses(t *testing.T) { + var s Selector - s.SetClause(c) + require.Equal(t, netmap.UnspecifiedClause, s.m.GetClause()) - require.Equal(t, c, s.Clause()) + s.SelectDistinct() + require.Equal(t, netmap.Distinct, s.m.GetClause()) + + s.SelectSame() + require.Equal(t, netmap.Same, s.m.GetClause()) } -func TestSelector_Attribute(t *testing.T) { - s := NewSelector() - a := "some attribute" +func TestSelector_SelectByBucketAttribute(t *testing.T) { + const attr = "some attribute" + var s Selector - s.SetAttribute(a) + require.Zero(t, s.m.GetAttribute()) - require.Equal(t, a, s.Attribute()) + s.SelectByBucketAttribute(attr) + require.Equal(t, attr, s.m.GetAttribute()) } -func TestSelector_Filter(t *testing.T) { - s := NewSelector() - f := "some filter" +func TestSelector_SetFilterName(t *testing.T) { + const fName = "some filter" + var s Selector - s.SetFilter(f) + require.Zero(t, s.m.GetFilter()) - require.Equal(t, f, s.Filter()) -} - -func TestSelectorEncoding(t *testing.T) { - s := newSelector("name", "atte", ClauseSame, 1, "filter") - - t.Run("binary", func(t *testing.T) { - data, err := s.Marshal() - require.NoError(t, err) - - s2 := *NewSelector() - require.NoError(t, s2.Unmarshal(data)) - - require.Equal(t, s, s2) - }) - - t.Run("json", func(t *testing.T) { - data, err := s.MarshalJSON() - require.NoError(t, err) - - s2 := *NewSelector() - require.NoError(t, s2.UnmarshalJSON(data)) - - require.Equal(t, s, s2) - }) -} - -func TestSelector_ToV2(t *testing.T) { - t.Run("nil", func(t *testing.T) { - var x *Selector - - require.Nil(t, x.ToV2()) - }) -} - -func TestNewSelectorFromV2(t *testing.T) { - t.Run("from nil", func(t *testing.T) { - var x *netmap.Selector - - require.Nil(t, NewSelectorFromV2(x)) - }) - - t.Run("from non-nil", func(t *testing.T) { - sV2 := testv2.GenerateSelector(false) - - s := NewSelectorFromV2(sV2) - - require.Equal(t, sV2, s.ToV2()) - }) -} - -func TestNewSelector(t *testing.T) { - t.Run("default values", func(t *testing.T) { - s := NewSelector() - - // check initial values - require.Zero(t, s.Count()) - require.Equal(t, ClauseUnspecified, s.Clause()) - require.Empty(t, s.Attribute()) - require.Empty(t, s.Name()) - require.Empty(t, s.Filter()) - - // convert to v2 message - sV2 := s.ToV2() - - require.Zero(t, sV2.GetCount()) - require.Equal(t, netmap.UnspecifiedClause, sV2.GetClause()) - require.Empty(t, sV2.GetAttribute()) - require.Empty(t, sV2.GetName()) - require.Empty(t, sV2.GetFilter()) - }) + s.SetFilterName(fName) + require.Equal(t, fName, s.m.GetFilter()) } diff --git a/netmap/subnet.go b/netmap/subnet.go deleted file mode 100644 index 41be1c4..0000000 --- a/netmap/subnet.go +++ /dev/null @@ -1,85 +0,0 @@ -package netmap - -import ( - "errors" - "fmt" - - "github.com/nspcc-dev/neofs-api-go/v2/netmap" - "github.com/nspcc-dev/neofs-api-go/v2/refs" - subnetid "github.com/nspcc-dev/neofs-sdk-go/subnet/id" -) - -// EnterSubnet writes to NodeInfo the intention to enter the subnet. Must not be called on nil. -// Zero NodeInfo belongs to zero subnet. -func (i *NodeInfo) EnterSubnet(id subnetid.ID) { - i.changeSubnet(id, true) -} - -// ExitSubnet writes to NodeInfo the intention to exit subnet. Must not be called on nil. -func (i *NodeInfo) ExitSubnet(id subnetid.ID) { - i.changeSubnet(id, false) -} - -func (i *NodeInfo) changeSubnet(id subnetid.ID, isMember bool) { - var ( - idv2 refs.SubnetID - info netmap.NodeSubnetInfo - ) - - id.WriteToV2(&idv2) - - info.SetID(&idv2) - info.SetEntryFlag(isMember) - - if i.m == nil { - i.m = new(netmap.NodeInfo) - } - - netmap.WriteSubnetInfo(i.m, info) -} - -// ErrRemoveSubnet is returned when a node needs to leave the subnet. -var ErrRemoveSubnet = netmap.ErrRemoveSubnet - -// IterateSubnets iterates over all subnets the node belongs to and passes the IDs to f. -// Must not be called on nil. Handler must not be nil. -// -// If f returns ErrRemoveSubnet, then removes subnet entry. Note that this leads to an instant mutation of NodeInfo. -// Breaks on any other non-nil error and returns it. -// -// Returns an error if subnet incorrectly enabled/disabled. -// Returns an error if the node is not included in any subnet by the end of the loop. -func (i *NodeInfo) IterateSubnets(f func(subnetid.ID) error) error { - var id subnetid.ID - - return netmap.IterateSubnets(i.m, func(idv2 refs.SubnetID) error { - err := id.ReadFromV2(idv2) - if err != nil { - return fmt.Errorf("invalid subnet: %w", err) - } - - err = f(id) - if errors.Is(err, ErrRemoveSubnet) { - return netmap.ErrRemoveSubnet - } - - return err - }) -} - -var errAbortSubnetIter = errors.New("abort subnet iterator") - -// BelongsToSubnet checks if node belongs to subnet by ID. -// -// Function is NPE-safe: nil NodeInfo always belongs to zero subnet only. -func BelongsToSubnet(node *NodeInfo, id subnetid.ID) bool { - err := node.IterateSubnets(func(id_ subnetid.ID) error { - if id.Equals(id_) { - return errAbortSubnetIter - } - - return nil - }) - - return errors.Is(err, errAbortSubnetIter) -} diff --git a/netmap/subnet_test.go b/netmap/subnet_test.go index 7e1150b..11dac77 100644 --- a/netmap/subnet_test.go +++ b/netmap/subnet_test.go @@ -92,22 +92,22 @@ func TestEnterSubnet(t *testing.T) { node netmap.NodeInfo ) - require.True(t, netmap.BelongsToSubnet(&node, id)) + require.True(t, netmap.BelongsToSubnet(node, id)) node.EnterSubnet(id) - require.True(t, netmap.BelongsToSubnet(&node, id)) + require.True(t, netmap.BelongsToSubnet(node, id)) node.ExitSubnet(id) - require.False(t, netmap.BelongsToSubnet(&node, id)) + require.False(t, netmap.BelongsToSubnet(node, id)) id.SetNumeric(10) node.EnterSubnet(id) - require.True(t, netmap.BelongsToSubnet(&node, id)) - require.False(t, netmap.BelongsToSubnet(&node, subnetid.ID{})) + require.True(t, netmap.BelongsToSubnet(node, id)) + require.False(t, netmap.BelongsToSubnet(node, subnetid.ID{})) node.ExitSubnet(id) - require.False(t, netmap.BelongsToSubnet(&node, id)) - require.False(t, netmap.BelongsToSubnet(&node, subnetid.ID{})) + require.False(t, netmap.BelongsToSubnet(node, id)) + require.False(t, netmap.BelongsToSubnet(node, subnetid.ID{})) } func TestBelongsToSubnet(t *testing.T) { @@ -120,7 +120,7 @@ func TestBelongsToSubnet(t *testing.T) { node.EnterSubnet(id) - require.True(t, netmap.BelongsToSubnet(&node, idZero)) - require.True(t, netmap.BelongsToSubnet(&node, id)) - require.False(t, netmap.BelongsToSubnet(&node, idMiss)) + require.True(t, netmap.BelongsToSubnet(node, idZero)) + require.True(t, netmap.BelongsToSubnet(node, id)) + require.False(t, netmap.BelongsToSubnet(node, idMiss)) } diff --git a/netmap/test/generate.go b/netmap/test/generate.go index a6a54a4..b0874d2 100644 --- a/netmap/test/generate.go +++ b/netmap/test/generate.go @@ -1,92 +1,70 @@ package netmaptest -import "github.com/nspcc-dev/neofs-sdk-go/netmap" - -func filter(withInner bool) *netmap.Filter { - x := netmap.NewFilter() +import ( + "github.com/nspcc-dev/neofs-sdk-go/netmap" + subnetidtest "github.com/nspcc-dev/neofs-sdk-go/subnet/id/test" +) +func filter(withInner bool) (x netmap.Filter) { x.SetName("name") - x.SetKey("key") - x.SetValue("value") - x.SetOperation(netmap.OpAND) - if withInner { - x.SetInnerFilters(*filter(false), *filter(false)) + x.LogicalOR(filter(false), filter(false)) + } else { + x.NumericGE("epoch", 13) } return x } // Filter returns random netmap.Filter. -func Filter() *netmap.Filter { +func Filter() netmap.Filter { return filter(true) } -// Replica returns random netmap.Replica. -func Replica() *netmap.Replica { - x := netmap.NewReplica() +// Replica returns random netmap.ReplicaDescriptor. +func Replica() (x netmap.ReplicaDescriptor) { + x.SetAmount(666) + x.SetSelectorName("selector") - x.SetCount(666) - x.SetSelector("selector") - - return x + return } // Selector returns random netmap.Selector. -func Selector() *netmap.Selector { - x := netmap.NewSelector() - - x.SetCount(11) +func Selector() (x netmap.Selector) { + x.SetNodeAmount(11) x.SetName("name") - x.SetFilter("filter") - x.SetAttribute("attribute") - x.SetClause(netmap.ClauseDistinct) + x.SetFilterName("filter") + x.SelectByBucketAttribute("attribute") + x.SelectDistinct() - return x + return } // PlacementPolicy returns random netmap.PlacementPolicy. -func PlacementPolicy() *netmap.PlacementPolicy { - x := netmap.NewPlacementPolicy() +func PlacementPolicy() (p netmap.PlacementPolicy) { + p.SetContainerBackupFactor(9) + p.AddFilters(Filter(), Filter()) + p.AddReplicas(Replica(), Replica()) + p.AddSelectors(Selector(), Selector()) + p.RestrictSubnet(subnetidtest.ID()) - x.SetContainerBackupFactor(9) - x.SetFilters(*Filter(), *Filter()) - x.SetReplicas(*Replica(), *Replica()) - x.SetSelectors(*Selector(), *Selector()) - - return x -} - -// NetworkParameter returns random netmap.NetworkParameter. -func NetworkParameter() *netmap.NetworkParameter { - x := netmap.NewNetworkParameter() - - x.SetKey([]byte("key")) - x.SetValue([]byte("value")) - - return x -} - -// NetworkConfig returns random netmap.NetworkConfig. -func NetworkConfig() *netmap.NetworkConfig { - x := netmap.NewNetworkConfig() - - x.SetParameters( - *NetworkParameter(), - *NetworkParameter(), - ) - - return x + return } // NetworkInfo returns random netmap.NetworkInfo. -func NetworkInfo() *netmap.NetworkInfo { - x := netmap.NewNetworkInfo() - +func NetworkInfo() (x netmap.NetworkInfo) { x.SetCurrentEpoch(21) x.SetMagicNumber(32) x.SetMsPerBlock(43) - x.SetNetworkConfig(NetworkConfig()) + x.SetAuditFee(1) + x.SetStoragePrice(2) + x.SetContainerFee(3) + x.SetEigenTrustAlpha(0.4) + x.SetEigenTrustIterationAmount(5) + x.SetEpochDuration(6) + x.SetIRCandidateFee(7) + x.SetMaxObjectSize(8) + x.SetWithdrawalFee(9) - return x + return } diff --git a/policy/doc.go b/policy/doc.go deleted file mode 100644 index cceebca..0000000 --- a/policy/doc.go +++ /dev/null @@ -1,19 +0,0 @@ -// Package policy provides facilities for creating policy from SQL-like language. -// ANTLRv4 grammar is provided in `parser/Query.g4` and `parser/QueryLexer.g4`. -// -// Current limitations: -// 1. Filters must be defined before they are used. -// This requirement may be relaxed in future. -// 2. Keywords are key-sensitive. This can be changed if necessary -// https://github.com/antlr/antlr4/blob/master/doc/case-insensitive-lexing.md . -// -// Example query: -// REP 1 IN SPB -// REP 2 IN Americas -// CBF 4 -// SELECT 1 IN City FROM SPBSSD AS SPB -// SELECT 2 IN SAME City FROM Americas AS Americas -// FILTER SSD EQ true AS IsSSD -// FILTER @IsSSD AND Country EQ "RU" AND City EQ "St.Petersburg" AS SPBSSD -// FILTER 'Continent' EQ 'North America' OR Continent EQ 'South America' AS Americas -package policy diff --git a/policy/encode.go b/policy/encode.go deleted file mode 100644 index 39c9dcc..0000000 --- a/policy/encode.go +++ /dev/null @@ -1,141 +0,0 @@ -package policy - -import ( - "fmt" - "strconv" - "strings" - - "github.com/nspcc-dev/neofs-sdk-go/netmap" -) - -// Encode parses data of PlacementPolicy to a string. -func Encode(p *netmap.PlacementPolicy) []string { - if p == nil { - return nil - } - - var ( - replicas = p.Replicas() - selectors = p.Selectors() - filters = p.Filters() - ) - - // 1 for container backup factor - result := make([]string, 0, len(replicas)+len(selectors)+len(filters)+1) - - // first print replicas - encodeReplicas(replicas, &result) - - // then backup factor - if backupFactor := p.ContainerBackupFactor(); backupFactor != 0 { - result = append(result, fmt.Sprintf("CBF %d", backupFactor)) - } - - // then selectors - encodeSelectors(selectors, &result) - - // then filters - encodeFilters(filters, &result) - - return result -} - -func encodeReplicas(replicas []netmap.Replica, dst *[]string) { - builder := new(strings.Builder) - - for _, replica := range replicas { - builder.WriteString("REP ") - builder.WriteString(strconv.FormatUint(uint64(replica.Count()), 10)) - - if s := replica.Selector(); s != "" { - builder.WriteString(" IN ") - builder.WriteString(s) - } - - *dst = append(*dst, builder.String()) - builder.Reset() - } -} - -func encodeSelectors(selectors []netmap.Selector, dst *[]string) { - builder := new(strings.Builder) - - for _, selector := range selectors { - builder.WriteString("SELECT ") - builder.WriteString(strconv.FormatUint(uint64(selector.Count()), 10)) - - if a := selector.Attribute(); a != "" { - builder.WriteString(" IN") - - switch selector.Clause() { - case netmap.ClauseSame: - builder.WriteString(" SAME ") - case netmap.ClauseDistinct: - builder.WriteString(" DISTINCT ") - default: - builder.WriteString(" ") - } - - builder.WriteString(a) - } - - if f := selector.Filter(); f != "" { - builder.WriteString(" FROM ") - builder.WriteString(f) - } - - if n := selector.Name(); n != "" { - builder.WriteString(" AS ") - builder.WriteString(n) - } - - *dst = append(*dst, builder.String()) - builder.Reset() - } -} - -func encodeFilters(filters []netmap.Filter, dst *[]string) { - builder := new(strings.Builder) - - for _, filter := range filters { - builder.WriteString("FILTER ") - - builder.WriteString(encodeFilter(&filter)) - - *dst = append(*dst, builder.String()) - builder.Reset() - } -} - -func encodeFilter(filter *netmap.Filter) string { - builder := new(strings.Builder) - unspecified := filter.Operation() == 0 - - if k := filter.Key(); k != "" { - builder.WriteString(k) - builder.WriteString(" ") - builder.WriteString(filter.Operation().String()) - builder.WriteString(" ") - builder.WriteString(filter.Value()) - } else if n := filter.Name(); unspecified && n != "" { - builder.WriteString("@") - builder.WriteString(n) - } - - for i, subfilter := range filter.InnerFilters() { - if i != 0 { - builder.WriteString(" ") - builder.WriteString(filter.Operation().String()) - builder.WriteString(" ") - } - - builder.WriteString(encodeFilter(&subfilter)) - } - - if n := filter.Name(); n != "" && !unspecified { - builder.WriteString(" AS ") - builder.WriteString(n) - } - - return builder.String() -} diff --git a/policy/encode_test.go b/policy/encode_test.go deleted file mode 100644 index 813e716..0000000 --- a/policy/encode_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package policy_test - -import ( - "strings" - "testing" - - "github.com/nspcc-dev/neofs-sdk-go/policy" - "github.com/stretchr/testify/require" -) - -func TestEncode(t *testing.T) { - testCases := []string{ - `REP 1 IN X -CBF 1 -SELECT 2 IN SAME Location FROM * AS X`, - - `REP 1 -SELECT 2 IN City FROM Good -FILTER Country EQ RU AS FromRU -FILTER @FromRU AND Rating GT 7 AS Good`, - - `REP 7 IN SPB -SELECT 1 IN City FROM SPBSSD AS SPB -FILTER City EQ SPB AND SSD EQ true OR City EQ SPB AND Rating GE 5 AS SPBSSD`, - } - - for _, testCase := range testCases { - q, err := policy.Parse(testCase) - require.NoError(t, err) - - got := policy.Encode(q) - require.Equal(t, testCase, strings.Join(got, "\n")) - } -} diff --git a/policy/json.go b/policy/json.go deleted file mode 100644 index 595fceb..0000000 --- a/policy/json.go +++ /dev/null @@ -1,225 +0,0 @@ -package policy - -import ( - "encoding/json" - "fmt" - "strings" - - "github.com/nspcc-dev/neofs-sdk-go/netmap" -) - -type ( - filter struct { - Name string `json:"name,omitempty"` - Key string `json:"key,omitempty"` - Op string `json:"op,omitempty"` - Value string `json:"value,omitempty"` - Filters []filter `json:"filters,omitempty"` - } - replica struct { - Count uint32 `json:"count"` - Selector string `json:"selector,omitempty"` - } - selector struct { - Count uint32 `json:"count"` - Attribute string `json:"attribute"` - Filter string `json:"filter,omitempty"` - Name string `json:"name,omitempty"` - Clause string `json:"clause,omitempty"` - } - placement struct { - Replicas []replica `json:"replicas"` - CBF uint32 `json:"container_backup_factor,omitempty"` - Selectors []selector `json:"selectors,omitempty"` - Filters []filter `json:"filters,omitempty"` - } -) - -// ToJSON converts placement policy to JSON. -func ToJSON(np *netmap.PlacementPolicy) ([]byte, error) { - p := new(placement) - p.CBF = np.ContainerBackupFactor() - p.Filters = make([]filter, len(np.Filters())) - for i, f := range np.Filters() { - p.Filters[i].fromNetmap(&f) - } - p.Selectors = make([]selector, len(np.Selectors())) - for i, s := range np.Selectors() { - p.Selectors[i].fromNetmap(&s) - } - p.Replicas = make([]replica, len(np.Replicas())) - for i, r := range np.Replicas() { - p.Replicas[i].fromNetmap(&r) - } - return json.Marshal(p) -} - -// FromJSON creates placement policy from JSON. -func FromJSON(data []byte) (*netmap.PlacementPolicy, error) { - p := new(placement) - if err := json.Unmarshal(data, p); err != nil { - return nil, err - } - - rs := make([]netmap.Replica, len(p.Replicas)) - for i := range p.Replicas { - rs[i] = *p.Replicas[i].toNetmap() - } - - var fs []netmap.Filter - if len(p.Filters) != 0 { - fs = make([]netmap.Filter, len(p.Filters)) - for i := range p.Filters { - f, err := p.Filters[i].toNetmap() - if err != nil { - return nil, err - } - fs[i] = *f - } - } - - var ss []netmap.Selector - if len(p.Selectors) != 0 { - ss = make([]netmap.Selector, len(p.Selectors)) - for i := range p.Selectors { - s, err := p.Selectors[i].toNetmap() - if err != nil { - return nil, err - } - ss[i] = *s - } - } - - pp := new(netmap.PlacementPolicy) - pp.SetReplicas(rs...) - pp.SetContainerBackupFactor(p.CBF) - pp.SetFilters(fs...) - pp.SetSelectors(ss...) - return pp, nil -} - -func (r *replica) toNetmap() *netmap.Replica { - nr := new(netmap.Replica) - nr.SetCount(r.Count) - nr.SetSelector(r.Selector) - return nr -} - -func (r *replica) fromNetmap(nr *netmap.Replica) { - r.Count = nr.Count() - r.Selector = nr.Selector() -} - -func (f *filter) toNetmap() (*netmap.Filter, error) { - var op netmap.Operation - switch strings.ToUpper(f.Op) { - case "EQ": - op = netmap.OpEQ - case "NE": - op = netmap.OpNE - case "GT": - op = netmap.OpGT - case "GE": - op = netmap.OpGE - case "LT": - op = netmap.OpLT - case "LE": - op = netmap.OpLE - case "AND": - op = netmap.OpAND - case "OR": - op = netmap.OpOR - case "": - default: - return nil, fmt.Errorf("%w: '%s'", ErrUnknownOp, f.Op) - } - - var fs []netmap.Filter - if len(f.Filters) != 0 { - fs = make([]netmap.Filter, len(f.Filters)) - for i := range f.Filters { - var err error - fsp, err := f.Filters[i].toNetmap() - if err != nil { - return nil, err - } - fs[i] = *fsp - } - } - - nf := new(netmap.Filter) - nf.SetInnerFilters(fs...) - nf.SetOperation(op) - nf.SetName(f.Name) - nf.SetValue(f.Value) - nf.SetKey(f.Key) - return nf, nil -} - -func (f *filter) fromNetmap(nf *netmap.Filter) { - f.Name = nf.Name() - f.Key = nf.Key() - f.Value = nf.Value() - switch nf.Operation() { - case netmap.OpEQ: - f.Op = "EQ" - case netmap.OpNE: - f.Op = "NE" - case netmap.OpGT: - f.Op = "GT" - case netmap.OpGE: - f.Op = "GE" - case netmap.OpLT: - f.Op = "LT" - case netmap.OpLE: - f.Op = "LE" - case netmap.OpAND: - f.Op = "AND" - case netmap.OpOR: - f.Op = "OR" - default: - // do nothing - } - if nf.InnerFilters() != nil { - f.Filters = make([]filter, len(nf.InnerFilters())) - for i, sf := range nf.InnerFilters() { - f.Filters[i].fromNetmap(&sf) - } - } -} - -func (s *selector) toNetmap() (*netmap.Selector, error) { - var c netmap.Clause - switch strings.ToUpper(s.Clause) { - case "SAME": - c = netmap.ClauseSame - case "DISTINCT": - c = netmap.ClauseDistinct - case "": - default: - return nil, fmt.Errorf("%w: '%s'", ErrUnknownClause, s.Clause) - } - ns := new(netmap.Selector) - ns.SetName(s.Name) - ns.SetAttribute(s.Attribute) - ns.SetCount(s.Count) - ns.SetClause(c) - ns.SetFilter(s.Filter) - return ns, nil -} - -func (s *selector) fromNetmap(ns *netmap.Selector) { - s.Name = ns.Name() - s.Filter = ns.Filter() - s.Count = ns.Count() - s.Attribute = ns.Attribute() - switch ns.Clause() { - case netmap.ClauseSame: - s.Clause = "same" - case netmap.ClauseDistinct: - s.Clause = "distinct" - default: - // do nothing - } - s.Name = ns.Name() -} diff --git a/policy/json_test.go b/policy/json_test.go deleted file mode 100644 index 96ef68f..0000000 --- a/policy/json_test.go +++ /dev/null @@ -1,82 +0,0 @@ -package policy - -import ( - "testing" - - "github.com/nspcc-dev/neofs-sdk-go/netmap" - "github.com/stretchr/testify/require" -) - -func TestToJSON(t *testing.T) { - check := func(t *testing.T, p *netmap.PlacementPolicy, json string) { - data, err := ToJSON(p) - require.NoError(t, err) - require.JSONEq(t, json, string(data)) - - np, err := FromJSON(data) - require.NoError(t, err) - require.Equal(t, p, np) - } - t.Run("SimpleREP", func(t *testing.T) { - p := new(netmap.PlacementPolicy) - p.SetReplicas(newReplica("", 3)) - check(t, p, `{"replicas":[{"count":3}]}`) - }) - t.Run("REPWithCBF", func(t *testing.T) { - p := new(netmap.PlacementPolicy) - p.SetReplicas(newReplica("", 3)) - p.SetContainerBackupFactor(3) - check(t, p, `{"replicas":[{"count":3}],"container_backup_factor":3}`) - }) - t.Run("REPFromSelector", func(t *testing.T) { - p := new(netmap.PlacementPolicy) - p.SetReplicas(newReplica("Nodes", 3)) - p.SetContainerBackupFactor(3) - p.SetSelectors( - newSelector(1, netmap.ClauseDistinct, "City", "", "Nodes")) - check(t, p, `{ - "replicas":[{"count":3,"selector":"Nodes"}], - "container_backup_factor":3, - "selectors": [{ - "name":"Nodes", - "attribute":"City", - "clause":"distinct", - "count":1 - }]}`) - }) - t.Run("FilterOps", func(t *testing.T) { - p := new(netmap.PlacementPolicy) - p.SetReplicas(newReplica("Nodes", 3)) - p.SetContainerBackupFactor(3) - p.SetSelectors( - newSelector(1, netmap.ClauseSame, "City", "Good", "Nodes")) - p.SetFilters( - newFilter("GoodRating", "Rating", "5", netmap.OpGE), - newFilter("Good", "", "", netmap.OpOR, - newFilter("GoodRating", "", "", 0), - newFilter("", "Attr1", "Val1", netmap.OpEQ), - newFilter("", "Attr2", "Val2", netmap.OpNE), - newFilter("", "", "", netmap.OpAND, - newFilter("", "Attr4", "2", netmap.OpLT), - newFilter("", "Attr5", "3", netmap.OpLE)), - newFilter("", "Attr3", "1", netmap.OpGT)), - ) - check(t, p, `{ - "replicas":[{"count":3,"selector":"Nodes"}], - "container_backup_factor":3, - "selectors": [{"name":"Nodes","attribute":"City","clause":"same","count":1,"filter":"Good"}], - "filters": [ - {"name":"GoodRating","key":"Rating","op":"GE","value":"5"}, - {"name":"Good","op":"OR","filters":[ - {"name":"GoodRating"}, - {"key":"Attr1","op":"EQ","value":"Val1"}, - {"key":"Attr2","op":"NE","value":"Val2"}, - {"op":"AND","filters":[ - {"key":"Attr4","op":"LT","value":"2"}, - {"key":"Attr5","op":"LE","value":"3"} - ]}, - {"key":"Attr3","op":"GT","value":"1"} - ]} - ]}`) - }) -} diff --git a/policy/query.go b/policy/query.go deleted file mode 100644 index b1abd62..0000000 --- a/policy/query.go +++ /dev/null @@ -1,311 +0,0 @@ -package policy - -import ( - "errors" - "fmt" - "strconv" - "strings" - - "github.com/antlr/antlr4/runtime/Go/antlr" - "github.com/nspcc-dev/neofs-sdk-go/netmap" - "github.com/nspcc-dev/neofs-sdk-go/policy/parser" -) - -var ( - // ErrInvalidNumber is returned when a value of SELECT is 0. - ErrInvalidNumber = errors.New("policy: expected positive integer") - // ErrUnknownClause is returned when a statement(clause) in a query is unknown. - ErrUnknownClause = errors.New("policy: unknown clause") - // ErrUnknownOp is returned when an operation in a query is unknown. - ErrUnknownOp = errors.New("policy: unknown operation") - // ErrUnknownFilter is returned when a value of FROM in a query is unknown. - ErrUnknownFilter = errors.New("policy: filter not found") - // ErrUnknownSelector is returned when a value of IN is unknown. - ErrUnknownSelector = errors.New("policy: selector not found") - // ErrSyntaxError is returned for errors found by ANTLR parser. - ErrSyntaxError = errors.New("policy: syntax error") -) - -type policyVisitor struct { - errors []error - parser.BaseQueryVisitor - antlr.DefaultErrorListener -} - -// Parse parses s into a placement policy. -func Parse(s string) (*netmap.PlacementPolicy, error) { - return parse(s) -} - -func newPolicyVisitor() *policyVisitor { - return &policyVisitor{} -} - -func parse(s string) (*netmap.PlacementPolicy, error) { - input := antlr.NewInputStream(s) - lexer := parser.NewQueryLexer(input) - stream := antlr.NewCommonTokenStream(lexer, 0) - - p := parser.NewQuery(stream) - p.BuildParseTrees = true - - v := newPolicyVisitor() - p.RemoveErrorListeners() - p.AddErrorListener(v) - pl := p.Policy().Accept(v) - - if len(v.errors) != 0 { - return nil, v.errors[0] - } - if err := validatePolicy(pl.(*netmap.PlacementPolicy)); err != nil { - return nil, err - } - return pl.(*netmap.PlacementPolicy), nil -} - -func (p *policyVisitor) SyntaxError(recognizer antlr.Recognizer, offendingSymbol interface{}, line, column int, msg string, e antlr.RecognitionException) { - p.reportError(fmt.Errorf("%w: line %d:%d %s", ErrSyntaxError, line, column, msg)) -} - -func (p *policyVisitor) reportError(err error) interface{} { - p.errors = append(p.errors, err) - return nil -} - -// VisitPolicy implements parser.QueryVisitor interface. -func (p *policyVisitor) VisitPolicy(ctx *parser.PolicyContext) interface{} { - if len(p.errors) != 0 { - return nil - } - - pl := new(netmap.PlacementPolicy) - - repStmts := ctx.AllRepStmt() - rs := make([]netmap.Replica, 0, len(repStmts)) - for _, r := range repStmts { - res, ok := r.Accept(p).(*netmap.Replica) - if !ok { - return nil - } - - rs = append(rs, *res) - } - pl.SetReplicas(rs...) - - if cbfStmt := ctx.CbfStmt(); cbfStmt != nil { - cbf, ok := cbfStmt.(*parser.CbfStmtContext).Accept(p).(uint32) - if !ok { - return nil - } - pl.SetContainerBackupFactor(cbf) - } - - selStmts := ctx.AllSelectStmt() - ss := make([]netmap.Selector, 0, len(selStmts)) - for _, s := range selStmts { - res, ok := s.Accept(p).(*netmap.Selector) - if !ok { - return nil - } - - ss = append(ss, *res) - } - pl.SetSelectors(ss...) - - filtStmts := ctx.AllFilterStmt() - fs := make([]netmap.Filter, 0, len(filtStmts)) - for _, f := range filtStmts { - fs = append(fs, *f.Accept(p).(*netmap.Filter)) - } - pl.SetFilters(fs...) - - return pl -} - -func (p *policyVisitor) VisitCbfStmt(ctx *parser.CbfStmtContext) interface{} { - cbf, err := strconv.ParseUint(ctx.GetBackupFactor().GetText(), 10, 32) - if err != nil { - return p.reportError(ErrInvalidNumber) - } - - return uint32(cbf) -} - -// VisitRepStmt implements parser.QueryVisitor interface. -func (p *policyVisitor) VisitRepStmt(ctx *parser.RepStmtContext) interface{} { - num, err := strconv.ParseUint(ctx.GetCount().GetText(), 10, 32) - if err != nil { - return p.reportError(ErrInvalidNumber) - } - - rs := new(netmap.Replica) - rs.SetCount(uint32(num)) - - if sel := ctx.GetSelector(); sel != nil { - rs.SetSelector(sel.GetText()) - } - - return rs -} - -// VisitSelectStmt implements parser.QueryVisitor interface. -func (p *policyVisitor) VisitSelectStmt(ctx *parser.SelectStmtContext) interface{} { - res, err := strconv.ParseUint(ctx.GetCount().GetText(), 10, 32) - if err != nil { - return p.reportError(ErrInvalidNumber) - } - - s := new(netmap.Selector) - s.SetCount(uint32(res)) - - if clStmt := ctx.Clause(); clStmt != nil { - s.SetClause(clauseFromString(clStmt.GetText())) - } - - if bStmt := ctx.GetBucket(); bStmt != nil { - s.SetAttribute(ctx.GetBucket().GetText()) - } - - s.SetFilter(ctx.GetFilter().GetText()) // either ident or wildcard - - if ctx.AS() != nil { - s.SetName(ctx.GetName().GetText()) - } - return s -} - -// VisitFilterStmt implements parser.QueryVisitor interface. -func (p *policyVisitor) VisitFilterStmt(ctx *parser.FilterStmtContext) interface{} { - f := p.VisitFilterExpr(ctx.GetExpr().(*parser.FilterExprContext)).(*netmap.Filter) - f.SetName(ctx.GetName().GetText()) - return f -} - -func (p *policyVisitor) VisitFilterExpr(ctx *parser.FilterExprContext) interface{} { - if eCtx := ctx.Expr(); eCtx != nil { - return eCtx.Accept(p) - } - - if inner := ctx.GetInner(); inner != nil { - return inner.Accept(p) - } - - f := new(netmap.Filter) - op := operationFromString(ctx.GetOp().GetText()) - f.SetOperation(op) - - f1 := *ctx.GetF1().Accept(p).(*netmap.Filter) - f2 := *ctx.GetF2().Accept(p).(*netmap.Filter) - - // Consider f1=(.. AND ..) AND f2. This can be merged because our AND operation - // is of arbitrary arity. ANTLR generates left-associative parse-tree by default. - if f1.Operation() == op { - f.SetInnerFilters(append(f1.InnerFilters(), f2)...) - return f - } - - f.SetInnerFilters(f1, f2) - return f -} - -// VisitFilterKey implements parser.QueryVisitor interface. -func (p *policyVisitor) VisitFilterKey(ctx *parser.FilterKeyContext) interface{} { - if id := ctx.Ident(); id != nil { - return id.GetText() - } - - str := ctx.STRING().GetText() - return str[1 : len(str)-1] -} - -func (p *policyVisitor) VisitFilterValue(ctx *parser.FilterValueContext) interface{} { - if id := ctx.Ident(); id != nil { - return id.GetText() - } - - if num := ctx.Number(); num != nil { - return num.GetText() - } - - str := ctx.STRING().GetText() - return str[1 : len(str)-1] -} - -// VisitExpr implements parser.QueryVisitor interface. -func (p *policyVisitor) VisitExpr(ctx *parser.ExprContext) interface{} { - f := new(netmap.Filter) - if flt := ctx.GetFilter(); flt != nil { - f.SetName(flt.GetText()) - return f - } - - key := ctx.GetKey().Accept(p) - opStr := ctx.SIMPLE_OP().GetText() - value := ctx.GetValue().Accept(p) - - f.SetKey(key.(string)) - f.SetOperation(operationFromString(opStr)) - f.SetValue(value.(string)) - return f -} - -// validatePolicy checks high-level constraints such as filter link in SELECT -// being actually defined in FILTER section. -func validatePolicy(p *netmap.PlacementPolicy) error { - seenFilters := map[string]bool{} - for _, f := range p.Filters() { - seenFilters[f.Name()] = true - } - - seenSelectors := map[string]bool{} - for _, s := range p.Selectors() { - if flt := s.Filter(); flt != netmap.MainFilterName && !seenFilters[flt] { - return fmt.Errorf("%w: '%s'", ErrUnknownFilter, flt) - } - seenSelectors[s.Name()] = true - } - - for _, r := range p.Replicas() { - if sel := r.Selector(); sel != "" && !seenSelectors[sel] { - return fmt.Errorf("%w: '%s'", ErrUnknownSelector, sel) - } - } - - return nil -} - -func clauseFromString(s string) netmap.Clause { - switch strings.ToUpper(s) { - case "SAME": - return netmap.ClauseSame - case "DISTINCT": - return netmap.ClauseDistinct - default: - // Such errors should be handled by ANTLR code thus this panic. - panic(fmt.Errorf("BUG: invalid clause: %s", s)) - } -} - -func operationFromString(op string) netmap.Operation { - switch strings.ToUpper(op) { - case "AND": - return netmap.OpAND - case "OR": - return netmap.OpOR - case "EQ": - return netmap.OpEQ - case "NE": - return netmap.OpNE - case "GE": - return netmap.OpGE - case "GT": - return netmap.OpGT - case "LE": - return netmap.OpLE - case "LT": - return netmap.OpLT - default: - // Such errors should be handled by ANTLR code thus this panic. - panic(fmt.Errorf("BUG: invalid operation: %s", op)) - } -} diff --git a/policy/query_test.go b/policy/query_test.go deleted file mode 100644 index 888e481..0000000 --- a/policy/query_test.go +++ /dev/null @@ -1,324 +0,0 @@ -package policy - -import ( - "errors" - "fmt" - "math" - "testing" - - "github.com/nspcc-dev/neofs-sdk-go/netmap" - "github.com/stretchr/testify/require" -) - -func TestSimple(t *testing.T) { - q := `REP 3` - expected := new(netmap.PlacementPolicy) - expected.SetFilters([]netmap.Filter{}...) - expected.SetSelectors([]netmap.Selector{}...) - expected.SetReplicas(newReplica("", 3)) - - r, err := Parse(q) - require.NoError(t, err) - require.EqualValues(t, expected, r) -} - -func TestSimpleWithHRWB(t *testing.T) { - q := `REP 3 CBF 4` - expected := new(netmap.PlacementPolicy) - expected.SetFilters([]netmap.Filter{}...) - expected.SetSelectors([]netmap.Selector{}...) - expected.SetReplicas(newReplica("", 3)) - expected.SetContainerBackupFactor(4) - - r, err := Parse(q) - require.NoError(t, err) - require.EqualValues(t, expected, r) -} - -func TestFromSelect(t *testing.T) { - q := `REP 1 IN SPB -SELECT 1 IN City FROM * AS SPB` - expected := new(netmap.PlacementPolicy) - expected.SetFilters([]netmap.Filter{}...) - expected.SetSelectors(newSelector(1, netmap.ClauseUnspecified, "City", "*", "SPB")) - expected.SetReplicas(newReplica("SPB", 1)) - - r, err := Parse(q) - require.NoError(t, err) - require.EqualValues(t, expected, r) -} - -// https://github.com/nspcc-dev/neofs-node/issues/46 -func TestFromSelectNoAttribute(t *testing.T) { - t.Run("Simple", func(t *testing.T) { - q := `REP 2 - SELECT 6 FROM *` - - expected := new(netmap.PlacementPolicy) - expected.SetFilters([]netmap.Filter{}...) - expected.SetSelectors(newSelector(6, netmap.ClauseUnspecified, "", "*", "")) - expected.SetReplicas(newReplica("", 2)) - - r, err := Parse(q) - require.NoError(t, err) - require.EqualValues(t, expected, r) - }) - - t.Run("with filter", func(t *testing.T) { - q := `REP 2 - SELECT 6 FROM F - FILTER StorageType EQ SSD AS F` - - expected := new(netmap.PlacementPolicy) - expected.SetFilters(newFilter("F", "StorageType", "SSD", netmap.OpEQ)) - expected.SetSelectors(newSelector(6, netmap.ClauseUnspecified, "", "F", "")) - expected.SetReplicas(newReplica("", 2)) - - r, err := Parse(q) - require.NoError(t, err) - require.EqualValues(t, expected, r) - }) -} - -func TestString(t *testing.T) { - qTemplate := `REP 1 -SELECT 1 IN City FROM Filt -FILTER Property EQ %s AND Something NE 7 AS Filt` - - testCases := []string{ - `"double-quoted"`, - `"with ' single"`, - `'single-quoted'`, - `'with " double'`, - } - - for _, s := range testCases { - t.Run(s, func(t *testing.T) { - q := fmt.Sprintf(qTemplate, s) - r, err := Parse(q) - require.NoError(t, err) - - expected := newFilter("Filt", "", "", netmap.OpAND, - newFilter("", "Property", s[1:len(s)-1], netmap.OpEQ), - newFilter("", "Something", "7", netmap.OpNE)) - require.EqualValues(t, []netmap.Filter{expected}, r.Filters()) - }) - } -} - -func TestFromSelectClause(t *testing.T) { - q := `REP 4 -SELECT 3 IN Country FROM * -SELECT 2 IN SAME City FROM * -SELECT 1 IN DISTINCT Continent FROM *` - expected := new(netmap.PlacementPolicy) - expected.SetFilters([]netmap.Filter{}...) - expected.SetSelectors( - newSelector(3, netmap.ClauseUnspecified, "Country", "*", ""), - newSelector(2, netmap.ClauseSame, "City", "*", ""), - newSelector(1, netmap.ClauseDistinct, "Continent", "*", "")) - expected.SetReplicas(newReplica("", 4)) - - r, err := Parse(q) - require.NoError(t, err) - require.EqualValues(t, expected, r) -} - -func TestSimpleFilter(t *testing.T) { - q := `REP 1 -SELECT 1 IN City FROM Good -FILTER Rating GT 7 AS Good` - expected := new(netmap.PlacementPolicy) - expected.SetReplicas(newReplica("", 1)) - expected.SetSelectors( - newSelector(1, netmap.ClauseUnspecified, "City", "Good", "")) - expected.SetFilters(newFilter("Good", "Rating", "7", netmap.OpGT)) - - r, err := Parse(q) - require.NoError(t, err) - require.EqualValues(t, expected, r) -} - -func TestFilterReference(t *testing.T) { - q := `REP 1 -SELECT 2 IN City FROM Good -FILTER Country EQ "RU" AS FromRU -FILTER @FromRU AND Rating GT 7 AS Good` - expected := new(netmap.PlacementPolicy) - expected.SetReplicas(newReplica("", 1)) - expected.SetSelectors( - newSelector(2, netmap.ClauseUnspecified, "City", "Good", "")) - expected.SetFilters( - newFilter("FromRU", "Country", "RU", netmap.OpEQ), - newFilter("Good", "", "", netmap.OpAND, - newFilter("FromRU", "", "", 0), - newFilter("", "Rating", "7", netmap.OpGT)), - ) - - r, err := Parse(q) - require.NoError(t, err) - require.EqualValues(t, expected, r) -} - -func TestFilterOps(t *testing.T) { - q := `REP 1 -SELECT 2 IN City FROM Good -FILTER A GT 1 AND B GE 2 AND C LT 3 AND D LE 4 - AND E EQ 5 AND F NE 6 AS Good` - expected := new(netmap.PlacementPolicy) - expected.SetReplicas(newReplica("", 1)) - expected.SetSelectors( - newSelector(2, netmap.ClauseUnspecified, "City", "Good", "")) - expected.SetFilters(newFilter("Good", "", "", netmap.OpAND, - newFilter("", "A", "1", netmap.OpGT), - newFilter("", "B", "2", netmap.OpGE), - newFilter("", "C", "3", netmap.OpLT), - newFilter("", "D", "4", netmap.OpLE), - newFilter("", "E", "5", netmap.OpEQ), - newFilter("", "F", "6", netmap.OpNE))) - - r, err := Parse(q) - require.NoError(t, err) - require.EqualValues(t, expected, r) -} - -func TestWithFilterPrecedence(t *testing.T) { - q := `REP 7 IN SPB -SELECT 1 IN City FROM SPBSSD AS SPB -FILTER City EQ "SPB" AND SSD EQ true OR City EQ "SPB" AND Rating GE 5 AS SPBSSD` - expected := new(netmap.PlacementPolicy) - expected.SetReplicas(newReplica("SPB", 7)) - expected.SetSelectors( - newSelector(1, netmap.ClauseUnspecified, "City", "SPBSSD", "SPB")) - expected.SetFilters( - newFilter("SPBSSD", "", "", netmap.OpOR, - newFilter("", "", "", netmap.OpAND, - newFilter("", "City", "SPB", netmap.OpEQ), - newFilter("", "SSD", "true", netmap.OpEQ)), - newFilter("", "", "", netmap.OpAND, - newFilter("", "City", "SPB", netmap.OpEQ), - newFilter("", "Rating", "5", netmap.OpGE)))) - - r, err := Parse(q) - require.NoError(t, err) - require.EqualValues(t, expected, r) -} - -func TestBrackets(t *testing.T) { - q := `REP 7 IN SPB -SELECT 1 IN City FROM SPBSSD AS SPB -FILTER ( City EQ "SPB" OR SSD EQ true ) AND (City EQ "SPB" OR Rating GE 5) AS SPBSSD` - - expected := new(netmap.PlacementPolicy) - expected.SetReplicas(newReplica("SPB", 7)) - expected.SetSelectors( - newSelector(1, netmap.ClauseUnspecified, "City", "SPBSSD", "SPB")) - expected.SetFilters( - newFilter("SPBSSD", "", "", netmap.OpAND, - newFilter("", "", "", netmap.OpOR, - newFilter("", "City", "SPB", netmap.OpEQ), - newFilter("", "SSD", "true", netmap.OpEQ)), - newFilter("", "", "", netmap.OpOR, - newFilter("", "City", "SPB", netmap.OpEQ), - newFilter("", "Rating", "5", netmap.OpGE)))) - - r, err := Parse(q) - require.NoError(t, err) - require.EqualValues(t, expected, r) -} - -func TestValidation(t *testing.T) { - t.Run("MissingSelector", func(t *testing.T) { - q := `REP 3 IN RU` - _, err := Parse(q) - require.True(t, errors.Is(err, ErrUnknownSelector), "got: %v", err) - }) - t.Run("MissingFilter", func(t *testing.T) { - q := `REP 3 - SELECT 1 IN City FROM MissingFilter` - _, err := Parse(q) - require.True(t, errors.Is(err, ErrUnknownFilter), "got: %v", err) - }) - t.Run("UnknownOp", func(t *testing.T) { - q := `REP 3 - SELECT 1 IN City FROM F - FILTER Country KEK RU AS F` - _, err := Parse(q) - require.True(t, errors.Is(err, ErrSyntaxError), "got: %v", err) - }) - t.Run("TypoInREP", func(t *testing.T) { - q := `REK 3` - _, err := Parse(q) - require.True(t, errors.Is(err, ErrSyntaxError)) - }) - t.Run("InvalidFilterName", func(t *testing.T) { - q := `REP 3 - SELECT 1 IN City FROM F - FILTER Good AND Country EQ RU AS F - FILTER Rating EQ 5 AS Good` - _, err := Parse(q) - require.Error(t, err) - }) -} - -// Checks that an error is returned in cases when positive 32-bit integer is expected. -func TestInvalidNumbers(t *testing.T) { - tmpls := []string{ - "REP %d", - "REP 1 CBF %d", - "REP 1 SELECT %d FROM *", - } - for i := range tmpls { - zero := fmt.Sprintf(tmpls[i], 0) - t.Run(zero, func(t *testing.T) { - _, err := Parse(zero) - require.Error(t, err) - }) - - big := fmt.Sprintf(tmpls[i], int64(math.MaxUint32)+1) - t.Run(big, func(t *testing.T) { - _, err := Parse(big) - require.Error(t, err) - }) - } -} - -func TestFilterStringSymbols(t *testing.T) { - q := `REP 1 IN S -SELECT 1 FROM F AS S -FILTER "UN-LOCODE" EQ "RU LED" AS F` - - expected := new(netmap.PlacementPolicy) - expected.SetReplicas(newReplica("S", 1)) - expected.SetSelectors( - newSelector(1, netmap.ClauseUnspecified, "", "F", "S")) - expected.SetFilters(newFilter("F", "UN-LOCODE", "RU LED", netmap.OpEQ)) - - r, err := Parse(q) - require.NoError(t, err) - require.EqualValues(t, expected, r) -} - -func newFilter(name, key, value string, op netmap.Operation, sub ...netmap.Filter) (f netmap.Filter) { - f.SetName(name) - f.SetKey(key) - f.SetValue(value) - f.SetOperation(op) - f.SetInnerFilters(sub...) - return f -} - -func newReplica(s string, c uint32) (r netmap.Replica) { - r.SetSelector(s) - r.SetCount(c) - return r -} - -func newSelector(count uint32, c netmap.Clause, attr, f, name string) (s netmap.Selector) { - s.SetCount(count) - s.SetClause(c) - s.SetAttribute(attr) - s.SetFilter(f) - s.SetName(name) - return s -} diff --git a/pool/pool_test.go b/pool/pool_test.go index 49dcbb5..805bca8 100644 --- a/pool/pool_test.go +++ b/pool/pool_test.go @@ -45,7 +45,7 @@ func TestBuildPoolCreateSessionFailed(t *testing.T) { ctrl := gomock.NewController(t) ni := &netmap.NodeInfo{} - ni.SetAddresses("addr1", "addr2") + ni.SetNetworkEndpoints("addr1", "addr2") clientBuilder := func(_ string) (client, error) { mockClient := NewMockClient(ctrl)