[#199] sdk/netmap: Correct linter's remarks

Signed-off-by: Leonard Lyubich <leonard@nspcc.ru>
This commit is contained in:
Leonard Lyubich 2020-11-16 17:52:15 +03:00 committed by Alex Vanin
parent 459d295788
commit 3a966ee5df
9 changed files with 69 additions and 37 deletions

View file

@ -73,12 +73,6 @@ var (
_ normalizer = (*constNorm)(nil) _ normalizer = (*constNorm)(nil)
) )
// capWeightFunc calculates weight which is equal to capacity.
func capWeightFunc(n *Node) float64 { return float64(n.Capacity) }
// priceWeightFunc calculates weight which is equal to price.
func priceWeightFunc(n *Node) float64 { return float64(n.Price) }
// newWeightFunc returns weightFunc which multiplies normalized // newWeightFunc returns weightFunc which multiplies normalized
// capacity and price. // capacity and price.
func newWeightFunc(capNorm, priceNorm normalizer) weightFunc { func newWeightFunc(capNorm, priceNorm normalizer) weightFunc {
@ -87,12 +81,6 @@ func newWeightFunc(capNorm, priceNorm normalizer) weightFunc {
} }
} }
// newMeanSumAgg returns an aggregator which
// computes mean value by keeping total sum.
func newMeanSumAgg() aggregator {
return new(meanSumAgg)
}
// newMeanAgg returns an aggregator which // newMeanAgg returns an aggregator which
// computes mean value by recalculating it on // computes mean value by recalculating it on
// every addition. // every addition.
@ -106,12 +94,6 @@ func newMinAgg() aggregator {
return new(minAgg) return new(minAgg)
} }
// newMaxAgg returns an aggregator which
// computes max value.
func newMaxAgg() aggregator {
return new(maxAgg)
}
// newMeanIQRAgg returns an aggregator which // newMeanIQRAgg returns an aggregator which
// computes mean value of values from IQR interval. // computes mean value of values from IQR interval.
func newMeanIQRAgg() aggregator { func newMeanIQRAgg() aggregator {
@ -124,24 +106,12 @@ func newReverseMinNorm(min float64) normalizer {
return &reverseMinNorm{min: min} return &reverseMinNorm{min: min}
} }
// newMaxNorm returns a normalizer which
// normalize values in range of 0.0 to 1.0 to a maximum value.
func newMaxNorm(max float64) normalizer {
return &maxNorm{max: max}
}
// newSigmoidNorm returns a normalizer which // newSigmoidNorm returns a normalizer which
// normalize values in range of 0.0 to 1.0 to a scaled sigmoid. // normalize values in range of 0.0 to 1.0 to a scaled sigmoid.
func newSigmoidNorm(scale float64) normalizer { func newSigmoidNorm(scale float64) normalizer {
return &sigmoidNorm{scale: scale} return &sigmoidNorm{scale: scale}
} }
// newConstNorm returns a normalizer which
// returns a constant values
func newConstNorm(value float64) normalizer {
return &constNorm{value: value}
}
func (a *meanSumAgg) Add(n float64) { func (a *meanSumAgg) Add(n float64) {
a.sum += n a.sum += n
a.count++ a.count++
@ -151,6 +121,7 @@ func (a *meanSumAgg) Compute() float64 {
if a.count == 0 { if a.count == 0 {
return 0 return 0
} }
return a.sum / float64(a.count) return a.sum / float64(a.count)
} }
@ -197,22 +168,27 @@ func (a *meanIQRAgg) Compute() float64 {
sort.Slice(a.arr, func(i, j int) bool { return a.arr[i] < a.arr[j] }) sort.Slice(a.arr, func(i, j int) bool { return a.arr[i] < a.arr[j] })
var min, max float64 var min, max float64
if l < 4 {
const minLn = 4
if l < minLn {
min, max = a.arr[0], a.arr[l-1] min, max = a.arr[0], a.arr[l-1]
} else { } else {
start, end := l/4, l*3/4-1 start, end := l/minLn, l*3/minLn-1
iqr := a.k * (a.arr[end] - a.arr[start]) iqr := a.k * (a.arr[end] - a.arr[start])
min, max = a.arr[start]-iqr, a.arr[end]+iqr min, max = a.arr[start]-iqr, a.arr[end]+iqr
} }
count := 0 count := 0
sum := float64(0) sum := float64(0)
for _, e := range a.arr { for _, e := range a.arr {
if e >= min && e <= max { if e >= min && e <= max {
sum += e sum += e
count++ count++
} }
} }
return sum / float64(count) return sum / float64(count)
} }
@ -220,6 +196,7 @@ func (r *reverseMinNorm) Normalize(w float64) float64 {
if w == 0 { if w == 0 {
return 0 return 0
} }
return r.min / w return r.min / w
} }
@ -227,6 +204,7 @@ func (r *maxNorm) Normalize(w float64) float64 {
if r.max == 0 { if r.max == 0 {
return 0 return 0
} }
return w / r.max return w / r.max
} }
@ -234,7 +212,9 @@ func (r *sigmoidNorm) Normalize(w float64) float64 {
if r.scale == 0 { if r.scale == 0 {
return 0 return 0
} }
x := w / r.scale x := w / r.scale
return x / (1 + x) return x / (1 + x)
} }

View file

@ -45,7 +45,7 @@ func (c Clause) ToV2() netmap.Clause {
func (c Clause) String() string { func (c Clause) String() string {
switch c { switch c {
default: default:
return "UNSPECIFIED" return "CLAUSE_UNSPECIFIED"
case ClauseDistinct: case ClauseDistinct:
return "DISTINCT" return "DISTINCT"
case ClauseSame: case ClauseSame:

View file

@ -70,10 +70,12 @@ func (c *Context) setPivot(pivot []byte) {
func GetDefaultWeightFunc(ns Nodes) weightFunc { func GetDefaultWeightFunc(ns Nodes) weightFunc {
mean := newMeanAgg() mean := newMeanAgg()
min := newMinAgg() min := newMinAgg()
for i := range ns { for i := range ns {
mean.Add(float64(ns[i].Capacity)) mean.Add(float64(ns[i].Capacity))
min.Add(float64(ns[i].Price)) min.Add(float64(ns[i].Price))
} }
return newWeightFunc( return newWeightFunc(
newSigmoidNorm(mean.Compute()), newSigmoidNorm(mean.Compute()),
newReverseMinNorm(min.Compute())) newReverseMinNorm(min.Compute()))

View file

@ -26,6 +26,7 @@ func (c *Context) processFilters(p *PlacementPolicy) error {
return err return err
} }
} }
return nil return nil
} }
@ -33,15 +34,19 @@ func (c *Context) processFilter(f *Filter, top bool) error {
if f == nil { if f == nil {
return fmt.Errorf("%w: FILTER", ErrMissingField) return fmt.Errorf("%w: FILTER", ErrMissingField)
} }
if f.Name() == MainFilterName { if f.Name() == MainFilterName {
return fmt.Errorf("%w: '*' is reserved", ErrInvalidFilterName) return fmt.Errorf("%w: '*' is reserved", ErrInvalidFilterName)
} }
if top && f.Name() == "" { if top && f.Name() == "" {
return ErrUnnamedTopFilter return ErrUnnamedTopFilter
} }
if !top && f.Name() != "" && c.Filters[f.Name()] == nil { if !top && f.Name() != "" && c.Filters[f.Name()] == nil {
return fmt.Errorf("%w: '%s'", ErrFilterNotFound, f.Name()) return fmt.Errorf("%w: '%s'", ErrFilterNotFound, f.Name())
} }
switch f.Operation() { switch f.Operation() {
case OpAND, OpOR: case OpAND, OpOR:
for _, flt := range f.InnerFilters() { for _, flt := range f.InnerFilters() {
@ -55,6 +60,7 @@ func (c *Context) processFilter(f *Filter, top bool) error {
} else if !top && f.Name() != "" { // named reference } else if !top && f.Name() != "" { // named reference
return nil return nil
} }
switch f.Operation() { switch f.Operation() {
case OpEQ, OpNE: case OpEQ, OpNE:
case OpGT, OpGE, OpLT, OpLE: case OpGT, OpGE, OpLT, OpLE:
@ -62,14 +68,17 @@ func (c *Context) processFilter(f *Filter, top bool) error {
if err != nil { if err != nil {
return fmt.Errorf("%w: '%s'", ErrInvalidNumber, f.Value()) return fmt.Errorf("%w: '%s'", ErrInvalidNumber, f.Value())
} }
c.numCache[f] = n c.numCache[f] = n
default: default:
return fmt.Errorf("%w: %s", ErrInvalidFilterOp, f.Operation()) return fmt.Errorf("%w: %s", ErrInvalidFilterOp, f.Operation())
} }
} }
if top { if top {
c.Filters[f.Name()] = f c.Filters[f.Name()] = f
} }
return nil return nil
} }
@ -83,11 +92,13 @@ func (c *Context) match(f *Filter, b *Node) bool {
if lf.Name() != "" { if lf.Name() != "" {
lf = c.Filters[lf.Name()] lf = c.Filters[lf.Name()]
} }
ok := c.match(lf, b) ok := c.match(lf, b)
if ok == (f.Operation() == OpOR) { if ok == (f.Operation() == OpOR) {
return ok return ok
} }
} }
return f.Operation() == OpAND return f.Operation() == OpAND
default: default:
return c.matchKeyValue(f, b) return c.matchKeyValue(f, b)
@ -102,6 +113,7 @@ func (c *Context) matchKeyValue(f *Filter, b *Node) bool {
return b.Attribute(f.Key()) != f.Value() return b.Attribute(f.Key()) != f.Value()
default: default:
var attr uint64 var attr uint64
switch f.Key() { switch f.Key() {
case PriceAttr: case PriceAttr:
attr = b.Price attr = b.Price
@ -109,6 +121,7 @@ func (c *Context) matchKeyValue(f *Filter, b *Node) bool {
attr = b.Capacity attr = b.Capacity
default: default:
var err error var err error
attr, err = strconv.ParseUint(b.Attribute(f.Key()), 10, 64) attr, err = strconv.ParseUint(b.Attribute(f.Key()), 10, 64)
if err != nil { if err != nil {
// Note: because filters are somewhat independent from nodes attributes, // Note: because filters are somewhat independent from nodes attributes,
@ -116,6 +129,7 @@ func (c *Context) matchKeyValue(f *Filter, b *Node) bool {
return false return false
} }
} }
switch f.Operation() { switch f.Operation() {
case OpGT: case OpGT:
return attr > c.numCache[f] return attr > c.numCache[f]

View file

@ -23,6 +23,7 @@ func flattenNodes(ns []Nodes) Nodes {
for i := range ns { for i := range ns {
result = append(result, ns[i]...) result = append(result, ns[i]...)
} }
return result return result
} }
@ -31,11 +32,13 @@ func (m *Netmap) GetPlacementVectors(cnt ContainerNodes, pivot []byte) ([]Nodes,
h := hrw.Hash(pivot) h := hrw.Hash(pivot)
wf := GetDefaultWeightFunc(m.Nodes) wf := GetDefaultWeightFunc(m.Nodes)
result := make([]Nodes, len(cnt.Replicas())) result := make([]Nodes, len(cnt.Replicas()))
for i, rep := range cnt.Replicas() { for i, rep := range cnt.Replicas() {
result[i] = make(Nodes, len(rep)) result[i] = make(Nodes, len(rep))
copy(result[i], rep) copy(result[i], rep)
hrw.SortSliceByWeightValue(result[i], result[i].Weights(wf), h) hrw.SortSliceByWeightValue(result[i], result[i].Weights(wf), h)
} }
return result, nil return result, nil
} }
@ -45,28 +48,35 @@ func (m *Netmap) GetPlacementVectors(cnt ContainerNodes, pivot []byte) ([]Nodes,
func (m *Netmap) GetContainerNodes(p *PlacementPolicy, pivot []byte) (ContainerNodes, error) { func (m *Netmap) GetContainerNodes(p *PlacementPolicy, pivot []byte) (ContainerNodes, error) {
c := NewContext(m) c := NewContext(m)
c.setPivot(pivot) c.setPivot(pivot)
if err := c.processFilters(p); err != nil { if err := c.processFilters(p); err != nil {
return nil, err return nil, err
} }
if err := c.processSelectors(p); err != nil { if err := c.processSelectors(p); err != nil {
return nil, err return nil, err
} }
result := make([]Nodes, len(p.Replicas())) result := make([]Nodes, len(p.Replicas()))
for i, r := range p.Replicas() { for i, r := range p.Replicas() {
if r == nil { if r == nil {
return nil, fmt.Errorf("%w: REPLICA", ErrMissingField) return nil, fmt.Errorf("%w: REPLICA", ErrMissingField)
} }
if r.Selector() == "" { if r.Selector() == "" {
for _, s := range p.Selectors() { for _, s := range p.Selectors() {
result[i] = append(result[i], flattenNodes(c.Selections[s.Name()])...) result[i] = append(result[i], flattenNodes(c.Selections[s.Name()])...)
} }
} }
nodes, ok := c.Selections[r.Selector()] nodes, ok := c.Selections[r.Selector()]
if !ok { if !ok {
return nil, fmt.Errorf("%w: REPLICA '%s'", ErrSelectorNotFound, r.Selector()) return nil, fmt.Errorf("%w: REPLICA '%s'", ErrSelectorNotFound, r.Selector())
} }
result[i] = append(result[i], flattenNodes(nodes)...)
result[i] = append(result[i], flattenNodes(nodes)...)
} }
return containerNodes(result), nil return containerNodes(result), nil
} }

View file

@ -63,6 +63,7 @@ func NodesFromInfo(infos []NodeInfo) Nodes {
for i := range infos { for i := range infos {
nodes[i] = newNodeV2(i, &infos[i]) nodes[i] = newNodeV2(i, &infos[i])
} }
return nodes return nodes
} }
@ -73,6 +74,7 @@ func newNodeV2(index int, ni *NodeInfo) *Node {
AttrMap: make(map[string]string, len(ni.Attributes())), AttrMap: make(map[string]string, len(ni.Attributes())),
NodeInfo: ni, NodeInfo: ni,
} }
for _, attr := range ni.Attributes() { for _, attr := range ni.Attributes() {
switch attr.Key() { switch attr.Key() {
case CapacityAttr: case CapacityAttr:
@ -80,8 +82,10 @@ func newNodeV2(index int, ni *NodeInfo) *Node {
case PriceAttr: case PriceAttr:
n.Price, _ = strconv.ParseUint(attr.Value(), 10, 64) n.Price, _ = strconv.ParseUint(attr.Value(), 10, 64)
} }
n.AttrMap[attr.Key()] = attr.Value() n.AttrMap[attr.Key()] = attr.Value()
} }
return n return n
} }
@ -91,6 +95,7 @@ func (n Nodes) Weights(wf weightFunc) []float64 {
for i := range n { for i := range n {
w = append(w, wf(n[i])) w = append(w, wf(n[i]))
} }
return w return w
} }
@ -104,6 +109,7 @@ func GetBucketWeight(ns Nodes, a aggregator, wf weightFunc) float64 {
for i := range ns { for i := range ns {
a.Add(wf(ns[i])) a.Add(wf(ns[i]))
} }
return a.Compute() return a.Compute()
} }
@ -134,7 +140,7 @@ func (s NodeState) ToV2() netmap.NodeState {
func (s NodeState) String() string { func (s NodeState) String() string {
switch s { switch s {
default: default:
return "UNSPECIFIED" return "STATE_UNSPECIFIED"
case NodeStateOffline: case NodeStateOffline:
return "OFFLINE" return "OFFLINE"
case NodeStateOnline: case NodeStateOnline:

View file

@ -86,7 +86,7 @@ func (op Operation) ToV2() netmap.Operation {
func (op Operation) String() string { func (op Operation) String() string {
switch op { switch op {
default: default:
return "UNSPECIFIED" return "OPERATION_UNSPECIFIED"
case OpNE: case OpNE:
return "NE" return "NE"
case OpEQ: case OpEQ:

View file

@ -22,13 +22,17 @@ func (c *Context) processSelectors(p *PlacementPolicy) error {
return fmt.Errorf("%w: SELECT FROM '%s'", ErrFilterNotFound, s.Filter()) return fmt.Errorf("%w: SELECT FROM '%s'", ErrFilterNotFound, s.Filter())
} }
} }
c.Selectors[s.Name()] = s c.Selectors[s.Name()] = s
result, err := c.getSelection(p, s) result, err := c.getSelection(p, s)
if err != nil { if err != nil {
return err return err
} }
c.Selections[s.Name()] = result c.Selections[s.Name()] = result
} }
return nil return nil
} }
@ -47,6 +51,7 @@ func GetNodesCount(p *PlacementPolicy, s *Selector) (int, int) {
func (c *Context) getSelection(p *PlacementPolicy, s *Selector) ([]Nodes, error) { func (c *Context) getSelection(p *PlacementPolicy, s *Selector) ([]Nodes, error) {
bucketCount, nodesInBucket := GetNodesCount(p, s) bucketCount, nodesInBucket := GetNodesCount(p, s)
buckets := c.getSelectionBase(s) buckets := c.getSelectionBase(s)
if len(buckets) < bucketCount { if len(buckets) < bucketCount {
return nil, fmt.Errorf("%w: '%s'", ErrNotEnoughNodes, s.Name()) return nil, fmt.Errorf("%w: '%s'", ErrNotEnoughNodes, s.Name())
} }
@ -65,22 +70,27 @@ func (c *Context) getSelection(p *PlacementPolicy, s *Selector) ([]Nodes, error)
} }
nodes := make([]Nodes, 0, len(buckets)) nodes := make([]Nodes, 0, len(buckets))
for i := range buckets { for i := range buckets {
ns := buckets[i].nodes ns := buckets[i].nodes
if len(ns) >= nodesInBucket { if len(ns) >= nodesInBucket {
nodes = append(nodes, ns[:nodesInBucket]) nodes = append(nodes, ns[:nodesInBucket])
} }
} }
if len(nodes) < bucketCount { if len(nodes) < bucketCount {
return nil, fmt.Errorf("%w: '%s'", ErrNotEnoughNodes, s.Name()) return nil, fmt.Errorf("%w: '%s'", ErrNotEnoughNodes, s.Name())
} }
if len(c.pivot) != 0 { if len(c.pivot) != 0 {
weights := make([]float64, len(nodes)) weights := make([]float64, len(nodes))
for i := range nodes { for i := range nodes {
weights[i] = GetBucketWeight(nodes[i], c.aggregator(), c.weightFunc) weights[i] = GetBucketWeight(nodes[i], c.aggregator(), c.weightFunc)
} }
hrw.SortSliceByWeightIndex(nodes, weights, c.pivotHash) hrw.SortSliceByWeightIndex(nodes, weights, c.pivotHash)
} }
return nodes[:bucketCount], nil return nodes[:bucketCount], nil
} }
@ -97,6 +107,7 @@ func (c *Context) getSelectionBase(s *Selector) []nodeAttrPair {
result := []nodeAttrPair{} result := []nodeAttrPair{}
nodeMap := map[string]Nodes{} nodeMap := map[string]Nodes{}
attr := s.Attribute() attr := s.Attribute()
for i := range c.Netmap.Nodes { for i := range c.Netmap.Nodes {
if isMain || c.match(f, c.Netmap.Nodes[i]) { if isMain || c.match(f, c.Netmap.Nodes[i]) {
if attr == "" { if attr == "" {
@ -108,6 +119,7 @@ func (c *Context) getSelectionBase(s *Selector) []nodeAttrPair {
} }
} }
} }
if attr != "" { if attr != "" {
for k, ns := range nodeMap { for k, ns := range nodeMap {
result = append(result, nodeAttrPair{attr: k, nodes: ns}) result = append(result, nodeAttrPair{attr: k, nodes: ns})
@ -119,6 +131,7 @@ func (c *Context) getSelectionBase(s *Selector) []nodeAttrPair {
hrw.SortSliceByWeightValue(result[i].nodes, result[i].nodes.Weights(c.weightFunc), c.pivotHash) hrw.SortSliceByWeightValue(result[i].nodes, result[i].nodes.Weights(c.weightFunc), c.pivotHash)
} }
} }
return result return result
} }

View file

@ -163,7 +163,10 @@ func TestPlacementPolicy_ProcessSelectorsHRW(t *testing.T) {
c := NewContext(nm) c := NewContext(nm)
c.setPivot([]byte("containerID")) c.setPivot([]byte("containerID"))
c.weightFunc = newWeightFunc(newMaxNorm(10000), newReverseMinNorm(1)) c.weightFunc = newWeightFunc(newMaxNorm(10000), newReverseMinNorm(1))
c.aggregator = newMaxAgg c.aggregator = func() aggregator {
return new(maxAgg)
}
require.NoError(t, c.processFilters(p)) require.NoError(t, c.processFilters(p))
require.NoError(t, c.processSelectors(p)) require.NoError(t, c.processSelectors(p))
@ -186,6 +189,10 @@ func TestPlacementPolicy_ProcessSelectorsHRW(t *testing.T) {
require.Equal(t, res, cnt) require.Equal(t, res, cnt)
} }
func newMaxNorm(max float64) normalizer {
return &maxNorm{max: max}
}
func TestPlacementPolicy_ProcessSelectorsInvalid(t *testing.T) { func TestPlacementPolicy_ProcessSelectorsInvalid(t *testing.T) {
testCases := []struct { testCases := []struct {
name string name string