frostfs-node/lib/netmap/netmap.go

package netmap

import (
	"crypto/sha256"
	"encoding/json"
	"reflect"
	"sort"
	"sync"

	"github.com/nspcc-dev/neofs-api-go/bootstrap"
	"github.com/nspcc-dev/netmap"
	"github.com/pkg/errors"
	"github.com/spaolacci/murmur3"
)

type (
	// Bucket is an alias for github.com/nspcc-dev/netmap.Bucket
	Bucket = netmap.Bucket
	// SFGroup is an alias for github.com/nspcc-dev/netmap.SFGroup
	SFGroup = netmap.SFGroup
	// Select is an alias for github.com/nspcc-dev/netmap.Select
	Select = netmap.Select
	// Filter is an alias for github.com/nspcc-dev/netmap.Filter
	Filter = netmap.Filter
	// SimpleFilter is an alias for github.com/nspcc-dev/netmap.Filter
	SimpleFilter = netmap.SimpleFilter
	// PlacementRule is an alias for github.com/nspcc-dev/netmap.Filter
	PlacementRule = netmap.PlacementRule

	// NetMap is a general network map structure for NeoFS
	NetMap struct {
		mu    *sync.RWMutex
		root  Bucket
		items Nodes
	}

	// Nodes is an alias for slice of NodeInfo which is structure that describes every host
	Nodes []bootstrap.NodeInfo
)

const (
	// Separator separates key:value pairs in string representation of options.
	Separator = netmap.Separator

	// NodesBucket is the name for optionless bucket containing only nodes.
	NodesBucket = netmap.NodesBucket
)

var (
	// FilterIn returns filter, which checks if value is in specified list.
	FilterIn = netmap.FilterIn
	// FilterNotIn returns filter, which checks if value is not in specified list.
	FilterNotIn = netmap.FilterNotIn
	// FilterOR returns OR combination of filters.
	FilterOR = netmap.FilterOR
	// FilterAND returns AND combination of filters.
	FilterAND = netmap.FilterAND
	// FilterEQ returns filter, which checks if value is equal to v.
	FilterEQ = netmap.FilterEQ
	// FilterNE returns filter, which checks if value is not equal to v.
	FilterNE = netmap.FilterNE
	// FilterGT returns filter, which checks if value is greater than v.
	FilterGT = netmap.FilterGT
	// FilterGE returns filter, which checks if value is greater or equal than v.
	FilterGE = netmap.FilterGE
	// FilterLT returns filter, which checks if value is less than v.
	FilterLT = netmap.FilterLT
	// FilterLE returns filter, which checks if value is less or equal than v.
	FilterLE = netmap.FilterLE
)

var errNetMapsConflict = errors.New("netmaps are in conflict")

// Copy creates new slice of copied nodes.
func (n Nodes) Copy() Nodes {
	res := make(Nodes, len(n))
	for i := range n {
		res[i].Address = n[i].Address
		res[i].Status = n[i].Status

		if n[i].PubKey != nil {
			res[i].PubKey = make([]byte, len(n[i].PubKey))
			copy(res[i].PubKey, n[i].PubKey)
		}

		if n[i].Options != nil {
			res[i].Options = make([]string, len(n[i].Options))
			copy(res[i].Options, n[i].Options)
		}
	}

	return res
}

// NewNetmap is an constructor.
func NewNetmap() *NetMap {
	return &NetMap{
		items: make([]bootstrap.NodeInfo, 0),
		mu:    new(sync.RWMutex),
	}
}

// Equals return whether two netmap are identical.
func (n *NetMap) Equals(nm *NetMap) bool {
	n.mu.RLock()
	defer n.mu.RUnlock()

	return len(n.items) == len(nm.items) &&
		n.root.Equals(nm.root) &&
		reflect.DeepEqual(n.items, nm.items)
}

// Root returns netmap root-bucket.
func (n *NetMap) Root() *Bucket {
	n.mu.RLock()
	cp := n.root.Copy()
	n.mu.RUnlock()

	return &cp
}

// Copy creates and returns full copy of target netmap.
func (n *NetMap) Copy() *NetMap {
	n.mu.RLock()
	defer n.mu.RUnlock()

	nm := NewNetmap()
	nm.items = n.items.Copy()
	nm.root = n.root.Copy()

	return nm
}

type hashedItem struct {
	h    uint32
	info *bootstrap.NodeInfo
}

// Normalise reorders netmap items into some canonical order.
func (n *NetMap) Normalise() *NetMap {
	nm := NewNetmap()
	items := n.items.Copy()

	if len(items) == 0 {
		return nm
	}

	itemsH := make([]hashedItem, len(n.items))
	for i := range itemsH {
		itemsH[i].h = murmur3.Sum32(n.items[i].PubKey)
		itemsH[i].info = &items[i]
	}

	sort.Slice(itemsH, func(i, j int) bool {
		if itemsH[i].h == itemsH[j].h {
			return itemsH[i].info.Address < itemsH[j].info.Address
		}
		return itemsH[i].h < itemsH[j].h
	})

	lastHash := ^itemsH[0].h
	lastAddr := ""

	for i := range itemsH {
		if itemsH[i].h != lastHash || itemsH[i].info.Address != lastAddr {
			_ = nm.AddNode(itemsH[i].info)
			lastHash = itemsH[i].h
		}
	}

	return nm
}

// Hash returns hash of n.
func (n *NetMap) Hash() (sum [32]byte) {
	items := n.Normalise().Items()
	w := sha256.New()

	for i := range items {
		data, _ := items[i].Marshal()
		_, _ = w.Write(data)
	}

	s := w.Sum(nil)
	copy(sum[:], s)

	return
}

// InheritWeights calculates average capacity and minimal price, then provides buckets with IQR weight.
func (n *NetMap) InheritWeights() *NetMap {
	nm := n.Copy()

	// find average capacity in the network map
	meanCap := nm.root.Traverse(netmap.NewMeanAgg(), netmap.CapWeightFunc).Compute()
	capNorm := netmap.NewSigmoidNorm(meanCap)

	// find minimal price in the network map
	minPrice := nm.root.Traverse(netmap.NewMinAgg(), netmap.PriceWeightFunc).Compute()
	priceNorm := netmap.NewReverseMinNorm(minPrice)

	// provide all buckets with
	wf := netmap.NewWeightFunc(capNorm, priceNorm)
	meanAF := netmap.AggregatorFactory{New: netmap.NewMeanIQRAgg}
	nm.root.TraverseTree(meanAF, wf)

	return nm
}

// Merge checks if merge is possible and then add new elements from given netmap.
func (n *NetMap) Merge(n1 *NetMap) error {
	n.mu.Lock()
	defer n.mu.Unlock()

	var (
		tr    = make(map[uint32]netmap.Node, len(n1.items))
		items = n.items
	)

loop:
	for j := range n1.items {
		for i := range n.items {
			if n.items[i].Equals(n1.items[j]) {
				tr[uint32(j)] = netmap.Node{
					N: uint32(i),
					C: n.items[i].Capacity(),
					P: n.items[i].Price(),
				}
				continue loop
			}
		}
		tr[uint32(j)] = netmap.Node{
			N: uint32(len(items)),
			C: n1.items[j].Capacity(),
			P: n1.items[j].Price(),
		}
		items = append(items, n1.items[j])
	}

	root := n1.root.UpdateIndices(tr)
	if n.root.CheckConflicts(root) {
		return errNetMapsConflict
	}

	n.items = items
	n.root.Merge(root)

	return nil
}

// FindGraph finds sub-graph filtered by given SFGroup.
func (n *NetMap) FindGraph(pivot []byte, ss ...SFGroup) (c *Bucket) {
	n.mu.RLock()
	defer n.mu.RUnlock()

	return n.root.FindGraph(pivot, ss...)
}

// FindNodes finds sub-graph filtered by given SFGroup and returns all sub-graph items.
func (n *NetMap) FindNodes(pivot []byte, ss ...SFGroup) (nodes []uint32) {
	n.mu.RLock()
	defer n.mu.RUnlock()

	return n.root.FindNodes(pivot, ss...).Nodes()
}

// Items return slice of all NodeInfo in netmap.
func (n *NetMap) Items() []bootstrap.NodeInfo {
	n.mu.RLock()
	defer n.mu.RUnlock()

	return n.items
}

// ItemsCopy return copied slice of all NodeInfo in netmap (is it useful?).
func (n *NetMap) ItemsCopy() Nodes {
	n.mu.RLock()
	defer n.mu.RUnlock()

	return n.items.Copy()
}

// Add adds node with given address and given options.
func (n *NetMap) Add(addr string, pk []byte, st bootstrap.NodeStatus, opts ...string) error {
	return n.AddNode(&bootstrap.NodeInfo{Address: addr, PubKey: pk, Status: st, Options: opts})
}

// Update replaces netmap with given netmap.
func (n *NetMap) Update(nxt *NetMap) {
	n.mu.Lock()
	defer n.mu.Unlock()

	n.root = nxt.root
	n.items = nxt.items
}

// GetMaxSelection returns 'maximal container' -- subgraph which contains
// any other subgraph satisfying specified selects and filters.
func (n *NetMap) GetMaxSelection(ss []Select, fs []Filter) (r *Bucket) {
	return n.root.GetMaxSelection(netmap.SFGroup{Selectors: ss, Filters: fs})
}

// AddNode adds to exited or new node slice of given options.
func (n *NetMap) AddNode(nodeInfo *bootstrap.NodeInfo, opts ...string) error {
	n.mu.Lock()
	defer n.mu.Unlock()

	info := *nodeInfo

	info.Options = append(info.Options, opts...)

	num := -1

	// looking for existed node info item
	for i := range n.items {
		if n.items[i].Equals(info) {
			num = i
			break
		}
	}
	// if item is not existed - add it
	if num < 0 {
		num = len(n.items)
		n.items = append(n.items, info)
	}

	return n.root.AddStrawNode(netmap.Node{
		N: uint32(num),
		C: n.items[num].Capacity(),
		P: n.items[num].Price(),
	}, info.Options...)
}

// GetNodesByOption returns slice of NodeInfo that has given option.
func (n *NetMap) GetNodesByOption(opts ...string) []bootstrap.NodeInfo {
	n.mu.RLock()
	defer n.mu.RUnlock()

	ns := n.root.GetNodesByOption(opts...)
	nodes := make([]bootstrap.NodeInfo, 0, len(ns))

	for _, info := range ns {
		nodes = append(nodes, n.items[info.N])
	}

	return nodes
}

// MarshalJSON custom marshaller.
func (n *NetMap) MarshalJSON() ([]byte, error) {
	n.mu.RLock()
	defer n.mu.RUnlock()

	return json.Marshal(n.items)
}

// UnmarshalJSON custom unmarshaller.
func (n *NetMap) UnmarshalJSON(data []byte) error {
	var (
		nm    = NewNetmap()
		items []bootstrap.NodeInfo
	)

	if err := json.Unmarshal(data, &items); err != nil {
		return err
	}

	for i := range items {
		if err := nm.Add(items[i].Address, items[i].PubKey, items[i].Status, items[i].Options...); err != nil {
			return err
		}
	}

	if n.mu == nil {
		n.mu = new(sync.RWMutex)
	}

	n.mu.Lock()
	n.root = nm.root
	n.items = nm.items
	n.mu.Unlock()

	return nil
}

// Size returns number of nodes in network map.
func (n *NetMap) Size() int {
	n.mu.RLock()
	defer n.mu.RUnlock()

	return len(n.items)
}