frostfs-node/pkg/services/object_manager/replication/implementations.go
Stanislav Bogatyrev b7b5079934 Add Inner Ring code
2020-07-24 17:07:37 +03:00

294 lines
7.1 KiB
Go

package replication
import (
"context"
"sync"
"github.com/multiformats/go-multiaddr"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/localstore"
"github.com/nspcc-dev/neofs-node/pkg/services/object_manager/placement"
"github.com/nspcc-dev/neofs-node/pkg/util/rand"
"github.com/nspcc-dev/netmap"
"github.com/pkg/errors"
)
type (
replicationScheduler struct {
cac ContainerActualityChecker
ls localstore.Iterator
}
// SchedulerParams groups the parameters of scheduler constructor.
SchedulerParams struct {
ContainerActualityChecker
localstore.Iterator
}
objectPool struct {
mu *sync.Mutex
tasks []Address
}
multiSolver struct {
as AddressStore
pl placement.Component
}
// MultiSolverParams groups the parameters of multi solver constructor.
MultiSolverParams struct {
AddressStore
Placement placement.Component
}
)
const (
objectPoolInstanceFailMsg = "could not create object pool"
multiSolverInstanceFailMsg = "could not create multi solver"
replicationSchedulerEntity = "replication scheduler"
)
var (
errPoolExhausted = errors.New("object pool is exhausted")
errEmptyLister = errors.New("empty local objects lister")
errEmptyContainerActual = errors.New("empty container actuality checker")
errEmptyAddressStore = errors.New("empty address store")
errEmptyPlacement = errors.New("empty placement")
)
// NewObjectPool is an object pool constructor.
func NewObjectPool() ObjectPool {
return &objectPool{mu: new(sync.Mutex)}
}
// NewReplicationScheduler is a replication scheduler constructor.
func NewReplicationScheduler(p SchedulerParams) (Scheduler, error) {
switch {
case p.ContainerActualityChecker == nil:
return nil, errors.Wrap(errEmptyContainerActual, objectPoolInstanceFailMsg)
case p.Iterator == nil:
return nil, errors.Wrap(errEmptyLister, objectPoolInstanceFailMsg)
}
return &replicationScheduler{
cac: p.ContainerActualityChecker,
ls: p.Iterator,
}, nil
}
// NewMultiSolver is a multi solver constructor.
func NewMultiSolver(p MultiSolverParams) (MultiSolver, error) {
switch {
case p.Placement == nil:
return nil, errors.Wrap(errEmptyPlacement, multiSolverInstanceFailMsg)
case p.AddressStore == nil:
return nil, errors.Wrap(errEmptyAddressStore, multiSolverInstanceFailMsg)
}
return &multiSolver{
as: p.AddressStore,
pl: p.Placement,
}, nil
}
func (s *objectPool) Update(pool []Address) {
s.mu.Lock()
defer s.mu.Unlock()
s.tasks = pool
}
func (s *objectPool) Undone() int {
s.mu.Lock()
defer s.mu.Unlock()
return len(s.tasks)
}
func (s *objectPool) Pop() (Address, error) {
s.mu.Lock()
defer s.mu.Unlock()
if len(s.tasks) == 0 {
return Address{}, errPoolExhausted
}
head := s.tasks[0]
s.tasks = s.tasks[1:]
return head, nil
}
func (s *replicationScheduler) SelectForReplication(limit int) ([]Address, error) {
// Attention! This routine might be inefficient with big number of objects
// and containers. Consider using fast traversal and filtering algorithms
// with sieve of bloom filters.
migration := make([]Address, 0, limit)
replication := make([]Address, 0)
ctx := context.Background()
if err := s.ls.Iterate(nil, func(meta *localstore.ObjectMeta) bool {
if s.cac.Actual(ctx, meta.Object.SystemHeader.CID) {
replication = append(replication, *meta.Object.Address())
} else {
migration = append(migration, *meta.Object.Address())
}
return len(migration) >= limit
}); err != nil {
return nil, err
}
lnM := len(migration)
lnR := len(replication)
edge := 0
// I considered using rand.Perm() and appending elements in `for` cycle.
// But it seems, that shuffling is efficient even when `limit-lnM`
// is 1000 times smaller than `lnR`. But it can be discussed and changed
// later anyway.
if lnM < limit {
r := rand.New()
r.Shuffle(lnR, func(i, j int) {
replication[i], replication[j] = replication[j], replication[i]
})
edge = min(limit-lnM, lnR)
}
return append(migration, replication[:edge]...), nil
}
func (s *multiSolver) Epoch() uint64 { return s.pl.NetworkState().Epoch }
func (s *multiSolver) SelfAddr() (multiaddr.Multiaddr, error) { return s.as.SelfAddr() }
func (s *multiSolver) ReservationRatio(ctx context.Context, addr Address) (int, error) {
graph, err := s.pl.Query(ctx, placement.ContainerID(addr.CID))
if err != nil {
return 0, errors.Wrap(err, "reservation ratio computation failed on placement query")
}
nodes, err := graph.Filter(func(group netmap.SFGroup, bucket *netmap.Bucket) *netmap.Bucket {
return bucket.GetSelection(group.Selectors, addr.ObjectID.Bytes())
}).NodeList()
if err != nil {
return 0, errors.Wrap(err, "reservation ratio computation failed on graph node list")
}
return len(nodes), nil
}
func (s *multiSolver) SelectRemoteStorages(ctx context.Context, addr Address, excl ...multiaddr.Multiaddr) ([]ObjectLocation, error) {
selfAddr, err := s.as.SelfAddr()
if err != nil {
return nil, errors.Wrap(err, "select remote storage nodes failed on get self address")
}
nodes, err := s.selectNodes(ctx, addr, excl...)
if err != nil {
return nil, errors.Wrap(err, "select remote storage nodes failed on get node list")
}
var (
metSelf bool
selfIndex = -1
res = make([]ObjectLocation, 0, len(nodes))
)
for i := range nodes {
if nodes[i].Equal(selfAddr) {
metSelf = true
selfIndex = i
}
res = append(res, ObjectLocation{
Node: nodes[i],
WeightGreater: !metSelf,
})
}
if selfIndex != -1 {
res = append(res[:selfIndex], res[selfIndex+1:]...)
}
return res, nil
}
func (s *multiSolver) selectNodes(ctx context.Context, addr Address, excl ...multiaddr.Multiaddr) ([]multiaddr.Multiaddr, error) {
graph, err := s.pl.Query(ctx, placement.ContainerID(addr.CID))
if err != nil {
return nil, errors.Wrap(err, "select remote storage nodes failed on placement query")
}
filter := func(group netmap.SFGroup, bucket *netmap.Bucket) *netmap.Bucket { return bucket }
if !addr.ObjectID.Empty() {
filter = func(group netmap.SFGroup, bucket *netmap.Bucket) *netmap.Bucket {
return bucket.GetSelection(group.Selectors, addr.ObjectID.Bytes())
}
}
return graph.Exclude(excl).Filter(filter).NodeList()
}
func (s *multiSolver) Actual(ctx context.Context, cid CID) bool {
graph, err := s.pl.Query(ctx, placement.ContainerID(cid))
if err != nil {
return false
}
nodes, err := graph.NodeList()
if err != nil {
return false
}
selfAddr, err := s.as.SelfAddr()
if err != nil {
return false
}
for i := range nodes {
if nodes[i].Equal(selfAddr) {
return true
}
}
return false
}
func (s *multiSolver) CompareWeight(ctx context.Context, addr Address, node multiaddr.Multiaddr) int {
selfAddr, err := s.as.SelfAddr()
if err != nil {
return -1
}
if selfAddr.Equal(node) {
return 0
}
excl := make([]multiaddr.Multiaddr, 0)
for {
nodes, err := s.selectNodes(ctx, addr, excl...)
if err != nil {
return -1
}
for j := range nodes {
if nodes[j].Equal(selfAddr) {
return -1
} else if nodes[j].Equal(node) {
return 1
}
}
excl = append(excl, nodes[0]) // TODO: when it will become relevant to append full nodes slice
}
}
func min(a, b int) int {
if a < b {
return a
}
return b
}