TrueCloudLab/frostfs-node
19
1
Fork 27
Code Issues 90 Pull requests 4 Projects Releases 1 Activity Actions

[#965] policer: Implement continuous replication

Browse source 
Signed-off-by: Alex Vanin <alexey@nspcc.ru>
This commit is contained in:
Alex Vanin 2021-11-10 11:34:00 +03:00 committed by Alex Vanin
parent 20f0b29a6e
commit a74a402a7d
5 changed files with 145 additions and 159 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
cmd/neofs-node/config.go
View file

@ -197,6 +197,10 @@ type cfgLocalStorage struct {
type cfgObjectRoutines struct {
putRemote *ants.Pool
putRemoteCapacity int
replication *ants.Pool
}
type cfgControlService struct {
@ -449,10 +453,13 @@ func initObjectPool(cfg *config.Config) (pool cfgObjectRoutines) {
optNonBlocking := ants.WithNonblocking(true)
pool.putRemote, err = ants.NewPool(objectconfig.Put(cfg).PoolSizeRemote(), optNonBlocking)
if err != nil {
pool.putRemoteCapacity = objectconfig.Put(cfg).PoolSizeRemote()
pool.putRemote, err = ants.NewPool(pool.putRemoteCapacity, optNonBlocking)
fatalOnErr(err)
pool.replication, err = ants.NewPool(pool.putRemoteCapacity)
fatalOnErr(err)
}
return pool
}
@ -486,3 +493,10 @@ func (c *cfg) bootstrap() error {
func (c *cfg) needBootstrap() bool {
return c.cfgNetmap.needBootstrap
}
// ObjectServiceLoad implements system loader interface for policer component.
// It is calculated as size/capacity ratio of "remote object put" worker.
// Returns float value between 0.0 and 1.0.
func (c *cfg) ObjectServiceLoad() float64 {
return float64(c.cfgObject.pool.putRemote.Running()) / float64(c.cfgObject.pool.putRemoteCapacity)
}

19
cmd/neofs-node/object.go
View file

@ -17,7 +17,6 @@ import (
morphClient "github.com/nspcc-dev/neofs-node/pkg/morph/client"
cntrwrp "github.com/nspcc-dev/neofs-node/pkg/morph/client/container/wrapper"
nmwrp "github.com/nspcc-dev/neofs-node/pkg/morph/client/netmap/wrapper"
"github.com/nspcc-dev/neofs-node/pkg/morph/event"
objectTransportGRPC "github.com/nspcc-dev/neofs-node/pkg/network/transport/object/grpc"
objectService "github.com/nspcc-dev/neofs-node/pkg/services/object"
"github.com/nspcc-dev/neofs-node/pkg/services/object/acl"
@ -232,8 +231,6 @@ func initObjectService(c *cfg) {
c.workers = append(c.workers, repl)
ch := make(chan *policer.Task, 1)
pol := policer.New(
policer.WithLogger(c.log),
policer.WithLocalStorage(ls),
@ -241,9 +238,6 @@ func initObjectService(c *cfg) {
policer.WithPlacementBuilder(
placement.NewNetworkMapSourceBuilder(c.cfgObject.netMapSource),
),
policer.WithWorkScope(100),
policer.WithExpansionRate(10),
policer.WithTrigger(ch),
policer.WithRemoteHeader(
headsvc.NewRemoteHeader(keyStorage, clientConstructor),
),
@ -260,18 +254,11 @@ func initObjectService(c *cfg) {
)
}
}),
policer.WithMaxCapacity(c.cfgObject.pool.putRemoteCapacity),
policer.WithPool(c.cfgObject.pool.replication),
policer.WithNodeLoader(c),
)
addNewEpochNotificationHandler(c, func(ev event.Event) {
select {
case ch <- new(policer.Task):
case <-c.ctx.Done():
close(ch)
default:
c.log.Info("policer is busy")
}
})
traverseGen := util.NewTraverserGenerator(c.cfgObject.netMapSource, c.cfgObject.cnrSource, c)
c.workers = append(c.workers, pol)

83
pkg/services/policer/policer.go
View file

@ -1,9 +1,9 @@
package policer
import (
"sync"
"time"
lru "github.com/hashicorp/golang-lru"
"github.com/nspcc-dev/neofs-node/pkg/core/container"
"github.com/nspcc-dev/neofs-node/pkg/core/netmap"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine"
@ -12,15 +12,22 @@ import (
"github.com/nspcc-dev/neofs-node/pkg/services/replicator"
"github.com/nspcc-dev/neofs-node/pkg/util/logger"
"github.com/nspcc-dev/neofs-sdk-go/object"
"github.com/panjf2000/ants/v2"
"go.uber.org/zap"
)
// NodeLoader provides application load statistics.
type nodeLoader interface {
// ObjectServiceLoad returns object service load value in [0:1] range.
ObjectServiceLoad() float64
}
// Policer represents the utility that verifies
// compliance with the object storage policy.
type Policer struct {
*cfg
prevTask prevTask
cache *lru.Cache
}
// Option is an option for Policer constructor.
@ -33,12 +40,8 @@ type RedundantCopyCallback func(*object.Address)
type cfg struct {
headTimeout time.Duration
workScope workScope
log *logger.Logger
trigger <-chan *Task
jobQueue jobQueue
cnrSrc container.Source
@ -52,11 +55,25 @@ type cfg struct {
replicator *replicator.Replicator
cbRedundantCopy RedundantCopyCallback
taskPool *ants.Pool
loader nodeLoader
maxCapacity int
batchSize, cacheSize uint32
rebalanceFreq, evictDuration time.Duration
}
func defaultCfg() *cfg {
return &cfg{
log: zap.L(),
batchSize: 10,
cacheSize: 200_000, // should not allocate more than 200 MiB
rebalanceFreq: 1 * time.Second,
evictDuration: 30 * time.Second,
}
}
@ -70,12 +87,14 @@ func New(opts ...Option) *Policer {
c.log = c.log.With(zap.String("component", "Object Policer"))
cache, err := lru.New(int(c.cacheSize))
if err != nil {
panic(err)
}
return &Policer{
cfg: c,
prevTask: prevTask{
cancel: func() {},
wait: new(sync.WaitGroup),
},
cache: cache,
}
}
@ -86,27 +105,6 @@ func WithHeadTimeout(v time.Duration) Option {
}
}
// WithWorkScope returns option to set job work scope value of Policer.
func WithWorkScope(v int) Option {
return func(c *cfg) {
c.workScope.val = v
}
}
// WithExpansionRate returns option to set expansion rate of Policer's works scope (in %).
func WithExpansionRate(v int) Option {
return func(c *cfg) {
c.workScope.expRate = v
}
}
// WithTrigger returns option to set triggering channel of Policer.
func WithTrigger(v <-chan *Task) Option {
return func(c *cfg) {
c.trigger = v
}
}
// WithLogger returns option to set Logger of Policer.
func WithLogger(v *logger.Logger) Option {
return func(c *cfg) {
@ -164,3 +162,26 @@ func WithRedundantCopyCallback(cb RedundantCopyCallback) Option {
c.cbRedundantCopy = cb
}
}
// WithMaxCapacity returns option to set max capacity
// that can be set to the pool.
func WithMaxCapacity(cap int) Option {
return func(c *cfg) {
c.maxCapacity = cap
}
}
// WithPool returns option to set pool for
// policy and replication operations.
func WithPool(p *ants.Pool) Option {
return func(c *cfg) {
c.taskPool = p
}
}
// WithNodeLoader returns option to set NeoFS node load source.
func WithNodeLoader(l nodeLoader) Option {
return func(c *cfg) {
c.loader = l
}
}

139
pkg/services/policer/process.go
View file

@ -2,113 +2,84 @@ package policer
import (
"context"
"sync"
"errors"
"time"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine"
"github.com/nspcc-dev/neofs-sdk-go/object"
"go.uber.org/zap"
)
// Task represents group of Policer tact parameters.
type Task struct{}
type prevTask struct {
undone int
cancel context.CancelFunc
wait *sync.WaitGroup
}
type workScope struct {
val int
expRate int // in %
}
func (p *Policer) Run(ctx context.Context) {
defer func() {
p.log.Info("routine stopped")
}()
p.log.Info("process routine",
zap.Int("work scope value", p.workScope.val),
zap.Int("expansion rate (%)", p.workScope.val),
zap.Duration("head timeout", p.headTimeout),
go p.poolCapacityWorker(ctx)
p.shardPolicyWorker(ctx)
}
func (p *Policer) shardPolicyWorker(ctx context.Context) {
var (
addrs []*object.Address
cursor *engine.Cursor
err error
)
for {
select {
case <-ctx.Done():
p.prevTask.cancel()
p.log.Warn("context is done",
zap.String("error", ctx.Err().Error()),
)
return
case task, ok := <-p.trigger:
if !ok {
p.log.Warn("trigger channel is closed")
return
}
p.prevTask.cancel()
p.prevTask.wait.Wait()
var taskCtx context.Context
taskCtx, p.prevTask.cancel = context.WithCancel(ctx)
go p.handleTask(taskCtx, task)
}
}
}
func (p *Policer) handleTask(ctx context.Context, task *Task) {
p.prevTask.wait.Add(1)
defer func() {
p.prevTask.wait.Done()
p.log.Info("finish work",
zap.Int("amount of unfinished objects", p.prevTask.undone),
)
}()
var delta int
// undone - amount of objects we couldn't process in last epoch
if p.prevTask.undone > 0 {
// if there are unprocessed objects, then lower your estimation
delta = -p.prevTask.undone
} else {
// otherwise try to expand
delta = p.workScope.val * p.workScope.expRate / 100
}
addrs, err := p.jobQueue.Select(p.workScope.val + delta)
addrs, cursor, err = p.jobQueue.Select(cursor, p.batchSize)
if err != nil {
p.log.Warn("could not select objects",
zap.String("error", err.Error()),
)
if errors.Is(err, engine.ErrEndOfListing) {
time.Sleep(time.Second) // finished whole cycle, sleep a bit
continue
}
// if there are NOT enough objects to fill the pool, do not change it
// otherwise expand or shrink it with the delta value
if len(addrs) >= p.workScope.val+delta {
p.workScope.val += delta
p.log.Warn("failure at object select for replication", zap.Error(err))
}
p.prevTask.undone = len(addrs)
for i := range addrs {
select {
case <-ctx.Done():
return
default:
addr := addrs[i]
addrStr := addr.String()
err = p.taskPool.Submit(func() {
v, ok := p.cache.Get(addrStr)
if ok && time.Since(v.(time.Time)) < p.evictDuration {
return
}
p.processObject(ctx, addrs[i])
p.processObject(ctx, addr)
p.cache.Add(addrStr, time.Now())
})
if err != nil {
p.log.Warn("pool submission", zap.Error(err))
}
}
}
}
}
p.prevTask.undone--
func (p *Policer) poolCapacityWorker(ctx context.Context) {
ticker := time.NewTicker(p.rebalanceFreq)
for {
select {
case <-ctx.Done():
ticker.Stop()
return
case <-ticker.C:
neofsSysLoad := p.loader.ObjectServiceLoad()
newCapacity := int((1.0 - neofsSysLoad) * float64(p.maxCapacity))
if newCapacity == 0 {
newCapacity++
}
if p.taskPool.Cap() != newCapacity {
p.taskPool.Tune(newCapacity)
p.log.Debug("tune replication capacity",
zap.Float64("system_load", neofsSysLoad),
zap.Int("new_capacity", newCapacity))
}
}
}
}

25
pkg/services/policer/queue.go
View file

@ -1,8 +1,9 @@
package policer
import (
"fmt"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine"
"github.com/nspcc-dev/neofs-node/pkg/util/rand"
"github.com/nspcc-dev/neofs-sdk-go/object"
)
@ -10,23 +11,15 @@ type jobQueue struct {
localStorage *engine.StorageEngine
}
func (q *jobQueue) Select(limit int) ([]*object.Address, error) {
// TODO: optimize the logic for selecting objects
// We can prioritize objects for migration, newly arrived objects, etc.
// It is recommended to make changes after updating the metabase
func (q *jobQueue) Select(cursor *engine.Cursor, count uint32) ([]*object.Address, *engine.Cursor, error) {
prm := new(engine.ListWithCursorPrm)
prm.WithCursor(cursor)
prm.WithCount(count)
res, err := engine.List(q.localStorage, 0) // consider some limit
res, err := q.localStorage.ListWithCursor(prm)
if err != nil {
return nil, err
return nil, nil, fmt.Errorf("cannot list objects in engine: %w", err)
}
rand.New().Shuffle(len(res), func(i, j int) {
res[i], res[j] = res[j], res[i]
})
if len(res) < limit {
return res, nil
}
return res[:limit], nil
return res.AddressList(), res.Cursor(), nil
}