From 80ef4492c16edb716ff31607a480dda368b36022 Mon Sep 17 00:00:00 2001 From: Leonard Lyubich Date: Wed, 12 May 2021 11:39:29 +0300 Subject: [PATCH] [#493] node: Get rid of outdated object GC worker Signed-off-by: Leonard Lyubich --- cmd/neofs-node/config.go | 8 -- cmd/neofs-node/object.go | 30 ------ pkg/services/object_manager/gc/gc.go | 137 ------------------------- pkg/services/object_manager/gc/opts.go | 42 -------- 4 files changed, 217 deletions(-) delete mode 100644 pkg/services/object_manager/gc/gc.go delete mode 100644 pkg/services/object_manager/gc/opts.go diff --git a/cmd/neofs-node/config.go b/cmd/neofs-node/config.go index 6866890f..b2317f83 100644 --- a/cmd/neofs-node/config.go +++ b/cmd/neofs-node/config.go @@ -92,10 +92,6 @@ const ( // config keys for cfgReputation cfgReputationContract = "reputation.scripthash" - cfgGCQueueSize = "gc.queuesize" - cfgGCQueueTick = "gc.duration.sleep" - cfgGCTimeout = "gc.duration.timeout" - cfgPolicerWorkScope = "policer.work_scope" cfgPolicerExpRate = "policer.expansion_rate" cfgPolicerHeadTimeout = "policer.head_timeout" @@ -466,10 +462,6 @@ func defaultConfiguration(v *viper.Viper) { v.SetDefault(cfgMetricsShutdownTimeout, "30s") - v.SetDefault(cfgGCQueueSize, 1000) - v.SetDefault(cfgGCQueueTick, "5s") - v.SetDefault(cfgGCTimeout, "5s") - v.SetDefault(cfgPolicerWorkScope, 100) v.SetDefault(cfgPolicerExpRate, 10) // in % v.SetDefault(cfgPolicerHeadTimeout, 5*time.Second) diff --git a/cmd/neofs-node/object.go b/cmd/neofs-node/object.go index c875b789..468ad61e 100644 --- a/cmd/neofs-node/object.go +++ b/cmd/neofs-node/object.go @@ -34,7 +34,6 @@ import ( searchsvc "github.com/nspcc-dev/neofs-node/pkg/services/object/search" searchsvcV2 "github.com/nspcc-dev/neofs-node/pkg/services/object/search/v2" "github.com/nspcc-dev/neofs-node/pkg/services/object/util" - "github.com/nspcc-dev/neofs-node/pkg/services/object_manager/gc" "github.com/nspcc-dev/neofs-node/pkg/services/object_manager/placement" "github.com/nspcc-dev/neofs-node/pkg/services/policer" "github.com/nspcc-dev/neofs-node/pkg/services/replicator" @@ -94,26 +93,12 @@ func (s *objectSvc) GetRangeHash(ctx context.Context, req *object.GetRangeHashRe return s.get.GetRangeHash(ctx, req) } -type localObjectRemover struct { - storage *engine.StorageEngine - - log *logger.Logger -} - type localObjectInhumer struct { storage *engine.StorageEngine log *logger.Logger } -func (r *localObjectRemover) Delete(addr ...*objectSDK.Address) error { - _, err := r.storage.Delete(new(engine.DeletePrm). - WithAddresses(addr...), - ) - - return err -} - func (r *localObjectInhumer) DeleteObjects(ts *objectSDK.Address, addr ...*objectSDK.Address) { prm := new(engine.InhumePrm) @@ -182,26 +167,11 @@ func initObjectService(c *cfg) { sidechain: c.cfgMorph.client, } - objRemover := &localObjectRemover{ - storage: ls, - log: c.log, - } - objInhumer := &localObjectInhumer{ storage: ls, log: c.log, } - objGC := gc.New( - gc.WithLogger(c.log), - gc.WithRemover(objRemover), - gc.WithQueueCapacity(c.viper.GetUint32(cfgGCQueueSize)), - gc.WithSleepInterval(c.viper.GetDuration(cfgGCQueueTick)), - gc.WithWorkingInterval(c.viper.GetDuration(cfgGCTimeout)), - ) - - c.workers = append(c.workers, objGC) - repl := replicator.New( replicator.WithLogger(c.log), replicator.WithPutTimeout( diff --git a/pkg/services/object_manager/gc/gc.go b/pkg/services/object_manager/gc/gc.go deleted file mode 100644 index e52ee84c..00000000 --- a/pkg/services/object_manager/gc/gc.go +++ /dev/null @@ -1,137 +0,0 @@ -package gc - -import ( - "context" - "time" - - "github.com/mr-tron/base58" - "github.com/nspcc-dev/neofs-api-go/pkg/container" - "github.com/nspcc-dev/neofs-api-go/pkg/object" - "github.com/nspcc-dev/neofs-node/pkg/util/logger" - "go.uber.org/zap" -) - -// GC represents an object garbage collector. -type GC struct { - *cfg - - timer *time.Timer - - queue chan *object.Address -} - -// Option represents GC constructor option. -type Option func(*cfg) - -type cfg struct { - sleepInterval, workInterval time.Duration - - queueCap uint32 - - log *logger.Logger - - remover Remover -} - -// Remover is an interface of the component that stores objects. -type Remover interface { - // Delete removes object from physical storage. - Delete(...*object.Address) error -} - -func defaultCfg() *cfg { - return &cfg{ - sleepInterval: 5 * time.Second, - workInterval: 5 * time.Second, - queueCap: 10, - log: zap.L(), - } -} - -// New creates, initializes and returns GC instance. -func New(opts ...Option) *GC { - cfg := defaultCfg() - - for i := range opts { - opts[i](cfg) - } - - cfg.log = cfg.log.With(zap.String("component", "Object GC")) - - return &GC{ - cfg: cfg, - } -} - -func (gc *GC) Run(ctx context.Context) { - defer func() { - close(gc.queue) - gc.timer.Stop() - gc.log.Info("routine stopped") - }() - - gc.log.Info("process routine", - zap.Uint32("queue capacity", gc.queueCap), - zap.Duration("sleep interval", gc.sleepInterval), - zap.Duration("working interval", gc.workInterval), - ) - - gc.queue = make(chan *object.Address, gc.queueCap) - gc.timer = time.NewTimer(gc.sleepInterval) - - for { - select { - case <-ctx.Done(): - gc.log.Warn("context is done", - zap.String("error", ctx.Err().Error()), - ) - - return - case _, ok := <-gc.timer.C: - if !ok { - gc.log.Warn("timer is stopped") - - return - } - - abort := time.After(gc.workInterval) - - loop: - for { - select { - case <-ctx.Done(): - gc.log.Warn("context is done", - zap.String("error", ctx.Err().Error()), - ) - - return - case <-abort: - break loop - case addr, ok := <-gc.queue: - if !ok { - gc.log.Warn("queue channel is closed") - } else if err := gc.remover.Delete(addr); err != nil { - gc.log.Error("could not remove object", - zap.String("error", err.Error()), - ) - } else { - gc.log.Info("object removed", - zap.String("CID", stringifyCID(addr.ContainerID())), - zap.String("ID", stringifyID(addr.ObjectID())), - ) - } - } - } - - gc.timer.Reset(gc.sleepInterval) - } - } -} - -func stringifyID(addr *object.ID) string { - return base58.Encode(addr.ToV2().GetValue()) -} - -func stringifyCID(addr *container.ID) string { - return base58.Encode(addr.ToV2().GetValue()) -} diff --git a/pkg/services/object_manager/gc/opts.go b/pkg/services/object_manager/gc/opts.go deleted file mode 100644 index fa159cf7..00000000 --- a/pkg/services/object_manager/gc/opts.go +++ /dev/null @@ -1,42 +0,0 @@ -package gc - -import ( - "time" - - "github.com/nspcc-dev/neofs-node/pkg/util/logger" -) - -// WithRemover returns option to set object remover. -func WithRemover(v Remover) Option { - return func(c *cfg) { - c.remover = v - } -} - -// WithLogger returns option to set logging component. -func WithLogger(v *logger.Logger) Option { - return func(c *cfg) { - c.log = v - } -} - -// WithQueueCapacity returns option to set delete queue capacity. -func WithQueueCapacity(v uint32) Option { - return func(c *cfg) { - c.queueCap = v - } -} - -// WithWorkingInterval returns option to set working interval of GC. -func WithWorkingInterval(v time.Duration) Option { - return func(c *cfg) { - c.workInterval = v - } -} - -// WithSleepInteval returns option to set sleep interval of GC. -func WithSleepInterval(v time.Duration) Option { - return func(c *cfg) { - c.sleepInterval = v - } -}