[#661] blobovniczatree: Make Rebuild concurrent

Different DBs can be rebuild concurrently.

Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>

pkg/local_object_storage/shard/control.go

@@ -162,7 +162,7 @@ func (s *Shard) Init(ctx context.Context) error {
 
 	s.gc.init(ctx)
 
-	s.rb = newRebuilder()
+	s.rb = newRebuilder(s.rebuildLimiter)
 	s.rb.Start(ctx, s.blobStor, s.metaBase, s.log)
 
 	return nil

pkg/local_object_storage/shard/rebuild_limiter.go

@@ -0,0 +1,13 @@
+package shard
+
+import "context"
+
+type RebuildWorkerLimiter interface {
+	AcquireWorkSlot(ctx context.Context) error
+	ReleaseWorkSlot()
+}
+
+type noopRebuildLimiter struct{}
+
+func (l *noopRebuildLimiter) AcquireWorkSlot(context.Context) error { return nil }
+func (l *noopRebuildLimiter) ReleaseWorkSlot()                      {}

pkg/local_object_storage/shard/rebuilder.go

@@ -14,16 +14,18 @@ import (
 )
 
 type rebuilder struct {
-	mtx    *sync.Mutex
-	wg     *sync.WaitGroup
-	cancel func()
+	mtx     *sync.Mutex
+	wg      *sync.WaitGroup
+	cancel  func()
+	limiter RebuildWorkerLimiter
 }
 
-func newRebuilder() *rebuilder {
+func newRebuilder(l RebuildWorkerLimiter) *rebuilder {
 	return &rebuilder{
-		mtx:    &sync.Mutex{},
-		wg:     &sync.WaitGroup{},
-		cancel: nil,
+		mtx:     &sync.Mutex{},
+		wg:      &sync.WaitGroup{},
+		cancel:  nil,
+		limiter: l,
 	}
 }
 
@@ -45,7 +47,7 @@ func (r *rebuilder) start(ctx context.Context, bs *blobstor.BlobStor, mb *meta.D
 		defer r.wg.Done()
 
 		log.Info(logs.BlobstoreRebuildStarted)
-		if err := bs.Rebuild(ctx, &mbStorageIDUpdate{mb: mb}); err != nil {
+		if err := bs.Rebuild(ctx, &mbStorageIDUpdate{mb: mb}, r.limiter); err != nil {
 			log.Warn(logs.FailedToRebuildBlobstore, zap.Error(err))
 		} else {
 			log.Info(logs.BlobstoreRebuildCompletedSuccessfully)

pkg/local_object_storage/shard/shard.go

@@ -127,6 +127,8 @@ type cfg struct {
 	metricsWriter MetricsWriter
 
 	reportErrorFunc func(selfID string, message string, err error)
+
+	rebuildLimiter RebuildWorkerLimiter
 }
 
 func defaultCfg() *cfg {
@@ -135,6 +137,7 @@ func defaultCfg() *cfg {
 		log:             &logger.Logger{Logger: zap.L()},
 		gcCfg:           defaultGCCfg(),
 		reportErrorFunc: func(string, string, error) {},
+		rebuildLimiter:  &noopRebuildLimiter{},
 	}
 }
 
@@ -372,6 +375,14 @@ func WithExpiredCollectorWorkerCount(count int) Option {
 	}
 }
 
+// WithRebuildWorkerLimiter return option to set concurrent
+// workers count of storage rebuild operation.
+func WithRebuildWorkerLimiter(l RebuildWorkerLimiter) Option {
+	return func(c *cfg) {
+		c.rebuildLimiter = l
+	}
+}
+
 func (s *Shard) fillInfo() {
 	s.cfg.info.MetaBaseInfo = s.metaBase.DumpInfo()
 	s.cfg.info.BlobStorInfo = s.blobStor.DumpInfo()