[#661] metrcis: Add rebuild percent metric

Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>

pkg/local_object_storage/blobstor/blobovniczatree/metrics.go

@@ -23,6 +23,7 @@ type Metrics interface {
 
 	SetRebuildStatus(status string)
 	ObjectMoved(d time.Duration)
+	SetRebuildPercent(value uint32)
 
 	Delete(d time.Duration, success, withStorageID bool)
 	Exists(d time.Duration, success, withStorageID bool)
@@ -38,6 +39,7 @@ func (m *noopMetrics) SetParentID(string)                      {}
 func (m *noopMetrics) SetMode(bool)                            {}
 func (m *noopMetrics) Close()                                  {}
 func (m *noopMetrics) SetRebuildStatus(string)                 {}
+func (m *noopMetrics) SetRebuildPercent(uint32)                {}
 func (m *noopMetrics) ObjectMoved(time.Duration)               {}
 func (m *noopMetrics) Delete(time.Duration, bool, bool)        {}
 func (m *noopMetrics) Exists(time.Duration, bool, bool)        {}

pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go

@@ -30,6 +30,7 @@ func (b *Blobovniczas) Rebuild(ctx context.Context, prm common.RebuildPrm) (comm
 	}
 
 	b.metrics.SetRebuildStatus(rebuildStatusRunning)
+	b.metrics.SetRebuildPercent(0)
 	success := true
 	defer func() {
 		if success {
@@ -71,6 +72,7 @@ func (b *Blobovniczas) Rebuild(ctx context.Context, prm common.RebuildPrm) (comm
 }
 
 func (b *Blobovniczas) migrateDBs(ctx context.Context, dbs []string, prm common.RebuildPrm, res common.RebuildRes) (common.RebuildRes, error) {
+	var completedDBCount uint32
 	for _, db := range dbs {
 		b.log.Debug(logs.BlobovniczaTreeRebuildingBlobovnicza, zap.String("path", db))
 		movedObjects, err := b.rebuildDB(ctx, db, prm.MetaStorage, prm.WorkerLimiter)
@@ -81,7 +83,10 @@ func (b *Blobovniczas) migrateDBs(ctx context.Context, dbs []string, prm common.
 		}
 		b.log.Debug(logs.BlobovniczaTreeRebuildingBlobovniczaSuccess, zap.String("path", db), zap.Uint64("moved_objects_count", movedObjects))
 		res.FilesRemoved++
+		completedDBCount++
+		b.metrics.SetRebuildPercent((100 * completedDBCount) / uint32(len(dbs)))
 	}
+	b.metrics.SetRebuildPercent(100)
 	return res, nil
 }
 

pkg/local_object_storage/metrics/blobovnicza.go

@@ -46,6 +46,10 @@ func (m *blobovniczaTreeMetrics) SetRebuildStatus(status string) {
 	m.m.BlobovniczaTreeRebuildStatus(m.shardID, m.path, status)
 }
 
+func (m *blobovniczaTreeMetrics) SetRebuildPercent(value uint32) {
+	m.m.BlobovniczaTreeRebuildPercent(m.shardID, m.path, value)
+}
+
 func (m *blobovniczaTreeMetrics) ObjectMoved(d time.Duration) {
 	m.m.BlobovniczaTreeObjectMoved(m.shardID, m.path, d)
 }

pkg/metrics/blobovnicza.go

@@ -25,6 +25,7 @@ type BlobobvnizcaMetrics interface {
 	DecOpenBlobovniczaCount(shardID, path string)
 
 	BlobovniczaTreeRebuildStatus(shardID, path, status string)
+	BlobovniczaTreeRebuildPercent(shardID, path string, value uint32)
 	BlobovniczaTreeObjectMoved(shardID, path string, d time.Duration)
 }
 
@@ -38,6 +39,7 @@ type blobovnicza struct {
 	treeOpenCounter        *prometheus.GaugeVec
 	treeObjectMoveDuration *prometheus.HistogramVec
 	treeRebuildStatus      *shardIDPathModeValue
+	treeRebuildPercent     *prometheus.GaugeVec
 }
 
 func newBlobovnicza() *blobovnicza {
@@ -87,6 +89,12 @@ func newBlobovnicza() *blobovnicza {
 			Help:      "Accumulated Blobovnicza tree object move duration",
 		}, []string{shardIDLabel, pathLabel}),
 		treeRebuildStatus: newShardIDPathMode(blobovniczaTreeSubSystem, "rebuild_status", "Blobovnicza tree rebuild status"),
+		treeRebuildPercent: metrics.NewGaugeVec(prometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: blobovniczaTreeSubSystem,
+			Name:      "rebuild_complete_percent",
+			Help:      "Percent of rebuild completeness",
+		}, []string{shardIDLabel, pathLabel}),
 	}
 }
 
@@ -112,6 +120,10 @@ func (b *blobovnicza) CloseBlobobvnizcaTree(shardID, path string) {
 		shardIDLabel: shardID,
 		pathLabel:    path,
 	})
+	b.treeRebuildPercent.DeletePartialMatch(prometheus.Labels{
+		shardIDLabel: shardID,
+		pathLabel:    path,
+	})
 	b.treeRebuildStatus.SetMode(shardID, path, undefinedStatus)
 }
 
@@ -191,3 +203,10 @@ func (b *blobovnicza) BlobovniczaTreeObjectMoved(shardID, path string, d time.Du
 		pathLabel:    path,
 	}).Observe(d.Seconds())
 }
+
+func (b *blobovnicza) BlobovniczaTreeRebuildPercent(shardID, path string, value uint32) {
+	b.treeRebuildPercent.With(prometheus.Labels{
+		shardIDLabel: shardID,
+		pathLabel:    path,
+	}).Set(float64(value))
+}