[#661] metrics: Add blobovniczatree rebuild metrics

Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>

pkg/local_object_storage/blobstor/blobovniczatree/control.go

@@ -19,6 +19,7 @@ var errFailedToChangeExtensionReadOnly = errors.New("failed to change blobovnicz
 func (b *Blobovniczas) Open(readOnly bool) error {
 	b.readOnly = readOnly
 	b.metrics.SetMode(readOnly)
+	b.metrics.SetRebuildStatus(rebuildStatusNotStarted)
 	b.openManagers()
 	return nil
 }

pkg/local_object_storage/blobstor/blobovniczatree/metrics.go

@@ -6,6 +6,13 @@ import (
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza"
 )
 
+const (
+	rebuildStatusNotStarted = "not_started"
+	rebuildStatusRunning    = "running"
+	rebuildStatusCompleted  = "completed"
+	rebuildStatusFailed     = "failed"
+)
+
 type Metrics interface {
 	Blobovnicza() blobovnicza.Metrics
 
@@ -14,6 +21,9 @@ type Metrics interface {
 	SetMode(readOnly bool)
 	Close()
 
+	SetRebuildStatus(status string)
+	ObjectMoved(d time.Duration)
+
 	Delete(d time.Duration, success, withStorageID bool)
 	Exists(d time.Duration, success, withStorageID bool)
 	GetRange(d time.Duration, size int, success, withStorageID bool)
@@ -27,6 +37,8 @@ type noopMetrics struct{}
 func (m *noopMetrics) SetParentID(string)                      {}
 func (m *noopMetrics) SetMode(bool)                            {}
 func (m *noopMetrics) Close()                                  {}
+func (m *noopMetrics) SetRebuildStatus(string)                 {}
+func (m *noopMetrics) ObjectMoved(time.Duration)               {}
 func (m *noopMetrics) Delete(time.Duration, bool, bool)        {}
 func (m *noopMetrics) Exists(time.Duration, bool, bool)        {}
 func (m *noopMetrics) GetRange(time.Duration, int, bool, bool) {}

pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go

@@ -23,6 +23,16 @@ func (b *Blobovniczas) Rebuild(ctx context.Context, prm common.RebuildPrm) (comm
 		return common.RebuildRes{}, common.ErrReadOnly
 	}
 
+	b.metrics.SetRebuildStatus(rebuildStatusRunning)
+	success := true
+	defer func() {
+		if success {
+			b.metrics.SetRebuildStatus(rebuildStatusCompleted)
+		} else {
+			b.metrics.SetRebuildStatus(rebuildStatusFailed)
+		}
+	}()
+
 	b.rebuildGuard.Lock()
 	defer b.rebuildGuard.Unlock()
 
@@ -33,6 +43,7 @@ func (b *Blobovniczas) Rebuild(ctx context.Context, prm common.RebuildPrm) (comm
 	res.ObjectsMoved += completedPreviosMoves
 	if err != nil {
 		b.log.Warn(logs.BlobovniczaTreeCompletedPreviousRebuildFailed, zap.Error(err))
+		success = false
 		return res, err
 	}
 	b.log.Debug(logs.BlobovniczaTreeCompletedPreviousRebuildSuccess)
@@ -41,6 +52,7 @@ func (b *Blobovniczas) Rebuild(ctx context.Context, prm common.RebuildPrm) (comm
 	dbsToMigrate, err := b.getDBsToRebuild(ctx)
 	if err != nil {
 		b.log.Warn(logs.BlobovniczaTreeCollectingDBToRebuildFailed, zap.Error(err))
+		success = false
 		return res, err
 	}
 	b.log.Info(logs.BlobovniczaTreeCollectingDBToRebuildSuccess, zap.Int("blobovniczas_to_rebuild", len(dbsToMigrate)))
@@ -50,6 +62,7 @@ func (b *Blobovniczas) Rebuild(ctx context.Context, prm common.RebuildPrm) (comm
 		res.ObjectsMoved += movedObjects
 		if err != nil {
 			b.log.Warn(logs.BlobovniczaTreeRebuildingBlobovniczaFailed, zap.String("path", db), zap.Uint64("moved_objects_count", movedObjects), zap.Error(err))
+			success = false
 			return res, err
 		}
 		b.log.Debug(logs.BlobovniczaTreeRebuildingBlobovniczaSuccess, zap.String("path", db), zap.Uint64("moved_objects_count", movedObjects))
@@ -121,6 +134,10 @@ func (b *Blobovniczas) moveObjects(ctx context.Context, blz *blobovnicza.Blobovn
 
 func (b *Blobovniczas) moveObject(ctx context.Context, source *blobovnicza.Blobovnicza, sourcePath string,
 	addr oid.Address, data []byte, metaStore common.MetaStorage) error {
+	startedAt := time.Now()
+	defer func() {
+		b.metrics.ObjectMoved(time.Since(startedAt))
+	}()
 	it := &moveIterator{
 		B:             b,
 		ID:            nil,

pkg/local_object_storage/metrics/blobovnicza.go

@@ -42,6 +42,14 @@ func (m *blobovniczaTreeMetrics) Close() {
 	m.m.CloseBlobobvnizcaTree(m.shardID, m.path)
 }
 
+func (m *blobovniczaTreeMetrics) SetRebuildStatus(status string) {
+	m.m.BlobovniczaTreeRebuildStatus(m.shardID, m.path, status)
+}
+
+func (m *blobovniczaTreeMetrics) ObjectMoved(d time.Duration) {
+	m.m.BlobovniczaTreeObjectMoved(m.shardID, m.path, d)
+}
+
 func (m *blobovniczaTreeMetrics) Delete(d time.Duration, success, withStorageID bool) {
 	m.m.BlobobvnizcaTreeMethodDuration(m.shardID, m.path, "Delete", d, success, metrics_impl.NullBool{Valid: true, Bool: withStorageID})
 }

pkg/metrics/blobovnicza.go

@@ -23,16 +23,21 @@ type BlobobvnizcaMetrics interface {
 
 	IncOpenBlobovniczaCount(shardID, path string)
 	DecOpenBlobovniczaCount(shardID, path string)
+
+	BlobovniczaTreeRebuildStatus(shardID, path, status string)
+	BlobovniczaTreeObjectMoved(shardID, path string, d time.Duration)
 }
 
 type blobovnicza struct {
-	treeMode        *shardIDPathModeValue
+	treeMode               *shardIDPathModeValue
-	treeReqDuration *prometheus.HistogramVec
+	treeReqDuration        *prometheus.HistogramVec
-	treePut         *prometheus.CounterVec
+	treePut                *prometheus.CounterVec
-	treeGet         *prometheus.CounterVec
+	treeGet                *prometheus.CounterVec
-	treeOpenSize    *prometheus.GaugeVec
+	treeOpenSize           *prometheus.GaugeVec
-	treeOpenItems   *prometheus.GaugeVec
+	treeOpenItems          *prometheus.GaugeVec
-	treeOpenCounter *prometheus.GaugeVec
+	treeOpenCounter        *prometheus.GaugeVec
+	treeObjectMoveDuration *prometheus.HistogramVec
+	treeRebuildStatus      *shardIDPathModeValue
 }
 
 func newBlobovnicza() *blobovnicza {
@@ -75,6 +80,13 @@ func newBlobovnicza() *blobovnicza {
 			Name:      "open_blobovnicza_count",
 			Help:      "Count of opened blobovniczas of Blobovnicza tree",
 		}, []string{shardIDLabel, pathLabel}),
+		treeObjectMoveDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{
+			Namespace: namespace,
+			Subsystem: blobovniczaTreeSubSystem,
+			Name:      "object_move_duration_seconds",
+			Help:      "Accumulated Blobovnicza tree object move duration",
+		}, []string{shardIDLabel, pathLabel}),
+		treeRebuildStatus: newShardIDPathMode(blobovniczaTreeSubSystem, "rebuild_status", "Blobovnicza tree rebuild status"),
 	}
 }
 
@@ -96,6 +108,11 @@ func (b *blobovnicza) CloseBlobobvnizcaTree(shardID, path string) {
 		shardIDLabel: shardID,
 		pathLabel:    path,
 	})
+	b.treeObjectMoveDuration.DeletePartialMatch(prometheus.Labels{
+		shardIDLabel: shardID,
+		pathLabel:    path,
+	})
+	b.treeRebuildStatus.SetMode(shardID, path, undefinedStatus)
 }
 
 func (b *blobovnicza) BlobobvnizcaTreeMethodDuration(shardID, path string, method string, d time.Duration, success bool, withStorageID NullBool) {
@@ -163,3 +180,14 @@ func (b *blobovnicza) SubOpenBlobovniczaItems(shardID, path string, items uint64
 		pathLabel:    path,
 	}).Sub(float64(items))
 }
+
+func (b *blobovnicza) BlobovniczaTreeRebuildStatus(shardID, path, status string) {
+	b.treeRebuildStatus.SetMode(shardID, path, status)
+}
+
+func (b *blobovnicza) BlobovniczaTreeObjectMoved(shardID, path string, d time.Duration) {
+	b.treeObjectMoveDuration.With(prometheus.Labels{
+		shardIDLabel: shardID,
+		pathLabel:    path,
+	}).Observe(d.Seconds())
+}

pkg/metrics/consts.go

@@ -45,4 +45,5 @@ const (
 
 	failedToDeleteStatus = "failed_to_delete"
 	deletedStatus        = "deleted"
+	undefinedStatus      = "undefined"
 )