diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go b/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go index ee0326bc..ed05f877 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go @@ -23,6 +23,7 @@ type Metrics interface { SetRebuildStatus(status string) ObjectMoved(d time.Duration) + SetRebuildPercent(value uint32) Delete(d time.Duration, success, withStorageID bool) Exists(d time.Duration, success, withStorageID bool) @@ -38,6 +39,7 @@ func (m *noopMetrics) SetParentID(string) {} func (m *noopMetrics) SetMode(bool) {} func (m *noopMetrics) Close() {} func (m *noopMetrics) SetRebuildStatus(string) {} +func (m *noopMetrics) SetRebuildPercent(uint32) {} func (m *noopMetrics) ObjectMoved(time.Duration) {} func (m *noopMetrics) Delete(time.Duration, bool, bool) {} func (m *noopMetrics) Exists(time.Duration, bool, bool) {} diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go index c7e44fdb..d65ca43f 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go @@ -30,6 +30,7 @@ func (b *Blobovniczas) Rebuild(ctx context.Context, prm common.RebuildPrm) (comm } b.metrics.SetRebuildStatus(rebuildStatusRunning) + b.metrics.SetRebuildPercent(0) success := true defer func() { if success { @@ -71,6 +72,7 @@ func (b *Blobovniczas) Rebuild(ctx context.Context, prm common.RebuildPrm) (comm } func (b *Blobovniczas) migrateDBs(ctx context.Context, dbs []string, prm common.RebuildPrm, res common.RebuildRes) (common.RebuildRes, error) { + var completedDBCount uint32 for _, db := range dbs { b.log.Debug(logs.BlobovniczaTreeRebuildingBlobovnicza, zap.String("path", db)) movedObjects, err := b.rebuildDB(ctx, db, prm.MetaStorage, prm.WorkerLimiter) @@ -81,7 +83,10 @@ func (b *Blobovniczas) migrateDBs(ctx context.Context, dbs []string, prm common. } b.log.Debug(logs.BlobovniczaTreeRebuildingBlobovniczaSuccess, zap.String("path", db), zap.Uint64("moved_objects_count", movedObjects)) res.FilesRemoved++ + completedDBCount++ + b.metrics.SetRebuildPercent((100 * completedDBCount) / uint32(len(dbs))) } + b.metrics.SetRebuildPercent(100) return res, nil } diff --git a/pkg/local_object_storage/metrics/blobovnicza.go b/pkg/local_object_storage/metrics/blobovnicza.go index 3d751ba3..3a92a7bc 100644 --- a/pkg/local_object_storage/metrics/blobovnicza.go +++ b/pkg/local_object_storage/metrics/blobovnicza.go @@ -46,6 +46,10 @@ func (m *blobovniczaTreeMetrics) SetRebuildStatus(status string) { m.m.BlobovniczaTreeRebuildStatus(m.shardID, m.path, status) } +func (m *blobovniczaTreeMetrics) SetRebuildPercent(value uint32) { + m.m.BlobovniczaTreeRebuildPercent(m.shardID, m.path, value) +} + func (m *blobovniczaTreeMetrics) ObjectMoved(d time.Duration) { m.m.BlobovniczaTreeObjectMoved(m.shardID, m.path, d) } diff --git a/pkg/metrics/blobovnicza.go b/pkg/metrics/blobovnicza.go index ddeeb406..f3a14a96 100644 --- a/pkg/metrics/blobovnicza.go +++ b/pkg/metrics/blobovnicza.go @@ -25,6 +25,7 @@ type BlobobvnizcaMetrics interface { DecOpenBlobovniczaCount(shardID, path string) BlobovniczaTreeRebuildStatus(shardID, path, status string) + BlobovniczaTreeRebuildPercent(shardID, path string, value uint32) BlobovniczaTreeObjectMoved(shardID, path string, d time.Duration) } @@ -38,6 +39,7 @@ type blobovnicza struct { treeOpenCounter *prometheus.GaugeVec treeObjectMoveDuration *prometheus.HistogramVec treeRebuildStatus *shardIDPathModeValue + treeRebuildPercent *prometheus.GaugeVec } func newBlobovnicza() *blobovnicza { @@ -87,6 +89,12 @@ func newBlobovnicza() *blobovnicza { Help: "Accumulated Blobovnicza tree object move duration", }, []string{shardIDLabel, pathLabel}), treeRebuildStatus: newShardIDPathMode(blobovniczaTreeSubSystem, "rebuild_status", "Blobovnicza tree rebuild status"), + treeRebuildPercent: metrics.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: blobovniczaTreeSubSystem, + Name: "rebuild_complete_percent", + Help: "Percent of rebuild completeness", + }, []string{shardIDLabel, pathLabel}), } } @@ -112,6 +120,10 @@ func (b *blobovnicza) CloseBlobobvnizcaTree(shardID, path string) { shardIDLabel: shardID, pathLabel: path, }) + b.treeRebuildPercent.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }) b.treeRebuildStatus.SetMode(shardID, path, undefinedStatus) } @@ -191,3 +203,10 @@ func (b *blobovnicza) BlobovniczaTreeObjectMoved(shardID, path string, d time.Du pathLabel: path, }).Observe(d.Seconds()) } + +func (b *blobovnicza) BlobovniczaTreeRebuildPercent(shardID, path string, value uint32) { + b.treeRebuildPercent.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }).Set(float64(value)) +}