[#661] metrcis: Add rebuild percent metric

Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
This commit is contained in:
Dmitrii Stepanov 2023-10-11 09:23:55 +03:00
parent 04b530197e
commit 4b20f846af
4 changed files with 30 additions and 0 deletions

View file

@ -23,6 +23,7 @@ type Metrics interface {
SetRebuildStatus(status string) SetRebuildStatus(status string)
ObjectMoved(d time.Duration) ObjectMoved(d time.Duration)
SetRebuildPercent(value uint32)
Delete(d time.Duration, success, withStorageID bool) Delete(d time.Duration, success, withStorageID bool)
Exists(d time.Duration, success, withStorageID bool) Exists(d time.Duration, success, withStorageID bool)
@ -38,6 +39,7 @@ func (m *noopMetrics) SetParentID(string) {}
func (m *noopMetrics) SetMode(bool) {} func (m *noopMetrics) SetMode(bool) {}
func (m *noopMetrics) Close() {} func (m *noopMetrics) Close() {}
func (m *noopMetrics) SetRebuildStatus(string) {} func (m *noopMetrics) SetRebuildStatus(string) {}
func (m *noopMetrics) SetRebuildPercent(uint32) {}
func (m *noopMetrics) ObjectMoved(time.Duration) {} func (m *noopMetrics) ObjectMoved(time.Duration) {}
func (m *noopMetrics) Delete(time.Duration, bool, bool) {} func (m *noopMetrics) Delete(time.Duration, bool, bool) {}
func (m *noopMetrics) Exists(time.Duration, bool, bool) {} func (m *noopMetrics) Exists(time.Duration, bool, bool) {}

View file

@ -30,6 +30,7 @@ func (b *Blobovniczas) Rebuild(ctx context.Context, prm common.RebuildPrm) (comm
} }
b.metrics.SetRebuildStatus(rebuildStatusRunning) b.metrics.SetRebuildStatus(rebuildStatusRunning)
b.metrics.SetRebuildPercent(0)
success := true success := true
defer func() { defer func() {
if success { if success {
@ -71,6 +72,7 @@ func (b *Blobovniczas) Rebuild(ctx context.Context, prm common.RebuildPrm) (comm
} }
func (b *Blobovniczas) migrateDBs(ctx context.Context, dbs []string, prm common.RebuildPrm, res common.RebuildRes) (common.RebuildRes, error) { func (b *Blobovniczas) migrateDBs(ctx context.Context, dbs []string, prm common.RebuildPrm, res common.RebuildRes) (common.RebuildRes, error) {
var completedDBCount uint32
for _, db := range dbs { for _, db := range dbs {
b.log.Debug(logs.BlobovniczaTreeRebuildingBlobovnicza, zap.String("path", db)) b.log.Debug(logs.BlobovniczaTreeRebuildingBlobovnicza, zap.String("path", db))
movedObjects, err := b.rebuildDB(ctx, db, prm.MetaStorage, prm.WorkerLimiter) movedObjects, err := b.rebuildDB(ctx, db, prm.MetaStorage, prm.WorkerLimiter)
@ -81,7 +83,10 @@ func (b *Blobovniczas) migrateDBs(ctx context.Context, dbs []string, prm common.
} }
b.log.Debug(logs.BlobovniczaTreeRebuildingBlobovniczaSuccess, zap.String("path", db), zap.Uint64("moved_objects_count", movedObjects)) b.log.Debug(logs.BlobovniczaTreeRebuildingBlobovniczaSuccess, zap.String("path", db), zap.Uint64("moved_objects_count", movedObjects))
res.FilesRemoved++ res.FilesRemoved++
completedDBCount++
b.metrics.SetRebuildPercent((100 * completedDBCount) / uint32(len(dbs)))
} }
b.metrics.SetRebuildPercent(100)
return res, nil return res, nil
} }

View file

@ -46,6 +46,10 @@ func (m *blobovniczaTreeMetrics) SetRebuildStatus(status string) {
m.m.BlobovniczaTreeRebuildStatus(m.shardID, m.path, status) m.m.BlobovniczaTreeRebuildStatus(m.shardID, m.path, status)
} }
func (m *blobovniczaTreeMetrics) SetRebuildPercent(value uint32) {
m.m.BlobovniczaTreeRebuildPercent(m.shardID, m.path, value)
}
func (m *blobovniczaTreeMetrics) ObjectMoved(d time.Duration) { func (m *blobovniczaTreeMetrics) ObjectMoved(d time.Duration) {
m.m.BlobovniczaTreeObjectMoved(m.shardID, m.path, d) m.m.BlobovniczaTreeObjectMoved(m.shardID, m.path, d)
} }

View file

@ -25,6 +25,7 @@ type BlobobvnizcaMetrics interface {
DecOpenBlobovniczaCount(shardID, path string) DecOpenBlobovniczaCount(shardID, path string)
BlobovniczaTreeRebuildStatus(shardID, path, status string) BlobovniczaTreeRebuildStatus(shardID, path, status string)
BlobovniczaTreeRebuildPercent(shardID, path string, value uint32)
BlobovniczaTreeObjectMoved(shardID, path string, d time.Duration) BlobovniczaTreeObjectMoved(shardID, path string, d time.Duration)
} }
@ -38,6 +39,7 @@ type blobovnicza struct {
treeOpenCounter *prometheus.GaugeVec treeOpenCounter *prometheus.GaugeVec
treeObjectMoveDuration *prometheus.HistogramVec treeObjectMoveDuration *prometheus.HistogramVec
treeRebuildStatus *shardIDPathModeValue treeRebuildStatus *shardIDPathModeValue
treeRebuildPercent *prometheus.GaugeVec
} }
func newBlobovnicza() *blobovnicza { func newBlobovnicza() *blobovnicza {
@ -87,6 +89,12 @@ func newBlobovnicza() *blobovnicza {
Help: "Accumulated Blobovnicza tree object move duration", Help: "Accumulated Blobovnicza tree object move duration",
}, []string{shardIDLabel, pathLabel}), }, []string{shardIDLabel, pathLabel}),
treeRebuildStatus: newShardIDPathMode(blobovniczaTreeSubSystem, "rebuild_status", "Blobovnicza tree rebuild status"), treeRebuildStatus: newShardIDPathMode(blobovniczaTreeSubSystem, "rebuild_status", "Blobovnicza tree rebuild status"),
treeRebuildPercent: metrics.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Subsystem: blobovniczaTreeSubSystem,
Name: "rebuild_complete_percent",
Help: "Percent of rebuild completeness",
}, []string{shardIDLabel, pathLabel}),
} }
} }
@ -112,6 +120,10 @@ func (b *blobovnicza) CloseBlobobvnizcaTree(shardID, path string) {
shardIDLabel: shardID, shardIDLabel: shardID,
pathLabel: path, pathLabel: path,
}) })
b.treeRebuildPercent.DeletePartialMatch(prometheus.Labels{
shardIDLabel: shardID,
pathLabel: path,
})
b.treeRebuildStatus.SetMode(shardID, path, undefinedStatus) b.treeRebuildStatus.SetMode(shardID, path, undefinedStatus)
} }
@ -191,3 +203,10 @@ func (b *blobovnicza) BlobovniczaTreeObjectMoved(shardID, path string, d time.Du
pathLabel: path, pathLabel: path,
}).Observe(d.Seconds()) }).Observe(d.Seconds())
} }
func (b *blobovnicza) BlobovniczaTreeRebuildPercent(shardID, path string, value uint32) {
b.treeRebuildPercent.With(prometheus.Labels{
shardIDLabel: shardID,
pathLabel: path,
}).Set(float64(value))
}