[#661] metrics: Add blobovniczatree rebuild metrics

Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
This commit is contained in:
Dmitrii Stepanov 2023-09-27 18:21:37 +03:00
parent b2769ca3de
commit d4d905ecc6
6 changed files with 74 additions and 7 deletions

View file

@ -19,6 +19,7 @@ var errFailedToChangeExtensionReadOnly = errors.New("failed to change blobovnicz
func (b *Blobovniczas) Open(readOnly bool) error { func (b *Blobovniczas) Open(readOnly bool) error {
b.readOnly = readOnly b.readOnly = readOnly
b.metrics.SetMode(readOnly) b.metrics.SetMode(readOnly)
b.metrics.SetRebuildStatus(rebuildStatusNotStarted)
b.openManagers() b.openManagers()
return nil return nil
} }

View file

@ -6,6 +6,13 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza"
) )
const (
rebuildStatusNotStarted = "not_started"
rebuildStatusRunning = "running"
rebuildStatusCompleted = "completed"
rebuildStatusFailed = "failed"
)
type Metrics interface { type Metrics interface {
Blobovnicza() blobovnicza.Metrics Blobovnicza() blobovnicza.Metrics
@ -14,6 +21,9 @@ type Metrics interface {
SetMode(readOnly bool) SetMode(readOnly bool)
Close() Close()
SetRebuildStatus(status string)
ObjectMoved(d time.Duration)
Delete(d time.Duration, success, withStorageID bool) Delete(d time.Duration, success, withStorageID bool)
Exists(d time.Duration, success, withStorageID bool) Exists(d time.Duration, success, withStorageID bool)
GetRange(d time.Duration, size int, success, withStorageID bool) GetRange(d time.Duration, size int, success, withStorageID bool)
@ -27,6 +37,8 @@ type noopMetrics struct{}
func (m *noopMetrics) SetParentID(string) {} func (m *noopMetrics) SetParentID(string) {}
func (m *noopMetrics) SetMode(bool) {} func (m *noopMetrics) SetMode(bool) {}
func (m *noopMetrics) Close() {} func (m *noopMetrics) Close() {}
func (m *noopMetrics) SetRebuildStatus(string) {}
func (m *noopMetrics) ObjectMoved(time.Duration) {}
func (m *noopMetrics) Delete(time.Duration, bool, bool) {} func (m *noopMetrics) Delete(time.Duration, bool, bool) {}
func (m *noopMetrics) Exists(time.Duration, bool, bool) {} func (m *noopMetrics) Exists(time.Duration, bool, bool) {}
func (m *noopMetrics) GetRange(time.Duration, int, bool, bool) {} func (m *noopMetrics) GetRange(time.Duration, int, bool, bool) {}

View file

@ -23,6 +23,16 @@ func (b *Blobovniczas) Rebuild(ctx context.Context, prm common.RebuildPrm) (comm
return common.RebuildRes{}, common.ErrReadOnly return common.RebuildRes{}, common.ErrReadOnly
} }
b.metrics.SetRebuildStatus(rebuildStatusRunning)
success := true
defer func() {
if success {
b.metrics.SetRebuildStatus(rebuildStatusCompleted)
} else {
b.metrics.SetRebuildStatus(rebuildStatusFailed)
}
}()
b.rebuildGuard.Lock() b.rebuildGuard.Lock()
defer b.rebuildGuard.Unlock() defer b.rebuildGuard.Unlock()
@ -33,6 +43,7 @@ func (b *Blobovniczas) Rebuild(ctx context.Context, prm common.RebuildPrm) (comm
res.ObjectsMoved += completedPreviosMoves res.ObjectsMoved += completedPreviosMoves
if err != nil { if err != nil {
b.log.Warn(logs.BlobovniczaTreeCompletedPreviousRebuildFailed, zap.Error(err)) b.log.Warn(logs.BlobovniczaTreeCompletedPreviousRebuildFailed, zap.Error(err))
success = false
return res, err return res, err
} }
b.log.Debug(logs.BlobovniczaTreeCompletedPreviousRebuildSuccess) b.log.Debug(logs.BlobovniczaTreeCompletedPreviousRebuildSuccess)
@ -41,6 +52,7 @@ func (b *Blobovniczas) Rebuild(ctx context.Context, prm common.RebuildPrm) (comm
dbsToMigrate, err := b.getDBsToRebuild(ctx) dbsToMigrate, err := b.getDBsToRebuild(ctx)
if err != nil { if err != nil {
b.log.Warn(logs.BlobovniczaTreeCollectingDBToRebuildFailed, zap.Error(err)) b.log.Warn(logs.BlobovniczaTreeCollectingDBToRebuildFailed, zap.Error(err))
success = false
return res, err return res, err
} }
b.log.Info(logs.BlobovniczaTreeCollectingDBToRebuildSuccess, zap.Int("blobovniczas_to_rebuild", len(dbsToMigrate))) b.log.Info(logs.BlobovniczaTreeCollectingDBToRebuildSuccess, zap.Int("blobovniczas_to_rebuild", len(dbsToMigrate)))
@ -50,6 +62,7 @@ func (b *Blobovniczas) Rebuild(ctx context.Context, prm common.RebuildPrm) (comm
res.ObjectsMoved += movedObjects res.ObjectsMoved += movedObjects
if err != nil { if err != nil {
b.log.Warn(logs.BlobovniczaTreeRebuildingBlobovniczaFailed, zap.String("path", db), zap.Uint64("moved_objects_count", movedObjects), zap.Error(err)) b.log.Warn(logs.BlobovniczaTreeRebuildingBlobovniczaFailed, zap.String("path", db), zap.Uint64("moved_objects_count", movedObjects), zap.Error(err))
success = false
return res, err return res, err
} }
b.log.Debug(logs.BlobovniczaTreeRebuildingBlobovniczaSuccess, zap.String("path", db), zap.Uint64("moved_objects_count", movedObjects)) b.log.Debug(logs.BlobovniczaTreeRebuildingBlobovniczaSuccess, zap.String("path", db), zap.Uint64("moved_objects_count", movedObjects))
@ -121,6 +134,10 @@ func (b *Blobovniczas) moveObjects(ctx context.Context, blz *blobovnicza.Blobovn
func (b *Blobovniczas) moveObject(ctx context.Context, source *blobovnicza.Blobovnicza, sourcePath string, func (b *Blobovniczas) moveObject(ctx context.Context, source *blobovnicza.Blobovnicza, sourcePath string,
addr oid.Address, data []byte, metaStore common.MetaStorage) error { addr oid.Address, data []byte, metaStore common.MetaStorage) error {
startedAt := time.Now()
defer func() {
b.metrics.ObjectMoved(time.Since(startedAt))
}()
it := &moveIterator{ it := &moveIterator{
B: b, B: b,
ID: nil, ID: nil,

View file

@ -42,6 +42,14 @@ func (m *blobovniczaTreeMetrics) Close() {
m.m.CloseBlobobvnizcaTree(m.shardID, m.path) m.m.CloseBlobobvnizcaTree(m.shardID, m.path)
} }
func (m *blobovniczaTreeMetrics) SetRebuildStatus(status string) {
m.m.BlobovniczaTreeRebuildStatus(m.shardID, m.path, status)
}
func (m *blobovniczaTreeMetrics) ObjectMoved(d time.Duration) {
m.m.BlobovniczaTreeObjectMoved(m.shardID, m.path, d)
}
func (m *blobovniczaTreeMetrics) Delete(d time.Duration, success, withStorageID bool) { func (m *blobovniczaTreeMetrics) Delete(d time.Duration, success, withStorageID bool) {
m.m.BlobobvnizcaTreeMethodDuration(m.shardID, m.path, "Delete", d, success, metrics_impl.NullBool{Valid: true, Bool: withStorageID}) m.m.BlobobvnizcaTreeMethodDuration(m.shardID, m.path, "Delete", d, success, metrics_impl.NullBool{Valid: true, Bool: withStorageID})
} }

View file

@ -23,16 +23,21 @@ type BlobobvnizcaMetrics interface {
IncOpenBlobovniczaCount(shardID, path string) IncOpenBlobovniczaCount(shardID, path string)
DecOpenBlobovniczaCount(shardID, path string) DecOpenBlobovniczaCount(shardID, path string)
BlobovniczaTreeRebuildStatus(shardID, path, status string)
BlobovniczaTreeObjectMoved(shardID, path string, d time.Duration)
} }
type blobovnicza struct { type blobovnicza struct {
treeMode *shardIDPathModeValue treeMode *shardIDPathModeValue
treeReqDuration *prometheus.HistogramVec treeReqDuration *prometheus.HistogramVec
treePut *prometheus.CounterVec treePut *prometheus.CounterVec
treeGet *prometheus.CounterVec treeGet *prometheus.CounterVec
treeOpenSize *prometheus.GaugeVec treeOpenSize *prometheus.GaugeVec
treeOpenItems *prometheus.GaugeVec treeOpenItems *prometheus.GaugeVec
treeOpenCounter *prometheus.GaugeVec treeOpenCounter *prometheus.GaugeVec
treeObjectMoveDuration *prometheus.HistogramVec
treeRebuildStatus *shardIDPathModeValue
} }
func newBlobovnicza() *blobovnicza { func newBlobovnicza() *blobovnicza {
@ -75,6 +80,13 @@ func newBlobovnicza() *blobovnicza {
Name: "open_blobovnicza_count", Name: "open_blobovnicza_count",
Help: "Count of opened blobovniczas of Blobovnicza tree", Help: "Count of opened blobovniczas of Blobovnicza tree",
}, []string{shardIDLabel, pathLabel}), }, []string{shardIDLabel, pathLabel}),
treeObjectMoveDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{
Namespace: namespace,
Subsystem: blobovniczaTreeSubSystem,
Name: "object_move_duration_seconds",
Help: "Accumulated Blobovnicza tree object move duration",
}, []string{shardIDLabel, pathLabel}),
treeRebuildStatus: newShardIDPathMode(blobovniczaTreeSubSystem, "rebuild_status", "Blobovnicza tree rebuild status"),
} }
} }
@ -96,6 +108,11 @@ func (b *blobovnicza) CloseBlobobvnizcaTree(shardID, path string) {
shardIDLabel: shardID, shardIDLabel: shardID,
pathLabel: path, pathLabel: path,
}) })
b.treeObjectMoveDuration.DeletePartialMatch(prometheus.Labels{
shardIDLabel: shardID,
pathLabel: path,
})
b.treeRebuildStatus.SetMode(shardID, path, undefinedStatus)
} }
func (b *blobovnicza) BlobobvnizcaTreeMethodDuration(shardID, path string, method string, d time.Duration, success bool, withStorageID NullBool) { func (b *blobovnicza) BlobobvnizcaTreeMethodDuration(shardID, path string, method string, d time.Duration, success bool, withStorageID NullBool) {
@ -163,3 +180,14 @@ func (b *blobovnicza) SubOpenBlobovniczaItems(shardID, path string, items uint64
pathLabel: path, pathLabel: path,
}).Sub(float64(items)) }).Sub(float64(items))
} }
func (b *blobovnicza) BlobovniczaTreeRebuildStatus(shardID, path, status string) {
b.treeRebuildStatus.SetMode(shardID, path, status)
}
func (b *blobovnicza) BlobovniczaTreeObjectMoved(shardID, path string, d time.Duration) {
b.treeObjectMoveDuration.With(prometheus.Labels{
shardIDLabel: shardID,
pathLabel: path,
}).Observe(d.Seconds())
}

View file

@ -45,4 +45,5 @@ const (
failedToDeleteStatus = "failed_to_delete" failedToDeleteStatus = "failed_to_delete"
deletedStatus = "deleted" deletedStatus = "deleted"
undefinedStatus = "undefined"
) )