forked from TrueCloudLab/frostfs-node
[#661] metrics: Add blobovniczatree rebuild metrics
Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>feature/804_override_storage
parent
b2769ca3de
commit
d4d905ecc6
|
@ -19,6 +19,7 @@ var errFailedToChangeExtensionReadOnly = errors.New("failed to change blobovnicz
|
||||||
func (b *Blobovniczas) Open(readOnly bool) error {
|
func (b *Blobovniczas) Open(readOnly bool) error {
|
||||||
b.readOnly = readOnly
|
b.readOnly = readOnly
|
||||||
b.metrics.SetMode(readOnly)
|
b.metrics.SetMode(readOnly)
|
||||||
|
b.metrics.SetRebuildStatus(rebuildStatusNotStarted)
|
||||||
b.openManagers()
|
b.openManagers()
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -6,6 +6,13 @@ import (
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza"
|
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
rebuildStatusNotStarted = "not_started"
|
||||||
|
rebuildStatusRunning = "running"
|
||||||
|
rebuildStatusCompleted = "completed"
|
||||||
|
rebuildStatusFailed = "failed"
|
||||||
|
)
|
||||||
|
|
||||||
type Metrics interface {
|
type Metrics interface {
|
||||||
Blobovnicza() blobovnicza.Metrics
|
Blobovnicza() blobovnicza.Metrics
|
||||||
|
|
||||||
|
@ -14,6 +21,9 @@ type Metrics interface {
|
||||||
SetMode(readOnly bool)
|
SetMode(readOnly bool)
|
||||||
Close()
|
Close()
|
||||||
|
|
||||||
|
SetRebuildStatus(status string)
|
||||||
|
ObjectMoved(d time.Duration)
|
||||||
|
|
||||||
Delete(d time.Duration, success, withStorageID bool)
|
Delete(d time.Duration, success, withStorageID bool)
|
||||||
Exists(d time.Duration, success, withStorageID bool)
|
Exists(d time.Duration, success, withStorageID bool)
|
||||||
GetRange(d time.Duration, size int, success, withStorageID bool)
|
GetRange(d time.Duration, size int, success, withStorageID bool)
|
||||||
|
@ -27,6 +37,8 @@ type noopMetrics struct{}
|
||||||
func (m *noopMetrics) SetParentID(string) {}
|
func (m *noopMetrics) SetParentID(string) {}
|
||||||
func (m *noopMetrics) SetMode(bool) {}
|
func (m *noopMetrics) SetMode(bool) {}
|
||||||
func (m *noopMetrics) Close() {}
|
func (m *noopMetrics) Close() {}
|
||||||
|
func (m *noopMetrics) SetRebuildStatus(string) {}
|
||||||
|
func (m *noopMetrics) ObjectMoved(time.Duration) {}
|
||||||
func (m *noopMetrics) Delete(time.Duration, bool, bool) {}
|
func (m *noopMetrics) Delete(time.Duration, bool, bool) {}
|
||||||
func (m *noopMetrics) Exists(time.Duration, bool, bool) {}
|
func (m *noopMetrics) Exists(time.Duration, bool, bool) {}
|
||||||
func (m *noopMetrics) GetRange(time.Duration, int, bool, bool) {}
|
func (m *noopMetrics) GetRange(time.Duration, int, bool, bool) {}
|
||||||
|
|
|
@ -23,6 +23,16 @@ func (b *Blobovniczas) Rebuild(ctx context.Context, prm common.RebuildPrm) (comm
|
||||||
return common.RebuildRes{}, common.ErrReadOnly
|
return common.RebuildRes{}, common.ErrReadOnly
|
||||||
}
|
}
|
||||||
|
|
||||||
|
b.metrics.SetRebuildStatus(rebuildStatusRunning)
|
||||||
|
success := true
|
||||||
|
defer func() {
|
||||||
|
if success {
|
||||||
|
b.metrics.SetRebuildStatus(rebuildStatusCompleted)
|
||||||
|
} else {
|
||||||
|
b.metrics.SetRebuildStatus(rebuildStatusFailed)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
b.rebuildGuard.Lock()
|
b.rebuildGuard.Lock()
|
||||||
defer b.rebuildGuard.Unlock()
|
defer b.rebuildGuard.Unlock()
|
||||||
|
|
||||||
|
@ -33,6 +43,7 @@ func (b *Blobovniczas) Rebuild(ctx context.Context, prm common.RebuildPrm) (comm
|
||||||
res.ObjectsMoved += completedPreviosMoves
|
res.ObjectsMoved += completedPreviosMoves
|
||||||
if err != nil {
|
if err != nil {
|
||||||
b.log.Warn(logs.BlobovniczaTreeCompletedPreviousRebuildFailed, zap.Error(err))
|
b.log.Warn(logs.BlobovniczaTreeCompletedPreviousRebuildFailed, zap.Error(err))
|
||||||
|
success = false
|
||||||
return res, err
|
return res, err
|
||||||
}
|
}
|
||||||
b.log.Debug(logs.BlobovniczaTreeCompletedPreviousRebuildSuccess)
|
b.log.Debug(logs.BlobovniczaTreeCompletedPreviousRebuildSuccess)
|
||||||
|
@ -41,6 +52,7 @@ func (b *Blobovniczas) Rebuild(ctx context.Context, prm common.RebuildPrm) (comm
|
||||||
dbsToMigrate, err := b.getDBsToRebuild(ctx)
|
dbsToMigrate, err := b.getDBsToRebuild(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
b.log.Warn(logs.BlobovniczaTreeCollectingDBToRebuildFailed, zap.Error(err))
|
b.log.Warn(logs.BlobovniczaTreeCollectingDBToRebuildFailed, zap.Error(err))
|
||||||
|
success = false
|
||||||
return res, err
|
return res, err
|
||||||
}
|
}
|
||||||
b.log.Info(logs.BlobovniczaTreeCollectingDBToRebuildSuccess, zap.Int("blobovniczas_to_rebuild", len(dbsToMigrate)))
|
b.log.Info(logs.BlobovniczaTreeCollectingDBToRebuildSuccess, zap.Int("blobovniczas_to_rebuild", len(dbsToMigrate)))
|
||||||
|
@ -50,6 +62,7 @@ func (b *Blobovniczas) Rebuild(ctx context.Context, prm common.RebuildPrm) (comm
|
||||||
res.ObjectsMoved += movedObjects
|
res.ObjectsMoved += movedObjects
|
||||||
if err != nil {
|
if err != nil {
|
||||||
b.log.Warn(logs.BlobovniczaTreeRebuildingBlobovniczaFailed, zap.String("path", db), zap.Uint64("moved_objects_count", movedObjects), zap.Error(err))
|
b.log.Warn(logs.BlobovniczaTreeRebuildingBlobovniczaFailed, zap.String("path", db), zap.Uint64("moved_objects_count", movedObjects), zap.Error(err))
|
||||||
|
success = false
|
||||||
return res, err
|
return res, err
|
||||||
}
|
}
|
||||||
b.log.Debug(logs.BlobovniczaTreeRebuildingBlobovniczaSuccess, zap.String("path", db), zap.Uint64("moved_objects_count", movedObjects))
|
b.log.Debug(logs.BlobovniczaTreeRebuildingBlobovniczaSuccess, zap.String("path", db), zap.Uint64("moved_objects_count", movedObjects))
|
||||||
|
@ -121,6 +134,10 @@ func (b *Blobovniczas) moveObjects(ctx context.Context, blz *blobovnicza.Blobovn
|
||||||
|
|
||||||
func (b *Blobovniczas) moveObject(ctx context.Context, source *blobovnicza.Blobovnicza, sourcePath string,
|
func (b *Blobovniczas) moveObject(ctx context.Context, source *blobovnicza.Blobovnicza, sourcePath string,
|
||||||
addr oid.Address, data []byte, metaStore common.MetaStorage) error {
|
addr oid.Address, data []byte, metaStore common.MetaStorage) error {
|
||||||
|
startedAt := time.Now()
|
||||||
|
defer func() {
|
||||||
|
b.metrics.ObjectMoved(time.Since(startedAt))
|
||||||
|
}()
|
||||||
it := &moveIterator{
|
it := &moveIterator{
|
||||||
B: b,
|
B: b,
|
||||||
ID: nil,
|
ID: nil,
|
||||||
|
|
|
@ -42,6 +42,14 @@ func (m *blobovniczaTreeMetrics) Close() {
|
||||||
m.m.CloseBlobobvnizcaTree(m.shardID, m.path)
|
m.m.CloseBlobobvnizcaTree(m.shardID, m.path)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (m *blobovniczaTreeMetrics) SetRebuildStatus(status string) {
|
||||||
|
m.m.BlobovniczaTreeRebuildStatus(m.shardID, m.path, status)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *blobovniczaTreeMetrics) ObjectMoved(d time.Duration) {
|
||||||
|
m.m.BlobovniczaTreeObjectMoved(m.shardID, m.path, d)
|
||||||
|
}
|
||||||
|
|
||||||
func (m *blobovniczaTreeMetrics) Delete(d time.Duration, success, withStorageID bool) {
|
func (m *blobovniczaTreeMetrics) Delete(d time.Duration, success, withStorageID bool) {
|
||||||
m.m.BlobobvnizcaTreeMethodDuration(m.shardID, m.path, "Delete", d, success, metrics_impl.NullBool{Valid: true, Bool: withStorageID})
|
m.m.BlobobvnizcaTreeMethodDuration(m.shardID, m.path, "Delete", d, success, metrics_impl.NullBool{Valid: true, Bool: withStorageID})
|
||||||
}
|
}
|
||||||
|
|
|
@ -23,16 +23,21 @@ type BlobobvnizcaMetrics interface {
|
||||||
|
|
||||||
IncOpenBlobovniczaCount(shardID, path string)
|
IncOpenBlobovniczaCount(shardID, path string)
|
||||||
DecOpenBlobovniczaCount(shardID, path string)
|
DecOpenBlobovniczaCount(shardID, path string)
|
||||||
|
|
||||||
|
BlobovniczaTreeRebuildStatus(shardID, path, status string)
|
||||||
|
BlobovniczaTreeObjectMoved(shardID, path string, d time.Duration)
|
||||||
}
|
}
|
||||||
|
|
||||||
type blobovnicza struct {
|
type blobovnicza struct {
|
||||||
treeMode *shardIDPathModeValue
|
treeMode *shardIDPathModeValue
|
||||||
treeReqDuration *prometheus.HistogramVec
|
treeReqDuration *prometheus.HistogramVec
|
||||||
treePut *prometheus.CounterVec
|
treePut *prometheus.CounterVec
|
||||||
treeGet *prometheus.CounterVec
|
treeGet *prometheus.CounterVec
|
||||||
treeOpenSize *prometheus.GaugeVec
|
treeOpenSize *prometheus.GaugeVec
|
||||||
treeOpenItems *prometheus.GaugeVec
|
treeOpenItems *prometheus.GaugeVec
|
||||||
treeOpenCounter *prometheus.GaugeVec
|
treeOpenCounter *prometheus.GaugeVec
|
||||||
|
treeObjectMoveDuration *prometheus.HistogramVec
|
||||||
|
treeRebuildStatus *shardIDPathModeValue
|
||||||
}
|
}
|
||||||
|
|
||||||
func newBlobovnicza() *blobovnicza {
|
func newBlobovnicza() *blobovnicza {
|
||||||
|
@ -75,6 +80,13 @@ func newBlobovnicza() *blobovnicza {
|
||||||
Name: "open_blobovnicza_count",
|
Name: "open_blobovnicza_count",
|
||||||
Help: "Count of opened blobovniczas of Blobovnicza tree",
|
Help: "Count of opened blobovniczas of Blobovnicza tree",
|
||||||
}, []string{shardIDLabel, pathLabel}),
|
}, []string{shardIDLabel, pathLabel}),
|
||||||
|
treeObjectMoveDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{
|
||||||
|
Namespace: namespace,
|
||||||
|
Subsystem: blobovniczaTreeSubSystem,
|
||||||
|
Name: "object_move_duration_seconds",
|
||||||
|
Help: "Accumulated Blobovnicza tree object move duration",
|
||||||
|
}, []string{shardIDLabel, pathLabel}),
|
||||||
|
treeRebuildStatus: newShardIDPathMode(blobovniczaTreeSubSystem, "rebuild_status", "Blobovnicza tree rebuild status"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -96,6 +108,11 @@ func (b *blobovnicza) CloseBlobobvnizcaTree(shardID, path string) {
|
||||||
shardIDLabel: shardID,
|
shardIDLabel: shardID,
|
||||||
pathLabel: path,
|
pathLabel: path,
|
||||||
})
|
})
|
||||||
|
b.treeObjectMoveDuration.DeletePartialMatch(prometheus.Labels{
|
||||||
|
shardIDLabel: shardID,
|
||||||
|
pathLabel: path,
|
||||||
|
})
|
||||||
|
b.treeRebuildStatus.SetMode(shardID, path, undefinedStatus)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *blobovnicza) BlobobvnizcaTreeMethodDuration(shardID, path string, method string, d time.Duration, success bool, withStorageID NullBool) {
|
func (b *blobovnicza) BlobobvnizcaTreeMethodDuration(shardID, path string, method string, d time.Duration, success bool, withStorageID NullBool) {
|
||||||
|
@ -163,3 +180,14 @@ func (b *blobovnicza) SubOpenBlobovniczaItems(shardID, path string, items uint64
|
||||||
pathLabel: path,
|
pathLabel: path,
|
||||||
}).Sub(float64(items))
|
}).Sub(float64(items))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (b *blobovnicza) BlobovniczaTreeRebuildStatus(shardID, path, status string) {
|
||||||
|
b.treeRebuildStatus.SetMode(shardID, path, status)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *blobovnicza) BlobovniczaTreeObjectMoved(shardID, path string, d time.Duration) {
|
||||||
|
b.treeObjectMoveDuration.With(prometheus.Labels{
|
||||||
|
shardIDLabel: shardID,
|
||||||
|
pathLabel: path,
|
||||||
|
}).Observe(d.Seconds())
|
||||||
|
}
|
||||||
|
|
|
@ -45,4 +45,5 @@ const (
|
||||||
|
|
||||||
failedToDeleteStatus = "failed_to_delete"
|
failedToDeleteStatus = "failed_to_delete"
|
||||||
deletedStatus = "deleted"
|
deletedStatus = "deleted"
|
||||||
|
undefinedStatus = "undefined"
|
||||||
)
|
)
|
||||||
|
|
Loading…
Reference in New Issue