forked from TrueCloudLab/frostfs-node
[#424] metrics: Refactor engine metrics
Use histogram vector to measure request duration. Fix naming like in Prometheus best practice. Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
This commit is contained in:
parent
c8023a9c8d
commit
1b364d8cf4
13 changed files with 46 additions and 122 deletions
|
@ -67,7 +67,7 @@ func ContainerSize(e *StorageEngine, id cid.ID) (uint64, error) {
|
||||||
|
|
||||||
func (e *StorageEngine) containerSize(prm ContainerSizePrm) (res ContainerSizeRes, err error) {
|
func (e *StorageEngine) containerSize(prm ContainerSizePrm) (res ContainerSizeRes, err error) {
|
||||||
if e.metrics != nil {
|
if e.metrics != nil {
|
||||||
defer elapsed(e.metrics.AddEstimateContainerSizeDuration)()
|
defer elapsed("EstimateContainerSize", e.metrics.AddMethodDuration)()
|
||||||
}
|
}
|
||||||
|
|
||||||
e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) {
|
e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) {
|
||||||
|
@ -115,7 +115,7 @@ func ListContainers(e *StorageEngine) ([]cid.ID, error) {
|
||||||
|
|
||||||
func (e *StorageEngine) listContainers() (ListContainersRes, error) {
|
func (e *StorageEngine) listContainers() (ListContainersRes, error) {
|
||||||
if e.metrics != nil {
|
if e.metrics != nil {
|
||||||
defer elapsed(e.metrics.AddListContainersDuration)()
|
defer elapsed("ListContainers", e.metrics.AddMethodDuration)()
|
||||||
}
|
}
|
||||||
|
|
||||||
uniqueIDs := make(map[string]cid.ID)
|
uniqueIDs := make(map[string]cid.ID)
|
||||||
|
|
|
@ -67,7 +67,7 @@ func (e *StorageEngine) Delete(ctx context.Context, prm DeletePrm) (res DeleteRe
|
||||||
|
|
||||||
func (e *StorageEngine) delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) {
|
func (e *StorageEngine) delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) {
|
||||||
if e.metrics != nil {
|
if e.metrics != nil {
|
||||||
defer elapsed(e.metrics.AddDeleteDuration)()
|
defer elapsed("Delete", e.metrics.AddMethodDuration)()
|
||||||
}
|
}
|
||||||
|
|
||||||
var locked struct {
|
var locked struct {
|
||||||
|
|
|
@ -64,7 +64,7 @@ func (e *StorageEngine) Get(ctx context.Context, prm GetPrm) (res GetRes, err er
|
||||||
|
|
||||||
func (e *StorageEngine) get(ctx context.Context, prm GetPrm) (GetRes, error) {
|
func (e *StorageEngine) get(ctx context.Context, prm GetPrm) (GetRes, error) {
|
||||||
if e.metrics != nil {
|
if e.metrics != nil {
|
||||||
defer elapsed(e.metrics.AddGetDuration)()
|
defer elapsed("Get", e.metrics.AddMethodDuration)()
|
||||||
}
|
}
|
||||||
|
|
||||||
var errNotFound apistatus.ObjectNotFound
|
var errNotFound apistatus.ObjectNotFound
|
||||||
|
|
|
@ -68,7 +68,7 @@ func (e *StorageEngine) head(ctx context.Context, prm HeadPrm) (HeadRes, error)
|
||||||
defer span.End()
|
defer span.End()
|
||||||
|
|
||||||
if e.metrics != nil {
|
if e.metrics != nil {
|
||||||
defer elapsed(e.metrics.AddHeadDuration)()
|
defer elapsed("Head", e.metrics.AddMethodDuration)()
|
||||||
}
|
}
|
||||||
|
|
||||||
var (
|
var (
|
||||||
|
|
|
@ -78,7 +78,7 @@ func (e *StorageEngine) Inhume(ctx context.Context, prm InhumePrm) (res InhumeRe
|
||||||
|
|
||||||
func (e *StorageEngine) inhume(ctx context.Context, prm InhumePrm) (InhumeRes, error) {
|
func (e *StorageEngine) inhume(ctx context.Context, prm InhumePrm) (InhumeRes, error) {
|
||||||
if e.metrics != nil {
|
if e.metrics != nil {
|
||||||
defer elapsed(e.metrics.AddInhumeDuration)()
|
defer elapsed("Inhume", e.metrics.AddMethodDuration)()
|
||||||
}
|
}
|
||||||
|
|
||||||
var shPrm shard.InhumePrm
|
var shPrm shard.InhumePrm
|
||||||
|
|
|
@ -7,17 +7,7 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
type MetricRegister interface {
|
type MetricRegister interface {
|
||||||
AddListContainersDuration(d time.Duration)
|
AddMethodDuration(method string, d time.Duration)
|
||||||
AddEstimateContainerSizeDuration(d time.Duration)
|
|
||||||
AddDeleteDuration(d time.Duration)
|
|
||||||
AddExistsDuration(d time.Duration)
|
|
||||||
AddGetDuration(d time.Duration)
|
|
||||||
AddHeadDuration(d time.Duration)
|
|
||||||
AddInhumeDuration(d time.Duration)
|
|
||||||
AddPutDuration(d time.Duration)
|
|
||||||
AddRangeDuration(d time.Duration)
|
|
||||||
AddSearchDuration(d time.Duration)
|
|
||||||
AddListObjectsDuration(d time.Duration)
|
|
||||||
|
|
||||||
SetObjectCounter(shardID, objectType string, v uint64)
|
SetObjectCounter(shardID, objectType string, v uint64)
|
||||||
AddToObjectCounter(shardID, objectType string, delta int)
|
AddToObjectCounter(shardID, objectType string, delta int)
|
||||||
|
@ -28,17 +18,17 @@ type MetricRegister interface {
|
||||||
AddToPayloadCounter(shardID string, size int64)
|
AddToPayloadCounter(shardID string, size int64)
|
||||||
IncErrorCounter(shardID string)
|
IncErrorCounter(shardID string)
|
||||||
ClearErrorCounter(shardID string)
|
ClearErrorCounter(shardID string)
|
||||||
DeleteErrorCounter(shardID string)
|
DeleteShardMetrics(shardID string)
|
||||||
|
|
||||||
WriteCache() metrics.WriteCacheMetrics
|
WriteCache() metrics.WriteCacheMetrics
|
||||||
GC() metrics.GCMetrics
|
GC() metrics.GCMetrics
|
||||||
}
|
}
|
||||||
|
|
||||||
func elapsed(addFunc func(d time.Duration)) func() {
|
func elapsed(method string, addFunc func(method string, d time.Duration)) func() {
|
||||||
t := time.Now()
|
t := time.Now()
|
||||||
|
|
||||||
return func() {
|
return func() {
|
||||||
addFunc(time.Since(t))
|
addFunc(method, time.Since(t))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -57,7 +57,7 @@ func (e *StorageEngine) Put(ctx context.Context, prm PutPrm) (err error) {
|
||||||
|
|
||||||
func (e *StorageEngine) put(ctx context.Context, prm PutPrm) error {
|
func (e *StorageEngine) put(ctx context.Context, prm PutPrm) error {
|
||||||
if e.metrics != nil {
|
if e.metrics != nil {
|
||||||
defer elapsed(e.metrics.AddPutDuration)()
|
defer elapsed("Put", e.metrics.AddMethodDuration)()
|
||||||
}
|
}
|
||||||
|
|
||||||
addr := object.AddressOf(prm.obj)
|
addr := object.AddressOf(prm.obj)
|
||||||
|
|
|
@ -80,7 +80,7 @@ func (e *StorageEngine) getRange(ctx context.Context, prm RngPrm) (RngRes, error
|
||||||
defer span.End()
|
defer span.End()
|
||||||
|
|
||||||
if e.metrics != nil {
|
if e.metrics != nil {
|
||||||
defer elapsed(e.metrics.AddRangeDuration)()
|
defer elapsed("GetRange", e.metrics.AddMethodDuration)()
|
||||||
}
|
}
|
||||||
|
|
||||||
var errNotFound apistatus.ObjectNotFound
|
var errNotFound apistatus.ObjectNotFound
|
||||||
|
|
|
@ -60,7 +60,7 @@ func (e *StorageEngine) Select(ctx context.Context, prm SelectPrm) (res SelectRe
|
||||||
|
|
||||||
func (e *StorageEngine) _select(ctx context.Context, prm SelectPrm) (SelectRes, error) {
|
func (e *StorageEngine) _select(ctx context.Context, prm SelectPrm) (SelectRes, error) {
|
||||||
if e.metrics != nil {
|
if e.metrics != nil {
|
||||||
defer elapsed(e.metrics.AddSearchDuration)()
|
defer elapsed("Search", e.metrics.AddMethodDuration)()
|
||||||
}
|
}
|
||||||
|
|
||||||
addrList := make([]oid.Address, 0)
|
addrList := make([]oid.Address, 0)
|
||||||
|
@ -109,7 +109,7 @@ func (e *StorageEngine) List(limit uint64) (res SelectRes, err error) {
|
||||||
|
|
||||||
func (e *StorageEngine) list(limit uint64) (SelectRes, error) {
|
func (e *StorageEngine) list(limit uint64) (SelectRes, error) {
|
||||||
if e.metrics != nil {
|
if e.metrics != nil {
|
||||||
defer elapsed(e.metrics.AddListObjectsDuration)()
|
defer elapsed("ListObjects", e.metrics.AddMethodDuration)()
|
||||||
}
|
}
|
||||||
|
|
||||||
addrList := make([]oid.Address, 0, limit)
|
addrList := make([]oid.Address, 0, limit)
|
||||||
|
|
|
@ -70,8 +70,8 @@ func (m *metricsWithID) ClearErrorCounter() {
|
||||||
m.mw.ClearErrorCounter(m.id)
|
m.mw.ClearErrorCounter(m.id)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *metricsWithID) DeleteErrorCounter() {
|
func (m *metricsWithID) DeleteShardMetrics() {
|
||||||
m.mw.DeleteErrorCounter(m.id)
|
m.mw.DeleteShardMetrics(m.id)
|
||||||
}
|
}
|
||||||
|
|
||||||
// AddShard adds a new shard to the storage engine.
|
// AddShard adds a new shard to the storage engine.
|
||||||
|
@ -186,7 +186,7 @@ func (e *StorageEngine) removeShards(ids ...string) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
sh.DeleteErrorCounter()
|
sh.DeleteShardMetrics()
|
||||||
|
|
||||||
ss = append(ss, sh)
|
ss = append(ss, sh)
|
||||||
delete(e.shards, id)
|
delete(e.shards, id)
|
||||||
|
|
|
@ -77,7 +77,7 @@ func (m *metricsStore) ClearErrorCounter() {
|
||||||
m.errCounter = 0
|
m.errCounter = 0
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *metricsStore) DeleteErrorCounter() {
|
func (m *metricsStore) DeleteShardMetrics() {
|
||||||
m.errCounter = 0
|
m.errCounter = 0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -79,8 +79,8 @@ type MetricsWriter interface {
|
||||||
IncErrorCounter()
|
IncErrorCounter()
|
||||||
// ClearErrorCounter clear error counter.
|
// ClearErrorCounter clear error counter.
|
||||||
ClearErrorCounter()
|
ClearErrorCounter()
|
||||||
// DeleteErrorCounter delete error counter.
|
// DeleteShardMetrics deletes shard metrics from registry.
|
||||||
DeleteErrorCounter()
|
DeleteShardMetrics()
|
||||||
}
|
}
|
||||||
|
|
||||||
type cfg struct {
|
type cfg struct {
|
||||||
|
@ -447,8 +447,8 @@ func (s *Shard) ClearErrorCounter() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Shard) DeleteErrorCounter() {
|
func (s *Shard) DeleteShardMetrics() {
|
||||||
if s.cfg.metricsWriter != nil {
|
if s.cfg.metricsWriter != nil {
|
||||||
s.cfg.metricsWriter.DeleteErrorCounter()
|
s.cfg.metricsWriter.DeleteShardMetrics()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,8 +1,6 @@
|
||||||
package metrics
|
package metrics
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
|
||||||
"strings"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-observability/metrics"
|
"git.frostfs.info/TrueCloudLab/frostfs-observability/metrics"
|
||||||
|
@ -11,60 +9,33 @@ import (
|
||||||
|
|
||||||
type (
|
type (
|
||||||
engineMetrics struct {
|
engineMetrics struct {
|
||||||
listContainersDuration prometheus.Counter
|
methodDuration *prometheus.HistogramVec
|
||||||
estimateContainerSizeDuration prometheus.Counter
|
|
||||||
deleteDuration prometheus.Counter
|
containerSize *prometheus.GaugeVec
|
||||||
existsDuration prometheus.Counter
|
payloadSize *prometheus.GaugeVec
|
||||||
getDuration prometheus.Counter
|
errorCounter *prometheus.GaugeVec
|
||||||
headDuration prometheus.Counter
|
|
||||||
inhumeDuration prometheus.Counter
|
|
||||||
putDuration prometheus.Counter
|
|
||||||
rangeDuration prometheus.Counter
|
|
||||||
searchDuration prometheus.Counter
|
|
||||||
listObjectsDuration prometheus.Counter
|
|
||||||
containerSize *prometheus.GaugeVec
|
|
||||||
payloadSize *prometheus.GaugeVec
|
|
||||||
errorCounter *prometheus.GaugeVec
|
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
const engineSubsystem = "engine"
|
const (
|
||||||
|
engineSubsystem = "engine"
|
||||||
|
engineMethod = "method"
|
||||||
|
)
|
||||||
|
|
||||||
func newEngineMetrics() engineMetrics {
|
func newEngineMetrics() engineMetrics {
|
||||||
return engineMetrics{
|
return engineMetrics{
|
||||||
listContainersDuration: newEngineMethodDurationCounter("list_containers_"),
|
containerSize: newEngineGaugeVector("container_size_bytes", "Accumulated size of all objects in a container", []string{containerIDLabelKey}),
|
||||||
estimateContainerSizeDuration: newEngineCounter("estimate_container_size_duration", "Accumulated duration of engine container size estimate operations"),
|
payloadSize: newEngineGaugeVector("payload_size_bytes", "Accumulated size of all objects in a shard", []string{shardIDLabelKey}),
|
||||||
deleteDuration: newEngineMethodDurationCounter("delete"),
|
errorCounter: newEngineGaugeVector("error_counter", "Shard's error counter", []string{shardIDLabelKey}),
|
||||||
existsDuration: newEngineMethodDurationCounter("exists"),
|
methodDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{
|
||||||
getDuration: newEngineMethodDurationCounter("get"),
|
Namespace: namespace,
|
||||||
headDuration: newEngineMethodDurationCounter("head"),
|
Subsystem: engineSubsystem,
|
||||||
inhumeDuration: newEngineMethodDurationCounter("inhume"),
|
Name: "request_duration_seconds",
|
||||||
putDuration: newEngineMethodDurationCounter("put"),
|
Help: "Duration of Engine requests",
|
||||||
rangeDuration: newEngineMethodDurationCounter("range"),
|
}, []string{engineMethod}),
|
||||||
searchDuration: newEngineMethodDurationCounter("search"),
|
|
||||||
listObjectsDuration: newEngineMethodDurationCounter("list_objects"),
|
|
||||||
containerSize: newEngineGaugeVector("container_size", "Accumulated size of all objects in a container", []string{containerIDLabelKey}),
|
|
||||||
payloadSize: newEngineGaugeVector("payload_size", "Accumulated size of all objects in a shard", []string{shardIDLabelKey}),
|
|
||||||
errorCounter: newEngineGaugeVector("error_counter", "Shard's error counter", []string{shardIDLabelKey}),
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func newEngineCounter(name, help string) prometheus.Counter {
|
|
||||||
return metrics.NewCounter(prometheus.CounterOpts{
|
|
||||||
Namespace: namespace,
|
|
||||||
Subsystem: engineSubsystem,
|
|
||||||
Name: name,
|
|
||||||
Help: help,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func newEngineMethodDurationCounter(method string) prometheus.Counter {
|
|
||||||
return newEngineCounter(
|
|
||||||
fmt.Sprintf("%s_duration", method),
|
|
||||||
fmt.Sprintf("Accumulated duration of engine %s operations", strings.ReplaceAll(method, "_", " ")),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
func newEngineGaugeVector(name, help string, labels []string) *prometheus.GaugeVec {
|
func newEngineGaugeVector(name, help string, labels []string) *prometheus.GaugeVec {
|
||||||
return metrics.NewGaugeVec(prometheus.GaugeOpts{
|
return metrics.NewGaugeVec(prometheus.GaugeOpts{
|
||||||
Namespace: namespace,
|
Namespace: namespace,
|
||||||
|
@ -74,48 +45,10 @@ func newEngineGaugeVector(name, help string, labels []string) *prometheus.GaugeV
|
||||||
}, labels)
|
}, labels)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m engineMetrics) AddListContainersDuration(d time.Duration) {
|
func (m *engineMetrics) AddMethodDuration(method string, d time.Duration) {
|
||||||
m.listObjectsDuration.Add(float64(d))
|
m.methodDuration.With(prometheus.Labels{
|
||||||
}
|
engineMethod: method,
|
||||||
|
}).Observe(d.Seconds())
|
||||||
func (m engineMetrics) AddEstimateContainerSizeDuration(d time.Duration) {
|
|
||||||
m.estimateContainerSizeDuration.Add(float64(d))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m engineMetrics) AddDeleteDuration(d time.Duration) {
|
|
||||||
m.deleteDuration.Add(float64(d))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m engineMetrics) AddExistsDuration(d time.Duration) {
|
|
||||||
m.existsDuration.Add(float64(d))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m engineMetrics) AddGetDuration(d time.Duration) {
|
|
||||||
m.getDuration.Add(float64(d))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m engineMetrics) AddHeadDuration(d time.Duration) {
|
|
||||||
m.headDuration.Add(float64(d))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m engineMetrics) AddInhumeDuration(d time.Duration) {
|
|
||||||
m.inhumeDuration.Add(float64(d))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m engineMetrics) AddPutDuration(d time.Duration) {
|
|
||||||
m.putDuration.Add(float64(d))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m engineMetrics) AddRangeDuration(d time.Duration) {
|
|
||||||
m.rangeDuration.Add(float64(d))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m engineMetrics) AddSearchDuration(d time.Duration) {
|
|
||||||
m.searchDuration.Add(float64(d))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m engineMetrics) AddListObjectsDuration(d time.Duration) {
|
|
||||||
m.listObjectsDuration.Add(float64(d))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m engineMetrics) AddToContainerSize(cnrID string, size int64) {
|
func (m engineMetrics) AddToContainerSize(cnrID string, size int64) {
|
||||||
|
@ -134,6 +67,7 @@ func (m engineMetrics) ClearErrorCounter(shardID string) {
|
||||||
m.errorCounter.With(prometheus.Labels{shardIDLabelKey: shardID}).Set(0)
|
m.errorCounter.With(prometheus.Labels{shardIDLabelKey: shardID}).Set(0)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m engineMetrics) DeleteErrorCounter(shardID string) {
|
func (m engineMetrics) DeleteShardMetrics(shardID string) {
|
||||||
m.errorCounter.Delete(prometheus.Labels{shardIDLabelKey: shardID})
|
m.errorCounter.Delete(prometheus.Labels{shardIDLabelKey: shardID})
|
||||||
|
m.payloadSize.Delete(prometheus.Labels{shardIDLabelKey: shardID})
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue