From ab342c4ebaf521c779fa484758f43ccd9cbe4c1e Mon Sep 17 00:00:00 2001
From: Dmitrii Stepanov <d.stepanov@yadro.com>
Date: Wed, 5 Mar 2025 15:25:35 +0300
Subject: [PATCH] [#1653] objectSvc: Add operations by IO tag metric
Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
---
internal/metrics/object.go | 19 +++++++++++++++----
internal/qos/tags.go | 15 ++++++++++++++-
pkg/services/object/metrics.go | 21 +++++++++++----------
3 files changed, 40 insertions(+), 15 deletions(-)
diff --git a/internal/metrics/object.go b/internal/metrics/object.go
index 0ba994ed3..e4f6dfde1 100644
--- a/internal/metrics/object.go
+++ b/internal/metrics/object.go
@@ -9,13 +9,14 @@ import (
)
type ObjectServiceMetrics interface {
- AddRequestDuration(method string, d time.Duration, success bool)
+ AddRequestDuration(method string, d time.Duration, success bool, ioTag string)
AddPayloadSize(method string, size int)
}
type objectServiceMetrics struct {
- methodDuration *prometheus.HistogramVec
- payloadCounter *prometheus.CounterVec
+ methodDuration *prometheus.HistogramVec
+ payloadCounter *prometheus.CounterVec
+ ioTagOpsCounter *prometheus.CounterVec
}
func newObjectServiceMetrics() *objectServiceMetrics {
@@ -32,14 +33,24 @@ func newObjectServiceMetrics() *objectServiceMetrics {
Name: "request_payload_bytes",
Help: "Object Service request payload",
}, []string{methodLabel}),
+ ioTagOpsCounter: metrics.NewCounterVec(prometheus.CounterOpts{
+ Namespace: namespace,
+ Subsystem: objectSubsystem,
+ Name: "requests_total",
+ Help: "Count of requests for each IO tag",
+ }, []string{methodLabel, ioTagLabel}),
}
}
-func (m *objectServiceMetrics) AddRequestDuration(method string, d time.Duration, success bool) {
+func (m *objectServiceMetrics) AddRequestDuration(method string, d time.Duration, success bool, ioTag string) {
m.methodDuration.With(prometheus.Labels{
methodLabel: method,
successLabel: strconv.FormatBool(success),
}).Observe(d.Seconds())
+ m.ioTagOpsCounter.With(prometheus.Labels{
+ ioTagLabel: ioTag,
+ methodLabel: method,
+ }).Inc()
}
func (m *objectServiceMetrics) AddPayloadSize(method string, size int) {
diff --git a/internal/qos/tags.go b/internal/qos/tags.go
index 6a9a7f7a4..9db45f190 100644
--- a/internal/qos/tags.go
+++ b/internal/qos/tags.go
@@ -1,6 +1,11 @@
package qos
-import "fmt"
+import (
+ "context"
+ "fmt"
+
+ "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging"
+)
type IOTag string
@@ -37,3 +42,11 @@ func FromRawString(s string) (IOTag, error) {
func (t IOTag) String() string {
return string(t)
}
+
+func IOTagFromContext(ctx context.Context) string {
+ tag, ok := tagging.IOTagFromContext(ctx)
+ if !ok {
+ tag = "undefined"
+ }
+ return tag
+}
diff --git a/pkg/services/object/metrics.go b/pkg/services/object/metrics.go
index 19748e938..6a6ee0f0f 100644
--- a/pkg/services/object/metrics.go
+++ b/pkg/services/object/metrics.go
@@ -4,6 +4,7 @@ import (
"context"
"time"
+ "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object"
)
@@ -34,7 +35,7 @@ type (
}
MetricRegister interface {
- AddRequestDuration(string, time.Duration, bool)
+ AddRequestDuration(string, time.Duration, bool, string)
AddPayloadSize(string, int)
}
)
@@ -51,7 +52,7 @@ func (m MetricCollector) Get(req *object.GetRequest, stream GetObjectStream) (er
if m.enabled {
t := time.Now()
defer func() {
- m.metrics.AddRequestDuration("Get", time.Since(t), err == nil)
+ m.metrics.AddRequestDuration("Get", time.Since(t), err == nil, qos.IOTagFromContext(stream.Context()))
}()
err = m.next.Get(req, &getStreamMetric{
ServerStream: stream,
@@ -106,7 +107,7 @@ func (m MetricCollector) PutSingle(ctx context.Context, request *object.PutSingl
res, err := m.next.PutSingle(ctx, request)
- m.metrics.AddRequestDuration("PutSingle", time.Since(t), err == nil)
+ m.metrics.AddRequestDuration("PutSingle", time.Since(t), err == nil, qos.IOTagFromContext(ctx))
if err == nil {
m.metrics.AddPayloadSize("PutSingle", len(request.GetBody().GetObject().GetPayload()))
}
@@ -122,7 +123,7 @@ func (m MetricCollector) Head(ctx context.Context, request *object.HeadRequest)
res, err := m.next.Head(ctx, request)
- m.metrics.AddRequestDuration("Head", time.Since(t), err == nil)
+ m.metrics.AddRequestDuration("Head", time.Since(t), err == nil, qos.IOTagFromContext(ctx))
return res, err
}
@@ -135,7 +136,7 @@ func (m MetricCollector) Search(req *object.SearchRequest, stream SearchStream)
err := m.next.Search(req, stream)
- m.metrics.AddRequestDuration("Search", time.Since(t), err == nil)
+ m.metrics.AddRequestDuration("Search", time.Since(t), err == nil, qos.IOTagFromContext(stream.Context()))
return err
}
@@ -148,7 +149,7 @@ func (m MetricCollector) Delete(ctx context.Context, request *object.DeleteReque
res, err := m.next.Delete(ctx, request)
- m.metrics.AddRequestDuration("Delete", time.Since(t), err == nil)
+ m.metrics.AddRequestDuration("Delete", time.Since(t), err == nil, qos.IOTagFromContext(ctx))
return res, err
}
return m.next.Delete(ctx, request)
@@ -160,7 +161,7 @@ func (m MetricCollector) GetRange(req *object.GetRangeRequest, stream GetObjectR
err := m.next.GetRange(req, stream)
- m.metrics.AddRequestDuration("GetRange", time.Since(t), err == nil)
+ m.metrics.AddRequestDuration("GetRange", time.Since(t), err == nil, qos.IOTagFromContext(stream.Context()))
return err
}
@@ -173,7 +174,7 @@ func (m MetricCollector) GetRangeHash(ctx context.Context, request *object.GetRa
res, err := m.next.GetRangeHash(ctx, request)
- m.metrics.AddRequestDuration("GetRangeHash", time.Since(t), err == nil)
+ m.metrics.AddRequestDuration("GetRangeHash", time.Since(t), err == nil, qos.IOTagFromContext(ctx))
return res, err
}
@@ -209,7 +210,7 @@ func (s putStreamMetric) Send(ctx context.Context, req *object.PutRequest) error
func (s putStreamMetric) CloseAndRecv(ctx context.Context) (*object.PutResponse, error) {
res, err := s.stream.CloseAndRecv(ctx)
- s.metrics.AddRequestDuration("Put", time.Since(s.start), err == nil)
+ s.metrics.AddRequestDuration("Put", time.Since(s.start), err == nil, qos.IOTagFromContext(ctx))
return res, err
}
@@ -223,7 +224,7 @@ func (s patchStreamMetric) Send(ctx context.Context, req *object.PatchRequest) e
func (s patchStreamMetric) CloseAndRecv(ctx context.Context) (*object.PatchResponse, error) {
res, err := s.stream.CloseAndRecv(ctx)
- s.metrics.AddRequestDuration("Patch", time.Since(s.start), err == nil)
+ s.metrics.AddRequestDuration("Patch", time.Since(s.start), err == nil, qos.IOTagFromContext(ctx))
return res, err
}