[#1653] objectSvc: Add operations by IO tag metric

Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>

internal/metrics/object.go

@@ -9,13 +9,14 @@ import (
 )
 
 type ObjectServiceMetrics interface {
-	AddRequestDuration(method string, d time.Duration, success bool)
+	AddRequestDuration(method string, d time.Duration, success bool, ioTag string)
 	AddPayloadSize(method string, size int)
 }
 
 type objectServiceMetrics struct {
 	methodDuration  *prometheus.HistogramVec
 	payloadCounter  *prometheus.CounterVec
+	ioTagOpsCounter *prometheus.CounterVec
 }
 
 func newObjectServiceMetrics() *objectServiceMetrics {
@@ -32,14 +33,24 @@ func newObjectServiceMetrics() *objectServiceMetrics {
 			Name:      "request_payload_bytes",
 			Help:      "Object Service request payload",
 		}, []string{methodLabel}),
+		ioTagOpsCounter: metrics.NewCounterVec(prometheus.CounterOpts{
+			Namespace: namespace,
+			Subsystem: objectSubsystem,
+			Name:      "requests_total",
+			Help:      "Count of requests for each IO tag",
+		}, []string{methodLabel, ioTagLabel}),
 	}
 }
 
-func (m *objectServiceMetrics) AddRequestDuration(method string, d time.Duration, success bool) {
+func (m *objectServiceMetrics) AddRequestDuration(method string, d time.Duration, success bool, ioTag string) {
 	m.methodDuration.With(prometheus.Labels{
 		methodLabel:  method,
 		successLabel: strconv.FormatBool(success),
 	}).Observe(d.Seconds())
+	m.ioTagOpsCounter.With(prometheus.Labels{
+		ioTagLabel:  ioTag,
+		methodLabel: method,
+	}).Inc()
 }
 
 func (m *objectServiceMetrics) AddPayloadSize(method string, size int) {

internal/qos/tags.go

@@ -1,6 +1,11 @@
 package qos
 
-import "fmt"
+import (
+	"context"
+	"fmt"
+
+	"git.frostfs.info/TrueCloudLab/frostfs-qos/tagging"
+)
 
 type IOTag string
 
@@ -37,3 +42,11 @@ func FromRawString(s string) (IOTag, error) {
 func (t IOTag) String() string {
 	return string(t)
 }
+
+func IOTagFromContext(ctx context.Context) string {
+	tag, ok := tagging.IOTagFromContext(ctx)
+	if !ok {
+		tag = "undefined"
+	}
+	return tag
+}

pkg/services/object/metrics.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"time"
 
+	"git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util"
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object"
 )
@@ -34,7 +35,7 @@ type (
 	}
 
 	MetricRegister interface {
-		AddRequestDuration(string, time.Duration, bool)
+		AddRequestDuration(string, time.Duration, bool, string)
 		AddPayloadSize(string, int)
 	}
 )
@@ -51,7 +52,7 @@ func (m MetricCollector) Get(req *object.GetRequest, stream GetObjectStream) (er
 	if m.enabled {
 		t := time.Now()
 		defer func() {
-			m.metrics.AddRequestDuration("Get", time.Since(t), err == nil)
+			m.metrics.AddRequestDuration("Get", time.Since(t), err == nil, qos.IOTagFromContext(stream.Context()))
 		}()
 		err = m.next.Get(req, &getStreamMetric{
 			ServerStream: stream,
@@ -106,7 +107,7 @@ func (m MetricCollector) PutSingle(ctx context.Context, request *object.PutSingl
 
 		res, err := m.next.PutSingle(ctx, request)
 
-		m.metrics.AddRequestDuration("PutSingle", time.Since(t), err == nil)
+		m.metrics.AddRequestDuration("PutSingle", time.Since(t), err == nil, qos.IOTagFromContext(ctx))
 		if err == nil {
 			m.metrics.AddPayloadSize("PutSingle", len(request.GetBody().GetObject().GetPayload()))
 		}
@@ -122,7 +123,7 @@ func (m MetricCollector) Head(ctx context.Context, request *object.HeadRequest)
 
 		res, err := m.next.Head(ctx, request)
 
-		m.metrics.AddRequestDuration("Head", time.Since(t), err == nil)
+		m.metrics.AddRequestDuration("Head", time.Since(t), err == nil, qos.IOTagFromContext(ctx))
 
 		return res, err
 	}
@@ -135,7 +136,7 @@ func (m MetricCollector) Search(req *object.SearchRequest, stream SearchStream)
 
 		err := m.next.Search(req, stream)
 
-		m.metrics.AddRequestDuration("Search", time.Since(t), err == nil)
+		m.metrics.AddRequestDuration("Search", time.Since(t), err == nil, qos.IOTagFromContext(stream.Context()))
 
 		return err
 	}
@@ -148,7 +149,7 @@ func (m MetricCollector) Delete(ctx context.Context, request *object.DeleteReque
 
 		res, err := m.next.Delete(ctx, request)
 
-		m.metrics.AddRequestDuration("Delete", time.Since(t), err == nil)
+		m.metrics.AddRequestDuration("Delete", time.Since(t), err == nil, qos.IOTagFromContext(ctx))
 		return res, err
 	}
 	return m.next.Delete(ctx, request)
@@ -160,7 +161,7 @@ func (m MetricCollector) GetRange(req *object.GetRangeRequest, stream GetObjectR
 
 		err := m.next.GetRange(req, stream)
 
-		m.metrics.AddRequestDuration("GetRange", time.Since(t), err == nil)
+		m.metrics.AddRequestDuration("GetRange", time.Since(t), err == nil, qos.IOTagFromContext(stream.Context()))
 
 		return err
 	}
@@ -173,7 +174,7 @@ func (m MetricCollector) GetRangeHash(ctx context.Context, request *object.GetRa
 
 		res, err := m.next.GetRangeHash(ctx, request)
 
-		m.metrics.AddRequestDuration("GetRangeHash", time.Since(t), err == nil)
+		m.metrics.AddRequestDuration("GetRangeHash", time.Since(t), err == nil, qos.IOTagFromContext(ctx))
 
 		return res, err
 	}
@@ -209,7 +210,7 @@ func (s putStreamMetric) Send(ctx context.Context, req *object.PutRequest) error
 func (s putStreamMetric) CloseAndRecv(ctx context.Context) (*object.PutResponse, error) {
 	res, err := s.stream.CloseAndRecv(ctx)
 
-	s.metrics.AddRequestDuration("Put", time.Since(s.start), err == nil)
+	s.metrics.AddRequestDuration("Put", time.Since(s.start), err == nil, qos.IOTagFromContext(ctx))
 
 	return res, err
 }
@@ -223,7 +224,7 @@ func (s patchStreamMetric) Send(ctx context.Context, req *object.PatchRequest) e
 func (s patchStreamMetric) CloseAndRecv(ctx context.Context) (*object.PatchResponse, error) {
 	res, err := s.stream.CloseAndRecv(ctx)
 
-	s.metrics.AddRequestDuration("Patch", time.Since(s.start), err == nil)
+	s.metrics.AddRequestDuration("Patch", time.Since(s.start), err == nil, qos.IOTagFromContext(ctx))
 
 	return res, err
 }