From b510d3b13f179a2e86e30f17a4660220d4f86c9b Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 18 Feb 2025 11:23:10 +0300 Subject: [PATCH] [#9999] blobstor: Add IO tag label to metrics Signed-off-by: Dmitrii Stepanov --- internal/metrics/blobstore.go | 7 ++-- internal/metrics/consts.go | 1 + internal/qos/tags.go | 16 ++++++++- pkg/local_object_storage/blobstor/delete.go | 3 +- pkg/local_object_storage/blobstor/exists.go | 3 +- pkg/local_object_storage/blobstor/get.go | 3 +- .../blobstor/get_range.go | 3 +- pkg/local_object_storage/blobstor/info.go | 3 +- pkg/local_object_storage/blobstor/iterate.go | 3 +- pkg/local_object_storage/blobstor/metrics.go | 34 +++++++++---------- pkg/local_object_storage/blobstor/put.go | 3 +- pkg/local_object_storage/metrics/blobstore.go | 28 +++++++-------- 12 files changed, 65 insertions(+), 42 deletions(-) diff --git a/internal/metrics/blobstore.go b/internal/metrics/blobstore.go index d9bb3f029..535c8732a 100644 --- a/internal/metrics/blobstore.go +++ b/internal/metrics/blobstore.go @@ -12,7 +12,7 @@ type BlobstoreMetrics interface { SetMode(shardID string, readOnly bool) Close(shardID string) - MethodDuration(shardID string, method string, d time.Duration, success bool, withStorageID NullBool) + MethodDuration(shardID string, method string, d time.Duration, success bool, withStorageID NullBool, ioTag string) AddPut(shardID string, size int) AddGet(shardID string, size int) } @@ -32,7 +32,7 @@ func newBlobstoreMetrics() *blobstoreMetrics { Subsystem: blobstoreSubSystem, Name: "request_duration_seconds", Help: "Accumulated Blobstore request process duration", - }, []string{shardIDLabel, successLabel, methodLabel, withStorageIDLabel}), + }, []string{shardIDLabel, successLabel, methodLabel, withStorageIDLabel, ioTagLabel}), put: metrics.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: blobstoreSubSystem, @@ -65,12 +65,13 @@ func (m *blobstoreMetrics) Close(shardID string) { }) } -func (m *blobstoreMetrics) MethodDuration(shardID string, method string, d time.Duration, success bool, withStorageID NullBool) { +func (m *blobstoreMetrics) MethodDuration(shardID string, method string, d time.Duration, success bool, withStorageID NullBool, ioTag string) { m.reqDuration.With(prometheus.Labels{ shardIDLabel: shardID, successLabel: strconv.FormatBool(success), methodLabel: method, withStorageIDLabel: withStorageID.String(), + ioTagLabel: ioTag, }).Observe(d.Seconds()) } diff --git a/internal/metrics/consts.go b/internal/metrics/consts.go index cb165de69..c384915d5 100644 --- a/internal/metrics/consts.go +++ b/internal/metrics/consts.go @@ -43,6 +43,7 @@ const ( hitLabel = "hit" cacheLabel = "cache" sourceIPLabel = "source_ip" + ioTagLabel = "io_tag" readWriteMode = "READ_WRITE" readOnlyMode = "READ_ONLY" diff --git a/internal/qos/tags.go b/internal/qos/tags.go index 6a9a7f7a4..0862c159b 100644 --- a/internal/qos/tags.go +++ b/internal/qos/tags.go @@ -1,6 +1,11 @@ package qos -import "fmt" +import ( + "context" + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" +) type IOTag string @@ -37,3 +42,12 @@ func FromRawString(s string) (IOTag, error) { func (t IOTag) String() string { return string(t) } + +var undefinedIOTag = "undefined" + +func IOTagStringFromContext(ctx context.Context) string { + if v, ok := tagging.IOTagFromContext(ctx); ok { + return v + } + return undefinedIOTag +} diff --git a/pkg/local_object_storage/blobstor/delete.go b/pkg/local_object_storage/blobstor/delete.go index 86d8f15e3..78e07a2cf 100644 --- a/pkg/local_object_storage/blobstor/delete.go +++ b/pkg/local_object_storage/blobstor/delete.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" @@ -20,7 +21,7 @@ func (b *BlobStor) Delete(ctx context.Context, prm common.DeletePrm) (common.Del success = false ) defer func() { - b.metrics.Delete(time.Since(startedAt), success, prm.StorageID != nil) + b.metrics.Delete(time.Since(startedAt), success, prm.StorageID != nil, qos.IOTagStringFromContext(ctx)) }() ctx, span := tracing.StartSpanFromContext(ctx, "BlobStor.Delete", diff --git a/pkg/local_object_storage/blobstor/exists.go b/pkg/local_object_storage/blobstor/exists.go index c155e15b8..25005dcd2 100644 --- a/pkg/local_object_storage/blobstor/exists.go +++ b/pkg/local_object_storage/blobstor/exists.go @@ -6,6 +6,7 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.opentelemetry.io/otel/attribute" @@ -23,7 +24,7 @@ func (b *BlobStor) Exists(ctx context.Context, prm common.ExistsPrm) (common.Exi startedAt = time.Now() ) defer func() { - b.metrics.Exists(time.Since(startedAt), exists, prm.StorageID != nil) + b.metrics.Exists(time.Since(startedAt), exists, prm.StorageID != nil, qos.IOTagStringFromContext(ctx)) }() ctx, span := tracing.StartSpanFromContext(ctx, "BlobStor.Exists", diff --git a/pkg/local_object_storage/blobstor/get.go b/pkg/local_object_storage/blobstor/get.go index d00ef2f21..41f9e3643 100644 --- a/pkg/local_object_storage/blobstor/get.go +++ b/pkg/local_object_storage/blobstor/get.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" @@ -20,7 +21,7 @@ import ( func (b *BlobStor) Get(ctx context.Context, prm common.GetPrm) (res common.GetRes, err error) { startedAt := time.Now() defer func() { - b.metrics.Get(time.Since(startedAt), len(res.RawData), err == nil, prm.StorageID != nil) + b.metrics.Get(time.Since(startedAt), len(res.RawData), err == nil, prm.StorageID != nil, qos.IOTagStringFromContext(ctx)) }() ctx, span := tracing.StartSpanFromContext(ctx, "BlobStor.Get", diff --git a/pkg/local_object_storage/blobstor/get_range.go b/pkg/local_object_storage/blobstor/get_range.go index 9bded4720..8da56334a 100644 --- a/pkg/local_object_storage/blobstor/get_range.go +++ b/pkg/local_object_storage/blobstor/get_range.go @@ -6,6 +6,7 @@ import ( "strconv" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" @@ -21,7 +22,7 @@ import ( func (b *BlobStor) GetRange(ctx context.Context, prm common.GetRangePrm) (res common.GetRangeRes, err error) { startedAt := time.Now() defer func() { - b.metrics.GetRange(time.Since(startedAt), len(res.Data), err == nil, prm.StorageID != nil) + b.metrics.GetRange(time.Since(startedAt), len(res.Data), err == nil, prm.StorageID != nil, qos.IOTagStringFromContext(ctx)) }() ctx, span := tracing.StartSpanFromContext(ctx, "BlobStor.GetRange", diff --git a/pkg/local_object_storage/blobstor/info.go b/pkg/local_object_storage/blobstor/info.go index c1c47f3bb..07cef9419 100644 --- a/pkg/local_object_storage/blobstor/info.go +++ b/pkg/local_object_storage/blobstor/info.go @@ -5,6 +5,7 @@ import ( "sync/atomic" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "golang.org/x/sync/errgroup" ) @@ -30,7 +31,7 @@ func (b *BlobStor) ObjectsCount(ctx context.Context) (uint64, error) { var err error startedAt := time.Now() defer func() { - b.metrics.ObjectsCount(time.Since(startedAt), err == nil) + b.metrics.ObjectsCount(time.Since(startedAt), err == nil, qos.IOTagStringFromContext(ctx)) }() ctx, span := tracing.StartSpanFromContext(ctx, "BlobStor.ObjectsCount") diff --git a/pkg/local_object_storage/blobstor/iterate.go b/pkg/local_object_storage/blobstor/iterate.go index ff1aa9d64..4979cb1d1 100644 --- a/pkg/local_object_storage/blobstor/iterate.go +++ b/pkg/local_object_storage/blobstor/iterate.go @@ -6,6 +6,7 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -27,7 +28,7 @@ func (b *BlobStor) Iterate(ctx context.Context, prm common.IteratePrm) (common.I success = false ) defer func() { - b.metrics.Iterate(time.Since(startedAt), success) + b.metrics.Iterate(time.Since(startedAt), success, qos.IOTagStringFromContext(ctx)) }() ctx, span := tracing.StartSpanFromContext(ctx, "BlobStor.Iterate", trace.WithAttributes( diff --git a/pkg/local_object_storage/blobstor/metrics.go b/pkg/local_object_storage/blobstor/metrics.go index aadc237af..fe755cfeb 100644 --- a/pkg/local_object_storage/blobstor/metrics.go +++ b/pkg/local_object_storage/blobstor/metrics.go @@ -7,24 +7,24 @@ type Metrics interface { SetMode(readOnly bool) Close() - Delete(d time.Duration, success, withStorageID bool) - Exists(d time.Duration, success, withStorageID bool) - GetRange(d time.Duration, size int, success, withStorageID bool) - Get(d time.Duration, size int, success, withStorageID bool) - Iterate(d time.Duration, success bool) - Put(d time.Duration, size int, success bool) - ObjectsCount(d time.Duration, success bool) + Delete(d time.Duration, success, withStorageID bool, ioTag string) + Exists(d time.Duration, success, withStorageID bool, ioTag string) + GetRange(d time.Duration, size int, success, withStorageID bool, ioTag string) + Get(d time.Duration, size int, success, withStorageID bool, ioTag string) + Iterate(d time.Duration, success bool, ioTag string) + Put(d time.Duration, size int, success bool, ioTag string) + ObjectsCount(d time.Duration, success bool, ioTag string) } type noopMetrics struct{} -func (m *noopMetrics) SetParentID(string) {} -func (m *noopMetrics) SetMode(bool) {} -func (m *noopMetrics) Close() {} -func (m *noopMetrics) Delete(time.Duration, bool, bool) {} -func (m *noopMetrics) Exists(time.Duration, bool, bool) {} -func (m *noopMetrics) GetRange(time.Duration, int, bool, bool) {} -func (m *noopMetrics) Get(time.Duration, int, bool, bool) {} -func (m *noopMetrics) Iterate(time.Duration, bool) {} -func (m *noopMetrics) Put(time.Duration, int, bool) {} -func (m *noopMetrics) ObjectsCount(time.Duration, bool) {} +func (m *noopMetrics) SetParentID(string) {} +func (m *noopMetrics) SetMode(bool) {} +func (m *noopMetrics) Close() {} +func (m *noopMetrics) Delete(time.Duration, bool, bool, string) {} +func (m *noopMetrics) Exists(time.Duration, bool, bool, string) {} +func (m *noopMetrics) GetRange(time.Duration, int, bool, bool, string) {} +func (m *noopMetrics) Get(time.Duration, int, bool, bool, string) {} +func (m *noopMetrics) Iterate(time.Duration, bool, string) {} +func (m *noopMetrics) Put(time.Duration, int, bool, string) {} +func (m *noopMetrics) ObjectsCount(time.Duration, bool, string) {} diff --git a/pkg/local_object_storage/blobstor/put.go b/pkg/local_object_storage/blobstor/put.go index fe9c109dd..f2f6ea298 100644 --- a/pkg/local_object_storage/blobstor/put.go +++ b/pkg/local_object_storage/blobstor/put.go @@ -5,6 +5,7 @@ import ( "fmt" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" @@ -32,7 +33,7 @@ func (b *BlobStor) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, e size = 0 ) defer func() { - b.metrics.Put(time.Since(startedAt), size, success) + b.metrics.Put(time.Since(startedAt), size, success, qos.IOTagStringFromContext(ctx)) }() ctx, span := tracing.StartSpanFromContext(ctx, "BlobStor.Put", diff --git a/pkg/local_object_storage/metrics/blobstore.go b/pkg/local_object_storage/metrics/blobstore.go index 9a41f01d0..8387b0a73 100644 --- a/pkg/local_object_storage/metrics/blobstore.go +++ b/pkg/local_object_storage/metrics/blobstore.go @@ -31,39 +31,39 @@ func (m *blobstoreMetrics) Close() { m.m.Close(m.shardID) } -func (m *blobstoreMetrics) Delete(d time.Duration, success, withStorageID bool) { - m.m.MethodDuration(m.shardID, "Delete", d, success, metrics_impl.NullBool{Bool: withStorageID, Valid: true}) +func (m *blobstoreMetrics) Delete(d time.Duration, success, withStorageID bool, ioTag string) { + m.m.MethodDuration(m.shardID, "Delete", d, success, metrics_impl.NullBool{Bool: withStorageID, Valid: true}, ioTag) } -func (m *blobstoreMetrics) Exists(d time.Duration, success, withStorageID bool) { - m.m.MethodDuration(m.shardID, "Exists", d, success, metrics_impl.NullBool{Bool: withStorageID, Valid: true}) +func (m *blobstoreMetrics) Exists(d time.Duration, success, withStorageID bool, ioTag string) { + m.m.MethodDuration(m.shardID, "Exists", d, success, metrics_impl.NullBool{Bool: withStorageID, Valid: true}, ioTag) } -func (m *blobstoreMetrics) GetRange(d time.Duration, size int, success, withStorageID bool) { - m.m.MethodDuration(m.shardID, "GetRange", d, success, metrics_impl.NullBool{Bool: withStorageID, Valid: true}) +func (m *blobstoreMetrics) GetRange(d time.Duration, size int, success, withStorageID bool, ioTag string) { + m.m.MethodDuration(m.shardID, "GetRange", d, success, metrics_impl.NullBool{Bool: withStorageID, Valid: true}, ioTag) if success { m.m.AddGet(m.shardID, size) } } -func (m *blobstoreMetrics) Get(d time.Duration, size int, success, withStorageID bool) { - m.m.MethodDuration(m.shardID, "Get", d, success, metrics_impl.NullBool{Bool: withStorageID, Valid: true}) +func (m *blobstoreMetrics) Get(d time.Duration, size int, success, withStorageID bool, ioTag string) { + m.m.MethodDuration(m.shardID, "Get", d, success, metrics_impl.NullBool{Bool: withStorageID, Valid: true}, ioTag) if success { m.m.AddGet(m.shardID, size) } } -func (m *blobstoreMetrics) Iterate(d time.Duration, success bool) { - m.m.MethodDuration(m.shardID, "Iterate", d, success, metrics_impl.NullBool{}) +func (m *blobstoreMetrics) Iterate(d time.Duration, success bool, ioTag string) { + m.m.MethodDuration(m.shardID, "Iterate", d, success, metrics_impl.NullBool{}, ioTag) } -func (m *blobstoreMetrics) Put(d time.Duration, size int, success bool) { - m.m.MethodDuration(m.shardID, "Put", d, success, metrics_impl.NullBool{}) +func (m *blobstoreMetrics) Put(d time.Duration, size int, success bool, ioTag string) { + m.m.MethodDuration(m.shardID, "Put", d, success, metrics_impl.NullBool{}, ioTag) if success { m.m.AddPut(m.shardID, size) } } -func (m *blobstoreMetrics) ObjectsCount(d time.Duration, success bool) { - m.m.MethodDuration(m.shardID, "ObjectsCount", d, success, metrics_impl.NullBool{}) +func (m *blobstoreMetrics) ObjectsCount(d time.Duration, success bool, ioTag string) { + m.m.MethodDuration(m.shardID, "ObjectsCount", d, success, metrics_impl.NullBool{}, ioTag) }