From 56f320dd85a5c065812becc9eeef15acecf0f371 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 7 Jun 2023 18:04:23 +0300 Subject: [PATCH] [#373] metrics: Add blobstor metrics Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 18 ++-- pkg/local_object_storage/metrics/blobstore.go | 65 ++++++++++++++ pkg/metrics/blobstore.go | 87 +++++++++++++++++++ pkg/metrics/consts.go | 14 +-- pkg/metrics/fstree.go | 6 +- pkg/metrics/mode.go | 8 ++ pkg/metrics/node.go | 42 ++++----- 7 files changed, 198 insertions(+), 42 deletions(-) create mode 100644 pkg/local_object_storage/metrics/blobstore.go create mode 100644 pkg/metrics/blobstore.go diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 44da2e981..bc73f8bc5 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -786,19 +786,23 @@ func (c *cfg) getShardOpts(shCfg shardCfg) shardOptsWithID { piloramaOpts := c.getPiloramaOpts(shCfg) ss := c.getSubstorageOpts(shCfg) + blobstoreOpts := []blobstor.Option{ + blobstor.WithCompressObjects(shCfg.compress), + blobstor.WithUncompressableContentTypes(shCfg.uncompressableContentType), + blobstor.WithStorages(ss), + blobstor.WithLogger(c.log), + } + if c.metricsCollector != nil { + blobstoreOpts = append(blobstoreOpts, blobstor.WithMetrics(lsmetrics.NewBlobstoreMetrics(c.metricsCollector.Blobstore()))) + } + var sh shardOptsWithID sh.configID = shCfg.id() sh.shOpts = []shard.Option{ shard.WithLogger(c.log), shard.WithRefillMetabase(shCfg.refillMetabase), shard.WithMode(shCfg.mode), - shard.WithBlobStorOptions( - blobstor.WithCompressObjects(shCfg.compress), - blobstor.WithUncompressableContentTypes(shCfg.uncompressableContentType), - blobstor.WithStorages(ss), - - blobstor.WithLogger(c.log), - ), + shard.WithBlobStorOptions(blobstoreOpts...), shard.WithMetaBaseOptions( meta.WithPath(shCfg.metaCfg.path), meta.WithPermissions(shCfg.metaCfg.perm), diff --git a/pkg/local_object_storage/metrics/blobstore.go b/pkg/local_object_storage/metrics/blobstore.go new file mode 100644 index 000000000..48249e89c --- /dev/null +++ b/pkg/local_object_storage/metrics/blobstore.go @@ -0,0 +1,65 @@ +package metrics + +import ( + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" + metrics_impl "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" +) + +type blobstoreMetrics struct { + shardID string + m metrics_impl.BlobstoreMetrics +} + +func NewBlobstoreMetrics(m metrics_impl.BlobstoreMetrics) blobstor.Metrics { + return &blobstoreMetrics{ + shardID: undefined, + m: m, + } +} + +func (m *blobstoreMetrics) SetParentID(parentID string) { + m.shardID = parentID +} + +func (m *blobstoreMetrics) SetMode(readOnly bool) { + m.m.SetMode(m.shardID, readOnly) +} + +func (m *blobstoreMetrics) Close() { + m.m.Close(m.shardID) +} + +func (m *blobstoreMetrics) Delete(d time.Duration, success, withStorageID bool) { + m.m.MethodDuration(m.shardID, "Delete", d, success, metrics_impl.NullBool{Bool: withStorageID, Valid: true}) +} + +func (m *blobstoreMetrics) Exists(d time.Duration, success, withStorageID bool) { + m.m.MethodDuration(m.shardID, "Exists", d, success, metrics_impl.NullBool{Bool: withStorageID, Valid: true}) +} + +func (m *blobstoreMetrics) GetRange(d time.Duration, size int, success, withStorageID bool) { + m.m.MethodDuration(m.shardID, "GetRange", d, success, metrics_impl.NullBool{Bool: withStorageID, Valid: true}) + if success { + m.m.AddGet(m.shardID, size) + } +} + +func (m *blobstoreMetrics) Get(d time.Duration, size int, success, withStorageID bool) { + m.m.MethodDuration(m.shardID, "Get", d, success, metrics_impl.NullBool{Bool: withStorageID, Valid: true}) + if success { + m.m.AddGet(m.shardID, size) + } +} + +func (m *blobstoreMetrics) Iterate(d time.Duration, success bool) { + m.m.MethodDuration(m.shardID, "Iterate", d, success, metrics_impl.NullBool{}) +} + +func (m *blobstoreMetrics) Put(d time.Duration, size int, success bool) { + m.m.MethodDuration(m.shardID, "Put", d, success, metrics_impl.NullBool{}) + if success { + m.m.AddPut(m.shardID, size) + } +} diff --git a/pkg/metrics/blobstore.go b/pkg/metrics/blobstore.go new file mode 100644 index 000000000..d9bb3f029 --- /dev/null +++ b/pkg/metrics/blobstore.go @@ -0,0 +1,87 @@ +package metrics + +import ( + "strconv" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +type BlobstoreMetrics interface { + SetMode(shardID string, readOnly bool) + Close(shardID string) + + MethodDuration(shardID string, method string, d time.Duration, success bool, withStorageID NullBool) + AddPut(shardID string, size int) + AddGet(shardID string, size int) +} + +type blobstoreMetrics struct { + mode *shardIDModeValue + reqDuration *prometheus.HistogramVec + put *prometheus.CounterVec + get *prometheus.CounterVec +} + +func newBlobstoreMetrics() *blobstoreMetrics { + return &blobstoreMetrics{ + mode: newShardIDMode(blobstoreSubSystem, "mode", "Blobstore mode value"), + reqDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: blobstoreSubSystem, + Name: "request_duration_seconds", + Help: "Accumulated Blobstore request process duration", + }, []string{shardIDLabel, successLabel, methodLabel, withStorageIDLabel}), + put: metrics.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: blobstoreSubSystem, + Name: "put_bytes", + Help: "Accumulated payload size written to Blobstore", + }, []string{shardIDLabel}), + get: metrics.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: blobstoreSubSystem, + Name: "get_bytes", + Help: "Accumulated payload size read from Blobstore", + }, []string{shardIDLabel}), + } +} + +func (m *blobstoreMetrics) SetMode(shardID string, readOnly bool) { + m.mode.SetMode(shardID, modeFromBool(readOnly)) +} + +func (m *blobstoreMetrics) Close(shardID string) { + m.mode.SetMode(shardID, closedMode) + m.reqDuration.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + }) + m.get.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + }) + m.put.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + }) +} + +func (m *blobstoreMetrics) MethodDuration(shardID string, method string, d time.Duration, success bool, withStorageID NullBool) { + m.reqDuration.With(prometheus.Labels{ + shardIDLabel: shardID, + successLabel: strconv.FormatBool(success), + methodLabel: method, + withStorageIDLabel: withStorageID.String(), + }).Observe(d.Seconds()) +} + +func (m *blobstoreMetrics) AddPut(shardID string, size int) { + m.put.With(prometheus.Labels{ + shardIDLabel: shardID, + }).Add(float64(size)) +} + +func (m *blobstoreMetrics) AddGet(shardID string, size int) { + m.get.With(prometheus.Labels{ + shardIDLabel: shardID, + }).Add(float64(size)) +} diff --git a/pkg/metrics/consts.go b/pkg/metrics/consts.go index d65ee51c1..d06d15902 100644 --- a/pkg/metrics/consts.go +++ b/pkg/metrics/consts.go @@ -1,13 +1,15 @@ package metrics const ( - fstreeSubSystem = "fstree" + fstreeSubSystem = "fstree" + blobstoreSubSystem = "blobstore" - successLabel = "success" - shardIDLabel = "shardID" - modeLabel = "mode" - pathLabel = "path" - methodLabel = "method" + successLabel = "success" + shardIDLabel = "shardID" + modeLabel = "mode" + pathLabel = "path" + methodLabel = "method" + withStorageIDLabel = "withStorageID" readWriteMode = "READ_WRITE" readOnlyMode = "READ_ONLY" diff --git a/pkg/metrics/fstree.go b/pkg/metrics/fstree.go index 5bc43757a..4d4f0693b 100644 --- a/pkg/metrics/fstree.go +++ b/pkg/metrics/fstree.go @@ -49,11 +49,7 @@ func newFSTreeMetrics() *fstreeMetrics { } func (m *fstreeMetrics) SetMode(shardID, path string, readOnly bool) { - modeValue := readWriteMode - if readOnly { - modeValue = readOnlyMode - } - m.mode.SetMode(shardID, path, modeValue) + m.mode.SetMode(shardID, path, modeFromBool(readOnly)) } func (m *fstreeMetrics) Close(shardID, path string) { diff --git a/pkg/metrics/mode.go b/pkg/metrics/mode.go index aa2246676..ad959e18e 100644 --- a/pkg/metrics/mode.go +++ b/pkg/metrics/mode.go @@ -73,3 +73,11 @@ func (m *shardIDPathModeValue) Delete(shardID, path string) { pathLabel: path, }) } + +func modeFromBool(readOnly bool) string { + modeValue := readWriteMode + if readOnly { + modeValue = readOnlyMode + } + return modeValue +} diff --git a/pkg/metrics/node.go b/pkg/metrics/node.go index 0b04b635b..bf26aab61 100644 --- a/pkg/metrics/node.go +++ b/pkg/metrics/node.go @@ -15,34 +15,24 @@ type NodeMetrics struct { treeService *treeServiceMetrics epoch prometheus.Gauge fstree *fstreeMetrics + blobstore *blobstoreMetrics } func NewNodeMetrics() *NodeMetrics { - objectService := newObjectServiceMetrics() - - engine := newEngineMetrics() - - state := newStateMetrics() - - replicator := newReplicatorMetrics() - - treeService := newTreeServiceMetrics() - - epoch := metrics.NewGauge(prometheus.GaugeOpts{ - Namespace: namespace, - Subsystem: innerRingSubsystem, - Name: "epoch", - Help: "Current epoch as seen by inner-ring node.", - }) - return &NodeMetrics{ - objectService: objectService, - engine: engine, - state: state, - replicator: replicator, - treeService: treeService, - epoch: epoch, - fstree: newFSTreeMetrics(), + objectService: newObjectServiceMetrics(), + engine: newEngineMetrics(), + state: newStateMetrics(), + replicator: newReplicatorMetrics(), + treeService: newTreeServiceMetrics(), + epoch: metrics.NewGauge(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: innerRingSubsystem, + Name: "epoch", + Help: "Current epoch as seen by inner-ring node.", + }), + fstree: newFSTreeMetrics(), + blobstore: newBlobstoreMetrics(), } } @@ -74,3 +64,7 @@ func (m *NodeMetrics) State() StateMetrics { func (m *NodeMetrics) FSTree() FSTreeMetrics { return m.fstree } + +func (m *NodeMetrics) Blobstore() BlobstoreMetrics { + return m.blobstore +}