From 16a142cd0c403c51556e4e49a48fa1be02db9102 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 7 Jun 2023 17:06:02 +0300 Subject: [PATCH] [#373] metrics: Add FSTree metrics Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 21 +++-- pkg/local_object_storage/metrics/consts.go | 3 + pkg/local_object_storage/metrics/fstree.go | 61 ++++++++++++++ pkg/local_object_storage/shard/id.go | 4 +- pkg/metrics/consts.go | 15 ++++ pkg/metrics/fstree.go | 96 ++++++++++++++++++++++ pkg/metrics/mode.go | 36 ++++++++ pkg/metrics/node.go | 6 ++ 8 files changed, 236 insertions(+), 6 deletions(-) create mode 100644 pkg/local_object_storage/metrics/consts.go create mode 100644 pkg/local_object_storage/metrics/fstree.go create mode 100644 pkg/metrics/consts.go create mode 100644 pkg/metrics/fstree.go diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index c88b7224..44da2e98 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -36,6 +36,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + lsmetrics "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" shardmode "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" @@ -752,12 +753,22 @@ func (c *cfg) getSubstorageOpts(shCfg shardCfg) []blobstor.SubStorage { }, }) case fstree.Type: + fstreeOpts := []fstree.Option{ + fstree.WithPath(sRead.path), + fstree.WithPerm(sRead.perm), + fstree.WithDepth(sRead.depth), + fstree.WithNoSync(sRead.noSync), + } + if c.metricsCollector != nil { + fstreeOpts = append(fstreeOpts, + fstree.WithMetrics( + lsmetrics.NewFSTreeMetricsWithoutShardID(sRead.path, c.metricsCollector.FSTree()), + ), + ) + } + ss = append(ss, blobstor.SubStorage{ - Storage: fstree.New( - fstree.WithPath(sRead.path), - fstree.WithPerm(sRead.perm), - fstree.WithDepth(sRead.depth), - fstree.WithNoSync(sRead.noSync)), + Storage: fstree.New(fstreeOpts...), Policy: func(_ *objectSDK.Object, data []byte) bool { return true }, diff --git a/pkg/local_object_storage/metrics/consts.go b/pkg/local_object_storage/metrics/consts.go new file mode 100644 index 00000000..51993071 --- /dev/null +++ b/pkg/local_object_storage/metrics/consts.go @@ -0,0 +1,3 @@ +package metrics + +const undefined = "undefined" diff --git a/pkg/local_object_storage/metrics/fstree.go b/pkg/local_object_storage/metrics/fstree.go new file mode 100644 index 00000000..d3749d9b --- /dev/null +++ b/pkg/local_object_storage/metrics/fstree.go @@ -0,0 +1,61 @@ +package metrics + +import ( + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" + metrics_impl "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" +) + +func NewFSTreeMetricsWithoutShardID(path string, m metrics_impl.FSTreeMetrics) fstree.Metrics { + return &fstreeMetrics{ + shardID: undefined, + path: path, + m: m, + } +} + +type fstreeMetrics struct { + shardID string + path string + m metrics_impl.FSTreeMetrics +} + +func (m *fstreeMetrics) SetParentID(parentID string) { + m.shardID = parentID +} + +func (m *fstreeMetrics) SetMode(readOnly bool) { + m.m.SetMode(m.shardID, m.path, readOnly) +} +func (m *fstreeMetrics) Close() { + m.m.Close(m.shardID, m.path) +} + +func (m *fstreeMetrics) Iterate(d time.Duration, success bool) { + m.m.MethodDuration(m.shardID, m.path, "Iterate", d, success) +} +func (m *fstreeMetrics) Delete(d time.Duration, success bool) { + m.m.MethodDuration(m.shardID, m.path, "Delete", d, success) +} +func (m *fstreeMetrics) Exists(d time.Duration, success bool) { + m.m.MethodDuration(m.shardID, m.path, "Exists", d, success) +} +func (m *fstreeMetrics) Put(d time.Duration, size int, success bool) { + m.m.MethodDuration(m.shardID, m.path, "Put", d, success) + if success { + m.m.AddPut(m.shardID, m.path, size) + } +} +func (m *fstreeMetrics) Get(d time.Duration, size int, success bool) { + m.m.MethodDuration(m.shardID, m.path, "Get", d, success) + if success { + m.m.AddGet(m.shardID, m.path, size) + } +} +func (m *fstreeMetrics) GetRange(d time.Duration, size int, success bool) { + m.m.MethodDuration(m.shardID, m.path, "GetRange", d, success) + if success { + m.m.AddGet(m.shardID, m.path, size) + } +} diff --git a/pkg/local_object_storage/shard/id.go b/pkg/local_object_storage/shard/id.go index 4f6fd065..e2ac423f 100644 --- a/pkg/local_object_storage/shard/id.go +++ b/pkg/local_object_storage/shard/id.go @@ -57,7 +57,9 @@ func (s *Shard) UpdateID() (err error) { } s.metaBase.SetParentID(s.info.ID.String()) s.blobStor.SetParentID(s.info.ID.String()) - s.pilorama.SetParentID(s.info.ID.String()) + if s.pilorama != nil { + s.pilorama.SetParentID(s.info.ID.String()) + } if len(id) != 0 { return nil diff --git a/pkg/metrics/consts.go b/pkg/metrics/consts.go new file mode 100644 index 00000000..d65ee51c --- /dev/null +++ b/pkg/metrics/consts.go @@ -0,0 +1,15 @@ +package metrics + +const ( + fstreeSubSystem = "fstree" + + successLabel = "success" + shardIDLabel = "shardID" + modeLabel = "mode" + pathLabel = "path" + methodLabel = "method" + + readWriteMode = "READ_WRITE" + readOnlyMode = "READ_ONLY" + closedMode = "CLOSED" +) diff --git a/pkg/metrics/fstree.go b/pkg/metrics/fstree.go new file mode 100644 index 00000000..5bc43757 --- /dev/null +++ b/pkg/metrics/fstree.go @@ -0,0 +1,96 @@ +package metrics + +import ( + "strconv" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +type FSTreeMetrics interface { + SetMode(shardID, path string, readOnly bool) + Close(shardID, path string) + + MethodDuration(shardID, path string, method string, d time.Duration, success bool) + AddGet(shardID, path string, size int) + AddPut(shardID, path string, size int) +} + +type fstreeMetrics struct { + mode *shardIDPathModeValue + reqDuration *prometheus.HistogramVec + put *prometheus.CounterVec + get *prometheus.CounterVec +} + +func newFSTreeMetrics() *fstreeMetrics { + return &fstreeMetrics{ + mode: newShardIDPathMode(fstreeSubSystem, "mode", "FSTree mode value"), + reqDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: fstreeSubSystem, + Name: "request_duration_seconds", + Help: "Accumulated FSTree request process duration", + }, []string{shardIDLabel, successLabel, pathLabel, methodLabel}), + put: metrics.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: fstreeSubSystem, + Name: "put_bytes", + Help: "Accumulated payload size written to FSTree", + }, []string{shardIDLabel, pathLabel}), + get: metrics.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: fstreeSubSystem, + Name: "get_bytes", + Help: "Accumulated payload size read from FSTree", + }, []string{shardIDLabel, pathLabel}), + } +} + +func (m *fstreeMetrics) SetMode(shardID, path string, readOnly bool) { + modeValue := readWriteMode + if readOnly { + modeValue = readOnlyMode + } + m.mode.SetMode(shardID, path, modeValue) +} + +func (m *fstreeMetrics) Close(shardID, path string) { + m.mode.SetMode(shardID, path, closedMode) + m.reqDuration.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }) + m.get.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }) + m.put.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }) +} + +func (m *fstreeMetrics) MethodDuration(shardID, path string, method string, d time.Duration, success bool) { + m.reqDuration.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + successLabel: strconv.FormatBool(success), + methodLabel: method, + }).Observe(d.Seconds()) +} + +func (m *fstreeMetrics) AddGet(shardID, path string, size int) { + m.get.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }).Add(float64(size)) +} + +func (m *fstreeMetrics) AddPut(shardID, path string, size int) { + m.put.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }).Add(float64(size)) +} diff --git a/pkg/metrics/mode.go b/pkg/metrics/mode.go index 7ae3fd5f..aa224667 100644 --- a/pkg/metrics/mode.go +++ b/pkg/metrics/mode.go @@ -37,3 +37,39 @@ func (m *shardIDModeValue) Delete(shardID string) { wcShardID: shardID, }) } + +type shardIDPathModeValue struct { + modeValue *prometheus.GaugeVec +} + +func newShardIDPathMode(subsystem, name, help string) *shardIDPathModeValue { + return &shardIDPathModeValue{ + modeValue: metrics.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: name, + Help: help, + }, []string{shardIDLabel, pathLabel, modeLabel}), + } +} + +func (m *shardIDPathModeValue) SetMode(shardID, path string, mode string) { + m.modeValue.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }) + + m.modeValue.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + modeLabel: mode, + }).Set(1) +} + +func (m *shardIDPathModeValue) Delete(shardID, path string) { + m.modeValue.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }) +} diff --git a/pkg/metrics/node.go b/pkg/metrics/node.go index 232c178d..0b04b635 100644 --- a/pkg/metrics/node.go +++ b/pkg/metrics/node.go @@ -14,6 +14,7 @@ type NodeMetrics struct { objectService *objectServiceMetrics treeService *treeServiceMetrics epoch prometheus.Gauge + fstree *fstreeMetrics } func NewNodeMetrics() *NodeMetrics { @@ -41,6 +42,7 @@ func NewNodeMetrics() *NodeMetrics { replicator: replicator, treeService: treeService, epoch: epoch, + fstree: newFSTreeMetrics(), } } @@ -68,3 +70,7 @@ func (m *NodeMetrics) Engine() EngineMetrics { func (m *NodeMetrics) State() StateMetrics { return m.state } + +func (m *NodeMetrics) FSTree() FSTreeMetrics { + return m.fstree +}