From af608da952dbf9c531c7a31b008a81bfee55ef16 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 9 Jun 2023 12:14:32 +0300 Subject: [PATCH] [#373] metrics: Add blobovnizca metrics Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 27 ++-- .../blobovnicza/control.go | 8 +- .../blobovnicza/metrics.go | 16 +- pkg/local_object_storage/blobovnicza/sizes.go | 4 +- .../blobstor/blobovniczatree/control.go | 2 +- .../blobstor/blobovniczatree/metrics.go | 8 +- .../metrics/blobovnizca.go | 98 ++++++++++++ pkg/metrics/blobovnizca.go | 141 ++++++++++++++++++ pkg/metrics/consts.go | 5 +- pkg/metrics/node.go | 10 +- 10 files changed, 287 insertions(+), 32 deletions(-) create mode 100644 pkg/local_object_storage/metrics/blobovnizca.go create mode 100644 pkg/metrics/blobovnizca.go diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index bc73f8bc5..0f2db8725 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -738,16 +738,25 @@ func (c *cfg) getSubstorageOpts(shCfg shardCfg) []blobstor.SubStorage { for _, sRead := range shCfg.subStorages { switch sRead.typ { case blobovniczatree.Type: - ss = append(ss, blobstor.SubStorage{ - Storage: blobovniczatree.NewBlobovniczaTree( - blobovniczatree.WithRootPath(sRead.path), - blobovniczatree.WithPermissions(sRead.perm), - blobovniczatree.WithBlobovniczaSize(sRead.size), - blobovniczatree.WithBlobovniczaShallowDepth(sRead.depth), - blobovniczatree.WithBlobovniczaShallowWidth(sRead.width), - blobovniczatree.WithOpenedCacheSize(sRead.openedCacheSize), + blobTreeOpts := []blobovniczatree.Option{ + blobovniczatree.WithRootPath(sRead.path), + blobovniczatree.WithPermissions(sRead.perm), + blobovniczatree.WithBlobovniczaSize(sRead.size), + blobovniczatree.WithBlobovniczaShallowDepth(sRead.depth), + blobovniczatree.WithBlobovniczaShallowWidth(sRead.width), + blobovniczatree.WithOpenedCacheSize(sRead.openedCacheSize), + blobovniczatree.WithLogger(c.log), + } - blobovniczatree.WithLogger(c.log)), + if c.metricsCollector != nil { + blobTreeOpts = append(blobTreeOpts, + blobovniczatree.WithMetrics( + lsmetrics.NewBlobovniczaTreeMetrics(sRead.path, c.metricsCollector.BlobobvnizcaTreeMetrics()), + ), + ) + } + ss = append(ss, blobstor.SubStorage{ + Storage: blobovniczatree.NewBlobovniczaTree(blobTreeOpts...), Policy: func(_ *objectSDK.Object, data []byte) bool { return uint64(len(data)) < shCfg.smallSizeObjectLimit }, diff --git a/pkg/local_object_storage/blobovnicza/control.go b/pkg/local_object_storage/blobovnicza/control.go index c776afe06..da4c870bd 100644 --- a/pkg/local_object_storage/blobovnicza/control.go +++ b/pkg/local_object_storage/blobovnicza/control.go @@ -36,7 +36,7 @@ func (b *Blobovnicza) Open() error { b.boltDB, err = bbolt.Open(b.path, b.perm, b.boltOptions) if err == nil { - b.metrics.IncOpenCount() + b.metrics.IncOpenBlobovnizcaCount() } return err @@ -86,7 +86,7 @@ func (b *Blobovnicza) Init() error { sz := uint64(info.Size()) b.filled.Store(sz) - b.metrics.IncSize(sz) + b.metrics.AddSize(sz) return err } @@ -98,8 +98,8 @@ func (b *Blobovnicza) Close() error { err := b.boltDB.Close() if err == nil { - b.metrics.DecOpenCount() - b.metrics.DecSize(b.filled.Load()) + b.metrics.DecOpenBlobovnizcaCount() + b.metrics.SubSize(b.filled.Load()) } return err } diff --git a/pkg/local_object_storage/blobovnicza/metrics.go b/pkg/local_object_storage/blobovnicza/metrics.go index 6127370bc..d511f90f1 100644 --- a/pkg/local_object_storage/blobovnicza/metrics.go +++ b/pkg/local_object_storage/blobovnicza/metrics.go @@ -1,16 +1,16 @@ package blobovnicza type Metrics interface { - IncOpenCount() - DecOpenCount() + IncOpenBlobovnizcaCount() + DecOpenBlobovnizcaCount() - IncSize(size uint64) - DecSize(size uint64) + AddSize(size uint64) + SubSize(size uint64) } type NoopMetrics struct{} -func (m *NoopMetrics) IncOpenCount() {} -func (m *NoopMetrics) DecOpenCount() {} -func (m *NoopMetrics) IncSize(uint64) {} -func (m *NoopMetrics) DecSize(uint64) {} +func (m *NoopMetrics) IncOpenBlobovnizcaCount() {} +func (m *NoopMetrics) DecOpenBlobovnizcaCount() {} +func (m *NoopMetrics) AddSize(uint64) {} +func (m *NoopMetrics) SubSize(uint64) {} diff --git a/pkg/local_object_storage/blobovnicza/sizes.go b/pkg/local_object_storage/blobovnicza/sizes.go index bdbc77d18..7e10b728e 100644 --- a/pkg/local_object_storage/blobovnicza/sizes.go +++ b/pkg/local_object_storage/blobovnicza/sizes.go @@ -41,12 +41,12 @@ func upperPowerOfTwo(v uint64) uint64 { func (b *Blobovnicza) incSize(sz uint64) { b.filled.Add(sz) - b.metrics.IncSize(sz) + b.metrics.AddSize(sz) } func (b *Blobovnicza) decSize(sz uint64) { b.filled.Add(^(sz - 1)) - b.metrics.DecSize(sz) + b.metrics.SubSize(sz) } func (b *Blobovnicza) full() bool { diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/control.go b/pkg/local_object_storage/blobstor/blobovniczatree/control.go index 0045e08db..bc3d7d60c 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/control.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/control.go @@ -130,7 +130,7 @@ func (b *Blobovniczas) openBlobovniczaNoCache(p string) (*blobovnicza.Blobovnicz blz := blobovnicza.New(append(b.blzOpts, blobovnicza.WithReadOnly(b.readOnly), blobovnicza.WithPath(path), - blobovnicza.WithMetrics(b.metrics.BlobovnicaMetrics(path)), + blobovnicza.WithMetrics(b.metrics.Blobovnizca()), )...) if err := blz.Open(); err != nil { diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go b/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go index 39fe34e28..6c9b600c0 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go @@ -7,7 +7,7 @@ import ( ) type Metrics interface { - BlobovnicaMetrics(path string) blobovnicza.Metrics + Blobovnizca() blobovnicza.Metrics SetParentID(parentID string) @@ -24,9 +24,6 @@ type Metrics interface { type noopMetrics struct{} -func (m *noopMetrics) BlobovnicaMetrics(string) blobovnicza.Metrics { - return &blobovnicza.NoopMetrics{} -} func (m *noopMetrics) SetParentID(string) {} func (m *noopMetrics) SetMode(bool) {} func (m *noopMetrics) Close() {} @@ -36,3 +33,6 @@ func (m *noopMetrics) GetRange(time.Duration, int, bool, bool) {} func (m *noopMetrics) Get(time.Duration, int, bool, bool) {} func (m *noopMetrics) Iterate(time.Duration, bool) {} func (m *noopMetrics) Put(time.Duration, int, bool) {} +func (m *noopMetrics) Blobovnizca() blobovnicza.Metrics { + return &blobovnicza.NoopMetrics{} +} diff --git a/pkg/local_object_storage/metrics/blobovnizca.go b/pkg/local_object_storage/metrics/blobovnizca.go new file mode 100644 index 000000000..a498d822e --- /dev/null +++ b/pkg/local_object_storage/metrics/blobovnizca.go @@ -0,0 +1,98 @@ +package metrics + +import ( + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" + metrics_impl "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" +) + +func NewBlobovniczaTreeMetrics(path string, m metrics_impl.BlobobvnizcaMetrics) blobovniczatree.Metrics { + return &blobovniczaTreeMetrics{ + path: path, + shardID: undefined, + m: m, + } +} + +type blobovniczaTreeMetrics struct { + shardID string + path string + m metrics_impl.BlobobvnizcaMetrics +} + +func (m *blobovniczaTreeMetrics) Blobovnizca() blobovnicza.Metrics { + return &blobovniczaMetrics{ + shardID: func() string { return m.shardID }, + path: m.path, + m: m.m, + } +} + +func (m *blobovniczaTreeMetrics) SetParentID(parentID string) { + m.shardID = parentID +} + +func (m *blobovniczaTreeMetrics) SetMode(readOnly bool) { + m.m.SetBlobobvnizcaTreeMode(m.shardID, m.path, readOnly) +} + +func (m *blobovniczaTreeMetrics) Close() { + m.m.CloseBlobobvnizcaTree(m.shardID, m.path) +} + +func (m *blobovniczaTreeMetrics) Delete(d time.Duration, success, withStorageID bool) { + m.m.BlobobvnizcaTreeMethodDuration(m.shardID, m.path, "Delete", d, success, metrics_impl.NullBool{Valid: true, Bool: withStorageID}) +} + +func (m *blobovniczaTreeMetrics) Exists(d time.Duration, success, withStorageID bool) { + m.m.BlobobvnizcaTreeMethodDuration(m.shardID, m.path, "Exists", d, success, metrics_impl.NullBool{Valid: true, Bool: withStorageID}) +} + +func (m *blobovniczaTreeMetrics) GetRange(d time.Duration, size int, success, withStorageID bool) { + m.m.BlobobvnizcaTreeMethodDuration(m.shardID, m.path, "GetRange", d, success, metrics_impl.NullBool{Valid: true, Bool: withStorageID}) + if success { + m.m.AddBlobobvnizcaTreeGet(m.shardID, m.path, size) + } +} + +func (m *blobovniczaTreeMetrics) Get(d time.Duration, size int, success, withStorageID bool) { + m.m.BlobobvnizcaTreeMethodDuration(m.shardID, m.path, "Get", d, success, metrics_impl.NullBool{Valid: true, Bool: withStorageID}) + if success { + m.m.AddBlobobvnizcaTreeGet(m.shardID, m.path, size) + } +} + +func (m *blobovniczaTreeMetrics) Iterate(d time.Duration, success bool) { + m.m.BlobobvnizcaTreeMethodDuration(m.shardID, m.path, "Iterate", d, success, metrics_impl.NullBool{}) +} + +func (m *blobovniczaTreeMetrics) Put(d time.Duration, size int, success bool) { + m.m.BlobobvnizcaTreeMethodDuration(m.shardID, m.path, "Put", d, success, metrics_impl.NullBool{}) + if success { + m.m.AddBlobobvnizcaTreePut(m.shardID, m.path, size) + } +} + +type blobovniczaMetrics struct { + m metrics_impl.BlobobvnizcaMetrics + shardID func() string + path string +} + +func (m *blobovniczaMetrics) AddSize(size uint64) { + m.m.AddTreeSize(m.shardID(), m.path, size) +} + +func (m *blobovniczaMetrics) SubSize(size uint64) { + m.m.SubTreeSize(m.shardID(), m.path, size) +} + +func (m *blobovniczaMetrics) IncOpenBlobovnizcaCount() { + m.m.IncOpenBlobovnizcaCount(m.shardID(), m.path) +} + +func (m *blobovniczaMetrics) DecOpenBlobovnizcaCount() { + m.m.DecOpenBlobovnizcaCount(m.shardID(), m.path) +} diff --git a/pkg/metrics/blobovnizca.go b/pkg/metrics/blobovnizca.go new file mode 100644 index 000000000..9dc3ed572 --- /dev/null +++ b/pkg/metrics/blobovnizca.go @@ -0,0 +1,141 @@ +package metrics + +import ( + "strconv" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +type BlobobvnizcaMetrics interface { + SetBlobobvnizcaTreeMode(shardID, path string, readOnly bool) + CloseBlobobvnizcaTree(shardID, path string) + BlobobvnizcaTreeMethodDuration(shardID, path string, method string, d time.Duration, success bool, withStorageID NullBool) + AddBlobobvnizcaTreePut(shardID, path string, size int) + AddBlobobvnizcaTreeGet(shardID, path string, size int) + + AddTreeSize(shardID, path string, size uint64) + SubTreeSize(shardID, path string, size uint64) + + IncOpenBlobovnizcaCount(shardID, path string) + DecOpenBlobovnizcaCount(shardID, path string) +} + +type blobovnizca struct { + treeMode *shardIDPathModeValue + treeReqDuration *prometheus.HistogramVec + treePut *prometheus.CounterVec + treeGet *prometheus.CounterVec + treeSize *prometheus.GaugeVec + treeOpenCounter *prometheus.GaugeVec +} + +func newBlobovnizca() *blobovnizca { + return &blobovnizca{ + treeMode: newShardIDPathMode(blobovnizaTreeSubSystem, "mode", "Blobovnizca tree mode"), + + treeReqDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: blobovnizaTreeSubSystem, + Name: "request_duration_seconds", + Help: "Accumulated Blobovnizca tree request process duration", + }, []string{shardIDLabel, pathLabel, successLabel, methodLabel, withStorageIDLabel}), + treePut: metrics.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: blobovnizaTreeSubSystem, + Name: "put_bytes", + Help: "Accumulated payload size written to Blobovnizca tree", + }, []string{shardIDLabel, pathLabel}), + treeGet: metrics.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: blobovnizaTreeSubSystem, + Name: "get_bytes", + Help: "Accumulated payload size read from Blobovnizca tree", + }, []string{shardIDLabel, pathLabel}), + treeSize: metrics.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: blobovnizaTreeSubSystem, + Name: "size_bytes", + Help: "Blobovnizca tree size", + }, []string{shardIDLabel, pathLabel}), + treeOpenCounter: metrics.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: blobovnizaTreeSubSystem, + Name: "open_blobovnizca_count", + Help: "Count of opened blobovnizcas of Blobovnizca tree", + }, []string{shardIDLabel, pathLabel}), + } +} + +func (b *blobovnizca) SetBlobobvnizcaTreeMode(shardID, path string, readOnly bool) { + b.treeMode.SetMode(shardID, path, modeFromBool(readOnly)) +} + +func (b *blobovnizca) CloseBlobobvnizcaTree(shardID, path string) { + b.treeMode.SetMode(shardID, path, closedMode) + b.treeReqDuration.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }) + b.treeGet.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }) + b.treePut.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }) +} + +func (b *blobovnizca) BlobobvnizcaTreeMethodDuration(shardID, path string, method string, d time.Duration, success bool, withStorageID NullBool) { + b.treeReqDuration.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + successLabel: strconv.FormatBool(success), + methodLabel: method, + withStorageIDLabel: withStorageID.String(), + }).Observe(d.Seconds()) +} + +func (b *blobovnizca) AddBlobobvnizcaTreePut(shardID, path string, size int) { + b.treePut.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }).Add(float64(size)) +} + +func (b *blobovnizca) AddBlobobvnizcaTreeGet(shardID, path string, size int) { + b.treeGet.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }).Add(float64(size)) +} + +func (b *blobovnizca) AddTreeSize(shardID, path string, size uint64) { + b.treeSize.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }).Add(float64(size)) +} + +func (b *blobovnizca) SubTreeSize(shardID, path string, size uint64) { + b.treeSize.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }).Sub(float64(size)) +} + +func (b *blobovnizca) IncOpenBlobovnizcaCount(shardID, path string) { + b.treeOpenCounter.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }).Inc() +} + +func (b *blobovnizca) DecOpenBlobovnizcaCount(shardID, path string) { + b.treeOpenCounter.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }).Dec() +} diff --git a/pkg/metrics/consts.go b/pkg/metrics/consts.go index d06d15902..d79e11696 100644 --- a/pkg/metrics/consts.go +++ b/pkg/metrics/consts.go @@ -1,8 +1,9 @@ package metrics const ( - fstreeSubSystem = "fstree" - blobstoreSubSystem = "blobstore" + fstreeSubSystem = "fstree" + blobstoreSubSystem = "blobstore" + blobovnizaTreeSubSystem = "blobovniza_tree" successLabel = "success" shardIDLabel = "shardID" diff --git a/pkg/metrics/node.go b/pkg/metrics/node.go index bf26aab61..babd6cdcf 100644 --- a/pkg/metrics/node.go +++ b/pkg/metrics/node.go @@ -16,6 +16,7 @@ type NodeMetrics struct { epoch prometheus.Gauge fstree *fstreeMetrics blobstore *blobstoreMetrics + blobobvnizca *blobovnizca } func NewNodeMetrics() *NodeMetrics { @@ -31,8 +32,9 @@ func NewNodeMetrics() *NodeMetrics { Name: "epoch", Help: "Current epoch as seen by inner-ring node.", }), - fstree: newFSTreeMetrics(), - blobstore: newBlobstoreMetrics(), + fstree: newFSTreeMetrics(), + blobstore: newBlobstoreMetrics(), + blobobvnizca: newBlobovnizca(), } } @@ -68,3 +70,7 @@ func (m *NodeMetrics) FSTree() FSTreeMetrics { func (m *NodeMetrics) Blobstore() BlobstoreMetrics { return m.blobstore } + +func (m *NodeMetrics) BlobobvnizcaTreeMetrics() BlobobvnizcaMetrics { + return m.blobobvnizca +}