[#373] metrics: Add blobstor metrics

Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>

cmd/frostfs-node/config.go

@@ -786,19 +786,23 @@ func (c *cfg) getShardOpts(shCfg shardCfg) shardOptsWithID {
 	piloramaOpts := c.getPiloramaOpts(shCfg)
 	ss := c.getSubstorageOpts(shCfg)
 
+	blobstoreOpts := []blobstor.Option{
+		blobstor.WithCompressObjects(shCfg.compress),
+		blobstor.WithUncompressableContentTypes(shCfg.uncompressableContentType),
+		blobstor.WithStorages(ss),
+		blobstor.WithLogger(c.log),
+	}
+	if c.metricsCollector != nil {
+		blobstoreOpts = append(blobstoreOpts, blobstor.WithMetrics(lsmetrics.NewBlobstoreMetrics(c.metricsCollector.Blobstore())))
+	}
+
 	var sh shardOptsWithID
 	sh.configID = shCfg.id()
 	sh.shOpts = []shard.Option{
 		shard.WithLogger(c.log),
 		shard.WithRefillMetabase(shCfg.refillMetabase),
 		shard.WithMode(shCfg.mode),
-		shard.WithBlobStorOptions(
+		shard.WithBlobStorOptions(blobstoreOpts...),
-			blobstor.WithCompressObjects(shCfg.compress),
-			blobstor.WithUncompressableContentTypes(shCfg.uncompressableContentType),
-			blobstor.WithStorages(ss),
-
-			blobstor.WithLogger(c.log),
-		),
 		shard.WithMetaBaseOptions(
 			meta.WithPath(shCfg.metaCfg.path),
 			meta.WithPermissions(shCfg.metaCfg.perm),

pkg/local_object_storage/metrics/blobstore.go

@@ -0,0 +1,65 @@
+package metrics
+
+import (
+	"time"
+
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor"
+	metrics_impl "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics"
+)
+
+type blobstoreMetrics struct {
+	shardID string
+	m       metrics_impl.BlobstoreMetrics
+}
+
+func NewBlobstoreMetrics(m metrics_impl.BlobstoreMetrics) blobstor.Metrics {
+	return &blobstoreMetrics{
+		shardID: undefined,
+		m:       m,
+	}
+}
+
+func (m *blobstoreMetrics) SetParentID(parentID string) {
+	m.shardID = parentID
+}
+
+func (m *blobstoreMetrics) SetMode(readOnly bool) {
+	m.m.SetMode(m.shardID, readOnly)
+}
+
+func (m *blobstoreMetrics) Close() {
+	m.m.Close(m.shardID)
+}
+
+func (m *blobstoreMetrics) Delete(d time.Duration, success, withStorageID bool) {
+	m.m.MethodDuration(m.shardID, "Delete", d, success, metrics_impl.NullBool{Bool: withStorageID, Valid: true})
+}
+
+func (m *blobstoreMetrics) Exists(d time.Duration, success, withStorageID bool) {
+	m.m.MethodDuration(m.shardID, "Exists", d, success, metrics_impl.NullBool{Bool: withStorageID, Valid: true})
+}
+
+func (m *blobstoreMetrics) GetRange(d time.Duration, size int, success, withStorageID bool) {
+	m.m.MethodDuration(m.shardID, "GetRange", d, success, metrics_impl.NullBool{Bool: withStorageID, Valid: true})
+	if success {
+		m.m.AddGet(m.shardID, size)
+	}
+}
+
+func (m *blobstoreMetrics) Get(d time.Duration, size int, success, withStorageID bool) {
+	m.m.MethodDuration(m.shardID, "Get", d, success, metrics_impl.NullBool{Bool: withStorageID, Valid: true})
+	if success {
+		m.m.AddGet(m.shardID, size)
+	}
+}
+
+func (m *blobstoreMetrics) Iterate(d time.Duration, success bool) {
+	m.m.MethodDuration(m.shardID, "Iterate", d, success, metrics_impl.NullBool{})
+}
+
+func (m *blobstoreMetrics) Put(d time.Duration, size int, success bool) {
+	m.m.MethodDuration(m.shardID, "Put", d, success, metrics_impl.NullBool{})
+	if success {
+		m.m.AddPut(m.shardID, size)
+	}
+}

pkg/metrics/blobstore.go

@@ -0,0 +1,87 @@
+package metrics
+
+import (
+	"strconv"
+	"time"
+
+	"git.frostfs.info/TrueCloudLab/frostfs-observability/metrics"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+type BlobstoreMetrics interface {
+	SetMode(shardID string, readOnly bool)
+	Close(shardID string)
+
+	MethodDuration(shardID string, method string, d time.Duration, success bool, withStorageID NullBool)
+	AddPut(shardID string, size int)
+	AddGet(shardID string, size int)
+}
+
+type blobstoreMetrics struct {
+	mode        *shardIDModeValue
+	reqDuration *prometheus.HistogramVec
+	put         *prometheus.CounterVec
+	get         *prometheus.CounterVec
+}
+
+func newBlobstoreMetrics() *blobstoreMetrics {
+	return &blobstoreMetrics{
+		mode: newShardIDMode(blobstoreSubSystem, "mode", "Blobstore mode value"),
+		reqDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{
+			Namespace: namespace,
+			Subsystem: blobstoreSubSystem,
+			Name:      "request_duration_seconds",
+			Help:      "Accumulated Blobstore request process duration",
+		}, []string{shardIDLabel, successLabel, methodLabel, withStorageIDLabel}),
+		put: metrics.NewCounterVec(prometheus.CounterOpts{
+			Namespace: namespace,
+			Subsystem: blobstoreSubSystem,
+			Name:      "put_bytes",
+			Help:      "Accumulated payload size written to Blobstore",
+		}, []string{shardIDLabel}),
+		get: metrics.NewCounterVec(prometheus.CounterOpts{
+			Namespace: namespace,
+			Subsystem: blobstoreSubSystem,
+			Name:      "get_bytes",
+			Help:      "Accumulated payload size read from Blobstore",
+		}, []string{shardIDLabel}),
+	}
+}
+
+func (m *blobstoreMetrics) SetMode(shardID string, readOnly bool) {
+	m.mode.SetMode(shardID, modeFromBool(readOnly))
+}
+
+func (m *blobstoreMetrics) Close(shardID string) {
+	m.mode.SetMode(shardID, closedMode)
+	m.reqDuration.DeletePartialMatch(prometheus.Labels{
+		shardIDLabel: shardID,
+	})
+	m.get.DeletePartialMatch(prometheus.Labels{
+		shardIDLabel: shardID,
+	})
+	m.put.DeletePartialMatch(prometheus.Labels{
+		shardIDLabel: shardID,
+	})
+}
+
+func (m *blobstoreMetrics) MethodDuration(shardID string, method string, d time.Duration, success bool, withStorageID NullBool) {
+	m.reqDuration.With(prometheus.Labels{
+		shardIDLabel:       shardID,
+		successLabel:       strconv.FormatBool(success),
+		methodLabel:        method,
+		withStorageIDLabel: withStorageID.String(),
+	}).Observe(d.Seconds())
+}
+
+func (m *blobstoreMetrics) AddPut(shardID string, size int) {
+	m.put.With(prometheus.Labels{
+		shardIDLabel: shardID,
+	}).Add(float64(size))
+}
+
+func (m *blobstoreMetrics) AddGet(shardID string, size int) {
+	m.get.With(prometheus.Labels{
+		shardIDLabel: shardID,
+	}).Add(float64(size))
+}

pkg/metrics/consts.go

@@ -1,13 +1,15 @@
 package metrics
 
 const (
-	fstreeSubSystem = "fstree"
+	fstreeSubSystem    = "fstree"
+	blobstoreSubSystem = "blobstore"
 
-	successLabel = "success"
+	successLabel       = "success"
-	shardIDLabel = "shardID"
+	shardIDLabel       = "shardID"
-	modeLabel    = "mode"
+	modeLabel          = "mode"
-	pathLabel    = "path"
+	pathLabel          = "path"
-	methodLabel  = "method"
+	methodLabel        = "method"
+	withStorageIDLabel = "withStorageID"
 
 	readWriteMode = "READ_WRITE"
 	readOnlyMode  = "READ_ONLY"

pkg/metrics/fstree.go

@@ -49,11 +49,7 @@ func newFSTreeMetrics() *fstreeMetrics {
 }
 
 func (m *fstreeMetrics) SetMode(shardID, path string, readOnly bool) {
-	modeValue := readWriteMode
+	m.mode.SetMode(shardID, path, modeFromBool(readOnly))
-	if readOnly {
-		modeValue = readOnlyMode
-	}
-	m.mode.SetMode(shardID, path, modeValue)
 }
 
 func (m *fstreeMetrics) Close(shardID, path string) {

pkg/metrics/mode.go

@@ -73,3 +73,11 @@ func (m *shardIDPathModeValue) Delete(shardID, path string) {
 		pathLabel:    path,
 	})
 }
+
+func modeFromBool(readOnly bool) string {
+	modeValue := readWriteMode
+	if readOnly {
+		modeValue = readOnlyMode
+	}
+	return modeValue
+}

pkg/metrics/node.go

@@ -15,34 +15,24 @@ type NodeMetrics struct {
 	treeService   *treeServiceMetrics
 	epoch         prometheus.Gauge
 	fstree        *fstreeMetrics
+	blobstore     *blobstoreMetrics
 }
 
 func NewNodeMetrics() *NodeMetrics {
-	objectService := newObjectServiceMetrics()
-
-	engine := newEngineMetrics()
-
-	state := newStateMetrics()
-
-	replicator := newReplicatorMetrics()
-
-	treeService := newTreeServiceMetrics()
-
-	epoch := metrics.NewGauge(prometheus.GaugeOpts{
-		Namespace: namespace,
-		Subsystem: innerRingSubsystem,
-		Name:      "epoch",
-		Help:      "Current epoch as seen by inner-ring node.",
-	})
-
 	return &NodeMetrics{
-		objectService: objectService,
+		objectService: newObjectServiceMetrics(),
-		engine:        engine,
+		engine:        newEngineMetrics(),
-		state:         state,
+		state:         newStateMetrics(),
-		replicator:    replicator,
+		replicator:    newReplicatorMetrics(),
-		treeService:   treeService,
+		treeService:   newTreeServiceMetrics(),
-		epoch:         epoch,
+		epoch: metrics.NewGauge(prometheus.GaugeOpts{
-		fstree:        newFSTreeMetrics(),
+			Namespace: namespace,
+			Subsystem: innerRingSubsystem,
+			Name:      "epoch",
+			Help:      "Current epoch as seen by inner-ring node.",
+		}),
+		fstree:    newFSTreeMetrics(),
+		blobstore: newBlobstoreMetrics(),
 	}
 }
 
@@ -74,3 +64,7 @@ func (m *NodeMetrics) State() StateMetrics {
 func (m *NodeMetrics) FSTree() FSTreeMetrics {
 	return m.fstree
 }
+
+func (m *NodeMetrics) Blobstore() BlobstoreMetrics {
+	return m.blobstore
+}