[#424] metrics: Use mode value as metric value for shard

Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
This commit is contained in:
Dmitrii Stepanov 2023-06-14 11:00:44 +03:00
parent 847732605c
commit 4449006862
8 changed files with 66 additions and 74 deletions

View file

@ -3,6 +3,7 @@ package engine
import ( import (
"time" "time"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics"
) )
@ -12,7 +13,7 @@ type MetricRegister interface {
SetObjectCounter(shardID, objectType string, v uint64) SetObjectCounter(shardID, objectType string, v uint64)
AddToObjectCounter(shardID, objectType string, delta int) AddToObjectCounter(shardID, objectType string, delta int)
SetReadonly(shardID string, readonly bool) SetMode(shardID string, mode mode.Mode)
AddToContainerSize(cnrID string, size int64) AddToContainerSize(cnrID string, size int64)
AddToPayloadCounter(shardID string, size int64) AddToPayloadCounter(shardID string, size int64)

View file

@ -50,8 +50,8 @@ func (m *metricsWithID) DecObjectCounter(objectType string) {
m.mw.AddToObjectCounter(m.id, objectType, -1) m.mw.AddToObjectCounter(m.id, objectType, -1)
} }
func (m *metricsWithID) SetReadonly(readonly bool) { func (m *metricsWithID) SetMode(mode mode.Mode) {
m.mw.SetReadonly(m.id, readonly) m.mw.SetMode(m.id, mode)
} }
func (m *metricsWithID) AddToContainerSize(cnr string, size int64) { func (m *metricsWithID) AddToContainerSize(cnr string, size int64) {
@ -90,7 +90,7 @@ func (e *StorageEngine) AddShard(opts ...shard.Option) (*shard.ID, error) {
} }
if e.cfg.metrics != nil { if e.cfg.metrics != nil {
e.cfg.metrics.SetReadonly(sh.ID().String(), sh.GetMode() != mode.ReadWrite) e.cfg.metrics.SetMode(sh.ID().String(), sh.GetMode())
} }
return sh.ID(), nil return sh.ID(), nil

View file

@ -22,7 +22,7 @@ type metricsStore struct {
objCounters map[string]uint64 objCounters map[string]uint64
cnrSize map[string]int64 cnrSize map[string]int64
pldSize int64 pldSize int64
readOnly bool mode mode.Mode
errCounter int64 errCounter int64
} }
@ -57,8 +57,8 @@ func (m metricsStore) DecObjectCounter(objectType string) {
m.AddToObjectCounter(objectType, -1) m.AddToObjectCounter(objectType, -1)
} }
func (m *metricsStore) SetReadonly(r bool) { func (m *metricsStore) SetMode(mode mode.Mode) {
m.readOnly = r m.mode = mode
} }
func (m metricsStore) AddToContainerSize(cnr string, size int64) { func (m metricsStore) AddToContainerSize(cnr string, size int64) {
@ -91,9 +91,9 @@ func TestCounters(t *testing.T) {
sh, mm := shardWithMetrics(t, dir) sh, mm := shardWithMetrics(t, dir)
sh.SetMode(mode.ReadOnly) sh.SetMode(mode.ReadOnly)
require.True(t, mm.readOnly) require.Equal(t, mode.ReadOnly, mm.mode)
sh.SetMode(mode.ReadWrite) sh.SetMode(mode.ReadWrite)
require.False(t, mm.readOnly) require.Equal(t, mode.ReadWrite, mm.mode)
const objNumber = 10 const objNumber = 10
oo := make([]*object.Object, objNumber) oo := make([]*object.Object, objNumber)

View file

@ -64,7 +64,7 @@ func (s *Shard) setMode(m mode.Mode) error {
s.info.Mode = m s.info.Mode = m
if s.metricsWriter != nil { if s.metricsWriter != nil {
s.metricsWriter.SetReadonly(s.info.Mode != mode.ReadWrite) s.metricsWriter.SetMode(s.info.Mode)
} }
s.log.Info(logs.ShardShardModeSetSuccessfully, s.log.Info(logs.ShardShardModeSetSuccessfully,

View file

@ -73,8 +73,8 @@ type MetricsWriter interface {
// SetShardID must set (update) the shard identifier that will be used in // SetShardID must set (update) the shard identifier that will be used in
// metrics. // metrics.
SetShardID(id string) SetShardID(id string)
// SetReadonly must set shard readonly state. // SetReadonly must set shard mode.
SetReadonly(readonly bool) SetMode(mode mode.Mode)
// IncErrorCounter increment error counter. // IncErrorCounter increment error counter.
IncErrorCounter() IncErrorCounter()
// ClearErrorCounter clear error counter. // ClearErrorCounter clear error counter.

View file

@ -3,6 +3,7 @@ package metrics
import ( import (
"time" "time"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
"git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
) )
@ -16,7 +17,7 @@ type EngineMetrics interface {
AddToObjectCounter(shardID, objectType string, delta int) AddToObjectCounter(shardID, objectType string, delta int)
SetObjectCounter(shardID, objectType string, v uint64) SetObjectCounter(shardID, objectType string, v uint64)
AddToPayloadCounter(shardID string, size int64) AddToPayloadCounter(shardID string, size int64)
SetReadonly(shardID string, readonly bool) SetMode(shardID string, mode mode.Mode)
WriteCache() WriteCacheMetrics WriteCache() WriteCacheMetrics
GC() GCMetrics GC() GCMetrics
@ -28,7 +29,7 @@ type engineMetrics struct {
containerSize *prometheus.GaugeVec containerSize *prometheus.GaugeVec
payloadSize *prometheus.GaugeVec payloadSize *prometheus.GaugeVec
errorCounter *prometheus.GaugeVec errorCounter *prometheus.GaugeVec
shardsReadonly *prometheus.GaugeVec mode *shardIDModeValue
gc *gcMetrics gc *gcMetrics
writeCache *writeCacheMetrics writeCache *writeCacheMetrics
@ -50,10 +51,10 @@ func newEngineMetrics() *engineMetrics {
Name: "request_duration_seconds", Name: "request_duration_seconds",
Help: "Duration of Engine requests", Help: "Duration of Engine requests",
}, []string{engineMethod}), }, []string{engineMethod}),
objectCounter: newEngineGaugeVector("objects_total", "Objects counters per shards", []string{shardIDLabelKey, counterTypeLabelKey}), objectCounter: newEngineGaugeVector("objects_total", "Objects counters per shards", []string{shardIDLabelKey, counterTypeLabelKey}),
shardsReadonly: newEngineGaugeVector("mode", "Shard mode", []string{shardIDLabelKey}), gc: newGCMetrics(),
gc: newGCMetrics(), writeCache: newWriteCacheMetrics(),
writeCache: newWriteCacheMetrics(), mode: newShardIDMode(engineSubsystem, "mode", "Shard mode"),
} }
} }
@ -92,7 +93,7 @@ func (m *engineMetrics) DeleteShardMetrics(shardID string) {
m.errorCounter.Delete(prometheus.Labels{shardIDLabelKey: shardID}) m.errorCounter.Delete(prometheus.Labels{shardIDLabelKey: shardID})
m.payloadSize.Delete(prometheus.Labels{shardIDLabelKey: shardID}) m.payloadSize.Delete(prometheus.Labels{shardIDLabelKey: shardID})
m.objectCounter.DeletePartialMatch(prometheus.Labels{shardIDLabelKey: shardID}) m.objectCounter.DeletePartialMatch(prometheus.Labels{shardIDLabelKey: shardID})
m.shardsReadonly.Delete(prometheus.Labels{shardIDLabelKey: shardID}) m.mode.Delete(shardID)
} }
func (m *engineMetrics) AddToObjectCounter(shardID, objectType string, delta int) { func (m *engineMetrics) AddToObjectCounter(shardID, objectType string, delta int) {
@ -113,16 +114,8 @@ func (m *engineMetrics) SetObjectCounter(shardID, objectType string, v uint64) {
).Set(float64(v)) ).Set(float64(v))
} }
func (m *engineMetrics) SetReadonly(shardID string, readonly bool) { func (m *engineMetrics) SetMode(shardID string, mode mode.Mode) {
var flag float64 m.mode.SetMode(shardID, mode.String())
if readonly {
flag = 1
}
m.shardsReadonly.With(
prometheus.Labels{
shardIDLabelKey: shardID,
},
).Set(flag)
} }
func (m *engineMetrics) WriteCache() WriteCacheMetrics { func (m *engineMetrics) WriteCache() WriteCacheMetrics {

39
pkg/metrics/mode.go Normal file
View file

@ -0,0 +1,39 @@
package metrics
import (
"git.frostfs.info/TrueCloudLab/frostfs-observability/metrics"
"github.com/prometheus/client_golang/prometheus"
)
type shardIDModeValue struct {
modeValue *prometheus.GaugeVec
}
func newShardIDMode(subsystem, name, help string) *shardIDModeValue {
return &shardIDModeValue{
modeValue: metrics.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: name,
Help: help,
}, []string{wcShardID, wcMode}),
}
}
func (m *shardIDModeValue) SetMode(shardID string, mode string) {
m.modeValue.DeletePartialMatch(prometheus.Labels{
wcShardID: shardID,
})
m.modeValue.With(prometheus.Labels{
wcShardID: shardID,
wcMode: mode,
}).Set(1)
}
func (m *shardIDModeValue) Delete(shardID string) {
m.modeValue.DeletePartialMatch(prometheus.Labels{
wcShardID: shardID,
})
}

View file

@ -2,7 +2,6 @@ package metrics
import ( import (
"fmt" "fmt"
"sync"
"time" "time"
"git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics"
@ -19,10 +18,6 @@ const (
wcOperation = "operation" wcOperation = "operation"
) )
type shardIDMode struct {
shardID, mode string
}
type WriteCacheMetrics interface { type WriteCacheMetrics interface {
AddMethodDuration(shardID string, method string, success bool, d time.Duration, storageType string) AddMethodDuration(shardID string, method string, success bool, d time.Duration, storageType string)
@ -46,9 +41,7 @@ type writeCacheMetrics struct {
estimatedSize *prometheus.GaugeVec estimatedSize *prometheus.GaugeVec
modeMetrics map[shardIDMode]prometheus.GaugeFunc mode *shardIDModeValue
modeValues map[string]string
modeMtx sync.RWMutex
} }
func newWriteCacheMetrics() *writeCacheMetrics { func newWriteCacheMetrics() *writeCacheMetrics {
@ -67,9 +60,7 @@ func newWriteCacheMetrics() *writeCacheMetrics {
}, []string{wcShardID, wcStorage, wcSuccess, wcOperation}), }, []string{wcShardID, wcStorage, wcSuccess, wcOperation}),
actualCount: newWCGaugeVec("actual_objects_total", "Actual objects count in writecache", []string{wcShardID, wcStorage}), actualCount: newWCGaugeVec("actual_objects_total", "Actual objects count in writecache", []string{wcShardID, wcStorage}),
estimatedSize: newWCGaugeVec("estimated_size_bytes", "Estimated writecache size", []string{wcShardID, wcStorage}), estimatedSize: newWCGaugeVec("estimated_size_bytes", "Estimated writecache size", []string{wcShardID, wcStorage}),
modeMtx: sync.RWMutex{}, mode: newShardIDMode(wcSubsystem, "mode", "Writecache mode value"),
modeMetrics: make(map[shardIDMode]prometheus.GaugeFunc),
modeValues: make(map[string]string),
} }
} }
@ -113,39 +104,7 @@ func (m *writeCacheMetrics) SetEstimateSize(shardID string, size uint64, storage
} }
func (m *writeCacheMetrics) SetMode(shardID string, mode string) { func (m *writeCacheMetrics) SetMode(shardID string, mode string) {
m.modeMtx.Lock() m.mode.SetMode(shardID, mode)
defer m.modeMtx.Unlock()
m.modeValues[shardID] = mode
key := shardIDMode{
shardID: shardID,
mode: mode,
}
if _, found := m.modeMetrics[key]; found {
return
}
metric := metrics.NewGaugeFunc(
prometheus.GaugeOpts{
Namespace: namespace,
Subsystem: wcSubsystem,
Name: "writecache_mode",
Help: "Writecache mode value",
ConstLabels: prometheus.Labels{
wcShardID: shardID,
wcMode: mode,
},
}, func() float64 {
m.modeMtx.RLock()
defer m.modeMtx.RUnlock()
value := m.modeValues[shardID]
if value == mode {
return 1
}
return 0
})
m.modeMetrics[key] = metric
} }
func (m *writeCacheMetrics) IncOperationCounter(shardID string, operation string, success NullBool, storageType string) { func (m *writeCacheMetrics) IncOperationCounter(shardID string, operation string, success NullBool, storageType string) {
@ -158,7 +117,7 @@ func (m *writeCacheMetrics) IncOperationCounter(shardID string, operation string
} }
func (m *writeCacheMetrics) Close(shardID string) { func (m *writeCacheMetrics) Close(shardID string) {
m.SetMode(shardID, "CLOSED") m.mode.Delete(shardID)
m.methodDuration.DeletePartialMatch(prometheus.Labels{wcShardID: shardID}) m.methodDuration.DeletePartialMatch(prometheus.Labels{wcShardID: shardID})
m.operationCounter.DeletePartialMatch(prometheus.Labels{wcShardID: shardID}) m.operationCounter.DeletePartialMatch(prometheus.Labels{wcShardID: shardID})
m.actualCount.DeletePartialMatch(prometheus.Labels{wcShardID: shardID}) m.actualCount.DeletePartialMatch(prometheus.Labels{wcShardID: shardID})