forked from TrueCloudLab/frostfs-node
WIP: Morph: Add unit tests #2
8 changed files with 66 additions and 74 deletions
|
@ -3,6 +3,7 @@ package engine
|
||||||
import (
|
import (
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics"
|
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -12,7 +13,7 @@ type MetricRegister interface {
|
||||||
SetObjectCounter(shardID, objectType string, v uint64)
|
SetObjectCounter(shardID, objectType string, v uint64)
|
||||||
AddToObjectCounter(shardID, objectType string, delta int)
|
AddToObjectCounter(shardID, objectType string, delta int)
|
||||||
|
|
||||||
SetReadonly(shardID string, readonly bool)
|
SetMode(shardID string, mode mode.Mode)
|
||||||
|
|
||||||
AddToContainerSize(cnrID string, size int64)
|
AddToContainerSize(cnrID string, size int64)
|
||||||
AddToPayloadCounter(shardID string, size int64)
|
AddToPayloadCounter(shardID string, size int64)
|
||||||
|
|
|
@ -50,8 +50,8 @@ func (m *metricsWithID) DecObjectCounter(objectType string) {
|
||||||
m.mw.AddToObjectCounter(m.id, objectType, -1)
|
m.mw.AddToObjectCounter(m.id, objectType, -1)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *metricsWithID) SetReadonly(readonly bool) {
|
func (m *metricsWithID) SetMode(mode mode.Mode) {
|
||||||
m.mw.SetReadonly(m.id, readonly)
|
m.mw.SetMode(m.id, mode)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *metricsWithID) AddToContainerSize(cnr string, size int64) {
|
func (m *metricsWithID) AddToContainerSize(cnr string, size int64) {
|
||||||
|
@ -90,7 +90,7 @@ func (e *StorageEngine) AddShard(opts ...shard.Option) (*shard.ID, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
if e.cfg.metrics != nil {
|
if e.cfg.metrics != nil {
|
||||||
e.cfg.metrics.SetReadonly(sh.ID().String(), sh.GetMode() != mode.ReadWrite)
|
e.cfg.metrics.SetMode(sh.ID().String(), sh.GetMode())
|
||||||
}
|
}
|
||||||
|
|
||||||
return sh.ID(), nil
|
return sh.ID(), nil
|
||||||
|
|
|
@ -22,7 +22,7 @@ type metricsStore struct {
|
||||||
objCounters map[string]uint64
|
objCounters map[string]uint64
|
||||||
cnrSize map[string]int64
|
cnrSize map[string]int64
|
||||||
pldSize int64
|
pldSize int64
|
||||||
readOnly bool
|
mode mode.Mode
|
||||||
errCounter int64
|
errCounter int64
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -57,8 +57,8 @@ func (m metricsStore) DecObjectCounter(objectType string) {
|
||||||
m.AddToObjectCounter(objectType, -1)
|
m.AddToObjectCounter(objectType, -1)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *metricsStore) SetReadonly(r bool) {
|
func (m *metricsStore) SetMode(mode mode.Mode) {
|
||||||
m.readOnly = r
|
m.mode = mode
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m metricsStore) AddToContainerSize(cnr string, size int64) {
|
func (m metricsStore) AddToContainerSize(cnr string, size int64) {
|
||||||
|
@ -91,9 +91,9 @@ func TestCounters(t *testing.T) {
|
||||||
sh, mm := shardWithMetrics(t, dir)
|
sh, mm := shardWithMetrics(t, dir)
|
||||||
|
|
||||||
sh.SetMode(mode.ReadOnly)
|
sh.SetMode(mode.ReadOnly)
|
||||||
require.True(t, mm.readOnly)
|
require.Equal(t, mode.ReadOnly, mm.mode)
|
||||||
sh.SetMode(mode.ReadWrite)
|
sh.SetMode(mode.ReadWrite)
|
||||||
require.False(t, mm.readOnly)
|
require.Equal(t, mode.ReadWrite, mm.mode)
|
||||||
|
|
||||||
const objNumber = 10
|
const objNumber = 10
|
||||||
oo := make([]*object.Object, objNumber)
|
oo := make([]*object.Object, objNumber)
|
||||||
|
|
|
@ -64,7 +64,7 @@ func (s *Shard) setMode(m mode.Mode) error {
|
||||||
|
|
||||||
s.info.Mode = m
|
s.info.Mode = m
|
||||||
if s.metricsWriter != nil {
|
if s.metricsWriter != nil {
|
||||||
s.metricsWriter.SetReadonly(s.info.Mode != mode.ReadWrite)
|
s.metricsWriter.SetMode(s.info.Mode)
|
||||||
}
|
}
|
||||||
|
|
||||||
s.log.Info(logs.ShardShardModeSetSuccessfully,
|
s.log.Info(logs.ShardShardModeSetSuccessfully,
|
||||||
|
|
|
@ -73,8 +73,8 @@ type MetricsWriter interface {
|
||||||
// SetShardID must set (update) the shard identifier that will be used in
|
// SetShardID must set (update) the shard identifier that will be used in
|
||||||
// metrics.
|
// metrics.
|
||||||
SetShardID(id string)
|
SetShardID(id string)
|
||||||
// SetReadonly must set shard readonly state.
|
// SetReadonly must set shard mode.
|
||||||
SetReadonly(readonly bool)
|
SetMode(mode mode.Mode)
|
||||||
// IncErrorCounter increment error counter.
|
// IncErrorCounter increment error counter.
|
||||||
IncErrorCounter()
|
IncErrorCounter()
|
||||||
// ClearErrorCounter clear error counter.
|
// ClearErrorCounter clear error counter.
|
||||||
|
|
|
@ -3,6 +3,7 @@ package metrics
|
||||||
import (
|
import (
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-observability/metrics"
|
"git.frostfs.info/TrueCloudLab/frostfs-observability/metrics"
|
||||||
"github.com/prometheus/client_golang/prometheus"
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
)
|
)
|
||||||
|
@ -16,7 +17,7 @@ type EngineMetrics interface {
|
||||||
AddToObjectCounter(shardID, objectType string, delta int)
|
AddToObjectCounter(shardID, objectType string, delta int)
|
||||||
SetObjectCounter(shardID, objectType string, v uint64)
|
SetObjectCounter(shardID, objectType string, v uint64)
|
||||||
AddToPayloadCounter(shardID string, size int64)
|
AddToPayloadCounter(shardID string, size int64)
|
||||||
SetReadonly(shardID string, readonly bool)
|
SetMode(shardID string, mode mode.Mode)
|
||||||
|
|
||||||
WriteCache() WriteCacheMetrics
|
WriteCache() WriteCacheMetrics
|
||||||
GC() GCMetrics
|
GC() GCMetrics
|
||||||
|
@ -28,7 +29,7 @@ type engineMetrics struct {
|
||||||
containerSize *prometheus.GaugeVec
|
containerSize *prometheus.GaugeVec
|
||||||
payloadSize *prometheus.GaugeVec
|
payloadSize *prometheus.GaugeVec
|
||||||
errorCounter *prometheus.GaugeVec
|
errorCounter *prometheus.GaugeVec
|
||||||
shardsReadonly *prometheus.GaugeVec
|
mode *shardIDModeValue
|
||||||
|
|
||||||
gc *gcMetrics
|
gc *gcMetrics
|
||||||
writeCache *writeCacheMetrics
|
writeCache *writeCacheMetrics
|
||||||
|
@ -50,10 +51,10 @@ func newEngineMetrics() *engineMetrics {
|
||||||
Name: "request_duration_seconds",
|
Name: "request_duration_seconds",
|
||||||
Help: "Duration of Engine requests",
|
Help: "Duration of Engine requests",
|
||||||
}, []string{engineMethod}),
|
}, []string{engineMethod}),
|
||||||
objectCounter: newEngineGaugeVector("objects_total", "Objects counters per shards", []string{shardIDLabelKey, counterTypeLabelKey}),
|
objectCounter: newEngineGaugeVector("objects_total", "Objects counters per shards", []string{shardIDLabelKey, counterTypeLabelKey}),
|
||||||
shardsReadonly: newEngineGaugeVector("mode", "Shard mode", []string{shardIDLabelKey}),
|
gc: newGCMetrics(),
|
||||||
gc: newGCMetrics(),
|
writeCache: newWriteCacheMetrics(),
|
||||||
writeCache: newWriteCacheMetrics(),
|
mode: newShardIDMode(engineSubsystem, "mode", "Shard mode"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -92,7 +93,7 @@ func (m *engineMetrics) DeleteShardMetrics(shardID string) {
|
||||||
m.errorCounter.Delete(prometheus.Labels{shardIDLabelKey: shardID})
|
m.errorCounter.Delete(prometheus.Labels{shardIDLabelKey: shardID})
|
||||||
m.payloadSize.Delete(prometheus.Labels{shardIDLabelKey: shardID})
|
m.payloadSize.Delete(prometheus.Labels{shardIDLabelKey: shardID})
|
||||||
m.objectCounter.DeletePartialMatch(prometheus.Labels{shardIDLabelKey: shardID})
|
m.objectCounter.DeletePartialMatch(prometheus.Labels{shardIDLabelKey: shardID})
|
||||||
m.shardsReadonly.Delete(prometheus.Labels{shardIDLabelKey: shardID})
|
m.mode.Delete(shardID)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *engineMetrics) AddToObjectCounter(shardID, objectType string, delta int) {
|
func (m *engineMetrics) AddToObjectCounter(shardID, objectType string, delta int) {
|
||||||
|
@ -113,16 +114,8 @@ func (m *engineMetrics) SetObjectCounter(shardID, objectType string, v uint64) {
|
||||||
).Set(float64(v))
|
).Set(float64(v))
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *engineMetrics) SetReadonly(shardID string, readonly bool) {
|
func (m *engineMetrics) SetMode(shardID string, mode mode.Mode) {
|
||||||
var flag float64
|
m.mode.SetMode(shardID, mode.String())
|
||||||
if readonly {
|
|
||||||
flag = 1
|
|
||||||
}
|
|
||||||
m.shardsReadonly.With(
|
|
||||||
prometheus.Labels{
|
|
||||||
shardIDLabelKey: shardID,
|
|
||||||
},
|
|
||||||
).Set(flag)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *engineMetrics) WriteCache() WriteCacheMetrics {
|
func (m *engineMetrics) WriteCache() WriteCacheMetrics {
|
||||||
|
|
39
pkg/metrics/mode.go
Normal file
39
pkg/metrics/mode.go
Normal file
|
@ -0,0 +1,39 @@
|
||||||
|
package metrics
|
||||||
|
|
||||||
|
import (
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-observability/metrics"
|
||||||
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
|
)
|
||||||
|
|
||||||
|
type shardIDModeValue struct {
|
||||||
|
modeValue *prometheus.GaugeVec
|
||||||
|
}
|
||||||
|
|
||||||
|
func newShardIDMode(subsystem, name, help string) *shardIDModeValue {
|
||||||
|
return &shardIDModeValue{
|
||||||
|
modeValue: metrics.NewGaugeVec(
|
||||||
|
prometheus.GaugeOpts{
|
||||||
|
Namespace: namespace,
|
||||||
|
Subsystem: subsystem,
|
||||||
|
Name: name,
|
||||||
|
Help: help,
|
||||||
|
}, []string{wcShardID, wcMode}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *shardIDModeValue) SetMode(shardID string, mode string) {
|
||||||
|
m.modeValue.DeletePartialMatch(prometheus.Labels{
|
||||||
|
wcShardID: shardID,
|
||||||
|
})
|
||||||
|
|
||||||
|
m.modeValue.With(prometheus.Labels{
|
||||||
|
wcShardID: shardID,
|
||||||
|
wcMode: mode,
|
||||||
|
}).Set(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *shardIDModeValue) Delete(shardID string) {
|
||||||
|
m.modeValue.DeletePartialMatch(prometheus.Labels{
|
||||||
|
wcShardID: shardID,
|
||||||
|
})
|
||||||
|
}
|
|
@ -2,7 +2,6 @@ package metrics
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"sync"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-observability/metrics"
|
"git.frostfs.info/TrueCloudLab/frostfs-observability/metrics"
|
||||||
|
@ -19,10 +18,6 @@ const (
|
||||||
wcOperation = "operation"
|
wcOperation = "operation"
|
||||||
)
|
)
|
||||||
|
|
||||||
type shardIDMode struct {
|
|
||||||
shardID, mode string
|
|
||||||
}
|
|
||||||
|
|
||||||
type WriteCacheMetrics interface {
|
type WriteCacheMetrics interface {
|
||||||
AddMethodDuration(shardID string, method string, success bool, d time.Duration, storageType string)
|
AddMethodDuration(shardID string, method string, success bool, d time.Duration, storageType string)
|
||||||
|
|
||||||
|
@ -46,9 +41,7 @@ type writeCacheMetrics struct {
|
||||||
|
|
||||||
estimatedSize *prometheus.GaugeVec
|
estimatedSize *prometheus.GaugeVec
|
||||||
|
|
||||||
modeMetrics map[shardIDMode]prometheus.GaugeFunc
|
mode *shardIDModeValue
|
||||||
modeValues map[string]string
|
|
||||||
modeMtx sync.RWMutex
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func newWriteCacheMetrics() *writeCacheMetrics {
|
func newWriteCacheMetrics() *writeCacheMetrics {
|
||||||
|
@ -67,9 +60,7 @@ func newWriteCacheMetrics() *writeCacheMetrics {
|
||||||
}, []string{wcShardID, wcStorage, wcSuccess, wcOperation}),
|
}, []string{wcShardID, wcStorage, wcSuccess, wcOperation}),
|
||||||
actualCount: newWCGaugeVec("actual_objects_total", "Actual objects count in writecache", []string{wcShardID, wcStorage}),
|
actualCount: newWCGaugeVec("actual_objects_total", "Actual objects count in writecache", []string{wcShardID, wcStorage}),
|
||||||
estimatedSize: newWCGaugeVec("estimated_size_bytes", "Estimated writecache size", []string{wcShardID, wcStorage}),
|
estimatedSize: newWCGaugeVec("estimated_size_bytes", "Estimated writecache size", []string{wcShardID, wcStorage}),
|
||||||
modeMtx: sync.RWMutex{},
|
mode: newShardIDMode(wcSubsystem, "mode", "Writecache mode value"),
|
||||||
modeMetrics: make(map[shardIDMode]prometheus.GaugeFunc),
|
|
||||||
modeValues: make(map[string]string),
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -113,39 +104,7 @@ func (m *writeCacheMetrics) SetEstimateSize(shardID string, size uint64, storage
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *writeCacheMetrics) SetMode(shardID string, mode string) {
|
func (m *writeCacheMetrics) SetMode(shardID string, mode string) {
|
||||||
m.modeMtx.Lock()
|
m.mode.SetMode(shardID, mode)
|
||||||
defer m.modeMtx.Unlock()
|
|
||||||
|
|
||||||
m.modeValues[shardID] = mode
|
|
||||||
key := shardIDMode{
|
|
||||||
shardID: shardID,
|
|
||||||
mode: mode,
|
|
||||||
}
|
|
||||||
if _, found := m.modeMetrics[key]; found {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
metric := metrics.NewGaugeFunc(
|
|
||||||
prometheus.GaugeOpts{
|
|
||||||
Namespace: namespace,
|
|
||||||
Subsystem: wcSubsystem,
|
|
||||||
Name: "writecache_mode",
|
|
||||||
Help: "Writecache mode value",
|
|
||||||
ConstLabels: prometheus.Labels{
|
|
||||||
wcShardID: shardID,
|
|
||||||
wcMode: mode,
|
|
||||||
},
|
|
||||||
}, func() float64 {
|
|
||||||
m.modeMtx.RLock()
|
|
||||||
defer m.modeMtx.RUnlock()
|
|
||||||
|
|
||||||
value := m.modeValues[shardID]
|
|
||||||
if value == mode {
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
return 0
|
|
||||||
})
|
|
||||||
m.modeMetrics[key] = metric
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *writeCacheMetrics) IncOperationCounter(shardID string, operation string, success NullBool, storageType string) {
|
func (m *writeCacheMetrics) IncOperationCounter(shardID string, operation string, success NullBool, storageType string) {
|
||||||
|
@ -158,7 +117,7 @@ func (m *writeCacheMetrics) IncOperationCounter(shardID string, operation string
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *writeCacheMetrics) Close(shardID string) {
|
func (m *writeCacheMetrics) Close(shardID string) {
|
||||||
m.SetMode(shardID, "CLOSED")
|
m.mode.Delete(shardID)
|
||||||
m.methodDuration.DeletePartialMatch(prometheus.Labels{wcShardID: shardID})
|
m.methodDuration.DeletePartialMatch(prometheus.Labels{wcShardID: shardID})
|
||||||
m.operationCounter.DeletePartialMatch(prometheus.Labels{wcShardID: shardID})
|
m.operationCounter.DeletePartialMatch(prometheus.Labels{wcShardID: shardID})
|
||||||
m.actualCount.DeletePartialMatch(prometheus.Labels{wcShardID: shardID})
|
m.actualCount.DeletePartialMatch(prometheus.Labels{wcShardID: shardID})
|
||||||
|
|
Loading…
Reference in a new issue