From 9c64c6a2c45134abb19eedbb8acade7558171c0a Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 24 Jun 2024 17:12:26 +0300 Subject: [PATCH 1/2] [#451] metrics: Move to internal `metrics` don't look like something others want to import. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-ir/main.go | 2 +- cmd/frostfs-node/config.go | 2 +- cmd/frostfs-node/netmap.go | 2 +- {pkg => internal}/metrics/blobovnicza.go | 0 {pkg => internal}/metrics/blobstore.go | 0 {pkg => internal}/metrics/consts.go | 0 {pkg => internal}/metrics/engine.go | 0 {pkg => internal}/metrics/fstree.go | 0 {pkg => internal}/metrics/gc.go | 0 {pkg => internal}/metrics/grpc.go | 0 {pkg => internal}/metrics/innerring.go | 0 {pkg => internal}/metrics/metabase.go | 0 {pkg => internal}/metrics/mode.go | 0 {pkg => internal}/metrics/morph.go | 0 {pkg => internal}/metrics/morphcache.go | 0 {pkg => internal}/metrics/node.go | 0 {pkg => internal}/metrics/object.go | 0 {pkg => internal}/metrics/pilorama.go | 0 {pkg => internal}/metrics/policer.go | 0 {pkg => internal}/metrics/replicator.go | 0 {pkg => internal}/metrics/state.go | 0 {pkg => internal}/metrics/treeservice.go | 0 {pkg => internal}/metrics/types.go | 0 {pkg => internal}/metrics/writecache.go | 0 pkg/innerring/innerring.go | 2 +- pkg/local_object_storage/engine/metrics.go | 2 +- pkg/local_object_storage/engine/writecache.go | 2 +- pkg/local_object_storage/metrics/blobovnicza.go | 2 +- pkg/local_object_storage/metrics/blobstore.go | 2 +- pkg/local_object_storage/metrics/fstree.go | 2 +- pkg/local_object_storage/metrics/metabase.go | 2 +- pkg/local_object_storage/metrics/pilorama.go | 2 +- pkg/morph/client/client.go | 2 +- pkg/morph/client/constructor.go | 2 +- scripts/export-metrics/main.go | 2 +- 35 files changed, 14 insertions(+), 14 deletions(-) rename {pkg => internal}/metrics/blobovnicza.go (100%) rename {pkg => internal}/metrics/blobstore.go (100%) rename {pkg => internal}/metrics/consts.go (100%) rename {pkg => internal}/metrics/engine.go (100%) rename {pkg => internal}/metrics/fstree.go (100%) rename {pkg => internal}/metrics/gc.go (100%) rename {pkg => internal}/metrics/grpc.go (100%) rename {pkg => internal}/metrics/innerring.go (100%) rename {pkg => internal}/metrics/metabase.go (100%) rename {pkg => internal}/metrics/mode.go (100%) rename {pkg => internal}/metrics/morph.go (100%) rename {pkg => internal}/metrics/morphcache.go (100%) rename {pkg => internal}/metrics/node.go (100%) rename {pkg => internal}/metrics/object.go (100%) rename {pkg => internal}/metrics/pilorama.go (100%) rename {pkg => internal}/metrics/policer.go (100%) rename {pkg => internal}/metrics/replicator.go (100%) rename {pkg => internal}/metrics/state.go (100%) rename {pkg => internal}/metrics/treeservice.go (100%) rename {pkg => internal}/metrics/types.go (100%) rename {pkg => internal}/metrics/writecache.go (100%) diff --git a/cmd/frostfs-ir/main.go b/cmd/frostfs-ir/main.go index 206059b5d..5b852c37c 100644 --- a/cmd/frostfs-ir/main.go +++ b/cmd/frostfs-ir/main.go @@ -9,9 +9,9 @@ import ( "sync/atomic" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + irMetrics "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/misc" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" - irMetrics "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "github.com/spf13/viper" "go.uber.org/zap" diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 2b185cfc8..ef2c0424b 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -31,6 +31,7 @@ import ( replicatorconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/replicator" tracingconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/chainbase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" frostfsidcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/frostfsid" @@ -45,7 +46,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" shardmode "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" containerClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index 2d424eec6..d2c0af52d 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -10,8 +10,8 @@ import ( netmapGRPC "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap/grpc" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" netmapEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/netmap" diff --git a/pkg/metrics/blobovnicza.go b/internal/metrics/blobovnicza.go similarity index 100% rename from pkg/metrics/blobovnicza.go rename to internal/metrics/blobovnicza.go diff --git a/pkg/metrics/blobstore.go b/internal/metrics/blobstore.go similarity index 100% rename from pkg/metrics/blobstore.go rename to internal/metrics/blobstore.go diff --git a/pkg/metrics/consts.go b/internal/metrics/consts.go similarity index 100% rename from pkg/metrics/consts.go rename to internal/metrics/consts.go diff --git a/pkg/metrics/engine.go b/internal/metrics/engine.go similarity index 100% rename from pkg/metrics/engine.go rename to internal/metrics/engine.go diff --git a/pkg/metrics/fstree.go b/internal/metrics/fstree.go similarity index 100% rename from pkg/metrics/fstree.go rename to internal/metrics/fstree.go diff --git a/pkg/metrics/gc.go b/internal/metrics/gc.go similarity index 100% rename from pkg/metrics/gc.go rename to internal/metrics/gc.go diff --git a/pkg/metrics/grpc.go b/internal/metrics/grpc.go similarity index 100% rename from pkg/metrics/grpc.go rename to internal/metrics/grpc.go diff --git a/pkg/metrics/innerring.go b/internal/metrics/innerring.go similarity index 100% rename from pkg/metrics/innerring.go rename to internal/metrics/innerring.go diff --git a/pkg/metrics/metabase.go b/internal/metrics/metabase.go similarity index 100% rename from pkg/metrics/metabase.go rename to internal/metrics/metabase.go diff --git a/pkg/metrics/mode.go b/internal/metrics/mode.go similarity index 100% rename from pkg/metrics/mode.go rename to internal/metrics/mode.go diff --git a/pkg/metrics/morph.go b/internal/metrics/morph.go similarity index 100% rename from pkg/metrics/morph.go rename to internal/metrics/morph.go diff --git a/pkg/metrics/morphcache.go b/internal/metrics/morphcache.go similarity index 100% rename from pkg/metrics/morphcache.go rename to internal/metrics/morphcache.go diff --git a/pkg/metrics/node.go b/internal/metrics/node.go similarity index 100% rename from pkg/metrics/node.go rename to internal/metrics/node.go diff --git a/pkg/metrics/object.go b/internal/metrics/object.go similarity index 100% rename from pkg/metrics/object.go rename to internal/metrics/object.go diff --git a/pkg/metrics/pilorama.go b/internal/metrics/pilorama.go similarity index 100% rename from pkg/metrics/pilorama.go rename to internal/metrics/pilorama.go diff --git a/pkg/metrics/policer.go b/internal/metrics/policer.go similarity index 100% rename from pkg/metrics/policer.go rename to internal/metrics/policer.go diff --git a/pkg/metrics/replicator.go b/internal/metrics/replicator.go similarity index 100% rename from pkg/metrics/replicator.go rename to internal/metrics/replicator.go diff --git a/pkg/metrics/state.go b/internal/metrics/state.go similarity index 100% rename from pkg/metrics/state.go rename to internal/metrics/state.go diff --git a/pkg/metrics/treeservice.go b/internal/metrics/treeservice.go similarity index 100% rename from pkg/metrics/treeservice.go rename to internal/metrics/treeservice.go diff --git a/pkg/metrics/types.go b/internal/metrics/types.go similarity index 100% rename from pkg/metrics/types.go rename to internal/metrics/types.go diff --git a/pkg/metrics/writecache.go b/internal/metrics/writecache.go similarity index 100% rename from pkg/metrics/writecache.go rename to internal/metrics/writecache.go diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index 944aa8b76..50a37845b 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -8,12 +8,12 @@ import ( "sync/atomic" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/config" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/alphabet" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/governance" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap" timerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/timers" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" balanceClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/balance" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" diff --git a/pkg/local_object_storage/engine/metrics.go b/pkg/local_object_storage/engine/metrics.go index 6b99dff7e..72b5ae252 100644 --- a/pkg/local_object_storage/engine/metrics.go +++ b/pkg/local_object_storage/engine/metrics.go @@ -3,8 +3,8 @@ package engine import ( "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" ) type MetricRegister interface { diff --git a/pkg/local_object_storage/engine/writecache.go b/pkg/local_object_storage/engine/writecache.go index 023576716..da488260a 100644 --- a/pkg/local_object_storage/engine/writecache.go +++ b/pkg/local_object_storage/engine/writecache.go @@ -5,10 +5,10 @@ import ( "sync" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" diff --git a/pkg/local_object_storage/metrics/blobovnicza.go b/pkg/local_object_storage/metrics/blobovnicza.go index 6886bbc1d..460d6e638 100644 --- a/pkg/local_object_storage/metrics/blobovnicza.go +++ b/pkg/local_object_storage/metrics/blobovnicza.go @@ -3,10 +3,10 @@ package metrics import ( "time" + metrics_impl "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" - metrics_impl "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" ) func NewBlobovniczaTreeMetrics(path string, m metrics_impl.BlobobvnizcaMetrics) blobovniczatree.Metrics { diff --git a/pkg/local_object_storage/metrics/blobstore.go b/pkg/local_object_storage/metrics/blobstore.go index b3871dfca..9a41f01d0 100644 --- a/pkg/local_object_storage/metrics/blobstore.go +++ b/pkg/local_object_storage/metrics/blobstore.go @@ -3,8 +3,8 @@ package metrics import ( "time" + metrics_impl "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" - metrics_impl "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" ) type blobstoreMetrics struct { diff --git a/pkg/local_object_storage/metrics/fstree.go b/pkg/local_object_storage/metrics/fstree.go index e6b3175be..76822ac2c 100644 --- a/pkg/local_object_storage/metrics/fstree.go +++ b/pkg/local_object_storage/metrics/fstree.go @@ -3,9 +3,9 @@ package metrics import ( "time" + metrics_impl "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" - metrics_impl "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" ) func NewFSTreeMetricsWithoutShardID(path string, m metrics_impl.FSTreeMetrics) fstree.Metrics { diff --git a/pkg/local_object_storage/metrics/metabase.go b/pkg/local_object_storage/metrics/metabase.go index 1a02f2705..e962e37cb 100644 --- a/pkg/local_object_storage/metrics/metabase.go +++ b/pkg/local_object_storage/metrics/metabase.go @@ -3,9 +3,9 @@ package metrics import ( "time" + metrics_impl "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" - metrics_impl "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" ) func NewMetabaseMetrics(path string, m metrics_impl.MetabaseMetrics) meta.Metrics { diff --git a/pkg/local_object_storage/metrics/pilorama.go b/pkg/local_object_storage/metrics/pilorama.go index 1b715c2bd..050b769a0 100644 --- a/pkg/local_object_storage/metrics/pilorama.go +++ b/pkg/local_object_storage/metrics/pilorama.go @@ -3,9 +3,9 @@ package metrics import ( "time" + metrics_impl "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" - metrics_impl "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" ) func NewPiloramaMetrics(m metrics_impl.PiloramaMetrics) pilorama.Metrics { diff --git a/pkg/morph/client/client.go b/pkg/morph/client/client.go index c9f819f04..df521f56b 100644 --- a/pkg/morph/client/client.go +++ b/pkg/morph/client/client.go @@ -10,7 +10,7 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" morphmetrics "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "github.com/google/uuid" diff --git a/pkg/morph/client/constructor.go b/pkg/morph/client/constructor.go index c499aa892..648c7d3c0 100644 --- a/pkg/morph/client/constructor.go +++ b/pkg/morph/client/constructor.go @@ -7,7 +7,7 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" morphmetrics "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" lru "github.com/hashicorp/golang-lru/v2" diff --git a/scripts/export-metrics/main.go b/scripts/export-metrics/main.go index 63571e96d..51705ee49 100644 --- a/scripts/export-metrics/main.go +++ b/scripts/export-metrics/main.go @@ -6,7 +6,7 @@ import ( "fmt" "os" - local_metrics "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" + local_metrics "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" ) -- 2.45.2 From 9f36dd6f5e26d257d0e601c72e2e984ef6a9878a Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 24 Jun 2024 18:04:35 +0300 Subject: [PATCH 2/2] [#451] frostfs-node: Add cache metrics Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/cache.go | 67 ++++++++++++++++++++++++++++------ cmd/frostfs-node/cache_test.go | 6 ++- internal/metrics/cache.go | 35 ++++++++++++++++++ internal/metrics/consts.go | 3 ++ 4 files changed, 99 insertions(+), 12 deletions(-) create mode 100644 internal/metrics/cache.go diff --git a/cmd/frostfs-node/cache.go b/cmd/frostfs-node/cache.go index bf68d245f..81d552729 100644 --- a/cmd/frostfs-node/cache.go +++ b/cmd/frostfs-node/cache.go @@ -4,6 +4,7 @@ import ( "sync" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" putsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/put" @@ -28,15 +29,17 @@ type ttlNetCache[K comparable, V any] struct { cache *expirable.LRU[K, *valueWithError[V]] netRdr netValueReader[K, V] keyLocker *utilSync.KeyLocker[K] + metrics cacheMetrics } // complicates netValueReader with TTL caching mechanism. -func newNetworkTTLCache[K comparable, V any](sz int, ttl time.Duration, netRdr netValueReader[K, V]) *ttlNetCache[K, V] { +func newNetworkTTLCache[K comparable, V any](sz int, ttl time.Duration, netRdr netValueReader[K, V], metrics cacheMetrics) *ttlNetCache[K, V] { cache := expirable.NewLRU[K, *valueWithError[V]](sz, nil, ttl) return &ttlNetCache[K, V]{ cache: cache, netRdr: netRdr, + metrics: metrics, keyLocker: utilSync.NewKeyLocker[K](), } } @@ -47,8 +50,15 @@ func newNetworkTTLCache[K comparable, V any](sz int, ttl time.Duration, netRdr n // // returned value should not be modified. func (c *ttlNetCache[K, V]) get(key K) (V, error) { + hit := false + startedAt := time.Now() + defer func() { + c.metrics.AddMethodDuration("Get", time.Since(startedAt), hit) + }() + val, ok := c.cache.Peek(key) if ok { + hit = true return val.v, val.e } @@ -57,6 +67,7 @@ func (c *ttlNetCache[K, V]) get(key K) (V, error) { val, ok = c.cache.Peek(key) if ok { + hit = true return val.v, val.e } @@ -71,6 +82,11 @@ func (c *ttlNetCache[K, V]) get(key K) (V, error) { } func (c *ttlNetCache[K, V]) set(k K, v V, e error) { + startedAt := time.Now() + defer func() { + c.metrics.AddMethodDuration("Set", time.Since(startedAt), false) + }() + c.keyLocker.Lock(k) defer c.keyLocker.Unlock(k) @@ -81,10 +97,16 @@ func (c *ttlNetCache[K, V]) set(k K, v V, e error) { } func (c *ttlNetCache[K, V]) remove(key K) { + hit := false + startedAt := time.Now() + defer func() { + c.metrics.AddMethodDuration("Remove", time.Since(startedAt), hit) + }() + c.keyLocker.Lock(key) defer c.keyLocker.Unlock(key) - c.cache.Remove(key) + hit = c.cache.Remove(key) } // entity that provides LRU cache interface. @@ -92,16 +114,19 @@ type lruNetCache struct { cache *lru.Cache[uint64, *netmapSDK.NetMap] netRdr netValueReader[uint64, *netmapSDK.NetMap] + + metrics cacheMetrics } // newNetworkLRUCache returns wrapper over netValueReader with LRU cache. -func newNetworkLRUCache(sz int, netRdr netValueReader[uint64, *netmapSDK.NetMap]) *lruNetCache { +func newNetworkLRUCache(sz int, netRdr netValueReader[uint64, *netmapSDK.NetMap], metrics cacheMetrics) *lruNetCache { cache, err := lru.New[uint64, *netmapSDK.NetMap](sz) fatalOnErr(err) return &lruNetCache{ - cache: cache, - netRdr: netRdr, + cache: cache, + netRdr: netRdr, + metrics: metrics, } } @@ -111,8 +136,15 @@ func newNetworkLRUCache(sz int, netRdr netValueReader[uint64, *netmapSDK.NetMap] // // returned value should not be modified. func (c *lruNetCache) get(key uint64) (*netmapSDK.NetMap, error) { + hit := false + startedAt := time.Now() + defer func() { + c.metrics.AddMethodDuration("Get", time.Since(startedAt), hit) + }() + val, ok := c.cache.Get(key) if ok { + hit = true return val, nil } @@ -138,10 +170,10 @@ func newCachedContainerStorage(v container.Source, ttl time.Duration) ttlContain lruCnrCache := newNetworkTTLCache(containerCacheSize, ttl, func(id cid.ID) (*container.Container, error) { return v.Get(id) - }) + }, metrics.NewCacheMetrics("container")) lruDelInfoCache := newNetworkTTLCache(containerCacheSize, ttl, func(id cid.ID) (*container.DelInfo, error) { return v.DeletionInfo(id) - }) + }, metrics.NewCacheMetrics("container_deletion_info")) return ttlContainerStorage{ containerCache: lruCnrCache, @@ -175,7 +207,7 @@ func newCachedEACLStorage(v container.EACLSource, ttl time.Duration) ttlEACLStor lruCnrCache := newNetworkTTLCache(eaclCacheSize, ttl, func(id cid.ID) (*container.EACL, error) { return v.GetEACL(id) - }) + }, metrics.NewCacheMetrics("eacl")) return ttlEACLStorage{lruCnrCache} } @@ -202,7 +234,7 @@ func newCachedNetmapStorage(s netmap.State, v netmap.Source) netmap.Source { lruNetmapCache := newNetworkLRUCache(netmapCacheSize, func(key uint64) (*netmapSDK.NetMap, error) { return v.GetNetMapByEpoch(key) - }) + }, metrics.NewCacheMetrics("netmap")) return &lruNetmapSource{ netState: s, @@ -251,7 +283,7 @@ func newCachedIRFetcher(f interface{ InnerRingKeys() ([][]byte, error) }) cached irFetcherCache := newNetworkTTLCache(irFetcherCacheSize, irFetcherCacheTTL, func(_ struct{}) ([][]byte, error) { return f.InnerRingKeys() - }, + }, metrics.NewCacheMetrics("ir_keys"), ) return cachedIRFetcher{irFetcherCache} @@ -274,23 +306,32 @@ type ttlMaxObjectSizeCache struct { lastUpdated time.Time lastSize uint64 src putsvc.MaxSizeSource + metrics cacheMetrics } func newCachedMaxObjectSizeSource(src putsvc.MaxSizeSource) putsvc.MaxSizeSource { return &ttlMaxObjectSizeCache{ - src: src, + src: src, + metrics: metrics.NewCacheMetrics("max_object_size"), } } func (c *ttlMaxObjectSizeCache) MaxObjectSize() uint64 { const ttl = time.Second * 30 + hit := false + startedAt := time.Now() + defer func() { + c.metrics.AddMethodDuration("Get", time.Since(startedAt), hit) + }() + c.mtx.RLock() prevUpdated := c.lastUpdated size := c.lastSize c.mtx.RUnlock() if time.Since(prevUpdated) < ttl { + hit = true return size } @@ -305,3 +346,7 @@ func (c *ttlMaxObjectSizeCache) MaxObjectSize() uint64 { return size } + +type cacheMetrics interface { + AddMethodDuration(method string, d time.Duration, hit bool) +} diff --git a/cmd/frostfs-node/cache_test.go b/cmd/frostfs-node/cache_test.go index 6981ac721..f8c324a2f 100644 --- a/cmd/frostfs-node/cache_test.go +++ b/cmd/frostfs-node/cache_test.go @@ -10,7 +10,7 @@ import ( func TestTTLNetCache(t *testing.T) { ttlDuration := time.Millisecond * 50 - cache := newNetworkTTLCache(10, ttlDuration, testNetValueReader) + cache := newNetworkTTLCache(10, ttlDuration, testNetValueReader, &noopCacheMetricts{}) key := "key" @@ -54,3 +54,7 @@ func testNetValueReader(key string) (time.Time, error) { } return time.Now(), nil } + +type noopCacheMetricts struct{} + +func (m *noopCacheMetricts) AddMethodDuration(method string, d time.Duration, hit bool) {} diff --git a/internal/metrics/cache.go b/internal/metrics/cache.go new file mode 100644 index 000000000..8181586e2 --- /dev/null +++ b/internal/metrics/cache.go @@ -0,0 +1,35 @@ +package metrics + +import ( + "strconv" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +var cacheRequests = metrics.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: commonCacheSubsystem, + Name: "request_duration_seconds", + Help: "Accumulated common cache request process duration", +}, []string{hitLabel, methodLabel, cacheLabel}) + +type CacheMetrics struct { + cache string +} + +// NewCacheMetrics returns new CacheMetrics instance for cache specified. +func NewCacheMetrics(cache string) *CacheMetrics { + return &CacheMetrics{ + cache: cache, + } +} + +func (m *CacheMetrics) AddMethodDuration(method string, d time.Duration, hit bool) { + cacheRequests.With(prometheus.Labels{ + hitLabel: strconv.FormatBool(hit), + methodLabel: method, + cacheLabel: m.cache, + }).Observe(d.Seconds()) +} diff --git a/internal/metrics/consts.go b/internal/metrics/consts.go index f3b83a2e5..3aa51c0f0 100644 --- a/internal/metrics/consts.go +++ b/internal/metrics/consts.go @@ -21,6 +21,7 @@ const ( writeCacheSubsystem = "writecache" grpcServerSubsystem = "grpc_server" policerSubsystem = "policer" + commonCacheSubsystem = "common_cache" successLabel = "success" shardIDLabel = "shard_id" @@ -38,6 +39,8 @@ const ( storageLabel = "storage" operationLabel = "operation" endpointLabel = "endpoint" + hitLabel = "hit" + cacheLabel = "cache" readWriteMode = "READ_WRITE" readOnlyMode = "READ_ONLY" -- 2.45.2