diff --git a/cmd/frostfs-node/metrics.go b/cmd/frostfs-node/metrics.go index f18bbe65d..cf621086d 100644 --- a/cmd/frostfs-node/metrics.go +++ b/cmd/frostfs-node/metrics.go @@ -2,7 +2,7 @@ package main import ( metricsconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/metrics" - "github.com/prometheus/client_golang/prometheus/promhttp" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" ) func metricsComponent(c *cfg) (*httpComponent, bool) { @@ -12,7 +12,7 @@ func metricsComponent(c *cfg) (*httpComponent, bool) { c.dynamicConfiguration.metrics = new(httpComponent) c.dynamicConfiguration.metrics.cfg = c c.dynamicConfiguration.metrics.name = "metrics" - c.dynamicConfiguration.metrics.handler = promhttp.Handler() + c.dynamicConfiguration.metrics.handler = metrics.Handler() updated = true } diff --git a/pkg/metrics/engine.go b/pkg/metrics/engine.go index 4c51a55aa..73606fd23 100644 --- a/pkg/metrics/engine.go +++ b/pkg/metrics/engine.go @@ -72,19 +72,19 @@ func newEngineGaugeVector(name, help string, labels []string) *prometheus.GaugeV } func (m engineMetrics) register() { - prometheus.MustRegister(m.listContainersDuration) - prometheus.MustRegister(m.estimateContainerSizeDuration) - prometheus.MustRegister(m.deleteDuration) - prometheus.MustRegister(m.existsDuration) - prometheus.MustRegister(m.getDuration) - prometheus.MustRegister(m.headDuration) - prometheus.MustRegister(m.inhumeDuration) - prometheus.MustRegister(m.putDuration) - prometheus.MustRegister(m.rangeDuration) - prometheus.MustRegister(m.searchDuration) - prometheus.MustRegister(m.listObjectsDuration) - prometheus.MustRegister(m.containerSize) - prometheus.MustRegister(m.payloadSize) + mustRegister(m.listContainersDuration) + mustRegister(m.estimateContainerSizeDuration) + mustRegister(m.deleteDuration) + mustRegister(m.existsDuration) + mustRegister(m.getDuration) + mustRegister(m.headDuration) + mustRegister(m.inhumeDuration) + mustRegister(m.putDuration) + mustRegister(m.rangeDuration) + mustRegister(m.searchDuration) + mustRegister(m.listObjectsDuration) + mustRegister(m.containerSize) + mustRegister(m.payloadSize) } func (m engineMetrics) AddListContainersDuration(d time.Duration) { diff --git a/pkg/metrics/innerring.go b/pkg/metrics/innerring.go index 55b0aa089..4171b7423 100644 --- a/pkg/metrics/innerring.go +++ b/pkg/metrics/innerring.go @@ -27,8 +27,8 @@ func NewInnerRingMetrics() InnerRingServiceMetrics { }) ) - prometheus.MustRegister(epoch) - prometheus.MustRegister(health) + mustRegister(epoch) + mustRegister(health) return InnerRingServiceMetrics{ epoch: epoch, diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 640da2d5f..4ebd2afa1 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -27,7 +27,7 @@ func NewNodeMetrics() *NodeMetrics { Name: "epoch", Help: "Current epoch as seen by inner-ring node.", }) - prometheus.MustRegister(epoch) + mustRegister(epoch) return &NodeMetrics{ objectServiceMetrics: objectService, diff --git a/pkg/metrics/object.go b/pkg/metrics/object.go index fae86cb4a..ccc7c129c 100644 --- a/pkg/metrics/object.go +++ b/pkg/metrics/object.go @@ -65,8 +65,8 @@ func newObjectMethodCallCounter(name string) methodCount { } func (m methodCount) mustRegister() { - prometheus.MustRegister(m.success) - prometheus.MustRegister(m.total) + mustRegister(m.success) + mustRegister(m.total) } func (m methodCount) Inc(success bool) { @@ -135,19 +135,19 @@ func (m objectServiceMetrics) register() { m.rangeCounter.mustRegister() m.rangeHashCounter.mustRegister() - prometheus.MustRegister(m.getDuration) - prometheus.MustRegister(m.putDuration) - prometheus.MustRegister(m.headDuration) - prometheus.MustRegister(m.searchDuration) - prometheus.MustRegister(m.deleteDuration) - prometheus.MustRegister(m.rangeDuration) - prometheus.MustRegister(m.rangeHashDuration) + mustRegister(m.getDuration) + mustRegister(m.putDuration) + mustRegister(m.headDuration) + mustRegister(m.searchDuration) + mustRegister(m.deleteDuration) + mustRegister(m.rangeDuration) + mustRegister(m.rangeHashDuration) - prometheus.MustRegister(m.putPayload) - prometheus.MustRegister(m.getPayload) + mustRegister(m.putPayload) + mustRegister(m.getPayload) - prometheus.MustRegister(m.shardMetrics) - prometheus.MustRegister(m.shardsReadonly) + mustRegister(m.shardMetrics) + mustRegister(m.shardsReadonly) } func (m objectServiceMetrics) IncGetReqCounter(success bool) { diff --git a/pkg/metrics/registry.go b/pkg/metrics/registry.go new file mode 100644 index 000000000..c21c80449 --- /dev/null +++ b/pkg/metrics/registry.go @@ -0,0 +1,40 @@ +package metrics + +import ( + "net/http" + "sync" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +// Handler returns an http.Handler for the local registry. +func Handler() http.Handler { + promhttp.Handler() + return promhttp.InstrumentMetricHandler( + registry, + promhttp.HandlerFor(registry, promhttp.HandlerOpts{})) +} + +var ( + registry = prometheus.NewRegistry() + // registeredCollectorsMtx protects collectors slice. + // It should not be acessed concurrently, but we can easily forget this in future, thus this mutex. + registeredCollectorsMtx sync.Mutex + registeredCollectors []prometheus.Collector +) + +func init() { + mustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})) + mustRegister(collectors.NewGoCollector()) +} + +func mustRegister(cs ...prometheus.Collector) { + for i := range cs { + registry.MustRegister(cs[i]) + } + registeredCollectorsMtx.Lock() + registeredCollectors = append(registeredCollectors, cs...) + registeredCollectorsMtx.Unlock() +} diff --git a/pkg/metrics/state.go b/pkg/metrics/state.go index 94e28af38..9c96743ff 100644 --- a/pkg/metrics/state.go +++ b/pkg/metrics/state.go @@ -20,7 +20,7 @@ func newStateMetrics() stateMetrics { } func (m stateMetrics) register() { - prometheus.MustRegister(m.healthCheck) + mustRegister(m.healthCheck) } func (m stateMetrics) SetHealth(s int32) {