From 69227b48454a5b68057ea0b12f3797c8762cc98d Mon Sep 17 00:00:00 2001
From: Marina Biryukova <m.biryukova@yadro.com>
Date: Tue, 29 Aug 2023 18:17:56 +0300
Subject: [PATCH] [#199] Add metrics for HTTP endpoint status

Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>
---
 cmd/s3-gw/app.go |  3 +++
 metrics/app.go   | 16 ++++++++++++++++
 metrics/desc.go  | 10 ++++++++++
 metrics/gate.go  | 26 ++++++++++++++++----------
 metrics/http.go  | 34 ++++++++++++++++++++++++++++++++++
 5 files changed, 79 insertions(+), 10 deletions(-)
 create mode 100644 metrics/http.go

diff --git a/cmd/s3-gw/app.go b/cmd/s3-gw/app.go
index 99735c9..a84ae16 100644
--- a/cmd/s3-gw/app.go
+++ b/cmd/s3-gw/app.go
@@ -455,6 +455,7 @@ func (a *App) Serve(ctx context.Context) {
 			a.log.Info(logs.StartingServer, zap.String("address", a.servers[i].Address()))
 
 			if err := srv.Serve(a.servers[i].Listener()); err != nil && err != http.ErrServerClosed {
+				a.metrics.MarkUnhealthy(a.servers[i].Address())
 				a.log.Fatal(logs.ListenAndServe, zap.Error(err))
 			}
 		}(i)
@@ -560,9 +561,11 @@ func (a *App) initServers(ctx context.Context) {
 		}
 		srv, err := newServer(ctx, serverInfo)
 		if err != nil {
+			a.metrics.MarkUnhealthy(serverInfo.Address)
 			a.log.Warn(logs.FailedToAddServer, append(fields, zap.Error(err))...)
 			continue
 		}
+		a.metrics.MarkHealthy(serverInfo.Address)
 
 		a.servers = append(a.servers, srv)
 		a.log.Info(logs.AddServer, fields...)
diff --git a/metrics/app.go b/metrics/app.go
index 00b591d..f07a8c9 100644
--- a/metrics/app.go
+++ b/metrics/app.go
@@ -84,3 +84,19 @@ func (m *AppMetrics) Statistic() *APIStatMetrics {
 func (m *AppMetrics) Gather() ([]*dto.MetricFamily, error) {
 	return m.gate.Gather()
 }
+
+func (m *AppMetrics) MarkHealthy(endpoint string) {
+	if !m.isEnabled() {
+		return
+	}
+
+	m.gate.HTTPServer.MarkHealthy(endpoint)
+}
+
+func (m *AppMetrics) MarkUnhealthy(endpoint string) {
+	if !m.isEnabled() {
+		return
+	}
+
+	m.gate.HTTPServer.MarkUnhealthy(endpoint)
+}
diff --git a/metrics/desc.go b/metrics/desc.go
index 0a748ac..318f57a 100644
--- a/metrics/desc.go
+++ b/metrics/desc.go
@@ -134,6 +134,16 @@ var appMetricsDesc = map[string]map[string]Description{
 			VariableLabels: []string{"direction"},
 		},
 	},
+	serverSubsystem: {
+		httpHealthMetric: Description{
+			Type:           dto.MetricType_GAUGE,
+			Namespace:      namespace,
+			Subsystem:      serverSubsystem,
+			Name:           httpHealthMetric,
+			Help:           "HTTP Server endpoint health",
+			VariableLabels: []string{"endpoint"},
+		},
+	},
 }
 
 type Description struct {
diff --git a/metrics/gate.go b/metrics/gate.go
index 1104ab2..02c2329 100644
--- a/metrics/gate.go
+++ b/metrics/gate.go
@@ -16,11 +16,12 @@ type StatisticScraper interface {
 }
 
 type GateMetrics struct {
-	registry prometheus.Registerer
-	State    *StateMetrics
-	Pool     *poolMetricsCollector
-	Billing  *billingMetrics
-	Stats    *APIStatMetrics
+	registry   prometheus.Registerer
+	State      *StateMetrics
+	Pool       *poolMetricsCollector
+	Billing    *billingMetrics
+	Stats      *APIStatMetrics
+	HTTPServer *httpServerMetrics
 }
 
 func NewGateMetrics(scraper StatisticScraper) *GateMetrics {
@@ -38,12 +39,16 @@ func NewGateMetrics(scraper StatisticScraper) *GateMetrics {
 	statsMetric := newAPIStatMetrics()
 	registry.MustRegister(statsMetric)
 
+	serverMetric := newHTTPServerMetrics()
+	registry.MustRegister(serverMetric)
+
 	return &GateMetrics{
-		registry: registry,
-		State:    stateMetric,
-		Pool:     poolMetric,
-		Billing:  billingMetric,
-		Stats:    statsMetric,
+		registry:   registry,
+		State:      stateMetric,
+		Pool:       poolMetric,
+		Billing:    billingMetric,
+		Stats:      statsMetric,
+		HTTPServer: serverMetric,
 	}
 }
 
@@ -52,6 +57,7 @@ func (g *GateMetrics) Unregister() {
 	g.registry.Unregister(g.Pool)
 	g.Billing.Unregister()
 	g.registry.Unregister(g.Stats)
+	g.registry.Unregister(g.HTTPServer)
 }
 
 func (g *GateMetrics) Handler() http.Handler {
diff --git a/metrics/http.go b/metrics/http.go
new file mode 100644
index 0000000..986f5d8
--- /dev/null
+++ b/metrics/http.go
@@ -0,0 +1,34 @@
+package metrics
+
+import "github.com/prometheus/client_golang/prometheus"
+
+const (
+	serverSubsystem  = "server"
+	httpHealthMetric = "health"
+)
+
+type httpServerMetrics struct {
+	endpointHealth *prometheus.GaugeVec
+}
+
+func newHTTPServerMetrics() *httpServerMetrics {
+	return &httpServerMetrics{
+		endpointHealth: mustNewGaugeVec(appMetricsDesc[serverSubsystem][httpHealthMetric]),
+	}
+}
+
+func (m *httpServerMetrics) Collect(ch chan<- prometheus.Metric) {
+	m.endpointHealth.Collect(ch)
+}
+
+func (m *httpServerMetrics) Describe(desc chan<- *prometheus.Desc) {
+	m.endpointHealth.Describe(desc)
+}
+
+func (m *httpServerMetrics) MarkHealthy(endpoint string) {
+	m.endpointHealth.WithLabelValues(endpoint).Set(float64(1))
+}
+
+func (m *httpServerMetrics) MarkUnhealthy(endpoint string) {
+	m.endpointHealth.WithLabelValues(endpoint).Set(float64(0))
+}