diff --git a/cmd/s3-gw/app.go b/cmd/s3-gw/app.go index 99735c9c..a84ae16c 100644 --- a/cmd/s3-gw/app.go +++ b/cmd/s3-gw/app.go @@ -455,6 +455,7 @@ func (a *App) Serve(ctx context.Context) { a.log.Info(logs.StartingServer, zap.String("address", a.servers[i].Address())) if err := srv.Serve(a.servers[i].Listener()); err != nil && err != http.ErrServerClosed { + a.metrics.MarkUnhealthy(a.servers[i].Address()) a.log.Fatal(logs.ListenAndServe, zap.Error(err)) } }(i) @@ -560,9 +561,11 @@ func (a *App) initServers(ctx context.Context) { } srv, err := newServer(ctx, serverInfo) if err != nil { + a.metrics.MarkUnhealthy(serverInfo.Address) a.log.Warn(logs.FailedToAddServer, append(fields, zap.Error(err))...) continue } + a.metrics.MarkHealthy(serverInfo.Address) a.servers = append(a.servers, srv) a.log.Info(logs.AddServer, fields...) diff --git a/metrics/app.go b/metrics/app.go index 00b591da..f07a8c96 100644 --- a/metrics/app.go +++ b/metrics/app.go @@ -84,3 +84,19 @@ func (m *AppMetrics) Statistic() *APIStatMetrics { func (m *AppMetrics) Gather() ([]*dto.MetricFamily, error) { return m.gate.Gather() } + +func (m *AppMetrics) MarkHealthy(endpoint string) { + if !m.isEnabled() { + return + } + + m.gate.HTTPServer.MarkHealthy(endpoint) +} + +func (m *AppMetrics) MarkUnhealthy(endpoint string) { + if !m.isEnabled() { + return + } + + m.gate.HTTPServer.MarkUnhealthy(endpoint) +} diff --git a/metrics/desc.go b/metrics/desc.go index 0a748ac1..318f57ad 100644 --- a/metrics/desc.go +++ b/metrics/desc.go @@ -134,6 +134,16 @@ var appMetricsDesc = map[string]map[string]Description{ VariableLabels: []string{"direction"}, }, }, + serverSubsystem: { + httpHealthMetric: Description{ + Type: dto.MetricType_GAUGE, + Namespace: namespace, + Subsystem: serverSubsystem, + Name: httpHealthMetric, + Help: "HTTP Server endpoint health", + VariableLabels: []string{"endpoint"}, + }, + }, } type Description struct { diff --git a/metrics/gate.go b/metrics/gate.go index 1104ab22..02c23293 100644 --- a/metrics/gate.go +++ b/metrics/gate.go @@ -16,11 +16,12 @@ type StatisticScraper interface { } type GateMetrics struct { - registry prometheus.Registerer - State *StateMetrics - Pool *poolMetricsCollector - Billing *billingMetrics - Stats *APIStatMetrics + registry prometheus.Registerer + State *StateMetrics + Pool *poolMetricsCollector + Billing *billingMetrics + Stats *APIStatMetrics + HTTPServer *httpServerMetrics } func NewGateMetrics(scraper StatisticScraper) *GateMetrics { @@ -38,12 +39,16 @@ func NewGateMetrics(scraper StatisticScraper) *GateMetrics { statsMetric := newAPIStatMetrics() registry.MustRegister(statsMetric) + serverMetric := newHTTPServerMetrics() + registry.MustRegister(serverMetric) + return &GateMetrics{ - registry: registry, - State: stateMetric, - Pool: poolMetric, - Billing: billingMetric, - Stats: statsMetric, + registry: registry, + State: stateMetric, + Pool: poolMetric, + Billing: billingMetric, + Stats: statsMetric, + HTTPServer: serverMetric, } } @@ -52,6 +57,7 @@ func (g *GateMetrics) Unregister() { g.registry.Unregister(g.Pool) g.Billing.Unregister() g.registry.Unregister(g.Stats) + g.registry.Unregister(g.HTTPServer) } func (g *GateMetrics) Handler() http.Handler { diff --git a/metrics/http.go b/metrics/http.go new file mode 100644 index 00000000..986f5d84 --- /dev/null +++ b/metrics/http.go @@ -0,0 +1,34 @@ +package metrics + +import "github.com/prometheus/client_golang/prometheus" + +const ( + serverSubsystem = "server" + httpHealthMetric = "health" +) + +type httpServerMetrics struct { + endpointHealth *prometheus.GaugeVec +} + +func newHTTPServerMetrics() *httpServerMetrics { + return &httpServerMetrics{ + endpointHealth: mustNewGaugeVec(appMetricsDesc[serverSubsystem][httpHealthMetric]), + } +} + +func (m *httpServerMetrics) Collect(ch chan<- prometheus.Metric) { + m.endpointHealth.Collect(ch) +} + +func (m *httpServerMetrics) Describe(desc chan<- *prometheus.Desc) { + m.endpointHealth.Describe(desc) +} + +func (m *httpServerMetrics) MarkHealthy(endpoint string) { + m.endpointHealth.WithLabelValues(endpoint).Set(float64(1)) +} + +func (m *httpServerMetrics) MarkUnhealthy(endpoint string) { + m.endpointHealth.WithLabelValues(endpoint).Set(float64(0)) +}