From 8b706168467e9352e0c567633c3c264a340de79d Mon Sep 17 00:00:00 2001
From: tifayuki <tifayuki@gmail.com>
Date: Tue, 13 Feb 2018 13:30:56 -0800
Subject: [PATCH] Add notification metrics

It adds notification related prometheus metrics, including:
  - total count for events/success/failure/error
  - total count for notification per each status code
  - gauge of the pending notification queue

Signed-off-by: tifayuki <tifayuki@gmail.com>
---
 metrics/prometheus.go    |  3 +++
 notifications/metrics.go | 28 ++++++++++++++++++++++++++++
 2 files changed, 31 insertions(+)

diff --git a/metrics/prometheus.go b/metrics/prometheus.go
index b5a53214..91b32b23 100644
--- a/metrics/prometheus.go
+++ b/metrics/prometheus.go
@@ -10,4 +10,7 @@ const (
 var (
 	// StorageNamespace is the prometheus namespace of blob/cache related operations
 	StorageNamespace = metrics.NewNamespace(NamespacePrefix, "storage", nil)
+
+	// NotificationsNamespace is the prometheus namespace of notification related metrics
+	NotificationsNamespace = metrics.NewNamespace(NamespacePrefix, "notifications", nil)
 )
diff --git a/notifications/metrics.go b/notifications/metrics.go
index a20af168..69960e9c 100644
--- a/notifications/metrics.go
+++ b/notifications/metrics.go
@@ -5,6 +5,18 @@ import (
 	"fmt"
 	"net/http"
 	"sync"
+
+	prometheus "github.com/docker/distribution/metrics"
+	"github.com/docker/go-metrics"
+)
+
+var (
+	// eventsCounter counts total events of incoming, success, failure, and errors
+	eventsCounter = prometheus.NotificationsNamespace.NewLabeledCounter("events", "The number of total events", "type")
+	// pendingGauge measures the pending queue size
+	pendingGauge = prometheus.NotificationsNamespace.NewGauge("pending", "The gauge of pending events in queue", metrics.Total)
+	// statusCounter counts the total notification call per each status code
+	statusCounter = prometheus.NotificationsNamespace.NewLabeledCounter("status", "The number of status code", "code")
 )
 
 // EndpointMetrics track various actions taken by the endpoint, typically by
@@ -61,6 +73,9 @@ func (emsl *endpointMetricsHTTPStatusListener) success(status int, events ...Eve
 	defer emsl.safeMetrics.Unlock()
 	emsl.Statuses[fmt.Sprintf("%d %s", status, http.StatusText(status))] += len(events)
 	emsl.Successes += len(events)
+
+	statusCounter.WithValues(fmt.Sprintf("%d %s", status, http.StatusText(status))).Inc(1)
+	eventsCounter.WithValues("Successes").Inc(1)
 }
 
 func (emsl *endpointMetricsHTTPStatusListener) failure(status int, events ...Event) {
@@ -68,12 +83,17 @@ func (emsl *endpointMetricsHTTPStatusListener) failure(status int, events ...Eve
 	defer emsl.safeMetrics.Unlock()
 	emsl.Statuses[fmt.Sprintf("%d %s", status, http.StatusText(status))] += len(events)
 	emsl.Failures += len(events)
+
+	statusCounter.WithValues(fmt.Sprintf("%d %s", status, http.StatusText(status))).Inc(1)
+	eventsCounter.WithValues("Failures").Inc(1)
 }
 
 func (emsl *endpointMetricsHTTPStatusListener) err(err error, events ...Event) {
 	emsl.safeMetrics.Lock()
 	defer emsl.safeMetrics.Unlock()
 	emsl.Errors += len(events)
+
+	eventsCounter.WithValues("Errors").Inc(1)
 }
 
 // endpointMetricsEventQueueListener maintains the incoming events counter and
@@ -87,12 +107,17 @@ func (eqc *endpointMetricsEventQueueListener) ingress(events ...Event) {
 	defer eqc.Unlock()
 	eqc.Events += len(events)
 	eqc.Pending += len(events)
+
+	eventsCounter.WithValues("Events").Inc()
+	pendingGauge.Inc(1)
 }
 
 func (eqc *endpointMetricsEventQueueListener) egress(events ...Event) {
 	eqc.Lock()
 	defer eqc.Unlock()
 	eqc.Pending -= len(events)
+
+	pendingGauge.Dec(1)
 }
 
 // endpoints is global registry of endpoints used to report metrics to expvar
@@ -149,4 +174,7 @@ func init() {
 	}))
 
 	registry.(*expvar.Map).Set("notifications", &notifications)
+
+	// register prometheus metrics
+	metrics.Register(prometheus.NotificationsNamespace)
 }