rcserver: implement prometheus metrics on a dedicated port - fixes #7940

cmd/cmd.go

@@ -421,11 +421,18 @@ func initConfig() {
 	}
 
 	// Start the remote control server if configured
-	_, err = rcserver.Start(context.Background(), &rc.Opt)
+	_, err = rcserver.Start(ctx, &rc.Opt)
 	if err != nil {
 		log.Fatalf("Failed to start remote control: %v", err)
 	}
 
+	// Start the metrics server if configured
+	_, err = rcserver.MetricsStart(ctx, &rc.Opt)
+	if err != nil {
+		log.Fatalf("Failed to start metrics server: %v", err)
+
+	}
+
 	// Setup CPU profiling if desired
 	if *cpuProfile != "" {
 		fs.Infof(nil, "Creating CPU profile %q\n", *cpuProfile)

docs/content/docs.md

@@ -2836,6 +2836,17 @@ Rclone prefixes all log messages with their level in capitals, e.g. INFO
 which makes it easy to grep the log file for different kinds of
 information.
 
+Metrics
+-------
+
+Rclone can publish metrics in the OpenMetrics/Prometheus format.
+
+To enable the metrics endpoint, use the `--metrics-addr` flag. Metrics can also be published on the `--rc-addr` port if the `--rc` flag and `--rc-enable-metrics` flags are supplied or if using rclone rcd `--rc-enable-metrics`
+
+Rclone provides extensive configuration options for the metrics HTTP endpoint. These settings are grouped under the Metrics section and have a prefix `--metrics-*`.
+
+When metrics are enabled with `--rc-enable-metrics`, they will be published on the same port as the rc API. In this case, the `--metrics-*` flags will be ignored, and the HTTP endpoint configuration will be managed by the `--rc-*` parameters.
+
 Exit Code
 ---------
 

docs/content/rc.md

@@ -100,6 +100,7 @@ Default Off.
 ### --rc-enable-metrics
 
 Enable OpenMetrics/Prometheus compatible endpoint at `/metrics`.
+If more control over the metrics is desired (for example running it on a different port or with different auth) then endpoint can be enabled with the `--metrics-*` flags instead.
 
 Default Off.
 

fs/config/flags/flags.go

@@ -124,6 +124,7 @@ func init() {
 	All.NewGroup("Logging", "Flags for logging and statistics")
 	All.NewGroup("Metadata", "Flags to control metadata")
 	All.NewGroup("RC", "Flags to control the Remote Control API")
+	All.NewGroup("Metrics", "Flags to control the Metrics HTTP endpoint.")
 }
 
 // installFlag constructs a name from the flag passed in and

fs/rc/rc.go

@@ -71,8 +71,8 @@ var OptionsInfo = fs.Options{{
 }, {
 	Name:    "rc_enable_metrics",
 	Default: false,
-	Help:    "Enable prometheus metrics on /metrics",
+	Help:    "Enable the Prometheus metrics path at the remote control server",
-	Groups:  "RC",
+	Groups:  "RC,Metrics",
 }, {
 	Name:    "rc_job_expire_duration",
 	Default: 60 * time.Second,
@@ -83,10 +83,18 @@ var OptionsInfo = fs.Options{{
 	Default: 10 * time.Second,
 	Help:    "Interval to check for expired async jobs",
 	Groups:  "RC",
+}, {
+	Name:    "metrics_addr",
+	Default: []string{""},
+	Help:    "IPaddress:Port or :Port to bind metrics server to",
+	Groups:  "Metrics",
 }}.
 	AddPrefix(libhttp.ConfigInfo, "rc", "RC").
 	AddPrefix(libhttp.AuthConfigInfo, "rc", "RC").
 	AddPrefix(libhttp.TemplateConfigInfo, "rc", "RC").
+	AddPrefix(libhttp.ConfigInfo, "metrics", "Metrics").
+	AddPrefix(libhttp.AuthConfigInfo, "metrics", "Metrics").
+	AddPrefix(libhttp.TemplateConfigInfo, "metrics", "Metrics").
 	SetDefault("rc_addr", []string{"localhost:5572"})
 
 func init() {
@@ -109,6 +117,9 @@ type Options struct {
 	WebGUINoOpenBrowser bool                   `config:"rc_web_gui_no_open_browser"` // set to disable auto opening browser
 	WebGUIFetchURL      string                 `config:"rc_web_fetch_url"`           // set the default url for fetching webgui
 	EnableMetrics       bool                   `config:"rc_enable_metrics"`          // set to disable prometheus metrics on /metrics
+	MetricsHTTP         libhttp.Config         `config:"metrics"`
+	MetricsAuth         libhttp.AuthConfig     `config:"metrics"`
+	MetricsTemplate     libhttp.TemplateConfig `config:"metrics"`
 	JobExpireDuration   time.Duration          `config:"rc_job_expire_duration"`
 	JobExpireInterval   time.Duration          `config:"rc_job_expire_interval"`
 }

fs/rc/rcserver/metrics.go

@@ -0,0 +1,97 @@
+// Package rcserver implements the HTTP endpoint to serve the remote control
+package rcserver
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/rclone/rclone/fs/accounting"
+	"github.com/rclone/rclone/fs/fshttp"
+	"github.com/rclone/rclone/fs/rc"
+	"github.com/rclone/rclone/fs/rc/jobs"
+	libhttp "github.com/rclone/rclone/lib/http"
+)
+
+const path = "/metrics"
+
+var promHandlerFunc http.HandlerFunc
+
+func init() {
+	rcloneCollector := accounting.NewRcloneCollector(context.Background())
+	prometheus.MustRegister(rcloneCollector)
+
+	m := fshttp.NewMetrics("rclone")
+	for _, c := range m.Collectors() {
+		prometheus.MustRegister(c)
+	}
+	fshttp.DefaultMetrics = m
+
+	promHandlerFunc = promhttp.Handler().ServeHTTP
+}
+
+// MetricsStart the remote control server if configured
+//
+// If the server wasn't configured the *Server returned may be nil
+func MetricsStart(ctx context.Context, opt *rc.Options) (*MetricsServer, error) {
+	jobs.SetOpt(opt) // set the defaults for jobs
+	if opt.MetricsHTTP.ListenAddr[0] != "" {
+		// Serve on the DefaultServeMux so can have global registrations appear
+		s, err := newMetricsServer(ctx, opt)
+		if err != nil {
+			return nil, err
+		}
+		return s, s.Serve()
+	}
+	return nil, nil
+}
+
+// MetricsServer contains everything to run the rc server
+type MetricsServer struct {
+	ctx             context.Context // for global config
+	server          *libhttp.Server
+	promHandlerFunc http.Handler
+	opt             *rc.Options
+}
+
+func newMetricsServer(ctx context.Context, opt *rc.Options) (*MetricsServer, error) {
+	s := &MetricsServer{
+		ctx:             ctx,
+		opt:             opt,
+		promHandlerFunc: promHandlerFunc,
+	}
+
+	var err error
+	s.server, err = libhttp.NewServer(ctx,
+		libhttp.WithConfig(opt.MetricsHTTP),
+		libhttp.WithAuth(opt.MetricsAuth),
+		libhttp.WithTemplate(opt.MetricsTemplate),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to init server: %w", err)
+	}
+
+	router := s.server.Router()
+	router.Get(path, promHandlerFunc)
+	return s, nil
+}
+
+// Serve runs the http server in the background.
+//
+// Use s.Close() and s.Wait() to shutdown server
+func (s *MetricsServer) Serve() error {
+	s.server.Serve()
+	return nil
+}
+
+// Wait blocks while the server is serving requests
+func (s *MetricsServer) Wait() {
+	s.server.Wait()
+}
+
+// Shutdown gracefully shuts down the server
+func (s *MetricsServer) Shutdown() error {
+	return s.server.Shutdown()
+}

fs/rc/rcserver/metrics_test.go

@@ -0,0 +1,88 @@
+package rcserver
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"regexp"
+	"testing"
+
+	_ "github.com/rclone/rclone/backend/local"
+	"github.com/rclone/rclone/fs/accounting"
+	"github.com/rclone/rclone/fs/config/configfile"
+	"github.com/rclone/rclone/fs/rc"
+	"github.com/stretchr/testify/require"
+)
+
+// Run a suite of tests
+func testMetricsServer(t *testing.T, tests []testRun, opt *rc.Options) {
+	t.Helper()
+	ctx := context.Background()
+	configfile.Install()
+	rcServer, err := newMetricsServer(ctx, opt)
+	require.NoError(t, err)
+	testURL := rcServer.server.URLs()[0]
+	mux := rcServer.server.Router()
+	emulateCalls(t, tests, mux, testURL)
+}
+
+// return an enabled rc
+func newMetricsTestOpt() rc.Options {
+	opt := rc.Opt
+	opt.MetricsHTTP.ListenAddr = []string{testBindAddress}
+	return opt
+}
+
+func TestMetrics(t *testing.T) {
+	stats := accounting.GlobalStats()
+	tests := makeMetricsTestCases(stats)
+	opt := newMetricsTestOpt()
+	testMetricsServer(t, tests, &opt)
+
+	// Test changing a couple options
+	stats.Bytes(500)
+	for i := 0; i < 30; i++ {
+		require.NoError(t, stats.DeleteFile(context.Background(), 0))
+	}
+	stats.Errors(2)
+	stats.Bytes(324)
+
+	tests = makeMetricsTestCases(stats)
+	testMetricsServer(t, tests, &opt)
+}
+
+func makeMetricsTestCases(stats *accounting.StatsInfo) (tests []testRun) {
+	tests = []testRun{{
+		Name:     "Bytes Transferred Metric",
+		URL:      "metrics",
+		Method:   "GET",
+		Status:   http.StatusOK,
+		Contains: regexp.MustCompile(fmt.Sprintf("rclone_bytes_transferred_total %d", stats.GetBytes())),
+	}, {
+		Name:     "Checked Files Metric",
+		URL:      "metrics",
+		Method:   "GET",
+		Status:   http.StatusOK,
+		Contains: regexp.MustCompile(fmt.Sprintf("rclone_checked_files_total %d", stats.GetChecks())),
+	}, {
+		Name:     "Errors Metric",
+		URL:      "metrics",
+		Method:   "GET",
+		Status:   http.StatusOK,
+		Contains: regexp.MustCompile(fmt.Sprintf("rclone_errors_total %d", stats.GetErrors())),
+	}, {
+		Name:     "Deleted Files Metric",
+		URL:      "metrics",
+		Method:   "GET",
+		Status:   http.StatusOK,
+		Contains: regexp.MustCompile(fmt.Sprintf("rclone_files_deleted_total %d", stats.GetDeletes())),
+	}, {
+		Name:     "Files Transferred Metric",
+		URL:      "metrics",
+		Method:   "GET",
+		Status:   http.StatusOK,
+		Contains: regexp.MustCompile(fmt.Sprintf("rclone_files_transferred_total %d", stats.GetTransfers())),
+	},
+	}
+	return
+}

fs/rc/rcserver/rcserver.go

@@ -18,13 +18,9 @@ import (
 	"time"
 
 	"github.com/go-chi/chi/v5/middleware"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/cache"
 	"github.com/rclone/rclone/fs/config"
-	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/list"
 	"github.com/rclone/rclone/fs/rc"
 	"github.com/rclone/rclone/fs/rc/jobs"
@@ -35,21 +31,6 @@ import (
 	"github.com/skratchdot/open-golang/open"
 )
 
-var promHandler http.Handler
-
-func init() {
-	rcloneCollector := accounting.NewRcloneCollector(context.Background())
-	prometheus.MustRegister(rcloneCollector)
-
-	m := fshttp.NewMetrics("rclone")
-	for _, c := range m.Collectors() {
-		prometheus.MustRegister(c)
-	}
-	fshttp.DefaultMetrics = m
-
-	promHandler = promhttp.Handler()
-}
-
 // Start the remote control server if configured
 //
 // If the server wasn't configured the *Server returned may be nil
@@ -376,7 +357,7 @@ func (s *Server) handleGet(w http.ResponseWriter, r *http.Request, path string)
 		s.serveRemote(w, r, fsMatchResult[2], fsMatchResult[1])
 		return
 	case path == "metrics" && s.opt.EnableMetrics:
-		promHandler.ServeHTTP(w, r)
+		promHandlerFunc(w, r)
 		return
 	case path == "*" && s.opt.Serve:
 		// Serve /* as the remote listing

fs/rc/rcserver/rcserver_test.go

@@ -15,9 +15,10 @@ import (
 	"testing"
 	"time"
 
+	"github.com/go-chi/chi/v5"
+
 	_ "github.com/rclone/rclone/backend/local"
 	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/config/configfile"
 	"github.com/rclone/rclone/fs/rc"
 	"github.com/stretchr/testify/assert"
@@ -115,6 +116,10 @@ func testServer(t *testing.T, tests []testRun, opt *rc.Options) {
 	require.NoError(t, err)
 	testURL := rcServer.server.URLs()[0]
 	mux := rcServer.server.Router()
+	emulateCalls(t, tests, mux, testURL)
+}
+
+func emulateCalls(t *testing.T, tests []testRun, mux chi.Router, testURL string) {
 	for _, test := range tests {
 		t.Run(test.Name, func(t *testing.T) {
 			t.Helper()
@@ -568,61 +573,6 @@ Unknown command
 	testServer(t, tests, &opt)
 }
 
-func TestMetrics(t *testing.T) {
-	stats := accounting.GlobalStats()
-	tests := makeMetricsTestCases(stats)
-	opt := newTestOpt()
-	opt.EnableMetrics = true
-	testServer(t, tests, &opt)
-
-	// Test changing a couple options
-	stats.Bytes(500)
-	for i := 0; i < 30; i++ {
-		require.NoError(t, stats.DeleteFile(context.Background(), 0))
-	}
-	stats.Errors(2)
-	stats.Bytes(324)
-
-	tests = makeMetricsTestCases(stats)
-	testServer(t, tests, &opt)
-}
-
-func makeMetricsTestCases(stats *accounting.StatsInfo) (tests []testRun) {
-	tests = []testRun{{
-		Name:     "Bytes Transferred Metric",
-		URL:      "/metrics",
-		Method:   "GET",
-		Status:   http.StatusOK,
-		Contains: regexp.MustCompile(fmt.Sprintf("rclone_bytes_transferred_total %d", stats.GetBytes())),
-	}, {
-		Name:     "Checked Files Metric",
-		URL:      "/metrics",
-		Method:   "GET",
-		Status:   http.StatusOK,
-		Contains: regexp.MustCompile(fmt.Sprintf("rclone_checked_files_total %d", stats.GetChecks())),
-	}, {
-		Name:     "Errors Metric",
-		URL:      "/metrics",
-		Method:   "GET",
-		Status:   http.StatusOK,
-		Contains: regexp.MustCompile(fmt.Sprintf("rclone_errors_total %d", stats.GetErrors())),
-	}, {
-		Name:     "Deleted Files Metric",
-		URL:      "/metrics",
-		Method:   "GET",
-		Status:   http.StatusOK,
-		Contains: regexp.MustCompile(fmt.Sprintf("rclone_files_deleted_total %d", stats.GetDeletes())),
-	}, {
-		Name:     "Files Transferred Metric",
-		URL:      "/metrics",
-		Method:   "GET",
-		Status:   http.StatusOK,
-		Contains: regexp.MustCompile(fmt.Sprintf("rclone_files_transferred_total %d", stats.GetTransfers())),
-	},
-	}
-	return
-}
-
 var matchRemoteDirListing = regexp.MustCompile(`<title>Directory listing of /</title>`)
 
 func TestServingRoot(t *testing.T) {