Merge pull request #536 from nspcc-dev/feature/pprof

network: add Pprof metrics

README.md

@@ -85,6 +85,19 @@ Available network flags:
 - `--privnet, -p`
 - `--testnet, -t`
 
+#Developer notes
+Nodes have such features as [Prometheus](https://prometheus.io/docs/guides/go-application) and 
+[Pprof](https://golang.org/pkg/net/http/pprof/) in order to have additional information about them for debugging.
+
+How to configure Prometheus or Pprof:
+In `config/protocol.*.yml` there is 
+```
+  Prometheus:
+    Enabled: true
+    Port: 2112
+```
+where you can switch on/off and define port. Prometheus is enabled and Pprof is disabled by default.
+
 # Contributing
 
 Feel free to contribute to this project after reading the

cli/server/server.go

@@ -134,6 +134,22 @@ func getCountAndSkipFromContext(ctx *cli.Context) (uint32, uint32) {
 	return count, skip
 }
 
+func initBCWithMetrics(cfg config.Config) (*core.Blockchain, *metrics.Service, *metrics.Service, error) {
+	chain, err := initBlockChain(cfg)
+	if err != nil {
+		return nil, nil, nil, cli.NewExitError(err, 1)
+	}
+	configureAddresses(cfg.ApplicationConfiguration)
+	prometheus := metrics.NewPrometheusService(cfg.ApplicationConfiguration.Prometheus)
+	pprof := metrics.NewPprofService(cfg.ApplicationConfiguration.Pprof)
+
+	go chain.Run()
+	go prometheus.Start()
+	go pprof.Start()
+
+	return chain, prometheus, pprof, nil
+}
+
 func dumpDB(ctx *cli.Context) error {
 	cfg, err := getConfigFromContext(ctx)
 	if err != nil {
@@ -154,11 +170,10 @@ func dumpDB(ctx *cli.Context) error {
 	defer outStream.Close()
 	writer := io.NewBinWriterFromIO(outStream)
 
-	chain, err := initBlockChain(cfg)
+	chain, prometheus, pprof, err := initBCWithMetrics(cfg)
 	if err != nil {
-		return cli.NewExitError(err, 1)
+		return err
 	}
-	go chain.Run()
 
 	chainHeight := chain.BlockHeight()
 	if skip+count > chainHeight {
@@ -182,9 +197,12 @@ func dumpDB(ctx *cli.Context) error {
 			return cli.NewExitError(err, 1)
 		}
 	}
+	pprof.ShutDown()
+	prometheus.ShutDown()
 	chain.Close()
 	return nil
 }
+
 func restoreDB(ctx *cli.Context) error {
 	cfg, err := getConfigFromContext(ctx)
 	if err != nil {
@@ -205,11 +223,10 @@ func restoreDB(ctx *cli.Context) error {
 	defer inStream.Close()
 	reader := io.NewBinReaderFromIO(inStream)
 
-	chain, err := initBlockChain(cfg)
+	chain, prometheus, pprof, err := initBCWithMetrics(cfg)
 	if err != nil {
 		return err
 	}
-	go chain.Run()
 
 	var allBlocks = reader.ReadU32LE()
 	if reader.Err != nil {
@@ -241,8 +258,9 @@ func restoreDB(ctx *cli.Context) error {
 			return cli.NewExitError(fmt.Errorf("failed to add block %d: %s", i, err), 1)
 		}
 	}
+	pprof.ShutDown()
+	prometheus.ShutDown()
 	chain.Close()
-
 	return nil
 }
 
@@ -271,21 +289,17 @@ func startServer(ctx *cli.Context) error {
 
 	serverConfig := network.NewServerConfig(cfg)
 
-	chain, err := initBlockChain(cfg)
+	chain, prometheus, pprof, err := initBCWithMetrics(cfg)
 	if err != nil {
 		return err
 	}
 
-	configureAddresses(cfg.ApplicationConfiguration)
 	server := network.NewServer(serverConfig, chain)
 	rpcServer := rpc.NewServer(chain, cfg.ApplicationConfiguration.RPC, server)
 	errChan := make(chan error)
-	monitoring := metrics.NewMetricsService(cfg.ApplicationConfiguration.Monitoring)
 
-	go chain.Run()
 	go server.Start(errChan)
 	go rpcServer.Start(errChan)
-	go monitoring.Start()
 
 	fmt.Println(logo())
 	fmt.Println(server.UserAgent)
@@ -304,7 +318,8 @@ Main:
 			if serverErr := rpcServer.Shutdown(); serverErr != nil {
 				shutdownErr = errors.Wrap(serverErr, "Error encountered whilst shutting down server")
 			}
-			monitoring.ShutDown()
+			prometheus.ShutDown()
+			pprof.ShutDown()
 			chain.Close()
 			break Main
 		}
@@ -317,17 +332,20 @@ Main:
 	return nil
 }
 
-// configureAddresses sets up addresses for RPC and Monitoring depending from the provided config.
+// configureAddresses sets up addresses for RPC, Prometheus and Pprof depending from the provided config.
-// In case RPC or Monitoring Address provided each of them will use it.
+// In case RPC or Prometheus or Pprof Address provided each of them will use it.
-// In case global Address (of the node) provided and RPC/Monitoring don't have configured addresses they will
+// In case global Address (of the node) provided and RPC/Prometheus/Pprof don't have configured addresses they will
-// use global one. So Node and RPC and Monitoring will run on one address.
+// use global one. So Node and RPC and Prometheus and Pprof will run on one address.
 func configureAddresses(cfg config.ApplicationConfiguration) {
 	if cfg.Address != "" {
 		if cfg.RPC.Address == "" {
 			cfg.RPC.Address = cfg.Address
 		}
-		if cfg.Monitoring.Address == "" {
+		if cfg.Prometheus.Address == "" {
-			cfg.Monitoring.Address = cfg.Address
+			cfg.Prometheus.Address = cfg.Address
+		}
+		if cfg.Pprof.Address == "" {
+			cfg.Pprof.Address = cfg.Address
 		}
 	}
 }

config/config.go

@@ -76,7 +76,8 @@ type (
 		MaxPeers          int                      `yaml:"MaxPeers"`
 		AttemptConnPeers  int                      `yaml:"AttemptConnPeers"`
 		MinPeers          int                      `yaml:"MinPeers"`
-		Monitoring        metrics.PrometheusConfig `yaml:"Monitoring"`
+		Prometheus        metrics.Config           `yaml:"Prometheus"`
+		Pprof             metrics.Config           `yaml:"Pprof"`
 		RPC               RPCConfig                `yaml:"RPC"`
 		UnlockWallet      WalletConfig             `yaml:"UnlockWallet"`
 	}

config/protocol.mainnet.yml

@@ -57,6 +57,9 @@ ApplicationConfiguration:
     Enabled: true
     EnableCORSWorkaround: false
     Port: 10332
-  Monitoring:
+  Prometheus:
     Enabled: true
     Port: 2112
+  Pprof:
+    Enabled: false
+    Port: 2113

config/protocol.privnet.docker.four.yml

@@ -48,9 +48,12 @@ ApplicationConfiguration:
     Enabled: true
     EnableCORSWorkaround: false
     Port: 30336
-  Monitoring:
+  Prometheus:
     Enabled: true
     Port: 20004
+  Pprof:
+    Enabled: false
+    Port: 20014
   UnlockWallet:
     Path: "6PYRXVwHSqFSukL3CuXxdQ75VmsKpjeLgQLEjt83FrtHf1gCVphHzdD4nc"
     Password: "four"

config/protocol.privnet.docker.one.yml

@@ -48,9 +48,12 @@ ApplicationConfiguration:
     Enabled: true
     EnableCORSWorkaround: false
     Port: 30333
-  Monitoring:
+  Prometheus:
     Enabled: true
     Port: 20001
+  Pprof:
+    Enabled: false
+    Port: 20011
   UnlockWallet:
     Path: "6PYLmjBYJ4wQTCEfqvnznGJwZeW9pfUcV5m5oreHxqryUgqKpTRAFt9L8Y"
     Password: "one"

config/protocol.privnet.docker.three.yml

@@ -48,9 +48,12 @@ ApplicationConfiguration:
     Enabled: true
     EnableCORSWorkaround: false
     Port: 30335
-  Monitoring:
+  Prometheus:
     Enabled: true
     Port: 20003
+  Pprof:
+    Enabled: false
+    Port: 20013
   UnlockWallet:
     Path: "6PYX86vYiHfUbpD95hfN1xgnvcSxy5skxfWYKu3ztjecxk6ikYs2kcWbeh"
     Password: "three"

config/protocol.privnet.docker.two.yml

@@ -48,9 +48,12 @@ ApplicationConfiguration:
     Enabled: true
     EnableCORSWorkaround: false
     Port: 30334
-  Monitoring:
+  Prometheus:
     Enabled: true
     Port: 20002
+  Pprof:
+    Enabled: false
+    Port: 20012
   UnlockWallet:
     Path: "6PYXHjPaNvW8YknSXaKsTWjf9FRxo1s4naV2jdmSQEgzaqKGX368rndN3L"
     Password: "two"

config/protocol.privnet.yml

@@ -48,6 +48,9 @@ ApplicationConfiguration:
     Enabled: true
     EnableCORSWorkaround: false
     Port: 20331
-  Monitoring:
+  Prometheus:
     Enabled: true
     Port: 2112
+  Pprof:
+    Enabled: false
+    Port: 2113

config/protocol.testnet.yml

@@ -57,6 +57,9 @@ ApplicationConfiguration:
     Enabled: true
     EnableCORSWorkaround: false
     Port: 20332
-  Monitoring:
+  Prometheus:
     Enabled: true
     Port: 2112
+  Pprof:
+    Enabled: false
+    Port: 2113

config/protocol.unit_testnet.yml

@@ -47,6 +47,9 @@ ApplicationConfiguration:
     Enabled: true
     EnableCORSWorkaround: false
     Port: 20332
-  Monitoring:
+  Prometheus:
     Enabled: false #since it's not useful for unit tests.
     Port: 2112
+  Pprof:
+    Enabled: false #since it's not useful for unit tests.
+    Port: 2113

pkg/network/metrics/metrics.go

@@ -4,57 +4,47 @@ import (
 	"context"
 	"net/http"
 
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 	log "github.com/sirupsen/logrus"
 )
 
-// Service serves metrics provided by prometheus.
+// Service serves metrics.
 type Service struct {
 	*http.Server
-	config PrometheusConfig
+	config      Config
+	serviceType string
 }
 
-// PrometheusConfig config for Prometheus used for monitoring.
+// Config config used for monitoring.
-// Additional information about Prometheus could be found here: https://prometheus.io/docs/guides/go-application.
+type Config struct {
-type PrometheusConfig struct {
 	Enabled bool   `yaml:"Enabled"`
 	Address string `yaml:"Address"`
 	Port    string `yaml:"Port"`
 }
 
-// NewMetricsService created new service for gathering metrics.
+// Start runs http service with exposed endpoint on configured port.
-func NewMetricsService(cfg PrometheusConfig) *Service {
-	return &Service{
-		&http.Server{
-			Addr:    cfg.Address + ":" + cfg.Port,
-			Handler: promhttp.Handler(),
-		}, cfg,
-	}
-}
-
-// Start runs http service with exposed `/metrics` endpoint on configured port.
 func (ms *Service) Start() {
 	if ms.config.Enabled {
+		log.WithFields(log.Fields{
+			"endpoint": ms.Addr,
+			"service": ms.serviceType,
+		}).Info("service running")
 		err := ms.ListenAndServe()
-		if err != nil {
+		if err != nil && err != http.ErrServerClosed {
-			log.WithFields(log.Fields{
+			log.Warnf("%s service couldn't start on configured port", ms.serviceType)
-				"endpoint": ms.Addr,
-			}).Info("metrics service up and running")
-		} else {
-			log.Warn("metrics service couldn't start on configured port")
 		}
 	} else {
-		log.Infof("metrics service hasn't started since it's disabled")
+		log.Infof("%s service hasn't started since it's disabled", ms.serviceType)
 	}
 }
 
 // ShutDown stops service.
 func (ms *Service) ShutDown() {
 	log.WithFields(log.Fields{
-		"endpoint": ms.config.Port,
+		"endpoint": ms.Addr,
-	}).Info("shutting down monitoring-service")
+		"service": ms.serviceType,
+	}).Info("shutting down service")
 	err := ms.Shutdown(context.Background())
 	if err != nil {
-		log.Fatal("can't shut down monitoring service")
+		log.Fatalf("can't shut down %s service",  ms.serviceType)
 	}
 }

pkg/network/metrics/pprof.go

@@ -0,0 +1,25 @@
+package metrics
+
+import (
+	"net/http"
+	"net/http/pprof"
+)
+// PprofService https://golang.org/pkg/net/http/pprof/.
+type PprofService Service
+
+// NewPprofService created new service for gathering pprof metrics.
+func NewPprofService(cfg Config) *Service {
+	handler := http.NewServeMux()
+	handler.HandleFunc("/debug/pprof", pprof.Index)
+	handler.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
+	handler.HandleFunc("/debug/pprof/profile", pprof.Profile)
+	handler.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
+	handler.HandleFunc("/debug/pprof/trace", pprof.Trace)
+
+	return &Service{
+		&http.Server{
+			Addr:    cfg.Address + ":" + cfg.Port,
+			Handler: handler,
+		}, cfg, "Pprof",
+	}
+}

pkg/network/metrics/prometheus.go

@@ -0,0 +1,20 @@
+package metrics
+
+import (
+	"net/http"
+
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+// PrometheusService https://prometheus.io/docs/guides/go-application.
+type PrometheusService Service
+
+// NewPrometheusService creates new service for gathering prometheus metrics.
+func NewPrometheusService(cfg Config) *Service {
+	return &Service{
+		&http.Server{
+			Addr:    cfg.Address + ":" + cfg.Port,
+			Handler: promhttp.Handler(),
+		}, cfg, "Prometheus",
+	}
+}