From 0085831ec5728226adfa5360ccf5c84c623d366b Mon Sep 17 00:00:00 2001 From: Vsevolod Brekelov Date: Wed, 4 Dec 2019 09:48:32 +0300 Subject: [PATCH 1/3] network: add Pprof metrics Since we have some perf issues from time to time it is good to have pprof debugger. Disabled by default. --- cli/server/server.go | 24 ++++++++----- config/config.go | 3 +- config/protocol.mainnet.yml | 5 ++- config/protocol.privnet.docker.four.yml | 5 ++- config/protocol.privnet.docker.one.yml | 5 ++- config/protocol.privnet.docker.three.yml | 5 ++- config/protocol.privnet.docker.two.yml | 5 ++- config/protocol.privnet.yml | 5 ++- config/protocol.testnet.yml | 5 ++- config/protocol.unit_testnet.yml | 5 ++- pkg/network/metrics/metrics.go | 44 +++++++++--------------- pkg/network/metrics/pprof.go | 25 ++++++++++++++ pkg/network/metrics/prometheus.go | 20 +++++++++++ 13 files changed, 111 insertions(+), 45 deletions(-) create mode 100644 pkg/network/metrics/pprof.go create mode 100644 pkg/network/metrics/prometheus.go diff --git a/cli/server/server.go b/cli/server/server.go index 9061a832c..f3511ab01 100644 --- a/cli/server/server.go +++ b/cli/server/server.go @@ -280,12 +280,14 @@ func startServer(ctx *cli.Context) error { server := network.NewServer(serverConfig, chain) rpcServer := rpc.NewServer(chain, cfg.ApplicationConfiguration.RPC, server) errChan := make(chan error) - monitoring := metrics.NewMetricsService(cfg.ApplicationConfiguration.Monitoring) + prometheus := metrics.NewPrometheusService(cfg.ApplicationConfiguration.Prometheus) + pprof := metrics.NewPprofService(cfg.ApplicationConfiguration.Pprof) go chain.Run() go server.Start(errChan) go rpcServer.Start(errChan) - go monitoring.Start() + go prometheus.Start() + go pprof.Start() fmt.Println(logo()) fmt.Println(server.UserAgent) @@ -304,7 +306,8 @@ Main: if serverErr := rpcServer.Shutdown(); serverErr != nil { shutdownErr = errors.Wrap(serverErr, "Error encountered whilst shutting down server") } - monitoring.ShutDown() + prometheus.ShutDown() + pprof.ShutDown() chain.Close() break Main } @@ -317,17 +320,20 @@ Main: return nil } -// configureAddresses sets up addresses for RPC and Monitoring depending from the provided config. -// In case RPC or Monitoring Address provided each of them will use it. -// In case global Address (of the node) provided and RPC/Monitoring don't have configured addresses they will -// use global one. So Node and RPC and Monitoring will run on one address. +// configureAddresses sets up addresses for RPC, Prometheus and Pprof depending from the provided config. +// In case RPC or Prometheus or Pprof Address provided each of them will use it. +// In case global Address (of the node) provided and RPC/Prometheus/Pprof don't have configured addresses they will +// use global one. So Node and RPC and Prometheus and Pprof will run on one address. func configureAddresses(cfg config.ApplicationConfiguration) { if cfg.Address != "" { if cfg.RPC.Address == "" { cfg.RPC.Address = cfg.Address } - if cfg.Monitoring.Address == "" { - cfg.Monitoring.Address = cfg.Address + if cfg.Prometheus.Address == "" { + cfg.Prometheus.Address = cfg.Address + } + if cfg.Pprof.Address == "" { + cfg.Pprof.Address = cfg.Address } } } diff --git a/config/config.go b/config/config.go index a96b9dac2..74b03f4ac 100644 --- a/config/config.go +++ b/config/config.go @@ -76,7 +76,8 @@ type ( MaxPeers int `yaml:"MaxPeers"` AttemptConnPeers int `yaml:"AttemptConnPeers"` MinPeers int `yaml:"MinPeers"` - Monitoring metrics.PrometheusConfig `yaml:"Monitoring"` + Prometheus metrics.Config `yaml:"Prometheus"` + Pprof metrics.Config `yaml:"Pprof"` RPC RPCConfig `yaml:"RPC"` UnlockWallet WalletConfig `yaml:"UnlockWallet"` } diff --git a/config/protocol.mainnet.yml b/config/protocol.mainnet.yml index 579df4d17..0a6b125f5 100644 --- a/config/protocol.mainnet.yml +++ b/config/protocol.mainnet.yml @@ -57,6 +57,9 @@ ApplicationConfiguration: Enabled: true EnableCORSWorkaround: false Port: 10332 - Monitoring: + Prometheus: Enabled: true Port: 2112 + Pprof: + Enabled: false + Port: 2113 diff --git a/config/protocol.privnet.docker.four.yml b/config/protocol.privnet.docker.four.yml index 73a9869ee..05fdd6fa4 100644 --- a/config/protocol.privnet.docker.four.yml +++ b/config/protocol.privnet.docker.four.yml @@ -48,9 +48,12 @@ ApplicationConfiguration: Enabled: true EnableCORSWorkaround: false Port: 30336 - Monitoring: + Prometheus: Enabled: true Port: 20004 + Pprof: + Enabled: false + Port: 20014 UnlockWallet: Path: "6PYRXVwHSqFSukL3CuXxdQ75VmsKpjeLgQLEjt83FrtHf1gCVphHzdD4nc" Password: "four" diff --git a/config/protocol.privnet.docker.one.yml b/config/protocol.privnet.docker.one.yml index ff939f0b7..f44bc8df5 100644 --- a/config/protocol.privnet.docker.one.yml +++ b/config/protocol.privnet.docker.one.yml @@ -48,9 +48,12 @@ ApplicationConfiguration: Enabled: true EnableCORSWorkaround: false Port: 30333 - Monitoring: + Prometheus: Enabled: true Port: 20001 + Pprof: + Enabled: false + Port: 20011 UnlockWallet: Path: "6PYLmjBYJ4wQTCEfqvnznGJwZeW9pfUcV5m5oreHxqryUgqKpTRAFt9L8Y" Password: "one" diff --git a/config/protocol.privnet.docker.three.yml b/config/protocol.privnet.docker.three.yml index 95749d4ae..ad746a52d 100644 --- a/config/protocol.privnet.docker.three.yml +++ b/config/protocol.privnet.docker.three.yml @@ -48,9 +48,12 @@ ApplicationConfiguration: Enabled: true EnableCORSWorkaround: false Port: 30335 - Monitoring: + Prometheus: Enabled: true Port: 20003 + Pprof: + Enabled: false + Port: 20013 UnlockWallet: Path: "6PYX86vYiHfUbpD95hfN1xgnvcSxy5skxfWYKu3ztjecxk6ikYs2kcWbeh" Password: "three" diff --git a/config/protocol.privnet.docker.two.yml b/config/protocol.privnet.docker.two.yml index 4f7801166..b250aab10 100644 --- a/config/protocol.privnet.docker.two.yml +++ b/config/protocol.privnet.docker.two.yml @@ -48,9 +48,12 @@ ApplicationConfiguration: Enabled: true EnableCORSWorkaround: false Port: 30334 - Monitoring: + Prometheus: Enabled: true Port: 20002 + Pprof: + Enabled: false + Port: 20012 UnlockWallet: Path: "6PYXHjPaNvW8YknSXaKsTWjf9FRxo1s4naV2jdmSQEgzaqKGX368rndN3L" Password: "two" diff --git a/config/protocol.privnet.yml b/config/protocol.privnet.yml index 992e62aeb..87c97d2e0 100644 --- a/config/protocol.privnet.yml +++ b/config/protocol.privnet.yml @@ -48,6 +48,9 @@ ApplicationConfiguration: Enabled: true EnableCORSWorkaround: false Port: 20331 - Monitoring: + Prometheus: Enabled: true Port: 2112 + Pprof: + Enabled: false + Port: 2113 diff --git a/config/protocol.testnet.yml b/config/protocol.testnet.yml index 47ba9c345..fe069eaf6 100644 --- a/config/protocol.testnet.yml +++ b/config/protocol.testnet.yml @@ -57,6 +57,9 @@ ApplicationConfiguration: Enabled: true EnableCORSWorkaround: false Port: 20332 - Monitoring: + Prometheus: Enabled: true Port: 2112 + Pprof: + Enabled: false + Port: 2113 diff --git a/config/protocol.unit_testnet.yml b/config/protocol.unit_testnet.yml index 6a76288b1..21c8b462d 100644 --- a/config/protocol.unit_testnet.yml +++ b/config/protocol.unit_testnet.yml @@ -47,6 +47,9 @@ ApplicationConfiguration: Enabled: true EnableCORSWorkaround: false Port: 20332 - Monitoring: + Prometheus: Enabled: false #since it's not useful for unit tests. Port: 2112 + Pprof: + Enabled: false #since it's not useful for unit tests. + Port: 2113 diff --git a/pkg/network/metrics/metrics.go b/pkg/network/metrics/metrics.go index 8755ff5a8..f0c6d1d16 100644 --- a/pkg/network/metrics/metrics.go +++ b/pkg/network/metrics/metrics.go @@ -4,57 +4,47 @@ import ( "context" "net/http" - "github.com/prometheus/client_golang/prometheus/promhttp" log "github.com/sirupsen/logrus" ) -// Service serves metrics provided by prometheus. +// Service serves metrics. type Service struct { *http.Server - config PrometheusConfig + config Config + serviceType string } -// PrometheusConfig config for Prometheus used for monitoring. -// Additional information about Prometheus could be found here: https://prometheus.io/docs/guides/go-application. -type PrometheusConfig struct { +// Config config used for monitoring. +type Config struct { Enabled bool `yaml:"Enabled"` Address string `yaml:"Address"` Port string `yaml:"Port"` } -// NewMetricsService created new service for gathering metrics. -func NewMetricsService(cfg PrometheusConfig) *Service { - return &Service{ - &http.Server{ - Addr: cfg.Address + ":" + cfg.Port, - Handler: promhttp.Handler(), - }, cfg, - } -} - -// Start runs http service with exposed `/metrics` endpoint on configured port. +// Start runs http service with exposed endpoint on configured port. func (ms *Service) Start() { if ms.config.Enabled { + log.WithFields(log.Fields{ + "endpoint": ms.Addr, + "service": ms.serviceType, + }).Info("service running") err := ms.ListenAndServe() - if err != nil { - log.WithFields(log.Fields{ - "endpoint": ms.Addr, - }).Info("metrics service up and running") - } else { - log.Warn("metrics service couldn't start on configured port") + if err != nil && err != http.ErrServerClosed { + log.Warnf("%s service couldn't start on configured port", ms.serviceType) } } else { - log.Infof("metrics service hasn't started since it's disabled") + log.Infof("%s service hasn't started since it's disabled", ms.serviceType) } } // ShutDown stops service. func (ms *Service) ShutDown() { log.WithFields(log.Fields{ - "endpoint": ms.config.Port, - }).Info("shutting down monitoring-service") + "endpoint": ms.Addr, + "service": ms.serviceType, + }).Info("shutting down service") err := ms.Shutdown(context.Background()) if err != nil { - log.Fatal("can't shut down monitoring service") + log.Fatalf("can't shut down %s service", ms.serviceType) } } diff --git a/pkg/network/metrics/pprof.go b/pkg/network/metrics/pprof.go new file mode 100644 index 000000000..1e531a84e --- /dev/null +++ b/pkg/network/metrics/pprof.go @@ -0,0 +1,25 @@ +package metrics + +import ( + "net/http" + "net/http/pprof" +) +// PprofService https://golang.org/pkg/net/http/pprof/. +type PprofService Service + +// NewPprofService created new service for gathering pprof metrics. +func NewPprofService(cfg Config) *Service { + handler := http.NewServeMux() + handler.HandleFunc("/debug/pprof", pprof.Index) + handler.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) + handler.HandleFunc("/debug/pprof/profile", pprof.Profile) + handler.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + handler.HandleFunc("/debug/pprof/trace", pprof.Trace) + + return &Service{ + &http.Server{ + Addr: cfg.Address + ":" + cfg.Port, + Handler: handler, + }, cfg, "Pprof", + } +} diff --git a/pkg/network/metrics/prometheus.go b/pkg/network/metrics/prometheus.go new file mode 100644 index 000000000..25cf80281 --- /dev/null +++ b/pkg/network/metrics/prometheus.go @@ -0,0 +1,20 @@ +package metrics + +import ( + "net/http" + + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +// PrometheusService https://prometheus.io/docs/guides/go-application. +type PrometheusService Service + +// NewPrometheusService creates new service for gathering prometheus metrics. +func NewPrometheusService(cfg Config) *Service { + return &Service{ + &http.Server{ + Addr: cfg.Address + ":" + cfg.Port, + Handler: promhttp.Handler(), + }, cfg, "Prometheus", + } +} From 4cffc54a55c2499b17aa0a51e83161e356e4e81c Mon Sep 17 00:00:00 2001 From: Vsevolod Brekelov Date: Wed, 4 Dec 2019 09:56:10 +0300 Subject: [PATCH 2/3] readme: add notes for developers about Prometheus and Pprof --- README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.md b/README.md index ebaeb271b..626c47125 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,19 @@ Available network flags: - `--privnet, -p` - `--testnet, -t` +#Developer notes +Nodes have such features as [Prometheus](https://prometheus.io/docs/guides/go-application) and +[Pprof](https://golang.org/pkg/net/http/pprof/) in order to have additional information about them for debugging. + +How to configure Prometheus or Pprof: +In `config/protocol.*.yml` there is +``` + Prometheus: + Enabled: true + Port: 2112 +``` +where you can switch on/off and define port. Prometheus is enabled and Pprof is disabled by default. + # Contributing Feel free to contribute to this project after reading the From 9541029303c3067aeef9d1be1caab00cf06f9732 Mon Sep 17 00:00:00 2001 From: Vsevolod Brekelov Date: Mon, 16 Dec 2019 17:04:18 +0300 Subject: [PATCH 3/3] cli: add metrics to dump and restore calls --- cli/server/server.go | 38 +++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/cli/server/server.go b/cli/server/server.go index f3511ab01..d742e0d1c 100644 --- a/cli/server/server.go +++ b/cli/server/server.go @@ -134,6 +134,22 @@ func getCountAndSkipFromContext(ctx *cli.Context) (uint32, uint32) { return count, skip } +func initBCWithMetrics(cfg config.Config) (*core.Blockchain, *metrics.Service, *metrics.Service, error) { + chain, err := initBlockChain(cfg) + if err != nil { + return nil, nil, nil, cli.NewExitError(err, 1) + } + configureAddresses(cfg.ApplicationConfiguration) + prometheus := metrics.NewPrometheusService(cfg.ApplicationConfiguration.Prometheus) + pprof := metrics.NewPprofService(cfg.ApplicationConfiguration.Pprof) + + go chain.Run() + go prometheus.Start() + go pprof.Start() + + return chain, prometheus, pprof, nil +} + func dumpDB(ctx *cli.Context) error { cfg, err := getConfigFromContext(ctx) if err != nil { @@ -154,11 +170,10 @@ func dumpDB(ctx *cli.Context) error { defer outStream.Close() writer := io.NewBinWriterFromIO(outStream) - chain, err := initBlockChain(cfg) + chain, prometheus, pprof, err := initBCWithMetrics(cfg) if err != nil { - return cli.NewExitError(err, 1) + return err } - go chain.Run() chainHeight := chain.BlockHeight() if skip+count > chainHeight { @@ -182,9 +197,12 @@ func dumpDB(ctx *cli.Context) error { return cli.NewExitError(err, 1) } } + pprof.ShutDown() + prometheus.ShutDown() chain.Close() return nil } + func restoreDB(ctx *cli.Context) error { cfg, err := getConfigFromContext(ctx) if err != nil { @@ -205,11 +223,10 @@ func restoreDB(ctx *cli.Context) error { defer inStream.Close() reader := io.NewBinReaderFromIO(inStream) - chain, err := initBlockChain(cfg) + chain, prometheus, pprof, err := initBCWithMetrics(cfg) if err != nil { return err } - go chain.Run() var allBlocks = reader.ReadU32LE() if reader.Err != nil { @@ -241,8 +258,9 @@ func restoreDB(ctx *cli.Context) error { return cli.NewExitError(fmt.Errorf("failed to add block %d: %s", i, err), 1) } } + pprof.ShutDown() + prometheus.ShutDown() chain.Close() - return nil } @@ -271,23 +289,17 @@ func startServer(ctx *cli.Context) error { serverConfig := network.NewServerConfig(cfg) - chain, err := initBlockChain(cfg) + chain, prometheus, pprof, err := initBCWithMetrics(cfg) if err != nil { return err } - configureAddresses(cfg.ApplicationConfiguration) server := network.NewServer(serverConfig, chain) rpcServer := rpc.NewServer(chain, cfg.ApplicationConfiguration.RPC, server) errChan := make(chan error) - prometheus := metrics.NewPrometheusService(cfg.ApplicationConfiguration.Prometheus) - pprof := metrics.NewPprofService(cfg.ApplicationConfiguration.Pprof) - go chain.Run() go server.Start(errChan) go rpcServer.Start(errChan) - go prometheus.Start() - go pprof.Start() fmt.Println(logo()) fmt.Println(server.UserAgent)