From c397efb1c22003cf7ba7494cb1387a45487c9be8 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Thu, 11 Aug 2022 17:07:53 +0300 Subject: [PATCH] [#44] Expose metrics Signed-off-by: Denis Kirillov --- CHANGELOG.md | 2 +- cmd/neofs-rest-gw/config.go | 33 +++++ cmd/neofs-rest-gw/main.go | 6 +- config/config.env | 9 +- config/config.yaml | 11 +- gen/restapi/server.go | 9 ++ go.mod | 12 +- go.sum | 28 ++-- handlers/api.go | 38 +++++- metrics/metrics.go | 225 +++++++++++++++++++++++++++++++++ metrics/pprof.go | 33 +++++ metrics/service.go | 44 +++++++ templates/server/server.gotmpl | 9 ++ 13 files changed, 426 insertions(+), 33 deletions(-) create mode 100644 metrics/metrics.go create mode 100644 metrics/pprof.go create mode 100644 metrics/service.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 177c66f..9038867 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ This document outlines major changes between releases. ### Added - CORS headers (#39) - +- Expose metrics (#44) ## [0.2.1] "Razor Hill" - 2022-07-22 diff --git a/cmd/neofs-rest-gw/config.go b/cmd/neofs-rest-gw/config.go index 8f6dd32..156535b 100644 --- a/cmd/neofs-rest-gw/config.go +++ b/cmd/neofs-rest-gw/config.go @@ -16,6 +16,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/wallet" "github.com/nspcc-dev/neofs-rest-gw/gen/restapi" "github.com/nspcc-dev/neofs-rest-gw/handlers" + "github.com/nspcc-dev/neofs-rest-gw/metrics" "github.com/nspcc-dev/neofs-sdk-go/pool" "github.com/spf13/pflag" "github.com/spf13/viper" @@ -28,11 +29,19 @@ const ( defaultRequestTimeout = 15 * time.Second defaultConnectTimeout = 30 * time.Second + defaultShutdownTimeout = 15 * time.Second + // Timeouts. cfgNodeDialTimeout = "node-dial-timeout" cfgHealthcheckTimeout = "healthcheck-timeout" cfgRebalance = "rebalance-timer" + // Metrics / Profiler. + cfgPrometheusEnabled = "prometheus.enabled" + cfgPrometheusAddress = "prometheus.address" + cfgPprofEnabled = "pprof.enabled" + cfgPprofAddress = "pprof.address" + // Logger. cfgLoggerLevel = "logger.level" @@ -97,13 +106,26 @@ func config() *viper.Viper { peers := flagSet.StringArrayP(cfgPeers, "p", nil, "NeoFS nodes") + // init server flags restapi.BindDefaultFlags(flagSet) // set defaults: + // metrics + v.SetDefault(cfgPprofAddress, "localhost:8091") + v.SetDefault(cfgPrometheusAddress, "localhost:8092") + // logger: v.SetDefault(cfgLoggerLevel, "debug") + // Bind flags + if err := v.BindPFlag(cfgPprofEnabled, flagSet.Lookup(cmdPprof)); err != nil { + panic(err) + } + if err := v.BindPFlag(cfgPrometheusEnabled, flagSet.Lookup(cmdMetrics)); err != nil { + panic(err) + } + if err := v.BindPFlags(flagSet); err != nil { panic(err) } @@ -313,6 +335,17 @@ func newNeofsAPI(ctx context.Context, logger *zap.Logger, v *viper.Viper) (*hand apiPrm.Key = key apiPrm.Logger = logger + pprofConfig := metrics.Config{Enabled: v.GetBool(cfgPprofEnabled), Address: v.GetString(cfgPprofAddress)} + apiPrm.PprofService = metrics.NewPprofService(logger, pprofConfig) + + prometheusConfig := metrics.Config{Enabled: v.GetBool(cfgPrometheusEnabled), Address: v.GetString(cfgPrometheusAddress)} + apiPrm.PrometheusService = metrics.NewPrometheusService(logger, prometheusConfig) + if prometheusConfig.Enabled { + apiPrm.GateMetric = metrics.NewGateMetrics(p) + } + + apiPrm.ServiceShutdownTimeout = defaultShutdownTimeout + return handlers.New(&apiPrm), nil } diff --git a/cmd/neofs-rest-gw/main.go b/cmd/neofs-rest-gw/main.go index a7e8790..5b7dce2 100644 --- a/cmd/neofs-rest-gw/main.go +++ b/cmd/neofs-rest-gw/main.go @@ -25,8 +25,11 @@ func main() { logger.Fatal("init spec", zap.Error(err)) } + serverCfg := serverConfig(v) + serverCfg.SuccessfulStartCallback = neofsAPI.StartCallback + api := operations.NewNeofsRestGwAPI(swaggerSpec) - server := restapi.NewServer(api, serverConfig(v)) + server := restapi.NewServer(api, serverCfg) defer func() { if err = server.Shutdown(); err != nil { logger.Error("shutdown", zap.Error(err)) @@ -34,6 +37,7 @@ func main() { }() server.ConfigureAPI(neofsAPI.Configure) + neofsAPI.RunServices() // serve API if err = server.Serve(); err != nil { diff --git a/config/config.env b/config/config.env index 075c5ab..66b5650 100644 --- a/config/config.env +++ b/config/config.env @@ -6,9 +6,12 @@ REST_GW_WALLET_ADDRESS=NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP REST_GW_WALLET_PASSPHRASE=pwd # Enable metrics. -REST_GW_METRICS=true -# Enable pprof. -REST_GW_PPROF=true +REST_GW_PPROF_ENABLED=true +REST_GW_PPROF_ADDRESS=localhost:8091 + +REST_GW_PROMETHEUS_ENABLED=true +REST_GW_PROMETHEUS_ADDRESS=localhost:8092 + # Log level. REST_GW_LOGGER_LEVEL=debug diff --git a/config/config.yaml b/config/config.yaml index 6292f4d..cf0767a 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -6,10 +6,13 @@ wallet: # Password to decrypt wallet. passphrase: pwd -# Enable metrics. -metrics: true -# Enable pprof. -pprof: true +pprof: + enabled: true # Enable pprof. + address: localhost:8091 +prometheus: + enabled: true # Enable metrics. + address: localhost:8092 + logger: # Log level. level: debug diff --git a/gen/restapi/server.go b/gen/restapi/server.go index 50bb812..73ce434 100644 --- a/gen/restapi/server.go +++ b/gen/restapi/server.go @@ -58,6 +58,8 @@ type ServerConfig struct { TLSCertificate string TLSCertificateKey string TLSCACertificate string + + SuccessfulStartCallback func() } // NewServer creates a new api neofs rest gw server but does not configure it @@ -85,6 +87,7 @@ func NewServer(api *operations.NeofsRestGwAPI, cfg *ServerConfig) *Server { s.TLSWriteTimeout = cfg.TLSWriteTimeout s.shutdown = make(chan struct{}) s.api = api + s.startCallback = cfg.SuccessfulStartCallback s.interrupt = make(chan os.Signal, 1) return s } @@ -120,6 +123,8 @@ type Server struct { TLSWriteTimeout time.Duration httpsServerL net.Listener + startCallback func() + cfgTLSFn func(tlsConfig *tls.Config) cfgServerFn func(s *http.Server, scheme, addr string) @@ -322,6 +327,10 @@ func (s *Server) Serve() (err error) { wg.Add(1) go s.handleShutdown(wg, &servers) + if s.startCallback != nil { + s.startCallback() + } + wg.Wait() return nil } diff --git a/go.mod b/go.mod index e98d803..28f3bd7 100644 --- a/go.mod +++ b/go.mod @@ -11,9 +11,9 @@ require ( github.com/go-openapi/swag v0.21.1 github.com/go-openapi/validate v0.21.0 github.com/google/uuid v1.3.0 - github.com/nspcc-dev/neo-go v0.98.2 + github.com/nspcc-dev/neo-go v0.99.1 github.com/nspcc-dev/neofs-api-go/v2 v2.13.0 - github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.5.0.20220707112018-7d10b432d139 + github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.6 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.7.0 github.com/testcontainers/testcontainers-go v0.13.0 @@ -48,9 +48,7 @@ require ( github.com/go-stack/stack v1.8.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/mock v1.6.0 // indirect github.com/golang/protobuf v1.5.2 // indirect - github.com/golang/snappy v0.0.3 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/josharian/intern v1.0.0 // indirect @@ -74,7 +72,7 @@ require ( github.com/pelletier/go-toml v1.9.4 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_golang v1.11.0 // indirect + github.com/prometheus/client_golang v1.11.0 github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.26.0 // indirect github.com/prometheus/procfs v0.6.0 // indirect @@ -87,9 +85,7 @@ require ( github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/viper v1.10.1 github.com/subosito/gotenv v1.2.0 // indirect - github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 // indirect github.com/urfave/cli v1.22.5 // indirect - go.etcd.io/bbolt v1.3.6 // indirect go.mongodb.org/mongo-driver v1.8.4 // indirect go.opencensus.io v0.23.0 // indirect go.uber.org/atomic v1.9.0 // indirect @@ -104,5 +100,5 @@ require ( google.golang.org/protobuf v1.27.1 // indirect gopkg.in/ini.v1 v1.66.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 6f91a44..7802129 100644 --- a/go.sum +++ b/go.sum @@ -279,6 +279,7 @@ github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmeka github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20161114122254-48702e0da86b/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= @@ -470,7 +471,6 @@ github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= -github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -492,7 +492,6 @@ github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -549,6 +548,7 @@ github.com/gorilla/mux v1.7.2 h1:zoNxOV7WjqXptQOVngLmcSQgXmgk4NMz1HibBchjl/I= github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= @@ -730,6 +730,7 @@ github.com/nspcc-dev/dbft v0.0.0-20191209120240-0d6b7568d9ae/go.mod h1:3FjXOoHmA github.com/nspcc-dev/dbft v0.0.0-20200117124306-478e5cfbf03a/go.mod h1:/YFK+XOxxg0Bfm6P92lY5eDSLYfp06XOdL8KAVgXjVk= github.com/nspcc-dev/dbft v0.0.0-20200219114139-199d286ed6c1/go.mod h1:O0qtn62prQSqizzoagHmuuKoz8QMkU3SzBoKdEvm3aQ= github.com/nspcc-dev/dbft v0.0.0-20210721160347-1b03241391ac/go.mod h1:U8MSnEShH+o5hexfWJdze6uMFJteP0ko7J2frO7Yu1Y= +github.com/nspcc-dev/dbft v0.0.0-20220629112714-fd49ca59d354/go.mod h1:U8MSnEShH+o5hexfWJdze6uMFJteP0ko7J2frO7Yu1Y= github.com/nspcc-dev/go-ordered-json v0.0.0-20210915112629-e1b6cce73d02/go.mod h1:79bEUDEviBHJMFV6Iq6in57FEOCMcRhfQnfaf0ETA5U= github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22 h1:n4ZaFCKt1pQJd7PXoMJabZWK9ejjbLOVrkl/lOUmshg= github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22/go.mod h1:79bEUDEviBHJMFV6Iq6in57FEOCMcRhfQnfaf0ETA5U= @@ -737,28 +738,29 @@ github.com/nspcc-dev/hrw v1.0.9 h1:17VcAuTtrstmFppBjfRiia4K2wA/ukXZhLFS8Y8rz5Y= github.com/nspcc-dev/hrw v1.0.9/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkPG06MU= github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1:pPYwPZ2ks+uMnlRLUyXOpLieaDQSEaf4NM3zHVbRjmg= github.com/nspcc-dev/neo-go v0.98.0/go.mod h1:E3cc1x6RXSXrJb2nDWXTXjnXk3rIqVN8YdFyWv+FrqM= -github.com/nspcc-dev/neo-go v0.98.2 h1:aNTQR0BjkojCVXv17/dh1sD88a0A1L+7GNympylTKig= -github.com/nspcc-dev/neo-go v0.98.2/go.mod h1:KXKqJwfTyVJzDarSCDqFaKrVbg/qz0ZBk2c3AtzqS5M= -github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220321113211-526c423a6152/go.mod h1:QBE0I30F2kOAISNpT5oks82yF4wkkUq3SCfI3Hqgx/Y= +github.com/nspcc-dev/neo-go v0.99.1-pre.0.20220714084516-54849ef3e58e/go.mod h1:/y5Sl8p3YheTygriBtCCMWKkDOek8HcvSo5ds2rJtKI= +github.com/nspcc-dev/neo-go v0.99.1 h1:/dlDVVCqlmazusIu+Emd4TlwxjIOpNNU5ob7bwUH6Qk= +github.com/nspcc-dev/neo-go v0.99.1/go.mod h1:wvKMQXlE/M3eqtN0KIi1MjkEVTvYIJRrOMnJtqEkCxs= +github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220713145417-4f184498bc42/go.mod h1:QBE0I30F2kOAISNpT5oks82yF4wkkUq3SCfI3Hqgx/Y= +github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220727202624-6c7a401f776a/go.mod h1:QBE0I30F2kOAISNpT5oks82yF4wkkUq3SCfI3Hqgx/Y= github.com/nspcc-dev/neofs-api-go/v2 v2.11.0-pre.0.20211201134523-3604d96f3fe1/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs= github.com/nspcc-dev/neofs-api-go/v2 v2.11.1/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs= github.com/nspcc-dev/neofs-api-go/v2 v2.13.0 h1:7BcBiSjmWqJx0SPFlGRYt9ZFbMjucRIz9+Mv8UBLeq8= github.com/nspcc-dev/neofs-api-go/v2 v2.13.0/go.mod h1:73j09Xa7I2zQbM3HCvAHnDHPYiiWnEHa1d6Z6RDMBLU= -github.com/nspcc-dev/neofs-contract v0.15.1/go.mod h1:kxO5ZTqdzFnRM5RMvM+Fhd+3GGrJo6AmG2ZyA9OCqqQ= +github.com/nspcc-dev/neofs-contract v0.15.3/go.mod h1:BXVZUZUJxrmmDETglXHI8+5DSgn84B9y5DoSWqEjYCs= github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-crypto v0.3.0 h1:zlr3pgoxuzrmGCxc5W8dGVfA9Rro8diFvVnBg0L4ifM= github.com/nspcc-dev/neofs-crypto v0.3.0/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-sdk-go v0.0.0-20211201182451-a5b61c4f6477/go.mod h1:dfMtQWmBHYpl9Dez23TGtIUKiFvCIxUZq/CkSIhEpz4= github.com/nspcc-dev/neofs-sdk-go v0.0.0-20220113123743-7f3162110659/go.mod h1:/jay1lr3w7NQd/VDBkEhkJmDmyPNsu4W+QV2obsUV40= -github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.5.0.20220707112018-7d10b432d139 h1:wI/sH3DmfGrVCF7QATyGGI1fr8+IthcRDZQxNqiHB+Y= -github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.5.0.20220707112018-7d10b432d139/go.mod h1:KqyyYJT/Wyb5kQrIA6k6XIToSeqGkGPjYOilVCKFaEA= +github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.6 h1:RgkytZY51qCFL7Bbo2GRAAGHukxM9CpNkkDYr15AOA0= +github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.6/go.mod h1:lis8Maqsac+fwSGsEbqHqsA8ttQxSo3xENLmo4QhcW4= github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/tzhash v1.6.1 h1:8dUrWFpjkmoHF+7GxuGUmarj9LLHWFcuyF3CTrqq9JE= github.com/nspcc-dev/tzhash v1.6.1/go.mod h1:BoflzCVp+DO/f1mvbcsJQWoFzidIFBhWFZMglbUW648= -github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= @@ -771,7 +773,6 @@ github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+ github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= @@ -781,7 +782,6 @@ github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.10.3 h1:gph6h/qe9GSUw1NhH1gp+qb+h8rXD8Cy60Z32Qw3ELA= github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= @@ -944,7 +944,6 @@ github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/goleveldb v0.0.0-20180307113352-169b1b37be73/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= -github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 h1:xQdMZ1WLrgkkvOZ/LDQxjVxMLdby7osSh4ZEVa5sIjs= github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= github.com/testcontainers/testcontainers-go v0.13.0 h1:OUujSlEGsXVo/ykPVZk3KanBNGN0TYb/7oKIPVn15JA= @@ -994,7 +993,6 @@ github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= @@ -1566,7 +1564,6 @@ gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -1581,8 +1578,9 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/gotestsum v1.7.0/go.mod h1:V1m4Jw3eBerhI/A6qCxUE07RnCg7ACkKj9BYcAm09V8= diff --git a/handlers/api.go b/handlers/api.go index 484478e..34214a6 100644 --- a/handlers/api.go +++ b/handlers/api.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" "strings" + "time" "github.com/go-openapi/errors" "github.com/google/uuid" @@ -13,6 +14,7 @@ import ( "github.com/nspcc-dev/neofs-rest-gw/gen/models" "github.com/nspcc-dev/neofs-rest-gw/gen/restapi/operations" "github.com/nspcc-dev/neofs-rest-gw/internal/util" + "github.com/nspcc-dev/neofs-rest-gw/metrics" "github.com/nspcc-dev/neofs-sdk-go/pool" "github.com/nspcc-dev/neofs-sdk-go/user" "go.uber.org/zap" @@ -25,6 +27,11 @@ type API struct { key *keys.PrivateKey owner *user.ID defaultTimestamp bool + + gateMetric *metrics.GateMetrics + prometheusService *metrics.Service + pprofService *metrics.Service + serviceShutdownTimeout time.Duration } // PrmAPI groups parameters to init rest API. @@ -33,6 +40,11 @@ type PrmAPI struct { Pool *pool.Pool Key *keys.PrivateKey DefaultTimestamp bool + + GateMetric *metrics.GateMetrics + PrometheusService *metrics.Service + PprofService *metrics.Service + ServiceShutdownTimeout time.Duration } type BearerToken struct { @@ -70,6 +82,11 @@ func New(prm *PrmAPI) *API { key: prm.Key, owner: &owner, defaultTimestamp: prm.DefaultTimestamp, + + prometheusService: prm.PrometheusService, + pprofService: prm.PprofService, + gateMetric: prm.GateMetric, + serviceShutdownTimeout: prm.ServiceShutdownTimeout, } } @@ -118,7 +135,13 @@ func (a *API) Configure(api *operations.NeofsRestGwAPI) http.Handler { api.PreServerShutdown = func() {} - api.ServerShutdown = func() {} + api.ServerShutdown = func() { + shutDownCtx, cancel := context.WithTimeout(context.Background(), a.serviceShutdownTimeout) + defer cancel() + + a.prometheusService.ShutDown(shutDownCtx) + a.pprofService.ShutDown(shutDownCtx) + } return a.setupGlobalMiddleware(a.docMiddleware(api.Serve(setupMiddlewares))) } @@ -161,3 +184,16 @@ func (a *API) logAndGetErrorResponse(msg string, err error, fields ...zap.Field) a.log.Error(msg, fields...) return util.NewErrorResponse(fmt.Errorf("%s: %w", msg, err)) } + +func (a API) StartCallback() { + if a.gateMetric == nil { + return + } + + a.gateMetric.SetHealth(1) +} + +func (a API) RunServices() { + go a.pprofService.Start() + go a.prometheusService.Start() +} diff --git a/metrics/metrics.go b/metrics/metrics.go new file mode 100644 index 0000000..a1232c5 --- /dev/null +++ b/metrics/metrics.go @@ -0,0 +1,225 @@ +package metrics + +import ( + "net/http" + + "github.com/nspcc-dev/neofs-sdk-go/pool" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "go.uber.org/zap" +) + +const ( + namespace = "neofs_rest_gw" + stateSubsystem = "state" + poolSubsystem = "pool" + + methodGetBalance = "get_balance" + methodPutContainer = "put_container" + methodGetContainer = "get_container" + methodListContainer = "list_container" + methodDeleteContainer = "delete_container" + methodGetContainerEacl = "get_container_eacl" + methodSetContainerEacl = "set_container_eacl" + methodEndpointInfo = "endpoint_info" + methodNetworkInfo = "network_info" + methodPutObject = "put_object" + methodDeleteObject = "delete_object" + methodGetObject = "get_object" + methodHeadObject = "head_object" + methodRangeObject = "range_object" + methodCreateSession = "create_session" +) + +type GateMetrics struct { + stateMetrics + poolMetricsCollector +} + +type stateMetrics struct { + healthCheck prometheus.Gauge +} + +type poolMetricsCollector struct { + pool *pool.Pool + overallErrors prometheus.Counter + overallNodeErrors *prometheus.CounterVec + overallNodeRequests *prometheus.CounterVec + currentErrors *prometheus.GaugeVec + requestDuration *prometheus.GaugeVec +} + +// NewGateMetrics creates new metrics for rest gate. +func NewGateMetrics(p *pool.Pool) *GateMetrics { + stateMetric := newStateMetrics() + stateMetric.register() + + poolMetric := newPoolMetricsCollector(p) + poolMetric.register() + + return &GateMetrics{ + stateMetrics: *stateMetric, + poolMetricsCollector: *poolMetric, + } +} + +func newStateMetrics() *stateMetrics { + return &stateMetrics{ + healthCheck: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: stateSubsystem, + Name: "health", + Help: "Current REST gateway state", + }), + } +} + +func (m stateMetrics) register() { + prometheus.MustRegister(m.healthCheck) +} + +func (m stateMetrics) SetHealth(s int32) { + m.healthCheck.Set(float64(s)) +} + +func newPoolMetricsCollector(p *pool.Pool) *poolMetricsCollector { + overallErrors := prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: poolSubsystem, + Name: "overall_errors", + Help: "Total number of errors in pool", + }, + ) + + overallNodeErrors := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: poolSubsystem, + Name: "overall_node_errors", + Help: "Total number of errors for connection in pool", + }, + []string{ + "node", + }, + ) + + overallNodeRequests := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: poolSubsystem, + Name: "overall_node_requests", + Help: "Total number of requests to specific node in pool", + }, + []string{ + "node", + }, + ) + + currentErrors := prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: poolSubsystem, + Name: "current_errors", + Help: "Number of errors on current connections that will be reset after the threshold", + }, + []string{ + "node", + }, + ) + + requestsDuration := prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: poolSubsystem, + Name: "avg_request_duration", + Help: "Average request duration (in milliseconds) for specific method on node in pool", + }, + []string{ + "node", + "method", + }, + ) + + return &poolMetricsCollector{ + pool: p, + overallErrors: overallErrors, + overallNodeErrors: overallNodeErrors, + overallNodeRequests: overallNodeRequests, + currentErrors: currentErrors, + requestDuration: requestsDuration, + } +} + +func (m *poolMetricsCollector) Collect(ch chan<- prometheus.Metric) { + m.updateStatistic() + m.overallErrors.Collect(ch) + m.overallNodeErrors.Collect(ch) + m.overallNodeRequests.Collect(ch) + m.currentErrors.Collect(ch) + m.requestDuration.Collect(ch) +} + +func (m poolMetricsCollector) Describe(descs chan<- *prometheus.Desc) { + m.overallErrors.Describe(descs) + m.overallNodeErrors.Describe(descs) + m.overallNodeRequests.Describe(descs) + m.currentErrors.Describe(descs) + m.requestDuration.Describe(descs) +} + +func (m *poolMetricsCollector) register() { + prometheus.MustRegister(m) +} + +func (m *poolMetricsCollector) updateStatistic() { + stat := m.pool.Statistic() + + m.currentErrors.Reset() + m.requestDuration.Reset() + + for _, node := range stat.Nodes() { + m.overallNodeErrors.WithLabelValues(node.Address()).Add(float64(node.OverallErrors())) + m.overallNodeRequests.WithLabelValues(node.Address()).Add(float64(node.Requests())) + + m.currentErrors.WithLabelValues(node.Address()).Set(float64(node.CurrentErrors())) + m.updateRequestsDuration(node) + } + + m.overallErrors.Add(float64(stat.OverallErrors())) +} + +func (m *poolMetricsCollector) updateRequestsDuration(node pool.NodeStatistic) { + m.requestDuration.WithLabelValues(node.Address(), methodGetBalance).Set(float64(node.AverageGetBalance().Milliseconds())) + m.requestDuration.WithLabelValues(node.Address(), methodPutContainer).Set(float64(node.AveragePutContainer().Milliseconds())) + m.requestDuration.WithLabelValues(node.Address(), methodGetContainer).Set(float64(node.AverageGetContainer().Milliseconds())) + m.requestDuration.WithLabelValues(node.Address(), methodListContainer).Set(float64(node.AverageListContainer().Milliseconds())) + m.requestDuration.WithLabelValues(node.Address(), methodDeleteContainer).Set(float64(node.AverageDeleteContainer().Milliseconds())) + m.requestDuration.WithLabelValues(node.Address(), methodGetContainerEacl).Set(float64(node.AverageGetContainerEACL().Milliseconds())) + m.requestDuration.WithLabelValues(node.Address(), methodSetContainerEacl).Set(float64(node.AverageSetContainerEACL().Milliseconds())) + m.requestDuration.WithLabelValues(node.Address(), methodEndpointInfo).Set(float64(node.AverageEndpointInfo().Milliseconds())) + m.requestDuration.WithLabelValues(node.Address(), methodNetworkInfo).Set(float64(node.AverageNetworkInfo().Milliseconds())) + m.requestDuration.WithLabelValues(node.Address(), methodPutObject).Set(float64(node.AveragePutObject().Milliseconds())) + m.requestDuration.WithLabelValues(node.Address(), methodDeleteObject).Set(float64(node.AverageDeleteObject().Milliseconds())) + m.requestDuration.WithLabelValues(node.Address(), methodGetObject).Set(float64(node.AverageGetObject().Milliseconds())) + m.requestDuration.WithLabelValues(node.Address(), methodHeadObject).Set(float64(node.AverageHeadObject().Milliseconds())) + m.requestDuration.WithLabelValues(node.Address(), methodRangeObject).Set(float64(node.AverageRangeObject().Milliseconds())) + m.requestDuration.WithLabelValues(node.Address(), methodCreateSession).Set(float64(node.AverageCreateSession().Milliseconds())) +} + +// NewPrometheusService creates a new service for gathering prometheus metrics. +func NewPrometheusService(log *zap.Logger, cfg Config) *Service { + if log == nil { + return nil + } + + return &Service{ + Server: &http.Server{ + Addr: cfg.Address, + Handler: promhttp.Handler(), + }, + enabled: cfg.Enabled, + serviceType: "Prometheus", + log: log.With(zap.String("service", "Prometheus")), + } +} diff --git a/metrics/pprof.go b/metrics/pprof.go new file mode 100644 index 0000000..4719a69 --- /dev/null +++ b/metrics/pprof.go @@ -0,0 +1,33 @@ +package metrics + +import ( + "net/http" + "net/http/pprof" + + "go.uber.org/zap" +) + +// NewPprofService creates a new service for gathering pprof metrics. +func NewPprofService(l *zap.Logger, cfg Config) *Service { + handler := http.NewServeMux() + handler.HandleFunc("/debug/pprof/", pprof.Index) + handler.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) + handler.HandleFunc("/debug/pprof/profile", pprof.Profile) + handler.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + handler.HandleFunc("/debug/pprof/trace", pprof.Trace) + + // Manually add support for paths linked to by index page at /debug/pprof/ + for _, item := range []string{"allocs", "block", "heap", "goroutine", "mutex", "threadcreate"} { + handler.Handle("/debug/pprof/"+item, pprof.Handler(item)) + } + + return &Service{ + Server: &http.Server{ + Addr: cfg.Address, + Handler: handler, + }, + enabled: cfg.Enabled, + serviceType: "Pprof", + log: l.With(zap.String("service", "Pprof")), + } +} diff --git a/metrics/service.go b/metrics/service.go new file mode 100644 index 0000000..6070462 --- /dev/null +++ b/metrics/service.go @@ -0,0 +1,44 @@ +package metrics + +import ( + "context" + "net/http" + + "go.uber.org/zap" +) + +// Service serves metrics. +type Service struct { + *http.Server + enabled bool + log *zap.Logger + serviceType string +} + +// Config is a params to configure service. +type Config struct { + Address string + Enabled bool +} + +// Start runs http service with the exposed endpoint on the configured port. +func (ms *Service) Start() { + if ms.enabled { + ms.log.Info("service is running", zap.String("endpoint", ms.Addr)) + err := ms.ListenAndServe() + if err != nil && err != http.ErrServerClosed { + ms.log.Warn("service couldn't start on configured port") + } + } else { + ms.log.Info("service hasn't started since it's disabled") + } +} + +// ShutDown stops the service. +func (ms *Service) ShutDown(ctx context.Context) { + ms.log.Info("shutting down service", zap.String("endpoint", ms.Addr)) + err := ms.Shutdown(ctx) + if err != nil { + ms.log.Panic("can't shut down service", zap.Error(err)) + } +} diff --git a/templates/server/server.gotmpl b/templates/server/server.gotmpl index a4376d3..3b0e098 100644 --- a/templates/server/server.gotmpl +++ b/templates/server/server.gotmpl @@ -63,6 +63,8 @@ type ServerConfig struct { TLSCertificate string TLSCertificateKey string TLSCACertificate string + + SuccessfulStartCallback func() } // NewServer creates a new api {{ humanize .Name }} server but does not configure it @@ -90,6 +92,7 @@ func NewServer(api *{{ .APIPackageAlias }}.{{ pascalize .Name }}API, cfg *Server s.TLSWriteTimeout = cfg.TLSWriteTimeout s.shutdown = make(chan struct{}) s.api = api + s.startCallback = cfg.SuccessfulStartCallback s.interrupt = make(chan os.Signal, 1) return s } @@ -125,6 +128,8 @@ type Server struct { TLSWriteTimeout time.Duration httpsServerL net.Listener + startCallback func() + cfgTLSFn func (tlsConfig *tls.Config) cfgServerFn func(s *http.Server, scheme, addr string) @@ -329,6 +334,10 @@ func (s *Server) Serve() (err error) { wg.Add(1) go s.handleShutdown(wg, &servers) + if s.startCallback != nil { + s.startCallback() + } + wg.Wait() return nil }