diff --git a/CHANGELOG.md b/CHANGELOG.md index 9038867..31211c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ This document outlines major changes between releases. ### Added - CORS headers (#39) - Expose metrics (#44) +- Documentation for default params (#45) ## [0.2.1] "Razor Hill" - 2022-07-22 diff --git a/README.md b/README.md index 608150e..0dbf804 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,6 @@ Before building make sure you have the following tools: * git * curl - To build neofs-rest-gw call `make` the cloned repository (the binary will end up in `bin/neofs-rest-gw`). Notable make targets: @@ -59,14 +58,14 @@ an IP address of the node in output of `make hosts` command These two commands are functionally equivalent, they run the gate with one backend node (and otherwise default settings): -``` +```shell $ neofs-rest-gw -p 192.168.130.72:8080 $ REST_GW_PEERS_0_ADDRESS=192.168.130.72:8080 neofs-rest-gw ``` It's also possible to specify uri scheme (grpc or grpcs) when using `-p`: -``` +```shell $ neofs-rest-gw -p grpc://192.168.130.72:8080 $ REST_GW_PEERS_0_ADDRESS=grpcs://192.168.130.72:8080 neofs-rest-gw ``` @@ -74,19 +73,24 @@ $ REST_GW_PEERS_0_ADDRESS=grpcs://192.168.130.72:8080 neofs-rest-gw ## Configuration In general, everything available as CLI parameter can also be specified via environment variables, so they're not -specifically mentioned in most cases -(see `--help` also). If you prefer a config file you can use it in yaml format. See config [examples](./config) for -details. +specifically mentioned in most cases (see `--help` also). If you prefer a config file you can use it in yaml format. +See [config](./config/config.yaml) and [defaults](./docs/gate-configuration.md) for example. + +```shell +$ neofs-rest-gw --config config.yaml +``` ## Docs -You can see additional docs and swagger specification using the following url (suppose you ran rest-gw on `localhost:8090`): + +You can see additional docs and swagger specification using the following url +(suppose you ran rest-gw on `localhost:8090`): + * http://localhost:8090/docs - rest-gw documentation * http://localhost:8090/v1/docs - swagger specification ## Contributing -Feel free to contribute to this project after reading the [contributing -guidelines](CONTRIBUTING.md). +Feel free to contribute to this project after reading the [contributing guidelines](CONTRIBUTING.md). Before starting to work on a certain topic, create a new issue first, describing the feature/topic you are going to implement. diff --git a/cmd/neofs-rest-gw/config.go b/cmd/neofs-rest-gw/config.go index 156535b..01b487d 100644 --- a/cmd/neofs-rest-gw/config.go +++ b/cmd/neofs-rest-gw/config.go @@ -25,16 +25,19 @@ import ( ) const ( - defaultRebalanceTimer = 15 * time.Second - defaultRequestTimeout = 15 * time.Second - defaultConnectTimeout = 30 * time.Second + defaultConnectTimeout = 10 * time.Second + defaultHealthcheckTimeout = 15 * time.Second + defaultRebalanceTimer = 60 * time.Second defaultShutdownTimeout = 15 * time.Second - // Timeouts. + defaultPoolErrorThreshold uint32 = 100 + + // Pool config. cfgNodeDialTimeout = "node-dial-timeout" cfgHealthcheckTimeout = "healthcheck-timeout" cfgRebalance = "rebalance-timer" + cfgPoolErrorThreshold = "pool-error-threshold" // Metrics / Profiler. cfgPrometheusEnabled = "prometheus.enabled" @@ -100,8 +103,8 @@ func config() *viper.Viper { flagSet.StringP(cmdWallet, "w", "", `path to the wallet`) flagSet.String(cmdAddress, "", `address of wallet account`) config := flagSet.String(cmdConfig, "", "config path") - flagSet.Duration(cfgNodeDialTimeout, defaultConnectTimeout, "gRPC connect timeout") - flagSet.Duration(cfgHealthcheckTimeout, defaultRequestTimeout, "gRPC request timeout") + flagSet.Duration(cfgNodeDialTimeout, defaultConnectTimeout, "gRPC node connect timeout") + flagSet.Duration(cfgHealthcheckTimeout, defaultHealthcheckTimeout, "gRPC healthcheck timeout") flagSet.Duration(cfgRebalance, defaultRebalanceTimer, "gRPC connection rebalance timer") peers := flagSet.StringArrayP(cfgPeers, "p", nil, "NeoFS nodes") @@ -110,6 +113,8 @@ func config() *viper.Viper { restapi.BindDefaultFlags(flagSet) // set defaults: + // pool + v.SetDefault(cfgPoolErrorThreshold, defaultPoolErrorThreshold) // metrics v.SetDefault(cfgPprofAddress, "localhost:8091") @@ -316,6 +321,7 @@ func newNeofsAPI(ctx context.Context, logger *zap.Logger, v *viper.Viper) (*hand prm.SetNodeDialTimeout(v.GetDuration(cfgNodeDialTimeout)) prm.SetHealthcheckTimeout(v.GetDuration(cfgHealthcheckTimeout)) prm.SetClientRebalanceInterval(v.GetDuration(cfgRebalance)) + prm.SetErrorThreshold(v.GetUint32(cfgPoolErrorThreshold)) for _, peer := range fetchPeers(logger, v) { prm.AddNode(peer) diff --git a/config/config.env b/config/config.env index 66b5650..deb58cb 100644 --- a/config/config.env +++ b/config/config.env @@ -37,11 +37,13 @@ REST_GW_PEERS_2_PRIORITY=2 REST_GW_PEERS_3_WEIGHT=9 # Timeout to dial node. -node_dial_timeout=5s +REST_GW_NODE_DIAL_TIMEOUT=10s # Timeout to check node health during rebalance. -healthcheck_timeout=5s +REST_GW_HEALTHCHECK_TIMEOUT=15s # Interval to check nodes health. -rebalance_timer=30s +REST_GW_REBALANCE_TIMER=60s +# The number of errors on connection after which node is considered as unhealthy. +REST_GW_POOL_ERROR_THRESHOLD=100 # Grace period for which to wait before killing idle connections REST_GW_CLEANUP_TIMEOUT=10s @@ -80,5 +82,3 @@ REST_GW_TLS_KEEP_ALIVE=3m REST_GW_TLS_READ_TIMEOUT=30s # Maximum duration before timing out write of the response. REST_GW_TLS_WRITE_TIMEOUT=30s - - diff --git a/config/config.yaml b/config/config.yaml index cf0767a..f562b8d 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -45,7 +45,9 @@ node-dial-timeout: 5s # Timeout to check node health during rebalance. healthcheck-timeout: 5s # Interval to check nodes health. -rebalance_timer: 30s +rebalance-timer: 30s +# The number of errors on connection after which node is considered as unhealthy. +pool-error-threshold: 100 # The listeners to enable, this can be repeated and defaults to the schemes in the swagger spec. scheme: [ http ] @@ -86,5 +88,3 @@ tls-keep-alive: 3m tls-read-timeout: 30s # Maximum duration before timing out write of the response. tls-write-timeout: 30s - - diff --git a/docs/gate-configuration.md b/docs/gate-configuration.md new file mode 100644 index 0000000..4991a74 --- /dev/null +++ b/docs/gate-configuration.md @@ -0,0 +1,160 @@ +# NeoFS REST Gateway configuration file + +This section contains detailed NeoFS REST Gateway configuration file description +including default config values and some tips to set up configurable values. + +There are some custom types used for brevity: + +* `duration` -- string consisting of a number and a suffix. Suffix examples include `s` (seconds), `m` (minutes), `ms` ( + milliseconds). + +# Structure + +| Section | Description | +|-----------------|-------------------------------------------------------| +| no section | [General parameters](#general-section) | +| `wallet` | [Wallet configuration](#wallet-section) | +| `peers` | [Nodes configuration](#peers-section) | +| `logger` | [Logger configuration](#logger-section) | +| `pprof` | [Pprof configuration](#pprof-section) | +| `prometheus` | [Prometheus configuration](#prometheus-section) | + +# General section + +```yaml +node-dial-timeout: 10s +healthcheck-timeout: 15s +rebalance-timer: 60s +pool-error-threshold: 100 + +scheme: [ http ] +cleanup-timeout: 10s +graceful-timeout: 15s +max-header-size: 1000000 + +listen-address: localhost:8080 +listen-limit: 0 +keep-alive: 3m +read-timeout: 30s +write-timeout: 30s + +tls-listen-address: localhost:8081 +tls-certificate: /path/to/tls/cert +tls-key: /path/to/tls/key +tls-ca: /path/to/tls/ca +tls-listen-limit: 0 +tls-keep-alive: 3m +tls-read-timeout: 30s +tls-write-timeout: 30s +``` + +| Parameter | Type | Default value | Description | +|------------------------|------------|------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `node-dial-timeout` | `duration` | `10s` | Timeout to connect to a node. | +| `healthcheck-timeout` | `duration` | `15s` | Timeout to check node health during rebalance. | +| `rebalance-timer` | `duration` | `60s` | Interval to check node health. | +| `pool-error-threshold` | `uint32` | `100` | The number of errors on connection after which node is considered as unhealthy. | +| `scheme` | `[]string` | `[http]` | The listeners to enable, this can be repeated and defaults to the schemes in the swagger spec. | +| `cleanup-timeout` | `duration` | `10s` | Grace period for which to wait before killing idle connections. | +| `graceful-timeout` | `duration` | `15s` | Grace period for which to wait before shutting down the server. | +| `max-header-size` | `int` | `1000000` | Controls the maximum number of bytes the server will read parsing the request header's keys and values, including the request line. It does not limit the size of the request body. | +| `listen-address` | `string` | `localhost:8080` | The IP and port to listen on. | +| `listen-limit` | `int` | `0` | Limit the number of outstanding requests. `0` means no limit | | +| `keep-alive` | `duration` | `3m` | Sets the TCP keep-alive timeouts on accepted connections. | +| `read-timeout` | `duration` | `30s` | Maximum duration before timing out read of the request. It prunes dead TCP connections (e.g. closing laptop mid-download). | +| `write-timeout` | `duration` | `30s` | Maximum duration before timing out write of the response. | +| `tls-listen-address` | `string` | `localhost:8081` | The IP and port to listen on for TLS. | +| `tls-certificate` | `string` | | The certificate file to use for secure connections. | +| `tls-key` | `string` | | The private key file to use for secure connections (without passphrase). | +| `tls-ca` | `string` | | The certificate authority certificate file to be used with mutual tls auth. | +| `tls-listen-limit` | `int` | `0` | Limit the number of outstanding requests for TLS. `0` means no limit | | +| `tls-keep-alive` | `duration` | `3m` | Sets the TCP keep-alive timeouts on accepted connections for TLS. | +| `tls-read-timeout` | `duration` | `30s` | Maximum duration before timing out read of the request for TLS. It prunes dead TCP connections (e.g. closing laptop mid-download). | +| `tls-write-timeout` | `duration` | `30s` | Maximum duration before timing out write of the response for TLS. | + +# `wallet` section + +```yaml +wallet: + path: /path/to/wallet.json + address: NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP + passphrase: pwd +``` + +| Parameter | Type | Default value | Description | +|--------------|----------|---------------|--------------------------------------------------------------------------| +| `path` | `string` | | Path to the wallet. | +| `address` | `string` | | Account address to get from wallet. If omitted default one will be used. | +| `passphrase` | `string` | | Passphrase to decrypt wallet. | + +# `peers` section + +```yaml +# Nodes configuration +# This configuration makes the gateway use the first node (node1.neofs:8080) +# while it's healthy. Otherwise, gateway uses the second node (node2.neofs:8080) +# for 10% of requests and the third node (node3.neofs:8080) for 90% of requests. +# Until nodes with the same priority level are healthy +# nodes with other priority are not used. +# The lower the value, the higher the priority. +peers: + 0: + address: node1.neofs:8080 + priority: 1 + weight: 1 + 1: + address: node2.neofs:8080 + priority: 2 + weight: 0.1 + 2: + address: node3.neofs:8080 + priority: 2 + weight: 0.9 +``` + +| Parameter | Type | Default value | Description | +|------------|----------|---------------|---------------------------------------------------------------------------------------------------------------------------------------------------------| +| `address` | `string` | | Address of storage node. | +| `priority` | `int` | `1` | It allows to group nodes and don't switch group until all nodes with the same priority will be unhealthy. The lower the value, the higher the priority. | +| `weight` | `float` | `1` | Weight of node in the group with the same priority. Distribute requests to nodes proportionally to these values. | + +# `logger` section + +```yaml +logger: + level: debug +``` + +| Parameter | Type | Default value | Description | +|-----------|----------|---------------|----------------------------------------------------------------------------------------------------| +| `level` | `string` | `debug` | Logging level.
Possible values: `debug`, `info`, `warn`, `error`, `dpanic`, `panic`, `fatal`. | + +# `pprof` section + +Contains configuration for the `pprof` profiler. + +```yaml +pprof: + enabled: true + address: localhost:8091 +``` + +| Parameter | Type | Default value | Description | +|-----------|----------|------------------|-----------------------------------------| +| `enabled` | `bool` | `false` | Flag to enable the service. | +| `address` | `string` | `localhost:8091` | Address that service listener binds to. | + +# `prometheus` section + +Contains configuration for the `prometheus` metrics service. + +```yaml +prometheus: + enabled: true + address: localhost:8092 +``` + +| Parameter | Type | Default value | Description | +|-----------|----------|------------------|-----------------------------------------| +| `enabled` | `bool` | `false` | Flag to enable the service. | +| `address` | `string` | `localhost:8092` | Address that service listener binds to. |