realloc/frostfs-rest-gw
1
0
Fork 0
forked from TrueCloudLab/frostfs-rest-gw
Code Issues Pull requests Projects Releases Packages Wiki Activity

[#66] Group pool config parameters

Browse source 
Signed-off-by: Denis Kirillov <denis@nspcc.ru>
This commit is contained in:
Denis Kirillov 2022-10-13 09:41:57 +03:00 committed by Alex Vanin
parent 8d83320120
commit ac5750670f
6 changed files with 131 additions and 100 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
CHANGELOG.md
View file

@ -7,6 +7,19 @@ This document outlines major changes between releases.
### Added ### Added
- Stop pool dial on SIGINT (#76) - Stop pool dial on SIGINT (#76)
### Changed
- Pool configuration parameters (#66)
### Updating from v0.5.0
Now all pool config parameters moved to `pool` section. So you need to change:
* `peers` -> `pool.peers` (`REST_GW_PEERS` -> `REST_GW_POOL_PEERS`)
* `node-dial-timeout` -> `pool.node-dial-timeout` (`REST_GW_NODE_DIAL_TIMEOUT` -> `REST_GW_POOL_NODE_DIAL_TIMEOUT`)
* `healthcheck-timeout` -> `pool.healthcheck-timeout` (`REST_GW_HEALTHCHECK_TIMEOUT` -> `REST_GW_POOL_HEALTHCHECK_TIMEOUT`)
* `rebalance-timer` -> `pool.rebalance-timer` (`REST_GW_REBALANCE_TIMER` -> `REST_GW_POOL_REBALANCE_TIMER`)
* `pool-error-threshold` -> `pool.error-threshold`
## [0.5.0] "Undercity" - 2022-10-07 ## [0.5.0] "Undercity" - 2022-10-07
### Added ### Added

8
README.md
View file

@ -90,8 +90,8 @@ Or you can also use a [Docker image](https://hub.docker.com/r/nspccdev/neofs-res
## Execution ## Execution
REST gateway itself is not a NeoFS node, so to access NeoFS it uses node's gRPC interface and you need to provide some REST gateway itself is not a NeoFS node, so to access NeoFS it uses node's gRPC interface and you need to provide some
node that it will connect to. This can be done either via `-p` parameter or via `REST_GW_PEERS_<N>_ADDRESS` and node that it will connect to. This can be done either via `-p` parameter or via `REST_GW_POOL_PEERS_<N>_ADDRESS` and
`REST_GW_PEERS_<N>_WEIGHT` environment variables (the gate supports multiple NeoFS nodes with weighted load balancing). `REST_GW_POOL_PEERS_<N>_WEIGHT` environment variables (the gate supports multiple NeoFS nodes with weighted load balancing).
If you're launching REST gateway in bundle with [neofs-dev-env](https://github.com/nspcc-dev/neofs-dev-env), you can get If you're launching REST gateway in bundle with [neofs-dev-env](https://github.com/nspcc-dev/neofs-dev-env), you can get
an IP address of the node in output of `make hosts` command an IP address of the node in output of `make hosts` command
@ -102,14 +102,14 @@ settings):
```shell ```shell
$ neofs-rest-gw -p 192.168.130.72:8080 $ neofs-rest-gw -p 192.168.130.72:8080
$ REST_GW_PEERS_0_ADDRESS=192.168.130.72:8080 neofs-rest-gw $ REST_GW_POOL_PEERS_0_ADDRESS=192.168.130.72:8080 neofs-rest-gw
``` ```
It's also possible to specify uri scheme (grpc or grpcs) when using `-p`: It's also possible to specify uri scheme (grpc or grpcs) when using `-p`:
```shell ```shell
$ neofs-rest-gw -p grpc://192.168.130.72:8080 $ neofs-rest-gw -p grpc://192.168.130.72:8080
$ REST_GW_PEERS_0_ADDRESS=grpcs://192.168.130.72:8080 neofs-rest-gw $ REST_GW_POOL_PEERS_0_ADDRESS=grpcs://192.168.130.72:8080 neofs-rest-gw
``` ```
## Configuration ## Configuration

33
cmd/neofs-rest-gw/config.go
View file

@ -35,10 +35,15 @@ const (
defaultPoolErrorThreshold uint32 = 100 defaultPoolErrorThreshold uint32 = 100
// Pool config. // Pool config.
cfgNodeDialTimeout = "node-dial-timeout" cmdNodeDialTimeout = "node-dial-timeout"
cfgHealthcheckTimeout = "healthcheck-timeout" cfgNodeDialTimeout = "pool." + cmdNodeDialTimeout
cfgRebalance = "rebalance-timer" cmdHealthcheckTimeout = "healthcheck-timeout"
cfgPoolErrorThreshold = "pool-error-threshold" cfgHealthcheckTimeout = "pool." + cmdHealthcheckTimeout
cmdRebalance = "rebalance-timer"
cfgRebalance = "pool." + cmdRebalance
cfgPoolErrorThreshold = "pool.error-threshold"
cmdPeers = "peers"
cfgPeers = "pool." + cmdPeers
// Metrics / Profiler. // Metrics / Profiler.
cfgPrometheusEnabled = "prometheus.enabled" cfgPrometheusEnabled = "prometheus.enabled"
@ -54,9 +59,6 @@ const (
cfgWalletAddress = "wallet.address" cfgWalletAddress = "wallet.address"
cfgWalletPassphrase = "wallet.passphrase" cfgWalletPassphrase = "wallet.passphrase"
// Peers.
cfgPeers = "peers"
// Command line args. // Command line args.
cmdHelp = "help" cmdHelp = "help"
cmdVersion = "version" cmdVersion = "version"
@ -104,11 +106,11 @@ func config() *viper.Viper {
flagSet.StringP(cmdWallet, "w", "", `path to the wallet`) flagSet.StringP(cmdWallet, "w", "", `path to the wallet`)
flagSet.String(cmdAddress, "", `address of wallet account`) flagSet.String(cmdAddress, "", `address of wallet account`)
config := flagSet.String(cmdConfig, "", "config path") config := flagSet.String(cmdConfig, "", "config path")
flagSet.Duration(cfgNodeDialTimeout, defaultConnectTimeout, "gRPC node connect timeout") flagSet.Duration(cmdNodeDialTimeout, defaultConnectTimeout, "gRPC node connect timeout")
flagSet.Duration(cfgHealthcheckTimeout, defaultHealthcheckTimeout, "gRPC healthcheck timeout") flagSet.Duration(cmdHealthcheckTimeout, defaultHealthcheckTimeout, "gRPC healthcheck timeout")
flagSet.Duration(cfgRebalance, defaultRebalanceTimer, "gRPC connection rebalance timer") flagSet.Duration(cmdRebalance, defaultRebalanceTimer, "gRPC connection rebalance timer")
peers := flagSet.StringArrayP(cfgPeers, "p", nil, "NeoFS nodes") peers := flagSet.StringArrayP(cmdPeers, "p", nil, "NeoFS nodes")
// init server flags // init server flags
restapi.BindDefaultFlags(flagSet) restapi.BindDefaultFlags(flagSet)
@ -131,6 +133,15 @@ func config() *viper.Viper {
if err := v.BindPFlag(cfgPrometheusEnabled, flagSet.Lookup(cmdMetrics)); err != nil { if err := v.BindPFlag(cfgPrometheusEnabled, flagSet.Lookup(cmdMetrics)); err != nil {
panic(err) panic(err)
} }
if err := v.BindPFlag(cfgNodeDialTimeout, flagSet.Lookup(cmdNodeDialTimeout)); err != nil {
panic(err)
}
if err := v.BindPFlag(cfgHealthcheckTimeout, flagSet.Lookup(cmdHealthcheckTimeout)); err != nil {
panic(err)
}
if err := v.BindPFlag(cfgRebalance, flagSet.Lookup(cmdRebalance)); err != nil {
panic(err)
}
if err := v.BindPFlags(flagSet); err != nil { if err := v.BindPFlags(flagSet); err != nil {
panic(err) panic(err)

24
config/config.env
View file

@ -20,28 +20,28 @@ REST_GW_LOGGER_LEVEL=debug
# while it's healthy. Otherwise, gateway use the second node (grpc://s01.neofs.devenv:8080) # while it's healthy. Otherwise, gateway use the second node (grpc://s01.neofs.devenv:8080)
# for 10% of requests and the third node for 90% of requests. # for 10% of requests and the third node for 90% of requests.
# Endpoint. # Endpoint.
REST_GW_PEERS_0_ADDRESS=grpc://s01.neofs.devenv:8080 REST_GW_POOL_PEERS_0_ADDRESS=grpc://s01.neofs.devenv:8080
# Until nodes with the same priority level are healthy # Until nodes with the same priority level are healthy
# nodes with other priority are not used. # nodes with other priority are not used.
# Еhe lower the value, the higher the priority. # Еhe lower the value, the higher the priority.
REST_GW_PEERS_0_PRIORITY=1 REST_GW_POOL_PEERS_0_PRIORITY=1
# Load distribution proportion for nodes with the same priority. # Load distribution proportion for nodes with the same priority.
REST_GW_PEERS_0_WEIGHT=1 REST_GW_POOL_PEERS_0_WEIGHT=1
REST_GW_PEERS_1_ADDRESS=grpc://s02.neofs.devenv:8080 REST_GW_POOL_PEERS_1_ADDRESS=grpc://s02.neofs.devenv:8080
REST_GW_PEERS_1_PRIORITY=2 REST_GW_POOL_PEERS_1_PRIORITY=2
REST_GW_PEERS_1_WEIGHT=1 REST_GW_POOL_PEERS_1_WEIGHT=1
REST_GW_PEERS_2_ADDRESS=grpc://s03.neofs.devenv:8080 REST_GW_POOL_PEERS_2_ADDRESS=grpc://s03.neofs.devenv:8080
REST_GW_PEERS_2_PRIORITY=2 REST_GW_POOL_PEERS_2_PRIORITY=2
REST_GW_PEERS_3_WEIGHT=9 REST_GW_POOL_PEERS_3_WEIGHT=9
# Timeout to dial node. # Timeout to dial node.
REST_GW_NODE_DIAL_TIMEOUT=10s REST_GW_POOL_NODE_DIAL_TIMEOUT=10s
# Timeout to check node health during rebalance. # Timeout to check node health during rebalance.
REST_GW_HEALTHCHECK_TIMEOUT=15s REST_GW_POOL_HEALTHCHECK_TIMEOUT=15s
# Interval to check nodes health. # Interval to check nodes health.
REST_GW_REBALANCE_TIMER=60s REST_GW_POOL_REBALANCE_TIMER=60s
# The number of errors on connection after which node is considered as unhealthy. # The number of errors on connection after which node is considered as unhealthy.
REST_GW_POOL_ERROR_THRESHOLD=100 REST_GW_POOL_ERROR_THRESHOLD=100

61
config/config.yaml
View file

@ -17,37 +17,38 @@ logger:
# Log level. # Log level.
level: debug level: debug
# Nodes configuration. pool:
# This configuration make gateway use the first node (grpc://s01.neofs.devenv:8080) # Timeout to dial node.
# while it's healthy. Otherwise, gateway use the second node (grpc://s01.neofs.devenv:8080) node-dial-timeout: 5s
# for 10% of requests and the third node for 90% of requests. # Timeout to check node health during rebalance.
peers: healthcheck-timeout: 5s
0: # Interval to check nodes' health.
# Endpoint. rebalance-timer: 30s
address: grpc://s01.neofs.devenv:8080 # The number of errors on connection after which node is considered as unhealthy.
# Until nodes with the same priority level are healthy error-threshold: 100
# nodes with other priority are not used.
# Еhe lower the value, the higher the priority.
priority: 1
# Load distribution proportion for nodes with the same priority.
weight: 1
1:
address: grpc://s02.neofs.devenv:8080
priority: 2
weight: 1
2:
address: grpc://s03.neofs.devenv:8080
priority: 2
weight: 9
# Timeout to dial node. # Nodes configuration.
node-dial-timeout: 5s # This configuration make gateway use the first node (grpc://s01.neofs.devenv:8080)
# Timeout to check node health during rebalance. # while it's healthy. Otherwise, gateway use the second node (grpc://s01.neofs.devenv:8080)
healthcheck-timeout: 5s # for 10% of requests and the third node for 90% of requests.
# Interval to check nodes health. peers:
rebalance-timer: 30s 0:
# The number of errors on connection after which node is considered as unhealthy. # Endpoint.
pool-error-threshold: 100 address: grpc://s01.neofs.devenv:8080
# Until nodes with the same priority level are healthy
# nodes with other priority are not used.
# Еhe lower the value, the higher the priority.
priority: 1
# Load distribution proportion for nodes with the same priority.
weight: 1
1:
address: grpc://s02.neofs.devenv:8080
priority: 2
weight: 1
2:
address: grpc://s03.neofs.devenv:8080
priority: 2
weight: 9
# The listeners to enable, this can be repeated and defaults to the schemes in the swagger spec. # The listeners to enable, this can be repeated and defaults to the schemes in the swagger spec.
scheme: [ http ] scheme: [ http ]

92
docs/gate-configuration.md
View file

@ -10,23 +10,18 @@ There are some custom types used for brevity:
# Structure # Structure
| Section | Description | | Section | Description |
|-----------------|-------------------------------------------------------| |--------------|-------------------------------------------------|
| no section | [General parameters](#general-section) | | no section | [General parameters](#general-section) |
| `wallet` | [Wallet configuration](#wallet-section) | | `wallet` | [Wallet configuration](#wallet-section) |
| `peers` | [Nodes configuration](#peers-section) | | `pool` | [Pool configuration](#pool-section) |
| `logger` | [Logger configuration](#logger-section) | | `logger` | [Logger configuration](#logger-section) |
| `pprof` | [Pprof configuration](#pprof-section) | | `pprof` | [Pprof configuration](#pprof-section) |
| `prometheus` | [Prometheus configuration](#prometheus-section) | | `prometheus` | [Prometheus configuration](#prometheus-section) |
# General section # General section
```yaml ```yaml
node-dial-timeout: 10s
healthcheck-timeout: 15s
rebalance-timer: 60s
pool-error-threshold: 100
scheme: [ http ] scheme: [ http ]
cleanup-timeout: 10s cleanup-timeout: 10s
graceful-timeout: 15s graceful-timeout: 15s
@ -50,10 +45,6 @@ tls-write-timeout: 30s
| Parameter | Type | Default value | Description | | Parameter | Type | Default value | Description |
|------------------------|------------|------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| |------------------------|------------|------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `node-dial-timeout` | `duration` | `10s` | Timeout to connect to a node. |
| `healthcheck-timeout` | `duration` | `15s` | Timeout to check node health during rebalance. |
| `rebalance-timer` | `duration` | `60s` | Interval to check node health. |
| `pool-error-threshold` | `uint32` | `100` | The number of errors on connection after which node is considered as unhealthy. |
| `scheme` | `[]string` | `[http]` | The listeners to enable, this can be repeated and defaults to the schemes in the swagger spec. | | `scheme` | `[]string` | `[http]` | The listeners to enable, this can be repeated and defaults to the schemes in the swagger spec. |
| `cleanup-timeout` | `duration` | `10s` | Grace period for which to wait before killing idle connections. | | `cleanup-timeout` | `duration` | `10s` | Grace period for which to wait before killing idle connections. |
| `graceful-timeout` | `duration` | `15s` | Grace period for which to wait before shutting down the server. | | `graceful-timeout` | `duration` | `15s` | Grace period for which to wait before shutting down the server. |
@ -87,36 +78,51 @@ wallet:
| `address` | `string` | | Account address to get from wallet. If omitted default one will be used. | | `address` | `string` | | Account address to get from wallet. If omitted default one will be used. |
| `passphrase` | `string` | | Passphrase to decrypt wallet. | | `passphrase` | `string` | | Passphrase to decrypt wallet. |
# `peers` section # `pool` section
```yaml ```yaml
# Nodes configuration pool:
# This configuration makes the gateway use the first node (node1.neofs:8080) node-dial-timeout: 10s
# while it's healthy. Otherwise, gateway uses the second node (node2.neofs:8080) healthcheck-timeout: 15s
# for 10% of requests and the third node (node3.neofs:8080) for 90% of requests. rebalance-timer: 60s
# Until nodes with the same priority level are healthy error-threshold: 100
# nodes with other priority are not used.
# The lower the value, the higher the priority. # Nodes configuration
peers: # This configuration makes the gateway use the first node (node1.neofs:8080)
0: # while it's healthy. Otherwise, gateway uses the second node (node2.neofs:8080)
address: node1.neofs:8080 # for 10% of requests and the third node (node3.neofs:8080) for 90% of requests.
priority: 1 # Until nodes with the same priority level are healthy
weight: 1 # nodes with other priority are not used.
1: # The lower the value, the higher the priority.
address: node2.neofs:8080 peers:
priority: 2 0:
weight: 0.1 address: node1.neofs:8080
2: priority: 1
address: node3.neofs:8080 weight: 1
priority: 2 1:
weight: 0.9 address: node2.neofs:8080
priority: 2
weight: 0.1
2:
address: node3.neofs:8080
priority: 2
weight: 0.9
``` ```
| Parameter | Type | Default value | Description | | Parameter | Type | Default value | Description |
|------------|----------|---------------|---------------------------------------------------------------------------------------------------------------------------------------------------------| |-----------------------|------------|---------------|---------------------------------------------------------------------------------|
| `address` | `string` | | Address of storage node. | | `node-dial-timeout` | `duration` | `10s` | Timeout to connect to a node. |
| `priority` | `int` | `1` | It allows to group nodes and don't switch group until all nodes with the same priority will be unhealthy. The lower the value, the higher the priority. | | `healthcheck-timeout` | `duration` | `15s` | Timeout to check node health during rebalance. |
| `weight` | `float` | `1` | Weight of node in the group with the same priority. Distribute requests to nodes proportionally to these values. | | `rebalance-timer` | `duration` | `60s` | Interval to check node health. |
| `error-threshold` | `uint32` | `100` | The number of errors on connection after which node is considered as unhealthy. |
## `peers` section
| Parameter | Type | Default value | Description |
|------------------------|------------|---------------|---------------------------------------------------------------------------------------------------------------------------------------------------------|
| `address` | `string` | | Address of storage node. |
| `priority` | `int` | `1` | It allows to group nodes and don't switch group until all nodes with the same priority will be unhealthy. The lower the value, the higher the priority. |
| `weight` | `float` | `1` | Weight of node in the group with the same priority. Distribute requests to nodes proportionally to these values. |
# `logger` section # `logger` section