From 976f16803d5b901e171b39d7cc03fb19286af22e Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 28 Apr 2023 11:14:14 +0300 Subject: [PATCH 001/107] [#18] frostfs-adm.yml: Add alphabet-wallets Signed-off-by: Evgenii Stratonikov --- Makefile | 4 ++-- frostfs-adm.yml | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 61782ba..1f00ce8 100644 --- a/Makefile +++ b/Makefile @@ -75,8 +75,8 @@ up/basic: up/bootstrap up/bootstrap: get vendor/hosts @$(foreach SVC, $(START_BOOTSTRAP), $(shell docker-compose -f services/$(SVC)/docker-compose.yml up -d)) @source ./bin/helper.sh - @./vendor/frostfs-adm --config frostfs-adm.yml morph init --alphabet-wallets ./services/ir --contracts vendor/contracts || die "Failed to initialize Alphabet wallets" - @for f in ./services/storage/wallet*.json; do echo "Transfer GAS to wallet $${f}" && ./vendor/frostfs-adm -c frostfs-adm.yml morph refill-gas --storage-wallet $${f} --gas 10.0 --alphabet-wallets services/ir || die "Failed to transfer GAS to alphabet wallets"; done + @./vendor/frostfs-adm --config frostfs-adm.yml morph init --contracts vendor/contracts || die "Failed to initialize Alphabet wallets" + @for f in ./services/storage/wallet*.json; do echo "Transfer GAS to wallet $${f}" && ./vendor/frostfs-adm -c frostfs-adm.yml morph refill-gas --storage-wallet $${f} --gas 10.0 || die "Failed to transfer GAS to alphabet wallets"; done @echo "FrostFS sidechain environment is deployed" # Build up certain service diff --git a/frostfs-adm.yml b/frostfs-adm.yml index dcc4ba2..c44046d 100644 --- a/frostfs-adm.yml +++ b/frostfs-adm.yml @@ -1,4 +1,5 @@ rpc-endpoint: http://morph-chain.frostfs.devenv:30333 +alphabet-wallets: ./services/ir network: max_object_size: 67108864 epoch_duration: 240 From db0ebb7349745a11b241c994fc287cf6469f144b Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 28 Apr 2023 11:22:37 +0300 Subject: [PATCH 002/107] [#18] Makefile: Split long `for` line Signed-off-by: Evgenii Stratonikov --- Makefile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1f00ce8..d0dca8a 100644 --- a/Makefile +++ b/Makefile @@ -76,7 +76,11 @@ up/bootstrap: get vendor/hosts @$(foreach SVC, $(START_BOOTSTRAP), $(shell docker-compose -f services/$(SVC)/docker-compose.yml up -d)) @source ./bin/helper.sh @./vendor/frostfs-adm --config frostfs-adm.yml morph init --contracts vendor/contracts || die "Failed to initialize Alphabet wallets" - @for f in ./services/storage/wallet*.json; do echo "Transfer GAS to wallet $${f}" && ./vendor/frostfs-adm -c frostfs-adm.yml morph refill-gas --storage-wallet $${f} --gas 10.0 || die "Failed to transfer GAS to alphabet wallets"; done + @for f in ./services/storage/wallet*.json; do \ + echo "Transfer GAS to wallet $${f}" \ + && ./vendor/frostfs-adm -c frostfs-adm.yml morph refill-gas --storage-wallet $${f} --gas 10.0 \ + || die "Failed to transfer GAS to alphabet wallets"; \ + done @echo "FrostFS sidechain environment is deployed" # Build up certain service From 6d6aaefaec3af2440ffbc62246dc7a04bd19a67a Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 28 Apr 2023 11:24:32 +0300 Subject: [PATCH 003/107] [#18] bin: Replace some commands with frostfs-adm Signed-off-by: Evgenii Stratonikov --- Makefile | 6 ++--- bin/config.sh | 50 ------------------------------------------ bin/resolve.sh | 22 ------------------- bin/tick.sh | 49 ----------------------------------------- frostfs_config.mk | 24 +++++++++----------- services/ir/prepare.mk | 3 +-- 6 files changed, 14 insertions(+), 140 deletions(-) delete mode 100755 bin/config.sh delete mode 100755 bin/resolve.sh delete mode 100755 bin/tick.sh diff --git a/Makefile b/Makefile index d0dca8a..0322582 100644 --- a/Makefile +++ b/Makefile @@ -66,8 +66,8 @@ up: up/basic .PHONY: up/basic up/basic: up/bootstrap @$(foreach SVC, $(START_BASIC), $(shell docker-compose -f services/$(SVC)/docker-compose.yml up -d)) - @./bin/tick.sh - @./bin/config.sh string SystemDNS container + @./vendor/frostfs-adm -c ./frostfs-adm.yml morph force-new-epoch + @./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config SystemDNS=container --force @echo "Basic FrostFS Developer Environment is ready" # Start bootstrap services @@ -75,7 +75,7 @@ up/basic: up/bootstrap up/bootstrap: get vendor/hosts @$(foreach SVC, $(START_BOOTSTRAP), $(shell docker-compose -f services/$(SVC)/docker-compose.yml up -d)) @source ./bin/helper.sh - @./vendor/frostfs-adm --config frostfs-adm.yml morph init --contracts vendor/contracts || die "Failed to initialize Alphabet wallets" + @./vendor/frostfs-adm --config frostfs-adm.yml morph init --contracts vendor/contracts @for f in ./services/storage/wallet*.json; do \ echo "Transfer GAS to wallet $${f}" \ && ./vendor/frostfs-adm -c frostfs-adm.yml morph refill-gas --storage-wallet $${f} --gas 10.0 \ diff --git a/bin/config.sh b/bin/config.sh deleted file mode 100755 index 31b613f..0000000 --- a/bin/config.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env bash - -echo "Running bin/config.sh" - -# Source env settings -. .env -. services/ir/.ir.env -source bin/helper.sh - -# NeoGo binary path. -NEOGO="${NEOGO:-docker exec morph_chain neo-go}" - -# Wallet files to change config value -WALLET="${WALLET:-services/morph_chain/node-wallet.json}" -CONFIG_IMG="${CONFIG_IMG:-/wallets/config.yml}" - -NETMAP_ADDR=$(bin/resolve.sh netmap.frostfs) || die "Failed to resolve 'netmap.frostfs' domain name" - -# FrostFS configuration record: variable type [string|int|etc], -# key is a string and value is a constant of given type -TYPE=${1} -KEY=${2} -VALUE="${3}" - -[ -z "$TYPE" ] && echo "Empty config value type" && exit 1 -[ -z "$KEY" ] && echo "Empty config key" && exit 1 -[ -z "$VALUE" ] && echo "Empty config value" && exit 1 - -# Internal variables -if [[ -z "${FROSTFS_NOTARY_DISABLED}" ]]; then - ADDR=$(jq -r .accounts[2].address < "${WALLET}" || die "Cannot get address from ${WALLET}") -else - ADDR=$(jq -r .accounts[0].address < "${WALLET}" || die "Cannot get address from ${WALLET}") -fi - -# Change config value in side chain -echo "Changing ${KEY} configration value to ${VALUE}" - -# shellcheck disable=SC2086 -${NEOGO} contract invokefunction \ - --wallet-config ${CONFIG_IMG} \ - -a ${ADDR} --force \ - -r http://morph-chain.${LOCAL_DOMAIN}:30333 \ - ${NETMAP_ADDR} \ - setConfig bytes:beefcafe \ - string:${KEY} \ - ${TYPE}:${VALUE} -- ${ADDR} || exit 1 - -# Update epoch to apply new configuration value -./bin/tick.sh diff --git a/bin/resolve.sh b/bin/resolve.sh deleted file mode 100755 index 9aba65b..0000000 --- a/bin/resolve.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash - -# Source env settings -. .env -source bin/helper.sh - -# NeoGo binary path. -NEOGO="${NEOGO:-docker exec morph_chain neo-go}" -# NNS contract script hash -output=$(curl -s --data '{ "id": 1, "jsonrpc": "2.0", "method": "getcontractstate", "params": [1] }' \ - "http://morph-chain.${LOCAL_DOMAIN}:30333/") \ - || die "Cannot fetch NNS contract state" - -NNS_ADDR=$(jq -r '.result.hash' <<< "$output") \ - || die "Cannot parse NNS contract hash: $NNS_ADDR" - -${NEOGO} contract testinvokefunction \ - -r "http://morph-chain.${LOCAL_DOMAIN}:30333" \ - "${NNS_ADDR}" resolve string:"${1}" int:16 \ - | jq -r '.stack[0].value | if type=="array" then .[0].value else . end' \ - | base64 -d \ - || die "Cannot invoke 'NNS.resolve' $output" diff --git a/bin/tick.sh b/bin/tick.sh deleted file mode 100755 index 75482f0..0000000 --- a/bin/tick.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env bash - -echo "Running bin/tick.sh" - -# Source env settings -. .env -. services/ir/.ir.env -source bin/helper.sh - -# NeoGo binary path. -NEOGO="${NEOGO:-docker exec morph_chain neo-go}" - -# Wallet files to change config value -WALLET="${WALLET:-services/morph_chain/node-wallet.json}" -CONFIG_IMG="${CONFIG_IMG:-/wallets/config.yml}" - -# Internal variables -if [[ -z "${FROSTFS_NOTARY_DISABLED}" ]]; then - ADDR=$(jq -r .accounts[2].address < "${WALLET}" || die "Cannot get address from ${WALLET}") -else - ADDR=$(jq -r .accounts[0].address < "${WALLET}" || die "Cannot get address from ${WALLET}") -fi - -# Grep Morph block time -SIDECHAIN_PROTO="${SIDECHAIN_PROTO:-services/morph_chain/protocol.privnet.yml}" -BLOCK_DURATION=$(grep SecondsPerBlock < "$SIDECHAIN_PROTO" | awk '{print $2}') \ - || die "Cannot fetch block duration" -NETMAP_ADDR=$(bin/resolve.sh netmap.frostfs) || die "Cannot resolve netmap.frostfs" - -# Fetch current epoch value -EPOCH=$(${NEOGO} contract testinvokefunction \ - -r "http://morph-chain.${LOCAL_DOMAIN}:30333" "${NETMAP_ADDR}" epoch \ - | grep 'value' | awk -F'"' '{ print $4 }') \ - || die "Cannot fetch epoch from netmap contract" - -echo "Updating FrostFS epoch to $((EPOCH+1))" - -# shellcheck disable=SC2086 -${NEOGO} contract invokefunction \ - --wallet-config ${CONFIG_IMG} \ - -a ${ADDR} --force \ - -r http://morph-chain.${LOCAL_DOMAIN}:30333 \ - ${NETMAP_ADDR} \ - newEpoch int:$((EPOCH+1)) -- ${ADDR}:Global \ - || die "Cannot increment an epoch" - -# Wait one Morph block to ensure the transaction broadcasted -# shellcheck disable=SC2086 -sleep $BLOCK_DURATION diff --git a/frostfs_config.mk b/frostfs_config.mk index 8f164d1..71eb8f1 100644 --- a/frostfs_config.mk +++ b/frostfs_config.mk @@ -1,44 +1,40 @@ # Update epoch duration in side chain blocks (make update.epoch_duration val=30) update.epoch_duration: - @./bin/config.sh int EpochDuration $(val) + @./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config EpochDuration=$(val) # Update max object size in bytes (make update.max_object_size val=1000) update.max_object_size: - @./bin/config.sh int MaxObjectSize $(val) + @./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config MaxObjectSize=$(val) # Update audit fee per result in fixed 12 (make update.audit_fee val=100) update.audit_fee: - @./bin/config.sh int AuditFee $(val) + @./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config AuditFee=$(val) # Update container fee per alphabet node in fixed 12 (make update.container_fee val=500) update.container_fee: - @./bin/config.sh int ContainerFee $(val) + @./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config ContainerFee=$(val) # Update container alias fee per alphabet node in fixed 12 (make update.container_alias_fee val=100) update.container_alias_fee: - @./bin/config.sh int ContainerAliasFee $(val) + @./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config ContainerAliasFee=$(val) # Update amount of EigenTrust iterations (make update.eigen_trust_iterations val=2) update.eigen_trust_iterations: - @./bin/config.sh int EigenTrustIterations $(val) - + @./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config EigenTrustIterations=$(val) # Update system dns to resolve container names (make update.system_dns val=container) update.system_dns: - @./bin/config.sh string SystemDNS $(val) + @./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config SystemDNS=$(val) --force # Update alpha parameter of EigenTrust algorithm in 0 <= f <= 1.0 (make update.eigen_trust_alpha val=0.2) update.eigen_trust_alpha: - @./bin/config.sh string EigenTrustAlpha $(val) + @./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config EigenTrustAlpha=$(val) # Update basic income rate in fixed 12 (make update.basic_income_rate val=1000) update.basic_income_rate: - @./bin/config.sh int BasicIncomeRate $(val) + @./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config BasicIncomeRate=$(val) # Update homomorphic hashing disabled flag (make update.homomorphic_hashing_disable val=true) update.homomorphic_hashing_disable: - @./bin/config.sh bool HomomorphicHashingDisabled $(val) + @./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config HomomorphicHashingDisabled=$(val) -# Tick new epoch in side chain -tick.epoch: - @./bin/tick.sh diff --git a/services/ir/prepare.mk b/services/ir/prepare.mk index fd7f253..af13d12 100644 --- a/services/ir/prepare.mk +++ b/services/ir/prepare.mk @@ -1,4 +1,3 @@ # Deposit GAS from default wallet to FrostFS privnet contract prepare.ir: - @./bin/config.sh int ContainerFee 0 - @./bin/config.sh int ContainerAliasFee 0 + @./vendor/frostfs-adm -c ./frostfs-adm.yml ContainerFee=0 ContainerAliasFee=0 From 045b55250ab7193fd992320cc61480681598e1e6 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 28 Apr 2023 12:08:41 +0300 Subject: [PATCH 004/107] [#18] bin: Remove passwd.exp It was here from older times Signed-off-by: Evgenii Stratonikov --- bin/passwd.exp | 23 ----------------------- 1 file changed, 23 deletions(-) delete mode 100755 bin/passwd.exp diff --git a/bin/passwd.exp b/bin/passwd.exp deleted file mode 100755 index 08fd3c7..0000000 --- a/bin/passwd.exp +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/expect - -set passwd [lindex $argv 0] -set args [lrange $argv 1 end] - -spawn -noecho {*}$args -expect -re {^.*assword.*$} - -if { $passwd == "-"} { - send -- "\r" -} else { - send -- "$passwd\r" -} - -expect { - "Relay transaction" { - send "y\r" - exp_continue - } - EOF -} -lassign [wait] pid spawnid os_error_flag value -exit $value From 1b6b2d125993548efdaa17472f755239bbd7ccd3 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Tue, 2 May 2023 18:44:26 +0300 Subject: [PATCH 005/107] [#21] Update FrostFS Core components to the latest master Signed-off-by: Alex Vanin --- .env | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.env b/.env index c33accc..78c1aec 100644 --- a/.env +++ b/.env @@ -12,11 +12,11 @@ NEOGO_VERSION=0.100.1 NEOGO_IMAGE=nspccdev/neo-go # FrostFS InnerRing nodes -IR_VERSION=5ffa8268 +IR_VERSION=eca5c210 IR_IMAGE=truecloudlab/frostfs-ir # FrostFS Storage nodes -NODE_VERSION=5ffa8268 +NODE_VERSION=eca5c210 NODE_IMAGE=truecloudlab/frostfs-storage # NATS Server @@ -40,17 +40,17 @@ LOCODE_DB_URL=https://github.com/nspcc-dev/neofs-locode-db/releases/download/v0. #LOCODE_DB_PATH=/path/to/locode_db # FrostFS CLI binary -FROSTFS_CLI_URL=https://http.t5.fs.neo.org/AQgse8bPCZx4zScMuAKxowJdZPbKHp8NDcp15o6VUNmk/C6BNLpYg5gWLHp3DrXozSxxGLDahBuSBCyJoYSSR1M3Q +FROSTFS_CLI_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/BbngJDdRJEDJTJk7qptq3SxKqrJqtvVYWU6R5AaFGbtG #FROSTFS_CLI_PATH=/path/to/frostfs-cli-binary # FrostFS ADM tool binary -FROSTFS_ADM_VERSION=e3554425 -FROSTFS_ADM_URL=https://http.t5.fs.neo.org/AQgse8bPCZx4zScMuAKxowJdZPbKHp8NDcp15o6VUNmk/sXZxy9vbFyJiLhN9qTSXozXK7SN9H8ZC6dpvAt59Zaj +FROSTFS_ADM_VERSION=eca5c210 +FROSTFS_ADM_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/2GxarAjGUb3RevxvqFGYT3hDQxNNaHzK6aFxhJCAMehq #FROSTFS_ADM_PATH=/path/to/frostfs-adm-binary # Compiled FrostFS Smart Contracts -FROSTFS_CONTRACTS_VERSION=4f3c08f5 -FROSTFS_CONTRACTS_URL=https://http.t5.fs.neo.org/AQgse8bPCZx4zScMuAKxowJdZPbKHp8NDcp15o6VUNmk/c1nGtturFrSeygYP3AyNHDDLNbs7HhJiH2BQkgZxEmZ +FROSTFS_CONTRACTS_VERSION=8537293e +FROSTFS_CONTRACTS_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/6ccZoj4HxoN1G1qvJAX2Qw9p2D6qdyzAjNMaNkEKYQpA #FROSTFS_CONTRACTS_PATH=/path/to/unpacked/frostfs-contracts-dir # Jaeger tracing @@ -59,4 +59,4 @@ JAEGER_IMAGE=jaegertracing/all-in-one # Prometheus monitoring PROMETHEUS_VERSION=v2.43.0 -PROMETHEUS_IMAGE=prom/prometheus \ No newline at end of file +PROMETHEUS_IMAGE=prom/prometheus From f57a739c3f07ff8394d679be6cae213b5a41c271 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 3 May 2023 11:38:04 +0300 Subject: [PATCH 006/107] [#22] .env: Update neo-go to v0.101.1 Signed-off-by: Evgenii Stratonikov --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 78c1aec..be4f6ff 100644 --- a/.env +++ b/.env @@ -8,7 +8,7 @@ BASTION_VERSION=10 BASTION_IMAGE=debian # NeoGo privnet -NEOGO_VERSION=0.100.1 +NEOGO_VERSION=0.101.1 NEOGO_IMAGE=nspccdev/neo-go # FrostFS InnerRing nodes From 53007be0474333df3981a31cfb617d96d0522aa0 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 3 May 2023 11:46:38 +0300 Subject: [PATCH 007/107] [#22] Makefile: Remove unneeded escape Signed-off-by: Evgenii Stratonikov --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0322582..9f791cf 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ HOSTS_LINES = $(shell grep -Rl IPV4_PREFIX ./services/* | grep .hosts) MORPH_CHAIN_PROTOCOL = './services/morph_chain/protocol.privnet.yml' # List of grepped environment variables from *.env -GREP_DOTENV = $(shell find . -name '*.env' -exec grep -rhv -e '^\#' -e '^$$' {} + | sort -u ) +GREP_DOTENV = $(shell find . -name '*.env' -exec grep -rhv -e '^#' -e '^$$' {} + | sort -u ) # Pull all required Docker images .PHONY: pull From 7171e152e0f3a258409a0f22cce352aaddbc0d38 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 3 May 2023 11:56:23 +0300 Subject: [PATCH 008/107] [#22] morph_chain: Do not use deprecated config fields Signed-off-by: Evgenii Stratonikov --- Makefile | 2 +- services/morph_chain/protocol.privnet.yml | 32 +++++++++++++---------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index 9f791cf..4caa292 100644 --- a/Makefile +++ b/Makefile @@ -152,7 +152,7 @@ clean: .PHONY: env env: @$(foreach envvar,$(GREP_DOTENV),echo $(envvar);) - @echo MORPH_BLOCK_TIME=$(shell grep 'SecondsPerBlock' $(MORPH_CHAIN_PROTOCOL) | awk '{print $$2}')s + @echo MORPH_BLOCK_TIME=$(shell grep 'TimePerBlock' $(MORPH_CHAIN_PROTOCOL) | awk '{print $$2}')s @echo MORPH_MAGIC=$(shell grep 'Magic' $(MORPH_CHAIN_PROTOCOL) | awk '{print $$2}') # Restart storage nodes with clean volumes diff --git a/services/morph_chain/protocol.privnet.yml b/services/morph_chain/protocol.privnet.yml index 39155c8..13e03b5 100644 --- a/services/morph_chain/protocol.privnet.yml +++ b/services/morph_chain/protocol.privnet.yml @@ -1,50 +1,54 @@ ProtocolConfiguration: Magic: 15405 MaxTraceableBlocks: 200000 - SecondsPerBlock: 1 + TimePerBlock: 1s MemPoolSize: 50000 StandbyCommittee: - 02b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc2 ValidatorsCount: 1 SeedList: - 172.200.0.1:20333 - VerifyBlocks: true VerifyTransactions: true StateRootInHeader: true P2PSigExtensions: true ApplicationConfiguration: + SkipBlockVerification: false DBConfiguration: Type: "boltdb" BoltDBOptions: FilePath: "./db/morph.bolt" - NodePort: 20333 + P2P: + Addresses: + - ":20333" + DialTimeout: 3s + ProtoTickInterval: 2s + PingInterval: 30s + PingTimeout: 90s + MaxPeers: 10 + AttemptConnPeers: 5 + MinPeers: 0 Relay: true - DialTimeout: 3 - ProtoTickInterval: 2 - PingInterval: 30 - PingTimeout: 90 - MaxPeers: 10 - AttemptConnPeers: 5 - MinPeers: 0 RPC: - Address: 192.168.130.90 + Addresses: + - "192.168.130.90:30333" Enabled: true SessionEnabled: true EnableCORSWorkaround: false MaxGasInvoke: 100 - Port: 30333 P2PNotary: Enabled: true UnlockWallet: Path: "./wallets/node-wallet.json" Password: "one" Prometheus: + Addresses: + - ":20001" Enabled: true - Port: 20001 Pprof: + Addresses: + - ":20011" Enabled: true - Port: 20011 UnlockWallet: Path: "./wallets/node-wallet.json" Password: "one" From 45b98196ccb9fdbb1668c49b58a3fbf591c3815a Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 4 May 2023 17:42:48 +0300 Subject: [PATCH 009/107] [#24] Update frostfsid id Signed-off-by: Anton Nikiforov --- services/ir/.ir.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/ir/.ir.env b/services/ir/.ir.env index 33df607..08d7e17 100644 --- a/services/ir/.ir.env +++ b/services/ir/.ir.env @@ -1,3 +1,3 @@ -FROSTFS_IR_CONTRACTS_FROSTFSID=1943e9bb78a0fe2fe0c95fd2677eec2da6aa4aa5 +FROSTFS_IR_CONTRACTS_FROSTFSID=27407c76feabc407908f3d09a3d845d45e7c981a FROSTFS_IR_CONTROL_GRPC_ENDPOINT=127.0.0.1:16512 From cbed3ce798036a893b641db95e8b9ac21c9794da Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Wed, 10 May 2023 12:31:06 +0300 Subject: [PATCH 010/107] [#27] Fix step `make prepare.ir` Signed-off-by: Anton Nikiforov --- services/ir/prepare.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/ir/prepare.mk b/services/ir/prepare.mk index af13d12..6649502 100644 --- a/services/ir/prepare.mk +++ b/services/ir/prepare.mk @@ -1,3 +1,3 @@ # Deposit GAS from default wallet to FrostFS privnet contract prepare.ir: - @./vendor/frostfs-adm -c ./frostfs-adm.yml ContainerFee=0 ContainerAliasFee=0 + @./vendor/frostfs-adm morph set-config -c ./frostfs-adm.yml ContainerFee=0 ContainerAliasFee=0 From 2452e4469b86f116a1abd9e2b353f45a16457cd9 Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Tue, 16 May 2023 11:58:23 +0300 Subject: [PATCH 011/107] [#29] Remove prepare.ir make target Signed-off-by: Alejandro Lopez --- README.md | 11 ----------- services/ir/prepare.mk | 3 --- 2 files changed, 14 deletions(-) delete mode 100644 services/ir/prepare.mk diff --git a/README.md b/README.md index b9af689..197d680 100644 --- a/README.md +++ b/README.md @@ -57,17 +57,6 @@ Run all services with command: $ make up ``` -When all services are up, you need to make GAS deposit for test wallet to be -able to pay for FrostFS operations. Test wallet is located in -`wallets/wallet.json` with the corresponding key in `wallets/wallet.key`. The -password is empty. - -``` -$ make prepare.ir -password > -fa6ba62bffb04030d303dcc95bda7413e03aa3c7e6ca9c2f999d65db9ec9b82c -``` - Also, you should add self-signed node (`s04.frostfs.devenv`) certificate to trusted store (default location might be changed using `CA_CERTS_TRUSTED_STORE` variable). This step is required for client services (frostfs-http-gw, diff --git a/services/ir/prepare.mk b/services/ir/prepare.mk deleted file mode 100644 index 6649502..0000000 --- a/services/ir/prepare.mk +++ /dev/null @@ -1,3 +0,0 @@ -# Deposit GAS from default wallet to FrostFS privnet contract -prepare.ir: - @./vendor/frostfs-adm morph set-config -c ./frostfs-adm.yml ContainerFee=0 ContainerAliasFee=0 From 26e290efcbea7c842e179333ac6a59e34b23d2c8 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 23 May 2023 13:23:52 +0300 Subject: [PATCH 012/107] [#31] jaeger: Use badger as storage backend Signed-off-by: Dmitrii Stepanov --- services/jaeger/docker-compose.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/services/jaeger/docker-compose.yml b/services/jaeger/docker-compose.yml index be045f8..a460548 100644 --- a/services/jaeger/docker-compose.yml +++ b/services/jaeger/docker-compose.yml @@ -20,7 +20,10 @@ services: env_file: [ ".env", ".jaeger.env", ".int_test.env" ] environment: - COLLECTOR_OTLP_ENABLED=true - - MEMORY_MAX_TRACES=100000 + - SPAN_STORAGE_TYPE=badger + - BADGER_EPHEMERAL=false + - BADGER_DIRECTORY_VALUE=/badger/data + - BADGER_DIRECTORY_KEY=/badger/key networks: jaeger_int: From 5534204706097eacc4ee35eda96cdcf8bfc2e6b4 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 24 May 2023 11:45:25 +0300 Subject: [PATCH 013/107] [#33] s3: Use all storage nodes as tree service Signed-off-by: Dmitrii Stepanov --- services/s3_gate/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/s3_gate/docker-compose.yml b/services/s3_gate/docker-compose.yml index f86caf5..1445480 100644 --- a/services/s3_gate/docker-compose.yml +++ b/services/s3_gate/docker-compose.yml @@ -25,7 +25,7 @@ services: - S3_GW_RPC_ENDPOINT=http://morph-chain.${LOCAL_DOMAIN}:30333 - S3_GW_SERVER_0_ADDRESS=s3.${LOCAL_DOMAIN}:8080 - S3_GW_LISTEN_DOMAINS=s3.${LOCAL_DOMAIN} - - S3_GW_TREE_SERVICE=s01.${LOCAL_DOMAIN}:8080 + - S3_GW_TREE_SERVICE=s01.${LOCAL_DOMAIN}:8080 s02.${LOCAL_DOMAIN}:8080 s03.${LOCAL_DOMAIN}:8080 s04.${LOCAL_DOMAIN}:8080 - S3_GW_PEERS_0_ADDRESS=s01.${LOCAL_DOMAIN}:8080 - S3_GW_PEERS_0_WEIGHT=0.2 - S3_GW_PEERS_1_ADDRESS=s02.${LOCAL_DOMAIN}:8080 From ee4d5ee6a6c91a1a4e1f82679f8746d4db64fd61 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 23 May 2023 16:08:14 +0300 Subject: [PATCH 014/107] [#32] storage: Enable writecache by default Signed-off-by: Dmitrii Stepanov --- services/storage/cfg/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/storage/cfg/config.yml b/services/storage/cfg/config.yml index fa50f4e..0a85096 100644 --- a/services/storage/cfg/config.yml +++ b/services/storage/cfg/config.yml @@ -48,7 +48,7 @@ storage: shard: 0: writecache: - enabled: false + enabled: true path: /storage/wc0 # Write-cache root directory metabase: @@ -68,7 +68,7 @@ storage: 1: writecache: - enabled: false + enabled: true path: /storage/wc1 # Write-cache root directory metabase: From dc2a2862d410eab6aab07fe34fde744e2183d31b Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Tue, 30 May 2023 15:49:43 +0300 Subject: [PATCH 015/107] [#34] Bump FrostFS versions Signed-off-by: Alex Vanin --- .env | 8 ++++---- services/http_gate/docker-compose.yml | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.env b/.env index be4f6ff..8f69cc5 100644 --- a/.env +++ b/.env @@ -12,11 +12,11 @@ NEOGO_VERSION=0.101.1 NEOGO_IMAGE=nspccdev/neo-go # FrostFS InnerRing nodes -IR_VERSION=eca5c210 +IR_VERSION=365a7ca0 IR_IMAGE=truecloudlab/frostfs-ir # FrostFS Storage nodes -NODE_VERSION=eca5c210 +NODE_VERSION=365a7ca0 NODE_IMAGE=truecloudlab/frostfs-storage # NATS Server @@ -24,7 +24,7 @@ NATS_VERSION=2.7.2 NATS_IMAGE=nats # HTTP Gate -HTTP_GW_VERSION=6abd500b +HTTP_GW_VERSION=0.27.0-rc.1-15-g1776db28 HTTP_GW_IMAGE=truecloudlab/frostfs-http-gw # REST Gate @@ -32,7 +32,7 @@ REST_GW_VERSION=c9c85e90 REST_GW_IMAGE=truecloudlab/frostfs-rest-gw # S3 Gate -S3_GW_VERSION=000d9ed4 +S3_GW_VERSION=0.27.0-rc.1-30-gce929468 S3_GW_IMAGE=truecloudlab/frostfs-s3-gw # FrostFS LOCODE database diff --git a/services/http_gate/docker-compose.yml b/services/http_gate/docker-compose.yml index 0dd3ad5..7108ee5 100644 --- a/services/http_gate/docker-compose.yml +++ b/services/http_gate/docker-compose.yml @@ -21,6 +21,7 @@ services: command: [ "frostfs-http-gw", "--config", "/etc/frostfs/http/config.yml" ] environment: - HTTP_GW_RPC_ENDPOINT=http://morph-chain.${LOCAL_DOMAIN}:30333 + - HTTP_GW_TREE_SERVICE=s01.${LOCAL_DOMAIN}:8080 - HTTP_GW_PEERS_0_ADDRESS=s01.${LOCAL_DOMAIN}:8080 - HTTP_GW_PEERS_0_WEIGHT=0.2 - HTTP_GW_PEERS_1_ADDRESS=s02.${LOCAL_DOMAIN}:8080 From c50ea0ea5c7dc47f3653f0db24b029fcadb8f19a Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Wed, 7 Jun 2023 14:12:54 +0300 Subject: [PATCH 016/107] [#1] Add DCO check for forgejo runner Signed-off-by: Alex Vanin --- .dockerignore | 1 + .forgejo/workflows/dco.yml | 20 ++++++++++++++++++++ .github/workflows/dco.yml | 21 --------------------- 3 files changed, 21 insertions(+), 21 deletions(-) create mode 100644 .forgejo/workflows/dco.yml delete mode 100644 .github/workflows/dco.yml diff --git a/.dockerignore b/.dockerignore index 73b336a..2f9d4b4 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,6 @@ .docker .github +.forgejo vendor tmp .secrets diff --git a/.forgejo/workflows/dco.yml b/.forgejo/workflows/dco.yml new file mode 100644 index 0000000..880addd --- /dev/null +++ b/.forgejo/workflows/dco.yml @@ -0,0 +1,20 @@ +on: [pull_request] + +jobs: + dco: + name: DCO + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Setup Go + uses: actions/setup-go@v3 + with: + go-version: '1.20' + + - name: Run commit format checker + uses: https://git.alexvan.in/alexvanin/dco-go@v1 + with: + from: dca6ff62 diff --git a/.github/workflows/dco.yml b/.github/workflows/dco.yml deleted file mode 100644 index 40ed8fc..0000000 --- a/.github/workflows/dco.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: DCO check - -on: - pull_request: - branches: - - master - -jobs: - commits_check_job: - runs-on: ubuntu-latest - name: Commits Check - steps: - - name: Get PR Commits - id: 'get-pr-commits' - uses: tim-actions/get-pr-commits@master - with: - token: ${{ secrets.GITHUB_TOKEN }} - - name: DCO Check - uses: tim-actions/dco@master - with: - commits: ${{ steps.get-pr-commits.outputs.commits }} From 80dc0faae9a3cf8819cd21f305524b0a7ab0aa4c Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Wed, 7 Jun 2023 12:29:52 +0300 Subject: [PATCH 017/107] [#1] Use frostfs.info hosted locode URL Signed-off-by: Alex Vanin --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 8f69cc5..f921d46 100644 --- a/.env +++ b/.env @@ -36,7 +36,7 @@ S3_GW_VERSION=0.27.0-rc.1-30-gce929468 S3_GW_IMAGE=truecloudlab/frostfs-s3-gw # FrostFS LOCODE database -LOCODE_DB_URL=https://github.com/nspcc-dev/neofs-locode-db/releases/download/v0.3.0/locode_db.gz +LOCODE_DB_URL=https://git.frostfs.info/attachments/a2e8def7-52b6-49f1-89cd-a056712e8e54 #LOCODE_DB_PATH=/path/to/locode_db # FrostFS CLI binary From 6b0ce07808659bd67856f584c9d4b390462afc14 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Wed, 7 Jun 2023 12:30:48 +0300 Subject: [PATCH 018/107] [#1] Update cert generation scripts Signed-off-by: Alex Vanin --- services/nats/generate_cert.sh | 2 +- services/storage/generate_cert.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/services/nats/generate_cert.sh b/services/nats/generate_cert.sh index 071f9df..4c4dd0b 100755 --- a/services/nats/generate_cert.sh +++ b/services/nats/generate_cert.sh @@ -16,7 +16,7 @@ CLI_KEY=$WORKDIR/client-key.pem CLI_REQ=$WORKDIR/client-req.csr CLI_CRT=$WORKDIR/client-cert.pem -SUBJ="/O=NSPCC" +SUBJ="/O=TrueCloudLab" if [[ ! -f $CA_KEY || ! -f $CA_CRT ]]; then openssl req -newkey rsa:4096 -x509 -days 365 -nodes -keyout $CA_KEY -out $CA_CRT -subj $SUBJ 2>&1 || diff --git a/services/storage/generate_cert.sh b/services/storage/generate_cert.sh index ddc7aa1..87672d2 100755 --- a/services/storage/generate_cert.sh +++ b/services/storage/generate_cert.sh @@ -19,7 +19,7 @@ if [[ ! -f ${CERT} ]]; then ) > ${SSL_CONFIG} openssl req -new -newkey rsa:4096 -x509 -sha256 -days 365 -nodes \ - -subj "/C=RU/ST=SPB/L=St.Petersburg/O=NSPCC/OU=NSPCC/CN=s04.${LOCAL_DOMAIN}" \ + -subj "/C=RU/ST=SPB/L=St.Petersburg/O=TrueCloudLab/OU=TrueCloudLab/CN=s04.${LOCAL_DOMAIN}" \ -keyout "${KEY}" -out "${CERT}" -extensions san -config "${SSL_CONFIG}" &> /dev/null || { die "Failed to generate SSL certificate for s04" } From e551fac84d94cda36a0673fbb0815c168050456a Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Wed, 7 Jun 2023 12:27:13 +0300 Subject: [PATCH 019/107] [#1] Update notary service description Signed-off-by: Alex Vanin --- docs/notary.md | 66 +++----------------------------------------------- 1 file changed, 4 insertions(+), 62 deletions(-) diff --git a/docs/notary.md b/docs/notary.md index 66cd713..2b16689 100644 --- a/docs/notary.md +++ b/docs/notary.md @@ -7,66 +7,12 @@ create containers, approve balance changes, update network map, tick epochs, etc. With notary service, it takes up to seven times fewer transactions to do these operations. Notary service calculates the exact amount of GAS to execute transaction, therefore operations are cheaper (withdraw fee **with** -notary is less than 0.5 GAS; withdraw fee **without** notary is up to 7.0 GAS). +notary is less than 0.5 GAS; withdraw fee **without** notary is up to 7.0 GAS). -By default, main chain service is running without notary service, and side chain -running with notary service. However, you can change that in configuration. +Currently, frostfs-dev-env contains single chain (see morph service) and it +enables notary service from the genesis block. -# Disable notary service in side chain - -To disable notary service in side chain do these steps. - -1. Update `.env` and choose notary disabled chain dump for side chain. - -``` -MORPH_CHAIN_URL="https://github.com/nspcc-dev/neofs-contract/releases/download/v0.9.0/devenv_sidechain_notary_disabled.gz" -``` - -Make sure to update chain dump files with `make get` target. - -2. Update `service/morph_chain/protocol.privnet.yml` and disable notary settings -and state root in header. - -```yaml -ProtocolConfiguration: - StateRootInHeader: false - P2PSigExtensions: false -ApplicationConfiguration: - P2PNotary: - Enabled: false -``` - -Chain dump without notary service does not have predefined network map. -Therefore, you need to wait about 5 minutes until new epoch tick with updated -network map. - - -3. Enable helper commands - -To enable helper commands such as `make tick.epoch` or `make update.epoch_duration` -make sure to export non-empty `FROSTFS_NOTARY_DISABLED` environment variable. -``` -$ export FROSTFS_NOTARY_DISABLED=1 -``` - -Use `unset` command to return it back. -``` -$ unset FROSTFS_NOTARY_DISABLED -``` - -# Enable notary service in main chain - -To enable notary service in main chain do these steps. - -1. Update `.env` and choose notary enabled chain dump for main chain. - -``` -CHAIN_URL="https://github.com/nspcc-dev/neofs-contract/releases/download/v0.9.0/devenv_mainchain.gz" -``` - -Make sure to update chain dump files with `make get` target. - -2. Update `service/chain/protocol.privnet.yml` and enable notary settings. +To enable notary service, use neo-go configuration below. ```yaml ProtocolConfiguration: @@ -75,7 +21,3 @@ ApplicationConfiguration: P2PNotary: Enabled: true ``` - -Main chain generates a block once per 15 seconds, so Inner Ring takes about -15-30 seconds to make a notary deposit in main chain after startup. Then -frostfs-dev-env is ready to work. From 5050dff55aa556584cdfdbfddd81235fb4f2d34a Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Wed, 7 Jun 2023 12:37:51 +0300 Subject: [PATCH 020/107] [#1] Update morph service description Signed-off-by: Alex Vanin --- docs/morph.md | 69 ++++++++++++++------------------------------------- 1 file changed, 19 insertions(+), 50 deletions(-) diff --git a/docs/morph.md b/docs/morph.md index 4429298..56edfbc 100644 --- a/docs/morph.md +++ b/docs/morph.md @@ -3,52 +3,23 @@ A single-node N3 privnet deployment, running on [neo-go](https://github.com/nspcc-dev/neo-go). Represents N3 FrostFS SideChain. Contracts deployed: -- Alphabet (AZ) [contract](https://github.com/TrueCloudLab/frostfs-contract/tree/master/alphabet) -- Audit [contract](https://github.com/TrueCloudLab/frostfs-contract/tree/master/audit) -- Balance [contract](https://github.com/TrueCloudLab/frostfs-contract/tree/master/balance) -- Container [contract](https://github.com/TrueCloudLab/frostfs-contract/tree/master/container) -- Netmap [contract](https://github.com/TrueCloudLab/frostfs-contract/tree/master/netmap) -- NeoFSID [contract](https://github.com/TrueCloudLab/frostfs-contract/tree/master/neofsid) -- Proxy [contract](https://github.com/TrueCloudLab/frostfs-contract/tree/master/proxy) -- Reputation [contract](https://github.com/TrueCloudLab/frostfs-contract/tree/master/reputation) +- Alphabet (AZ) [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/alphabet) +- Audit [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/audit) +- Balance [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/balance) +- Container [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/container) +- Netmap [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/netmap) +- NeoFSID [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/neofsid) +- Proxy [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/proxy) +- Reputation [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/reputation) RPC available at `http://morph-chain.frostfs.devenv:30333`. ## .env settings -### MORPH_CHAIN_URL - -URL to get side chain dump. Used on artifact get stage. - -### MORPH_CHAIN_PATH - -Path to get side chain dump. If set, overrides `CHAIN_URL`. - ### NEOGO_VERSION Version of neo-go docker container for side chain deployment. -## Side chain wallets - -There is a wallet with GAS that used for contract deployment: -`wallets/wallet.json`. This wallet has one account with **empty password**. - -``` -$ neo-go wallet nep17 balance \ - -w wallets/wallet.json \ - -r http://morph-chain.frostfs.devenv:30333 - -Account NbUgTSFvPmsRxmGeWpuuGeJUoRoi6PErcM -GAS: GasToken (d2a4cff31913016155e38e474a2c06d08be276cf) - Amount : 189826.0515316 - Updated: 3909 -FROSTFS: FrostFS Balance (69550190e740b93f92dbd5dea52246f550391057) - Amount : 50 - Updated: 3909 -``` - -This way you can also monitor FrostFS internal balance of your account. - ## FrostFS global config FrostFS uses global configuration to store epoch duration, maximum object size, @@ -57,16 +28,10 @@ netmap contract and managed by Inner Ring (Alphabet) nodes. To change these parameters use `make update.*` commands. Command down below changes epoch duration from 300 blocks (about 300 seconds with 1bps) to 30. -Script enters passwords automatically with `expect` utility. ``` $ make update.epoch_duration val=30 -Changing EpochDuration configration value to 30 -Enter account NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP password > -Sent invocation transaction bdc0fa88cd6719ef6df2b9c82de423ddec6141ca24255c2d0072688083b1de9d -Updating FrostFS epoch to 20 -Enter account NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP password > -Sent invocation transaction 12296e1ce24dd6c04edb9c56d0a1d0e26d3226adefb0333c74a28788f44a8d0f +Waiting for transactions to persist... ``` Read more about available configuration in Makefile help. @@ -76,10 +41,14 @@ $ make help ... Targets: ... - update.audit_fee Update audit fee per result in fixed 12 (make update.audit_fee val=100) - update.basic_income_rate Update basic income rate in fixed 12 (make update.basic_income_rate val=1000) - update.container_fee Update container fee per alphabet node in fixed 12 (make update.container_fee val=500) - update.eigen_trust_iterations Update amount of EigenTrust iterations (make update.eigen_trust_iterations val=2) - update.epoch_duration Update epoch duration in side chain blocks (make update.epoch_duration val=30) - update.max_object_size Update max object size in bytes (make update.max_object_size val=1000) + update.audit_fee Update audit fee per result in fixed 12 (make update.audit_fee val=100) + update.basic_income_rate Update basic income rate in fixed 12 (make update.basic_income_rate val=1000) + update.container_alias_fee Update container alias fee per alphabet node in fixed 12 (make update.container_alias_fee val=100) + update.container_fee Update container fee per alphabet node in fixed 12 (make update.container_fee val=500) + update.eigen_trust_alpha Update alpha parameter of EigenTrust algorithm in 0 <= f <= 1.0 (make update.eigen_trust_alpha val=0.2) + update.eigen_trust_iterations Update amount of EigenTrust iterations (make update.eigen_trust_iterations val=2) + update.epoch_duration Update epoch duration in side chain blocks (make update.epoch_duration val=30) + update.homomorphic_hashing_disable Update homomorphic hashing disabled flag (make update.homomorphic_hashing_disable val=true) + update.max_object_size Update max object size in bytes (make update.max_object_size val=1000) + update.system_dns Update system dns to resolve container names (make update.system_dns val=container) ``` From aded88f09a0a2da3cd690b3a0d202219839d7b7c Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Wed, 7 Jun 2023 12:48:03 +0300 Subject: [PATCH 021/107] [#1] Update HTTP gateway description Signed-off-by: Alex Vanin --- docs/http_gate.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/http_gate.md b/docs/http_gate.md index 1aadd27..e2159ad 100644 --- a/docs/http_gate.md +++ b/docs/http_gate.md @@ -2,7 +2,7 @@ Protocol Gateway to access data in FrostFS using HTTP protocol. -Source code and more information can be found in [project's GitHub repository](https://github.com/TrueCloudLab/frostfs-http-gate) +Source code and more information can be found in [project's repository](https://git.frostfs.info/TrueCloudLab/frostfs-http-gw) ## .env settings From 656833d37e9780e9729d1e155756f88fb188884f Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Wed, 7 Jun 2023 12:49:23 +0300 Subject: [PATCH 022/107] [#1] Update REST gateway description Signed-off-by: Alex Vanin --- docs/rest_gate.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/rest_gate.md b/docs/rest_gate.md index e511280..ea2e95b 100644 --- a/docs/rest_gate.md +++ b/docs/rest_gate.md @@ -2,7 +2,7 @@ REST Gateway to access data in FrostFS using REST. -Source code and more information can be found in [project's GitHub repository](https://github.com/TrueCloudLab/frostfs-rest-gw) +Source code and more information can be found in [project's repository](https://git.frostfs.info/TrueCloudLab/frostfs-rest-gw) ## .env settings From 463521d511e04b195154985557f0bae153f62153 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Wed, 7 Jun 2023 12:52:57 +0300 Subject: [PATCH 023/107] [#1] Update S3 gateway description Signed-off-by: Alex Vanin --- docs/s3_gate.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/s3_gate.md b/docs/s3_gate.md index 0a110d6..717fded 100644 --- a/docs/s3_gate.md +++ b/docs/s3_gate.md @@ -2,7 +2,7 @@ Protocol Gateway to access data in FrostFS using AWS S3 protocol -Source code and more information can be found in [project's GitHub repository](https://github.com/TrueCloudLab/frostfs-s3-gw) +Source code and more information can be found in [project's repository](https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw) ## .env settings From 8eca14f331cf22e1f1b28e2542a0966f480845f9 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Wed, 7 Jun 2023 13:01:44 +0300 Subject: [PATCH 024/107] [#1] Update CONTRIBUTING Signed-off-by: Alex Vanin --- CONTRIBUTING.md | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f3ae1f1..fb57514 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3,8 +3,8 @@ First, thank you for contributing! We love and encourage pull requests from everyone. Please follow the guidelines: -- Check the open [issues](https://github.com/TrueCloudLab/frostfs-dev-env/issues) and - [pull requests](https://github.com/TrueCloudLab/frostfs-dev-env/pulls) for existing +- Check the open [issues](https://git.frostfs.info/TrueCloudLab/frostfs-dev-env/issues) and + [pull requests](https://git.frostfs.info/TrueCloudLab/frostfs-dev-env/pulls) for existing discussions. - Open an issue first, to discuss a new feature or enhancement. @@ -25,19 +25,19 @@ Start by forking the `frostfs-dev-env` repository, make changes in a branch and send a pull request. We encourage pull requests to discuss code changes. Here are the steps in details: -### Set up your GitHub Repository -Fork [FrostFS node upstream](https://github.com/TrueCloudLab/frostfs-dev-env/fork) source +### Set up your git repository +Fork [FrostFS node upstream](https://git.frostfs.info/repo/fork/24) source repository to your own personal repository. Copy the URL of your fork (you will need it for the `git clone` command below). ```sh -$ git clone https://github.com/TrueCloudLab/frostfs-dev-env +$ git clone https://git.frostfs.info//frostfs-dev-env.git ``` ### Set up git remote as ``upstream`` ```sh $ cd frostfs-dev-env -$ git remote add upstream https://github.com/TrueCloudLab/frostfs-dev-env +$ git remote add upstream https://git.frostfs.info/TrueCloudLab/frostfs-dev-env.git $ git fetch upstream $ git merge upstream/master ... @@ -55,8 +55,7 @@ $ git checkout -b feature/123-something_awesome ### Test your changes After your code changes, make sure -- To add test cases for the new code. -- To run `make lint` +- To run `make up` to check dev-env is not broken. - To squash your commits into a single commit or a series of logically separated commits run `git rebase -i`. It's okay to force update your pull request. @@ -86,8 +85,8 @@ $ git push origin feature/123-something_awesome ``` ### Create a Pull Request -Pull requests can be created via GitHub. Refer to [this -document](https://help.github.com/articles/creating-a-pull-request/) for +Pull requests can be created via Forgejo. Refer to [this +document](https://docs.codeberg.org/collaborating/pull-requests-and-git-flow/) for detailed steps on how to create a pull request. After a Pull Request gets peer reviewed and approved, it will be merged. From c31b7d9c84bc1f38f51252ee8eb7150d61ab845c Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Wed, 7 Jun 2023 12:44:09 +0300 Subject: [PATCH 025/107] [#1] Update README Signed-off-by: Alex Vanin --- README.md | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 197d680..aa07a08 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ Make sure you have installed all of the following prerequisites on your machine: Clone repo: ``` -$ git clone https://github.com/TrueCloudLab/frostfs-dev-env.git +$ git clone https://git.frostfs.info/TrueCloudLab/frostfs-dev-env.git ``` Run next commands from project's root: @@ -71,12 +71,7 @@ password of inner ring wallet is `one`. See examples in `make help`. ``` $ make update.epoch_duration val=30 -Changing EpochDuration configration value to 30 -Enter account NNudMSGzEoktFzdYGYoNb3bzHzbmM1genF password > -Sent invocation transaction dbb8c1145b6d10f150135630e13bb0dc282023163f5956c6945a60db0cb45cb0 -Updating FrostFS epoch to 2 -Enter account NNudMSGzEoktFzdYGYoNb3bzHzbmM1genF password > -Sent invocation transaction 0e6eb5e190f36332e5e5f4e866c7e100826e285fd949e11c085e15224f343ba6 +Waiting for transactions to persist... ``` For instructions on how to set up DevEnv on macOS, please refer [the @@ -112,7 +107,7 @@ Maybe you will find the answer for your question in [F.A.Q.](docs/faq.md) ## Using FrostFS Admin Tool in `dev-env` -Devenv supports FrostFS network management via [frostfs-adm](https://github.com/TrueCloudLab/frostfs-node/tree/master/cmd/frostfs-adm). +Devenv supports FrostFS network management via [frostfs-adm](https://git.frostfs.info/TrueCloudLab/frostfs-node/src/branch/master/cmd/frostfs-adm). `services/ir` contains the Alphabet wallet in a proper format, specify it with `--alphabet-wallets` flag. From d08b338e06c43f0889c26940a086488b01f673ee Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 14 Jun 2023 12:38:24 +0300 Subject: [PATCH 026/107] [#36] prometheus: Add IR metrics Signed-off-by: Dmitrii Stepanov --- services/prometheus/prometheus.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/services/prometheus/prometheus.yml b/services/prometheus/prometheus.yml index 1ea7300..9ea7aed 100644 --- a/services/prometheus/prometheus.yml +++ b/services/prometheus/prometheus.yml @@ -16,4 +16,7 @@ scrape_configs: - targets: ['s3.frostfs.devenv:9090'] - job_name: 'neo-go' static_configs: - - targets: ['morph-chain.frostfs.devenv:20001'] \ No newline at end of file + - targets: ['morph-chain.frostfs.devenv:20001'] + - job_name: 'inner-ring' + static_configs: + - targets: ['ir01.frostfs.devenv:9090'] \ No newline at end of file From 3400bb5736f769f5cf9720a3dbfff99809337c7a Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Thu, 13 Jul 2023 12:13:27 +0300 Subject: [PATCH 027/107] [#37] Add grafana service to visualize prometheus metrics Signed-off-by: Alex Vanin --- .env | 4 + .services | 1 + services/grafana/.env | 1 + services/grafana/.hosts | 1 + services/grafana/.int_test.env | 1 + services/grafana/docker-compose.yml | 26 + services/grafana/grafana.ini | 3 + .../provisioning/dashboards/14061_rev1.json | 1162 +++++++++++++++++ .../provisioning/dashboards/dashboard.yml | 10 + .../provisioning/datasources/datasource.yml | 8 + 10 files changed, 1217 insertions(+) create mode 120000 services/grafana/.env create mode 100644 services/grafana/.hosts create mode 120000 services/grafana/.int_test.env create mode 100644 services/grafana/docker-compose.yml create mode 100644 services/grafana/grafana.ini create mode 100644 services/grafana/provisioning/dashboards/14061_rev1.json create mode 100644 services/grafana/provisioning/dashboards/dashboard.yml create mode 100644 services/grafana/provisioning/datasources/datasource.yml diff --git a/.env b/.env index f921d46..e04d5ca 100644 --- a/.env +++ b/.env @@ -60,3 +60,7 @@ JAEGER_IMAGE=jaegertracing/all-in-one # Prometheus monitoring PROMETHEUS_VERSION=v2.43.0 PROMETHEUS_IMAGE=prom/prometheus + +# Grafana versions +GRAFANA_VERSION=9.5.6 +GRAFANA_IMAGE=grafana/grafana diff --git a/.services b/.services index d9f34c7..768e4e6 100644 --- a/.services +++ b/.services @@ -3,3 +3,4 @@ http_gate s3_gate rest_gate +grafana diff --git a/services/grafana/.env b/services/grafana/.env new file mode 120000 index 0000000..c7360fb --- /dev/null +++ b/services/grafana/.env @@ -0,0 +1 @@ +../../.env \ No newline at end of file diff --git a/services/grafana/.hosts b/services/grafana/.hosts new file mode 100644 index 0000000..8f36ea6 --- /dev/null +++ b/services/grafana/.hosts @@ -0,0 +1 @@ +IPV4_PREFIX.122 grafana.LOCAL_DOMAIN diff --git a/services/grafana/.int_test.env b/services/grafana/.int_test.env new file mode 120000 index 0000000..582b6a2 --- /dev/null +++ b/services/grafana/.int_test.env @@ -0,0 +1 @@ +../../.int_test.env \ No newline at end of file diff --git a/services/grafana/docker-compose.yml b/services/grafana/docker-compose.yml new file mode 100644 index 0000000..795b87c --- /dev/null +++ b/services/grafana/docker-compose.yml @@ -0,0 +1,26 @@ +version: '2.4' +services: + grafana: + image: ${GRAFANA_IMAGE}:${GRAFANA_VERSION} + domainname: ${LOCAL_DOMAIN} + hostname: grafana + container_name: grafana + restart: on-failure + networks: + grafana_int: + internet: + ipv4_address: ${IPV4_PREFIX}.122 + volumes: + - ./../../vendor/hosts:/etc/hosts + - ./grafana.ini:/etc/grafana/grafana.ini + - ./provisioning:/etc/grafana/provisioning + ports: + - '3000:3000' + stop_signal: SIGKILL + env_file: [ ".env", ".int_test.env" ] + +networks: + grafana_int: + internet: + external: true + name: basenet_internet diff --git a/services/grafana/grafana.ini b/services/grafana/grafana.ini new file mode 100644 index 0000000..256888f --- /dev/null +++ b/services/grafana/grafana.ini @@ -0,0 +1,3 @@ +[auth.anonymous] +# enable anonymous access +enabled = true diff --git a/services/grafana/provisioning/dashboards/14061_rev1.json b/services/grafana/provisioning/dashboards/14061_rev1.json new file mode 100644 index 0000000..deec62b --- /dev/null +++ b/services/grafana/provisioning/dashboards/14061_rev1.json @@ -0,0 +1,1162 @@ +{ + "__inputs": [], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "7.2.0" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "A quickstart to setup the Prometheus Go runtime exporter with preconfigured dashboards, alerting rules, and recording rules.", + "editable": true, + "graphTooltip": 0, + "id": null, + "iteration": 1602794777869, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Average total bytes of memory reserved across all process instances of a job.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg by(job)(go_memstats_sys_bytes{job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{job}} (avg)", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total Reserved Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Average stack memory usage across all instances of a job.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg by (job) (go_memstats_stack_sys_bytes{job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{job}}: stack inuse (avg)", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Stack Memory Use", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Average memory reservations by the runtime, not for stack or heap, across all instances of a job.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 26, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg by (job)(go_memstats_mspan_sys_bytes{job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{instance}}: mspan (avg)", + "refId": "B" + }, + { + "expr": "avg by (job)(go_memstats_mcache_sys_bytes{job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{instance}}: mcache (avg)", + "refId": "D" + }, + { + "expr": "avg by (job)(go_memstats_buck_hash_sys_bytes{job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{instance}}: buck hash (avg)", + "refId": "E" + }, + { + "expr": "avg by (job)(go_memstats_gc_sys_bytes{job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{job}}: gc (avg)", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Other Memory Reservations", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Average memory reserved, and actually in use, by the heap, across all instances of a job.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg by (job)(go_memstats_heap_sys_bytes{job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{job}}: heap reserved (avg)", + "refId": "B" + }, + { + "expr": "avg by (job)(go_memstats_heap_inuse_bytes{job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{job}}: heap in use (avg)", + "refId": "A" + }, + { + "expr": "avg by (job)(go_memstats_heap_alloc_bytes{job=~\"tns_app\",instance=~\".*\"})", + "interval": "", + "legendFormat": "{{job}}: heap alloc (avg)", + "refId": "C" + }, + { + "expr": "avg by (job)(go_memstats_heap_idle_bytes{job=~\"tns_app\",instance=~\".*\"})", + "interval": "", + "legendFormat": "{{job}}: heap idle (avg)", + "refId": "D" + }, + { + "expr": "avg by (job)(go_memstats_heap_released_bytes{job=~\"tns_app\",instance=~\".*\"})", + "interval": "", + "legendFormat": "{{job}}: heap released (avg)", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Heap Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Average allocation rate in bytes per second, across all instances of a job.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 1, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg by (job)(rate(go_memstats_alloc_bytes_total{job=\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "{{job}}: bytes malloced/s (avg)", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Allocation Rate, Bytes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Average rate of heap object allocation, across all instances of a job.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(go_memstats_mallocs_total{job=\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "interval": "", + "legendFormat": "{{job}}: obj mallocs/s (avg)", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Heap Object Allocation Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Average number of live memory objects across all instances of a job.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg by(job)(go_memstats_mallocs_total{job=\"$job\", instance=~\"$instance\"} - go_memstats_frees_total{job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{job}}: object count (avg)", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of Live Objects", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Average number of goroutines across instances of a job.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg by (job)(go_goroutines{job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{job}}: goroutine count (avg)", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg by (job)(go_gc_duration_seconds{quantile=\"0\", job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{job}}: min gc time (avg)", + "refId": "A" + }, + { + "expr": "avg by (job)(go_gc_duration_seconds{quantile=\"1\", job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{job}}: max gc time (avg)", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "GC min & max duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "The number used bytes at which the runtime plans to perform the next GC, averaged across all instances of a job.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "hiddenSeries": false, + "id": 27, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg by (job)(go_memstats_next_gc_bytes{job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{job}} next gc bytes (avg)", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Next GC, Bytes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "5s", + "schemaVersion": 25, + "style": "dark", + "tags": [ + "go", + "golang" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": null, + "current": {}, + "datasource": "$datasource", + "definition": "label_values(go_info, job)", + "hide": 0, + "includeAll": false, + "label": "job", + "multi": false, + "name": "job", + "options": [], + "query": "label_values(go_info, job)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": "", + "current": {}, + "datasource": "$datasource", + "definition": "label_values(go_info{job=\"$job\"}, instance)", + "hide": 0, + "includeAll": true, + "label": "instance", + "multi": true, + "name": "instance", + "options": [], + "query": "label_values(go_info{job=\"$job\"}, instance)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Go Runtime Exporter Quickstart and Dashboardby Grafana Labs", + "uid": "CgCw8jKZz3", + "version": 3, + "gnetId": 14061 +} \ No newline at end of file diff --git a/services/grafana/provisioning/dashboards/dashboard.yml b/services/grafana/provisioning/dashboards/dashboard.yml new file mode 100644 index 0000000..bc9f26e --- /dev/null +++ b/services/grafana/provisioning/dashboards/dashboard.yml @@ -0,0 +1,10 @@ +apiVersion: 1 + +providers: + - name: Prometheus + orgId: 1 + type: file + disableDeletion: false + editable: true + options: + path: /etc/grafana/provisioning/dashboards diff --git a/services/grafana/provisioning/datasources/datasource.yml b/services/grafana/provisioning/datasources/datasource.yml new file mode 100644 index 0000000..81b6a7b --- /dev/null +++ b/services/grafana/provisioning/datasources/datasource.yml @@ -0,0 +1,8 @@ +apiVersion: 1 + +datasources: +- name: Prometheus + type: prometheus + access: proxy + orgId: 1 + url: http://prometheus:9090 From 2744f675aa6efb52b046e9ee9a57d6fd9d9f0a1a Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 28 Jul 2023 18:40:52 +0300 Subject: [PATCH 028/107] [#38] grafana: Add storage dashboard Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 489 ++++++++++++++++++ 1 file changed, 489 insertions(+) create mode 100644 services/grafana/provisioning/dashboards/storage-node.json diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json new file mode 100644 index 0000000..8f38c0d --- /dev/null +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -0,0 +1,489 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 2, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [], + "title": "Server", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by gRPC method", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, grpc_service, grpc_method) (\n rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed requests", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(grpc_server_handled_total{instance=\"$instance\",grpc_code!=\"OK\"}[$__rate_interval])) by (grpc_service, grpc_method)\n / \nsum(rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])) by (grpc_service, grpc_method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "95%-tile latency of the requests.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile(0.95, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency, 0.95q", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "99%-tile latency of the requests.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile(0.99, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency, 0.99q", + "transformations": [], + "type": "timeseries" + } + ], + "refresh": "1m", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "s01.frostfs.devenv:9090", + "value": "s01.frostfs.devenv:9090" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_state_health,instance)", + "hide": 0, + "includeAll": false, + "label": "", + "multi": false, + "name": "instance", + "options": [], + "query": { + "query": "label_values(frostfs_node_state_health,instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Storage node", + "uid": "aa6318b6-4780-4a9a-9fe7-a4a81917265f", + "version": 1, + "weekStart": "" +} \ No newline at end of file From 584fa43ca7c0617d97eb874b4e38a806ad8a9e37 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 31 Jul 2023 12:05:40 +0300 Subject: [PATCH 029/107] [#38] grafana: Add overview board Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/overview.json | 327 ++++++++++++++++++ 1 file changed, 327 insertions(+) create mode 100644 services/grafana/provisioning/dashboards/overview.json diff --git a/services/grafana/provisioning/dashboards/overview.json b/services/grafana/provisioning/dashboards/overview.json new file mode 100644 index 0000000..46a5987 --- /dev/null +++ b/services/grafana/provisioning/dashboards/overview.json @@ -0,0 +1,327 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Overview panel for all services", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 3, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by service instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance) (\n rate(grpc_server_started_total[$__rate_interval])\n)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Error rate by service instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(rate(grpc_server_handled_total{grpc_code!=\"OK\"}[$__rate_interval])) by (instance)\n / \nsum(rate(grpc_server_started_total{}[$__rate_interval])) by (instance)\n * 100.0", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Resident memory size by service instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance) (\n process_resident_memory_bytes\n)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Memory", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Overview", + "uid": "da5f53f7-c069-44b5-8372-462333c9fe4a", + "version": 1, + "weekStart": "" + } \ No newline at end of file From 3abb217d3028937a4f50c506b7a16137dec9d4c6 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 31 Jul 2023 12:06:15 +0300 Subject: [PATCH 030/107] [#38] grafana: Change start order Start grafana right after prometheus. Signed-off-by: Dmitrii Stepanov --- .bootstrap_services | 1 + .services | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.bootstrap_services b/.bootstrap_services index 02237d1..a56729d 100644 --- a/.bootstrap_services +++ b/.bootstrap_services @@ -4,3 +4,4 @@ basenet morph_chain jaeger prometheus +grafana diff --git a/.services b/.services index 768e4e6..d9f34c7 100644 --- a/.services +++ b/.services @@ -3,4 +3,3 @@ http_gate s3_gate rest_gate -grafana From 7de23fe789d7817d553287d10ba312e2f867c2df Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 31 Jul 2023 12:06:51 +0300 Subject: [PATCH 031/107] [#38] grafana: Fix config Allow to acces grafana anonymous. Change home page. Signed-off-by: Dmitrii Stepanov --- services/grafana/grafana.ini | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/services/grafana/grafana.ini b/services/grafana/grafana.ini index 256888f..3c14906 100644 --- a/services/grafana/grafana.ini +++ b/services/grafana/grafana.ini @@ -1,3 +1,7 @@ [auth.anonymous] -# enable anonymous access enabled = true +org_name = Main Org. +org_role = Editor + +[dashboards] +default_home_dashboard_path= /etc/grafana/provisioning/dashboards/overview.json \ No newline at end of file From 2efc0442f18120cc05ea8fc287a70f284b4a6a37 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 1 Aug 2023 16:14:18 +0300 Subject: [PATCH 032/107] [#38] grafana: Add instance to panels Signed-off-by: Dmitrii Stepanov --- services/grafana/provisioning/dashboards/storage-node.json | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index 8f38c0d..62de402 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -19,7 +19,6 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 2, "links": [], "liveNow": false, "panels": [ @@ -327,7 +326,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "histogram_quantile(0.95, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", + "expr": "histogram_quantile(0.95, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", "format": "time_series", "instant": false, "interval": "", @@ -427,7 +426,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "histogram_quantile(0.99, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", + "expr": "histogram_quantile(0.99, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", "format": "time_series", "instant": false, "interval": "", From f749581c4ebd4e43932965fdf93f16b070dc2380 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 1 Aug 2023 17:18:54 +0300 Subject: [PATCH 033/107] [#38] grafana: Add tree service row Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 666 +++++++++++++++--- 1 file changed, 575 insertions(+), 91 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index 62de402..df05dc0 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -244,7 +244,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "95%-tile latency of the requests.", + "description": "Latency of the requests for selected quantile", "fieldConfig": { "defaults": { "color": { @@ -326,7 +326,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "histogram_quantile(0.95, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", + "expr": "histogram_quantile($quantile, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", "format": "time_series", "instant": false, "interval": "", @@ -335,109 +335,560 @@ "refId": "A" } ], - "title": "Latency, 0.95q", + "title": "Latency", "transformations": [], "type": "timeseries" }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "99%-tile latency of the requests.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, + "collapsed": true, "gridPos": { - "h": 10, + "h": 1, "w": 24, "x": 0, "y": 31 }, - "id": 5, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ + "id": 6, + "panels": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile(0.99, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{grpc_service}}.{{grpc_method}}", - "range": true, - "refId": "A" + "description": "Requests per second by Pilorama method for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id, method) (\n rate(frostfs_node_pilorama_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Pilorama RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed pilorama requests for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 42 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(frostfs_node_pilorama_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (shard_id, method)\n / \nsum(rate(frostfs_node_pilorama_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (shard_id, method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Pilorama error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the pilorama requests for selected quantile for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 52 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_pilorama_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Pilorama latency", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Duration of waiting time for replication, replication task duration and synchronization for selected quantile", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 62 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_treeservice_replicate_wait_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (success,le)\n)", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "replicate wait success:{{success}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile,\n sum(rate(frostfs_node_treeservice_replicate_task_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (success,le)\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "replicate task success:{{success}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_treeservice_sync_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (success,le)\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "sync success:{{success}}", + "range": true, + "refId": "C" + } + ], + "title": "Replication and synchronization", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Pilorama mode for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1.5, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 72 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(frostfs_node_pilorama_mode{job=\"node\", instance=\"$instance\"}) by (shard_id, mode)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, mode {{mode}}", + "range": true, + "refId": "A" + } + ], + "title": "Pilorama mode", + "transformations": [], + "type": "timeseries" } ], - "title": "Latency, 0.99q", - "transformations": [], - "type": "timeseries" + "title": "Tree service", + "type": "row" } ], "refresh": "1m", @@ -459,7 +910,7 @@ "definition": "label_values(frostfs_node_state_health,instance)", "hide": 0, "includeAll": false, - "label": "", + "label": "Instance", "multi": false, "name": "instance", "options": [], @@ -472,11 +923,44 @@ "skipUrlSync": false, "sort": 1, "type": "query" + }, + { + "current": { + "selected": false, + "text": "0.95", + "value": "0.95" + }, + "hide": 0, + "includeAll": false, + "label": "Quantile", + "multi": false, + "name": "quantile", + "options": [ + { + "selected": true, + "text": "0.95", + "value": "0.95" + }, + { + "selected": false, + "text": "0.99", + "value": "0.99" + }, + { + "selected": false, + "text": "1.00", + "value": "1.00" + } + ], + "query": "0.95, 0.99, 1.00", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" } ] }, "time": { - "from": "now-30m", + "from": "now-1h", "to": "now" }, "timepicker": {}, From 4f9285251ffcf271861f44c6c05dc2359c3931c3 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 2 Aug 2023 15:06:03 +0300 Subject: [PATCH 034/107] [#38] grafana: Add writecache boards Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 727 +++++++++++++++++- 1 file changed, 724 insertions(+), 3 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index df05dc0..ce27576 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -354,7 +354,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Requests per second by Pilorama method for each shard", + "description": "Requests per second by pilorama method for each shard", "fieldConfig": { "defaults": { "color": { @@ -640,7 +640,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_pilorama_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method,le)\n)\n", + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_pilorama_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method, le)\n)\n", "format": "time_series", "instant": false, "interval": "", @@ -833,7 +833,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -889,6 +890,726 @@ ], "title": "Tree service", "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 12, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by writecache method for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id, method) (\n rate(frostfs_node_writecache_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed writecache requests for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 43 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(frostfs_node_writecache_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (shard_id, method)\n / \nsum(rate(frostfs_node_writecache_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (shard_id, method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the Writecache requests for selected quantile for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 53 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_writecache_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Actual count of the objects, that writecache stores", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 63 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_writecache_actual_objects_total{job=\"node\", instance=\"$instance\"}", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, storage {{storage}}", + "range": true, + "refId": "A" + } + ], + "title": "Actual count", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Estimated size of the objects, that writecache stores", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 73 + }, + "id": 17, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_writecache_estimated_size_bytes{job=\"node\", instance=\"$instance\"}", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, storage {{storage}}", + "range": true, + "refId": "A" + } + ], + "title": "Estimated size", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Rate of the writecache background operations", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 83 + }, + "id": 18, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (shard_id, storage, operation) (rate(frostfs_node_writecache_operations_total{job=\"node\", instance=\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, storage {{storage}}, operation {{operation}}", + "range": true, + "refId": "A" + } + ], + "title": "Operations rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Writecache mode for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1.5, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 93 + }, + "id": 19, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(frostfs_node_writecache_mode_info{job=\"node\", instance=\"$instance\"}) by (shard_id, mode)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, mode {{mode}}", + "range": true, + "refId": "A" + } + ], + "title": "Mode", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Writecache", + "type": "row" } ], "refresh": "1m", From d5ee2907403686eb8df604bec19c9d6f05f94c4b Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 2 Aug 2023 16:14:29 +0300 Subject: [PATCH 035/107] [#38] grafana: Add node instance state Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 746 +++++++++++------- 1 file changed, 446 insertions(+), 300 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index ce27576..4bd7a0e 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -23,93 +23,102 @@ "liveNow": false, "panels": [ { - "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Instance state", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [ + { + "options": { + "0": { + "index": 0, + "text": "Undefined" + }, + "1": { + "index": 1, + "text": "Starting" + }, + "2": { + "index": 2, + "text": "Ready" + }, + "3": { + "index": 3, + "text": "Shutting down" + } + }, + "type": "value" + } + ], + "max": 3, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "string" + }, + "overrides": [] + }, "gridPos": { - "h": 1, + "h": 6, "w": 24, "x": 0, "y": 0 }, - "id": 2, - "panels": [], - "title": "Server", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Requests per second by gRPC method", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 1 - }, - "id": 1, + "id": 21, "options": { "legend": { "calcs": [], "displayMode": "table", "placement": "right", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "multi", @@ -125,217 +134,16 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum by (instance, grpc_service, grpc_method) (\n rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])\n)\n", + "expr": "frostfs_node_state_health{job=\"node\", instance=\"$instance\"}\n", "format": "time_series", "instant": false, "interval": "", - "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "legendFormat": "state", "range": true, "refId": "A" } ], - "title": "RPS", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Percentage of the failed requests", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 11 - }, - "id": 3, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(rate(grpc_server_handled_total{instance=\"$instance\",grpc_code!=\"OK\"}[$__rate_interval])) by (grpc_service, grpc_method)\n / \nsum(rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])) by (grpc_service, grpc_method)\n * 100.0\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{grpc_service}}.{{grpc_method}}", - "range": true, - "refId": "A" - } - ], - "title": "Error rate", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Latency of the requests for selected quantile", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 21 - }, - "id": 4, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile($quantile, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{grpc_service}}.{{grpc_method}}", - "range": true, - "refId": "A" - } - ], - "title": "Latency", + "title": "State", "transformations": [], "type": "timeseries" }, @@ -345,7 +153,350 @@ "h": 1, "w": 24, "x": 0, - "y": 31 + "y": 6 + }, + "id": 2, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by gRPC method", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, grpc_service, grpc_method) (\n rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed requests", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "neo.fs.v2.object.ObjectService.Search" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(grpc_server_handled_total{instance=\"$instance\",grpc_code!=\"OK\"}[$__rate_interval])) by (grpc_service, grpc_method)\n / \nsum(rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])) by (grpc_service, grpc_method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the requests for selected quantile", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Server", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 7 }, "id": 6, "panels": [ @@ -398,8 +549,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -499,8 +649,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -599,8 +748,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -699,8 +847,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -833,8 +980,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -897,7 +1043,7 @@ "h": 1, "w": 24, "x": 0, - "y": 32 + "y": 8 }, "id": 12, "panels": [ @@ -967,7 +1113,7 @@ "h": 10, "w": 24, "x": 0, - "y": 33 + "y": 39 }, "id": 13, "options": { @@ -1068,7 +1214,7 @@ "h": 10, "w": 24, "x": 0, - "y": 43 + "y": 49 }, "id": 15, "options": { @@ -1168,7 +1314,7 @@ "h": 10, "w": 24, "x": 0, - "y": 53 + "y": 59 }, "id": 14, "options": { @@ -1268,7 +1414,7 @@ "h": 10, "w": 24, "x": 0, - "y": 63 + "y": 69 }, "id": 16, "options": { @@ -1368,7 +1514,7 @@ "h": 10, "w": 24, "x": 0, - "y": 73 + "y": 79 }, "id": 17, "options": { @@ -1468,7 +1614,7 @@ "h": 10, "w": 24, "x": 0, - "y": 83 + "y": 89 }, "id": 18, "options": { @@ -1570,7 +1716,7 @@ "h": 10, "w": 24, "x": 0, - "y": 93 + "y": 99 }, "id": 19, "options": { From d76cc2e48a0b61534e024707d038184881ba5e5b Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 3 Aug 2023 15:31:19 +0300 Subject: [PATCH 036/107] [#38] grafana: Add replicator dashbords Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 982 ++++++++++++------ 1 file changed, 654 insertions(+), 328 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index 4bd7a0e..0b97983 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -22,6 +22,19 @@ "links": [], "liveNow": false, "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 22, + "panels": [], + "title": "Health", + "type": "row" + }, { "datasource": { "type": "prometheus", @@ -110,7 +123,7 @@ "h": 6, "w": 24, "x": 0, - "y": 0 + "y": 1 }, "id": 21, "options": { @@ -148,347 +161,346 @@ "type": "timeseries" }, { - "collapsed": true, + "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 6 + "y": 7 }, "id": 2, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "panels": [], + "title": "Server", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by gRPC method", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "description": "Requests per second by gRPC method", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 7 - }, - "id": 1, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" }, - "tooltip": { - "mode": "multi", - "sort": "desc" + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" } }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum by (instance, grpc_service, grpc_method) (\n rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])\n)\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{grpc_service}}.{{grpc_method}}", - "range": true, - "refId": "A" - } - ], - "title": "RPS", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Percentage of the failed requests", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [ + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "neo.fs.v2.object.ObjectService.Search" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 } ] }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 17 - }, - "id": 3, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(rate(grpc_server_handled_total{instance=\"$instance\",grpc_code!=\"OK\"}[$__rate_interval])) by (grpc_service, grpc_method)\n / \nsum(rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])) by (grpc_service, grpc_method)\n * 100.0\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{grpc_service}}.{{grpc_method}}", - "range": true, - "refId": "A" - } - ], - "title": "Error rate", - "transformations": [], - "type": "timeseries" + "unit": "reqps" }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Latency of the requests for selected quantile", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 27 - }, - "id": 4, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile($quantile, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{grpc_service}}.{{grpc_method}}", - "range": true, - "refId": "A" - } - ], - "title": "Latency", - "transformations": [], - "type": "timeseries" + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, grpc_service, grpc_method) (\n rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" } ], - "title": "Server", - "type": "row" + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed requests", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "neo.fs.v2.object.ObjectService.Search" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(grpc_server_handled_total{instance=\"$instance\",grpc_code!=\"OK\"}[$__rate_interval])) by (grpc_service, grpc_method)\n / \nsum(rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])) by (grpc_service, grpc_method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the requests for selected quantile", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" }, { "collapsed": true, @@ -496,7 +508,7 @@ "h": 1, "w": 24, "x": 0, - "y": 7 + "y": 38 }, "id": 6, "panels": [ @@ -1043,7 +1055,7 @@ "h": 1, "w": 24, "x": 0, - "y": 8 + "y": 39 }, "id": 12, "panels": [ @@ -1113,7 +1125,7 @@ "h": 10, "w": 24, "x": 0, - "y": 39 + "y": 4 }, "id": 13, "options": { @@ -1214,7 +1226,7 @@ "h": 10, "w": 24, "x": 0, - "y": 49 + "y": 14 }, "id": 15, "options": { @@ -1314,7 +1326,7 @@ "h": 10, "w": 24, "x": 0, - "y": 59 + "y": 24 }, "id": 14, "options": { @@ -1414,7 +1426,7 @@ "h": 10, "w": 24, "x": 0, - "y": 69 + "y": 34 }, "id": 16, "options": { @@ -1514,7 +1526,7 @@ "h": 10, "w": 24, "x": 0, - "y": 79 + "y": 44 }, "id": 17, "options": { @@ -1614,7 +1626,7 @@ "h": 10, "w": 24, "x": 0, - "y": 89 + "y": 54 }, "id": 18, "options": { @@ -1716,7 +1728,7 @@ "h": 10, "w": 24, "x": 0, - "y": 99 + "y": 64 }, "id": 19, "options": { @@ -1756,6 +1768,320 @@ ], "title": "Writecache", "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 23, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Count of in-flight requests", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 75 + }, + "id": 24, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_replicator_in_flight_requests_total{job=\"node\", instance=\"$instance\"}", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "in-flight requests", + "range": true, + "refId": "A" + } + ], + "title": "In-flight request", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Rate of the objects replicated", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 85 + }, + "id": 25, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "rate(frostfs_node_replicator_processed_objects_total{job=\"node\", instance=\"$instance\"}[$__rate_interval])", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "object processing rate", + "range": true, + "refId": "A" + } + ], + "title": "Objects processing rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Rate of the objects payload replicated", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 95 + }, + "id": 26, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "rate(frostfs_node_replicator_total_replicated_payload_size_bytes{job=\"node\", instance=\"$instance\"}[$__rate_interval])", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "payload processing rate", + "range": true, + "refId": "A" + } + ], + "title": "Payload processing rate", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Replicator", + "type": "row" } ], "refresh": "1m", From 74d43f48f2c31ea20d4238efd9a24218239077c4 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 3 Aug 2023 15:48:53 +0300 Subject: [PATCH 037/107] [#38] grafana: Add object service dashboards Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 435 +++++++++++++++++- 1 file changed, 428 insertions(+), 7 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index 0b97983..264ad20 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -1125,7 +1125,7 @@ "h": 10, "w": 24, "x": 0, - "y": 4 + "y": 40 }, "id": 13, "options": { @@ -1226,7 +1226,7 @@ "h": 10, "w": 24, "x": 0, - "y": 14 + "y": 50 }, "id": 15, "options": { @@ -1326,7 +1326,7 @@ "h": 10, "w": 24, "x": 0, - "y": 24 + "y": 60 }, "id": 14, "options": { @@ -1426,7 +1426,7 @@ "h": 10, "w": 24, "x": 0, - "y": 34 + "y": 70 }, "id": 16, "options": { @@ -1526,7 +1526,7 @@ "h": 10, "w": 24, "x": 0, - "y": 44 + "y": 80 }, "id": 17, "options": { @@ -1626,7 +1626,7 @@ "h": 10, "w": 24, "x": 0, - "y": 54 + "y": 90 }, "id": 18, "options": { @@ -1728,7 +1728,7 @@ "h": 10, "w": 24, "x": 0, - "y": 64 + "y": 100 }, "id": 19, "options": { @@ -2082,6 +2082,427 @@ ], "title": "Replicator", "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 27, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by object service method", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 112 + }, + "id": 28, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, method) (\n rate(frostfs_node_object_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed object service requests", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 122 + }, + "id": 29, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(frostfs_node_object_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (method)\n / \nsum(rate(frostfs_node_object_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the object service requests for selected quantile", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 132 + }, + "id": 30, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_object_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Bytes per second by object service method", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 142 + }, + "id": 31, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, method) (\n rate(frostfs_node_object_request_payload_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Payload rate", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Object service", + "type": "row" } ], "refresh": "1m", From 201855e72958978118c8ff9e2aede2c757acf5bc Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 3 Aug 2023 15:57:07 +0300 Subject: [PATCH 038/107] [#38] grafana: Add epoch dashboard Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 108 ++++++++++++++---- 1 file changed, 87 insertions(+), 21 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index 264ad20..4aaaebf 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -32,7 +32,7 @@ }, "id": 22, "panels": [], - "title": "Health", + "title": "General", "type": "row" }, { @@ -121,7 +121,7 @@ }, "gridPos": { "h": 6, - "w": 24, + "w": 12, "x": 0, "y": 1 }, @@ -160,6 +160,72 @@ "transformations": [], "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Current epoch on the instance", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 32, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "frostfs_node_ir_epoch{job=\"node\", instance=\"$instance\"}", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Epoch", + "type": "gauge" + }, { "collapsed": false, "gridPos": { @@ -577,7 +643,7 @@ "h": 10, "w": 24, "x": 0, - "y": 32 + "y": 40 }, "id": 8, "options": { @@ -677,7 +743,7 @@ "h": 10, "w": 24, "x": 0, - "y": 42 + "y": 50 }, "id": 9, "options": { @@ -776,7 +842,7 @@ "h": 10, "w": 24, "x": 0, - "y": 52 + "y": 60 }, "id": 10, "options": { @@ -875,7 +941,7 @@ "h": 10, "w": 24, "x": 0, - "y": 62 + "y": 70 }, "id": 7, "options": { @@ -1008,7 +1074,7 @@ "h": 10, "w": 24, "x": 0, - "y": 72 + "y": 80 }, "id": 11, "options": { @@ -1125,7 +1191,7 @@ "h": 10, "w": 24, "x": 0, - "y": 40 + "y": 48 }, "id": 13, "options": { @@ -1226,7 +1292,7 @@ "h": 10, "w": 24, "x": 0, - "y": 50 + "y": 58 }, "id": 15, "options": { @@ -1326,7 +1392,7 @@ "h": 10, "w": 24, "x": 0, - "y": 60 + "y": 68 }, "id": 14, "options": { @@ -1426,7 +1492,7 @@ "h": 10, "w": 24, "x": 0, - "y": 70 + "y": 78 }, "id": 16, "options": { @@ -1526,7 +1592,7 @@ "h": 10, "w": 24, "x": 0, - "y": 80 + "y": 88 }, "id": 17, "options": { @@ -1626,7 +1692,7 @@ "h": 10, "w": 24, "x": 0, - "y": 90 + "y": 98 }, "id": 18, "options": { @@ -1728,7 +1794,7 @@ "h": 10, "w": 24, "x": 0, - "y": 100 + "y": 108 }, "id": 19, "options": { @@ -1842,7 +1908,7 @@ "h": 10, "w": 24, "x": 0, - "y": 75 + "y": 83 }, "id": 24, "options": { @@ -1942,7 +2008,7 @@ "h": 10, "w": 24, "x": 0, - "y": 85 + "y": 93 }, "id": 25, "options": { @@ -2042,7 +2108,7 @@ "h": 10, "w": 24, "x": 0, - "y": 95 + "y": 103 }, "id": 26, "options": { @@ -2159,7 +2225,7 @@ "h": 10, "w": 24, "x": 0, - "y": 112 + "y": 120 }, "id": 28, "options": { @@ -2260,7 +2326,7 @@ "h": 10, "w": 24, "x": 0, - "y": 122 + "y": 130 }, "id": 29, "options": { @@ -2360,7 +2426,7 @@ "h": 10, "w": 24, "x": 0, - "y": 132 + "y": 140 }, "id": 30, "options": { @@ -2463,7 +2529,7 @@ "h": 10, "w": 24, "x": 0, - "y": 142 + "y": 150 }, "id": 31, "options": { From 8218440525c4129ef97e18534522a879a7a909dd Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 3 Aug 2023 16:11:35 +0300 Subject: [PATCH 039/107] [#38] grafana: Add metabase dashboards Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 3332 ++++++++++------- 1 file changed, 1876 insertions(+), 1456 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index 4aaaebf..20a3ed2 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -576,6 +576,1881 @@ "x": 0, "y": 38 }, + "id": 27, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by object service method", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 120 + }, + "id": 28, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, method) (\n rate(frostfs_node_object_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed object service requests", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 130 + }, + "id": 29, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(frostfs_node_object_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (method)\n / \nsum(rate(frostfs_node_object_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the object service requests for selected quantile", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 140 + }, + "id": 30, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_object_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Bytes per second by object service method", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 150 + }, + "id": 31, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, method) (\n rate(frostfs_node_object_request_payload_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Payload rate", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Object service", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 33, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by metabase method for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 35, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id, method) (\n rate(frostfs_node_metabase_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed metabase requests for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 37, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(frostfs_node_metabase_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (shard_id, method)\n / \nsum(rate(frostfs_node_metabase_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (shard_id, method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the metabase requests for selected quantile for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 36, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_metabase_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Metabase mode for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1.5, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 34, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(frostfs_node_metabase_mode{job=\"node\", instance=\"$instance\"}) by (shard_id, mode)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, mode {{mode}}", + "range": true, + "refId": "A" + } + ], + "title": "Mode", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Metabase", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 12, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by writecache method for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id, method) (\n rate(frostfs_node_writecache_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed writecache requests for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 51 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(frostfs_node_writecache_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (shard_id, method)\n / \nsum(rate(frostfs_node_writecache_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (shard_id, method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the writecache requests for selected quantile for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 61 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_writecache_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Actual count of the objects, that writecache stores", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 71 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_writecache_actual_objects_total{job=\"node\", instance=\"$instance\"}", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, storage {{storage}}", + "range": true, + "refId": "A" + } + ], + "title": "Actual count", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Estimated size of the objects, that writecache stores", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 81 + }, + "id": 17, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_writecache_estimated_size_bytes{job=\"node\", instance=\"$instance\"}", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, storage {{storage}}", + "range": true, + "refId": "A" + } + ], + "title": "Estimated size", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Rate of the writecache background operations", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 91 + }, + "id": 18, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (shard_id, storage, operation) (rate(frostfs_node_writecache_operations_total{job=\"node\", instance=\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, storage {{storage}}, operation {{operation}}", + "range": true, + "refId": "A" + } + ], + "title": "Operations rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Writecache mode for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1.5, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 101 + }, + "id": 19, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(frostfs_node_writecache_mode_info{job=\"node\", instance=\"$instance\"}) by (shard_id, mode)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, mode {{mode}}", + "range": true, + "refId": "A" + } + ], + "title": "Mode", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Writecache", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 23, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Count of in-flight requests", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 83 + }, + "id": 24, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_replicator_in_flight_requests_total{job=\"node\", instance=\"$instance\"}", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "in-flight requests", + "range": true, + "refId": "A" + } + ], + "title": "In-flight request", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Rate of the objects replicated", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 93 + }, + "id": 25, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "rate(frostfs_node_replicator_processed_objects_total{job=\"node\", instance=\"$instance\"}[$__rate_interval])", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "object processing rate", + "range": true, + "refId": "A" + } + ], + "title": "Objects processing rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Rate of the objects payload replicated", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 103 + }, + "id": 26, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "rate(frostfs_node_replicator_total_replicated_payload_size_bytes{job=\"node\", instance=\"$instance\"}[$__rate_interval])", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "payload processing rate", + "range": true, + "refId": "A" + } + ], + "title": "Payload processing rate", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Replicator", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 42 + }, "id": 6, "panels": [ { @@ -1114,1461 +2989,6 @@ ], "title": "Tree service", "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 39 - }, - "id": 12, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Requests per second by writecache method for each shard", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 48 - }, - "id": 13, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum by (instance, shard_id, method) (\n rate(frostfs_node_writecache_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{method}}, shard {{shard_id}}", - "range": true, - "refId": "A" - } - ], - "title": "RPS", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Percentage of the failed writecache requests for each shard", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 58 - }, - "id": 15, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(rate(frostfs_node_writecache_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (shard_id, method)\n / \nsum(rate(frostfs_node_writecache_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (shard_id, method)\n * 100.0\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{method}}, shard {{shard_id}}", - "range": true, - "refId": "A" - } - ], - "title": "Error rate", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Latency of the Writecache requests for selected quantile for each shard", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 68 - }, - "id": 14, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_writecache_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method,le)\n)\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{method}}, shard {{shard_id}}", - "range": true, - "refId": "A" - } - ], - "title": "Latency", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Actual count of the objects, that writecache stores", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 78 - }, - "id": 16, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "frostfs_node_writecache_actual_objects_total{job=\"node\", instance=\"$instance\"}", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "Shard {{shard_id}}, storage {{storage}}", - "range": true, - "refId": "A" - } - ], - "title": "Actual count", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Estimated size of the objects, that writecache stores", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 88 - }, - "id": 17, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "frostfs_node_writecache_estimated_size_bytes{job=\"node\", instance=\"$instance\"}", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "Shard {{shard_id}}, storage {{storage}}", - "range": true, - "refId": "A" - } - ], - "title": "Estimated size", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Rate of the writecache background operations", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 98 - }, - "id": 18, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum by (shard_id, storage, operation) (rate(frostfs_node_writecache_operations_total{job=\"node\", instance=\"$instance\"}[$__rate_interval]))", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "Shard {{shard_id}}, storage {{storage}}, operation {{operation}}", - "range": true, - "refId": "A" - } - ], - "title": "Operations rate", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Writecache mode for each shard", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 1.5, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 108 - }, - "id": 19, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(frostfs_node_writecache_mode_info{job=\"node\", instance=\"$instance\"}) by (shard_id, mode)\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "Shard {{shard_id}}, mode {{mode}}", - "range": true, - "refId": "A" - } - ], - "title": "Mode", - "transformations": [], - "type": "timeseries" - } - ], - "title": "Writecache", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 40 - }, - "id": 23, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Count of in-flight requests", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 83 - }, - "id": 24, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "frostfs_node_replicator_in_flight_requests_total{job=\"node\", instance=\"$instance\"}", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "in-flight requests", - "range": true, - "refId": "A" - } - ], - "title": "In-flight request", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Rate of the objects replicated", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 93 - }, - "id": 25, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "rate(frostfs_node_replicator_processed_objects_total{job=\"node\", instance=\"$instance\"}[$__rate_interval])", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "object processing rate", - "range": true, - "refId": "A" - } - ], - "title": "Objects processing rate", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Rate of the objects payload replicated", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 103 - }, - "id": 26, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "rate(frostfs_node_replicator_total_replicated_payload_size_bytes{job=\"node\", instance=\"$instance\"}[$__rate_interval])", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "payload processing rate", - "range": true, - "refId": "A" - } - ], - "title": "Payload processing rate", - "transformations": [], - "type": "timeseries" - } - ], - "title": "Replicator", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 41 - }, - "id": 27, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Requests per second by object service method", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 120 - }, - "id": 28, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum by (instance, method) (\n rate(frostfs_node_object_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{method}}", - "range": true, - "refId": "A" - } - ], - "title": "RPS", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Percentage of the failed object service requests", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 130 - }, - "id": 29, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(rate(frostfs_node_object_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (method)\n / \nsum(rate(frostfs_node_object_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (method)\n * 100.0\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{method}}, shard {{shard_id}}", - "range": true, - "refId": "A" - } - ], - "title": "Error rate", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Latency of the object service requests for selected quantile", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 140 - }, - "id": 30, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_object_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (method,le)\n)\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{method}}", - "range": true, - "refId": "A" - } - ], - "title": "Latency", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Bytes per second by object service method", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 150 - }, - "id": 31, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum by (instance, method) (\n rate(frostfs_node_object_request_payload_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{method}}", - "range": true, - "refId": "A" - } - ], - "title": "Payload rate", - "transformations": [], - "type": "timeseries" - } - ], - "title": "Object service", - "type": "row" } ], "refresh": "1m", @@ -2579,7 +2999,7 @@ "list": [ { "current": { - "selected": false, + "selected": true, "text": "s01.frostfs.devenv:9090", "value": "s01.frostfs.devenv:9090" }, From 1408558631b490387e17474775a704cdd1c9e4c1 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 4 Aug 2023 13:59:12 +0300 Subject: [PATCH 040/107] [#38] grafana: Fix units Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 681 +++++++++--------- 1 file changed, 341 insertions(+), 340 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index 20a3ed2..a2a37aa 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -227,7 +227,7 @@ "type": "gauge" }, { - "collapsed": false, + "collapsed": true, "gridPos": { "h": 1, "w": 24, @@ -235,338 +235,339 @@ "y": 7 }, "id": 2, - "panels": [], - "title": "Server", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Requests per second by gRPC method", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 8 - }, - "id": 1, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ + "panels": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "editorMode": "code", - "exemplar": false, - "expr": "sum by (instance, grpc_service, grpc_method) (\n rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])\n)\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{grpc_service}}.{{grpc_method}}", - "range": true, - "refId": "A" - } - ], - "title": "RPS", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Percentage of the failed requests", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "description": "Requests per second by gRPC method", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "neo.fs.v2.object.ObjectService.Search" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, - "viz": true + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 18 - }, - "id": 3, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(rate(grpc_server_handled_total{instance=\"$instance\",grpc_code!=\"OK\"}[$__rate_interval])) by (grpc_service, grpc_method)\n / \nsum(rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])) by (grpc_service, grpc_method)\n * 100.0\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{grpc_service}}.{{grpc_method}}", - "range": true, - "refId": "A" - } - ], - "title": "Error rate", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Latency of the requests for selected quantile", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 8 }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "multi", + "sort": "desc" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, grpc_service, grpc_method) (\n rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 28 - }, - "id": 4, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile($quantile, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{grpc_service}}.{{grpc_method}}", - "range": true, - "refId": "A" + "description": "Percentage of the failed requests", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "neo.fs.v2.object.ObjectService.Search" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(grpc_server_handled_total{instance=\"$instance\",grpc_code!=\"OK\"}[$__rate_interval])) by (grpc_service, grpc_method)\n / \nsum(rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])) by (grpc_service, grpc_method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the requests for selected quantile", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" } ], - "title": "Latency", - "transformations": [], - "type": "timeseries" + "title": "Server", + "type": "row" }, { "collapsed": true, @@ -574,7 +575,7 @@ "h": 1, "w": 24, "x": 0, - "y": 38 + "y": 8 }, "id": 27, "panels": [ @@ -644,7 +645,7 @@ "h": 10, "w": 24, "x": 0, - "y": 120 + "y": 9 }, "id": 28, "options": { @@ -745,7 +746,7 @@ "h": 10, "w": 24, "x": 0, - "y": 130 + "y": 19 }, "id": 29, "options": { @@ -837,7 +838,7 @@ } ] }, - "unit": "ms" + "unit": "s" }, "overrides": [] }, @@ -845,7 +846,7 @@ "h": 10, "w": 24, "x": 0, - "y": 140 + "y": 29 }, "id": 30, "options": { @@ -940,7 +941,7 @@ } ] }, - "unit": "Bps" + "unit": "binBps" }, "overrides": [] }, @@ -948,7 +949,7 @@ "h": 10, "w": 24, "x": 0, - "y": 150 + "y": 39 }, "id": 31, "options": { @@ -995,7 +996,7 @@ "h": 1, "w": 24, "x": 0, - "y": 39 + "y": 9 }, "id": 33, "panels": [ @@ -1065,7 +1066,7 @@ "h": 10, "w": 24, "x": 0, - "y": 1 + "y": 10 }, "id": 35, "options": { @@ -1166,7 +1167,7 @@ "h": 10, "w": 24, "x": 0, - "y": 11 + "y": 20 }, "id": 37, "options": { @@ -1258,7 +1259,7 @@ } ] }, - "unit": "ms" + "unit": "s" }, "overrides": [] }, @@ -1266,7 +1267,7 @@ "h": 10, "w": 24, "x": 0, - "y": 21 + "y": 30 }, "id": 36, "options": { @@ -1368,7 +1369,7 @@ "h": 10, "w": 24, "x": 0, - "y": 31 + "y": 40 }, "id": 34, "options": { @@ -1415,7 +1416,7 @@ "h": 1, "w": 24, "x": 0, - "y": 40 + "y": 10 }, "id": 12, "panels": [ @@ -1485,7 +1486,7 @@ "h": 10, "w": 24, "x": 0, - "y": 41 + "y": 11 }, "id": 13, "options": { @@ -1586,7 +1587,7 @@ "h": 10, "w": 24, "x": 0, - "y": 51 + "y": 21 }, "id": 15, "options": { @@ -1678,7 +1679,7 @@ } ] }, - "unit": "ms" + "unit": "s" }, "overrides": [] }, @@ -1686,7 +1687,7 @@ "h": 10, "w": 24, "x": 0, - "y": 61 + "y": 31 }, "id": 14, "options": { @@ -1786,7 +1787,7 @@ "h": 10, "w": 24, "x": 0, - "y": 71 + "y": 41 }, "id": 16, "options": { @@ -1878,7 +1879,7 @@ } ] }, - "unit": "decbytes" + "unit": "bytes" }, "overrides": [] }, @@ -1886,7 +1887,7 @@ "h": 10, "w": 24, "x": 0, - "y": 81 + "y": 51 }, "id": 17, "options": { @@ -1986,7 +1987,7 @@ "h": 10, "w": 24, "x": 0, - "y": 91 + "y": 61 }, "id": 18, "options": { @@ -2088,7 +2089,7 @@ "h": 10, "w": 24, "x": 0, - "y": 101 + "y": 71 }, "id": 19, "options": { @@ -2135,7 +2136,7 @@ "h": 1, "w": 24, "x": 0, - "y": 41 + "y": 11 }, "id": 23, "panels": [ @@ -2202,7 +2203,7 @@ "h": 10, "w": 24, "x": 0, - "y": 83 + "y": 12 }, "id": 24, "options": { @@ -2302,7 +2303,7 @@ "h": 10, "w": 24, "x": 0, - "y": 93 + "y": 22 }, "id": 25, "options": { @@ -2394,7 +2395,7 @@ } ] }, - "unit": "Bps" + "unit": "binBps" }, "overrides": [] }, @@ -2402,7 +2403,7 @@ "h": 10, "w": 24, "x": 0, - "y": 103 + "y": 32 }, "id": 26, "options": { @@ -2449,7 +2450,7 @@ "h": 1, "w": 24, "x": 0, - "y": 42 + "y": 12 }, "id": 6, "panels": [ @@ -2999,7 +3000,7 @@ "list": [ { "current": { - "selected": true, + "selected": false, "text": "s01.frostfs.devenv:9090", "value": "s01.frostfs.devenv:9090" }, From 0231b2bbf0b0a2cf95b255839c79bbbbc28527e7 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 4 Aug 2023 14:34:54 +0300 Subject: [PATCH 041/107] [#38] grafana: Add blobstore dashboard Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 565 +++++++++++++++++- 1 file changed, 551 insertions(+), 14 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index a2a37aa..0d8213f 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -168,10 +168,6 @@ "description": "Current epoch on the instance", "fieldConfig": { "defaults": { - "color": { - "fixedColor": "green", - "mode": "fixed" - }, "mappings": [], "thresholds": { "mode": "absolute", @@ -186,7 +182,7 @@ } ] }, - "unit": "none" + "unit": "short" }, "overrides": [] }, @@ -198,6 +194,9 @@ }, "id": 32, "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ @@ -206,8 +205,7 @@ "fields": "", "values": false }, - "showThresholdLabels": false, - "showThresholdMarkers": true + "textMode": "auto" }, "pluginVersion": "9.5.6", "targets": [ @@ -224,7 +222,7 @@ } ], "title": "Epoch", - "type": "gauge" + "type": "stat" }, { "collapsed": true, @@ -1066,7 +1064,7 @@ "h": 10, "w": 24, "x": 0, - "y": 10 + "y": 51 }, "id": 35, "options": { @@ -1167,7 +1165,7 @@ "h": 10, "w": 24, "x": 0, - "y": 20 + "y": 61 }, "id": 37, "options": { @@ -1267,7 +1265,7 @@ "h": 10, "w": 24, "x": 0, - "y": 30 + "y": 71 }, "id": 36, "options": { @@ -1369,7 +1367,7 @@ "h": 10, "w": 24, "x": 0, - "y": 40 + "y": 81 }, "id": 34, "options": { @@ -2138,6 +2136,545 @@ "x": 0, "y": 11 }, + "id": 38, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by blobstore method for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 39, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id, method) (\n rate(frostfs_node_blobstore_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed blobstore requests for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 40, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(frostfs_node_blobstore_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (shard_id, method)\n / \nsum(rate(frostfs_node_blobstore_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (shard_id, method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the metabase requests for selected quantile for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 41, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_blobstore_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Bytes per second by blobstore shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 42 + }, + "id": 43, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id) (\n rate(frostfs_node_blobstore_put_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Write rate, shard {{shard_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id) (\n rate(frostfs_node_blobstore_get_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Read rate, shard {{shard_id}}", + "range": true, + "refId": "B" + } + ], + "title": "Payload rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Blobstore mode for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1.5, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 52 + }, + "id": 42, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(frostfs_node_blobstore_mode{job=\"node\", instance=\"$instance\"}) by (shard_id, mode)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, mode {{mode}}", + "range": true, + "refId": "A" + } + ], + "title": "Mode", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Blobstore", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 12 + }, "id": 23, "panels": [ { @@ -2450,7 +2987,7 @@ "h": 1, "w": 24, "x": 0, - "y": 12 + "y": 13 }, "id": 6, "panels": [ @@ -3000,7 +3537,7 @@ "list": [ { "current": { - "selected": false, + "selected": true, "text": "s01.frostfs.devenv:9090", "value": "s01.frostfs.devenv:9090" }, From acd32cb87720840754556a255b15f81c6faf6fb8 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 4 Aug 2023 14:50:04 +0300 Subject: [PATCH 042/107] [#38] grafana: Add blobovnizca dashboards Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 757 +++++++++++++++++- 1 file changed, 751 insertions(+), 6 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index 0d8213f..cd8f32c 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -2204,7 +2204,7 @@ "h": 10, "w": 24, "x": 0, - "y": 12 + "y": 13 }, "id": 39, "options": { @@ -2305,7 +2305,7 @@ "h": 10, "w": 24, "x": 0, - "y": 22 + "y": 23 }, "id": 40, "options": { @@ -2405,7 +2405,7 @@ "h": 10, "w": 24, "x": 0, - "y": 32 + "y": 33 }, "id": 41, "options": { @@ -2508,7 +2508,7 @@ "h": 10, "w": 24, "x": 0, - "y": 42 + "y": 43 }, "id": 43, "options": { @@ -2626,7 +2626,7 @@ "h": 10, "w": 24, "x": 0, - "y": 52 + "y": 53 }, "id": 42, "options": { @@ -2675,6 +2675,751 @@ "x": 0, "y": 12 }, + "id": 44, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by blobovnizca method for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 45, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id, method) (\n rate(frostfs_node_blobovniza_tree_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed blobovnizca requests for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 46, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(frostfs_node_blobsovniza_tree_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (shard_id, method)\n / \nsum(rate(frostfs_node_blobsovniza_tree_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (shard_id, method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the blobovnizca requests for selected quantile for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 47, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_blobovniza_tree_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Bytes per second by blobovnizca shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 48, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id) (\n rate(frostfs_node_blobovniza_tree_put_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Write rate, shard {{shard_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id) (\n rate(frostfs_node_blobovniza_tree_get_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Read rate, shard {{shard_id}}", + "range": true, + "refId": "B" + } + ], + "title": "Payload rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Blobovnizca size for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 50, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_blobovniza_tree_size_bytes{instance=\"$instance\"}\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Size", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Count of open DB's of blobovnizca for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 51 + }, + "id": 51, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_blobovniza_tree_open_blobovnizca_count{instance=\"$instance\"}\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Open DB count", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Blobovnizca mode for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1.5, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 61 + }, + "id": 49, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(frostfs_node_blobovniza_tree_mode{job=\"node\", instance=\"$instance\"}) by (shard_id, mode)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, mode {{mode}}", + "range": true, + "refId": "A" + } + ], + "title": "Mode", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Blobovnizca", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 13 + }, "id": 23, "panels": [ { @@ -2987,7 +3732,7 @@ "h": 1, "w": 24, "x": 0, - "y": 13 + "y": 14 }, "id": 6, "panels": [ From cc963b78b5ca1f4b16c67d516a18467954dcce63 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 4 Aug 2023 15:03:48 +0300 Subject: [PATCH 043/107] [#38] grafana: Add fstree dashboards Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 567 +++++++++++++++++- 1 file changed, 553 insertions(+), 14 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index cd8f32c..6fd6bf5 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -2204,7 +2204,7 @@ "h": 10, "w": 24, "x": 0, - "y": 13 + "y": 12 }, "id": 39, "options": { @@ -2305,7 +2305,7 @@ "h": 10, "w": 24, "x": 0, - "y": 23 + "y": 22 }, "id": 40, "options": { @@ -2405,7 +2405,7 @@ "h": 10, "w": 24, "x": 0, - "y": 33 + "y": 32 }, "id": 41, "options": { @@ -2508,7 +2508,7 @@ "h": 10, "w": 24, "x": 0, - "y": 43 + "y": 42 }, "id": 43, "options": { @@ -2626,7 +2626,7 @@ "h": 10, "w": 24, "x": 0, - "y": 53 + "y": 52 }, "id": 42, "options": { @@ -2675,6 +2675,545 @@ "x": 0, "y": 12 }, + "id": 52, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by fstree method for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 63 + }, + "id": 53, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id, method) (\n rate(frostfs_node_fstree_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed fstree requests for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 73 + }, + "id": 54, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(frostfs_node_fstree_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (shard_id, method)\n / \nsum(rate(frostfs_node_fstree_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (shard_id, method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the fstree requests for selected quantile for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 83 + }, + "id": 55, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_fstree_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Bytes per second by fstree shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 93 + }, + "id": 56, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id) (\n rate(frostfs_node_fstree_put_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Write rate, shard {{shard_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id) (\n rate(frostfs_node_fstree_get_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Read rate, shard {{shard_id}}", + "range": true, + "refId": "B" + } + ], + "title": "Payload rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "FSTree mode for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1.5, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 103 + }, + "id": 57, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(frostfs_node_fstree_mode{job=\"node\", instance=\"$instance\"}) by (shard_id, mode)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, mode {{mode}}", + "range": true, + "refId": "A" + } + ], + "title": "Mode", + "transformations": [], + "type": "timeseries" + } + ], + "title": "FSTree", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 13 + }, "id": 44, "panels": [ { @@ -2743,7 +3282,7 @@ "h": 10, "w": 24, "x": 0, - "y": 1 + "y": 64 }, "id": 45, "options": { @@ -2844,7 +3383,7 @@ "h": 10, "w": 24, "x": 0, - "y": 11 + "y": 74 }, "id": 46, "options": { @@ -2944,7 +3483,7 @@ "h": 10, "w": 24, "x": 0, - "y": 21 + "y": 84 }, "id": 47, "options": { @@ -3047,7 +3586,7 @@ "h": 10, "w": 24, "x": 0, - "y": 31 + "y": 94 }, "id": 48, "options": { @@ -3166,7 +3705,7 @@ "h": 10, "w": 24, "x": 0, - "y": 41 + "y": 104 }, "id": 50, "options": { @@ -3269,7 +3808,7 @@ "h": 10, "w": 24, "x": 0, - "y": 51 + "y": 114 }, "id": 51, "options": { @@ -3371,7 +3910,7 @@ "h": 10, "w": 24, "x": 0, - "y": 61 + "y": 124 }, "id": 49, "options": { @@ -3418,7 +3957,7 @@ "h": 1, "w": 24, "x": 0, - "y": 13 + "y": 14 }, "id": 23, "panels": [ @@ -3732,7 +4271,7 @@ "h": 1, "w": 24, "x": 0, - "y": 14 + "y": 15 }, "id": 6, "panels": [ From d8df46b4d19ae673d44a0e55585c1a3fa68f8787 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Sat, 5 Aug 2023 21:16:47 +0300 Subject: [PATCH 044/107] [#38] grafana: Add engine dashboards Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 743 ++++++++++++++++-- 1 file changed, 672 insertions(+), 71 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index 6fd6bf5..0d08ded 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -403,7 +403,7 @@ "options": { "mode": "exclude", "names": [ - "neo.fs.v2.object.ObjectService.Search" + "neo.fs.v2.object.ObjectService.Put" ], "prefix": "All except:", "readOnly": true @@ -643,7 +643,7 @@ "h": 10, "w": 24, "x": 0, - "y": 9 + "y": 20 }, "id": 28, "options": { @@ -744,7 +744,7 @@ "h": 10, "w": 24, "x": 0, - "y": 19 + "y": 30 }, "id": 29, "options": { @@ -844,7 +844,7 @@ "h": 10, "w": 24, "x": 0, - "y": 29 + "y": 40 }, "id": 30, "options": { @@ -930,8 +930,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -947,7 +946,7 @@ "h": 10, "w": 24, "x": 0, - "y": 39 + "y": 50 }, "id": 31, "options": { @@ -996,6 +995,634 @@ "x": 0, "y": 9 }, + "id": 58, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by storage engine service method", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 59, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, method) (\n rate(frostfs_node_engine_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the engine requests for selected quantile for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 60, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_engine_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Payload size by shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 61, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_engine_payload_size_bytes{instance=\"$instance\"}\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Payload", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Objects count by shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 62, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_engine_objects_total{instance=\"$instance\"}\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, type {{type}}", + "range": true, + "refId": "A" + } + ], + "title": "Objects count", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Errors count by shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 50 + }, + "id": 63, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_engine_errors_total{instance=\"$instance\"}\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Errors count", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Mode for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1.5, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 60 + }, + "id": 64, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(frostfs_node_engine_mode_info{job=\"node\", instance=\"$instance\"}) by (shard_id, mode)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, mode {{mode}}", + "range": true, + "refId": "A" + } + ], + "title": "Shard mode", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Engine", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, "id": 33, "panels": [ { @@ -1047,8 +1674,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1148,8 +1774,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1248,8 +1873,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1350,8 +1974,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1414,7 +2037,7 @@ "h": 1, "w": 24, "x": 0, - "y": 10 + "y": 11 }, "id": 12, "panels": [ @@ -1467,8 +2090,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1568,8 +2190,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1668,8 +2289,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1768,8 +2388,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1868,8 +2487,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1968,8 +2586,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2070,8 +2687,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2134,7 +2750,7 @@ "h": 1, "w": 24, "x": 0, - "y": 11 + "y": 12 }, "id": 38, "panels": [ @@ -2204,7 +2820,7 @@ "h": 10, "w": 24, "x": 0, - "y": 12 + "y": 13 }, "id": 39, "options": { @@ -2305,7 +2921,7 @@ "h": 10, "w": 24, "x": 0, - "y": 22 + "y": 23 }, "id": 40, "options": { @@ -2405,7 +3021,7 @@ "h": 10, "w": 24, "x": 0, - "y": 32 + "y": 33 }, "id": 41, "options": { @@ -2508,7 +3124,7 @@ "h": 10, "w": 24, "x": 0, - "y": 42 + "y": 43 }, "id": 43, "options": { @@ -2626,7 +3242,7 @@ "h": 10, "w": 24, "x": 0, - "y": 52 + "y": 53 }, "id": 42, "options": { @@ -2673,7 +3289,7 @@ "h": 1, "w": 24, "x": 0, - "y": 12 + "y": 13 }, "id": 52, "panels": [ @@ -2726,8 +3342,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2827,8 +3442,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2927,8 +3541,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3030,8 +3643,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3148,8 +3760,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3212,7 +3823,7 @@ "h": 1, "w": 24, "x": 0, - "y": 13 + "y": 14 }, "id": 44, "panels": [ @@ -3265,8 +3876,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3366,8 +3976,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3466,8 +4075,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3569,8 +4177,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3688,8 +4295,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3791,8 +4397,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3893,8 +4498,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3957,7 +4561,7 @@ "h": 1, "w": 24, "x": 0, - "y": 14 + "y": 15 }, "id": 23, "panels": [ @@ -4007,8 +4611,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4107,8 +4710,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4207,8 +4809,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4271,7 +4872,7 @@ "h": 1, "w": 24, "x": 0, - "y": 15 + "y": 16 }, "id": 6, "panels": [ @@ -4821,7 +5422,7 @@ "list": [ { "current": { - "selected": true, + "selected": false, "text": "s01.frostfs.devenv:9090", "value": "s01.frostfs.devenv:9090" }, From b6f47cb2c260b7c12377f1cecddee42cb105e05a Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Sun, 6 Aug 2023 13:44:37 +0300 Subject: [PATCH 045/107] [#38] grafana: Add GC dashboard Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 180 +++++++++++++++--- 1 file changed, 152 insertions(+), 28 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index 0d08ded..0d9b437 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -283,8 +283,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -384,8 +383,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -509,8 +507,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -643,7 +640,7 @@ "h": 10, "w": 24, "x": 0, - "y": 20 + "y": 9 }, "id": 28, "options": { @@ -744,7 +741,7 @@ "h": 10, "w": 24, "x": 0, - "y": 30 + "y": 19 }, "id": 29, "options": { @@ -827,8 +824,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -844,7 +840,7 @@ "h": 10, "w": 24, "x": 0, - "y": 40 + "y": 29 }, "id": 30, "options": { @@ -946,7 +942,7 @@ "h": 10, "w": 24, "x": 0, - "y": 50 + "y": 39 }, "id": 31, "options": { @@ -1063,7 +1059,7 @@ "h": 10, "w": 24, "x": 0, - "y": 10 + "y": 11 }, "id": 59, "options": { @@ -1163,7 +1159,7 @@ "h": 10, "w": 24, "x": 0, - "y": 20 + "y": 21 }, "id": 60, "options": { @@ -1266,7 +1262,7 @@ "h": 10, "w": 24, "x": 0, - "y": 30 + "y": 31 }, "id": 61, "options": { @@ -1369,7 +1365,7 @@ "h": 10, "w": 24, "x": 0, - "y": 40 + "y": 41 }, "id": 62, "options": { @@ -1472,7 +1468,7 @@ "h": 10, "w": 24, "x": 0, - "y": 50 + "y": 51 }, "id": 63, "options": { @@ -1574,7 +1570,7 @@ "h": 10, "w": 24, "x": 0, - "y": 60 + "y": 61 }, "id": 64, "options": { @@ -2803,8 +2799,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2904,8 +2899,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3004,8 +2998,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3107,8 +3100,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3225,8 +3217,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5412,6 +5403,139 @@ ], "title": "Tree service", "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 66, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Count of objects GC marked to remove, deleted or failed to delete from disk", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 65, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_garbage_collector_deleted_objects_total{instance=\"$instance\"}\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, {{status}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_garbage_collector_marked_for_removal_objects_total{instance=\"$instance\"}\n", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, marked for remove, {{object_type}}", + "range": true, + "refId": "B" + } + ], + "title": "Objects count", + "transformations": [], + "type": "timeseries" + } + ], + "title": "GC", + "type": "row" } ], "refresh": "1m", From 9bb0385b85e20a8149c599c1fbdefcdf7143e737 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Sun, 6 Aug 2023 15:15:54 +0300 Subject: [PATCH 046/107] [#38] grafana: Add morph dashboards Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 999 ++++++++++++++++-- 1 file changed, 904 insertions(+), 95 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index 0d9b437..f7d827c 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -182,7 +182,7 @@ } ] }, - "unit": "short" + "unit": "none" }, "overrides": [] }, @@ -283,7 +283,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -299,7 +300,7 @@ "h": 10, "w": 24, "x": 0, - "y": 8 + "y": 2 }, "id": 1, "options": { @@ -383,7 +384,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -424,7 +426,7 @@ "h": 10, "w": 24, "x": 0, - "y": 18 + "y": 12 }, "id": 3, "options": { @@ -507,7 +509,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -523,7 +526,7 @@ "h": 10, "w": 24, "x": 0, - "y": 28 + "y": 22 }, "id": 4, "options": { @@ -640,7 +643,7 @@ "h": 10, "w": 24, "x": 0, - "y": 9 + "y": 3 }, "id": 28, "options": { @@ -741,7 +744,7 @@ "h": 10, "w": 24, "x": 0, - "y": 19 + "y": 13 }, "id": 29, "options": { @@ -824,7 +827,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -840,7 +844,7 @@ "h": 10, "w": 24, "x": 0, - "y": 29 + "y": 23 }, "id": 30, "options": { @@ -926,7 +930,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -942,7 +947,7 @@ "h": 10, "w": 24, "x": 0, - "y": 39 + "y": 33 }, "id": 31, "options": { @@ -1059,7 +1064,7 @@ "h": 10, "w": 24, "x": 0, - "y": 11 + "y": 4 }, "id": 59, "options": { @@ -1159,7 +1164,7 @@ "h": 10, "w": 24, "x": 0, - "y": 21 + "y": 14 }, "id": 60, "options": { @@ -1262,7 +1267,7 @@ "h": 10, "w": 24, "x": 0, - "y": 31 + "y": 24 }, "id": 61, "options": { @@ -1365,7 +1370,7 @@ "h": 10, "w": 24, "x": 0, - "y": 41 + "y": 34 }, "id": 62, "options": { @@ -1468,7 +1473,7 @@ "h": 10, "w": 24, "x": 0, - "y": 51 + "y": 44 }, "id": 63, "options": { @@ -1570,7 +1575,7 @@ "h": 10, "w": 24, "x": 0, - "y": 61 + "y": 54 }, "id": 64, "options": { @@ -1670,7 +1675,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1686,7 +1692,7 @@ "h": 10, "w": 24, "x": 0, - "y": 51 + "y": 5 }, "id": 35, "options": { @@ -1770,7 +1776,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1786,7 +1793,7 @@ "h": 10, "w": 24, "x": 0, - "y": 61 + "y": 15 }, "id": 37, "options": { @@ -1869,7 +1876,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1885,7 +1893,7 @@ "h": 10, "w": 24, "x": 0, - "y": 71 + "y": 25 }, "id": 36, "options": { @@ -1970,7 +1978,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1986,7 +1995,7 @@ "h": 10, "w": 24, "x": 0, - "y": 81 + "y": 35 }, "id": 34, "options": { @@ -2086,7 +2095,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2102,7 +2112,7 @@ "h": 10, "w": 24, "x": 0, - "y": 11 + "y": 6 }, "id": 13, "options": { @@ -2186,7 +2196,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2202,7 +2213,7 @@ "h": 10, "w": 24, "x": 0, - "y": 21 + "y": 16 }, "id": 15, "options": { @@ -2285,7 +2296,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2301,7 +2313,7 @@ "h": 10, "w": 24, "x": 0, - "y": 31 + "y": 26 }, "id": 14, "options": { @@ -2384,7 +2396,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2400,7 +2413,7 @@ "h": 10, "w": 24, "x": 0, - "y": 41 + "y": 36 }, "id": 16, "options": { @@ -2483,7 +2496,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2499,7 +2513,7 @@ "h": 10, "w": 24, "x": 0, - "y": 51 + "y": 46 }, "id": 17, "options": { @@ -2582,7 +2596,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2590,7 +2605,7 @@ } ] }, - "unit": "none" + "unit": "ops" }, "overrides": [] }, @@ -2598,7 +2613,7 @@ "h": 10, "w": 24, "x": 0, - "y": 61 + "y": 56 }, "id": 18, "options": { @@ -2683,7 +2698,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2699,7 +2715,7 @@ "h": 10, "w": 24, "x": 0, - "y": 71 + "y": 66 }, "id": 19, "options": { @@ -2799,7 +2815,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2815,7 +2832,7 @@ "h": 10, "w": 24, "x": 0, - "y": 13 + "y": 7 }, "id": 39, "options": { @@ -2899,7 +2916,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2915,7 +2933,7 @@ "h": 10, "w": 24, "x": 0, - "y": 23 + "y": 17 }, "id": 40, "options": { @@ -2998,7 +3016,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3014,7 +3033,7 @@ "h": 10, "w": 24, "x": 0, - "y": 33 + "y": 27 }, "id": 41, "options": { @@ -3100,7 +3119,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3116,7 +3136,7 @@ "h": 10, "w": 24, "x": 0, - "y": 43 + "y": 37 }, "id": 43, "options": { @@ -3217,7 +3237,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3233,7 +3254,7 @@ "h": 10, "w": 24, "x": 0, - "y": 53 + "y": 47 }, "id": 42, "options": { @@ -3333,7 +3354,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3349,7 +3371,7 @@ "h": 10, "w": 24, "x": 0, - "y": 63 + "y": 8 }, "id": 53, "options": { @@ -3433,7 +3455,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3449,7 +3472,7 @@ "h": 10, "w": 24, "x": 0, - "y": 73 + "y": 18 }, "id": 54, "options": { @@ -3532,7 +3555,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3548,7 +3572,7 @@ "h": 10, "w": 24, "x": 0, - "y": 83 + "y": 28 }, "id": 55, "options": { @@ -3634,7 +3658,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3650,7 +3675,7 @@ "h": 10, "w": 24, "x": 0, - "y": 93 + "y": 38 }, "id": 56, "options": { @@ -3751,7 +3776,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3767,7 +3793,7 @@ "h": 10, "w": 24, "x": 0, - "y": 103 + "y": 48 }, "id": 57, "options": { @@ -3867,7 +3893,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3883,7 +3910,7 @@ "h": 10, "w": 24, "x": 0, - "y": 64 + "y": 9 }, "id": 45, "options": { @@ -3967,7 +3994,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3983,7 +4011,7 @@ "h": 10, "w": 24, "x": 0, - "y": 74 + "y": 19 }, "id": 46, "options": { @@ -4066,7 +4094,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4082,7 +4111,7 @@ "h": 10, "w": 24, "x": 0, - "y": 84 + "y": 29 }, "id": 47, "options": { @@ -4168,7 +4197,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4184,7 +4214,7 @@ "h": 10, "w": 24, "x": 0, - "y": 94 + "y": 39 }, "id": 48, "options": { @@ -4286,7 +4316,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4302,7 +4333,7 @@ "h": 10, "w": 24, "x": 0, - "y": 104 + "y": 49 }, "id": 50, "options": { @@ -4388,7 +4419,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4404,7 +4436,7 @@ "h": 10, "w": 24, "x": 0, - "y": 114 + "y": 59 }, "id": 51, "options": { @@ -4489,7 +4521,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4505,7 +4538,7 @@ "h": 10, "w": 24, "x": 0, - "y": 124 + "y": 69 }, "id": 49, "options": { @@ -4602,7 +4635,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4618,7 +4652,7 @@ "h": 10, "w": 24, "x": 0, - "y": 12 + "y": 10 }, "id": 24, "options": { @@ -4701,7 +4735,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4709,7 +4744,7 @@ } ] }, - "unit": "none" + "unit": "cps" }, "overrides": [] }, @@ -4717,7 +4752,7 @@ "h": 10, "w": 24, "x": 0, - "y": 22 + "y": 20 }, "id": 25, "options": { @@ -4800,7 +4835,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4816,7 +4852,7 @@ "h": 10, "w": 24, "x": 0, - "y": 32 + "y": 30 }, "id": 26, "options": { @@ -4916,7 +4952,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4932,7 +4969,7 @@ "h": 10, "w": 24, "x": 0, - "y": 40 + "y": 11 }, "id": 8, "options": { @@ -5016,7 +5053,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -5032,7 +5070,7 @@ "h": 10, "w": 24, "x": 0, - "y": 50 + "y": 21 }, "id": 9, "options": { @@ -5115,7 +5153,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -5131,7 +5170,7 @@ "h": 10, "w": 24, "x": 0, - "y": 60 + "y": 31 }, "id": 10, "options": { @@ -5214,7 +5253,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -5230,7 +5270,7 @@ "h": 10, "w": 24, "x": 0, - "y": 70 + "y": 41 }, "id": 7, "options": { @@ -5347,7 +5387,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -5363,7 +5404,7 @@ "h": 10, "w": 24, "x": 0, - "y": 80 + "y": 51 }, "id": 11, "options": { @@ -5480,7 +5521,7 @@ "h": 10, "w": 24, "x": 0, - "y": 18 + "y": 12 }, "id": 65, "options": { @@ -5536,6 +5577,774 @@ ], "title": "GC", "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 67, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Number of endpoint switches", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 68, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "frostfs_node_morph_switches_total{job=\"node\", instance=\"$instance\"}", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Endpoint switches", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Index of the last received block", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 69, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "frostfs_node_morph_last_block{job=\"node\", instance=\"$instance\"}", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Last block", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Rate of notifications received by notification type", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 70, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "rate(frostfs_node_morph_notifications_total{job=\"node\", instance=\"$instance\"}[$__rate_interval])", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{notification_type}}", + "range": true, + "refId": "A" + } + ], + "title": "Notifications rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Rate of of contract invocations", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 71, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, contract, invoke_type, method) (\n rate(frostfs_node_morph_invoke_duration_seconds_count{job=\"node\", instance=\"$instance\"}[$__rate_interval])\n)", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Contract {{contract}}, type {{invoke_type}}, method {{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Contract method invoke rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed contract invocations", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "neo.fs.v2.object.ObjectService.Put" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 72, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, contract, invoke_type, method) (\n rate(frostfs_node_morph_invoke_duration_seconds_count{job=\"node\", instance=\"$instance\", success!=\"true\"}[$__rate_interval])\n)\n / \nsum by (instance, contract, invoke_type, method) (\n rate(frostfs_node_morph_invoke_duration_seconds_count{job=\"node\", instance=\"$instance\"}[$__rate_interval])\n)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Contract {{contract}}, type {{invoke_type}}, method {{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Contract invocations error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the contract invocations for selected quantile", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 73, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_morph_invoke_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (contract, invoke_type, method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Contract {{contract}}, type {{invoke_type}}, method {{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Contract invocations latency", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Rate of of morph cache methods invocations", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 47 + }, + "id": 75, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, method) (\n rate(frostfs_node_morphcache_request_duration_seconds_count{job=\"node\", instance=\"$instance\"}[$__rate_interval])\n)", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache method rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the morph cache for selected quantile", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 57 + }, + "id": 74, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_morphcache_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache invocations latency", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Morph", + "type": "row" } ], "refresh": "1m", @@ -5546,7 +6355,7 @@ "list": [ { "current": { - "selected": false, + "selected": true, "text": "s01.frostfs.devenv:9090", "value": "s01.frostfs.devenv:9090" }, From 04260ad0d8eba6dd33cf181371be3fa83996ed9c Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 7 Aug 2023 12:17:06 +0300 Subject: [PATCH 047/107] [#38] grafana: Fix writecache boards Stack count and size dashborads. Signed-off-by: Dmitrii Stepanov --- .gitattributes | 1 + .../provisioning/dashboards/storage-node.json | 161 +++++++----------- 2 files changed, 66 insertions(+), 96 deletions(-) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..5436e42 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +/services/grafana/provisioning/dashboards/* -diff -merge \ No newline at end of file diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index f7d827c..e0cd8a8 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -300,7 +300,7 @@ "h": 10, "w": 24, "x": 0, - "y": 2 + "y": 8 }, "id": 1, "options": { @@ -426,7 +426,7 @@ "h": 10, "w": 24, "x": 0, - "y": 12 + "y": 18 }, "id": 3, "options": { @@ -526,7 +526,7 @@ "h": 10, "w": 24, "x": 0, - "y": 22 + "y": 28 }, "id": 4, "options": { @@ -626,8 +626,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -727,8 +726,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -827,8 +825,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -930,8 +927,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1047,8 +1043,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1147,8 +1142,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1250,8 +1244,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1353,8 +1346,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1456,8 +1448,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1558,8 +1549,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1675,8 +1665,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1776,8 +1765,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1876,8 +1864,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1978,8 +1965,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2112,7 +2098,7 @@ "h": 10, "w": 24, "x": 0, - "y": 6 + "y": 12 }, "id": 13, "options": { @@ -2213,7 +2199,7 @@ "h": 10, "w": 24, "x": 0, - "y": 16 + "y": 22 }, "id": 15, "options": { @@ -2313,7 +2299,7 @@ "h": 10, "w": 24, "x": 0, - "y": 26 + "y": 32 }, "id": 14, "options": { @@ -2368,7 +2354,7 @@ "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 100, "gradientMode": "none", "hideFrom": { "legend": false, @@ -2385,7 +2371,7 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "none" + "mode": "normal" }, "thresholdsStyle": { "mode": "off" @@ -2413,7 +2399,7 @@ "h": 10, "w": 24, "x": 0, - "y": 36 + "y": 42 }, "id": 16, "options": { @@ -2468,7 +2454,7 @@ "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 100, "gradientMode": "none", "hideFrom": { "legend": false, @@ -2485,7 +2471,7 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "none" + "mode": "normal" }, "thresholdsStyle": { "mode": "off" @@ -2513,7 +2499,7 @@ "h": 10, "w": 24, "x": 0, - "y": 46 + "y": 52 }, "id": 17, "options": { @@ -2568,7 +2554,7 @@ "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 100, "gradientMode": "none", "hideFrom": { "legend": false, @@ -2585,7 +2571,7 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "none" + "mode": "normal" }, "thresholdsStyle": { "mode": "off" @@ -2613,7 +2599,7 @@ "h": 10, "w": 24, "x": 0, - "y": 56 + "y": 62 }, "id": 18, "options": { @@ -2715,7 +2701,7 @@ "h": 10, "w": 24, "x": 0, - "y": 66 + "y": 72 }, "id": 19, "options": { @@ -2832,7 +2818,7 @@ "h": 10, "w": 24, "x": 0, - "y": 7 + "y": 83 }, "id": 39, "options": { @@ -2933,7 +2919,7 @@ "h": 10, "w": 24, "x": 0, - "y": 17 + "y": 93 }, "id": 40, "options": { @@ -3033,7 +3019,7 @@ "h": 10, "w": 24, "x": 0, - "y": 27 + "y": 103 }, "id": 41, "options": { @@ -3136,7 +3122,7 @@ "h": 10, "w": 24, "x": 0, - "y": 37 + "y": 113 }, "id": 43, "options": { @@ -3254,7 +3240,7 @@ "h": 10, "w": 24, "x": 0, - "y": 47 + "y": 123 }, "id": 42, "options": { @@ -3371,7 +3357,7 @@ "h": 10, "w": 24, "x": 0, - "y": 8 + "y": 84 }, "id": 53, "options": { @@ -3472,7 +3458,7 @@ "h": 10, "w": 24, "x": 0, - "y": 18 + "y": 94 }, "id": 54, "options": { @@ -3572,7 +3558,7 @@ "h": 10, "w": 24, "x": 0, - "y": 28 + "y": 104 }, "id": 55, "options": { @@ -3675,7 +3661,7 @@ "h": 10, "w": 24, "x": 0, - "y": 38 + "y": 114 }, "id": 56, "options": { @@ -3793,7 +3779,7 @@ "h": 10, "w": 24, "x": 0, - "y": 48 + "y": 124 }, "id": 57, "options": { @@ -3910,7 +3896,7 @@ "h": 10, "w": 24, "x": 0, - "y": 9 + "y": 85 }, "id": 45, "options": { @@ -4011,7 +3997,7 @@ "h": 10, "w": 24, "x": 0, - "y": 19 + "y": 95 }, "id": 46, "options": { @@ -4111,7 +4097,7 @@ "h": 10, "w": 24, "x": 0, - "y": 29 + "y": 105 }, "id": 47, "options": { @@ -4214,7 +4200,7 @@ "h": 10, "w": 24, "x": 0, - "y": 39 + "y": 115 }, "id": 48, "options": { @@ -4333,7 +4319,7 @@ "h": 10, "w": 24, "x": 0, - "y": 49 + "y": 125 }, "id": 50, "options": { @@ -4436,7 +4422,7 @@ "h": 10, "w": 24, "x": 0, - "y": 59 + "y": 135 }, "id": 51, "options": { @@ -4538,7 +4524,7 @@ "h": 10, "w": 24, "x": 0, - "y": 69 + "y": 145 }, "id": 49, "options": { @@ -4635,8 +4621,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4735,8 +4720,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4835,8 +4819,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4952,8 +4935,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5053,8 +5035,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5153,8 +5134,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5253,8 +5233,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5387,8 +5366,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5504,8 +5482,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5601,8 +5578,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5665,8 +5641,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5762,8 +5737,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5862,8 +5836,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5963,8 +5936,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6088,8 +6060,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6188,8 +6159,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6288,8 +6258,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6355,7 +6324,7 @@ "list": [ { "current": { - "selected": true, + "selected": false, "text": "s01.frostfs.devenv:9090", "value": "s01.frostfs.devenv:9090" }, From 51053e331720b36b3b52adb8420b3883f530ce6f Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 11 Aug 2023 15:57:45 +0300 Subject: [PATCH 048/107] [#46] .forgejo: Update DCO action Signed-off-by: Evgenii Stratonikov --- .forgejo/workflows/dco.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.forgejo/workflows/dco.yml b/.forgejo/workflows/dco.yml index 880addd..6746408 100644 --- a/.forgejo/workflows/dco.yml +++ b/.forgejo/workflows/dco.yml @@ -1,3 +1,4 @@ +name: DCO action on: [pull_request] jobs: @@ -12,9 +13,9 @@ jobs: - name: Setup Go uses: actions/setup-go@v3 with: - go-version: '1.20' + go-version: '1.21' - name: Run commit format checker - uses: https://git.alexvan.in/alexvanin/dco-go@v1 + uses: https://git.frostfs.info/TrueCloudLab/dco-go@v2 with: - from: dca6ff62 + from: 'origin/${{ github.event.pull_request.base.ref }}' From 731976cc57de38e699032428695d0a276b6ecc13 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 11 Aug 2023 15:48:57 +0300 Subject: [PATCH 049/107] [#44] grafana: Add Client dashboard Client dashboard shows outgoing requests. Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 410 +++++++++++++++--- 1 file changed, 352 insertions(+), 58 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index e0cd8a8..f78962c 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -575,6 +575,324 @@ "x": 0, "y": 8 }, + "id": 76, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by gRPC method", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 77, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, grpc_service, grpc_method) (\n rate(grpc_client_started_total{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed requests", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 78, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(grpc_client_handled_total{instance=\"$instance\",grpc_code!=\"OK\"}[$__rate_interval])) by (grpc_service, grpc_method)\n / \nsum(rate(grpc_client_started_total{instance=\"$instance\"}[$__rate_interval])) by (grpc_service, grpc_method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the requests for selected quantile", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 79, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(grpc_client_handling_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Client", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, "id": 27, "panels": [ { @@ -990,7 +1308,7 @@ "h": 1, "w": 24, "x": 0, - "y": 9 + "y": 10 }, "id": 58, "panels": [ @@ -1612,7 +1930,7 @@ "h": 1, "w": 24, "x": 0, - "y": 10 + "y": 11 }, "id": 33, "panels": [ @@ -2028,7 +2346,7 @@ "h": 1, "w": 24, "x": 0, - "y": 11 + "y": 12 }, "id": 12, "panels": [ @@ -2081,8 +2399,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2182,8 +2499,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2282,8 +2598,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2382,8 +2697,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2482,8 +2796,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2582,8 +2895,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2684,8 +2996,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2748,7 +3059,7 @@ "h": 1, "w": 24, "x": 0, - "y": 12 + "y": 13 }, "id": 38, "panels": [ @@ -2801,8 +3112,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2902,8 +3212,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3002,8 +3311,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3105,8 +3413,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3223,8 +3530,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3287,7 +3593,7 @@ "h": 1, "w": 24, "x": 0, - "y": 13 + "y": 14 }, "id": 52, "panels": [ @@ -3340,8 +3646,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3441,8 +3746,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3541,8 +3845,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3644,8 +3947,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3762,8 +4064,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3826,7 +4127,7 @@ "h": 1, "w": 24, "x": 0, - "y": 14 + "y": 15 }, "id": 44, "panels": [ @@ -3879,8 +4180,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3980,8 +4280,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4080,8 +4379,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4183,8 +4481,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4302,8 +4599,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4405,8 +4701,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4507,8 +4802,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4571,7 +4865,7 @@ "h": 1, "w": 24, "x": 0, - "y": 15 + "y": 16 }, "id": 23, "panels": [ @@ -4882,7 +5176,7 @@ "h": 1, "w": 24, "x": 0, - "y": 16 + "y": 17 }, "id": 6, "panels": [ @@ -5429,7 +5723,7 @@ "h": 1, "w": 24, "x": 0, - "y": 17 + "y": 18 }, "id": 66, "panels": [ @@ -5561,7 +5855,7 @@ "h": 1, "w": 24, "x": 0, - "y": 18 + "y": 19 }, "id": 67, "panels": [ From 6eedab3d83db60fc68d4a7288ba463c7683b46b7 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 18 Aug 2023 13:40:03 +0300 Subject: [PATCH 050/107] [#48] grafana: Fix blobovnicza board Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 199 +++++++++++++----- 1 file changed, 152 insertions(+), 47 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index f78962c..318f8c7 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -283,8 +283,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -384,8 +383,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -509,8 +507,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -626,8 +623,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -727,8 +723,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -827,8 +822,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4136,7 +4130,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Requests per second by blobovnizca method for each shard", + "description": "Requests per second by blobovnicza method for each shard", "fieldConfig": { "defaults": { "color": { @@ -4180,7 +4174,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4196,7 +4191,7 @@ "h": 10, "w": 24, "x": 0, - "y": 85 + "y": 16 }, "id": 45, "options": { @@ -4220,7 +4215,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum by (instance, shard_id, method) (\n rate(frostfs_node_blobovniza_tree_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", + "expr": "sum by (instance, shard_id, method) (\n rate(frostfs_node_blobovnicza_tree_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", "format": "time_series", "instant": false, "interval": "", @@ -4238,7 +4233,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Percentage of the failed blobovnizca requests for each shard", + "description": "Percentage of the failed blobovnicza requests for each shard", "fieldConfig": { "defaults": { "color": { @@ -4280,7 +4275,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4296,7 +4292,7 @@ "h": 10, "w": 24, "x": 0, - "y": 95 + "y": 26 }, "id": 46, "options": { @@ -4320,7 +4316,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum(rate(frostfs_node_blobsovniza_tree_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (shard_id, method)\n / \nsum(rate(frostfs_node_blobsovniza_tree_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (shard_id, method)\n * 100.0\n", + "expr": "sum(rate(frostfs_node_blobsovnicza_tree_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (shard_id, method)\n / \nsum(rate(frostfs_node_blobsovnicza_tree_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (shard_id, method)\n * 100.0\n", "format": "time_series", "instant": false, "interval": "", @@ -4338,7 +4334,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Latency of the blobovnizca requests for selected quantile for each shard", + "description": "Latency of the blobovnicza requests for selected quantile for each shard", "fieldConfig": { "defaults": { "color": { @@ -4379,7 +4375,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4395,7 +4392,7 @@ "h": 10, "w": 24, "x": 0, - "y": 105 + "y": 36 }, "id": 47, "options": { @@ -4419,7 +4416,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_blobovniza_tree_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method,le)\n)\n", + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method,le)\n)\n", "format": "time_series", "instant": false, "interval": "", @@ -4437,7 +4434,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Bytes per second by blobovnizca shard", + "description": "Bytes per second by blobovnicza shard", "fieldConfig": { "defaults": { "color": { @@ -4481,7 +4478,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4497,7 +4495,7 @@ "h": 10, "w": 24, "x": 0, - "y": 115 + "y": 46 }, "id": 48, "options": { @@ -4521,7 +4519,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum by (instance, shard_id) (\n rate(frostfs_node_blobovniza_tree_put_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n", + "expr": "sum by (instance, shard_id) (\n rate(frostfs_node_blobovnicza_tree_put_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n", "format": "time_series", "instant": false, "interval": "", @@ -4536,7 +4534,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum by (instance, shard_id) (\n rate(frostfs_node_blobovniza_tree_get_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n", + "expr": "sum by (instance, shard_id) (\n rate(frostfs_node_blobovnicza_tree_get_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n", "format": "time_series", "hide": false, "instant": false, @@ -4555,7 +4553,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Blobovnizca size for each shard", + "description": "Open blobovnicza size for each shard", "fieldConfig": { "defaults": { "color": { @@ -4599,7 +4597,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4615,7 +4614,7 @@ "h": 10, "w": 24, "x": 0, - "y": 125 + "y": 56 }, "id": 50, "options": { @@ -4639,7 +4638,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "frostfs_node_blobovniza_tree_size_bytes{instance=\"$instance\"}\n", + "expr": "frostfs_node_blobovnicza_tree_open_blobovnicza_size_bytes{instance=\"$instance\"}\n", "format": "time_series", "instant": false, "interval": "", @@ -4648,7 +4647,7 @@ "refId": "A" } ], - "title": "Size", + "title": "Open blobovnicza size", "transformations": [], "type": "timeseries" }, @@ -4657,7 +4656,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Count of open DB's of blobovnizca for each shard", + "description": "Items count of open blobovnicza for each shard", "fieldConfig": { "defaults": { "color": { @@ -4701,7 +4700,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4717,7 +4717,110 @@ "h": 10, "w": 24, "x": 0, - "y": 135 + "y": 66 + }, + "id": 80, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_blobovnicza_tree_open_blobovnicza_items_total{instance=\"$instance\"}\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Open blobovnicza items", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Count of open blobovniczas for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 76 }, "id": 51, "options": { @@ -4741,7 +4844,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "frostfs_node_blobovniza_tree_open_blobovnizca_count{instance=\"$instance\"}\n", + "expr": "frostfs_node_blobovnicza_tree_open_blobovnicza_count{instance=\"$instance\"}\n", "format": "time_series", "instant": false, "interval": "", @@ -4750,7 +4853,7 @@ "refId": "A" } ], - "title": "Open DB count", + "title": "Open blobovnicza count", "transformations": [], "type": "timeseries" }, @@ -4759,7 +4862,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Blobovnizca mode for each shard", + "description": "Blobovnicza mode for each shard", "fieldConfig": { "defaults": { "color": { @@ -4802,7 +4905,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4818,7 +4922,7 @@ "h": 10, "w": 24, "x": 0, - "y": 145 + "y": 86 }, "id": 49, "options": { @@ -4842,7 +4946,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum(frostfs_node_blobovniza_tree_mode{job=\"node\", instance=\"$instance\"}) by (shard_id, mode)\n", + "expr": "sum(frostfs_node_blobovnicza_tree_mode{job=\"node\", instance=\"$instance\"}) by (shard_id, mode)\n", "format": "time_series", "instant": false, "interval": "", @@ -4856,7 +4960,7 @@ "type": "timeseries" } ], - "title": "Blobovnizca", + "title": "Blobovnicza", "type": "row" }, { @@ -5776,7 +5880,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -5792,7 +5897,7 @@ "h": 10, "w": 24, "x": 0, - "y": 12 + "y": 19 }, "id": 65, "options": { @@ -6618,7 +6723,7 @@ "list": [ { "current": { - "selected": false, + "selected": true, "text": "s01.frostfs.devenv:9090", "value": "s01.frostfs.devenv:9090" }, From b7ac6f30cf476556afa8f0bbb6aafecaeeb3d54f Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 21 Aug 2023 10:19:37 +0300 Subject: [PATCH 051/107] [#50] services/ir: Take all contract hashes from NNS Signed-off-by: Evgenii Stratonikov --- services/ir/.ir.env | 2 -- 1 file changed, 2 deletions(-) diff --git a/services/ir/.ir.env b/services/ir/.ir.env index 08d7e17..810cf6d 100644 --- a/services/ir/.ir.env +++ b/services/ir/.ir.env @@ -1,3 +1 @@ -FROSTFS_IR_CONTRACTS_FROSTFSID=27407c76feabc407908f3d09a3d845d45e7c981a - FROSTFS_IR_CONTROL_GRPC_ENDPOINT=127.0.0.1:16512 From eba763ff79829617896ed3e3a2471e82d7538fdd Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 25 Aug 2023 09:54:40 +0300 Subject: [PATCH 052/107] [#49] grafana: Server bucket distribution Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/grpc-buckets.json | 362 ++++++++++++++++++ 1 file changed, 362 insertions(+) create mode 100644 services/grafana/provisioning/dashboards/grpc-buckets.json diff --git a/services/grafana/provisioning/dashboards/grpc-buckets.json b/services/grafana/provisioning/dashboards/grpc-buckets.json new file mode 100644 index 0000000..ba7cd5f --- /dev/null +++ b/services/grafana/provisioning/dashboards/grpc-buckets.json @@ -0,0 +1,362 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 4, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "$service", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 20, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(grpc_server_handling_seconds_bucket{le=\"0.005\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])", + "hide": false, + "legendFormat": "< 0.005s", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"0.01\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"0.005\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.005s - 0.01s", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"0.025\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"0.01\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.01s - 0.025s", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"0.05\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"0.025\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.025s - 0.05s", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"0.1\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"0.05\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.05s - 0.1s", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"0.25\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"0.1\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.1s - 0.25s", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"0.5\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"0.25\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.25s - 0.5s", + "range": true, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"1\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"0.5\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.5s - 1s", + "range": true, + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"2.5\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"1\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "1s - 2.5s", + "range": true, + "refId": "I" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"5\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"2.5\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "2.5s - 5s", + "range": true, + "refId": "J" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"10\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"5\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "5s - 10s", + "range": true, + "refId": "K" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"+Inf\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"10\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "> 10s", + "range": true, + "refId": "L" + } + ], + "title": "$method", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "s01.frostfs.devenv:9090", + "value": "s01.frostfs.devenv:9090" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(grpc_server_handling_seconds_bucket,instance)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "instance", + "options": [], + "query": { + "query": "label_values(grpc_server_handling_seconds_bucket,instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": "", + "current": { + "selected": false, + "text": "neo.fs.v2.object.ObjectService", + "value": "neo.fs.v2.object.ObjectService" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(grpc_server_handling_seconds_bucket{instance=\"$instance\"},grpc_service)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "service", + "options": [], + "query": { + "query": "label_values(grpc_server_handling_seconds_bucket{instance=\"$instance\"},grpc_service)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "Put", + "value": "Put" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(grpc_server_handling_seconds_bucket{grpc_service=\"$service\", instance=\"$instance\"},grpc_method)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "method", + "options": [], + "query": { + "query": "label_values(grpc_server_handling_seconds_bucket{grpc_service=\"$service\", instance=\"$instance\"},grpc_method)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "GRPC server bucket distribution", + "uid": "a6830bc6-cffa-4e29-9622-6a10921748b0", + "version": 8, + "weekStart": "" + } \ No newline at end of file From 42e634927653e641430bac1dd07ea3571003aceb Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 25 Aug 2023 10:18:39 +0300 Subject: [PATCH 053/107] [#49] grafana: Storage engine bucket distribution Signed-off-by: Dmitrii Stepanov --- .../dashboards/engine-buckets.json | 335 ++++++++++++++++++ 1 file changed, 335 insertions(+) create mode 100644 services/grafana/provisioning/dashboards/engine-buckets.json diff --git a/services/grafana/provisioning/dashboards/engine-buckets.json b/services/grafana/provisioning/dashboards/engine-buckets.json new file mode 100644 index 0000000..8bc704b --- /dev/null +++ b/services/grafana/provisioning/dashboards/engine-buckets.json @@ -0,0 +1,335 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 5, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "$method", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 20, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])", + "hide": false, + "legendFormat": "< 0.005s", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.005s - 0.01s", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.01s - 0.025s", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.025s - 0.05s", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.05s - 0.1s", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.1s - 0.25s", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.25s - 0.5s", + "range": true, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.5s - 1s", + "range": true, + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "1s - 2.5s", + "range": true, + "refId": "I" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "2.5s - 5s", + "range": true, + "refId": "J" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "5s - 10s", + "range": true, + "refId": "K" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"+Inf\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "> 10s", + "range": true, + "refId": "L" + } + ], + "title": "$method", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "s01.frostfs.devenv:9090", + "value": "s01.frostfs.devenv:9090" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_engine_request_duration_seconds_bucket,instance)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "instance", + "options": [], + "query": { + "query": "label_values(frostfs_node_engine_request_duration_seconds_bucket,instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": "Put", + "value": "Put" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_engine_request_duration_seconds_bucket{instance=\"$instance\"},method)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "method", + "options": [], + "query": { + "query": "label_values(frostfs_node_engine_request_duration_seconds_bucket{instance=\"$instance\"},method)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Storage node engine bucket distribution", + "uid": "f886d27c-b877-495a-a1cb-d02839115cea", + "version": 2, + "weekStart": "" + } \ No newline at end of file From 4a6b481618f3eaa991a0f8db1101791136ce57ec Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 25 Aug 2023 10:43:51 +0300 Subject: [PATCH 054/107] [#49] grafana: Writecache bucket distribution Signed-off-by: Dmitrii Stepanov --- .../dashboards/writecache-buckets.json | 361 ++++++++++++++++++ 1 file changed, 361 insertions(+) create mode 100644 services/grafana/provisioning/dashboards/writecache-buckets.json diff --git a/services/grafana/provisioning/dashboards/writecache-buckets.json b/services/grafana/provisioning/dashboards/writecache-buckets.json new file mode 100644 index 0000000..cd37893 --- /dev/null +++ b/services/grafana/provisioning/dashboards/writecache-buckets.json @@ -0,0 +1,361 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 6, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "$method", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 20, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "< 0.005s", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.005s - 0.01s", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.01s - 0.025s", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.025s - 0.05s", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.05s - 0.1s", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.1s - 0.25s", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.25s - 0.5s", + "range": true, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.5s - 1s", + "range": true, + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "1s - 2.5s", + "range": true, + "refId": "I" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "2.5s - 5s", + "range": true, + "refId": "J" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "5s - 10s", + "range": true, + "refId": "K" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"+Inf\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "> 10s", + "range": true, + "refId": "L" + } + ], + "title": "$method", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "s01.frostfs.devenv:9090", + "value": "s01.frostfs.devenv:9090" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_writecache_request_duration_seconds_bucket,instance)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "instance", + "options": [], + "query": { + "query": "label_values(frostfs_node_writecache_request_duration_seconds_bucket,instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": "BGH3NNzxkiW7JB1BKLHb5z", + "value": "BGH3NNzxkiW7JB1BKLHb5z" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_writecache_request_duration_seconds_bucket{instance=\"$instance\"},shard_id)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "shard_id", + "options": [], + "query": { + "query": "label_values(frostfs_node_writecache_request_duration_seconds_bucket{instance=\"$instance\"},shard_id)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": "Delete", + "value": "Delete" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_writecache_request_duration_seconds_bucket{instance=\"$instance\", shard_id=\"$shard_id\"},method)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "method", + "options": [], + "query": { + "query": "label_values(frostfs_node_writecache_request_duration_seconds_bucket{instance=\"$instance\", shard_id=\"$shard_id\"},method)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Storage node writecache bucket distribution", + "uid": "d2fef1d1-9ef3-437e-ad21-670a20e0e9b4", + "version": 3, + "weekStart": "" + } \ No newline at end of file From f0c3c02943fd24e14b0a63d20f8cce0ee78f7edc Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 25 Aug 2023 10:54:38 +0300 Subject: [PATCH 055/107] [#49] grafana: Blobovnicza bucket distribution Signed-off-by: Dmitrii Stepanov --- .../dashboards/blobovnicza-buckets.json | 361 ++++++++++++++++++ 1 file changed, 361 insertions(+) create mode 100644 services/grafana/provisioning/dashboards/blobovnicza-buckets.json diff --git a/services/grafana/provisioning/dashboards/blobovnicza-buckets.json b/services/grafana/provisioning/dashboards/blobovnicza-buckets.json new file mode 100644 index 0000000..e35272a --- /dev/null +++ b/services/grafana/provisioning/dashboards/blobovnicza-buckets.json @@ -0,0 +1,361 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 7, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "$method", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 20, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "< 0.005s", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.005s - 0.01s", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.01s - 0.025s", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.025s - 0.05s", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.05s - 0.1s", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.1s - 0.25s", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.25s - 0.5s", + "range": true, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.5s - 1s", + "range": true, + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "1s - 2.5s", + "range": true, + "refId": "I" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "2.5s - 5s", + "range": true, + "refId": "J" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "5s - 10s", + "range": true, + "refId": "K" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"+Inf\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "> 10s", + "range": true, + "refId": "L" + } + ], + "title": "$method", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "s01.frostfs.devenv:9090", + "value": "s01.frostfs.devenv:9090" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket,instance)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "instance", + "options": [], + "query": { + "query": "label_values(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket,instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "3Xd2VDqghZnMnaRoP4bKeP", + "value": "3Xd2VDqghZnMnaRoP4bKeP" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{instance=\"$instance\"},shard_id)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "shard_id", + "options": [], + "query": { + "query": "label_values(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{instance=\"$instance\"},shard_id)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": "Put", + "value": "Put" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{instance=\"$instance\", shard_id=\"$shard_id\"},method)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "method", + "options": [], + "query": { + "query": "label_values(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{instance=\"$instance\", shard_id=\"$shard_id\"},method)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Storage node blobovnicza bucket distribution", + "uid": "e4097619-1ce9-4039-bd62-4f5b4bba8d26", + "version": 2, + "weekStart": "" + } \ No newline at end of file From fdcf71d5b6fe4d18b3060b8aec3244c19303a174 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 25 Aug 2023 11:12:11 +0300 Subject: [PATCH 056/107] [#49] grafana: Add metabase bucket distribution Signed-off-by: Dmitrii Stepanov --- .../dashboards/metabase-buckets.json | 361 ++++++++++++++++++ 1 file changed, 361 insertions(+) create mode 100644 services/grafana/provisioning/dashboards/metabase-buckets.json diff --git a/services/grafana/provisioning/dashboards/metabase-buckets.json b/services/grafana/provisioning/dashboards/metabase-buckets.json new file mode 100644 index 0000000..2fba386 --- /dev/null +++ b/services/grafana/provisioning/dashboards/metabase-buckets.json @@ -0,0 +1,361 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 8, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "$method", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 20, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "< 0.005s", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.005s - 0.01s", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.01s - 0.025s", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.025s - 0.05s", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.05s - 0.1s", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.1s - 0.25s", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.25s - 0.5s", + "range": true, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.5s - 1s", + "range": true, + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "1s - 2.5s", + "range": true, + "refId": "I" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "2.5s - 5s", + "range": true, + "refId": "J" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "5s - 10s", + "range": true, + "refId": "K" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"+Inf\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "> 10s", + "range": true, + "refId": "L" + } + ], + "title": "$method", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "s01.frostfs.devenv:9090", + "value": "s01.frostfs.devenv:9090" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_metabase_request_duration_seconds_bucket,instance)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "instance", + "options": [], + "query": { + "query": "label_values(frostfs_node_metabase_request_duration_seconds_bucket,instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "3Xd2VDqghZnMnaRoP4bKeP", + "value": "3Xd2VDqghZnMnaRoP4bKeP" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_metabase_request_duration_seconds_bucket{instance=\"$instance\"},shard_id)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "shard_id", + "options": [], + "query": { + "query": "label_values(frostfs_node_metabase_request_duration_seconds_bucket{instance=\"$instance\"},shard_id)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": "Delete", + "value": "Delete" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_metabase_request_duration_seconds_bucket{instance=\"$instance\", shard_id=\"$shard_id\"},method)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "method", + "options": [], + "query": { + "query": "label_values(frostfs_node_metabase_request_duration_seconds_bucket{instance=\"$instance\", shard_id=\"$shard_id\"},method)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Storage node metabase bucket distribution", + "uid": "e110ed66-d086-461d-958a-efc6c55607d2", + "version": 1, + "weekStart": "" + } \ No newline at end of file From 90bd39d7171fb7d43c71a008e78762aaf61d8e1a Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 25 Aug 2023 11:23:07 +0300 Subject: [PATCH 057/107] [#43] grafana: FSTree bucket distribution Signed-off-by: Dmitrii Stepanov --- .../dashboards/fstree-buckets.json | 361 ++++++++++++++++++ 1 file changed, 361 insertions(+) create mode 100644 services/grafana/provisioning/dashboards/fstree-buckets.json diff --git a/services/grafana/provisioning/dashboards/fstree-buckets.json b/services/grafana/provisioning/dashboards/fstree-buckets.json new file mode 100644 index 0000000..151376b --- /dev/null +++ b/services/grafana/provisioning/dashboards/fstree-buckets.json @@ -0,0 +1,361 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 9, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "$method", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 20, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "< 0.005s", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.005s - 0.01s", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.01s - 0.025s", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.025s - 0.05s", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.05s - 0.1s", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.1s - 0.25s", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.25s - 0.5s", + "range": true, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.5s - 1s", + "range": true, + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "1s - 2.5s", + "range": true, + "refId": "I" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "2.5s - 5s", + "range": true, + "refId": "J" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "5s - 10s", + "range": true, + "refId": "K" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"+Inf\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "> 10s", + "range": true, + "refId": "L" + } + ], + "title": "$method", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "s01.frostfs.devenv:9090", + "value": "s01.frostfs.devenv:9090" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_fstree_request_duration_seconds_bucket,instance)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "instance", + "options": [], + "query": { + "query": "label_values(frostfs_node_fstree_request_duration_seconds_bucket,instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "3Xd2VDqghZnMnaRoP4bKeP", + "value": "3Xd2VDqghZnMnaRoP4bKeP" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_fstree_request_duration_seconds_bucket{instance=\"$instance\"},shard_id)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "shard_id", + "options": [], + "query": { + "query": "label_values(frostfs_node_fstree_request_duration_seconds_bucket{instance=\"$instance\"},shard_id)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "Get", + "value": "Get" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_fstree_request_duration_seconds_bucket{instance=\"$instance\", shard_id=\"$shard_id\"},method)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "method", + "options": [], + "query": { + "query": "label_values(frostfs_node_fstree_request_duration_seconds_bucket{instance=\"$instance\", shard_id=\"$shard_id\"},method)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Storage node fstree bucket distribution", + "uid": "e6407e43-9419-445b-b715-284d69201519", + "version": 2, + "weekStart": "" + } \ No newline at end of file From e254eba6a86e5e0bef92e30ae33a94bdf9be7811 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 25 Aug 2023 14:09:45 +0300 Subject: [PATCH 058/107] [#52] frostfs-adm.yml: Allow maintenance mode by default It is a DEV-env after all. Signed-off-by: Evgenii Stratonikov --- frostfs-adm.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/frostfs-adm.yml b/frostfs-adm.yml index c44046d..5077431 100644 --- a/frostfs-adm.yml +++ b/frostfs-adm.yml @@ -5,6 +5,7 @@ network: epoch_duration: 240 basic_income_rate: 100000000 homomorphic_hash_disabled: false + maintenance_mode_allowed: true fee: audit: 10000 candidate: 10000000000 From 4413251994bc8480c038d1acc16810eeb005de0f Mon Sep 17 00:00:00 2001 From: Marina Biryukova Date: Wed, 13 Sep 2023 16:20:46 +0300 Subject: [PATCH 059/107] [#20] Add frostfs-cli configurations Signed-off-by: Marina Biryukova --- configs/s01-cli.yml | 4 ++++ configs/s02-cli.yml | 4 ++++ configs/s03-cli.yml | 4 ++++ configs/s04-cli.yml | 4 ++++ 4 files changed, 16 insertions(+) create mode 100644 configs/s01-cli.yml create mode 100644 configs/s02-cli.yml create mode 100644 configs/s03-cli.yml create mode 100644 configs/s04-cli.yml diff --git a/configs/s01-cli.yml b/configs/s01-cli.yml new file mode 100644 index 0000000..3bd7c43 --- /dev/null +++ b/configs/s01-cli.yml @@ -0,0 +1,4 @@ +wallet: services/storage/wallet01.json +password: "" +rpc-endpoint: s01.frostfs.devenv:8080 +endpoint: s01.frostfs.devenv:8081 \ No newline at end of file diff --git a/configs/s02-cli.yml b/configs/s02-cli.yml new file mode 100644 index 0000000..ac5a8bb --- /dev/null +++ b/configs/s02-cli.yml @@ -0,0 +1,4 @@ +wallet: services/storage/wallet02.json +password: "" +rpc-endpoint: s02.frostfs.devenv:8080 +endpoint: s02.frostfs.devenv:8081 \ No newline at end of file diff --git a/configs/s03-cli.yml b/configs/s03-cli.yml new file mode 100644 index 0000000..ed288dc --- /dev/null +++ b/configs/s03-cli.yml @@ -0,0 +1,4 @@ +wallet: services/storage/wallet03.json +password: "" +rpc-endpoint: s03.frostfs.devenv:8080 +endpoint: s03.frostfs.devenv:8081 \ No newline at end of file diff --git a/configs/s04-cli.yml b/configs/s04-cli.yml new file mode 100644 index 0000000..f4353d5 --- /dev/null +++ b/configs/s04-cli.yml @@ -0,0 +1,4 @@ +wallet: services/storage/wallet04.json +password: "" +rpc-endpoint: s04.frostfs.devenv:8080 +endpoint: s04.frostfs.devenv:8081 \ No newline at end of file From c2e2b6442c970022b5dc0a7dba100840c3daa652 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 29 Aug 2023 12:45:13 +0300 Subject: [PATCH 060/107] [#47] grafana: Don't bind port to localhost Signed-off-by: Denis Kirillov --- services/grafana/docker-compose.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/services/grafana/docker-compose.yml b/services/grafana/docker-compose.yml index 795b87c..6d27299 100644 --- a/services/grafana/docker-compose.yml +++ b/services/grafana/docker-compose.yml @@ -14,8 +14,6 @@ services: - ./../../vendor/hosts:/etc/hosts - ./grafana.ini:/etc/grafana/grafana.ini - ./provisioning:/etc/grafana/provisioning - ports: - - '3000:3000' stop_signal: SIGKILL env_file: [ ".env", ".int_test.env" ] From de8b58911e498b6a522297093ae221d93e2bd997 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 29 Aug 2023 14:06:09 +0300 Subject: [PATCH 061/107] [#47] prometheus: Don't bind port to localhost Signed-off-by: Denis Kirillov --- services/prometheus/docker-compose.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/services/prometheus/docker-compose.yml b/services/prometheus/docker-compose.yml index aef2c67..e9eb4f9 100644 --- a/services/prometheus/docker-compose.yml +++ b/services/prometheus/docker-compose.yml @@ -15,8 +15,6 @@ services: - ./prometheus.yml:/etc/prometheus/prometheus.yml command: - --config.file=/etc/prometheus/prometheus.yml - ports: - - '9090:9090' stop_signal: SIGKILL env_file: [ ".env", ".prometheus.env", ".int_test.env" ] @@ -24,4 +22,4 @@ networks: prometheus_int: internet: external: true - name: basenet_internet \ No newline at end of file + name: basenet_internet From 9c9ec639f05f1479cc84be713149df0937f33ac4 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 25 Oct 2023 10:56:08 +0300 Subject: [PATCH 062/107] [#58] Update neo-go to v0.103.0 Signed-off-by: Evgenii Stratonikov --- .env | 2 +- services/morph_chain/protocol.privnet.yml | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.env b/.env index e04d5ca..de7b691 100644 --- a/.env +++ b/.env @@ -8,7 +8,7 @@ BASTION_VERSION=10 BASTION_IMAGE=debian # NeoGo privnet -NEOGO_VERSION=0.101.1 +NEOGO_VERSION=0.103.0 NEOGO_IMAGE=nspccdev/neo-go # FrostFS InnerRing nodes diff --git a/services/morph_chain/protocol.privnet.yml b/services/morph_chain/protocol.privnet.yml index 13e03b5..560730a 100644 --- a/services/morph_chain/protocol.privnet.yml +++ b/services/morph_chain/protocol.privnet.yml @@ -29,6 +29,11 @@ ApplicationConfiguration: AttemptConnPeers: 5 MinPeers: 0 Relay: true + Consensus: + Enabled: true + UnlockWallet: + Path: "./wallets/node-wallet.json" + Password: "one" RPC: Addresses: - "192.168.130.90:30333" @@ -49,6 +54,3 @@ ApplicationConfiguration: Addresses: - ":20011" Enabled: true - UnlockWallet: - Path: "./wallets/node-wallet.json" - Password: "one" From 40454b55076c9590d9e9ded5a85069820cf88cbd Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Wed, 15 Nov 2023 16:08:15 +0300 Subject: [PATCH 063/107] [#56] Add loki Signed-off-by: Alexander Chuprov --- .env | 8 +- services/grafana/.hosts | 1 + services/grafana/docker-compose.yml | 8 + .../dashboards/storage-node-logs.json | 358 ++++++++++++++++++ .../provisioning/datasources/datasource.yml | 5 + services/storage/cfg/config.yml | 7 +- 6 files changed, 384 insertions(+), 3 deletions(-) create mode 100644 services/grafana/provisioning/dashboards/storage-node-logs.json diff --git a/.env b/.env index de7b691..0969ff5 100644 --- a/.env +++ b/.env @@ -12,11 +12,11 @@ NEOGO_VERSION=0.103.0 NEOGO_IMAGE=nspccdev/neo-go # FrostFS InnerRing nodes -IR_VERSION=365a7ca0 +IR_VERSION=0.37.0-rc.1-93-g364f835b IR_IMAGE=truecloudlab/frostfs-ir # FrostFS Storage nodes -NODE_VERSION=365a7ca0 +NODE_VERSION=0.37.0-rc.1-93-g364f835b NODE_IMAGE=truecloudlab/frostfs-storage # NATS Server @@ -64,3 +64,7 @@ PROMETHEUS_IMAGE=prom/prometheus # Grafana versions GRAFANA_VERSION=9.5.6 GRAFANA_IMAGE=grafana/grafana + +# Loki versions +LOKI_VERSION=2.9.1 +LOKI_IMAGE=grafana/loki \ No newline at end of file diff --git a/services/grafana/.hosts b/services/grafana/.hosts index 8f36ea6..537a644 100644 --- a/services/grafana/.hosts +++ b/services/grafana/.hosts @@ -1 +1,2 @@ IPV4_PREFIX.122 grafana.LOCAL_DOMAIN +IPV4_PREFIX.123 loki.LOCAL_DOMAIN diff --git a/services/grafana/docker-compose.yml b/services/grafana/docker-compose.yml index 6d27299..90b67f2 100644 --- a/services/grafana/docker-compose.yml +++ b/services/grafana/docker-compose.yml @@ -17,6 +17,14 @@ services: stop_signal: SIGKILL env_file: [ ".env", ".int_test.env" ] + loki: + image: ${LOKI_IMAGE}:${LOKI_VERSION} + command: -config.file=/etc/loki/local-config.yaml + networks: + grafana_int: + internet: + ipv4_address: ${IPV4_PREFIX}.123 + networks: grafana_int: internet: diff --git a/services/grafana/provisioning/dashboards/storage-node-logs.json b/services/grafana/provisioning/dashboards/storage-node-logs.json new file mode 100644 index 0000000..3c91d52 --- /dev/null +++ b/services/grafana/provisioning/dashboards/storage-node-logs.json @@ -0,0 +1,358 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Overview panel for all services", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "count_over_time({hostname=~\"$hostname\"} |~ \"$level\" |~ `$filter` [1s])", + "queryType": "range", + "refId": "A" + } + ], + "title": "Log intensity", + "type": "timeseries" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.width", + "value": 226 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "labels" + }, + "properties": [ + { + "id": "custom.width", + "value": 424 + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 4, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "{hostname=~\"$hostname\"} |~ `$filter` | json level |~ \"$level\"", + "queryType": "range", + "refId": "A" + } + ], + "title": "Log", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "id": true, + "tsNs": true + }, + "indexByName": {}, + "renameByName": {} + } + } + ], + "type": "table" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "label": "Hostname", + "multi": true, + "name": "hostname", + "options": [], + "query": { + "label": "hostname", + "refId": "LokiVariableQueryEditor-VariableQuery", + "stream": "", + "type": 1 + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "hide": 0, + "includeAll": true, + "label": "Level", + "multi": true, + "name": "level", + "options": [ + { + "selected": true, + "text": "All", + "value": "$__all" + }, + { + "selected": false, + "text": "debug", + "value": "debug" + }, + { + "selected": false, + "text": "info", + "value": "info" + }, + { + "selected": false, + "text": "warn", + "value": "warn" + }, + { + "selected": false, + "text": "error", + "value": "error" + }, + { + "selected": false, + "text": "dpanic", + "value": "dpanic" + }, + { + "selected": false, + "text": "panic", + "value": "panic" + }, + { + "selected": false, + "text": "fatal", + "value": "fatal" + } + ], + "query": "debug,info,warn,error,dpanic,panic,fatal", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + }, + { + "current": { + "selected": false, + "text": "", + "value": "" + }, + "hide": 0, + "label": "Filter", + "name": "filter", + "options": [ + { + "selected": true, + "text": "", + "value": "" + } + ], + "query": "", + "skipUrlSync": false, + "type": "textbox" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Storage node logs", + "uid": "bef4b42d-c74e-4241-96cb-d239113f44bc", + "version": 1, + "weekStart": "" + } \ No newline at end of file diff --git a/services/grafana/provisioning/datasources/datasource.yml b/services/grafana/provisioning/datasources/datasource.yml index 81b6a7b..7d7427f 100644 --- a/services/grafana/provisioning/datasources/datasource.yml +++ b/services/grafana/provisioning/datasources/datasource.yml @@ -6,3 +6,8 @@ datasources: access: proxy orgId: 1 url: http://prometheus:9090 +- name: Loki + type: loki + access: proxy + orgId: 1 + url: http://loki:3100 \ No newline at end of file diff --git a/services/storage/cfg/config.yml b/services/storage/cfg/config.yml index 0a85096..6b3e7a8 100644 --- a/services/storage/cfg/config.yml +++ b/services/storage/cfg/config.yml @@ -1,6 +1,11 @@ # Logger section logger: level: debug # Minimum enabled logging level + loki: + enabled: true + endpoint: "loki.frostfs.devenv:3100/api/prom/push" + max_batch_delay: 1s + max_batch_size: 200 # Profiler section pprof: @@ -14,7 +19,7 @@ prometheus: address: :9090 # Server address shutdown_timeout: 15s # Timeout for metrics HTTP server graceful shutdown - # Application tracing section +# Application tracing section tracing: enabled: true exporter: otlp_grpc From 1077c9d35898ec5dd4ba71436d9f4e6489ab26d3 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 6 Dec 2023 14:08:47 +0300 Subject: [PATCH 064/107] [#59] .env: Update neo-go to v0.104.0 Signed-off-by: Evgenii Stratonikov --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 0969ff5..16d2da5 100644 --- a/.env +++ b/.env @@ -8,7 +8,7 @@ BASTION_VERSION=10 BASTION_IMAGE=debian # NeoGo privnet -NEOGO_VERSION=0.103.0 +NEOGO_VERSION=0.104.0 NEOGO_IMAGE=nspccdev/neo-go # FrostFS InnerRing nodes From dfad34fdeae470937d797b065ead0e29fe4a01dc Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Thu, 14 Dec 2023 16:50:16 +0300 Subject: [PATCH 065/107] [#61] Update frostfs-node components with contracts Signed-off-by: Alex Vanin --- .env | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.env b/.env index 16d2da5..7752b67 100644 --- a/.env +++ b/.env @@ -12,11 +12,11 @@ NEOGO_VERSION=0.104.0 NEOGO_IMAGE=nspccdev/neo-go # FrostFS InnerRing nodes -IR_VERSION=0.37.0-rc.1-93-g364f835b +IR_VERSION=0.37.0-rc.1-164-gbdd43f62 IR_IMAGE=truecloudlab/frostfs-ir # FrostFS Storage nodes -NODE_VERSION=0.37.0-rc.1-93-g364f835b +NODE_VERSION=0.37.0-rc.1-164-gbdd43f62 NODE_IMAGE=truecloudlab/frostfs-storage # NATS Server @@ -40,17 +40,17 @@ LOCODE_DB_URL=https://git.frostfs.info/attachments/a2e8def7-52b6-49f1-89cd-a0567 #LOCODE_DB_PATH=/path/to/locode_db # FrostFS CLI binary -FROSTFS_CLI_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/BbngJDdRJEDJTJk7qptq3SxKqrJqtvVYWU6R5AaFGbtG +FROSTFS_CLI_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/HUu3vMojqiopmAzWgUPPPSN7ThosapsLj5WrBvZR67tC #FROSTFS_CLI_PATH=/path/to/frostfs-cli-binary # FrostFS ADM tool binary FROSTFS_ADM_VERSION=eca5c210 -FROSTFS_ADM_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/2GxarAjGUb3RevxvqFGYT3hDQxNNaHzK6aFxhJCAMehq +FROSTFS_ADM_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/NMw4AktoK2zo5Kftj23wGEVzE1d52ULufjtcLjAVqZc #FROSTFS_ADM_PATH=/path/to/frostfs-adm-binary # Compiled FrostFS Smart Contracts FROSTFS_CONTRACTS_VERSION=8537293e -FROSTFS_CONTRACTS_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/6ccZoj4HxoN1G1qvJAX2Qw9p2D6qdyzAjNMaNkEKYQpA +FROSTFS_CONTRACTS_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/BLG2NrQW5UAF1RSMSBK8Z4mGanhvzD9aTRk4NsRGVDV9 #FROSTFS_CONTRACTS_PATH=/path/to/unpacked/frostfs-contracts-dir # Jaeger tracing @@ -67,4 +67,4 @@ GRAFANA_IMAGE=grafana/grafana # Loki versions LOKI_VERSION=2.9.1 -LOKI_IMAGE=grafana/loki \ No newline at end of file +LOKI_IMAGE=grafana/loki From 9654b77236aa20f4badcc706a77bb7ed2c930e83 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Thu, 14 Dec 2023 16:53:12 +0300 Subject: [PATCH 066/107] [#61] Update frostfs-http-gw Signed-off-by: Alex Vanin --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 7752b67..99c5ed0 100644 --- a/.env +++ b/.env @@ -24,7 +24,7 @@ NATS_VERSION=2.7.2 NATS_IMAGE=nats # HTTP Gate -HTTP_GW_VERSION=0.27.0-rc.1-15-g1776db28 +HTTP_GW_VERSION=0.28.0-rc.1-15-g5ae75eb9 HTTP_GW_IMAGE=truecloudlab/frostfs-http-gw # REST Gate From 0f6f2722c2fe672a1246c6f6732c50711d1af271 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Thu, 14 Dec 2023 16:57:17 +0300 Subject: [PATCH 067/107] [#61] Update frostfs-s3-gw Signed-off-by: Alex Vanin --- .env | 2 +- services/s3_gate/cfg/config.yml | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/.env b/.env index 99c5ed0..793cffc 100644 --- a/.env +++ b/.env @@ -32,7 +32,7 @@ REST_GW_VERSION=c9c85e90 REST_GW_IMAGE=truecloudlab/frostfs-rest-gw # S3 Gate -S3_GW_VERSION=0.27.0-rc.1-30-gce929468 +S3_GW_VERSION=0.28.0-rc.1-51-g9272f4e1 S3_GW_IMAGE=truecloudlab/frostfs-s3-gw # FrostFS LOCODE database diff --git a/services/s3_gate/cfg/config.yml b/services/s3_gate/cfg/config.yml index 7fa041a..03e84e0 100644 --- a/services/s3_gate/cfg/config.yml +++ b/services/s3_gate/cfg/config.yml @@ -33,3 +33,17 @@ server: wallet: path: /wallet.json # Path to wallet passphrase: "s3" # Passphrase to decrypt wallet + +features: + md5: + enabled: true + +control: + grpc: + endpoint: localhost:16515 + +frostfsid: + enabled: false + +policy: + enabled: false From 19e5cec49fdd5fbf244ee234b5bc8cdb81873964 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 20 Feb 2024 15:47:55 +0300 Subject: [PATCH 068/107] [#63] Use SIGTERM to stop Go services Signed-off-by: Evgenii Stratonikov --- services/ir/docker-compose.yml | 3 ++- services/rest_gate/docker-compose.yml | 3 ++- services/s3_gate/docker-compose.yml | 3 ++- services/storage/docker-compose.yml | 12 ++++++++---- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/services/ir/docker-compose.yml b/services/ir/docker-compose.yml index 77ec50d..0f8e0f9 100644 --- a/services/ir/docker-compose.yml +++ b/services/ir/docker-compose.yml @@ -13,7 +13,8 @@ services: ir_int: internet: ipv4_address: ${IPV4_PREFIX}.61 - stop_signal: SIGKILL + stop_signal: SIGTERM + stop_grace_period: 15s volumes: - ./az.json:/wallet.json - ./az.key:/wallet01.key diff --git a/services/rest_gate/docker-compose.yml b/services/rest_gate/docker-compose.yml index f5bac6b..ee1ab5e 100644 --- a/services/rest_gate/docker-compose.yml +++ b/services/rest_gate/docker-compose.yml @@ -16,7 +16,8 @@ services: - ./wallet.json:/wallet.json - ./../../vendor/hosts:/etc/hosts - ./cfg:/etc/frostfs/rest - stop_signal: SIGKILL + stop_signal: SIGTERM + stop_grace_period: 15s env_file: [ ".env", ".int_test.env" ] command: [ "frostfs-rest-gw", "--config", "/etc/frostfs/rest/config.yml" ] environment: diff --git a/services/s3_gate/docker-compose.yml b/services/s3_gate/docker-compose.yml index 1445480..d1b7601 100644 --- a/services/s3_gate/docker-compose.yml +++ b/services/s3_gate/docker-compose.yml @@ -18,7 +18,8 @@ services: - ./tls.crt:/tls.crt - ./../../vendor/hosts:/etc/hosts - ./cfg:/etc/frostfs/s3 - stop_signal: SIGKILL + stop_signal: SIGTERM + stop_grace_period: 15s env_file: [ ".env", ".s3.env", ".int_test.env" ] command: [ "frostfs-s3-gw", "--config", "/etc/frostfs/s3/config.yml" ] environment: diff --git a/services/storage/docker-compose.yml b/services/storage/docker-compose.yml index b32ba21..1aabc5e 100644 --- a/services/storage/docker-compose.yml +++ b/services/storage/docker-compose.yml @@ -24,7 +24,8 @@ services: - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key - ../nats/ca-cert.pem:/etc/frostfs-node/nats.ca.crt - ./cfg:/etc/frostfs/storage - stop_signal: SIGKILL + stop_signal: SIGTERM + stop_grace_period: 15s env_file: [ ".env", ".storage.env", ".int_test.env" ] command: [ "frostfs-node", "--config", "/etc/frostfs/storage/config.yml" ] environment: @@ -64,7 +65,8 @@ services: - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key - ../nats/ca-cert.pem:/etc/frostfs-node/nats.ca.crt - ./cfg:/etc/frostfs/storage - stop_signal: SIGKILL + stop_signal: SIGTERM + stop_grace_period: 15s env_file: [ ".env", ".storage.env", ".int_test.env" ] command: [ "frostfs-node", "--config", "/etc/frostfs/storage/config.yml" ] environment: @@ -104,7 +106,8 @@ services: - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key - ../nats/ca-cert.pem:/etc/frostfs-node/nats.ca.crt - ./cfg:/etc/frostfs/storage - stop_signal: SIGKILL + stop_signal: SIGTERM + stop_grace_period: 15s env_file: [ ".env", ".storage.env", ".int_test.env" ] command: [ "frostfs-node", "--config", "/etc/frostfs/storage/config.yml" ] environment: @@ -145,7 +148,8 @@ services: - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key - ../nats/ca-cert.pem:/etc/frostfs-node/nats.ca.crt - ./cfg:/etc/frostfs/storage - stop_signal: SIGKILL + stop_signal: SIGTERM + stop_grace_period: 15s env_file: [ ".env", ".storage.env", ".int_test.env" ] command: [ "frostfs-node", "--config", "/etc/frostfs/storage/config.yml" ] environment: From 8edfcb364d9a104e15f976117d20485a864a0d41 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Thu, 22 Feb 2024 12:41:55 +0300 Subject: [PATCH 069/107] [#64] adm: Create default Allow policy for root namespace Signed-off-by: Airat Arifullin --- Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile b/Makefile index 4caa292..051aa7d 100644 --- a/Makefile +++ b/Makefile @@ -76,6 +76,10 @@ up/bootstrap: get vendor/hosts @$(foreach SVC, $(START_BOOTSTRAP), $(shell docker-compose -f services/$(SVC)/docker-compose.yml up -d)) @source ./bin/helper.sh @./vendor/frostfs-adm --config frostfs-adm.yml morph init --contracts vendor/contracts + echo "Set rule chain to policy contract" + @./vendor/frostfs-adm --config frostfs-adm.yml morph \ + ape add-rule-chain --target-type namespace --target-name "" \ + --rule 'allow Container.* *' --chain-id "allow_container_ops" @for f in ./services/storage/wallet*.json; do \ echo "Transfer GAS to wallet $${f}" \ && ./vendor/frostfs-adm -c frostfs-adm.yml morph refill-gas --storage-wallet $${f} --gas 10.0 \ From 47b4917e7bed1eae19b72e2d4dadc521b9b9c870 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 4 Apr 2024 11:38:41 +0300 Subject: [PATCH 070/107] [#65] services: Fix docker-compose warnings There were multiple warning like this one. ``` WARN[0000] /secret/services/rest_gate/docker-compose.yml: `version` is obsolete ``` According to docker-compose spec, the parameter is indeed purely informative: https://github.com/compose-spec/compose-spec/blob/master/spec.md#version-and-name-top-level-elements Signed-off-by: Evgenii Stratonikov --- services/basenet/docker-compose.yml | 1 - services/grafana/docker-compose.yml | 1 - services/http_gate/docker-compose.yml | 1 - services/ir/docker-compose.yml | 1 - services/jaeger/docker-compose.yml | 1 - services/morph_chain/docker-compose.yml | 1 - services/nats/docker-compose.yml | 1 - services/prometheus/docker-compose.yml | 1 - services/rest_gate/docker-compose.yml | 1 - services/s3_gate/docker-compose.yml | 1 - services/storage/docker-compose.yml | 1 - 11 files changed, 11 deletions(-) diff --git a/services/basenet/docker-compose.yml b/services/basenet/docker-compose.yml index abac92a..b2905a3 100644 --- a/services/basenet/docker-compose.yml +++ b/services/basenet/docker-compose.yml @@ -1,6 +1,5 @@ --- -version: "2.4" services: basenet: diff --git a/services/grafana/docker-compose.yml b/services/grafana/docker-compose.yml index 90b67f2..3953400 100644 --- a/services/grafana/docker-compose.yml +++ b/services/grafana/docker-compose.yml @@ -1,4 +1,3 @@ -version: '2.4' services: grafana: image: ${GRAFANA_IMAGE}:${GRAFANA_VERSION} diff --git a/services/http_gate/docker-compose.yml b/services/http_gate/docker-compose.yml index 7108ee5..c2e68ae 100644 --- a/services/http_gate/docker-compose.yml +++ b/services/http_gate/docker-compose.yml @@ -1,6 +1,5 @@ --- -version: "2.4" services: http_gate: image: ${HTTP_GW_IMAGE}:${HTTP_GW_VERSION} diff --git a/services/ir/docker-compose.yml b/services/ir/docker-compose.yml index 0f8e0f9..048651b 100644 --- a/services/ir/docker-compose.yml +++ b/services/ir/docker-compose.yml @@ -1,6 +1,5 @@ --- -version: "2.4" services: ir01: diff --git a/services/jaeger/docker-compose.yml b/services/jaeger/docker-compose.yml index a460548..1fb367b 100644 --- a/services/jaeger/docker-compose.yml +++ b/services/jaeger/docker-compose.yml @@ -1,4 +1,3 @@ -version: '2.4' services: jaeger: image: ${JAEGER_IMAGE}:${JAEGER_VERSION} diff --git a/services/morph_chain/docker-compose.yml b/services/morph_chain/docker-compose.yml index 4cba4b3..dd44b2c 100644 --- a/services/morph_chain/docker-compose.yml +++ b/services/morph_chain/docker-compose.yml @@ -1,6 +1,5 @@ --- -version: "2.4" services: frostfs_morph_chain: image: ${NEOGO_IMAGE}:${NEOGO_VERSION} diff --git a/services/nats/docker-compose.yml b/services/nats/docker-compose.yml index d05dd04..cb89948 100644 --- a/services/nats/docker-compose.yml +++ b/services/nats/docker-compose.yml @@ -1,6 +1,5 @@ --- -version: "2.4" services: nats: image: ${NATS_IMAGE}:${NATS_VERSION} diff --git a/services/prometheus/docker-compose.yml b/services/prometheus/docker-compose.yml index e9eb4f9..31f5fbe 100644 --- a/services/prometheus/docker-compose.yml +++ b/services/prometheus/docker-compose.yml @@ -1,4 +1,3 @@ -version: '2.4' services: prometheus: image: ${PROMETHEUS_IMAGE}:${PROMETHEUS_VERSION} diff --git a/services/rest_gate/docker-compose.yml b/services/rest_gate/docker-compose.yml index ee1ab5e..09cfe38 100644 --- a/services/rest_gate/docker-compose.yml +++ b/services/rest_gate/docker-compose.yml @@ -1,6 +1,5 @@ --- -version: "2.4" services: rest_gate: image: ${REST_GW_IMAGE}:${REST_GW_VERSION} diff --git a/services/s3_gate/docker-compose.yml b/services/s3_gate/docker-compose.yml index d1b7601..6b43a0b 100644 --- a/services/s3_gate/docker-compose.yml +++ b/services/s3_gate/docker-compose.yml @@ -1,6 +1,5 @@ --- -version: "2.4" services: s3_gate: image: ${S3_GW_IMAGE}:${S3_GW_VERSION} diff --git a/services/storage/docker-compose.yml b/services/storage/docker-compose.yml index 1aabc5e..8a8698e 100644 --- a/services/storage/docker-compose.yml +++ b/services/storage/docker-compose.yml @@ -1,6 +1,5 @@ --- -version: "2.4" services: storage01: image: ${NODE_IMAGE}:${NODE_VERSION} From 773ea2339bf1eebc4946a0adbad904d7a8c8debf Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 15 Apr 2024 20:16:51 +0300 Subject: [PATCH 071/107] [#66] Update frostfs-* to v0.38.5 Signed-off-by: Alexander Chuprov --- .env | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.env b/.env index 793cffc..ad96218 100644 --- a/.env +++ b/.env @@ -12,11 +12,11 @@ NEOGO_VERSION=0.104.0 NEOGO_IMAGE=nspccdev/neo-go # FrostFS InnerRing nodes -IR_VERSION=0.37.0-rc.1-164-gbdd43f62 +IR_VERSION=0.38.5 IR_IMAGE=truecloudlab/frostfs-ir # FrostFS Storage nodes -NODE_VERSION=0.37.0-rc.1-164-gbdd43f62 +NODE_VERSION=0.38.5 NODE_IMAGE=truecloudlab/frostfs-storage # NATS Server @@ -40,17 +40,17 @@ LOCODE_DB_URL=https://git.frostfs.info/attachments/a2e8def7-52b6-49f1-89cd-a0567 #LOCODE_DB_PATH=/path/to/locode_db # FrostFS CLI binary -FROSTFS_CLI_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/HUu3vMojqiopmAzWgUPPPSN7ThosapsLj5WrBvZR67tC +FROSTFS_CLI_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/BtA6zWLtoDuDnzxSNkKPjyN5hGuSZFxGuJD1gh8kBwkP #FROSTFS_CLI_PATH=/path/to/frostfs-cli-binary # FrostFS ADM tool binary -FROSTFS_ADM_VERSION=eca5c210 -FROSTFS_ADM_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/NMw4AktoK2zo5Kftj23wGEVzE1d52ULufjtcLjAVqZc +FROSTFS_ADM_VERSION=498f9955ea +FROSTFS_ADM_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/CjV4S6ENzo2FLK4KxXZHHNW4veR1ATtynGY6Mc1xQ6RB #FROSTFS_ADM_PATH=/path/to/frostfs-adm-binary # Compiled FrostFS Smart Contracts -FROSTFS_CONTRACTS_VERSION=8537293e -FROSTFS_CONTRACTS_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/BLG2NrQW5UAF1RSMSBK8Z4mGanhvzD9aTRk4NsRGVDV9 +FROSTFS_CONTRACTS_VERSION=694daebb19 +FROSTFS_CONTRACTS_URL=https://git.frostfs.info/TrueCloudLab/frostfs-contract/releases/download/v0.19.2/frostfs-contract-v0.19.2.tar.gz #FROSTFS_CONTRACTS_PATH=/path/to/unpacked/frostfs-contracts-dir # Jaeger tracing From 2b6122192a4c26a84e6f8faa5533ab52fb2a88ef Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 8 May 2024 11:33:19 +0300 Subject: [PATCH 072/107] [#67] services/ir: Remove deprecated flag in healthcheck Signed-off-by: Evgenii Stratonikov --- services/ir/healthcheck.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/ir/healthcheck.sh b/services/ir/healthcheck.sh index 747bc8b..88fcdaa 100755 --- a/services/ir/healthcheck.sh +++ b/services/ir/healthcheck.sh @@ -1,6 +1,6 @@ #!/bin/sh -/frostfs-cli control healthcheck \ +/frostfs-cli control ir healthcheck \ --endpoint "$FROSTFS_IR_CONTROL_GRPC_ENDPOINT" \ - --wallet /wallet01.key --ir | + --wallet /wallet01.key | grep "Health status: READY" From 0be22a937536ae5ef425b338caa676ea0711bbde Mon Sep 17 00:00:00 2001 From: Alexey Savchuk Date: Mon, 8 Jul 2024 10:34:28 +0300 Subject: [PATCH 073/107] [#73] Update HTTP gate docs Signed-off-by: Aleksey Savchuk --- docs/http_gate.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/http_gate.md b/docs/http_gate.md index e2159ad..b6b2591 100644 --- a/docs/http_gate.md +++ b/docs/http_gate.md @@ -22,8 +22,8 @@ Image label prefix to use for containers. - Create a new container ``` $ frostfs-cli --rpc-endpoint s01.frostfs.devenv:8080 \ - --key wallets/wallet.key \ - container create --basic-acl readonly --await \ + --wallet wallets/wallet.key \ + container create --basic-acl private --await \ --policy "REP 1 SELECT 1 FROM *" container ID: 4LfREK1cetL4PUji5fqj9SgRTSmaC5jExEDK9HKCDjdP awaiting... @@ -33,7 +33,7 @@ container has been persisted on sidechain - Put an object into the newly created container ``` $ frostfs-cli --rpc-endpoint s01.frostfs.devenv:8080 \ - --key wallets/wallet.key \ + --wallet wallets/wallet.key \ object put --file /tmp/backup.jpeg \ --cid 4LfREK1cetL4PUji5fqj9SgRTSmaC5jExEDK9HKCDjdP [/tmp/backup.jpeg] Object successfully stored From d03be14312da6b1bd2f5bcbb113c4594eeaff258 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Thu, 8 Aug 2024 13:18:13 +0300 Subject: [PATCH 074/107] [#75] Makefile: Add subjects for storage and client wallets to FrostfsID Signed-off-by: Airat Arifullin --- Makefile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 051aa7d..9483dc7 100644 --- a/Makefile +++ b/Makefile @@ -85,7 +85,12 @@ up/bootstrap: get vendor/hosts && ./vendor/frostfs-adm -c frostfs-adm.yml morph refill-gas --storage-wallet $${f} --gas 10.0 \ || die "Failed to transfer GAS to alphabet wallets"; \ done - @echo "FrostFS sidechain environment is deployed" + @echo "Create frostfsid subject for ./wallets/wallet.json" \ + && subj_key=`docker container exec -it morph_chain neo-go wallet dump-keys -w /wallets/wallet.json | tail -1 | tr -d ' \r\n'` \ + && echo "Subject key: $${subj_key}" \ + && ./vendor/frostfs-adm -c frostfs-adm.yml morph frostfsid create-subject --namespace "" --subject-key $${subj_key} --subject-name walletsubject \ + || die "Failed to create subject for the wallet"; \ + echo "FrostFS sidechain environment is deployed" # Build up certain service .PHONY: up/% From f94fa284ece87feb5c6dd9bf18b664da916b2656 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Sat, 17 Aug 2024 05:50:27 +0300 Subject: [PATCH 075/107] [#76] Update frostfs-core components to v0.42.9 Signed-off-by: Alexander Chuprov --- .env | 12 ++++++------ services/ir/artifacts.mk | 7 ++----- services/morph_chain/artifacts.mk | 7 ++----- 3 files changed, 10 insertions(+), 16 deletions(-) diff --git a/.env b/.env index ad96218..b414b0d 100644 --- a/.env +++ b/.env @@ -12,12 +12,12 @@ NEOGO_VERSION=0.104.0 NEOGO_IMAGE=nspccdev/neo-go # FrostFS InnerRing nodes -IR_VERSION=0.38.5 -IR_IMAGE=truecloudlab/frostfs-ir +IR_VERSION=v0.42.9 +IR_IMAGE=git.frostfs.info/truecloudlab/frostfs-ir # FrostFS Storage nodes -NODE_VERSION=0.38.5 -NODE_IMAGE=truecloudlab/frostfs-storage +NODE_VERSION=v0.42.9 +NODE_IMAGE=git.frostfs.info/truecloudlab/frostfs-storage # NATS Server NATS_VERSION=2.7.2 @@ -40,12 +40,12 @@ LOCODE_DB_URL=https://git.frostfs.info/attachments/a2e8def7-52b6-49f1-89cd-a0567 #LOCODE_DB_PATH=/path/to/locode_db # FrostFS CLI binary -FROSTFS_CLI_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/BtA6zWLtoDuDnzxSNkKPjyN5hGuSZFxGuJD1gh8kBwkP +FROSTFS_CLI_URL=https://git.frostfs.info/TrueCloudLab/frostfs-node/releases/download/${NODE_VERSION}//frostfs-cli #FROSTFS_CLI_PATH=/path/to/frostfs-cli-binary # FrostFS ADM tool binary FROSTFS_ADM_VERSION=498f9955ea -FROSTFS_ADM_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/CjV4S6ENzo2FLK4KxXZHHNW4veR1ATtynGY6Mc1xQ6RB +FROSTFS_ADM_URL=https://git.frostfs.info/TrueCloudLab/frostfs-node/releases/download/${NODE_VERSION}/frostfs-adm #FROSTFS_ADM_PATH=/path/to/frostfs-adm-binary # Compiled FrostFS Smart Contracts diff --git a/services/ir/artifacts.mk b/services/ir/artifacts.mk index 95b2c22..0cdbdbb 100644 --- a/services/ir/artifacts.mk +++ b/services/ir/artifacts.mk @@ -25,7 +25,6 @@ endif # Download FrostFS CLI .ONESHELL: get.cli: FROSTFS_CLI_FILE=./vendor/frostfs-cli -get.cli: FROSTFS_CLI_ARCHIVE_FILE=${FROSTFS_CLI_FILE}.tar.gz get.cli: FROSTFS_CLI_PATH?= get.cli: @mkdir -p ./vendor @@ -34,10 +33,8 @@ ifeq (${FROSTFS_CLI_PATH},) @echo "⇒ Download FrostFS CLI binary from ${FROSTFS_CLI_URL}" @curl \ -ksSL "${FROSTFS_CLI_URL}" \ - -o ${FROSTFS_CLI_ARCHIVE_FILE} - @tar -xvf ${FROSTFS_CLI_ARCHIVE_FILE} -C ./vendor | xargs -I {} \ - mv ./vendor/{} ${FROSTFS_CLI_FILE} - @rm ${FROSTFS_CLI_ARCHIVE_FILE} + -o ${FROSTFS_CLI_FILE} + @chmod +x ${FROSTFS_CLI_FILE} else @echo "⇒ Copy local binary from ${FROSTFS_CLI_PATH}" @cp ${FROSTFS_CLI_PATH} ${FROSTFS_CLI_FILE} diff --git a/services/morph_chain/artifacts.mk b/services/morph_chain/artifacts.mk index 3572e69..77a7ae3 100644 --- a/services/morph_chain/artifacts.mk +++ b/services/morph_chain/artifacts.mk @@ -20,15 +20,12 @@ endif # Download FrostFS ADM tool get.adm: FROSTFS_ADM_DEST=./vendor/frostfs-adm -get.adm: FROSTFS_ADM_ARCHIVE=frostfs-adm.tar.gz get.adm: ifeq (${FROSTFS_ADM_PATH},) @echo "⇒ Download FrostFS ADM binary from ${FROSTFS_ADM_URL}" - @curl -skSL ${FROSTFS_ADM_URL} -o ${FROSTFS_ADM_ARCHIVE} - @tar -xvf ${FROSTFS_ADM_ARCHIVE} -C ./vendor | xargs -I {} \ - mv ./vendor/{} ${FROSTFS_ADM_DEST} - @rm ${FROSTFS_ADM_ARCHIVE} + @curl -skSL ${FROSTFS_ADM_URL} -o ${FROSTFS_ADM_DEST} + @chmod +x ${FROSTFS_ADM_DEST} else @echo "⇒ Copy frostfs-adm binary from ${FROSTFS_ADM_PATH}" @cp ${FROSTFS_ADM_PATH} ${FROSTFS_ADM_DEST} From 155042343b40c94507af61d0cbc7598dfed995a0 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Sat, 17 Aug 2024 06:25:11 +0300 Subject: [PATCH 076/107] [#69] service/storage: Add support -q flag in healthcheck Signed-off-by: Alexander Chuprov --- services/storage/docker-compose.yml | 12 ++++-------- services/storage/healthcheck.sh | 5 ----- 2 files changed, 4 insertions(+), 13 deletions(-) delete mode 100755 services/storage/healthcheck.sh diff --git a/services/storage/docker-compose.yml b/services/storage/docker-compose.yml index 8a8698e..fb10974 100644 --- a/services/storage/docker-compose.yml +++ b/services/storage/docker-compose.yml @@ -17,7 +17,6 @@ services: - storage_s01:/storage - ./../../vendor/frostfs-cli:/frostfs-cli - ./cli-cfg.yml:/cli-cfg.yml - - ./healthcheck.sh:/healthcheck.sh - ./s04tls.crt:/etc/ssl/certs/s04tls.crt - ../nats/client-cert.pem:/etc/frostfs-node/nats.tls.cert - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key @@ -36,7 +35,7 @@ services: - FROSTFS_NODE_ATTRIBUTE_1=UN-LOCODE:RU MOW - FROSTFS_NODE_ATTRIBUTE_2=Price:22 healthcheck: - test: ["CMD-SHELL", "/healthcheck.sh"] + test: ["CMD-SHELL", "/frostfs-cli control healthcheck -q -c /cli-cfg.yml --endpoint \"$$FROSTFS_CONTROL_GRPC_ENDPOINT\""] interval: 2s timeout: 1s retries: 5 @@ -58,7 +57,6 @@ services: - storage_s02:/storage - ./../../vendor/frostfs-cli:/frostfs-cli - ./cli-cfg.yml:/cli-cfg.yml - - ./healthcheck.sh:/healthcheck.sh - ./s04tls.crt:/etc/ssl/certs/s04tls.crt - ../nats/client-cert.pem:/etc/frostfs-node/nats.tls.cert - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key @@ -77,7 +75,7 @@ services: - FROSTFS_NODE_ATTRIBUTE_1=UN-LOCODE:RU LED - FROSTFS_NODE_ATTRIBUTE_2=Price:33 healthcheck: - test: ["CMD-SHELL", "/healthcheck.sh"] + test: ["CMD-SHELL", "/frostfs-cli control healthcheck -q -c /cli-cfg.yml --endpoint \"$$FROSTFS_CONTROL_GRPC_ENDPOINT\""] interval: 2s timeout: 1s retries: 5 @@ -99,7 +97,6 @@ services: - storage_s03:/storage - ./../../vendor/frostfs-cli:/frostfs-cli - ./cli-cfg.yml:/cli-cfg.yml - - ./healthcheck.sh:/healthcheck.sh - ./s04tls.crt:/etc/ssl/certs/s04tls.crt - ../nats/client-cert.pem:/etc/frostfs-node/nats.tls.cert - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key @@ -118,7 +115,7 @@ services: - FROSTFS_NODE_ATTRIBUTE_1=UN-LOCODE:SE STO - FROSTFS_NODE_ATTRIBUTE_2=Price:11 healthcheck: - test: ["CMD-SHELL", "/healthcheck.sh"] + test: ["CMD-SHELL", "/frostfs-cli control healthcheck -q -c /cli-cfg.yml --endpoint \"$$FROSTFS_CONTROL_GRPC_ENDPOINT\""] interval: 2s timeout: 1s retries: 5 @@ -140,7 +137,6 @@ services: - storage_s04:/storage - ./../../vendor/frostfs-cli:/frostfs-cli - ./cli-cfg.yml:/cli-cfg.yml - - ./healthcheck.sh:/healthcheck.sh - ./s04tls.crt:/tls.crt - ./s04tls.key:/tls.key - ../nats/client-cert.pem:/etc/frostfs-node/nats.tls.cert @@ -165,7 +161,7 @@ services: - FROSTFS_NODE_ATTRIBUTE_1=UN-LOCODE:FI HEL - FROSTFS_NODE_ATTRIBUTE_2=Price:44 healthcheck: - test: ["CMD-SHELL", "/healthcheck.sh"] + test: ["CMD-SHELL", "/frostfs-cli control healthcheck -q -c /cli-cfg.yml --endpoint \"$$FROSTFS_CONTROL_GRPC_ENDPOINT\""] interval: 2s timeout: 1s retries: 5 diff --git a/services/storage/healthcheck.sh b/services/storage/healthcheck.sh deleted file mode 100755 index 792d2ad..0000000 --- a/services/storage/healthcheck.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh - -/frostfs-cli control healthcheck -c /cli-cfg.yml \ - --endpoint "$FROSTFS_CONTROL_GRPC_ENDPOINT" | - grep "Health status: READY" From ae658469a5d44191e51df539054b3f0367bfdb73 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Sat, 17 Aug 2024 06:25:24 +0300 Subject: [PATCH 077/107] [#69] service/ir: Add support -q flag in healthcheck Signed-off-by: Alexander Chuprov --- services/ir/docker-compose.yml | 3 +-- services/ir/healthcheck.sh | 6 ------ 2 files changed, 1 insertion(+), 8 deletions(-) delete mode 100755 services/ir/healthcheck.sh diff --git a/services/ir/docker-compose.yml b/services/ir/docker-compose.yml index 048651b..4e671b2 100644 --- a/services/ir/docker-compose.yml +++ b/services/ir/docker-compose.yml @@ -20,12 +20,11 @@ services: - ./../../vendor/hosts:/etc/hosts - ./../../vendor/locode_db:/locode/db - ./../../vendor/frostfs-cli:/frostfs-cli - - ./healthcheck.sh:/healthcheck.sh - ./cfg:/etc/frostfs/ir env_file: [ ".env", ".ir.env", ".int_test.env" ] command: [ "frostfs-ir", "--config", "/etc/frostfs/ir/config.yml" ] healthcheck: - test: ["CMD-SHELL", "/healthcheck.sh"] + test: ["CMD-SHELL", "/frostfs-cli control ir healthcheck -q --wallet /wallet01.key --endpoint \"$$FROSTFS_IR_CONTROL_GRPC_ENDPOINT\""] interval: 2s timeout: 1s retries: 5 diff --git a/services/ir/healthcheck.sh b/services/ir/healthcheck.sh deleted file mode 100755 index 88fcdaa..0000000 --- a/services/ir/healthcheck.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh - -/frostfs-cli control ir healthcheck \ - --endpoint "$FROSTFS_IR_CONTROL_GRPC_ENDPOINT" \ - --wallet /wallet01.key | - grep "Health status: READY" From 044cf99e8dae647abb3706b498b27d00b58b6edc Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Sat, 17 Aug 2024 08:08:24 +0300 Subject: [PATCH 078/107] [#70] Makefile: Make bootstrap idempotent Signed-off-by: Alexander Chuprov --- Makefile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 9483dc7..2f505a9 100644 --- a/Makefile +++ b/Makefile @@ -85,11 +85,15 @@ up/bootstrap: get vendor/hosts && ./vendor/frostfs-adm -c frostfs-adm.yml morph refill-gas --storage-wallet $${f} --gas 10.0 \ || die "Failed to transfer GAS to alphabet wallets"; \ done - @echo "Create frostfsid subject for ./wallets/wallet.json" \ - && subj_key=`docker container exec -it morph_chain neo-go wallet dump-keys -w /wallets/wallet.json | tail -1 | tr -d ' \r\n'` \ + @echo "Create frostfsid subject for ./wallets/wallet.json"; \ + if [ -n "$$(./vendor/frostfs-adm -c frostfs-adm.yml morph frostfsid list-subjects --namespace '')" ]; then \ + echo "Subject already exists"; \ + else \ + subj_key=`docker container exec -it morph_chain neo-go wallet dump-keys -w /wallets/wallet.json | tail -1 | tr -d ' \r\n'` \ && echo "Subject key: $${subj_key}" \ && ./vendor/frostfs-adm -c frostfs-adm.yml morph frostfsid create-subject --namespace "" --subject-key $${subj_key} --subject-name walletsubject \ || die "Failed to create subject for the wallet"; \ + fi echo "FrostFS sidechain environment is deployed" # Build up certain service From dd382f8ce048d9e0880f43471d7896de000c0843 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Sat, 17 Aug 2024 08:08:55 +0300 Subject: [PATCH 079/107] [#68] service/morph: Add volume for morph_chain Signed-off-by: Alexander Chuprov --- services/morph_chain/docker-compose.yml | 4 ++++ services/morph_chain/protocol.privnet.yml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/services/morph_chain/docker-compose.yml b/services/morph_chain/docker-compose.yml index dd44b2c..68535ea 100644 --- a/services/morph_chain/docker-compose.yml +++ b/services/morph_chain/docker-compose.yml @@ -19,9 +19,13 @@ services: - ./config.yml:/wallets/config.yml - ./../../vendor/hosts:/etc/hosts - ./../../wallets/wallet.json:/wallets/wallet.json + - chains:/chains networks: chain_int: internet: external: true name: basenet_internet + +volumes: + chains: diff --git a/services/morph_chain/protocol.privnet.yml b/services/morph_chain/protocol.privnet.yml index 560730a..2066029 100644 --- a/services/morph_chain/protocol.privnet.yml +++ b/services/morph_chain/protocol.privnet.yml @@ -17,7 +17,7 @@ ApplicationConfiguration: DBConfiguration: Type: "boltdb" BoltDBOptions: - FilePath: "./db/morph.bolt" + FilePath: "/chains/morph.bolt" P2P: Addresses: - ":20333" From 439a9e71cfaf3837f16d9c501385c8ed2ff67dbc Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Fri, 6 Sep 2024 14:09:08 +0300 Subject: [PATCH 080/107] [#81] env: Fix typo Signed-off-by: Alexander Chuprov --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index b414b0d..c62cf17 100644 --- a/.env +++ b/.env @@ -40,7 +40,7 @@ LOCODE_DB_URL=https://git.frostfs.info/attachments/a2e8def7-52b6-49f1-89cd-a0567 #LOCODE_DB_PATH=/path/to/locode_db # FrostFS CLI binary -FROSTFS_CLI_URL=https://git.frostfs.info/TrueCloudLab/frostfs-node/releases/download/${NODE_VERSION}//frostfs-cli +FROSTFS_CLI_URL=https://git.frostfs.info/TrueCloudLab/frostfs-node/releases/download/${NODE_VERSION}/frostfs-cli #FROSTFS_CLI_PATH=/path/to/frostfs-cli-binary # FrostFS ADM tool binary From 2e67acbcb2af952933b87856083a6a926b840723 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Fri, 6 Sep 2024 14:37:48 +0300 Subject: [PATCH 081/107] [#78] env: Restore version format Signed-off-by: Alexander Chuprov --- .env | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.env b/.env index c62cf17..dea5726 100644 --- a/.env +++ b/.env @@ -12,11 +12,11 @@ NEOGO_VERSION=0.104.0 NEOGO_IMAGE=nspccdev/neo-go # FrostFS InnerRing nodes -IR_VERSION=v0.42.9 +IR_VERSION=0.42.9 IR_IMAGE=git.frostfs.info/truecloudlab/frostfs-ir # FrostFS Storage nodes -NODE_VERSION=v0.42.9 +NODE_VERSION=0.42.9 NODE_IMAGE=git.frostfs.info/truecloudlab/frostfs-storage # NATS Server @@ -40,12 +40,12 @@ LOCODE_DB_URL=https://git.frostfs.info/attachments/a2e8def7-52b6-49f1-89cd-a0567 #LOCODE_DB_PATH=/path/to/locode_db # FrostFS CLI binary -FROSTFS_CLI_URL=https://git.frostfs.info/TrueCloudLab/frostfs-node/releases/download/${NODE_VERSION}/frostfs-cli +FROSTFS_CLI_URL=https://git.frostfs.info/TrueCloudLab/frostfs-node/releases/download/v${NODE_VERSION}/frostfs-cli #FROSTFS_CLI_PATH=/path/to/frostfs-cli-binary # FrostFS ADM tool binary FROSTFS_ADM_VERSION=498f9955ea -FROSTFS_ADM_URL=https://git.frostfs.info/TrueCloudLab/frostfs-node/releases/download/${NODE_VERSION}/frostfs-adm +FROSTFS_ADM_URL=https://git.frostfs.info/TrueCloudLab/frostfs-node/releases/download/v${NODE_VERSION}/frostfs-adm #FROSTFS_ADM_PATH=/path/to/frostfs-adm-binary # Compiled FrostFS Smart Contracts From 7538bd9b171409fc5eeb5c08d074efe5792db133 Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Tue, 10 Sep 2024 16:45:06 +0300 Subject: [PATCH 082/107] [#83] Honor IPV4_PREFIX in morph_chain config Signed-off-by: Vitaliy Potyarkin --- services/morph_chain/protocol.privnet.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/morph_chain/protocol.privnet.yml b/services/morph_chain/protocol.privnet.yml index 2066029..f1cc754 100644 --- a/services/morph_chain/protocol.privnet.yml +++ b/services/morph_chain/protocol.privnet.yml @@ -36,7 +36,7 @@ ApplicationConfiguration: Password: "one" RPC: Addresses: - - "192.168.130.90:30333" + - ":30333" Enabled: true SessionEnabled: true EnableCORSWorkaround: false From d0c32731f2c468dafd979337ad04af1075d2bca2 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 9 Sep 2024 18:59:33 +0300 Subject: [PATCH 083/107] [#80] Update frostfs-service components to v0.30.* Signed-off-by: Alexander Chuprov --- .env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.env b/.env index dea5726..17697f6 100644 --- a/.env +++ b/.env @@ -24,7 +24,7 @@ NATS_VERSION=2.7.2 NATS_IMAGE=nats # HTTP Gate -HTTP_GW_VERSION=0.28.0-rc.1-15-g5ae75eb9 +HTTP_GW_VERSION=0.30.2 HTTP_GW_IMAGE=truecloudlab/frostfs-http-gw # REST Gate @@ -32,7 +32,7 @@ REST_GW_VERSION=c9c85e90 REST_GW_IMAGE=truecloudlab/frostfs-rest-gw # S3 Gate -S3_GW_VERSION=0.28.0-rc.1-51-g9272f4e1 +S3_GW_VERSION=0.30.4 S3_GW_IMAGE=truecloudlab/frostfs-s3-gw # FrostFS LOCODE database From 636be7352ebf5221ed66b02a8df85cead7c3633e Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Mon, 30 Sep 2024 17:04:11 +0300 Subject: [PATCH 084/107] [#84] Make targets for issuing credentials Signed-off-by: Nikita Zinkevich --- Makefile | 1 + README.md | 59 +++++++++++++++++++++++++ services/morph_chain/docker-compose.yml | 1 + services/s3_gate/docker-compose.yml | 8 ++++ services/s3_gate/issue-creds.sh | 41 +++++++++++++++++ services/s3_gate/prepare.mk | 14 ++++++ 6 files changed, 124 insertions(+) create mode 100755 services/s3_gate/issue-creds.sh create mode 100644 services/s3_gate/prepare.mk diff --git a/Makefile b/Makefile index 2f505a9..dff81df 100644 --- a/Makefile +++ b/Makefile @@ -60,6 +60,7 @@ get: $(foreach SVC, $(GET_SVCS), get.$(SVC)) .PHONY: up up: up/basic @$(foreach SVC, $(START_SVCS), $(shell docker-compose -f services/$(SVC)/docker-compose.yml up -d)) + ./vendor/frostfs-adm morph proxy-add-account --config frostfs-adm.yml --account=`docker container exec morph_chain neo-go wallet dump-keys -w /wallets/s3-wallet.json | head -1 | awk '{print $1}'` || die "Couldn't set s3-gw wallet as proxy wallet" @echo "Full FrostFS Developer Environment is ready" # Build up FrostFS diff --git a/README.md b/README.md index aa07a08..ad57655 100644 --- a/README.md +++ b/README.md @@ -137,6 +137,65 @@ Display addresses and host names for each running service, if available. Clean up `vendor` directory. +### s3cred + +Registers user wallet and issues s3 credentials. + +Usage and default parameter values: +```sh +make s3cred [password=""] [contract_password=s3] [wallet=/user_wallet.json] [gate_public_key=0313b1ac3a8076e155a7e797b24f0b650cccad5941ea59d7cfd51a024a8b2a06bf] +``` + +As soon as the storage node is in the network map (see above) you can generate S3 +credentials: + +``` sh +$ make s3cred +{ + "access_key_id": "EXArWh8x1zeHG3851s1RtoCo7dowxF6rhLGA15nbMffT0AKRSjJ5fmcqf3Ht2VCAkfmPQUVARghRB77xHCA1BoN2p", + "secret_access_key": "d70c1dba83f0f90bb231f06f1ce0e0dfbcfb122f4b4345a3c18d3869c359b79f", + "owner_private_key": "140947599afd9ca89af4b358c3176eb046e554d942a0dc99a8e06f3e43c8f4ad", + "wallet_public_key": "0324e76288fcb900100d01802a14ef977cca45ad073561230446df14b344c858b6", + "container_id": "EXArWh8x1zeHG3851s1RtoCo7dowxF6rhLGA15nbMffT" +} +``` +Running without any parameters will result in defaults which are based on the private key from +`/user-wallet.json` file and `/wallet.json` contract wallet. + +Now let's configure an S3 client (AWS CLI will be used as example): + +``` sh +$ aws configure +AWS Access Key ID []: EXArWh8x1zeHG3851s1RtoCo7dowxF6rhLGA15nbMffT0AKRSjJ5fmcqf3Ht2VCAkfmPQUVARghRB77xHCA1BoN2p +AWS Secret Access Key []: d70c1dba83f0f90bb231f06f1ce0e0dfbcfb122f4b4345a3c18d3869c359b79f +Default region name []: us-east-1 +Default output format []: json +``` + +If you need to create credentials for different users, put user wallets to `wallets` dir and specify them via `wallet` parameter. +Pass wallet password in `password` parameter if it's not default. The same is for `contract_wallet` and `gate_public_key` params. + +```sh +$ make s3cred wallet=custom_wallet.json password=test +{ + "access_key_id": "jHhL5B33o16R4jQsb8wm9A3RRdS6KrTB5N4bja9Jys904W7xXFNKqem2ACvTRWRYJsZMCUikYFSokN7pPJziWyDi", + "secret_access_key": "21bb64fafa32c82417fd8b97ac56cc8a085998a3852632d52fe7042453daa440", + "owner_private_key": "10f6f9d7a47bb0bf68363ad8a99fe69f1493f8b6e1665b3e4e83feb2d5c7ee39", + "wallet_public_key": "03e38759973a6bb722baabc2dd84036a39f0b2f53d32fec45a4dacde8a50fe4b70", + "container_id": "jHhL5B33o16R4jQsb8wm9A3RRdS6KrTB5N4bja9Jys9" +} +``` + +To get credentials from custom wallet, place it in `wallets` dir before start. + +### cred + +Usage and default parameter values: +```sh +make cred [password=""] [contract_password=s3] [wallet=/user_wallet.json] +``` +The same as `s3cred`, but it doesn't issues s3 credentials. + ## Contributing Feel free to contribute to this project after reading the [contributing diff --git a/services/morph_chain/docker-compose.yml b/services/morph_chain/docker-compose.yml index 68535ea..2c2facb 100644 --- a/services/morph_chain/docker-compose.yml +++ b/services/morph_chain/docker-compose.yml @@ -19,6 +19,7 @@ services: - ./config.yml:/wallets/config.yml - ./../../vendor/hosts:/etc/hosts - ./../../wallets/wallet.json:/wallets/wallet.json + - ./../s3_gate/wallet.json:/wallets/s3-wallet.json - chains:/chains networks: diff --git a/services/s3_gate/docker-compose.yml b/services/s3_gate/docker-compose.yml index 6b43a0b..f7fc28d 100644 --- a/services/s3_gate/docker-compose.yml +++ b/services/s3_gate/docker-compose.yml @@ -12,11 +12,17 @@ services: internet: ipv4_address: ${IPV4_PREFIX}.82 volumes: + # Gate wallet - ./wallet.json:/wallet.json + # Custom user wallets + - ./wallets:/wallets + # Default user wallet + - ./../../wallets/wallet.json:/wallets/wallet.json - ./tls.key:/tls.key - ./tls.crt:/tls.crt - ./../../vendor/hosts:/etc/hosts - ./cfg:/etc/frostfs/s3 + - ./issue-creds.sh:/usr/bin/issue-creds.sh stop_signal: SIGTERM stop_grace_period: 15s env_file: [ ".env", ".s3.env", ".int_test.env" ] @@ -34,6 +40,8 @@ services: - S3_GW_PEERS_2_WEIGHT=0.2 - S3_GW_PEERS_3_ADDRESS=s04.${LOCAL_DOMAIN}:8080 - S3_GW_PEERS_3_WEIGHT=0.2 + - AUTHMATE_WALLET_PASSPHRASE= + - AUTHMATE_WALLET_CONTRACT_PASSPHRASE=s3 networks: s3_gate_int: diff --git a/services/s3_gate/issue-creds.sh b/services/s3_gate/issue-creds.sh new file mode 100755 index 0000000..e2355f8 --- /dev/null +++ b/services/s3_gate/issue-creds.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +initUser() { + /bin/frostfs-s3-authmate register-user \ + --wallet $WALLET_PATH \ + --rpc-endpoint http://morph-chain.frostfs.devenv:30333 \ + --username $USERNAME \ + --contract-wallet /wallet.json 1> /dev/null && touch $WALLET_CACHE/$USERNAME +} + +issueCreds() { + /bin/frostfs-s3-authmate issue-secret \ + --wallet $WALLET_PATH \ + --peer s01.frostfs.devenv:8080 \ + --gate-public-key $S3_GATE_PUBLIC_KEY \ + --container-placement-policy "REP 3" +} + +set -e + +WALLET_PATH=/wallets/$2 +if [[ -z "$2" ]]; then + WALLET_PATH=/wallets/wallet.json +fi + +S3_GATE_PUBLIC_KEY=$3 +if [[ -z "$3" ]]; then + S3_GATE_PUBLIC_KEY=0313b1ac3a8076e155a7e797b24f0b650cccad5941ea59d7cfd51a024a8b2a06bf +fi + +WALLET_CACHE=/data/wallets +mkdir -p $WALLET_CACHE + +USERNAME=$(echo $WALLET_PATH | md5sum | cut -d' ' -f1) +if [ ! -e $WALLET_CACHE/$USERNAME ]; then + initUser +fi + +if [ $1 == "s3" ]; then + issueCreds +fi diff --git a/services/s3_gate/prepare.mk b/services/s3_gate/prepare.mk new file mode 100644 index 0000000..c97add8 --- /dev/null +++ b/services/s3_gate/prepare.mk @@ -0,0 +1,14 @@ +.PHONY: s3cred register + +password?= +contract_password?=s3 +gate_public_key?= +wallet?= + +# Register wallet & generate S3 credentials +s3cred: + @docker exec -e AUTHMATE_WALLET_PASSPHRASE="$(password)" -e AUTHMATE_WALLET_CONTRACT_PASSPHRASE="$(contract_password)" s3_gate /usr/bin/issue-creds.sh s3 "$(wallet)" "$(gate_public_key)" + +# Only registers user wallet +register: + @docker exec -e AUTHMATE_WALLET_PASSPHRASE="$(password)" -e AUTHMATE_WALLET_CONTRACT_PASSPHRASE="$(contract_password)" s3_gate /usr/bin/issue-creds.sh native "$(wallet)" From 7152f59232ac5ac763cbb6433331d161f64c56d0 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 31 Oct 2024 10:24:30 +0300 Subject: [PATCH 085/107] [#88] Remove nats service It was removed from node in TrueCloudLab/frostfs-node#1161. Signed-off-by: Evgenii Stratonikov --- .basic_services | 1 - .env | 4 --- .gitignore | 1 - Makefile | 2 +- services/nats/.env | 1 - services/nats/.hosts | 1 - services/nats/.int_test.env | 1 - services/nats/artifacts.mk | 7 ----- services/nats/docker-compose.yml | 30 ------------------ services/nats/generate_cert.sh | 49 ----------------------------- services/nats/nats.conf | 15 --------- services/storage/cfg/config.yml | 8 ----- services/storage/docker-compose.yml | 12 ------- 13 files changed, 1 insertion(+), 131 deletions(-) delete mode 120000 services/nats/.env delete mode 100644 services/nats/.hosts delete mode 120000 services/nats/.int_test.env delete mode 100644 services/nats/artifacts.mk delete mode 100644 services/nats/docker-compose.yml delete mode 100755 services/nats/generate_cert.sh delete mode 100644 services/nats/nats.conf diff --git a/.basic_services b/.basic_services index 201ea70..845c731 100644 --- a/.basic_services +++ b/.basic_services @@ -1,5 +1,4 @@ # Services start/stop order # Will start from top to bottom and stop in reverse -nats ir storage diff --git a/.env b/.env index 17697f6..6bb3728 100644 --- a/.env +++ b/.env @@ -19,10 +19,6 @@ IR_IMAGE=git.frostfs.info/truecloudlab/frostfs-ir NODE_VERSION=0.42.9 NODE_IMAGE=git.frostfs.info/truecloudlab/frostfs-storage -# NATS Server -NATS_VERSION=2.7.2 -NATS_IMAGE=nats - # HTTP Gate HTTP_GW_VERSION=0.30.2 HTTP_GW_IMAGE=truecloudlab/frostfs-http-gw diff --git a/.gitignore b/.gitignore index 36e495d..a672692 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,3 @@ sites/* # Runtime generation keys services/storage/*tls.crt services/storage/*tls.key -services/nats/*.pem diff --git a/Makefile b/Makefile index dff81df..c4a6c48 100644 --- a/Makefile +++ b/Makefile @@ -150,7 +150,7 @@ hosts: vendor/hosts .PHONY: clean .ONESHELL: clean: - @rm -rf vendor/* services/storage/s04tls.* services/nats/*.pem + @rm -rf vendor/* services/storage/s04tls.* @> .int_test.env @for svc in $(PULL_SVCS) do diff --git a/services/nats/.env b/services/nats/.env deleted file mode 120000 index c7360fb..0000000 --- a/services/nats/.env +++ /dev/null @@ -1 +0,0 @@ -../../.env \ No newline at end of file diff --git a/services/nats/.hosts b/services/nats/.hosts deleted file mode 100644 index 6ed5417..0000000 --- a/services/nats/.hosts +++ /dev/null @@ -1 +0,0 @@ -IPV4_PREFIX.101 nats.LOCAL_DOMAIN diff --git a/services/nats/.int_test.env b/services/nats/.int_test.env deleted file mode 120000 index 582b6a2..0000000 --- a/services/nats/.int_test.env +++ /dev/null @@ -1 +0,0 @@ -../../.int_test.env \ No newline at end of file diff --git a/services/nats/artifacts.mk b/services/nats/artifacts.mk deleted file mode 100644 index b66cebf..0000000 --- a/services/nats/artifacts.mk +++ /dev/null @@ -1,7 +0,0 @@ -# Create new TLS certs for NATS server and clients - -NATS_DIR=$(abspath services/nats) - -get.nats: - @echo "⇒ Creating certs for NATS server and clients" - ${NATS_DIR}/generate_cert.sh ${LOCAL_DOMAIN} > /dev/null diff --git a/services/nats/docker-compose.yml b/services/nats/docker-compose.yml deleted file mode 100644 index cb89948..0000000 --- a/services/nats/docker-compose.yml +++ /dev/null @@ -1,30 +0,0 @@ ---- - -services: - nats: - image: ${NATS_IMAGE}:${NATS_VERSION} - domainname: ${LOCAL_DOMAIN} - hostname: nats - container_name: nats - restart: on-failure - dns: - - ${IPV4_PREFIX}.101 - networks: - nats_int: - internet: - ipv4_address: ${IPV4_PREFIX}.101 - volumes: - - ./../../vendor/hosts:/etc/hosts - - ./nats.conf:/etc/nats/frostfs-nats-server.conf - - ./server-cert.pem:/certs/server-cert.pem - - ./server-key.pem:/certs/server-key.pem - - ./ca-cert.pem:/certs/ca-cert.pem - stop_signal: SIGKILL - env_file: [ ".env", ".int_test.env" ] - command: ["-c", "/etc/nats/frostfs-nats-server.conf"] - -networks: - nats_int: - internet: - external: true - name: basenet_internet diff --git a/services/nats/generate_cert.sh b/services/nats/generate_cert.sh deleted file mode 100755 index 4c4dd0b..0000000 --- a/services/nats/generate_cert.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash - -source bin/helper.sh - -WORKDIR=$(dirname "$0") -LOCAL_DOMAIN=$1 - -CA_KEY=$WORKDIR/ca-key.pem -CA_CRT=$WORKDIR/ca-cert.pem - -SRV_KEY=$WORKDIR/server-key.pem -SRV_REQ=$WORKDIR/server-req.csr -SRV_CRT=$WORKDIR/server-cert.pem - -CLI_KEY=$WORKDIR/client-key.pem -CLI_REQ=$WORKDIR/client-req.csr -CLI_CRT=$WORKDIR/client-cert.pem - -SUBJ="/O=TrueCloudLab" - -if [[ ! -f $CA_KEY || ! -f $CA_CRT ]]; then - openssl req -newkey rsa:4096 -x509 -days 365 -nodes -keyout $CA_KEY -out $CA_CRT -subj $SUBJ 2>&1 || - die "CA certificate was not created" -fi - -if [[ ! -f $SRV_KEY || ! -f $SRV_CRT ]]; then - openssl req -newkey rsa:4096 -nodes -keyout $SRV_KEY -out $SRV_REQ -subj $SUBJ 2>&1 || - die "Server certificate was not created" - - openssl x509 -req -days 365 -set_serial 01 -in $SRV_REQ -out $SRV_CRT -CA $CA_CRT -CAkey $CA_KEY \ - -extensions san -extfile <(printf "[san]\nsubjectAltName=DNS:nats.$LOCAL_DOMAIN") 2>&1 || { - rm $SRV_REQ - die "Server certificate was not signed by CA" - } - - rm $SRV_REQ -fi - -if [[ ! -f $CLI_KEY || ! -f $CLI_CRT ]]; then - openssl req -newkey rsa:4096 -nodes -keyout $CLI_KEY -out $CLI_REQ -subj $SUBJ 2>&1 || - die "Client certificate was not created" - - openssl x509 -req -days 365 -set_serial 01 -in $CLI_REQ -out $CLI_CRT -CA $CA_CRT -CAkey $CA_KEY 2>&1 || { - rm $CLI_REQ - die "Client certificate was not signed by CA" - } - - rm $CLI_REQ -fi diff --git a/services/nats/nats.conf b/services/nats/nats.conf deleted file mode 100644 index 0c7af0a..0000000 --- a/services/nats/nats.conf +++ /dev/null @@ -1,15 +0,0 @@ -port: 4222 -monitor_port: 8222 - -jetstream { - store_dir=nats - max_memory_store: 1GB - max_file_store: 2GB -} - -tls { - cert_file: /certs/server-cert.pem - key_file: /certs/server-key.pem - ca_file: /certs/ca-cert.pem - verify: true -} diff --git a/services/storage/cfg/config.yml b/services/storage/cfg/config.yml index 6b3e7a8..8672069 100644 --- a/services/storage/cfg/config.yml +++ b/services/storage/cfg/config.yml @@ -35,14 +35,6 @@ morph: # Common storage node settings node: attribute_0: "User-Agent:FrostFS/0.34" - notification: - enabled: true # Turn on object notification service - endpoint: "tls://nats.frostfs.devenv:4222" # Notification server endpoint - timeout: "6s" # Timeout for object notification client connection - default_topic: "test" # Default topic for object notifications if not found in object's meta - certificate: "/etc/frostfs-node/nats.tls.cert" # Path to TLS certificate - key: "/etc/frostfs-node/nats.tls.key" # Path to TLS key - ca: "/etc/frostfs-node/nats.ca.crt" # Path to optional CA certificate # Tree section tree: diff --git a/services/storage/docker-compose.yml b/services/storage/docker-compose.yml index fb10974..6a070be 100644 --- a/services/storage/docker-compose.yml +++ b/services/storage/docker-compose.yml @@ -18,9 +18,6 @@ services: - ./../../vendor/frostfs-cli:/frostfs-cli - ./cli-cfg.yml:/cli-cfg.yml - ./s04tls.crt:/etc/ssl/certs/s04tls.crt - - ../nats/client-cert.pem:/etc/frostfs-node/nats.tls.cert - - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key - - ../nats/ca-cert.pem:/etc/frostfs-node/nats.ca.crt - ./cfg:/etc/frostfs/storage stop_signal: SIGTERM stop_grace_period: 15s @@ -58,9 +55,6 @@ services: - ./../../vendor/frostfs-cli:/frostfs-cli - ./cli-cfg.yml:/cli-cfg.yml - ./s04tls.crt:/etc/ssl/certs/s04tls.crt - - ../nats/client-cert.pem:/etc/frostfs-node/nats.tls.cert - - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key - - ../nats/ca-cert.pem:/etc/frostfs-node/nats.ca.crt - ./cfg:/etc/frostfs/storage stop_signal: SIGTERM stop_grace_period: 15s @@ -98,9 +92,6 @@ services: - ./../../vendor/frostfs-cli:/frostfs-cli - ./cli-cfg.yml:/cli-cfg.yml - ./s04tls.crt:/etc/ssl/certs/s04tls.crt - - ../nats/client-cert.pem:/etc/frostfs-node/nats.tls.cert - - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key - - ../nats/ca-cert.pem:/etc/frostfs-node/nats.ca.crt - ./cfg:/etc/frostfs/storage stop_signal: SIGTERM stop_grace_period: 15s @@ -139,9 +130,6 @@ services: - ./cli-cfg.yml:/cli-cfg.yml - ./s04tls.crt:/tls.crt - ./s04tls.key:/tls.key - - ../nats/client-cert.pem:/etc/frostfs-node/nats.tls.cert - - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key - - ../nats/ca-cert.pem:/etc/frostfs-node/nats.ca.crt - ./cfg:/etc/frostfs/storage stop_signal: SIGTERM stop_grace_period: 15s From 10e5bed2afec13cd9803f012360c1bd7dcd184c7 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 31 Oct 2024 10:38:09 +0300 Subject: [PATCH 086/107] [#79] storage: Take User-Agent from NODE_VERSION Signed-off-by: Evgenii Stratonikov --- services/storage/cfg/config.yml | 4 ---- services/storage/docker-compose.yml | 4 ++++ 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/services/storage/cfg/config.yml b/services/storage/cfg/config.yml index 8672069..0e2526d 100644 --- a/services/storage/cfg/config.yml +++ b/services/storage/cfg/config.yml @@ -32,10 +32,6 @@ morph: - address: ws://morph-chain:30333/ws priority: 1 -# Common storage node settings -node: - attribute_0: "User-Agent:FrostFS/0.34" - # Tree section tree: enabled: true diff --git a/services/storage/docker-compose.yml b/services/storage/docker-compose.yml index 6a070be..785ce79 100644 --- a/services/storage/docker-compose.yml +++ b/services/storage/docker-compose.yml @@ -29,6 +29,7 @@ services: - FROSTFS_NODE_ADDRESSES=s01.${LOCAL_DOMAIN}:8080 - FROSTFS_GRPC_0_ENDPOINT=s01.${LOCAL_DOMAIN}:8080 - FROSTFS_CONTROL_GRPC_ENDPOINT=s01.${LOCAL_DOMAIN}:8081 + - FROSTFS_NODE_ATTRIBUTE_0=User-Agent:FrostFS/${NODE_VERSION} - FROSTFS_NODE_ATTRIBUTE_1=UN-LOCODE:RU MOW - FROSTFS_NODE_ATTRIBUTE_2=Price:22 healthcheck: @@ -66,6 +67,7 @@ services: - FROSTFS_NODE_ADDRESSES=s02.${LOCAL_DOMAIN}:8080 - FROSTFS_GRPC_0_ENDPOINT=s02.${LOCAL_DOMAIN}:8080 - FROSTFS_CONTROL_GRPC_ENDPOINT=s02.${LOCAL_DOMAIN}:8081 + - FROSTFS_NODE_ATTRIBUTE_0=User-Agent:FrostFS/${NODE_VERSION} - FROSTFS_NODE_ATTRIBUTE_1=UN-LOCODE:RU LED - FROSTFS_NODE_ATTRIBUTE_2=Price:33 healthcheck: @@ -103,6 +105,7 @@ services: - FROSTFS_NODE_ADDRESSES=s03.${LOCAL_DOMAIN}:8080 - FROSTFS_GRPC_0_ENDPOINT=s03.${LOCAL_DOMAIN}:8080 - FROSTFS_CONTROL_GRPC_ENDPOINT=s03.${LOCAL_DOMAIN}:8081 + - FROSTFS_NODE_ATTRIBUTE_0=User-Agent:FrostFS/${NODE_VERSION} - FROSTFS_NODE_ATTRIBUTE_1=UN-LOCODE:SE STO - FROSTFS_NODE_ATTRIBUTE_2=Price:11 healthcheck: @@ -146,6 +149,7 @@ services: - FROSTFS_GRPC_1_TLS_ENABLED=true - FROSTFS_GRPC_1_TLS_CERTIFICATE=/tls.crt - FROSTFS_GRPC_1_TLS_KEY=/tls.key + - FROSTFS_NODE_ATTRIBUTE_0=User-Agent:FrostFS/${NODE_VERSION} - FROSTFS_NODE_ATTRIBUTE_1=UN-LOCODE:FI HEL - FROSTFS_NODE_ATTRIBUTE_2=Price:44 healthcheck: From a0fdaebbf41d847badeac4e39961ef4cf95df3d0 Mon Sep 17 00:00:00 2001 From: Pavel Pogodaev Date: Fri, 25 Oct 2024 16:24:26 +0300 Subject: [PATCH 087/107] [#85] Add s3 lifecycler Signed-off-by: Pavel Pogodaev --- .env | 6 +++- .services | 1 + services/s3_lifecycler/.env | 1 + services/s3_lifecycler/.hosts | 1 + services/s3_lifecycler/.int_test.env | 1 + services/s3_lifecycler/cfg/config.yml | 42 +++++++++++++++++++++++ services/s3_lifecycler/docker-compose.yml | 38 ++++++++++++++++++++ services/s3_lifecycler/wallet.json | 30 ++++++++++++++++ 8 files changed, 119 insertions(+), 1 deletion(-) create mode 120000 services/s3_lifecycler/.env create mode 100644 services/s3_lifecycler/.hosts create mode 120000 services/s3_lifecycler/.int_test.env create mode 100644 services/s3_lifecycler/cfg/config.yml create mode 100644 services/s3_lifecycler/docker-compose.yml create mode 100644 services/s3_lifecycler/wallet.json diff --git a/.env b/.env index 6bb3728..a1a1586 100644 --- a/.env +++ b/.env @@ -28,9 +28,13 @@ REST_GW_VERSION=c9c85e90 REST_GW_IMAGE=truecloudlab/frostfs-rest-gw # S3 Gate -S3_GW_VERSION=0.30.4 +S3_GW_VERSION=0.31.0-rc.4 S3_GW_IMAGE=truecloudlab/frostfs-s3-gw +# Lifecycler +S3_LIFECYCLER_VERSION=0.1.3 +S3_LIFECYCLER_IMAGE=truecloudlab/frostfs-s3-lifecycler + # FrostFS LOCODE database LOCODE_DB_URL=https://git.frostfs.info/attachments/a2e8def7-52b6-49f1-89cd-a056712e8e54 #LOCODE_DB_PATH=/path/to/locode_db diff --git a/.services b/.services index d9f34c7..d165241 100644 --- a/.services +++ b/.services @@ -3,3 +3,4 @@ http_gate s3_gate rest_gate +s3_lifecycler diff --git a/services/s3_lifecycler/.env b/services/s3_lifecycler/.env new file mode 120000 index 0000000..c7360fb --- /dev/null +++ b/services/s3_lifecycler/.env @@ -0,0 +1 @@ +../../.env \ No newline at end of file diff --git a/services/s3_lifecycler/.hosts b/services/s3_lifecycler/.hosts new file mode 100644 index 0000000..8eb1796 --- /dev/null +++ b/services/s3_lifecycler/.hosts @@ -0,0 +1 @@ +IPV4_PREFIX.84 lifecycler.LOCAL_DOMAIN diff --git a/services/s3_lifecycler/.int_test.env b/services/s3_lifecycler/.int_test.env new file mode 120000 index 0000000..582b6a2 --- /dev/null +++ b/services/s3_lifecycler/.int_test.env @@ -0,0 +1 @@ +../../.int_test.env \ No newline at end of file diff --git a/services/s3_lifecycler/cfg/config.yml b/services/s3_lifecycler/cfg/config.yml new file mode 100644 index 0000000..2555980 --- /dev/null +++ b/services/s3_lifecycler/cfg/config.yml @@ -0,0 +1,42 @@ +logger: + level: debug + +prometheus: + enabled: true + address: :9090 + +lifecycle: + job_fetcher_buffer: 1000 + executor_pool_size: 100 + +frostfs: + stream_timeout: 10s + connect_timeout: 10s + healthcheck_timeout: 15s + rebalance_interval: 60s + pool_error_threshold: 100 + tree_pool_max_attempts: 4 + +credential: + use: wallets + source: + wallets: + - path: /wallet.json + address: NTt1rxvmEDxEuuogLxs2xgxA71qhVaUcN7 + passphrase: "cycle" + - path: /user-wallet.json + address: NbUgTSFvPmsRxmGeWpuuGeJUoRoi6PErcM + passphrase: "" + +morph: + reconnect_clients_interval: 30s + dial_timeout: 5s + contract: + netmap: netmap.frostfs + frostfsid: frostfsid.frostfs + container: container.frostfs + +# Wallet configuration +wallet: + path: /wallet.json # Path to wallet + passphrase: "cycle" # Passphrase to decrypt wallet diff --git a/services/s3_lifecycler/docker-compose.yml b/services/s3_lifecycler/docker-compose.yml new file mode 100644 index 0000000..3456d0c --- /dev/null +++ b/services/s3_lifecycler/docker-compose.yml @@ -0,0 +1,38 @@ +--- + +version: "2.4" +services: + s3_lifecycler: + image: ${S3_LIFECYCLER_IMAGE}:${S3_LIFECYCLER_VERSION} + domainname: ${LOCAL_DOMAIN} + hostname: s3_lifecycler + container_name: s3_lifecycler + restart: on-failure + networks: + s3_lifecycler_int: + internet: + ipv4_address: ${IPV4_PREFIX}.84 + volumes: + - ./wallet.json:/wallet.json + - ./../../vendor/hosts:/etc/hosts + - ./cfg:/etc/frostfs/s3-lifecycler + - ./../../wallets/wallet.json:/user-wallet.json + stop_signal: SIGKILL + env_file: [ ".env", ".int_test.env" ] + command: [ "frostfs-s3-lifecycler", "--config", "/etc/frostfs/s3-lifecycler/config.yml" ] + environment: + - S3_LIFECYCLER_MORPH_RPC_ENDPOINT_0_ADDRESS=ws://morph-chain:30333/ws + - S3_LIFECYCLER_FROSTFS_PEERS_0_ADDRESS=s01.${LOCAL_DOMAIN}:8080 + - S3_LIFECYCLER_FROSTFS_PEERS_0_WEIGHT=0.2 + - S3_LIFECYCLER_FROSTFS_PEERS_1_ADDRESS=s02.${LOCAL_DOMAIN}:8080 + - S3_LIFECYCLER_FROSTFS_PEERS_1_WEIGHT=0.2 + - S3_LIFECYCLER_FROSTFS_PEERS_2_ADDRESS=s03.${LOCAL_DOMAIN}:8080 + - S3_LIFECYCLER_FROSTFS_PEERS_2_WEIGHT=0.2 + - S3_LIFECYCLER_FROSTFS_PEERS_3_ADDRESS=s04.${LOCAL_DOMAIN}:8080 + - S3_LIFECYCLER_FROSTFS_PEERS_3_WEIGHT=0.2 + +networks: + s3_lifecycler_int: + internet: + external: true + name: basenet_internet diff --git a/services/s3_lifecycler/wallet.json b/services/s3_lifecycler/wallet.json new file mode 100644 index 0000000..ff8f34e --- /dev/null +++ b/services/s3_lifecycler/wallet.json @@ -0,0 +1,30 @@ +{ + "version": "1.0", + "accounts": [ + { + "address": "NTt1rxvmEDxEuuogLxs2xgxA71qhVaUcN7", + "key": "6PYR3XurAyTzVeDG5WV2Z8vnGdySw3mTLuKjr6Nwo7tae64SJ7XjZSMMPQ", + "label": "lifecycler", + "contract": { + "script": "DCED9z0M+WSGfXZGxYLj1yYwmgxJXE/kNA4+oWNi0q1uKCdBVuezJw==", + "parameters": [ + { + "name": "parameter0", + "type": "Signature" + } + ], + "deployed": false + }, + "lock": false, + "isDefault": false + } + ], + "scrypt": { + "n": 16384, + "r": 8, + "p": 8 + }, + "extra": { + "Tokens": null + } +} From 326578f0ab2ead1423e137b012ed7dd0e3707f33 Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Wed, 6 Nov 2024 11:31:27 +0300 Subject: [PATCH 088/107] [#90] Stop using obsolete .github directory This commit is a part of multi-repo cleanup effort: https://git.frostfs.info/TrueCloudLab/frostfs-infra/issues/136 Signed-off-by: Vitaliy Potyarkin --- .dockerignore | 1 - {.github => .forgejo}/ISSUE_TEMPLATE/bug_report.md | 0 {.github => .forgejo}/ISSUE_TEMPLATE/config.yml | 0 {.github => .forgejo}/ISSUE_TEMPLATE/feature_request.md | 0 {.github => .forgejo}/logo.svg | 0 .github/CODEOWNERS | 1 - CODEOWNERS | 1 + README.md | 2 +- 8 files changed, 2 insertions(+), 3 deletions(-) rename {.github => .forgejo}/ISSUE_TEMPLATE/bug_report.md (100%) rename {.github => .forgejo}/ISSUE_TEMPLATE/config.yml (100%) rename {.github => .forgejo}/ISSUE_TEMPLATE/feature_request.md (100%) rename {.github => .forgejo}/logo.svg (100%) delete mode 100644 .github/CODEOWNERS create mode 100644 CODEOWNERS diff --git a/.dockerignore b/.dockerignore index 2f9d4b4..3867906 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,4 @@ .docker -.github .forgejo vendor tmp diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.forgejo/ISSUE_TEMPLATE/bug_report.md similarity index 100% rename from .github/ISSUE_TEMPLATE/bug_report.md rename to .forgejo/ISSUE_TEMPLATE/bug_report.md diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.forgejo/ISSUE_TEMPLATE/config.yml similarity index 100% rename from .github/ISSUE_TEMPLATE/config.yml rename to .forgejo/ISSUE_TEMPLATE/config.yml diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.forgejo/ISSUE_TEMPLATE/feature_request.md similarity index 100% rename from .github/ISSUE_TEMPLATE/feature_request.md rename to .forgejo/ISSUE_TEMPLATE/feature_request.md diff --git a/.github/logo.svg b/.forgejo/logo.svg similarity index 100% rename from .github/logo.svg rename to .forgejo/logo.svg diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS deleted file mode 100644 index a4413b0..0000000 --- a/.github/CODEOWNERS +++ /dev/null @@ -1 +0,0 @@ -* @alexvanin @fyrchik diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 0000000..d28053e --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1 @@ +.* @alexvanin @fyrchik diff --git a/README.md b/README.md index ad57655..8e4b714 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@

-FrostFS logo +FrostFS logo

FrostFS local Development and Testing environment From df6859d2463d768691d0d65d85961748e2309802 Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Fri, 13 Dec 2024 18:10:57 +0300 Subject: [PATCH 089/107] [#92] docs: Update contract list Contract list got outdated: it was mentioning NeoFS, Audit and Reputation Signed-off-by: Vitaliy Potyarkin --- docs/morph.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/morph.md b/docs/morph.md index 56edfbc..d713bf9 100644 --- a/docs/morph.md +++ b/docs/morph.md @@ -4,14 +4,16 @@ A single-node N3 privnet deployment, running on Contracts deployed: - Alphabet (AZ) [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/alphabet) -- Audit [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/audit) - Balance [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/balance) - Container [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/container) +- FrostFS [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/frostfs) +- FrostFSID [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/frostfsid) +- NNS [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/nns) - Netmap [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/netmap) -- NeoFSID [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/neofsid) +- Policy [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/policy) +- Processing [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/processing) - Proxy [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/proxy) -- Reputation [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/reputation) - + RPC available at `http://morph-chain.frostfs.devenv:30333`. ## .env settings From 0f9000bce6aedc9f1f376275ced6bc6a38cf6994 Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Tue, 10 Dec 2024 15:42:13 +0300 Subject: [PATCH 090/107] [#91] Refine CODEOWNERS settings Signed-off-by: Vitaliy Potyarkin --- CODEOWNERS | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index d28053e..3b080ce 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1 +1,3 @@ -.* @alexvanin @fyrchik +.* @alexvanin @fyrchik +.forgejo/.* @potyarkin +Makefile @potyarkin From 98484b97396d71d5cad90cc4ba504a743e9ba843 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 20 Dec 2024 14:58:00 +0300 Subject: [PATCH 091/107] [#93] Update neo-go to the latest version used by frostfs-node Signed-off-by: Alex Vanin --- .env | 2 +- services/morph_chain/protocol.privnet.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.env b/.env index a1a1586..4727fb1 100644 --- a/.env +++ b/.env @@ -8,7 +8,7 @@ BASTION_VERSION=10 BASTION_IMAGE=debian # NeoGo privnet -NEOGO_VERSION=0.104.0 +NEOGO_VERSION=0.106.3 NEOGO_IMAGE=nspccdev/neo-go # FrostFS InnerRing nodes diff --git a/services/morph_chain/protocol.privnet.yml b/services/morph_chain/protocol.privnet.yml index f1cc754..7cea631 100644 --- a/services/morph_chain/protocol.privnet.yml +++ b/services/morph_chain/protocol.privnet.yml @@ -11,6 +11,7 @@ ProtocolConfiguration: VerifyTransactions: true StateRootInHeader: true P2PSigExtensions: true + Hardforks: {} ApplicationConfiguration: SkipBlockVerification: false From e1b8fe791902f87b1347b3f2e948e4fc0fc583f1 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 20 Dec 2024 15:17:21 +0300 Subject: [PATCH 092/107] [#93] Update frostfs-node to the latest version Signed-off-by: Alex Vanin --- .env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.env b/.env index 4727fb1..a94341c 100644 --- a/.env +++ b/.env @@ -12,11 +12,11 @@ NEOGO_VERSION=0.106.3 NEOGO_IMAGE=nspccdev/neo-go # FrostFS InnerRing nodes -IR_VERSION=0.42.9 +IR_VERSION=0.44.4 IR_IMAGE=git.frostfs.info/truecloudlab/frostfs-ir # FrostFS Storage nodes -NODE_VERSION=0.42.9 +NODE_VERSION=0.44.4 NODE_IMAGE=git.frostfs.info/truecloudlab/frostfs-storage # HTTP Gate From 4db8ca356d5ebe8490ddfad69892da46b80e0ab6 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 20 Dec 2024 15:17:54 +0300 Subject: [PATCH 093/107] [#93] Remove unused env variables These variables were used when binaries were available in public storage network Signed-off-by: Alex Vanin --- .env | 2 -- 1 file changed, 2 deletions(-) diff --git a/.env b/.env index a94341c..aa96d95 100644 --- a/.env +++ b/.env @@ -44,12 +44,10 @@ FROSTFS_CLI_URL=https://git.frostfs.info/TrueCloudLab/frostfs-node/releases/down #FROSTFS_CLI_PATH=/path/to/frostfs-cli-binary # FrostFS ADM tool binary -FROSTFS_ADM_VERSION=498f9955ea FROSTFS_ADM_URL=https://git.frostfs.info/TrueCloudLab/frostfs-node/releases/download/v${NODE_VERSION}/frostfs-adm #FROSTFS_ADM_PATH=/path/to/frostfs-adm-binary # Compiled FrostFS Smart Contracts -FROSTFS_CONTRACTS_VERSION=694daebb19 FROSTFS_CONTRACTS_URL=https://git.frostfs.info/TrueCloudLab/frostfs-contract/releases/download/v0.19.2/frostfs-contract-v0.19.2.tar.gz #FROSTFS_CONTRACTS_PATH=/path/to/unpacked/frostfs-contracts-dir From 6a5817e15c1031b1aa06ebcb77dd8086be3079ee Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 20 Dec 2024 16:08:25 +0300 Subject: [PATCH 094/107] [#93] Register storage nodes in proxy contract during bootstrap This allows to send APE requests to the nodes right after `make up` command Signed-off-by: Alex Vanin --- Makefile | 2 ++ services/morph_chain/docker-compose.yml | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/Makefile b/Makefile index c4a6c48..58a12e0 100644 --- a/Makefile +++ b/Makefile @@ -85,6 +85,8 @@ up/bootstrap: get vendor/hosts echo "Transfer GAS to wallet $${f}" \ && ./vendor/frostfs-adm -c frostfs-adm.yml morph refill-gas --storage-wallet $${f} --gas 10.0 \ || die "Failed to transfer GAS to alphabet wallets"; \ + echo "Register storage wallet $${f} in proxy contract" \ + && ./vendor/frostfs-adm morph proxy-add-account --config frostfs-adm.yml --account=`docker container exec morph_chain neo-go wallet dump-keys -w /wallets/storage/$${f##*/} | head -1 | awk '{print $1}'` || die "Couldn't set storage allet as proxy wallet" done @echo "Create frostfsid subject for ./wallets/wallet.json"; \ if [ -n "$$(./vendor/frostfs-adm -c frostfs-adm.yml morph frostfsid list-subjects --namespace '')" ]; then \ diff --git a/services/morph_chain/docker-compose.yml b/services/morph_chain/docker-compose.yml index 2c2facb..6b83716 100644 --- a/services/morph_chain/docker-compose.yml +++ b/services/morph_chain/docker-compose.yml @@ -20,6 +20,10 @@ services: - ./../../vendor/hosts:/etc/hosts - ./../../wallets/wallet.json:/wallets/wallet.json - ./../s3_gate/wallet.json:/wallets/s3-wallet.json + - ./../storage/wallet01.json:/wallets/storage/wallet01.json + - ./../storage/wallet02.json:/wallets/storage/wallet02.json + - ./../storage/wallet03.json:/wallets/storage/wallet03.json + - ./../storage/wallet04.json:/wallets/storage/wallet04.json - chains:/chains networks: From d29d50a002c672a9ddfc6f6950dcba97e45344bc Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 20 Dec 2024 16:12:06 +0300 Subject: [PATCH 095/107] [#93] Update frostfs-contract to the latest version Signed-off-by: Alex Vanin --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index aa96d95..fe484c6 100644 --- a/.env +++ b/.env @@ -48,7 +48,7 @@ FROSTFS_ADM_URL=https://git.frostfs.info/TrueCloudLab/frostfs-node/releases/down #FROSTFS_ADM_PATH=/path/to/frostfs-adm-binary # Compiled FrostFS Smart Contracts -FROSTFS_CONTRACTS_URL=https://git.frostfs.info/TrueCloudLab/frostfs-contract/releases/download/v0.19.2/frostfs-contract-v0.19.2.tar.gz +FROSTFS_CONTRACTS_URL=https://git.frostfs.info/TrueCloudLab/frostfs-contract/releases/download/v0.21.1/frostfs-contract-v0.21.1.tar.gz #FROSTFS_CONTRACTS_PATH=/path/to/unpacked/frostfs-contracts-dir # Jaeger tracing From b08bb663f93b0aca97905e6bf8d4796cd8570328 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 20 Dec 2024 16:28:55 +0300 Subject: [PATCH 096/107] [#93] Update gateway components Signed-off-by: Alex Vanin --- .env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.env b/.env index fe484c6..7a58e35 100644 --- a/.env +++ b/.env @@ -20,7 +20,7 @@ NODE_VERSION=0.44.4 NODE_IMAGE=git.frostfs.info/truecloudlab/frostfs-storage # HTTP Gate -HTTP_GW_VERSION=0.30.2 +HTTP_GW_VERSION=0.32.0 HTTP_GW_IMAGE=truecloudlab/frostfs-http-gw # REST Gate @@ -28,7 +28,7 @@ REST_GW_VERSION=c9c85e90 REST_GW_IMAGE=truecloudlab/frostfs-rest-gw # S3 Gate -S3_GW_VERSION=0.31.0-rc.4 +S3_GW_VERSION=0.32.0 S3_GW_IMAGE=truecloudlab/frostfs-s3-gw # Lifecycler From d34d842700821a4eb8bcfb650734acb196957c64 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 20 Dec 2024 16:38:39 +0300 Subject: [PATCH 097/107] [#93] Remove frostfs-rest-gw from dev-env This repo is being archived and not maintained at the moment. Signed-off-by: Alex Vanin --- .env | 4 ---- .services | 1 - services/rest_gate/.env | 1 - services/rest_gate/.hosts | 1 - services/rest_gate/.int_test.env | 1 - services/rest_gate/cfg/config.yml | 12 ---------- services/rest_gate/docker-compose.yml | 32 --------------------------- services/rest_gate/wallet.json | 30 ------------------------- 8 files changed, 82 deletions(-) delete mode 120000 services/rest_gate/.env delete mode 100644 services/rest_gate/.hosts delete mode 120000 services/rest_gate/.int_test.env delete mode 100644 services/rest_gate/cfg/config.yml delete mode 100644 services/rest_gate/docker-compose.yml delete mode 100644 services/rest_gate/wallet.json diff --git a/.env b/.env index 7a58e35..824a4db 100644 --- a/.env +++ b/.env @@ -23,10 +23,6 @@ NODE_IMAGE=git.frostfs.info/truecloudlab/frostfs-storage HTTP_GW_VERSION=0.32.0 HTTP_GW_IMAGE=truecloudlab/frostfs-http-gw -# REST Gate -REST_GW_VERSION=c9c85e90 -REST_GW_IMAGE=truecloudlab/frostfs-rest-gw - # S3 Gate S3_GW_VERSION=0.32.0 S3_GW_IMAGE=truecloudlab/frostfs-s3-gw diff --git a/.services b/.services index d165241..ccc96f8 100644 --- a/.services +++ b/.services @@ -2,5 +2,4 @@ # Will start from top to bottom and stop in reverse http_gate s3_gate -rest_gate s3_lifecycler diff --git a/services/rest_gate/.env b/services/rest_gate/.env deleted file mode 120000 index c7360fb..0000000 --- a/services/rest_gate/.env +++ /dev/null @@ -1 +0,0 @@ -../../.env \ No newline at end of file diff --git a/services/rest_gate/.hosts b/services/rest_gate/.hosts deleted file mode 100644 index ee7578e..0000000 --- a/services/rest_gate/.hosts +++ /dev/null @@ -1 +0,0 @@ -IPV4_PREFIX.83 rest.LOCAL_DOMAIN diff --git a/services/rest_gate/.int_test.env b/services/rest_gate/.int_test.env deleted file mode 120000 index 582b6a2..0000000 --- a/services/rest_gate/.int_test.env +++ /dev/null @@ -1 +0,0 @@ -../../.int_test.env \ No newline at end of file diff --git a/services/rest_gate/cfg/config.yml b/services/rest_gate/cfg/config.yml deleted file mode 100644 index 0acdad3..0000000 --- a/services/rest_gate/cfg/config.yml +++ /dev/null @@ -1,12 +0,0 @@ -prometheus: - enabled: true - address: :9090 - -server: - # The IP and port to listen on. - listen-address: 0.0.0.0:8090 - -# Wallet settings -wallet: - path: /wallet.json # Path to wallet - passphrase: one # Password to decrypt wallet diff --git a/services/rest_gate/docker-compose.yml b/services/rest_gate/docker-compose.yml deleted file mode 100644 index 09cfe38..0000000 --- a/services/rest_gate/docker-compose.yml +++ /dev/null @@ -1,32 +0,0 @@ ---- - -services: - rest_gate: - image: ${REST_GW_IMAGE}:${REST_GW_VERSION} - domainname: ${LOCAL_DOMAIN} - hostname: rest - container_name: rest_gate - restart: on-failure - networks: - rest_gate_int: - internet: - ipv4_address: ${IPV4_PREFIX}.83 - volumes: - - ./wallet.json:/wallet.json - - ./../../vendor/hosts:/etc/hosts - - ./cfg:/etc/frostfs/rest - stop_signal: SIGTERM - stop_grace_period: 15s - env_file: [ ".env", ".int_test.env" ] - command: [ "frostfs-rest-gw", "--config", "/etc/frostfs/rest/config.yml" ] - environment: - - REST_GW_POOL_PEERS_0_ADDRESS=s01.${LOCAL_DOMAIN}:8080 - - REST_GW_POOL_PEERS_1_ADDRESS=s02.${LOCAL_DOMAIN}:8080 - - REST_GW_POOL_PEERS_2_ADDRESS=s03.${LOCAL_DOMAIN}:8080 - - REST_GW_POOL_PEERS_3_ADDRESS=s04.${LOCAL_DOMAIN}:8080 - -networks: - rest_gate_int: - internet: - external: true - name: basenet_internet diff --git a/services/rest_gate/wallet.json b/services/rest_gate/wallet.json deleted file mode 100644 index 2b60501..0000000 --- a/services/rest_gate/wallet.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "version": "3.0", - "accounts": [ - { - "address": "NPFCqWHfi9ixCJRu7DABRbVfXRbkSEr9Vo", - "key": "6PYTAGjdaeicUDPqGv9mmgwb9kTwimWJJmmfNqJSDGH9qM79zSRcL9oHiB", - "label": "REST Gateway", - "contract": { - "script": "DCECcuPzZCZ2VyDsm2jKEOMnU6xEWO2bF1dvOvBWTDFYB1ZBVuezJw==", - "parameters": [ - { - "name": "parameter0", - "type": "Signature" - } - ], - "deployed": false - }, - "lock": false, - "isDefault": false - } - ], - "scrypt": { - "n": 16384, - "r": 8, - "p": 8 - }, - "extra": { - "Tokens": null - } -} From c4d4fecb89ee1fbd2f7f02254f2ebf8264b47553 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 27 Dec 2024 10:42:19 +0300 Subject: [PATCH 098/107] [#96] s3_lifecycler: Fix docker-compose warnings Signed-off-by: Evgenii Stratonikov --- services/s3_lifecycler/docker-compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/services/s3_lifecycler/docker-compose.yml b/services/s3_lifecycler/docker-compose.yml index 3456d0c..2fffa50 100644 --- a/services/s3_lifecycler/docker-compose.yml +++ b/services/s3_lifecycler/docker-compose.yml @@ -1,6 +1,5 @@ --- -version: "2.4" services: s3_lifecycler: image: ${S3_LIFECYCLER_IMAGE}:${S3_LIFECYCLER_VERSION} From 90147c71081361e4348f0a041848e7ccf08d1235 Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Fri, 27 Dec 2024 09:33:04 +0300 Subject: [PATCH 099/107] [#95] Output errors during `make clean` It happens that a volume may not be deleted during `make clean`. For example, if the volume is being used by a container. If this happens, there are no errors printed to stdout. And old volumes may cause errors during subsequent `make up` Signed-off-by: Nikita Zinkevich --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 58a12e0..f24ec9d 100644 --- a/Makefile +++ b/Makefile @@ -159,7 +159,7 @@ clean: vols=`docker-compose -f services/$${svc}/docker-compose.yml config --volumes` if [[ ! -z "$${vols}" ]]; then for vol in $${vols}; do - docker volume rm -f "$${svc}_$${vol}" 2> /dev/null + docker volume rm -f "$${svc}_$${vol}" done fi done From 5471dbfc0e57babdc5f95c1c788fbd3a61c9e9d7 Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Thu, 9 Jan 2025 11:23:42 +0300 Subject: [PATCH 100/107] [#98] s3_gate: Fix custom user wallets folder creation during compose up Make custom wallets volume to point to the `wallets` directory in the project's root. Signed-off-by: Nikita Zinkevich --- README.md | 9 +++++---- services/s3_gate/docker-compose.yml | 6 ++---- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 8e4b714..415885e 100644 --- a/README.md +++ b/README.md @@ -143,7 +143,7 @@ Registers user wallet and issues s3 credentials. Usage and default parameter values: ```sh -make s3cred [password=""] [contract_password=s3] [wallet=/user_wallet.json] [gate_public_key=0313b1ac3a8076e155a7e797b24f0b650cccad5941ea59d7cfd51a024a8b2a06bf] +make s3cred [password=""] [contract_password=s3] [wallet=""] [gate_public_key=0313b1ac3a8076e155a7e797b24f0b650cccad5941ea59d7cfd51a024a8b2a06bf] ``` As soon as the storage node is in the network map (see above) you can generate S3 @@ -159,8 +159,9 @@ $ make s3cred "container_id": "EXArWh8x1zeHG3851s1RtoCo7dowxF6rhLGA15nbMffT" } ``` -Running without any parameters will result in defaults which are based on the private key from -`/user-wallet.json` file and `/wallet.json` contract wallet. +Running without any parameters results in defaults which are based on the private key from +`/wallets/wallet.json` user wallet and `/wallet.json` contract wallet. +If `wallet` parameter is set, gate searches custom user wallet file in `/wallets` directory. Now let's configure an S3 client (AWS CLI will be used as example): @@ -172,7 +173,7 @@ Default region name []: us-east-1 Default output format []: json ``` -If you need to create credentials for different users, put user wallets to `wallets` dir and specify them via `wallet` parameter. +If you need to create credentials for different users, put user wallet to `wallets` dir and specify it via `wallet` parameter. Pass wallet password in `password` parameter if it's not default. The same is for `contract_wallet` and `gate_public_key` params. ```sh diff --git a/services/s3_gate/docker-compose.yml b/services/s3_gate/docker-compose.yml index f7fc28d..3c1f9f6 100644 --- a/services/s3_gate/docker-compose.yml +++ b/services/s3_gate/docker-compose.yml @@ -14,10 +14,8 @@ services: volumes: # Gate wallet - ./wallet.json:/wallet.json - # Custom user wallets - - ./wallets:/wallets - # Default user wallet - - ./../../wallets/wallet.json:/wallets/wallet.json + # Folder for custom user wallets + - ./../../wallets/:/wallets/ - ./tls.key:/tls.key - ./tls.crt:/tls.crt - ./../../vendor/hosts:/etc/hosts From 0981202d6bcc0c454d55cfb1cb5c9b355e217170 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 21 Mar 2025 18:47:17 +0300 Subject: [PATCH 101/107] [#100] Use docker images as vendor binary source Signed-off-by: Alex Vanin --- .env | 6 ++++-- services/ir/artifacts.mk | 8 +++----- services/morph_chain/artifacts.mk | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.env b/.env index 824a4db..053f179 100644 --- a/.env +++ b/.env @@ -36,11 +36,13 @@ LOCODE_DB_URL=https://git.frostfs.info/attachments/a2e8def7-52b6-49f1-89cd-a0567 #LOCODE_DB_PATH=/path/to/locode_db # FrostFS CLI binary -FROSTFS_CLI_URL=https://git.frostfs.info/TrueCloudLab/frostfs-node/releases/download/v${NODE_VERSION}/frostfs-cli +FROSTFS_CLI_VERSION=0.44.4 +FROSTFS_CLI_IMAGE=git.frostfs.info/truecloudlab/frostfs-cli #FROSTFS_CLI_PATH=/path/to/frostfs-cli-binary # FrostFS ADM tool binary -FROSTFS_ADM_URL=https://git.frostfs.info/TrueCloudLab/frostfs-node/releases/download/v${NODE_VERSION}/frostfs-adm +FROSTFS_ADM_VERSION=0.44.4 +FROSTFS_ADM_IMAGE=git.frostfs.info/truecloudlab/frostfs-adm #FROSTFS_ADM_PATH=/path/to/frostfs-adm-binary # Compiled FrostFS Smart Contracts diff --git a/services/ir/artifacts.mk b/services/ir/artifacts.mk index 0cdbdbb..1c2cf5d 100644 --- a/services/ir/artifacts.mk +++ b/services/ir/artifacts.mk @@ -30,11 +30,9 @@ get.cli: @mkdir -p ./vendor ifeq (${FROSTFS_CLI_PATH},) - @echo "⇒ Download FrostFS CLI binary from ${FROSTFS_CLI_URL}" - @curl \ - -ksSL "${FROSTFS_CLI_URL}" \ - -o ${FROSTFS_CLI_FILE} - @chmod +x ${FROSTFS_CLI_FILE} + echo "⇒ Download FrostFS CLI binary from ${FROSTFS_CLI_IMAGE}:${FROSTFS_CLI_VERSION}" + $(shell docker cp `docker create --name tmp ${FROSTFS_CLI_IMAGE}:${FROSTFS_CLI_VERSION}`:/bin/frostfs-cli ${FROSTFS_CLI_FILE} && docker rm tmp >/dev/null) + chmod +x ${FROSTFS_CLI_FILE} else @echo "⇒ Copy local binary from ${FROSTFS_CLI_PATH}" @cp ${FROSTFS_CLI_PATH} ${FROSTFS_CLI_FILE} diff --git a/services/morph_chain/artifacts.mk b/services/morph_chain/artifacts.mk index 77a7ae3..4a4504a 100644 --- a/services/morph_chain/artifacts.mk +++ b/services/morph_chain/artifacts.mk @@ -23,8 +23,8 @@ get.adm: FROSTFS_ADM_DEST=./vendor/frostfs-adm get.adm: ifeq (${FROSTFS_ADM_PATH},) - @echo "⇒ Download FrostFS ADM binary from ${FROSTFS_ADM_URL}" - @curl -skSL ${FROSTFS_ADM_URL} -o ${FROSTFS_ADM_DEST} + @echo "⇒ Download FrostFS ADM binary from ${FROSTFS_ADM_IMAGE}:${FROSTFS_ADM_VERSION}" + $(shell docker cp `docker create --name tmp ${FROSTFS_ADM_IMAGE}:${FROSTFS_ADM_VERSION}`:/bin/frostfs-adm ${FROSTFS_ADM_DEST} && docker rm tmp >/dev/null) @chmod +x ${FROSTFS_ADM_DEST} else @echo "⇒ Copy frostfs-adm binary from ${FROSTFS_ADM_PATH}" From ea91d8823274fa5eac9bd86a49369c6267a3fc6f Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 21 Mar 2025 18:54:24 +0300 Subject: [PATCH 102/107] [#100] Add target to prepare storage for service components Signed-off-by: Alex Vanin --- Makefile | 22 ++++++++++++++++++++-- cli-cfg.yml | 3 +++ services/morph_chain/docker-compose.yml | 1 + wallets/system-wallet.json | 1 + 4 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 cli-cfg.yml create mode 100644 wallets/system-wallet.json diff --git a/Makefile b/Makefile index f24ec9d..b3b342d 100644 --- a/Makefile +++ b/Makefile @@ -58,9 +58,8 @@ get: $(foreach SVC, $(GET_SVCS), get.$(SVC)) # Start environment .PHONY: up -up: up/basic +up: up/basic up/pre-services @$(foreach SVC, $(START_SVCS), $(shell docker-compose -f services/$(SVC)/docker-compose.yml up -d)) - ./vendor/frostfs-adm morph proxy-add-account --config frostfs-adm.yml --account=`docker container exec morph_chain neo-go wallet dump-keys -w /wallets/s3-wallet.json | head -1 | awk '{print $1}'` || die "Couldn't set s3-gw wallet as proxy wallet" @echo "Full FrostFS Developer Environment is ready" # Build up FrostFS @@ -98,6 +97,25 @@ up/bootstrap: get vendor/hosts || die "Failed to create subject for the wallet"; \ fi echo "FrostFS sidechain environment is deployed" + +# Prepare to start services +.PHONY: up/pre-services +up/pre-services: + @source ./bin/helper.sh + @echo "Prepare storage for services"; \ + if [ -z "$$(./vendor/frostfs-cli -c cli-cfg.yml container list)" ]; then \ + subj_key=`docker container exec -it morph_chain neo-go wallet dump-keys -w /wallets/system-wallet.json | tail -1 | tr -d ' \r\n'` \ + && echo "Subject key: $${subj_key}" \ + && ./vendor/frostfs-adm -c frostfs-adm.yml morph frostfsid create-subject --namespace "" --subject-key $${subj_key} --subject-name system \ + || die "Failed to create subject for system wallet"; \ + proxy_acc=`docker container exec -it morph_chain neo-go wallet dump-keys -w /wallets/system-wallet.json | head -1 | cut -d" " -f1` \ + && echo "Proxy acc: $${proxy_acc}" \ + && ./vendor/frostfs-adm morph proxy-add-account --config frostfs-adm.yml --account=$${proxy_acc} || die "Failed to register S3 gateway as proxy acc"; \ + cid=`./vendor/frostfs-cli -c cli-cfg.yml container create -p "REP 4" --nns-name "cors" --nns-zone "container" --await | grep CID | cut -d" " -f2` \ + && echo "CORS Container: $${cid}" \ + && ./vendor/frostfs-cli -c cli-cfg.yml ape-manager add --target-type container --target-name $${cid} --rule "allow Object.* *" || die "Failed to create CORS container" + fi + @echo "Storage is prepared"; # Build up certain service .PHONY: up/% diff --git a/cli-cfg.yml b/cli-cfg.yml new file mode 100644 index 0000000..9a5c7eb --- /dev/null +++ b/cli-cfg.yml @@ -0,0 +1,3 @@ +wallet: ./wallets/system-wallet.json +password: "" +rpc-endpoint: s01.frostfs.devenv:8080 diff --git a/services/morph_chain/docker-compose.yml b/services/morph_chain/docker-compose.yml index 6b83716..c3a1481 100644 --- a/services/morph_chain/docker-compose.yml +++ b/services/morph_chain/docker-compose.yml @@ -19,6 +19,7 @@ services: - ./config.yml:/wallets/config.yml - ./../../vendor/hosts:/etc/hosts - ./../../wallets/wallet.json:/wallets/wallet.json + - ./../../wallets/system-wallet.json:/wallets/system-wallet.json - ./../s3_gate/wallet.json:/wallets/s3-wallet.json - ./../storage/wallet01.json:/wallets/storage/wallet01.json - ./../storage/wallet02.json:/wallets/storage/wallet02.json diff --git a/wallets/system-wallet.json b/wallets/system-wallet.json new file mode 100644 index 0000000..2f57db1 --- /dev/null +++ b/wallets/system-wallet.json @@ -0,0 +1 @@ +{"version":"1.0","accounts":[{"address":"NQijiVKHbL22PfF2AJQukv1CX75itxgzht","key":"6PYQKrpme57VqaucxuF7dDoSZRRA8d94oatHcScqhiFBauCXQvFDaYwEWa","label":"","contract":{"script":"DCEDRdLtpFIWeYyI7doTKRhIl4qYjaybGDveTyGpbqjsLZNBVuezJw==","parameters":[{"name":"parameter0","type":"Signature"}],"deployed":false},"lock":false,"isDefault":false}],"scrypt":{"n":16384,"r":8,"p":8},"extra":{"Tokens":null}} \ No newline at end of file From e0141c11545ecf107bf50fe5da293a1390ba2695 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 21 Mar 2025 18:55:43 +0300 Subject: [PATCH 103/107] [#100] Update components to the latest versions Signed-off-by: Alex Vanin --- .env | 16 ++++++++-------- services/http_gate/cfg/config.yml | 3 +++ services/s3_gate/cfg/config.yml | 3 +++ 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/.env b/.env index 053f179..dec29ce 100644 --- a/.env +++ b/.env @@ -12,20 +12,20 @@ NEOGO_VERSION=0.106.3 NEOGO_IMAGE=nspccdev/neo-go # FrostFS InnerRing nodes -IR_VERSION=0.44.4 +IR_VERSION=0.45.0-rc.3 IR_IMAGE=git.frostfs.info/truecloudlab/frostfs-ir # FrostFS Storage nodes -NODE_VERSION=0.44.4 +NODE_VERSION=0.45.0-rc.3 NODE_IMAGE=git.frostfs.info/truecloudlab/frostfs-storage # HTTP Gate -HTTP_GW_VERSION=0.32.0 -HTTP_GW_IMAGE=truecloudlab/frostfs-http-gw +HTTP_GW_VERSION=0.33.0-rc.2 +HTTP_GW_IMAGE=git.frostfs.info/truecloudlab/frostfs-http-gw # S3 Gate -S3_GW_VERSION=0.32.0 -S3_GW_IMAGE=truecloudlab/frostfs-s3-gw +S3_GW_VERSION=0.33.0-rc.2 +S3_GW_IMAGE=git.frostfs.info/truecloudlab/frostfs-s3-gw # Lifecycler S3_LIFECYCLER_VERSION=0.1.3 @@ -36,12 +36,12 @@ LOCODE_DB_URL=https://git.frostfs.info/attachments/a2e8def7-52b6-49f1-89cd-a0567 #LOCODE_DB_PATH=/path/to/locode_db # FrostFS CLI binary -FROSTFS_CLI_VERSION=0.44.4 +FROSTFS_CLI_VERSION=0.45.0-rc.3 FROSTFS_CLI_IMAGE=git.frostfs.info/truecloudlab/frostfs-cli #FROSTFS_CLI_PATH=/path/to/frostfs-cli-binary # FrostFS ADM tool binary -FROSTFS_ADM_VERSION=0.44.4 +FROSTFS_ADM_VERSION=0.45.0-rc.3 FROSTFS_ADM_IMAGE=git.frostfs.info/truecloudlab/frostfs-adm #FROSTFS_ADM_PATH=/path/to/frostfs-adm-binary diff --git a/services/http_gate/cfg/config.yml b/services/http_gate/cfg/config.yml index 1684c58..2f15e44 100644 --- a/services/http_gate/cfg/config.yml +++ b/services/http_gate/cfg/config.yml @@ -22,3 +22,6 @@ server: wallet: path: /wallet.json # Path to wallet passphrase: one # Passphrase to decrypt wallet + +containers: + cors: cors.container diff --git a/services/s3_gate/cfg/config.yml b/services/s3_gate/cfg/config.yml index 03e84e0..3d8e400 100644 --- a/services/s3_gate/cfg/config.yml +++ b/services/s3_gate/cfg/config.yml @@ -47,3 +47,6 @@ frostfsid: policy: enabled: false + +containers: + cors: cors.container From 018e33c1f9048df8e6c580bde535df535a297838 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Thu, 27 Mar 2025 17:34:37 +0300 Subject: [PATCH 104/107] [#101] Fix s3 gateway initialization Was broken in #100 Signed-off-by: Alex Vanin --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b3b342d..fcc09a3 100644 --- a/Makefile +++ b/Makefile @@ -108,7 +108,7 @@ up/pre-services: && echo "Subject key: $${subj_key}" \ && ./vendor/frostfs-adm -c frostfs-adm.yml morph frostfsid create-subject --namespace "" --subject-key $${subj_key} --subject-name system \ || die "Failed to create subject for system wallet"; \ - proxy_acc=`docker container exec -it morph_chain neo-go wallet dump-keys -w /wallets/system-wallet.json | head -1 | cut -d" " -f1` \ + proxy_acc=`docker container exec -it morph_chain neo-go wallet dump-keys -w /wallets/s3-wallet.json | head -1 | cut -d" " -f1` \ && echo "Proxy acc: $${proxy_acc}" \ && ./vendor/frostfs-adm morph proxy-add-account --config frostfs-adm.yml --account=$${proxy_acc} || die "Failed to register S3 gateway as proxy acc"; \ cid=`./vendor/frostfs-cli -c cli-cfg.yml container create -p "REP 4" --nns-name "cors" --nns-zone "container" --await | grep CID | cut -d" " -f2` \ From 6ef4d4aa0ad7f99f948c0cf19bda4f0faa191c37 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 28 Mar 2025 14:07:50 +0300 Subject: [PATCH 105/107] [#102] Makefile: Parallelize GAS transfer The last version of the `frostfs-adm` allows to work with multiple wallets in `refill-gas` and `proxy-add-account` commands. This makes `up/bootstrap` target take ~6s less time. The Makefile stuff is not obvious, but it seem to work as expected. Refs TrueCloudLab/frostfs-node#1590, TrueCloudLab/frostfs-node#1578 Signed-off-by: Evgenii Stratonikov --- Makefile | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index fcc09a3..911d168 100644 --- a/Makefile +++ b/Makefile @@ -72,6 +72,8 @@ up/basic: up/bootstrap # Start bootstrap services .PHONY: up/bootstrap +up/bootstrap: STORAGE_WALLETS = $(wildcard ./services/storage/wallet*.json) +up/bootstrap: STORAGE_ACCOUNTS = $(foreach wallet,$(STORAGE_WALLETS),$(shell docker container exec morph_chain neo-go wallet dump-keys -w /wallets/storage/$(notdir $(wallet)) | head -1 | awk '{print $$1}' )) up/bootstrap: get vendor/hosts @$(foreach SVC, $(START_BOOTSTRAP), $(shell docker-compose -f services/$(SVC)/docker-compose.yml up -d)) @source ./bin/helper.sh @@ -80,13 +82,16 @@ up/bootstrap: get vendor/hosts @./vendor/frostfs-adm --config frostfs-adm.yml morph \ ape add-rule-chain --target-type namespace --target-name "" \ --rule 'allow Container.* *' --chain-id "allow_container_ops" - @for f in ./services/storage/wallet*.json; do \ - echo "Transfer GAS to wallet $${f}" \ - && ./vendor/frostfs-adm -c frostfs-adm.yml morph refill-gas --storage-wallet $${f} --gas 10.0 \ - || die "Failed to transfer GAS to alphabet wallets"; \ - echo "Register storage wallet $${f} in proxy contract" \ - && ./vendor/frostfs-adm morph proxy-add-account --config frostfs-adm.yml --account=`docker container exec morph_chain neo-go wallet dump-keys -w /wallets/storage/$${f##*/} | head -1 | awk '{print $1}'` || die "Couldn't set storage allet as proxy wallet" - done + + echo -e "Transfer GAS to storage wallets: $(foreach wallet,$(STORAGE_WALLETS),\n\t$(wallet))" + ./vendor/frostfs-adm -c frostfs-adm.yml morph refill-gas \ + $(foreach wallet,$(STORAGE_WALLETS),--storage-wallet $(wallet)) \ + --gas 10.0 \ + || die "Failed to transfer GAS to alphabet wallets" + echo -e "Register storage accounts in proxy contract: $(foreach account,$(STORAGE_ACCOUNTS),\n\t$(account))" + ./vendor/frostfs-adm morph proxy-add-account --config frostfs-adm.yml \ + $(foreach account,$(STORAGE_ACCOUNTS),--account=$(account)) \ + || die "Couldn't set storage allet as proxy wallet" @echo "Create frostfsid subject for ./wallets/wallet.json"; \ if [ -n "$$(./vendor/frostfs-adm -c frostfs-adm.yml morph frostfsid list-subjects --namespace '')" ]; then \ echo "Subject already exists"; \ From 8608ba46ff2b6bc07a3d662ce0ace00bd137a31e Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Tue, 8 Apr 2025 14:51:37 +0300 Subject: [PATCH 106/107] [#104] Update components to the latest versions New S3 gateway version supports MFADelete feature that requires container with MFA objects. Signed-off-by: Alex Vanin --- .env | 12 ++++++------ Makefile | 5 ++++- services/s3_gate/cfg/config.yml | 1 + 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.env b/.env index dec29ce..9c4531d 100644 --- a/.env +++ b/.env @@ -12,19 +12,19 @@ NEOGO_VERSION=0.106.3 NEOGO_IMAGE=nspccdev/neo-go # FrostFS InnerRing nodes -IR_VERSION=0.45.0-rc.3 +IR_VERSION=0.45.0-rc.6 IR_IMAGE=git.frostfs.info/truecloudlab/frostfs-ir # FrostFS Storage nodes -NODE_VERSION=0.45.0-rc.3 +NODE_VERSION=0.45.0-rc.6 NODE_IMAGE=git.frostfs.info/truecloudlab/frostfs-storage # HTTP Gate -HTTP_GW_VERSION=0.33.0-rc.2 +HTTP_GW_VERSION=0.33.0-rc.3 HTTP_GW_IMAGE=git.frostfs.info/truecloudlab/frostfs-http-gw # S3 Gate -S3_GW_VERSION=0.33.0-rc.2 +S3_GW_VERSION=0.33.0-rc.3 S3_GW_IMAGE=git.frostfs.info/truecloudlab/frostfs-s3-gw # Lifecycler @@ -36,12 +36,12 @@ LOCODE_DB_URL=https://git.frostfs.info/attachments/a2e8def7-52b6-49f1-89cd-a0567 #LOCODE_DB_PATH=/path/to/locode_db # FrostFS CLI binary -FROSTFS_CLI_VERSION=0.45.0-rc.3 +FROSTFS_CLI_VERSION=0.45.0-rc.6 FROSTFS_CLI_IMAGE=git.frostfs.info/truecloudlab/frostfs-cli #FROSTFS_CLI_PATH=/path/to/frostfs-cli-binary # FrostFS ADM tool binary -FROSTFS_ADM_VERSION=0.45.0-rc.3 +FROSTFS_ADM_VERSION=0.45.0-rc.6 FROSTFS_ADM_IMAGE=git.frostfs.info/truecloudlab/frostfs-adm #FROSTFS_ADM_PATH=/path/to/frostfs-adm-binary diff --git a/Makefile b/Makefile index 911d168..56e5f7c 100644 --- a/Makefile +++ b/Makefile @@ -118,7 +118,10 @@ up/pre-services: && ./vendor/frostfs-adm morph proxy-add-account --config frostfs-adm.yml --account=$${proxy_acc} || die "Failed to register S3 gateway as proxy acc"; \ cid=`./vendor/frostfs-cli -c cli-cfg.yml container create -p "REP 4" --nns-name "cors" --nns-zone "container" --await | grep CID | cut -d" " -f2` \ && echo "CORS Container: $${cid}" \ - && ./vendor/frostfs-cli -c cli-cfg.yml ape-manager add --target-type container --target-name $${cid} --rule "allow Object.* *" || die "Failed to create CORS container" + && ./vendor/frostfs-cli -c cli-cfg.yml ape-manager add --target-type container --target-name $${cid} --rule "allow Object.* *" || die "Failed to create CORS container"; \ + cid=`./vendor/frostfs-cli -c cli-cfg.yml container create -p "REP 4" --nns-name "mfa" --nns-zone "container" --await | grep CID | cut -d" " -f2` \ + && echo "MFA Container: $${cid}" \ + && ./vendor/frostfs-cli -c cli-cfg.yml ape-manager add --target-type container --target-name $${cid} --rule "allow Object.* *" || die "Failed to create MFA container" fi @echo "Storage is prepared"; diff --git a/services/s3_gate/cfg/config.yml b/services/s3_gate/cfg/config.yml index 3d8e400..369325c 100644 --- a/services/s3_gate/cfg/config.yml +++ b/services/s3_gate/cfg/config.yml @@ -50,3 +50,4 @@ policy: containers: cors: cors.container + mfa: mfa.container From df8249f2aeabbbe8caab4229e42321027269cb74 Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Fri, 11 Apr 2025 15:49:20 +0300 Subject: [PATCH 107/107] [#105] Add website container creation Signed-off-by: Nikita Zinkevich --- Makefile | 3 +++ services/s3_gate/cfg/config.yml | 1 + 2 files changed, 4 insertions(+) diff --git a/Makefile b/Makefile index 56e5f7c..1f97c5f 100644 --- a/Makefile +++ b/Makefile @@ -122,6 +122,9 @@ up/pre-services: cid=`./vendor/frostfs-cli -c cli-cfg.yml container create -p "REP 4" --nns-name "mfa" --nns-zone "container" --await | grep CID | cut -d" " -f2` \ && echo "MFA Container: $${cid}" \ && ./vendor/frostfs-cli -c cli-cfg.yml ape-manager add --target-type container --target-name $${cid} --rule "allow Object.* *" || die "Failed to create MFA container" + cid=`./vendor/frostfs-cli -c cli-cfg.yml container create -p "REP 4" --nns-name "website" --nns-zone "container" --await | grep CID | cut -d" " -f2` \ + && echo "Website configuration Container: $${cid}" \ + && ./vendor/frostfs-cli -c cli-cfg.yml ape-manager add --target-type container --target-name $${cid} --rule "allow Object.* *" || die "Failed to create Website configuration container" fi @echo "Storage is prepared"; diff --git a/services/s3_gate/cfg/config.yml b/services/s3_gate/cfg/config.yml index 369325c..e5c0b53 100644 --- a/services/s3_gate/cfg/config.yml +++ b/services/s3_gate/cfg/config.yml @@ -51,3 +51,4 @@ policy: containers: cors: cors.container mfa: mfa.container + website: website.container