From 7a4a9b346a5a3c878beb74ccaac43b91526c2cd2 Mon Sep 17 00:00:00 2001 From: Artem Ivanov Date: Tue, 7 Jun 2022 10:11:57 +0300 Subject: [PATCH 001/149] Restart coredns on failure Signed-off-by: Alex Vanin --- services/coredns/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/services/coredns/docker-compose.yml b/services/coredns/docker-compose.yml index a65854d..0972a7c 100644 --- a/services/coredns/docker-compose.yml +++ b/services/coredns/docker-compose.yml @@ -7,6 +7,7 @@ services: container_name: coredns domainname: ${LOCAL_DOMAIN} hostname: coredns + restart: on-failure networks: coredns: internet: From cea4d0e26ea63bebeb2b197ec402f4e747d672a7 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 16 Sep 2022 16:54:40 +0300 Subject: [PATCH 002/149] Use latest neo-go with `mgmt-gethashbyid` patch Signed-off-by: Alex Vanin --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index d4f7dfa..906d3e8 100644 --- a/.env +++ b/.env @@ -10,7 +10,7 @@ BASTION_IMAGE=debian # NeoGo privnet #CHAIN_PATH="/path/to/devenv.dump.gz" CHAIN_URL="https://github.com/nspcc-dev/neofs-contract/releases/download/v0.15.5/devenv_mainchain_notary_disabled.gz" -NEOGO_VERSION=0.99.2 +NEOGO_VERSION=0.99.3-2-gaa57779d NEOGO_IMAGE=nspccdev/neo-go # NeoFS InnerRing nodes From 846311f12eab939b0d5c662be2dd84c79f9a25f5 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Wed, 28 Sep 2022 18:37:48 +0300 Subject: [PATCH 003/149] Disable caching on storage nodes Signed-off-by: Alex Vanin --- services/storage/.storage.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/storage/.storage.env b/services/storage/.storage.env index ad3a44f..c3605fb 100644 --- a/services/storage/.storage.env +++ b/services/storage/.storage.env @@ -27,7 +27,7 @@ NEOFS_MORPH_RPC_ENDPOINT_0_ADDRESS=ws://morph-chain:30333/ws ## Default: 5s #NEOFS_MORPH_DIAL_TIMEOUT=1m # Configure side chain caches (negative value to disable) -#NEOFS_MORPH_CACHE_TTL=15s +NEOFS_MORPH_CACHE_TTL=-1 # Cache of API clients section ## Dial timeout to connect to remote API server From 20a4e9f1ee25ac2b81f6d92a719221c35004e542 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 7 Oct 2022 13:20:52 +0300 Subject: [PATCH 004/149] [#226] morph_chain: Allow to use contracts from directory Signed-off-by: Evgenii Stratonikov --- services/morph_chain/artifacts.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/morph_chain/artifacts.mk b/services/morph_chain/artifacts.mk index 9c2a761..35efdec 100644 --- a/services/morph_chain/artifacts.mk +++ b/services/morph_chain/artifacts.mk @@ -13,7 +13,7 @@ ifeq (${NEOFS_CONTRACTS_PATH},) @rm ${NEOFS_CONTRACTS_ARCHIVE} else @echo "⇒ Copy compiled contracts from ${NEOFS_CONTRACTS_PATH}" - @cp -r ${NEOFS_CONTRACTS_PATH} ${NEOFS_CONTRACTS_DEST} + @cp -r ${NEOFS_CONTRACTS_PATH}/* ${NEOFS_CONTRACTS_DEST} endif get.adm: NEOFS_ADM_DEST=./vendor/neofs-adm From daab798d4121a0097cb13dc0328d63a235fc6ced Mon Sep 17 00:00:00 2001 From: anastasia prasolova Date: Sun, 16 Oct 2022 20:28:26 +0300 Subject: [PATCH 005/149] Add CODEOWNERS file Signed-off-by: anastasia prasolova --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..ce66aa1 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @aprasolova @realloc @anatoly-bogatyrev From 3a13c5398c0f7a2136ba3866a1b4e834926b1b2a Mon Sep 17 00:00:00 2001 From: anastasia prasolova Date: Sun, 16 Oct 2022 20:29:52 +0300 Subject: [PATCH 006/149] Fix CODEOWNERS file Signed-off-by: anastasia prasolova --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ce66aa1..a4413b0 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1 @@ -* @aprasolova @realloc @anatoly-bogatyrev +* @alexvanin @fyrchik From 55c1add70c8e71f1f634bdff460ee7717f59eb50 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 14 Oct 2022 12:39:30 +0300 Subject: [PATCH 007/149] [#225] makefile: pull only enabled services Signed-off-by: Denis Kirillov --- Makefile | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 7e7815a..d3dc47b 100644 --- a/Makefile +++ b/Makefile @@ -19,12 +19,6 @@ include services/*/artifacts.mk # Targets helpful to prepare service environment include services/*/prepare.mk -# Services that require artifacts -GET_SVCS = $(shell grep -Rl "get.*:" ./services/* | sort -u | grep artifacts.mk | xargs -I {} dirname {} | xargs basename -a) - -# Services that require pulling images -PULL_SVCS = $(shell find ./services -type f -name 'docker-compose.yml' | sort -u | xargs -I {} dirname {} | xargs basename -a) - # List of services to run START_SVCS = $(shell cat .services | grep -v \\\#) START_BASIC = $(shell cat .basic_services | grep -v \\\#) @@ -33,6 +27,15 @@ STOP_SVCS = $(shell tac .services | grep -v \\\#) STOP_BASIC = $(shell tac .basic_services | grep -v \\\#) STOP_BOOTSTRAP = $(shell tac .bootstrap_services | grep -v \\\#) +# Enabled services dirs +ENABLED_SVCS_DIRS = $(shell echo "${START_BOOTSTRAP} ${START_BASIC} ${START_SVCS}" | sed 's|[^ ]* *|./services/&|g') + +# Services that require artifacts +GET_SVCS = $(shell grep -Rl "get.*:" ./services/* | sort -u | grep artifacts.mk | xargs -I {} dirname {} | xargs basename -a) + +# Services that require pulling images +PULL_SVCS = $(shell find ${ENABLED_SVCS_DIRS} -type f -name 'docker-compose.yml' | sort -u | xargs -I {} dirname {} | xargs basename -a) + # List of hosts available in devenv HOSTS_LINES = $(shell grep -Rl IPV4_PREFIX ./services/* | grep .hosts) From 86a36a3ff430ae17845564c0c5f6ee6fa312ce34 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Sat, 22 Oct 2022 14:29:42 +0300 Subject: [PATCH 008/149] Update neofs-* to v0.33.0 Signed-off-by: Evgenii Stratonikov --- .env | 14 +++++++------- services/ir/.ir.env | 4 ++-- services/storage/.storage.env | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.env b/.env index d4f7dfa..6441384 100644 --- a/.env +++ b/.env @@ -9,16 +9,16 @@ BASTION_IMAGE=debian # NeoGo privnet #CHAIN_PATH="/path/to/devenv.dump.gz" -CHAIN_URL="https://github.com/nspcc-dev/neofs-contract/releases/download/v0.15.5/devenv_mainchain_notary_disabled.gz" -NEOGO_VERSION=0.99.2 +CHAIN_URL="https://github.com/nspcc-dev/neofs-contract/releases/download/v0.16.0/devenv_mainchain_notary_disabled.gz" +NEOGO_VERSION=0.99.4 NEOGO_IMAGE=nspccdev/neo-go # NeoFS InnerRing nodes -IR_VERSION=0.32.0 +IR_VERSION=0.33.0 IR_IMAGE=nspccdev/neofs-ir # NeoFS Storage nodes -NODE_VERSION=0.32.0 +NODE_VERSION=0.33.0 NODE_IMAGE=nspccdev/neofs-storage # NATS Server @@ -46,15 +46,15 @@ LOCODE_DB_URL=https://github.com/nspcc-dev/neofs-locode-db/releases/download/v0. #LOCODE_DB_PATH=/path/to/locode_db # NeoFS CLI binary -NEOFS_CLI_URL=https://github.com/nspcc-dev/neofs-node/releases/download/v0.32.0/neofs-cli-amd64.tar.gz +NEOFS_CLI_URL=https://github.com/nspcc-dev/neofs-node/releases/download/v0.33.0/neofs-cli-amd64.tar.gz #NEOFS_CLI_PATH=/path/to/neofs-cli-binary # NeoFS ADM tool binary -NEOFS_ADM_VERSION=v0.32.0 +NEOFS_ADM_VERSION=v0.33.0 NEOFS_ADM_URL=https://github.com/nspcc-dev/neofs-node/releases/download/${NEOFS_ADM_VERSION}/neofs-adm-amd64.tar.gz #NEOFS_ADM_PATH=/path/to/neofs-adm-binary # Compiled NeoFS Smart Contracts -NEOFS_CONTRACTS_VERSION=v0.15.5 +NEOFS_CONTRACTS_VERSION=v0.16.0 NEOFS_CONTRACTS_URL=https://github.com/nspcc-dev/neofs-contract/releases/download/${NEOFS_CONTRACTS_VERSION}/neofs-contract-${NEOFS_CONTRACTS_VERSION}.tar.gz #NEOFS_CONTRACTS_PATH=/path/to/unpacked/neofs-contracts-dir diff --git a/services/ir/.ir.env b/services/ir/.ir.env index 73f40bc..9c970b6 100644 --- a/services/ir/.ir.env +++ b/services/ir/.ir.env @@ -30,8 +30,8 @@ NEOFS_IR_TIMERS_DISTRIBUTE_BASIC_INCOME_DIV=4 NEOFS_IR_EMIT_STORAGE_AMOUNT=1000000000 NEOFS_IR_NETMAP_CLEANER_ENABLED=true -NEOFS_IR_CONTRACTS_NEOFS=913c08e0faa2b94516ac7effaea7a9363e492228 -NEOFS_IR_CONTRACTS_PROCESSING=ab239aa64ff6419419390038682a58b3e7dd0e86 +NEOFS_IR_CONTRACTS_NEOFS=f5dba2f67baa3cbb5595f0a7e46adf564ec9407a +NEOFS_IR_CONTRACTS_PROCESSING=37d3232083dd2c1bed76bb28d1c992fc6262f699 NEOFS_IR_MORPH_VALIDATORS=02b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc2 NEOFS_IR_AUDIT_PDP_MAX_SLEEP_INTERVAL=100ms diff --git a/services/storage/.storage.env b/services/storage/.storage.env index ad3a44f..748eac6 100644 --- a/services/storage/.storage.env +++ b/services/storage/.storage.env @@ -35,7 +35,7 @@ NEOFS_MORPH_RPC_ENDPOINT_0_ADDRESS=ws://morph-chain:30333/ws #NEOFS_APICLIENT_DIAL_TIMEOUT=1m # Common storage node attribute -NEOFS_NODE_ATTRIBUTE_0=User-Agent:NeoFS\/0.26 +NEOFS_NODE_ATTRIBUTE_0=User-Agent:NeoFS\/0.33 NEOFS_NODE_NOTIFICATION_ENABLED=true NEOFS_NODE_NOTIFICATION_ENDPOINT=tls://nats.neofs.devenv:4222 From f8d2708e1ac018a69bc286f71ab9da2d0b5037f6 Mon Sep 17 00:00:00 2001 From: Angira Kekteeva Date: Fri, 7 Oct 2022 14:30:19 +0400 Subject: [PATCH 009/149] [#172] Makefile: Fix help regex Made it print targets with / symbol Signed-off-by: Angira Kekteeva --- help.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/help.mk b/help.mk index ab30ca9..97f2667 100644 --- a/help.mk +++ b/help.mk @@ -8,4 +8,4 @@ help: @echo '' @echo ' Targets:' @echo '' - @awk '/^#/{ comment = substr($$0,3) } comment && /^[a-zA-Z][a-zA-Z0-9._-]+ ?:/{ print " ", $$1, comment }' $(MAKEFILE_LIST) | column -t -s ':' | grep -v 'IGNORE' | sort -u + @awk '/^#/{ comment = substr($$0,3) } comment && /^[a-zA-Z][a-zA-Z0-9._/-]+ ?:/{ print " ", $$1, comment }' $(MAKEFILE_LIST) | column -t -s ':' | grep -v 'IGNORE' | sort -u From b0087a53574f0e995073c4cf829d91747821ac5e Mon Sep 17 00:00:00 2001 From: Angira Kekteeva Date: Wed, 12 Oct 2022 18:04:37 +0400 Subject: [PATCH 010/149] [#172] Makefile: Fix comments of targets Signed-off-by: Angira Kekteeva --- Makefile | 5 +++-- services/chain/artifacts.mk | 1 + services/ir/artifacts.mk | 4 +++- services/morph_chain/artifacts.mk | 3 +++ services/nats/artifacts.mk | 2 +- services/storage/artifacts.mk | 4 ++-- services/storage/prepare.mk | 4 +++- 7 files changed, 16 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index d3dc47b..558765d 100644 --- a/Makefile +++ b/Makefile @@ -52,7 +52,7 @@ pull: $(foreach SVC, $(PULL_SVCS), $(shell cd services/$(SVC) && docker-compose pull)) @: -# Get all services artifacs +# Get all services artifacts .PHONY: get get: $(foreach SVC, $(GET_SVCS), get.$(SVC)) @: @@ -109,6 +109,7 @@ down/bootstrap: down/%: @docker-compose -f services/$*/docker-compose.yml down +# Generate changes for /etc/hosts .PHONY: vendor/hosts .ONESHELL: vendor/hosts: @@ -122,7 +123,7 @@ vendor/hosts: done < $${file}; done > $@ -# Display changes for /etc/hosts +# Generate and display changes for /etc/hosts .PHONY: hosts hosts: vendor/hosts @cat vendor/hosts diff --git a/services/chain/artifacts.mk b/services/chain/artifacts.mk index 535b867..26548de 100644 --- a/services/chain/artifacts.mk +++ b/services/chain/artifacts.mk @@ -1,4 +1,5 @@ # Download privnet chain dump with pre-deployed NeoFS contracts + get.chain: CHAIN_DUMP_NAME=devenv.dump. get.chain: CHAIN_PATH?= get.chain: diff --git a/services/ir/artifacts.mk b/services/ir/artifacts.mk index fa009b1..d85afcd 100644 --- a/services/ir/artifacts.mk +++ b/services/ir/artifacts.mk @@ -1,10 +1,11 @@ -# Get NeoFS LOCODE database +# Get NeoFS IR artifacts (LOCODE database and NeoFS CLI) LOCODE_DB_ARCHIVE_PATH=./vendor LOCODE_DB_ARCHIVE_FILE=locode_db.gz get.ir: get.locode get.cli +# Get NeoFS LOCODE database get.locode: LOCODE_DB_PATH?= get.locode: @mkdir -p ${LOCODE_DB_ARCHIVE_PATH} @@ -21,6 +22,7 @@ endif gzip -dfk ${LOCODE_DB_ARCHIVE_PATH}/${LOCODE_DB_ARCHIVE_FILE} +# Download NeoFS CLI .ONESHELL: get.cli: NEOFS_CLI_FILE=./vendor/neofs-cli get.cli: NEOFS_CLI_ARCHIVE_FILE=${NEOFS_CLI_FILE}.tar.gz diff --git a/services/morph_chain/artifacts.mk b/services/morph_chain/artifacts.mk index 35efdec..be6febf 100644 --- a/services/morph_chain/artifacts.mk +++ b/services/morph_chain/artifacts.mk @@ -1,6 +1,8 @@ # Download NeoFS Contracts and Adm tool to deploy environment + get.morph_chain: get.contracts get.adm +# Download NeoFS Contracts get.contracts: NEOFS_CONTRACTS_DEST=./vendor/contracts get.contracts: NEOFS_CONTRACTS_ARCHIVE=neofs-contracts.tar.gz get.contracts: @@ -16,6 +18,7 @@ else @cp -r ${NEOFS_CONTRACTS_PATH}/* ${NEOFS_CONTRACTS_DEST} endif +# Download NeoFS ADM tool get.adm: NEOFS_ADM_DEST=./vendor/neofs-adm get.adm: NEOFS_ADM_ARCHIVE=neofs-adm.tar.gz get.adm: diff --git a/services/nats/artifacts.mk b/services/nats/artifacts.mk index c68c57e..83a3bc9 100644 --- a/services/nats/artifacts.mk +++ b/services/nats/artifacts.mk @@ -1,4 +1,4 @@ -# Create new tls certs +# Create new TLS certs for NATS server and clients NATS_DIR=$(abspath services/nats) diff --git a/services/storage/artifacts.mk b/services/storage/artifacts.mk index 4866a32..b78c71a 100644 --- a/services/storage/artifacts.mk +++ b/services/storage/artifacts.mk @@ -1,11 +1,11 @@ -# Create new tls certs +# Create new TLS certs to NeoFS node CURRENT_DIR=$(dir $(abspath $(lastword $(MAKEFILE_LIST)))) STORAGE_DIR=$(patsubst %/,%,$(CURRENT_DIR)) SSL_CONFIG := $(shell mktemp) get.storage: - @echo "⇒ Creating tls certs to NeoFS node" + @echo "⇒ Creating TLS certs to NeoFS node" @(echo "[req]"; \ echo "distinguished_name=req"; \ echo "req_extensions=san"; \ diff --git a/services/storage/prepare.mk b/services/storage/prepare.mk index 22c276b..fad47b1 100644 --- a/services/storage/prepare.mk +++ b/services/storage/prepare.mk @@ -1,3 +1,5 @@ +# Add self-signed node TLS certificate to trusted store + prepare.storage: - @echo "Adding self-signed tls certs to trusted store" + @echo "Adding self-signed TLS certs to trusted store" @./bin/addCert.sh From f7ead4b014613d1cee652ea5825c3139a4cc118f Mon Sep 17 00:00:00 2001 From: Angira Kekteeva Date: Wed, 12 Oct 2022 13:57:45 +0400 Subject: [PATCH 011/149] [#172] Makefile: Fix SSL_CONFIG var assignment Signed-off-by: Angira Kekteeva --- services/storage/artifacts.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/storage/artifacts.mk b/services/storage/artifacts.mk index b78c71a..018b9d9 100644 --- a/services/storage/artifacts.mk +++ b/services/storage/artifacts.mk @@ -2,7 +2,7 @@ CURRENT_DIR=$(dir $(abspath $(lastword $(MAKEFILE_LIST)))) STORAGE_DIR=$(patsubst %/,%,$(CURRENT_DIR)) -SSL_CONFIG := $(shell mktemp) +SSL_CONFIG=$(shell mktemp) get.storage: @echo "⇒ Creating TLS certs to NeoFS node" From 5dc86e2aac7881f3edd983c598b81a273c04c7a7 Mon Sep 17 00:00:00 2001 From: Angira Kekteeva Date: Wed, 12 Oct 2022 13:58:25 +0400 Subject: [PATCH 012/149] [#172] Makefile: Add filter to help target Signed-off-by: Angira Kekteeva --- help.mk | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/help.mk b/help.mk index 97f2667..7964236 100644 --- a/help.mk +++ b/help.mk @@ -1,5 +1,7 @@ .PHONY: help +HELP_MAKEFILE_LIST=$(filter-out %/artifacts.mk, $(MAKEFILE_LIST)) + # Show this help prompt help: @echo ' Usage:' @@ -8,4 +10,4 @@ help: @echo '' @echo ' Targets:' @echo '' - @awk '/^#/{ comment = substr($$0,3) } comment && /^[a-zA-Z][a-zA-Z0-9._/-]+ ?:/{ print " ", $$1, comment }' $(MAKEFILE_LIST) | column -t -s ':' | grep -v 'IGNORE' | sort -u + @awk '/^#/{ comment = substr($$0,3) } comment && /^[a-zA-Z][a-zA-Z0-9._/-]+ ?:/{ print " ", $$1, comment }' $(HELP_MAKEFILE_LIST) | column -t -s ':' | grep -v 'IGNORE' | sort -u From f55dc15476b681b2a8149b1fe9a4ed11d8bc1810 Mon Sep 17 00:00:00 2001 From: Angira Kekteeva Date: Sat, 22 Oct 2022 02:09:27 +0400 Subject: [PATCH 013/149] [#172] Fix grep: warning: stray \ before # https://lists.gnu.org/archive/html/info-gnu/2022-09/msg00001.html Signed-off-by: Angira Kekteeva --- Makefile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 558765d..672f931 100644 --- a/Makefile +++ b/Makefile @@ -20,12 +20,12 @@ include services/*/artifacts.mk include services/*/prepare.mk # List of services to run -START_SVCS = $(shell cat .services | grep -v \\\#) -START_BASIC = $(shell cat .basic_services | grep -v \\\#) -START_BOOTSTRAP = $(shell cat .bootstrap_services | grep -v \\\#) -STOP_SVCS = $(shell tac .services | grep -v \\\#) -STOP_BASIC = $(shell tac .basic_services | grep -v \\\#) -STOP_BOOTSTRAP = $(shell tac .bootstrap_services | grep -v \\\#) +START_SVCS = $(shell cat .services | grep -v '#') +START_BASIC = $(shell cat .basic_services | grep -ve '#') +START_BOOTSTRAP = $(shell cat .bootstrap_services | grep -v '#') +STOP_SVCS = $(shell tac .services | grep -v '#') +STOP_BASIC = $(shell tac .basic_services | grep -v '#') +STOP_BOOTSTRAP = $(shell tac .bootstrap_services | grep -v '#') # Enabled services dirs ENABLED_SVCS_DIRS = $(shell echo "${START_BOOTSTRAP} ${START_BASIC} ${START_SVCS}" | sed 's|[^ ]* *|./services/&|g') From 9d2cc54f40db0b3167d53b5e14acb4e6111e17c8 Mon Sep 17 00:00:00 2001 From: Angira Kekteeva Date: Thu, 27 Oct 2022 13:53:27 +0400 Subject: [PATCH 014/149] [#231] Makefile: Fix SSL_CONFIG var assignment Signed-off-by: Angira Kekteeva --- services/storage/artifacts.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/storage/artifacts.mk b/services/storage/artifacts.mk index 018b9d9..af80ff1 100644 --- a/services/storage/artifacts.mk +++ b/services/storage/artifacts.mk @@ -2,7 +2,7 @@ CURRENT_DIR=$(dir $(abspath $(lastword $(MAKEFILE_LIST)))) STORAGE_DIR=$(patsubst %/,%,$(CURRENT_DIR)) -SSL_CONFIG=$(shell mktemp) +SSL_CONFIG:=$(shell mktemp) get.storage: @echo "⇒ Creating TLS certs to NeoFS node" From 1a7dcda7f545f18fc56a43ab5b5e7e806e151d75 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 1 Nov 2022 10:54:21 +0300 Subject: [PATCH 015/149] [#234] Update neofs-* to v0.34.0 Signed-off-by: Evgenii Stratonikov --- .env | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.env b/.env index 6441384..528e245 100644 --- a/.env +++ b/.env @@ -14,11 +14,11 @@ NEOGO_VERSION=0.99.4 NEOGO_IMAGE=nspccdev/neo-go # NeoFS InnerRing nodes -IR_VERSION=0.33.0 +IR_VERSION=0.34.0 IR_IMAGE=nspccdev/neofs-ir # NeoFS Storage nodes -NODE_VERSION=0.33.0 +NODE_VERSION=0.34.0 NODE_IMAGE=nspccdev/neofs-storage # NATS Server @@ -42,15 +42,15 @@ COREDNS_VERSION=v016 COREDNS_IMAGE=nspccdev/coredns # NeoFS LOCODE database -LOCODE_DB_URL=https://github.com/nspcc-dev/neofs-locode-db/releases/download/v0.2.1/locode_db.gz +LOCODE_DB_URL=https://github.com/nspcc-dev/neofs-locode-db/releases/download/v0.3.0/locode_db.gz #LOCODE_DB_PATH=/path/to/locode_db # NeoFS CLI binary -NEOFS_CLI_URL=https://github.com/nspcc-dev/neofs-node/releases/download/v0.33.0/neofs-cli-amd64.tar.gz +NEOFS_CLI_URL=https://github.com/nspcc-dev/neofs-node/releases/download/v0.34.0/neofs-cli-amd64.tar.gz #NEOFS_CLI_PATH=/path/to/neofs-cli-binary # NeoFS ADM tool binary -NEOFS_ADM_VERSION=v0.33.0 +NEOFS_ADM_VERSION=v0.34.0 NEOFS_ADM_URL=https://github.com/nspcc-dev/neofs-node/releases/download/${NEOFS_ADM_VERSION}/neofs-adm-amd64.tar.gz #NEOFS_ADM_PATH=/path/to/neofs-adm-binary From a2de0822fad5d0f35ccb035764d45e6eda222bd5 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 1 Nov 2022 10:57:10 +0300 Subject: [PATCH 016/149] [#233] Update gateways to v0.25.0 Signed-off-by: Evgenii Stratonikov --- .env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.env b/.env index 528e245..550e5b7 100644 --- a/.env +++ b/.env @@ -26,7 +26,7 @@ NATS_VERSION=2.7.2 NATS_IMAGE=nats # HTTP Gate -HTTP_GW_VERSION=0.24.0 +HTTP_GW_VERSION=0.25.0 HTTP_GW_IMAGE=nspccdev/neofs-http-gw # REST Gate @@ -34,7 +34,7 @@ REST_GW_VERSION=0.4.0 REST_GW_IMAGE=nspccdev/neofs-rest-gw # S3 Gate -S3_GW_VERSION=0.24.0 +S3_GW_VERSION=0.25.0 S3_GW_IMAGE=nspccdev/neofs-s3-gw # Coredns From f7fb6d5142ee69755cb240e481cd4438986931a2 Mon Sep 17 00:00:00 2001 From: anastasia prasolova Date: Thu, 3 Nov 2022 15:13:05 +0300 Subject: [PATCH 017/149] [#235]: Suppress output of certificates generation on `make up` Signed-off-by: anastasia prasolova --- Makefile | 5 +++-- services/nats/artifacts.mk | 2 +- services/storage/artifacts.mk | 17 ++--------------- services/storage/generate_cert.sh | 26 ++++++++++++++++++++++++++ 4 files changed, 32 insertions(+), 18 deletions(-) create mode 100755 services/storage/generate_cert.sh diff --git a/Makefile b/Makefile index 672f931..6e0ca96 100644 --- a/Makefile +++ b/Makefile @@ -75,8 +75,9 @@ up/basic: up/bootstrap .PHONY: up/bootstrap up/bootstrap: get vendor/hosts @$(foreach SVC, $(START_BOOTSTRAP), $(shell docker-compose -f services/$(SVC)/docker-compose.yml up -d)) - @./vendor/neofs-adm --config neofs-adm.yml morph init --alphabet-wallets ./services/ir --contracts vendor/contracts || exit 1 - @for f in ./services/storage/wallet*.json; do echo "Transfer GAS to wallet $${f}" && ./vendor/neofs-adm -c neofs-adm.yml morph refill-gas --storage-wallet $${f} --gas 10.0 --alphabet-wallets services/ir || exit 1; done + @source ./bin/helper.sh + @./vendor/neofs-adm --config neofs-adm.yml morph init --alphabet-wallets ./services/ir --contracts vendor/contracts || die "Failed to initialize Alphabet wallets" + @for f in ./services/storage/wallet*.json; do echo "Transfer GAS to wallet $${f}" && ./vendor/neofs-adm -c neofs-adm.yml morph refill-gas --storage-wallet $${f} --gas 10.0 --alphabet-wallets services/ir || die "Failed to transfer GAS to alphabet wallets"; done @echo "NeoFS sidechain environment is deployed" # Build up certain service diff --git a/services/nats/artifacts.mk b/services/nats/artifacts.mk index 83a3bc9..b66cebf 100644 --- a/services/nats/artifacts.mk +++ b/services/nats/artifacts.mk @@ -4,4 +4,4 @@ NATS_DIR=$(abspath services/nats) get.nats: @echo "⇒ Creating certs for NATS server and clients" - ${NATS_DIR}/generate_cert.sh ${LOCAL_DOMAIN} + ${NATS_DIR}/generate_cert.sh ${LOCAL_DOMAIN} > /dev/null diff --git a/services/storage/artifacts.mk b/services/storage/artifacts.mk index af80ff1..7132ac1 100644 --- a/services/storage/artifacts.mk +++ b/services/storage/artifacts.mk @@ -1,20 +1,7 @@ # Create new TLS certs to NeoFS node -CURRENT_DIR=$(dir $(abspath $(lastword $(MAKEFILE_LIST)))) -STORAGE_DIR=$(patsubst %/,%,$(CURRENT_DIR)) -SSL_CONFIG:=$(shell mktemp) +STORAGE_DIR=$(abspath services/storage) get.storage: @echo "⇒ Creating TLS certs to NeoFS node" - @(echo "[req]"; \ - echo "distinguished_name=req"; \ - echo "req_extensions=san"; \ - echo "[san]"; \ - echo "subjectAltName=DNS:s04.${LOCAL_DOMAIN}") > ${SSL_CONFIG} - @echo $(test -e "${STORAGE_DIR}/s04tls.key" && echo true) - @if [ ! -e "${STORAGE_DIR}/s04tls.key" ]; then \ - openssl req -new -newkey rsa:4096 -x509 -sha256 -days 365 -nodes \ - -subj "/C=RU/ST=SPB/L=St.Petersburg/O=NSPCC/OU=NSPCC/CN=s04.${LOCAL_DOMAIN}" \ - -keyout "${STORAGE_DIR}/s04tls.key" -out "${STORAGE_DIR}/s04tls.crt" \ - -extensions san -config "${SSL_CONFIG}" ; \ - fi + ${STORAGE_DIR}/generate_cert.sh ${LOCAL_DOMAIN} > /dev/null diff --git a/services/storage/generate_cert.sh b/services/storage/generate_cert.sh new file mode 100755 index 0000000..ddc7aa1 --- /dev/null +++ b/services/storage/generate_cert.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +source bin/helper.sh + +WORKDIR=$(dirname "$0") +LOCAL_DOMAIN=$1 +SSL_CONFIG=$(mktemp) +CERT="${WORKDIR}/s04tls.crt" +KEY="${WORKDIR}/s04tls.key" + + +if [[ ! -f ${CERT} ]]; then + ( + echo "[req]"; \ + echo "distinguished_name=req"; \ + echo "req_extensions=san"; \ + echo "[san]"; \ + echo "subjectAltName=DNS:s04.${LOCAL_DOMAIN}" + ) > ${SSL_CONFIG} + + openssl req -new -newkey rsa:4096 -x509 -sha256 -days 365 -nodes \ + -subj "/C=RU/ST=SPB/L=St.Petersburg/O=NSPCC/OU=NSPCC/CN=s04.${LOCAL_DOMAIN}" \ + -keyout "${KEY}" -out "${CERT}" -extensions san -config "${SSL_CONFIG}" &> /dev/null || { + die "Failed to generate SSL certificate for s04" + } +fi From 79a11641ed1b2bc7d9c41104569eae20ac3953cd Mon Sep 17 00:00:00 2001 From: anastasia prasolova Date: Mon, 7 Nov 2022 12:44:09 +0300 Subject: [PATCH 018/149] [#236]: Hardcode `morph_chain` RPC bind address Signed-off-by: anastasia prasolova --- services/morph_chain/protocol.privnet.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/services/morph_chain/protocol.privnet.yml b/services/morph_chain/protocol.privnet.yml index 6992ced..39155c8 100644 --- a/services/morph_chain/protocol.privnet.yml +++ b/services/morph_chain/protocol.privnet.yml @@ -28,6 +28,7 @@ ApplicationConfiguration: AttemptConnPeers: 5 MinPeers: 0 RPC: + Address: 192.168.130.90 Enabled: true SessionEnabled: true EnableCORSWorkaround: false From d1b15bbbedf646adc0ee2fa59b114bb7d9c12f1e Mon Sep 17 00:00:00 2001 From: Elizaveta Chichindaeva Date: Fri, 28 Oct 2022 16:08:41 +0300 Subject: [PATCH 019/149] FIX: current NEOFS_IR_CONTRACTS_NEOFS Signed-off-by: Elizaveta Chichindaeva --- services/ir/.ir.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/ir/.ir.env b/services/ir/.ir.env index 73f40bc..01dbaaf 100644 --- a/services/ir/.ir.env +++ b/services/ir/.ir.env @@ -30,7 +30,7 @@ NEOFS_IR_TIMERS_DISTRIBUTE_BASIC_INCOME_DIV=4 NEOFS_IR_EMIT_STORAGE_AMOUNT=1000000000 NEOFS_IR_NETMAP_CLEANER_ENABLED=true -NEOFS_IR_CONTRACTS_NEOFS=913c08e0faa2b94516ac7effaea7a9363e492228 +NEOFS_IR_CONTRACTS_NEOFS=b797c965c186811455430a7b39b81beb1dc3772a NEOFS_IR_CONTRACTS_PROCESSING=ab239aa64ff6419419390038682a58b3e7dd0e86 NEOFS_IR_MORPH_VALIDATORS=02b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc2 From d8e825443ee73cc5fda38f83f11e8443a111ea1b Mon Sep 17 00:00:00 2001 From: anastasia prasolova Date: Mon, 14 Nov 2022 15:15:00 +0300 Subject: [PATCH 020/149] [#238]: Remove interactive password input from setup scripts Signed-off-by: anastasia prasolova --- bin/config.sh | 17 +++++++++-------- bin/deposit.sh | 17 +++++++++-------- bin/resolve.sh | 2 +- bin/tick.sh | 21 ++++++++++----------- services/chain/config.yml | 4 ++++ services/chain/docker-compose.yml | 1 + 6 files changed, 34 insertions(+), 28 deletions(-) create mode 100644 services/chain/config.yml diff --git a/bin/config.sh b/bin/config.sh index 0f88192..f63a49c 100755 --- a/bin/config.sh +++ b/bin/config.sh @@ -1,19 +1,20 @@ #!/usr/bin/env bash +echo "Running bin/config.sh" + # Source env settings . .env . services/ir/.ir.env source bin/helper.sh # NeoGo binary path. -NEOGO="${NEOGO:-docker exec -it main_chain neo-go}" +NEOGO="${NEOGO:-docker exec main_chain neo-go}" # Wallet files to change config value WALLET="${WALLET:-services/chain/node-wallet.json}" -WALLET_IMG="${WALLET_IMG:-wallets/node-wallet.json}" -# Wallet password that would be entered automatically; '-' means no password -PASSWD="one" -NETMAP_ADDR=$(bin/resolve.sh netmap.neofs) +CONFIG_IMG="${CONFIG_IMG:-/wallets/config.yml}" + +NETMAP_ADDR=$(bin/resolve.sh netmap.neofs) || die "Failed to resolve 'netmap.neofs' domain name" # NeoFS configuration record: variable type [string|int|etc], # key is a string and value is a constant of given type @@ -36,9 +37,9 @@ fi echo "Changing ${KEY} configration value to ${VALUE}" # shellcheck disable=SC2086 -./bin/passwd.exp ${PASSWD} ${NEOGO} contract invokefunction \ - -w ${WALLET_IMG} \ - -a ${ADDR} \ +${NEOGO} contract invokefunction \ + --wallet-config ${CONFIG_IMG} \ + -a ${ADDR} --force \ -r http://morph-chain.${LOCAL_DOMAIN}:30333 \ ${NETMAP_ADDR} \ setConfig bytes:beefcafe \ diff --git a/bin/deposit.sh b/bin/deposit.sh index 95fbc3e..07610a5 100755 --- a/bin/deposit.sh +++ b/bin/deposit.sh @@ -1,16 +1,17 @@ #!/usr/bin/env bash +echo "Running bin/deposit.sh" + # Source env settings . .env . services/ir/.ir.env source bin/helper.sh # NeoGo binary path. -NEOGO="${NEOGO:-docker exec -it main_chain neo-go}" +NEOGO="${NEOGO:-docker exec main_chain neo-go}" # Wallet file to use for deposit GAS from -WALLET="${WALLET:-wallets/wallet.json}" -# Wallet password that would be entered automatically; '-' means no password -PASSWD="-" +WALLET="${WALLET:-services/chain/node-wallet.json}" +CONFIG="${CONFIG:-/wallets/config.yml}" # How much GAS to deposit. First cli argument or 50 by default DEPOSIT="${1:-50}" @@ -25,10 +26,10 @@ CONTRACT_ADDR=$(${NEOGO} util convert "${NEOFS_IR_CONTRACTS_NEOFS}" \ # Make deposit # shellcheck disable=SC2086 -./bin/passwd.exp ${PASSWD} ${NEOGO} wallet nep17 transfer \ - -w ${WALLET} \ +${NEOGO} wallet nep17 transfer \ + --wallet-config ${CONFIG} \ -r http://main-chain.${LOCAL_DOMAIN}:30333 \ - --from ${ADDR} \ + --from ${ADDR} --force \ --to ${CONTRACT_ADDR} \ --token GAS \ - --amount ${DEPOSIT} + --amount ${DEPOSIT} || die "Cannot transfer GAS to NeoFS contract" diff --git a/bin/resolve.sh b/bin/resolve.sh index 36a1ac2..9aba65b 100755 --- a/bin/resolve.sh +++ b/bin/resolve.sh @@ -5,7 +5,7 @@ source bin/helper.sh # NeoGo binary path. -NEOGO="${NEOGO:-docker exec -t morph_chain neo-go}" +NEOGO="${NEOGO:-docker exec morph_chain neo-go}" # NNS contract script hash output=$(curl -s --data '{ "id": 1, "jsonrpc": "2.0", "method": "getcontractstate", "params": [1] }' \ "http://morph-chain.${LOCAL_DOMAIN}:30333/") \ diff --git a/bin/tick.sh b/bin/tick.sh index ede13c2..8c89a90 100755 --- a/bin/tick.sh +++ b/bin/tick.sh @@ -1,20 +1,18 @@ #!/usr/bin/env bash +echo "Running bin/tick.sh" + # Source env settings . .env . services/ir/.ir.env source bin/helper.sh # NeoGo binary path. -NEOGO="${NEOGO:-docker exec -it main_chain neo-go}" -NEOGO_NONINTERACTIVE="${NEOGO_NONINTERACTIVE:-docker exec -t main_chain neo-go}" +NEOGO="${NEOGO:-docker exec main_chain neo-go}" # Wallet files to change config value WALLET="${WALLET:-services/chain/node-wallet.json}" -WALLET_IMG="${WALLET_IMG:-wallets/node-wallet.json}" - -# Wallet password that would be entered automatically; '-' means no password -PASSWD="one" +CONFIG_IMG="${CONFIG_IMG:-/wallets/config.yml}" # Internal variables if [[ -z "${NEOFS_NOTARY_DISABLED}" ]]; then @@ -30,7 +28,7 @@ BLOCK_DURATION=$(grep SecondsPerBlock < "$SIDECHAIN_PROTO" | awk '{print $2}') \ NETMAP_ADDR=$(bin/resolve.sh netmap.neofs) || die "Cannot resolve netmap.neofs" # Fetch current epoch value -EPOCH=$(${NEOGO_NONINTERACTIVE} contract testinvokefunction \ +EPOCH=$(${NEOGO} contract testinvokefunction \ -r "http://morph-chain.${LOCAL_DOMAIN}:30333" "${NETMAP_ADDR}" epoch \ | grep 'value' | awk -F'"' '{ print $4 }') \ || die "Cannot fetch epoch from netmap contract" @@ -38,12 +36,13 @@ EPOCH=$(${NEOGO_NONINTERACTIVE} contract testinvokefunction \ echo "Updating NeoFS epoch to $((EPOCH+1))" # shellcheck disable=SC2086 -./bin/passwd.exp ${PASSWD} ${NEOGO} contract invokefunction \ - -w ${WALLET_IMG} \ - -a ${ADDR} \ +${NEOGO} contract invokefunction \ + --wallet-config ${CONFIG_IMG} \ + -a ${ADDR} --force \ -r http://morph-chain.${LOCAL_DOMAIN}:30333 \ ${NETMAP_ADDR} \ - newEpoch int:$((EPOCH+1)) -- ${ADDR}:Global + newEpoch int:$((EPOCH+1)) -- ${ADDR}:Global \ + || die "Cannot increment an epoch" # Wait one Morph block to ensure the transaction broadcasted # shellcheck disable=SC2086 diff --git a/services/chain/config.yml b/services/chain/config.yml new file mode 100644 index 0000000..7b4bb29 --- /dev/null +++ b/services/chain/config.yml @@ -0,0 +1,4 @@ +--- + +Path: "/wallets/node-wallet.json" +Password: "one" diff --git a/services/chain/docker-compose.yml b/services/chain/docker-compose.yml index b3e0822..3fe1698 100644 --- a/services/chain/docker-compose.yml +++ b/services/chain/docker-compose.yml @@ -20,6 +20,7 @@ services: - ./../../vendor/chain.gz:/chain.gz - ./protocol.privnet.yml:/config/protocol.privnet.yml - ./node-wallet.json:/wallets/node-wallet.json + - ./config.yml:/wallets/config.yml - ./../../vendor/hosts:/etc/hosts - ./../../wallets/wallet.json:/wallets/wallet.json From 1099cdcef885efbbec56bc869dc04a64696ada23 Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Thu, 24 Nov 2022 15:53:06 +0300 Subject: [PATCH 021/149] [#242] gitignore: Add VSCode settings Signed-off-by: Pavel Karpy --- .gitignore | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.gitignore b/.gitignore index d5835aa..36e495d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,18 @@ +# IDE .idea +.vscode + +# Vendoring vendor + +temp tmp + .secrets sites/* !sites/.gitkeep + +# Runtime generation keys services/storage/*tls.crt services/storage/*tls.key services/nats/*.pem From 6838092a503fbeadfe4d2b2c2a52b2de43540fa5 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 21 Dec 2022 14:39:29 +0300 Subject: [PATCH 022/149] [TrueCloudLab#1] Use frostfs gates Signed-off-by: Denis Kirillov --- .env | 12 ++++++------ services/http_gate/.http.env | 2 +- services/rest_gate/docker-compose.yml | 10 +++++----- services/s3_gate/docker-compose.yml | 6 +++--- services/s3_gate/wallet.json | 2 +- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.env b/.env index 550e5b7..01ecbc3 100644 --- a/.env +++ b/.env @@ -26,16 +26,16 @@ NATS_VERSION=2.7.2 NATS_IMAGE=nats # HTTP Gate -HTTP_GW_VERSION=0.25.0 -HTTP_GW_IMAGE=nspccdev/neofs-http-gw +HTTP_GW_VERSION=6abd500b +HTTP_GW_IMAGE=truecloudlab/frostfs-http-gw # REST Gate -REST_GW_VERSION=0.4.0 -REST_GW_IMAGE=nspccdev/neofs-rest-gw +REST_GW_VERSION=c9c85e90 +REST_GW_IMAGE=truecloudlab/frostfs-rest-gw # S3 Gate -S3_GW_VERSION=0.25.0 -S3_GW_IMAGE=nspccdev/neofs-s3-gw +S3_GW_VERSION=000d9ed4 +S3_GW_IMAGE=truecloudlab/frostfs-s3-gw # Coredns COREDNS_VERSION=v016 diff --git a/services/http_gate/.http.env b/services/http_gate/.http.env index 97f6312..3e1b0dd 100644 --- a/services/http_gate/.http.env +++ b/services/http_gate/.http.env @@ -1,4 +1,4 @@ -HTTP_GW_LISTEN_ADDRESS=0.0.0.0:80 +HTTP_GW_SERVER_0_ADDRESS=0.0.0.0:80 HTTP_GW_LOGGER_LEVEL=info diff --git a/services/rest_gate/docker-compose.yml b/services/rest_gate/docker-compose.yml index 75b0b7a..478d25b 100644 --- a/services/rest_gate/docker-compose.yml +++ b/services/rest_gate/docker-compose.yml @@ -20,11 +20,11 @@ services: environment: - REST_GW_WALLET_PATH=/wallet.json - REST_GW_WALLET_PASSPHRASE=one - - REST_GW_LISTEN_ADDRESS=0.0.0.0:8090 - - REST_GW_PEERS_0_ADDRESS=s01.${LOCAL_DOMAIN}:8080 - - REST_GW_PEERS_1_ADDRESS=s02.${LOCAL_DOMAIN}:8080 - - REST_GW_PEERS_2_ADDRESS=s03.${LOCAL_DOMAIN}:8080 - - REST_GW_PEERS_3_ADDRESS=s04.${LOCAL_DOMAIN}:8080 + - REST_GW_SERVER_LISTEN_ADDRESS=0.0.0.0:8090 + - REST_GW_POOL_PEERS_0_ADDRESS=s01.${LOCAL_DOMAIN}:8080 + - REST_GW_POOL_PEERS_1_ADDRESS=s02.${LOCAL_DOMAIN}:8080 + - REST_GW_POOL_PEERS_2_ADDRESS=s03.${LOCAL_DOMAIN}:8080 + - REST_GW_POOL_PEERS_3_ADDRESS=s04.${LOCAL_DOMAIN}:8080 networks: rest_gate_int: diff --git a/services/s3_gate/docker-compose.yml b/services/s3_gate/docker-compose.yml index 82efcaa..8cce4ff 100644 --- a/services/s3_gate/docker-compose.yml +++ b/services/s3_gate/docker-compose.yml @@ -24,9 +24,9 @@ services: environment: - S3_GW_WALLET_PATH=/wallet.json - S3_GW_WALLET_PASSPHRASE=s3 - - S3_GW_TLS_KEY_FILE=/tls.key - - S3_GW_TLS_CERT_FILE=/tls.crt - - S3_GW_LISTEN_ADDRESS=s3.${LOCAL_DOMAIN}:8080 + - S3_GW_SERVER_0_TLS_KEY_FILE=/tls.key + - S3_GW_SERVER_0_TLS_CERT_FILE=/tls.crt + - S3_GW_SERVER_0_ADDRESS=s3.${LOCAL_DOMAIN}:8080 - S3_GW_LISTEN_DOMAINS=s3.${LOCAL_DOMAIN} - S3_GW_TREE_SERVICE=s01.${LOCAL_DOMAIN}:8080 - S3_GW_PEERS_0_ADDRESS=s01.${LOCAL_DOMAIN}:8080 diff --git a/services/s3_gate/wallet.json b/services/s3_gate/wallet.json index 8d3d4c2..ff1adfd 100644 --- a/services/s3_gate/wallet.json +++ b/services/s3_gate/wallet.json @@ -5,7 +5,7 @@ { "address": "NUUb82KR2JrVByHs2YSKgtK29gKnF5q6Vt", "key": "6PYSPET41jKtqie2cfkqgy7q9ueeucH8bX9Gotm2HwdNEDptj6aBe8dDg8", - "label": "NeoFS S3 Gate", + "label": "FrostFS S3 Gate", "contract": { "script": "DCEDE7GsOoB24VWn55eyTwtlDMytWUHqWdfP1RoCSosqBr9BVuezJw==", "parameters": [ From ffe5012ab6e82553973a9f6322d4f86cded292dc Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 21 Dec 2022 14:59:19 +0300 Subject: [PATCH 023/149] [TrueCloudLab#1] Update docs Signed-off-by: Denis Kirillov --- .github/logo.svg | 129 ---------------------------------------------- CONTRIBUTING.md | 18 +++---- README.md | 33 ++++++------ docs/basenet.md | 4 +- docs/chain.md | 38 +++++++------- docs/faq.md | 14 ++--- docs/http_gate.md | 16 +++--- docs/ir.md | 6 +-- docs/morph.md | 34 ++++++------ docs/notary.md | 2 +- docs/rest_gate.md | 12 ++--- docs/s3_gate.md | 8 +-- docs/storage.md | 4 +- 13 files changed, 93 insertions(+), 225 deletions(-) delete mode 100644 .github/logo.svg diff --git a/.github/logo.svg b/.github/logo.svg deleted file mode 100644 index b4da076..0000000 --- a/.github/logo.svg +++ /dev/null @@ -1,129 +0,0 @@ - - - - - - - - - - - - image/svg+xml - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7f6b65f..f3ae1f1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3,8 +3,8 @@ First, thank you for contributing! We love and encourage pull requests from everyone. Please follow the guidelines: -- Check the open [issues](https://github.com/nspcc-dev/neofs-dev-env/issues) and - [pull requests](https://github.com/nspcc-dev/neofs-dev-env/pulls) for existing +- Check the open [issues](https://github.com/TrueCloudLab/frostfs-dev-env/issues) and + [pull requests](https://github.com/TrueCloudLab/frostfs-dev-env/pulls) for existing discussions. - Open an issue first, to discuss a new feature or enhancement. @@ -21,23 +21,23 @@ everyone. Please follow the guidelines: ## Development Workflow -Start by forking the `neofs-dev-env` repository, make changes in a branch and then +Start by forking the `frostfs-dev-env` repository, make changes in a branch and then send a pull request. We encourage pull requests to discuss code changes. Here are the steps in details: -### Setup your GitHub Repository -Fork [NeoFS node upstream](https://github.com/nspcc-dev/neofs-dev-env/fork) source +### Set up your GitHub Repository +Fork [FrostFS node upstream](https://github.com/TrueCloudLab/frostfs-dev-env/fork) source repository to your own personal repository. Copy the URL of your fork (you will need it for the `git clone` command below). ```sh -$ git clone https://github.com/nspcc-dev/neofs-dev-env +$ git clone https://github.com/TrueCloudLab/frostfs-dev-env ``` ### Set up git remote as ``upstream`` ```sh -$ cd neofs-dev-env -$ git remote add upstream https://github.com/nspcc-dev/neofs-dev-env +$ cd frostfs-dev-env +$ git remote add upstream https://github.com/TrueCloudLab/frostfs-dev-env $ git fetch upstream $ git merge upstream/master ... @@ -103,7 +103,7 @@ contributors". To sign your work, just add a line like this at the end of your commit message: ``` -Signed-off-by: Samii Sakisaka +Signed-off-by: Samii Sakisaka ``` This can easily be done with the `--signoff` option to `git commit`. diff --git a/README.md b/README.md index b620273..42cb644 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,11 @@

-NeoFS -

-

- NeoFS local Development and Testing environment + FrostFS local Development and Testing environment

--- ## Overview -Tools to set up local NeoFS network and N3 privnets. Devenv, for short. +Tools to set up local FrostFS network and N3 privnets. Devenv, for short. ## Prerequisites @@ -27,7 +24,7 @@ Make sure you have installed all of the following prerequisites on your machine: Clone repo: ``` -$ git clone https://github.com/nspcc-dev/neofs-dev-env.git +$ git clone https://github.com/TrueCloudLab/frostfs-dev-env.git ``` Run next commands from project's root: @@ -42,11 +39,11 @@ been added already, there is no need to run it separately. ``` $ make hosts -192.168.130.10 bastion.neofs.devenv -192.168.130.50 main-chain.neofs.devenv -192.168.130.61 ir01.neofs.devenv +192.168.130.10 bastion.frostfs.devenv +192.168.130.50 main-chain.frostfs.devenv +192.168.130.61 ir01.frostfs.devenv ... -192.168.130.74 s04.neofs.devenv +192.168.130.74 s04.frostfs.devenv ``` This command shows addresses and hostnames of components. Add `make hosts` @@ -58,7 +55,7 @@ $ make up ``` When all services are up, you need to make GAS deposit for test wallet to be -able to pay for NeoFS operations. Test wallet is located in +able to pay for FrostFS operations. Test wallet is located in `wallets/wallet.json` with the corresponding key in `wallets/wallet.key`. The password is empty. @@ -68,16 +65,16 @@ password > fa6ba62bffb04030d303dcc95bda7413e03aa3c7e6ca9c2f999d65db9ec9b82c ``` -Also you should add self-signed node (`s04.neofs.devenv`) certificate to trusted +Also, you should add self-signed node (`s04.frostfs.devenv`) certificate to trusted store (default location might be changed using `CA_CERTS_TRUSTED_STORE` -variable). This step is required for client services (neofs-http-gw, -neofs-s3-gw) to interact with the node: +variable). This step is required for client services (frostfs-http-gw, +frostfs-s3-gw) to interact with the node: ``` $ sudo make prepare.storage ``` -Change NeoFS global configuration values with `make update.*` commands. The +Change FrostFS global configuration values with `make update.*` commands. The password of inner ring wallet is `one`. See examples in `make help`. ``` @@ -85,7 +82,7 @@ $ make update.epoch_duration val=30 Changing EpochDuration configration value to 30 Enter account NNudMSGzEoktFzdYGYoNb3bzHzbmM1genF password > Sent invocation transaction dbb8c1145b6d10f150135630e13bb0dc282023163f5956c6945a60db0cb45cb0 -Updating NeoFS epoch to 2 +Updating FrostFS epoch to 2 Enter account NNudMSGzEoktFzdYGYoNb3bzHzbmM1genF password > Sent invocation transaction 0e6eb5e190f36332e5e5f4e866c7e100826e285fd949e11c085e15224f343ba6 ``` @@ -121,9 +118,9 @@ You can find more information on each service in `docs` directory. Maybe you will find the answer for your question in [F.A.Q.](docs/faq.md) -## Using NeoFS Admin Tool in `dev-env` +## Using FrostFS Admin Tool in `dev-env` -Devenv supports NeoFS network management via [neofs-adm](https://github.com/nspcc-dev/neofs-node/tree/master/cmd/neofs-adm). +Devenv supports FrostFS network management via [frostfs-adm](https://github.com/TrueCloudLab/frostfs-node/tree/master/cmd/frostfs-adm). `services/ir` contains the Alphabet wallet in a proper format, specify it with `--alphabet-wallets` flag. diff --git a/docs/basenet.md b/docs/basenet.md index 7883889..0ce1f5e 100644 --- a/docs/basenet.md +++ b/docs/basenet.md @@ -7,7 +7,7 @@ host machine, so all programs running on host can connect to services exposed to ## .env settings -### LOCAL_DOMAIN=neofs.devenv +### LOCAL_DOMAIN=frostfs.devenv Domain to use for all containers exposed to `basenet_internet`. @@ -28,7 +28,7 @@ devenv services. Run shell in bastion: ``` -neofs-dev-env$ docker exec -ti bastion /bin/bash +frostfs-dev-env$ docker exec -ti bastion /bin/bash root@bastion:/# ip a sh 1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 diff --git a/docs/chain.md b/docs/chain.md index 0ed13ce..a1cb608 100644 --- a/docs/chain.md +++ b/docs/chain.md @@ -4,10 +4,10 @@ A single-node N3 privnet deployment, running on [neo-go](https://github.com/nspcc-dev/neo-go). Represents N3 MainNet. Contracts deployed: -- NeoFS [contract](https://github.com/nspcc-dev/neofs-contract/tree/master/neofs) -- Processing [contract](https://github.com/nspcc-dev/neofs-contract/tree/master/processing) +- FrostFS [contract](https://github.com/TrueCloudLab/frostfs-contract/tree/master/neofs) +- Processing [contract](https://github.com/TrueCloudLab/frostfs-contract/tree/master/processing) -RPC available at `http://main-chain.neofs.devenv:30333`. +RPC available at `http://main-chain.frostfs.devenv:30333`. ## .env settings @@ -31,7 +31,7 @@ There is a wallet with GAS that used for contract deployment: ``` $ neo-go wallet nep17 balance \ -w wallets/wallet.json \ - -r http://main-chain.neofs.devenv:30333 + -r http://main-chain.frostfs.devenv:30333 Account NbUgTSFvPmsRxmGeWpuuGeJUoRoi6PErcM GAS: GasToken (d2a4cff31913016155e38e474a2c06d08be276cf) @@ -40,14 +40,14 @@ GAS: GasToken (d2a4cff31913016155e38e474a2c06d08be276cf) ``` If you want to operate in main chain with your personal wallet (e.g. to make -a deposit in NeoFS contract), you can transfer GAS from there. +a deposit in FrostFS contract), you can transfer GAS from there. 1. Create new wallet. ``` -$ neo-go wallet init -a -w wallets/neofs1.json +$ neo-go wallet init -a -w wallets/frostfs1.json -Enter the name of the account > neofs1 +Enter the name of the account > frostfs1 Enter passphrase > Confirm passphrase > @@ -57,7 +57,7 @@ Confirm passphrase > { "address": "NXnzw3J9VvKXjM1BPAJK4QUpTtEQu4TpU6", ... -wallet successfully created, file location is wallets/neofs1.json +wallet successfully created, file location is wallets/frostfs1.json ``` 2. Transfer GAS from `wallets/wallet.json`. The password is empty. @@ -65,7 +65,7 @@ wallet successfully created, file location is wallets/neofs1.json ``` $ neo-go wallet nep17 transfer \ -w wallets/wallet.json \ - -r http://main-chain.neofs.devenv:30333 \ + -r http://main-chain.frostfs.devenv:30333 \ --from NbUgTSFvPmsRxmGeWpuuGeJUoRoi6PErcM \ --to NXnzw3J9VvKXjM1BPAJK4QUpTtEQu4TpU6 \ --amount 50 \ @@ -76,8 +76,8 @@ $ neo-go wallet nep17 transfer \ ``` $ neo-go wallet nep17 balance \ - -w wallets/neofs1.json \ - -r http://main-chain.neofs.devenv:30333 + -w wallets/frostfs1.json \ + -r http://main-chain.frostfs.devenv:30333 Account NXnzw3J9VvKXjM1BPAJK4QUpTtEQu4TpU6 GAS: GasToken (d2a4cff31913016155e38e474a2c06d08be276cf) @@ -98,7 +98,7 @@ Claim GAS to consensus node's wallet. Use account that contains NEO tokens. ``` $ neo-go wallet claim \ -w services/chain/node-wallet.json \ - -r http://main-chain.neofs.devenv:30333 \ + -r http://main-chain.frostfs.devenv:30333 \ -a NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP \ Password > 70e09bbd55846dcc7cee23905b737c63e5a80d32e387bce108bc6db8e641fb90 @@ -109,17 +109,17 @@ Then you can transfer GAS the same way as it was done in previous section. ``` $ neo-go wallet nep17 transfer \ -w services/chain/node-wallet.json \ - -r http://main-chain.neofs.devenv:30333 \ + -r http://main-chain.frostfs.devenv:30333 \ --from NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP \ --to NXnzw3J9VvKXjM1BPAJK4QUpTtEQu4TpU6 \ --amount 50 \ --token GAS ``` -## NeoFS GAS deposit +## FrostFS GAS deposit -NeoFS identifies users by their Neo wallet key pair. To start using NeoFS in -devenv you need to transfer some GAS to NeoFS contract in main chain. +FrostFS identifies users by their Neo wallet key pair. To start using FrostFS in +devenv you need to transfer some GAS to FrostFS contract in main chain. Invoke `bin/deposit.sh` script by running `make prepare.ir` command to transfer 50 GAS from account in `wallets/wallet.json` file. Script enters passwords @@ -136,7 +136,7 @@ Script converts addresses and executes this command: ``` $ neo-go wallet nep17 transfer \ -w wallets/wallet.json \ - -r http://main-chain.neofs.devenv:30333 \ + -r http://main-chain.frostfs.devenv:30333 \ --from NbUgTSFvPmsRxmGeWpuuGeJUoRoi6PErcM \ --to NerhjaqJsJt4LxMqUbkkVMpsF2d9TtcpFv \ --token GAS \ @@ -144,12 +144,12 @@ $ neo-go wallet nep17 transfer \ ``` You can specify any wallet address scripthash in the transfer's data argument, -and NeoFS deposit will be transferred to that address. +and FrostFS deposit will be transferred to that address. ``` $ neo-go wallet nep17 transfer \ -w wallets/wallet.json \ - -r http://main-chain.neofs.devenv:30333 \ + -r http://main-chain.frostfs.devenv:30333 \ --from NbUgTSFvPmsRxmGeWpuuGeJUoRoi6PErcM \ --to NerhjaqJsJt4LxMqUbkkVMpsF2d9TtcpFv \ --token GAS \ diff --git a/docs/faq.md b/docs/faq.md index 14f0c4f..12e5df7 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -1,9 +1,9 @@ # F.A.Q, tips and tricks -### How to export private key from Neo wallet for NeoFS use? +### How to export private key from Neo wallet for FrostFS use? -Private key for usage with NeoFS tools can be extracted from Neo wallet in three +Private key for usage with FrostFS tools can be extracted from Neo wallet in three simple steps. 1. Get the key in WIF format @@ -17,7 +17,7 @@ KxDgvEKzgSBPPfuVfw67oPQBSjidEiqTHURKSDL1R7yGaGYAeYnr 2. Convert form WIF to HEX ``` -$ neofs-cli util keyer KxDgvEKzgSBPPfuVfw67oPQBSjidEiqTHURKSDL1R7yGaGYAeYnr +$ frostfs-cli util keyer KxDgvEKzgSBPPfuVfw67oPQBSjidEiqTHURKSDL1R7yGaGYAeYnr PrivateKey 1dd37fba80fec4e6a6f13fd708d8dcb3b29def768017052f6c930fa1c5d90bbb PublicKey 031a6c6fbbdf02ca351745fa86b9ba5a9452d785ac4f7fc2b7548ca2a46c4fcf4a WIF KxDgvEKzgSBPPfuVfw67oPQBSjidEiqTHURKSDL1R7yGaGYAeYnr @@ -35,16 +35,16 @@ $ xxd wallets/wallet.key 00000010: b29d ef76 8017 052f 6c93 0fa1 c5d9 0bbb ...v.../l....... ``` -Later you will be able to provide wallet file in neofs-node config. +Later you will be able to provide wallet file in frostfs-node config. -### How to create Neo wallet JSON file using a NeoFS key file? +### How to create Neo wallet JSON file using a FrostFS key file? -You will need `neo-go` and `neofs-cli`. +You will need `neo-go` and `frostfs-cli`. 1. Get the WIF format of the private key ``` -$ neofs-cli util keyer -key ./services/ir/01.key | grep WIF | awk '{print $NF}' > temp_WIF +$ frostfs-cli util keyer -key ./services/ir/01.key | grep WIF | awk '{print $NF}' > temp_WIF ``` 2. Init a new empty Neo wallet diff --git a/docs/http_gate.md b/docs/http_gate.md index 9141929..1aadd27 100644 --- a/docs/http_gate.md +++ b/docs/http_gate.md @@ -1,8 +1,8 @@ # HTTP Protocol gateway -Protocol Gateway to access data in NeoFS using HTTP protocol. +Protocol Gateway to access data in FrostFS using HTTP protocol. -Source code and more information can be found in [project's GitHub repository](https://github.com/nspcc-dev/neofs-http-gate) +Source code and more information can be found in [project's GitHub repository](https://github.com/TrueCloudLab/frostfs-http-gate) ## .env settings @@ -10,10 +10,10 @@ Source code and more information can be found in [project's GitHub repository](h Image version label to use for containers. -If you want to use locally built image, just set it's label here. Instead of +If you want to use locally built image, just set its label here. Instead of pulling from DockerHub, the local image will be used. -### HTTP_GW_IMAGE=nspccdev/neofs-http-gw +### HTTP_GW_IMAGE=truecloudlab/frostfs-http-gw Image label prefix to use for containers. @@ -21,7 +21,7 @@ Image label prefix to use for containers. - Create a new container ``` -$ neofs-cli --rpc-endpoint s01.neofs.devenv:8080 \ +$ frostfs-cli --rpc-endpoint s01.frostfs.devenv:8080 \ --key wallets/wallet.key \ container create --basic-acl readonly --await \ --policy "REP 1 SELECT 1 FROM *" @@ -32,7 +32,7 @@ container has been persisted on sidechain ``` - Put an object into the newly created container ``` -$ neofs-cli --rpc-endpoint s01.neofs.devenv:8080 \ +$ frostfs-cli --rpc-endpoint s01.frostfs.devenv:8080 \ --key wallets/wallet.key \ object put --file /tmp/backup.jpeg \ --cid 4LfREK1cetL4PUji5fqj9SgRTSmaC5jExEDK9HKCDjdP @@ -40,9 +40,9 @@ $ neofs-cli --rpc-endpoint s01.neofs.devenv:8080 \ ID: 6EPpYqSFMGWrNLvYE9mNnut1CPKuPBKyi1ixHakzqsSB CID: 4LfREK1cetL4PUji5fqj9SgRTSmaC5jExEDK9HKCDjdP ``` -- Call `curl -sSI -XGET http://http.neofs.devenv/get//` +- Call `curl -sSI -XGET http://http.frostfs.devenv/get//` ``` -$ curl -sSI -XGET http://http.neofs.devenv/get/4LfREK1cetL4PUji5fqj9SgRTSmaC5jExEDK9HKCDjdP/6EPpYqSFMGWrNLvYE9mNnut1CPKuPBKyi1ixHakzqsSB +$ curl -sSI -XGET http://http.frostfs.devenv/get/4LfREK1cetL4PUji5fqj9SgRTSmaC5jExEDK9HKCDjdP/6EPpYqSFMGWrNLvYE9mNnut1CPKuPBKyi1ixHakzqsSB HTTP/1.1 200 OK Date: Thu, 03 Dec 2020 10:34:26 GMT Content-Type: image/jpeg diff --git a/docs/ir.md b/docs/ir.md index 28313f0..a870063 100644 --- a/docs/ir.md +++ b/docs/ir.md @@ -1,6 +1,6 @@ -# NeoFS Inner Ring +# FrostFS Inner Ring -NeoFS Inner Ring (Alphabet) node. According to governance scheme, Inner Ring +FrostFS Inner Ring (Alphabet) node. According to governance scheme, Inner Ring should contain Alphabet nodes that share key with one of side chain consensus nodes. In basic setup there is a single consensus node and single Inner Ring (Alphabet) node. @@ -14,6 +14,6 @@ Image version label to use for Inner Ring docker containers. If you want to use locally built image, just set it's label here. Instead of pulling from DockerHub, the local image will be used. -### IR_IMAGE=nspccdev/neofs-ir +### IR_IMAGE=truecloudlab/frostfs-ir Image label prefix to use for Inner Ring docker containers. diff --git a/docs/morph.md b/docs/morph.md index a30bfb5..4429298 100644 --- a/docs/morph.md +++ b/docs/morph.md @@ -1,18 +1,18 @@ -# N3 NeoFS side chain privnet service +# N3 FrostFS side chain privnet service A single-node N3 privnet deployment, running on -[neo-go](https://github.com/nspcc-dev/neo-go). Represents N3 NeoFS SideChain. +[neo-go](https://github.com/nspcc-dev/neo-go). Represents N3 FrostFS SideChain. Contracts deployed: -- Alphabet (AZ) [contract](https://github.com/nspcc-dev/neofs-contract/tree/master/alphabet) -- Audit [contract](https://github.com/nspcc-dev/neofs-contract/tree/master/audit) -- Balance [contract](https://github.com/nspcc-dev/neofs-contract/tree/master/balance) -- Container [contract](https://github.com/nspcc-dev/neofs-contract/tree/master/container) -- Netmap [contract](https://github.com/nspcc-dev/neofs-contract/tree/master/netmap) -- NeoFSID [contract](https://github.com/nspcc-dev/neofs-contract/tree/master/neofsid) -- Proxy [contract](https://github.com/nspcc-dev/neofs-contract/tree/master/proxy) -- Reputation [contract](https://github.com/nspcc-dev/neofs-contract/tree/master/reputation) +- Alphabet (AZ) [contract](https://github.com/TrueCloudLab/frostfs-contract/tree/master/alphabet) +- Audit [contract](https://github.com/TrueCloudLab/frostfs-contract/tree/master/audit) +- Balance [contract](https://github.com/TrueCloudLab/frostfs-contract/tree/master/balance) +- Container [contract](https://github.com/TrueCloudLab/frostfs-contract/tree/master/container) +- Netmap [contract](https://github.com/TrueCloudLab/frostfs-contract/tree/master/netmap) +- NeoFSID [contract](https://github.com/TrueCloudLab/frostfs-contract/tree/master/neofsid) +- Proxy [contract](https://github.com/TrueCloudLab/frostfs-contract/tree/master/proxy) +- Reputation [contract](https://github.com/TrueCloudLab/frostfs-contract/tree/master/reputation) -RPC available at `http://morph-chain.neofs.devenv:30333`. +RPC available at `http://morph-chain.frostfs.devenv:30333`. ## .env settings @@ -36,22 +36,22 @@ There is a wallet with GAS that used for contract deployment: ``` $ neo-go wallet nep17 balance \ -w wallets/wallet.json \ - -r http://morph-chain.neofs.devenv:30333 + -r http://morph-chain.frostfs.devenv:30333 Account NbUgTSFvPmsRxmGeWpuuGeJUoRoi6PErcM GAS: GasToken (d2a4cff31913016155e38e474a2c06d08be276cf) Amount : 189826.0515316 Updated: 3909 -NEOFS: NeoFS Balance (69550190e740b93f92dbd5dea52246f550391057) +FROSTFS: FrostFS Balance (69550190e740b93f92dbd5dea52246f550391057) Amount : 50 Updated: 3909 ``` -This way you can also monitor NeoFS internal balance of your account. +This way you can also monitor FrostFS internal balance of your account. -## NeoFS global config +## FrostFS global config -NeoFS uses global configuration to store epoch duration, maximum object size, +FrostFS uses global configuration to store epoch duration, maximum object size, container fee and other network parameters. Global configuration is stored in netmap contract and managed by Inner Ring (Alphabet) nodes. @@ -64,7 +64,7 @@ $ make update.epoch_duration val=30 Changing EpochDuration configration value to 30 Enter account NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP password > Sent invocation transaction bdc0fa88cd6719ef6df2b9c82de423ddec6141ca24255c2d0072688083b1de9d -Updating NeoFS epoch to 20 +Updating FrostFS epoch to 20 Enter account NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP password > Sent invocation transaction 12296e1ce24dd6c04edb9c56d0a1d0e26d3226adefb0333c74a28788f44a8d0f ``` diff --git a/docs/notary.md b/docs/notary.md index b973ead..eab01f7 100644 --- a/docs/notary.md +++ b/docs/notary.md @@ -78,4 +78,4 @@ ApplicationConfiguration: Main chain generates a block once per 15 seconds, so Inner Ring takes about 15-30 seconds to make a notary deposit in main chain after startup. Then -neofs-dev-env is ready to work. +frostfs-dev-env is ready to work. diff --git a/docs/rest_gate.md b/docs/rest_gate.md index 2597a2c..e511280 100644 --- a/docs/rest_gate.md +++ b/docs/rest_gate.md @@ -1,8 +1,8 @@ # REST Gateway -REST Gateway to access data in NeoFS using REST. +REST Gateway to access data in FrostFS using REST. -Source code and more information can be found in [project's GitHub repository](https://github.com/nspcc-dev/neofs-rest-gw) +Source code and more information can be found in [project's GitHub repository](https://github.com/TrueCloudLab/frostfs-rest-gw) ## .env settings @@ -13,7 +13,7 @@ Image version label to use for containers. If you want to use locally built image, just set its label here. Instead of pulling from DockerHub, the local image will be used. -### REST_GW_IMAGE=nspccdev/neofs-rest-gw +### REST_GW_IMAGE=truecloudlab/frostfs-rest-gw Image label prefix to use for containers. @@ -22,7 +22,7 @@ Image label prefix to use for containers. - List container for specific owner: ```shell -$ curl http://rest.neofs.devenv:8090/v1/containers?ownerId=NbUgTSFvPmsRxmGeWpuuGeJUoRoi6PErcM | jq +$ curl http://rest.frostfs.devenv:8090/v1/containers?ownerId=NbUgTSFvPmsRxmGeWpuuGeJUoRoi6PErcM | jq { "containers": [ { @@ -49,7 +49,7 @@ $ curl http://rest.neofs.devenv:8090/v1/containers?ownerId=NbUgTSFvPmsRxmGeWpuuG - Get container info: ```shell -$ curl http://rest.neofs.devenv:8090/v1/containers/BKcAvz8awKKy9NGsGKi1Hoxxu9AjTGvjKMNMQamvdLmX | jq +$ curl http://rest.frostfs.devenv:8090/v1/containers/BKcAvz8awKKy9NGsGKi1Hoxxu9AjTGvjKMNMQamvdLmX | jq { "attributes": [ { @@ -67,4 +67,4 @@ $ curl http://rest.neofs.devenv:8090/v1/containers/BKcAvz8awKKy9NGsGKi1Hoxxu9AjT } ``` -See all available routes http://rest.neofs.devenv:8090/v1/docs +See all available routes http://rest.frostfs.devenv:8090/v1/docs diff --git a/docs/s3_gate.md b/docs/s3_gate.md index 9c00573..0a110d6 100644 --- a/docs/s3_gate.md +++ b/docs/s3_gate.md @@ -1,8 +1,8 @@ # S3 Protocol gateway -Protocol Gateway to access data in NeoFS using AWS S3 protocol +Protocol Gateway to access data in FrostFS using AWS S3 protocol -Source code and more information can be found in [project's GitHub repository](https://github.com/nspcc-dev/neofs-s3-gw) +Source code and more information can be found in [project's GitHub repository](https://github.com/TrueCloudLab/frostfs-s3-gw) ## .env settings @@ -10,9 +10,9 @@ Source code and more information can be found in [project's GitHub repository](h Image version label to use for containers. -If you want to use locally built image, just set it's label here. Instead of +If you want to use locally built image, just set its label here. Instead of pulling from DockerHub, the local image will be used. -### S3_GW_IMAGE=nspccdev/neofs-s3-gw +### S3_GW_IMAGE=truecloudlab/frostfs-s3-gw Image label prefix to use for containers. diff --git a/docs/storage.md b/docs/storage.md index d46db7e..64ba5fe 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -6,9 +6,9 @@ Image version label to use for Storage docker containers. -If you want to use locally built image, just set it's label here. Instead of +If you want to use locally built image, just set its label here. Instead of pulling from DockerHub, the local image will be used. -### NODE_IMAGE=nspccdev/neofs-ir +### NODE_IMAGE=truecloudlab/frostfs-ir Image label prefix to use for Storage docker containers. From 09533ec3f776584d5e9779a70936a31254923f9f Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 21 Dec 2022 15:34:14 +0300 Subject: [PATCH 024/149] [TrueCloudLab#1] Update Makefile, scripts and compose files Signed-off-by: Denis Kirillov --- .env | 30 +++++++++--------- Makefile | 18 +++++------ bin/config.sh | 4 +-- bin/deposit.sh | 2 +- bin/tick.sh | 4 +-- docs/notary.md | 6 ++-- neofs-adm.yml => frostfs-adm.yml | 2 +- neofs_config.mk => frostfs_config.mk | 0 services/chain/artifacts.mk | 2 +- services/chain/docker-compose.yml | 2 +- services/chain/protocol.privnet.yml | 11 ------- services/ir/.ir.env | 2 +- services/ir/artifacts.mk | 34 ++++++++++---------- services/ir/docker-compose.yml | 2 +- services/ir/healthcheck.sh | 2 +- services/ir/prepare.mk | 2 +- services/morph_chain/artifacts.mk | 46 ++++++++++++++-------------- services/nats/docker-compose.yml | 4 +-- services/storage/.storage.env | 10 +++--- services/storage/artifacts.mk | 4 +-- services/storage/docker-compose.yml | 32 +++++++++---------- services/storage/healthcheck.sh | 2 +- 22 files changed, 105 insertions(+), 116 deletions(-) rename neofs-adm.yml => frostfs-adm.yml (84%) rename neofs_config.mk => frostfs_config.mk (100%) diff --git a/.env b/.env index 01ecbc3..cb42a52 100644 --- a/.env +++ b/.env @@ -1,5 +1,5 @@ # Basenet settings -LOCAL_DOMAIN=neofs.devenv +LOCAL_DOMAIN=frostfs.devenv IPV4_PREFIX=192.168.130 CA_CERTS_TRUSTED_STORE=/etc/ssl/certs @@ -13,11 +13,11 @@ CHAIN_URL="https://github.com/nspcc-dev/neofs-contract/releases/download/v0.16.0 NEOGO_VERSION=0.99.4 NEOGO_IMAGE=nspccdev/neo-go -# NeoFS InnerRing nodes +# FrostFS InnerRing nodes IR_VERSION=0.34.0 IR_IMAGE=nspccdev/neofs-ir -# NeoFS Storage nodes +# FrostFS Storage nodes NODE_VERSION=0.34.0 NODE_IMAGE=nspccdev/neofs-storage @@ -41,20 +41,20 @@ S3_GW_IMAGE=truecloudlab/frostfs-s3-gw COREDNS_VERSION=v016 COREDNS_IMAGE=nspccdev/coredns -# NeoFS LOCODE database +# FrostFS LOCODE database LOCODE_DB_URL=https://github.com/nspcc-dev/neofs-locode-db/releases/download/v0.3.0/locode_db.gz #LOCODE_DB_PATH=/path/to/locode_db -# NeoFS CLI binary -NEOFS_CLI_URL=https://github.com/nspcc-dev/neofs-node/releases/download/v0.34.0/neofs-cli-amd64.tar.gz -#NEOFS_CLI_PATH=/path/to/neofs-cli-binary +# FrostFS CLI binary +FROSTFS_CLI_URL=https://github.com/nspcc-dev/neofs-node/releases/download/v0.34.0/neofs-cli-amd64.tar.gz +#FROSTFS_CLI_PATH=/path/to/frostfs-cli-binary -# NeoFS ADM tool binary -NEOFS_ADM_VERSION=v0.34.0 -NEOFS_ADM_URL=https://github.com/nspcc-dev/neofs-node/releases/download/${NEOFS_ADM_VERSION}/neofs-adm-amd64.tar.gz -#NEOFS_ADM_PATH=/path/to/neofs-adm-binary +# FrostFS ADM tool binary +FROSTFS_ADM_VERSION=v0.34.0 +FROSTFS_ADM_URL=https://github.com/nspcc-dev/neofs-node/releases/download/${FROSTFS_ADM_VERSION}/neofs-adm-amd64.tar.gz +#FROSTFS_ADM_PATH=/path/to/frostfs-adm-binary -# Compiled NeoFS Smart Contracts -NEOFS_CONTRACTS_VERSION=v0.16.0 -NEOFS_CONTRACTS_URL=https://github.com/nspcc-dev/neofs-contract/releases/download/${NEOFS_CONTRACTS_VERSION}/neofs-contract-${NEOFS_CONTRACTS_VERSION}.tar.gz -#NEOFS_CONTRACTS_PATH=/path/to/unpacked/neofs-contracts-dir +# Compiled FrostFS Smart Contracts +FROSTFS_CONTRACTS_VERSION=v0.16.0 +FROSTFS_CONTRACTS_URL=https://github.com/nspcc-dev/neofs-contract/releases/download/${FROSTFS_CONTRACTS_VERSION}/neofs-contract-${FROSTFS_CONTRACTS_VERSION}.tar.gz +#FROSTFS_CONTRACTS_PATH=/path/to/unpacked/frostfs-contracts-dir diff --git a/Makefile b/Makefile index 6e0ca96..59fc415 100644 --- a/Makefile +++ b/Makefile @@ -10,8 +10,8 @@ include .env # help target include help.mk -# update NeoFS global config targets -include neofs_config.mk +# update FrostFS global config targets +include frostfs_config.mk # Targets to get required artifacts and external resources for each service include services/*/artifacts.mk @@ -61,24 +61,24 @@ get: $(foreach SVC, $(GET_SVCS), get.$(SVC)) .PHONY: up up: up/basic @$(foreach SVC, $(START_SVCS), $(shell docker-compose -f services/$(SVC)/docker-compose.yml up -d)) - @echo "Full NeoFS Developer Environment is ready" + @echo "Full FrostFS Developer Environment is ready" -# Build up NeoFS +# Build up FrostFS .PHONY: up/basic up/basic: up/bootstrap @$(foreach SVC, $(START_BASIC), $(shell docker-compose -f services/$(SVC)/docker-compose.yml up -d)) @./bin/tick.sh @./bin/config.sh string SystemDNS container - @echo "Basic NeoFS Developer Environment is ready" + @echo "Basic FrostFS Developer Environment is ready" # Start bootstrap services .PHONY: up/bootstrap up/bootstrap: get vendor/hosts @$(foreach SVC, $(START_BOOTSTRAP), $(shell docker-compose -f services/$(SVC)/docker-compose.yml up -d)) @source ./bin/helper.sh - @./vendor/neofs-adm --config neofs-adm.yml morph init --alphabet-wallets ./services/ir --contracts vendor/contracts || die "Failed to initialize Alphabet wallets" - @for f in ./services/storage/wallet*.json; do echo "Transfer GAS to wallet $${f}" && ./vendor/neofs-adm -c neofs-adm.yml morph refill-gas --storage-wallet $${f} --gas 10.0 --alphabet-wallets services/ir || die "Failed to transfer GAS to alphabet wallets"; done - @echo "NeoFS sidechain environment is deployed" + @./vendor/frostfs-adm --config frostfs-adm.yml morph init --alphabet-wallets ./services/ir --contracts vendor/contracts || die "Failed to initialize Alphabet wallets" + @for f in ./services/storage/wallet*.json; do echo "Transfer GAS to wallet $${f}" && ./vendor/frostfs-adm -c frostfs-adm.yml morph refill-gas --storage-wallet $${f} --gas 10.0 --alphabet-wallets services/ir || die "Failed to transfer GAS to alphabet wallets"; done + @echo "FrostFS sidechain environment is deployed" # Build up certain service .PHONY: up/% @@ -89,7 +89,7 @@ up/%: get vendor/hosts # Stop environment .PHONY: down down: down/add down/basic down/bootstrap - @echo "Full NeoFS Developer Environment is down" + @echo "Full FrostFS Developer Environment is down" .PHONY: down/add down/add: diff --git a/bin/config.sh b/bin/config.sh index f63a49c..0351f7c 100755 --- a/bin/config.sh +++ b/bin/config.sh @@ -16,7 +16,7 @@ CONFIG_IMG="${CONFIG_IMG:-/wallets/config.yml}" NETMAP_ADDR=$(bin/resolve.sh netmap.neofs) || die "Failed to resolve 'netmap.neofs' domain name" -# NeoFS configuration record: variable type [string|int|etc], +# FrostFS configuration record: variable type [string|int|etc], # key is a string and value is a constant of given type TYPE=${1} KEY=${2} @@ -27,7 +27,7 @@ VALUE="${3}" [ -z "$VALUE" ] && echo "Empty config value" && exit 1 # Internal variables -if [[ -z "${NEOFS_NOTARY_DISABLED}" ]]; then +if [[ -z "${FROSTFS_NOTARY_DISABLED}" ]]; then ADDR=$(jq -r .accounts[2].address < "${WALLET}" || die "Cannot get address from ${WALLET}") else ADDR=$(jq -r .accounts[0].address < "${WALLET}" || die "Cannot get address from ${WALLET}") diff --git a/bin/deposit.sh b/bin/deposit.sh index 07610a5..f591178 100755 --- a/bin/deposit.sh +++ b/bin/deposit.sh @@ -32,4 +32,4 @@ ${NEOGO} wallet nep17 transfer \ --from ${ADDR} --force \ --to ${CONTRACT_ADDR} \ --token GAS \ - --amount ${DEPOSIT} || die "Cannot transfer GAS to NeoFS contract" + --amount ${DEPOSIT} || die "Cannot transfer GAS to FrostFS contract" diff --git a/bin/tick.sh b/bin/tick.sh index 8c89a90..cbbaaa9 100755 --- a/bin/tick.sh +++ b/bin/tick.sh @@ -15,7 +15,7 @@ WALLET="${WALLET:-services/chain/node-wallet.json}" CONFIG_IMG="${CONFIG_IMG:-/wallets/config.yml}" # Internal variables -if [[ -z "${NEOFS_NOTARY_DISABLED}" ]]; then +if [[ -z "${FROSTFS_NOTARY_DISABLED}" ]]; then ADDR=$(jq -r .accounts[2].address < "${WALLET}" || die "Cannot get address from ${WALLET}") else ADDR=$(jq -r .accounts[0].address < "${WALLET}" || die "Cannot get address from ${WALLET}") @@ -33,7 +33,7 @@ EPOCH=$(${NEOGO} contract testinvokefunction \ | grep 'value' | awk -F'"' '{ print $4 }') \ || die "Cannot fetch epoch from netmap contract" -echo "Updating NeoFS epoch to $((EPOCH+1))" +echo "Updating FrostFS epoch to $((EPOCH+1))" # shellcheck disable=SC2086 ${NEOGO} contract invokefunction \ diff --git a/docs/notary.md b/docs/notary.md index eab01f7..66cd713 100644 --- a/docs/notary.md +++ b/docs/notary.md @@ -44,14 +44,14 @@ network map. 3. Enable helper commands To enable helper commands such as `make tick.epoch` or `make update.epoch_duration` -make sure to export non-empty `NEOFS_NOTARY_DISABLED` environment variable. +make sure to export non-empty `FROSTFS_NOTARY_DISABLED` environment variable. ``` -$ export NEOFS_NOTARY_DISABLED=1 +$ export FROSTFS_NOTARY_DISABLED=1 ``` Use `unset` command to return it back. ``` -$ unset NEOFS_NOTARY_DISABLED +$ unset FROSTFS_NOTARY_DISABLED ``` # Enable notary service in main chain diff --git a/neofs-adm.yml b/frostfs-adm.yml similarity index 84% rename from neofs-adm.yml rename to frostfs-adm.yml index dc10a6e..ba2039c 100644 --- a/neofs-adm.yml +++ b/frostfs-adm.yml @@ -1,4 +1,4 @@ -rpc-endpoint: http://morph-chain.neofs.devenv:30333 +rpc-endpoint: http://morph-chain.frostfs.devenv:30333 network: max_object_size: 67108864 epoch_duration: 240 diff --git a/neofs_config.mk b/frostfs_config.mk similarity index 100% rename from neofs_config.mk rename to frostfs_config.mk diff --git a/services/chain/artifacts.mk b/services/chain/artifacts.mk index 26548de..186b634 100644 --- a/services/chain/artifacts.mk +++ b/services/chain/artifacts.mk @@ -1,4 +1,4 @@ -# Download privnet chain dump with pre-deployed NeoFS contracts +# Download privnet chain dump with pre-deployed FrostFS contracts get.chain: CHAIN_DUMP_NAME=devenv.dump. get.chain: CHAIN_PATH?= diff --git a/services/chain/docker-compose.yml b/services/chain/docker-compose.yml index 3fe1698..1db47e2 100644 --- a/services/chain/docker-compose.yml +++ b/services/chain/docker-compose.yml @@ -2,7 +2,7 @@ version: "2.4" services: - neofs_main_chain: + frostfs_main_chain: image: ${NEOGO_IMAGE}:${NEOGO_VERSION} container_name: main_chain command: ["node", "--config-path", "/config", "--privnet", "--debug"] diff --git a/services/chain/protocol.privnet.yml b/services/chain/protocol.privnet.yml index d614613..d9a6189 100644 --- a/services/chain/protocol.privnet.yml +++ b/services/chain/protocol.privnet.yml @@ -41,17 +41,6 @@ ApplicationConfiguration: UnlockWallet: Path: "./wallets/node-wallet.json" Password: "one" - Oracle: - Enabled: true - NeoFS: - Nodes: - - s01.neofs.devenv:8080 - - s02.neofs.devenv:8080 - - s03.neofs.devenv:8080 - - s04.neofs.devenv:8080 - UnlockWallet: - Path: "./wallets/node-wallet.json" - Password: "one" P2PNotary: Enabled: false UnlockWallet: diff --git a/services/ir/.ir.env b/services/ir/.ir.env index 9c970b6..f5860e8 100644 --- a/services/ir/.ir.env +++ b/services/ir/.ir.env @@ -31,7 +31,7 @@ NEOFS_IR_EMIT_STORAGE_AMOUNT=1000000000 NEOFS_IR_NETMAP_CLEANER_ENABLED=true NEOFS_IR_CONTRACTS_NEOFS=f5dba2f67baa3cbb5595f0a7e46adf564ec9407a -NEOFS_IR_CONTRACTS_PROCESSING=37d3232083dd2c1bed76bb28d1c992fc6262f699 +NEOFS_IR_CONTRACTS_PROCESSING=332016219b9eb48348fb12694409dbefe29bdf30 NEOFS_IR_MORPH_VALIDATORS=02b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc2 NEOFS_IR_AUDIT_PDP_MAX_SLEEP_INTERVAL=100ms diff --git a/services/ir/artifacts.mk b/services/ir/artifacts.mk index d85afcd..1a8311b 100644 --- a/services/ir/artifacts.mk +++ b/services/ir/artifacts.mk @@ -1,44 +1,44 @@ -# Get NeoFS IR artifacts (LOCODE database and NeoFS CLI) +# Get FrostFS IR artifacts (LOCODE database and FrostFS CLI) LOCODE_DB_ARCHIVE_PATH=./vendor LOCODE_DB_ARCHIVE_FILE=locode_db.gz get.ir: get.locode get.cli -# Get NeoFS LOCODE database +# Get FrostFS LOCODE database get.locode: LOCODE_DB_PATH?= get.locode: @mkdir -p ${LOCODE_DB_ARCHIVE_PATH} ifeq (${LOCODE_DB_PATH},) - @echo "⇒ Download NeoFS LOCODE database from ${LOCODE_DB_URL}" + @echo "⇒ Download FrostFS LOCODE database from ${LOCODE_DB_URL}" @curl \ -sSL "${LOCODE_DB_URL}" \ -o ${LOCODE_DB_ARCHIVE_PATH}/${LOCODE_DB_ARCHIVE_FILE} else - @echo "⇒ Copy local archive of NeoFS LOCODE database from ${LOCODE_DB_PATH}" + @echo "⇒ Copy local archive of FrostFS LOCODE database from ${LOCODE_DB_PATH}" @cp ${LOCODE_DB_PATH} ${LOCODE_DB_ARCHIVE_PATH}/${LOCODE_DB_ARCHIVE_FILE} endif gzip -dfk ${LOCODE_DB_ARCHIVE_PATH}/${LOCODE_DB_ARCHIVE_FILE} -# Download NeoFS CLI +# Download FrostFS CLI .ONESHELL: -get.cli: NEOFS_CLI_FILE=./vendor/neofs-cli -get.cli: NEOFS_CLI_ARCHIVE_FILE=${NEOFS_CLI_FILE}.tar.gz -get.cli: NEOFS_CLI_PATH?= +get.cli: FROSTFS_CLI_FILE=./vendor/frostfs-cli +get.cli: FROSTFS_CLI_ARCHIVE_FILE=${FROSTFS_CLI_FILE}.tar.gz +get.cli: FROSTFS_CLI_PATH?= get.cli: @mkdir -p ./vendor -ifeq (${NEOFS_CLI_PATH},) - @echo "⇒ Download NeoFS CLI binary from ${NEOFS_CLI_URL}" +ifeq (${FROSTFS_CLI_PATH},) + @echo "⇒ Download FrostFS CLI binary from ${FROSTFS_CLI_URL}" @curl \ - -sSL "${NEOFS_CLI_URL}" \ - -o ${NEOFS_CLI_ARCHIVE_FILE} - @tar -xvf ${NEOFS_CLI_ARCHIVE_FILE} -C ./vendor | xargs -I {} \ - mv ./vendor/{} ${NEOFS_CLI_FILE} - @rm ${NEOFS_CLI_ARCHIVE_FILE} + -sSL "${FROSTFS_CLI_URL}" \ + -o ${FROSTFS_CLI_ARCHIVE_FILE} + @tar -xvf ${FROSTFS_CLI_ARCHIVE_FILE} -C ./vendor | xargs -I {} \ + mv ./vendor/{} ${FROSTFS_CLI_FILE} + @rm ${FROSTFS_CLI_ARCHIVE_FILE} else - @echo "⇒ Copy local binary from ${NEOFS_CLI_PATH}" - @cp ${NEOFS_CLI_PATH} ${NEOFS_CLI_FILE} + @echo "⇒ Copy local binary from ${FROSTFS_CLI_PATH}" + @cp ${FROSTFS_CLI_PATH} ${FROSTFS_CLI_FILE} endif diff --git a/services/ir/docker-compose.yml b/services/ir/docker-compose.yml index 929b99a..cf23e22 100644 --- a/services/ir/docker-compose.yml +++ b/services/ir/docker-compose.yml @@ -19,7 +19,7 @@ services: - ./az.key:/wallet01.key - ./../../vendor/hosts:/etc/hosts - ./../../vendor/locode_db:/locode/db - - ./../../vendor/neofs-cli:/neofs-cli + - ./../../vendor/frostfs-cli:/frostfs-cli - ./healthcheck.sh:/healthcheck.sh env_file: [ ".env", ".ir.env", ".int_test.env" ] environment: diff --git a/services/ir/healthcheck.sh b/services/ir/healthcheck.sh index 2934969..6e8ac3f 100755 --- a/services/ir/healthcheck.sh +++ b/services/ir/healthcheck.sh @@ -1,6 +1,6 @@ #!/bin/sh -/neofs-cli control healthcheck \ +/frostfs-cli control healthcheck \ --endpoint "$NEOFS_IR_CONTROL_GRPC_ENDPOINT" \ --wallet /wallet01.key --ir | grep "Health status: READY" diff --git a/services/ir/prepare.mk b/services/ir/prepare.mk index 00a9253..5959964 100644 --- a/services/ir/prepare.mk +++ b/services/ir/prepare.mk @@ -1,3 +1,3 @@ -# Deposit GAS from default wallet to NeoFS privnet contract +# Deposit GAS from default wallet to FrostFS privnet contract prepare.ir: @./bin/deposit.sh diff --git a/services/morph_chain/artifacts.mk b/services/morph_chain/artifacts.mk index be6febf..d3c6ec7 100644 --- a/services/morph_chain/artifacts.mk +++ b/services/morph_chain/artifacts.mk @@ -1,35 +1,35 @@ -# Download NeoFS Contracts and Adm tool to deploy environment +# Download FrostFS Contracts and Adm tool to deploy environment get.morph_chain: get.contracts get.adm -# Download NeoFS Contracts -get.contracts: NEOFS_CONTRACTS_DEST=./vendor/contracts -get.contracts: NEOFS_CONTRACTS_ARCHIVE=neofs-contracts.tar.gz +# Download FrostFS Contracts +get.contracts: FROSTFS_CONTRACTS_DEST=./vendor/contracts +get.contracts: FROSTFS_CONTRACTS_ARCHIVE=frostfs-contracts.tar.gz get.contracts: - @mkdir -p ${NEOFS_CONTRACTS_DEST} + @mkdir -p ${FROSTFS_CONTRACTS_DEST} -ifeq (${NEOFS_CONTRACTS_PATH},) - @echo "⇒ Download compiled NeoFS contracts from ${NEOFS_CONTRACTS_URL}" - @curl -sSL ${NEOFS_CONTRACTS_URL} -o ${NEOFS_CONTRACTS_ARCHIVE} - @tar -xf ${NEOFS_CONTRACTS_ARCHIVE} -C ${NEOFS_CONTRACTS_DEST} --strip-components 1 - @rm ${NEOFS_CONTRACTS_ARCHIVE} +ifeq (${FROSTFS_CONTRACTS_PATH},) + @echo "⇒ Download compiled FrostFS contracts from ${FROSTFS_CONTRACTS_URL}" + @curl -sSL ${FROSTFS_CONTRACTS_URL} -o ${FROSTFS_CONTRACTS_ARCHIVE} + @tar -xf ${FROSTFS_CONTRACTS_ARCHIVE} -C ${FROSTFS_CONTRACTS_DEST} --strip-components 1 + @rm ${FROSTFS_CONTRACTS_ARCHIVE} else - @echo "⇒ Copy compiled contracts from ${NEOFS_CONTRACTS_PATH}" - @cp -r ${NEOFS_CONTRACTS_PATH}/* ${NEOFS_CONTRACTS_DEST} + @echo "⇒ Copy compiled contracts from ${FROSTFS_CONTRACTS_PATH}" + @cp -r ${FROSTFS_CONTRACTS_PATH}/* ${FROSTFS_CONTRACTS_DEST} endif -# Download NeoFS ADM tool -get.adm: NEOFS_ADM_DEST=./vendor/neofs-adm -get.adm: NEOFS_ADM_ARCHIVE=neofs-adm.tar.gz +# Download FrostFS ADM tool +get.adm: FROSTFS_ADM_DEST=./vendor/frostfs-adm +get.adm: FROSTFS_ADM_ARCHIVE=frostfs-adm.tar.gz get.adm: -ifeq (${NEOFS_ADM_PATH},) - @echo "⇒ Download NeoFS ADM binary from ${NEOFS_ADM_URL}" - @curl -sSL ${NEOFS_ADM_URL} -o ${NEOFS_ADM_ARCHIVE} - @tar -xvf ${NEOFS_ADM_ARCHIVE} -C ./vendor | xargs -I {} \ - mv ./vendor/{} ${NEOFS_ADM_DEST} - @rm ${NEOFS_ADM_ARCHIVE} +ifeq (${FROSTFS_ADM_PATH},) + @echo "⇒ Download FrostFS ADM binary from ${FROSTFS_ADM_URL}" + @curl -sSL ${FROSTFS_ADM_URL} -o ${FROSTFS_ADM_ARCHIVE} + @tar -xvf ${FROSTFS_ADM_ARCHIVE} -C ./vendor | xargs -I {} \ + mv ./vendor/{} ${FROSTFS_ADM_DEST} + @rm ${FROSTFS_ADM_ARCHIVE} else - @echo "⇒ Copy neofs-adm binary from ${NEOFS_ADM_PATH}" - @cp ${NEOFS_ADM_PATH} ${NEOFS_ADM_DEST} + @echo "⇒ Copy frostfs-adm binary from ${FROSTFS_ADM_PATH}" + @cp ${FROSTFS_ADM_PATH} ${FROSTFS_ADM_DEST} endif diff --git a/services/nats/docker-compose.yml b/services/nats/docker-compose.yml index d69d10d..d05dd04 100644 --- a/services/nats/docker-compose.yml +++ b/services/nats/docker-compose.yml @@ -16,13 +16,13 @@ services: ipv4_address: ${IPV4_PREFIX}.101 volumes: - ./../../vendor/hosts:/etc/hosts - - ./nats.conf:/etc/nats/neofs-nats-server.conf + - ./nats.conf:/etc/nats/frostfs-nats-server.conf - ./server-cert.pem:/certs/server-cert.pem - ./server-key.pem:/certs/server-key.pem - ./ca-cert.pem:/certs/ca-cert.pem stop_signal: SIGKILL env_file: [ ".env", ".int_test.env" ] - command: ["-c", "/etc/nats/neofs-nats-server.conf"] + command: ["-c", "/etc/nats/frostfs-nats-server.conf"] networks: nats_int: diff --git a/services/storage/.storage.env b/services/storage/.storage.env index 748eac6..50c0d6f 100644 --- a/services/storage/.storage.env +++ b/services/storage/.storage.env @@ -35,14 +35,14 @@ NEOFS_MORPH_RPC_ENDPOINT_0_ADDRESS=ws://morph-chain:30333/ws #NEOFS_APICLIENT_DIAL_TIMEOUT=1m # Common storage node attribute -NEOFS_NODE_ATTRIBUTE_0=User-Agent:NeoFS\/0.33 +NEOFS_NODE_ATTRIBUTE_0=User-Agent:FrostFS\/0.33 NEOFS_NODE_NOTIFICATION_ENABLED=true -NEOFS_NODE_NOTIFICATION_ENDPOINT=tls://nats.neofs.devenv:4222 +NEOFS_NODE_NOTIFICATION_ENDPOINT=tls://nats.frostfs.devenv:4222 NEOFS_NODE_NOTIFICATION_DEFAULT_TOPIC=test -NEOFS_NODE_NOTIFICATION_CERTIFICATE=/etc/neofs-node/nats.tls.cert -NEOFS_NODE_NOTIFICATION_KEY=/etc/neofs-node/nats.tls.key -NEOFS_NODE_NOTIFICATION_CA=/etc/neofs-node/nats.ca.crt +NEOFS_NODE_NOTIFICATION_CERTIFICATE=/etc/frostfs-node/nats.tls.cert +NEOFS_NODE_NOTIFICATION_KEY=/etc/frostfs-node/nats.tls.key +NEOFS_NODE_NOTIFICATION_CA=/etc/frostfs-node/nats.ca.crt # Tree section NEOFS_TREE_ENABLED=True diff --git a/services/storage/artifacts.mk b/services/storage/artifacts.mk index 7132ac1..30d60be 100644 --- a/services/storage/artifacts.mk +++ b/services/storage/artifacts.mk @@ -1,7 +1,7 @@ -# Create new TLS certs to NeoFS node +# Create new TLS certs to FrostFS node STORAGE_DIR=$(abspath services/storage) get.storage: - @echo "⇒ Creating TLS certs to NeoFS node" + @echo "⇒ Creating TLS certs to FrostFS node" ${STORAGE_DIR}/generate_cert.sh ${LOCAL_DOMAIN} > /dev/null diff --git a/services/storage/docker-compose.yml b/services/storage/docker-compose.yml index 6bf8939..3e94644 100644 --- a/services/storage/docker-compose.yml +++ b/services/storage/docker-compose.yml @@ -16,13 +16,13 @@ services: - ./wallet01.json:/wallet.json - ./../../vendor/hosts:/etc/hosts - storage_s01:/storage - - ./../../vendor/neofs-cli:/neofs-cli + - ./../../vendor/frostfs-cli:/frostfs-cli - ./cli-cfg.yml:/cli-cfg.yml - ./healthcheck.sh:/healthcheck.sh - ./s04tls.crt:/etc/ssl/certs/s04tls.crt - - ../nats/client-cert.pem:/etc/neofs-node/nats.tls.cert - - ../nats/client-key.pem:/etc/neofs-node/nats.tls.key - - ../nats/ca-cert.pem:/etc/neofs-node/nats.ca.crt + - ../nats/client-cert.pem:/etc/frostfs-node/nats.tls.cert + - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key + - ../nats/ca-cert.pem:/etc/frostfs-node/nats.ca.crt stop_signal: SIGKILL env_file: [ ".env", ".storage.env", ".int_test.env" ] environment: @@ -54,13 +54,13 @@ services: - ./wallet02.json:/wallet.json - ./../../vendor/hosts:/etc/hosts - storage_s02:/storage - - ./../../vendor/neofs-cli:/neofs-cli + - ./../../vendor/frostfs-cli:/frostfs-cli - ./cli-cfg.yml:/cli-cfg.yml - ./healthcheck.sh:/healthcheck.sh - ./s04tls.crt:/etc/ssl/certs/s04tls.crt - - ../nats/client-cert.pem:/etc/neofs-node/nats.tls.cert - - ../nats/client-key.pem:/etc/neofs-node/nats.tls.key - - ../nats/ca-cert.pem:/etc/neofs-node/nats.ca.crt + - ../nats/client-cert.pem:/etc/frostfs-node/nats.tls.cert + - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key + - ../nats/ca-cert.pem:/etc/frostfs-node/nats.ca.crt stop_signal: SIGKILL env_file: [ ".env", ".storage.env", ".int_test.env" ] environment: @@ -92,13 +92,13 @@ services: - ./wallet03.json:/wallet.json - ./../../vendor/hosts:/etc/hosts - storage_s03:/storage - - ./../../vendor/neofs-cli:/neofs-cli + - ./../../vendor/frostfs-cli:/frostfs-cli - ./cli-cfg.yml:/cli-cfg.yml - ./healthcheck.sh:/healthcheck.sh - ./s04tls.crt:/etc/ssl/certs/s04tls.crt - - ../nats/client-cert.pem:/etc/neofs-node/nats.tls.cert - - ../nats/client-key.pem:/etc/neofs-node/nats.tls.key - - ../nats/ca-cert.pem:/etc/neofs-node/nats.ca.crt + - ../nats/client-cert.pem:/etc/frostfs-node/nats.tls.cert + - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key + - ../nats/ca-cert.pem:/etc/frostfs-node/nats.ca.crt stop_signal: SIGKILL env_file: [ ".env", ".storage.env", ".int_test.env" ] environment: @@ -130,14 +130,14 @@ services: - ./wallet04.json:/wallet.json - ./../../vendor/hosts:/etc/hosts - storage_s04:/storage - - ./../../vendor/neofs-cli:/neofs-cli + - ./../../vendor/frostfs-cli:/frostfs-cli - ./cli-cfg.yml:/cli-cfg.yml - ./healthcheck.sh:/healthcheck.sh - ./s04tls.crt:/tls.crt - ./s04tls.key:/tls.key - - ../nats/client-cert.pem:/etc/neofs-node/nats.tls.cert - - ../nats/client-key.pem:/etc/neofs-node/nats.tls.key - - ../nats/ca-cert.pem:/etc/neofs-node/nats.ca.crt + - ../nats/client-cert.pem:/etc/frostfs-node/nats.tls.cert + - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key + - ../nats/ca-cert.pem:/etc/frostfs-node/nats.ca.crt stop_signal: SIGKILL env_file: [ ".env", ".storage.env", ".int_test.env" ] environment: diff --git a/services/storage/healthcheck.sh b/services/storage/healthcheck.sh index fdbf390..bdc5baf 100755 --- a/services/storage/healthcheck.sh +++ b/services/storage/healthcheck.sh @@ -1,5 +1,5 @@ #!/bin/sh -/neofs-cli control healthcheck -c /cli-cfg.yml \ +/frostfs-cli control healthcheck -c /cli-cfg.yml \ --endpoint "$NEOFS_CONTROL_GRPC_ENDPOINT" | grep "Health status: READY" From 42847315af501e027c42d2285ed02b9825426ef6 Mon Sep 17 00:00:00 2001 From: Stanislav Bogatyrev Date: Sat, 7 Jan 2023 18:13:54 +0300 Subject: [PATCH 025/149] Change logo Signed-off-by: Stanislav Bogatyrev --- .github/logo.svg | 70 ++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 3 +++ 2 files changed, 73 insertions(+) create mode 100644 .github/logo.svg diff --git a/.github/logo.svg b/.github/logo.svg new file mode 100644 index 0000000..148c359 --- /dev/null +++ b/.github/logo.svg @@ -0,0 +1,70 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/README.md b/README.md index 42cb644..b9af689 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,6 @@ +

+FrostFS logo +

FrostFS local Development and Testing environment

From a9c05c1a98b33a6458e55d5ae303aec08fb29db4 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 10 Jan 2023 16:05:29 +0300 Subject: [PATCH 026/149] [TrueCloudLab#1] Update core images Signed-off-by: Denis Kirillov --- .env | 14 +++++++------- bin/config.sh | 2 +- bin/deposit.sh | 4 ++-- bin/tick.sh | 2 +- services/ir/.ir.env | 3 ++- services/ir/artifacts.mk | 2 +- services/morph_chain/artifacts.mk | 2 +- 7 files changed, 15 insertions(+), 14 deletions(-) diff --git a/.env b/.env index cb42a52..927233c 100644 --- a/.env +++ b/.env @@ -14,12 +14,12 @@ NEOGO_VERSION=0.99.4 NEOGO_IMAGE=nspccdev/neo-go # FrostFS InnerRing nodes -IR_VERSION=0.34.0 -IR_IMAGE=nspccdev/neofs-ir +IR_VERSION=e3554425 +IR_IMAGE=truecloudlab/frostfs-ir # FrostFS Storage nodes -NODE_VERSION=0.34.0 -NODE_IMAGE=nspccdev/neofs-storage +NODE_VERSION=e3554425 +NODE_IMAGE=truecloudlab/frostfs-storage # NATS Server NATS_VERSION=2.7.2 @@ -46,12 +46,12 @@ LOCODE_DB_URL=https://github.com/nspcc-dev/neofs-locode-db/releases/download/v0. #LOCODE_DB_PATH=/path/to/locode_db # FrostFS CLI binary -FROSTFS_CLI_URL=https://github.com/nspcc-dev/neofs-node/releases/download/v0.34.0/neofs-cli-amd64.tar.gz +FROSTFS_CLI_URL=https://http.t5.fs.neo.org/AQgse8bPCZx4zScMuAKxowJdZPbKHp8NDcp15o6VUNmk/DjKaXEXVmVunGgu2pMvpqZmzZ2omNxTQHx3sUYEz1s1b #FROSTFS_CLI_PATH=/path/to/frostfs-cli-binary # FrostFS ADM tool binary -FROSTFS_ADM_VERSION=v0.34.0 -FROSTFS_ADM_URL=https://github.com/nspcc-dev/neofs-node/releases/download/${FROSTFS_ADM_VERSION}/neofs-adm-amd64.tar.gz +FROSTFS_ADM_VERSION=e3554425 +FROSTFS_ADM_URL=https://http.t5.fs.neo.org/AQgse8bPCZx4zScMuAKxowJdZPbKHp8NDcp15o6VUNmk/JAqZZR6Rh7QL6NrJc9cbALNrUJenGD2VK6Mjdjsa7tzF #FROSTFS_ADM_PATH=/path/to/frostfs-adm-binary # Compiled FrostFS Smart Contracts diff --git a/bin/config.sh b/bin/config.sh index 0351f7c..e6d9cc5 100755 --- a/bin/config.sh +++ b/bin/config.sh @@ -14,7 +14,7 @@ NEOGO="${NEOGO:-docker exec main_chain neo-go}" WALLET="${WALLET:-services/chain/node-wallet.json}" CONFIG_IMG="${CONFIG_IMG:-/wallets/config.yml}" -NETMAP_ADDR=$(bin/resolve.sh netmap.neofs) || die "Failed to resolve 'netmap.neofs' domain name" +NETMAP_ADDR=$(bin/resolve.sh netmap.frostfs) || die "Failed to resolve 'netmap.frostfs' domain name" # FrostFS configuration record: variable type [string|int|etc], # key is a string and value is a constant of given type diff --git a/bin/deposit.sh b/bin/deposit.sh index f591178..af039e5 100755 --- a/bin/deposit.sh +++ b/bin/deposit.sh @@ -18,11 +18,11 @@ DEPOSIT="${1:-50}" # Internal variables ADDR=$(jq -r .accounts[0].address < "${WALLET}" \ || die "Cannot get address from wallet: ${WALLET}") -CONTRACT_ADDR=$(${NEOGO} util convert "${NEOFS_IR_CONTRACTS_NEOFS}" \ +CONTRACT_ADDR=$(${NEOGO} util convert "${NEOFS_IR_CONTRACTS_FROSTFS}" \ | grep 'LE ScriptHash to Address' \ | awk '{print $5}' \ | grep -oP "[A-z0-9]+" \ - || die "Cannot parse contract address: ${NEOFS_IR_CONTRACTS_NEOFS}") + || die "Cannot parse contract address: ${NEOFS_IR_CONTRACTS_FROSTFS}") # Make deposit # shellcheck disable=SC2086 diff --git a/bin/tick.sh b/bin/tick.sh index cbbaaa9..6120577 100755 --- a/bin/tick.sh +++ b/bin/tick.sh @@ -25,7 +25,7 @@ fi SIDECHAIN_PROTO="${SIDECHAIN_PROTO:-services/morph_chain/protocol.privnet.yml}" BLOCK_DURATION=$(grep SecondsPerBlock < "$SIDECHAIN_PROTO" | awk '{print $2}') \ || die "Cannot fetch block duration" -NETMAP_ADDR=$(bin/resolve.sh netmap.neofs) || die "Cannot resolve netmap.neofs" +NETMAP_ADDR=$(bin/resolve.sh netmap.frostfs) || die "Cannot resolve netmap.frostfs" # Fetch current epoch value EPOCH=$(${NEOGO} contract testinvokefunction \ diff --git a/services/ir/.ir.env b/services/ir/.ir.env index f5860e8..73904ca 100644 --- a/services/ir/.ir.env +++ b/services/ir/.ir.env @@ -30,7 +30,8 @@ NEOFS_IR_TIMERS_DISTRIBUTE_BASIC_INCOME_DIV=4 NEOFS_IR_EMIT_STORAGE_AMOUNT=1000000000 NEOFS_IR_NETMAP_CLEANER_ENABLED=true -NEOFS_IR_CONTRACTS_NEOFS=f5dba2f67baa3cbb5595f0a7e46adf564ec9407a +NEOFS_IR_CONTRACTS_FROSTFS=f5dba2f67baa3cbb5595f0a7e46adf564ec9407a +NEOFS_IR_CONTRACTS_FROSTFSID=1943e9bb78a0fe2fe0c95fd2677eec2da6aa4aa5 NEOFS_IR_CONTRACTS_PROCESSING=332016219b9eb48348fb12694409dbefe29bdf30 NEOFS_IR_MORPH_VALIDATORS=02b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc2 diff --git a/services/ir/artifacts.mk b/services/ir/artifacts.mk index 1a8311b..95b2c22 100644 --- a/services/ir/artifacts.mk +++ b/services/ir/artifacts.mk @@ -33,7 +33,7 @@ get.cli: ifeq (${FROSTFS_CLI_PATH},) @echo "⇒ Download FrostFS CLI binary from ${FROSTFS_CLI_URL}" @curl \ - -sSL "${FROSTFS_CLI_URL}" \ + -ksSL "${FROSTFS_CLI_URL}" \ -o ${FROSTFS_CLI_ARCHIVE_FILE} @tar -xvf ${FROSTFS_CLI_ARCHIVE_FILE} -C ./vendor | xargs -I {} \ mv ./vendor/{} ${FROSTFS_CLI_FILE} diff --git a/services/morph_chain/artifacts.mk b/services/morph_chain/artifacts.mk index d3c6ec7..0f24735 100644 --- a/services/morph_chain/artifacts.mk +++ b/services/morph_chain/artifacts.mk @@ -25,7 +25,7 @@ get.adm: ifeq (${FROSTFS_ADM_PATH},) @echo "⇒ Download FrostFS ADM binary from ${FROSTFS_ADM_URL}" - @curl -sSL ${FROSTFS_ADM_URL} -o ${FROSTFS_ADM_ARCHIVE} + @curl -skSL ${FROSTFS_ADM_URL} -o ${FROSTFS_ADM_ARCHIVE} @tar -xvf ${FROSTFS_ADM_ARCHIVE} -C ./vendor | xargs -I {} \ mv ./vendor/{} ${FROSTFS_ADM_DEST} @rm ${FROSTFS_ADM_ARCHIVE} From 769410a29fef22c65c5879d2dfa578e31dc0bda5 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 10 Jan 2023 17:54:08 +0300 Subject: [PATCH 027/149] [TrueCloudLab#1] Remove mainchain related components Signed-off-by: Denis Kirillov --- .bootstrap_services | 1 - .env | 2 - Makefile | 2 - bin/config.sh | 4 +- bin/deposit.sh | 35 ----- bin/tick.sh | 4 +- docs/chain.md | 158 --------------------- services/chain/.env | 1 - services/chain/.hosts | 1 - services/chain/.int_test.env | 1 - services/chain/artifacts.mk | 18 --- services/chain/docker-compose.yml | 31 ---- services/chain/node-wallet.json | 76 ---------- services/chain/protocol.privnet.yml | 48 ------- services/ir/.ir.env | 5 +- services/ir/prepare.mk | 3 +- services/{chain => morph_chain}/config.yml | 0 services/morph_chain/docker-compose.yml | 3 +- 18 files changed, 9 insertions(+), 384 deletions(-) delete mode 100755 bin/deposit.sh delete mode 100644 docs/chain.md delete mode 120000 services/chain/.env delete mode 100644 services/chain/.hosts delete mode 120000 services/chain/.int_test.env delete mode 100644 services/chain/artifacts.mk delete mode 100644 services/chain/docker-compose.yml delete mode 100644 services/chain/node-wallet.json delete mode 100644 services/chain/protocol.privnet.yml rename services/{chain => morph_chain}/config.yml (100%) diff --git a/.bootstrap_services b/.bootstrap_services index 3d2562c..3071b48 100644 --- a/.bootstrap_services +++ b/.bootstrap_services @@ -1,5 +1,4 @@ # Services start/stop order # Will start from top to bottom and stop in reverse basenet -chain morph_chain diff --git a/.env b/.env index 927233c..4fa3839 100644 --- a/.env +++ b/.env @@ -8,8 +8,6 @@ BASTION_VERSION=10 BASTION_IMAGE=debian # NeoGo privnet -#CHAIN_PATH="/path/to/devenv.dump.gz" -CHAIN_URL="https://github.com/nspcc-dev/neofs-contract/releases/download/v0.16.0/devenv_mainchain_notary_disabled.gz" NEOGO_VERSION=0.99.4 NEOGO_IMAGE=nspccdev/neo-go diff --git a/Makefile b/Makefile index 59fc415..61782ba 100644 --- a/Makefile +++ b/Makefile @@ -41,7 +41,6 @@ HOSTS_LINES = $(shell grep -Rl IPV4_PREFIX ./services/* | grep .hosts) # Paths to protocol.privnet.yml MORPH_CHAIN_PROTOCOL = './services/morph_chain/protocol.privnet.yml' -CHAIN_PROTOCOL = './services/chain/protocol.privnet.yml' # List of grepped environment variables from *.env GREP_DOTENV = $(shell find . -name '*.env' -exec grep -rhv -e '^\#' -e '^$$' {} + | sort -u ) @@ -150,7 +149,6 @@ clean: env: @$(foreach envvar,$(GREP_DOTENV),echo $(envvar);) @echo MORPH_BLOCK_TIME=$(shell grep 'SecondsPerBlock' $(MORPH_CHAIN_PROTOCOL) | awk '{print $$2}')s - @echo MAINNET_BLOCK_TIME=$(shell grep 'SecondsPerBlock' $(CHAIN_PROTOCOL) | awk '{print $$2}')s @echo MORPH_MAGIC=$(shell grep 'Magic' $(MORPH_CHAIN_PROTOCOL) | awk '{print $$2}') # Restart storage nodes with clean volumes diff --git a/bin/config.sh b/bin/config.sh index e6d9cc5..31b613f 100755 --- a/bin/config.sh +++ b/bin/config.sh @@ -8,10 +8,10 @@ echo "Running bin/config.sh" source bin/helper.sh # NeoGo binary path. -NEOGO="${NEOGO:-docker exec main_chain neo-go}" +NEOGO="${NEOGO:-docker exec morph_chain neo-go}" # Wallet files to change config value -WALLET="${WALLET:-services/chain/node-wallet.json}" +WALLET="${WALLET:-services/morph_chain/node-wallet.json}" CONFIG_IMG="${CONFIG_IMG:-/wallets/config.yml}" NETMAP_ADDR=$(bin/resolve.sh netmap.frostfs) || die "Failed to resolve 'netmap.frostfs' domain name" diff --git a/bin/deposit.sh b/bin/deposit.sh deleted file mode 100755 index af039e5..0000000 --- a/bin/deposit.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash - -echo "Running bin/deposit.sh" - -# Source env settings -. .env -. services/ir/.ir.env -source bin/helper.sh - -# NeoGo binary path. -NEOGO="${NEOGO:-docker exec main_chain neo-go}" -# Wallet file to use for deposit GAS from -WALLET="${WALLET:-services/chain/node-wallet.json}" -CONFIG="${CONFIG:-/wallets/config.yml}" -# How much GAS to deposit. First cli argument or 50 by default -DEPOSIT="${1:-50}" - -# Internal variables -ADDR=$(jq -r .accounts[0].address < "${WALLET}" \ - || die "Cannot get address from wallet: ${WALLET}") -CONTRACT_ADDR=$(${NEOGO} util convert "${NEOFS_IR_CONTRACTS_FROSTFS}" \ - | grep 'LE ScriptHash to Address' \ - | awk '{print $5}' \ - | grep -oP "[A-z0-9]+" \ - || die "Cannot parse contract address: ${NEOFS_IR_CONTRACTS_FROSTFS}") - -# Make deposit -# shellcheck disable=SC2086 -${NEOGO} wallet nep17 transfer \ - --wallet-config ${CONFIG} \ - -r http://main-chain.${LOCAL_DOMAIN}:30333 \ - --from ${ADDR} --force \ - --to ${CONTRACT_ADDR} \ - --token GAS \ - --amount ${DEPOSIT} || die "Cannot transfer GAS to FrostFS contract" diff --git a/bin/tick.sh b/bin/tick.sh index 6120577..75482f0 100755 --- a/bin/tick.sh +++ b/bin/tick.sh @@ -8,10 +8,10 @@ echo "Running bin/tick.sh" source bin/helper.sh # NeoGo binary path. -NEOGO="${NEOGO:-docker exec main_chain neo-go}" +NEOGO="${NEOGO:-docker exec morph_chain neo-go}" # Wallet files to change config value -WALLET="${WALLET:-services/chain/node-wallet.json}" +WALLET="${WALLET:-services/morph_chain/node-wallet.json}" CONFIG_IMG="${CONFIG_IMG:-/wallets/config.yml}" # Internal variables diff --git a/docs/chain.md b/docs/chain.md deleted file mode 100644 index a1cb608..0000000 --- a/docs/chain.md +++ /dev/null @@ -1,158 +0,0 @@ -# N3 main chain privnet service - -A single-node N3 privnet deployment, running on -[neo-go](https://github.com/nspcc-dev/neo-go). Represents N3 MainNet. - -Contracts deployed: -- FrostFS [contract](https://github.com/TrueCloudLab/frostfs-contract/tree/master/neofs) -- Processing [contract](https://github.com/TrueCloudLab/frostfs-contract/tree/master/processing) - -RPC available at `http://main-chain.frostfs.devenv:30333`. - -## .env settings - -### CHAIN_URL - -URL to get main chain dump. Used on artifact get stage. - -### CHAIN_PATH - -Path to get main chain dump. If set, overrides `CHAIN_URL`. - -### NEOGO_VERSION - -Version of neo-go docker container for main chain deployment. - -## Main chain wallets - -There is a wallet with GAS that used for contract deployment: -`wallets/wallet.json`. This wallet has one account with **empty password**. - -``` -$ neo-go wallet nep17 balance \ - -w wallets/wallet.json \ - -r http://main-chain.frostfs.devenv:30333 - -Account NbUgTSFvPmsRxmGeWpuuGeJUoRoi6PErcM -GAS: GasToken (d2a4cff31913016155e38e474a2c06d08be276cf) - Amount : 9978.0074623 - Updated: 34 -``` - -If you want to operate in main chain with your personal wallet (e.g. to make -a deposit in FrostFS contract), you can transfer GAS from there. - -1. Create new wallet. - -``` -$ neo-go wallet init -a -w wallets/frostfs1.json - -Enter the name of the account > frostfs1 -Enter passphrase > -Confirm passphrase > - -{ - "version": "3.0", - "accounts": [ - { - "address": "NXnzw3J9VvKXjM1BPAJK4QUpTtEQu4TpU6", -... -wallet successfully created, file location is wallets/frostfs1.json -``` - -2. Transfer GAS from `wallets/wallet.json`. The password is empty. - -``` -$ neo-go wallet nep17 transfer \ - -w wallets/wallet.json \ - -r http://main-chain.frostfs.devenv:30333 \ - --from NbUgTSFvPmsRxmGeWpuuGeJUoRoi6PErcM \ - --to NXnzw3J9VvKXjM1BPAJK4QUpTtEQu4TpU6 \ - --amount 50 \ - --token GAS -``` - -3. Check it's there. - -``` -$ neo-go wallet nep17 balance \ - -w wallets/frostfs1.json \ - -r http://main-chain.frostfs.devenv:30333 - -Account NXnzw3J9VvKXjM1BPAJK4QUpTtEQu4TpU6 -GAS: GasToken (d2a4cff31913016155e38e474a2c06d08be276cf) - Amount : 50 - Updated: 14689 -``` - -## Claim GAS from consensus node - -If there is no enough GAS on `wallets/wallet.json` account, you can claim some -GAS to consensus node's wallet and then transfer it. - -Consensus node is running with `services/chain/node-wallet.json` wallet. It has -multiple accounts with the password `one`. - - -Claim GAS to consensus node's wallet. Use account that contains NEO tokens. -``` -$ neo-go wallet claim \ - -w services/chain/node-wallet.json \ - -r http://main-chain.frostfs.devenv:30333 \ - -a NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP \ -Password > -70e09bbd55846dcc7cee23905b737c63e5a80d32e387bce108bc6db8e641fb90 -``` - -Then you can transfer GAS the same way as it was done in previous section. - -``` -$ neo-go wallet nep17 transfer \ - -w services/chain/node-wallet.json \ - -r http://main-chain.frostfs.devenv:30333 \ - --from NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP \ - --to NXnzw3J9VvKXjM1BPAJK4QUpTtEQu4TpU6 \ - --amount 50 \ - --token GAS -``` - -## FrostFS GAS deposit - -FrostFS identifies users by their Neo wallet key pair. To start using FrostFS in -devenv you need to transfer some GAS to FrostFS contract in main chain. - -Invoke `bin/deposit.sh` script by running `make prepare.ir` command to transfer -50 GAS from account in `wallets/wallet.json` file. Script enters passwords -automatically with `expect` utility. - -``` -$ make prepare.ir -Password > -Can't find matching token in the wallet. Querying RPC-node for balances. -6713c776f4102300691d9c3c493bcd3402434f5e32e8147e0a5bc72209a1e410 -``` - -Script converts addresses and executes this command: -``` -$ neo-go wallet nep17 transfer \ - -w wallets/wallet.json \ - -r http://main-chain.frostfs.devenv:30333 \ - --from NbUgTSFvPmsRxmGeWpuuGeJUoRoi6PErcM \ - --to NerhjaqJsJt4LxMqUbkkVMpsF2d9TtcpFv \ - --token GAS \ - --amount 50 -``` - -You can specify any wallet address scripthash in the transfer's data argument, -and FrostFS deposit will be transferred to that address. - -``` -$ neo-go wallet nep17 transfer \ - -w wallets/wallet.json \ - -r http://main-chain.frostfs.devenv:30333 \ - --from NbUgTSFvPmsRxmGeWpuuGeJUoRoi6PErcM \ - --to NerhjaqJsJt4LxMqUbkkVMpsF2d9TtcpFv \ - --token GAS \ - --amount 50 \ - hash160:bd711de066e9c2f7b502c7f3f0e0a6f1c8341edd -``` diff --git a/services/chain/.env b/services/chain/.env deleted file mode 120000 index c7360fb..0000000 --- a/services/chain/.env +++ /dev/null @@ -1 +0,0 @@ -../../.env \ No newline at end of file diff --git a/services/chain/.hosts b/services/chain/.hosts deleted file mode 100644 index a576e59..0000000 --- a/services/chain/.hosts +++ /dev/null @@ -1 +0,0 @@ -IPV4_PREFIX.50 main-chain.LOCAL_DOMAIN diff --git a/services/chain/.int_test.env b/services/chain/.int_test.env deleted file mode 120000 index 582b6a2..0000000 --- a/services/chain/.int_test.env +++ /dev/null @@ -1 +0,0 @@ -../../.int_test.env \ No newline at end of file diff --git a/services/chain/artifacts.mk b/services/chain/artifacts.mk deleted file mode 100644 index 186b634..0000000 --- a/services/chain/artifacts.mk +++ /dev/null @@ -1,18 +0,0 @@ -# Download privnet chain dump with pre-deployed FrostFS contracts - -get.chain: CHAIN_DUMP_NAME=devenv.dump. -get.chain: CHAIN_PATH?= -get.chain: - @mkdir -p ./vendor - -ifeq (${CHAIN_PATH},) - @echo "⇒ Download blockchain dump from ${CHAIN_URL}" - @curl \ - -sSL "${CHAIN_URL}" \ - -o ./vendor/chain.gz -else - @echo "⇒ Copy local archive ${CHAIN_PATH}" - @cp ${CHAIN_PATH} ./vendor/chain.gz -endif - - diff --git a/services/chain/docker-compose.yml b/services/chain/docker-compose.yml deleted file mode 100644 index 1db47e2..0000000 --- a/services/chain/docker-compose.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- - -version: "2.4" -services: - frostfs_main_chain: - image: ${NEOGO_IMAGE}:${NEOGO_VERSION} - container_name: main_chain - command: ["node", "--config-path", "/config", "--privnet", "--debug"] - domainname: ${LOCAL_DOMAIN} - hostname: main-chain - networks: - chain_int: - internet: - ipv4_address: ${IPV4_PREFIX}.50 - stop_signal: SIGKILL - env_file: [ ".int_test.env" ] - environment: - - ACC=/chain.gz - volumes: - - ./../../vendor/chain.gz:/chain.gz - - ./protocol.privnet.yml:/config/protocol.privnet.yml - - ./node-wallet.json:/wallets/node-wallet.json - - ./config.yml:/wallets/config.yml - - ./../../vendor/hosts:/etc/hosts - - ./../../wallets/wallet.json:/wallets/wallet.json - -networks: - chain_int: - internet: - external: true - name: basenet_internet diff --git a/services/chain/node-wallet.json b/services/chain/node-wallet.json deleted file mode 100644 index 00bfe8e..0000000 --- a/services/chain/node-wallet.json +++ /dev/null @@ -1,76 +0,0 @@ -{ - "version": "3.0", - "name":null, - "accounts": [ - { - "address": "Nhfg3TbpwogLvDGVvAvqyThbsHgoSUKwtn", - "key": "6PYM8VdX2BSm7BSXKzV4Fz6S3R9cDLLWNrD9nMjxW352jEv3fsC8N3wNLY", - "label": "", - "contract": { - "script": "DCECs2Ir9AF73+MXxYrtX0x1PyBrfbiWBG+n13S7xL9/jcJBVuezJw==", - "parameters": [ - { - "name": "parameter0", - "type": "Signature" - } - ], - "deployed": false - }, - "lock": false, - "extra":null, - "isDefault": false - }, - { - "address": "NVTiAjNgagDkTr5HTzDmQP9kPwPHN5BgVq", - "key": "6PYM8VdX2BSm7BSXKzV4Fz6S3R9cDLLWNrD9nMjxW352jEv3fsC8N3wNLY", - "label": "", - "contract": { - "script": "EwwhAhA6f33QFlWFl/eWDSfFFqQ5T9loueZRVetLAT5AQEBuDCECp7xV/oaE4BGXaNEEujB5W9zIZhnoZK3SYVZyPtGFzWIMIQKzYiv0AXvf4xfFiu1fTHU/IGt9uJYEb6fXdLvEv3+NwgwhA9kMB99j5pDOd5EuEKtRrMlEtmhgI3tgjE+PgwnnHuaZFEGe0Nw6", - "parameters": [ - { - "name": "parameter0", - "type": "Signature" - }, - { - "name": "parameter1", - "type": "Signature" - }, - { - "name": "parameter2", - "type": "Signature" - } - ], - "deployed": false - }, - "lock": false, - "extra":null, - "isDefault": false - }, - { - "address": "NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP", - "key": "6PYM8VdX2BSm7BSXKzV4Fz6S3R9cDLLWNrD9nMjxW352jEv3fsC8N3wNLY", - "label": "", - "contract": { - "script": "EQwhArNiK/QBe9/jF8WK7V9MdT8ga324lgRvp9d0u8S/f43CEUGe0Nw6", - "parameters": [ - { - "name": "parameter0", - "type": "Signature" - } - ], - "deployed": false - }, - "lock": false, - "extra":null, - "isDefault": false - } - ], - "scrypt": { - "n": 16384, - "r": 8, - "p": 8 - }, - "extra": { - "Tokens": null - } -} diff --git a/services/chain/protocol.privnet.yml b/services/chain/protocol.privnet.yml deleted file mode 100644 index d9a6189..0000000 --- a/services/chain/protocol.privnet.yml +++ /dev/null @@ -1,48 +0,0 @@ -ProtocolConfiguration: - Magic: 56753 - MaxTraceableBlocks: 200000 - SecondsPerBlock: 1 - MemPoolSize: 50000 - StandbyCommittee: - - 02b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc2 - ValidatorsCount: 1 - SeedList: - - 172.200.0.1:20333 - VerifyBlocks: true - VerifyTransactions: true - P2PSigExtensions: false - -ApplicationConfiguration: - DBConfiguration: - Type: "boltdb" - BoltDBOptions: - FilePath: "./db/privnet.bolt" - NodePort: 20333 - Relay: true - DialTimeout: 3 - ProtoTickInterval: 2 - PingInterval: 30 - PingTimeout: 90 - MaxPeers: 10 - AttemptConnPeers: 5 - MinPeers: 0 - RPC: - Enabled: true - SessionEnabled: true - EnableCORSWorkaround: false - MaxGasInvoke: 15 - Port: 30333 - Prometheus: - Enabled: true - Port: 20001 - Pprof: - Enabled: true - Port: 20011 - UnlockWallet: - Path: "./wallets/node-wallet.json" - Password: "one" - P2PNotary: - Enabled: false - UnlockWallet: - Path: "./wallets/node-wallet.json" - Password: "one" diff --git a/services/ir/.ir.env b/services/ir/.ir.env index 73904ca..ddc40e6 100644 --- a/services/ir/.ir.env +++ b/services/ir/.ir.env @@ -17,9 +17,8 @@ NEOFS_IR_PROMETHEUS_ADDRESS=:9090 #NEOFS_IR_METRICS_SHUTDOWN_TIMEOUT=1m # Toggling the sidechain-only mode -NEOFS_IR_WITHOUT_MAINNET=false +NEOFS_IR_WITHOUT_MAINNET=true -NEOFS_IR_MAINNET_ENDPOINT_CLIENT_0_ADDRESS=ws://main-chain:30333/ws NEOFS_IR_MORPH_ENDPOINT_CLIENT_0_ADDRESS=ws://morph-chain:30333/ws NEOFS_IR_TIMERS_EMIT=50 NEOFS_IR_TIMERS_STOP_ESTIMATION_DIV=4 @@ -30,9 +29,7 @@ NEOFS_IR_TIMERS_DISTRIBUTE_BASIC_INCOME_DIV=4 NEOFS_IR_EMIT_STORAGE_AMOUNT=1000000000 NEOFS_IR_NETMAP_CLEANER_ENABLED=true -NEOFS_IR_CONTRACTS_FROSTFS=f5dba2f67baa3cbb5595f0a7e46adf564ec9407a NEOFS_IR_CONTRACTS_FROSTFSID=1943e9bb78a0fe2fe0c95fd2677eec2da6aa4aa5 -NEOFS_IR_CONTRACTS_PROCESSING=332016219b9eb48348fb12694409dbefe29bdf30 NEOFS_IR_MORPH_VALIDATORS=02b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc2 NEOFS_IR_AUDIT_PDP_MAX_SLEEP_INTERVAL=100ms diff --git a/services/ir/prepare.mk b/services/ir/prepare.mk index 5959964..fd7f253 100644 --- a/services/ir/prepare.mk +++ b/services/ir/prepare.mk @@ -1,3 +1,4 @@ # Deposit GAS from default wallet to FrostFS privnet contract prepare.ir: - @./bin/deposit.sh + @./bin/config.sh int ContainerFee 0 + @./bin/config.sh int ContainerAliasFee 0 diff --git a/services/chain/config.yml b/services/morph_chain/config.yml similarity index 100% rename from services/chain/config.yml rename to services/morph_chain/config.yml diff --git a/services/morph_chain/docker-compose.yml b/services/morph_chain/docker-compose.yml index a4cebf4..4cba4b3 100644 --- a/services/morph_chain/docker-compose.yml +++ b/services/morph_chain/docker-compose.yml @@ -2,7 +2,7 @@ version: "2.4" services: - neofs_morph_chain: + frostfs_morph_chain: image: ${NEOGO_IMAGE}:${NEOGO_VERSION} container_name: morph_chain command: ["node", "--config-path", "/config", "--privnet", "--debug"] @@ -17,6 +17,7 @@ services: volumes: - ./protocol.privnet.yml:/config/protocol.privnet.yml - ./node-wallet.json:/wallets/node-wallet.json + - ./config.yml:/wallets/config.yml - ./../../vendor/hosts:/etc/hosts - ./../../wallets/wallet.json:/wallets/wallet.json From fc999d0a63c3904f14c98be7d5fc0554d87bc9a1 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 11 Jan 2023 09:48:22 +0300 Subject: [PATCH 028/149] [TrueCloudLab#1] Remove coredns Signed-off-by: Denis Kirillov --- .env | 4 ---- .services | 1 - services/coredns/.env | 1 - services/coredns/.hosts | 1 - services/coredns/.int_test.env | 1 - services/coredns/Corefile | 8 -------- services/coredns/docker-compose.yml | 24 ------------------------ services/http_gate/.http.env | 2 +- services/http_gate/docker-compose.yml | 3 +-- services/s3_gate/.s3.env | 2 ++ services/s3_gate/docker-compose.yml | 3 +-- 11 files changed, 5 insertions(+), 45 deletions(-) delete mode 120000 services/coredns/.env delete mode 100644 services/coredns/.hosts delete mode 120000 services/coredns/.int_test.env delete mode 100644 services/coredns/Corefile delete mode 100644 services/coredns/docker-compose.yml diff --git a/.env b/.env index 4fa3839..47987c9 100644 --- a/.env +++ b/.env @@ -35,10 +35,6 @@ REST_GW_IMAGE=truecloudlab/frostfs-rest-gw S3_GW_VERSION=000d9ed4 S3_GW_IMAGE=truecloudlab/frostfs-s3-gw -# Coredns -COREDNS_VERSION=v016 -COREDNS_IMAGE=nspccdev/coredns - # FrostFS LOCODE database LOCODE_DB_URL=https://github.com/nspcc-dev/neofs-locode-db/releases/download/v0.3.0/locode_db.gz #LOCODE_DB_PATH=/path/to/locode_db diff --git a/.services b/.services index 19b229e..d9f34c7 100644 --- a/.services +++ b/.services @@ -3,4 +3,3 @@ http_gate s3_gate rest_gate -coredns diff --git a/services/coredns/.env b/services/coredns/.env deleted file mode 120000 index c7360fb..0000000 --- a/services/coredns/.env +++ /dev/null @@ -1 +0,0 @@ -../../.env \ No newline at end of file diff --git a/services/coredns/.hosts b/services/coredns/.hosts deleted file mode 100644 index 5f77ce9..0000000 --- a/services/coredns/.hosts +++ /dev/null @@ -1 +0,0 @@ -IPV4_PREFIX.53 coredns.LOCAL_DOMAIN diff --git a/services/coredns/.int_test.env b/services/coredns/.int_test.env deleted file mode 120000 index 582b6a2..0000000 --- a/services/coredns/.int_test.env +++ /dev/null @@ -1 +0,0 @@ -../../.int_test.env \ No newline at end of file diff --git a/services/coredns/Corefile b/services/coredns/Corefile deleted file mode 100644 index 852c758..0000000 --- a/services/coredns/Corefile +++ /dev/null @@ -1,8 +0,0 @@ -. { - nns http://192.168.130.90:30333 - transfer { - to * - } - log - debug -} diff --git a/services/coredns/docker-compose.yml b/services/coredns/docker-compose.yml deleted file mode 100644 index a65854d..0000000 --- a/services/coredns/docker-compose.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- - -version: "2.4" -services: - neofs_coredns: - image: ${COREDNS_IMAGE}:${COREDNS_VERSION} - container_name: coredns - domainname: ${LOCAL_DOMAIN} - hostname: coredns - networks: - coredns: - internet: - ipv4_address: ${IPV4_PREFIX}.53 - stop_signal: SIGKILL - env_file: [ ".int_test.env" ] - volumes: - - ./Corefile:/Corefile - - ./../../vendor/hosts:/etc/hosts - -networks: - coredns: - internet: - external: true - name: basenet_internet diff --git a/services/http_gate/.http.env b/services/http_gate/.http.env index 3e1b0dd..efef237 100644 --- a/services/http_gate/.http.env +++ b/services/http_gate/.http.env @@ -6,4 +6,4 @@ HTTP_GW_REBALANCE_TIMER=5m HTTP_GW_CONNECT_TIMEOUT=60s HTTP_GW_REQUEST_TIMEOUT=300s -HTTP_GW_RESOLVE_ORDER=dns +HTTP_GW_RESOLVE_ORDER=nns diff --git a/services/http_gate/docker-compose.yml b/services/http_gate/docker-compose.yml index 1e822c8..d7f9c76 100644 --- a/services/http_gate/docker-compose.yml +++ b/services/http_gate/docker-compose.yml @@ -8,8 +8,6 @@ services: hostname: http_gate container_name: http_gate restart: on-failure - dns: - - ${IPV4_PREFIX}.53 networks: http_gate_int: internet: @@ -22,6 +20,7 @@ services: environment: - HTTP_GW_WALLET_PATH=/wallet.json - HTTP_GW_WALLET_PASSPHRASE=one + - HTTP_GW_RPC_ENDPOINT=http://morph-chain.${LOCAL_DOMAIN}:30333 - HTTP_GW_PEERS_0_ADDRESS=s01.${LOCAL_DOMAIN}:8080 - HTTP_GW_PEERS_0_WEIGHT=0.2 - HTTP_GW_PEERS_1_ADDRESS=s02.${LOCAL_DOMAIN}:8080 diff --git a/services/s3_gate/.s3.env b/services/s3_gate/.s3.env index 039939d..e89578a 100644 --- a/services/s3_gate/.s3.env +++ b/services/s3_gate/.s3.env @@ -6,3 +6,5 @@ S3_GW_LOGGER_LEVEL=debug S3_GW_MAX_CLIENTS_COUNT=100 S3_GW_MAX_CLIENTS_DEADLINE=30s + +S3_GW_RESOLVE_ORDER=nns diff --git a/services/s3_gate/docker-compose.yml b/services/s3_gate/docker-compose.yml index 8cce4ff..8dd9606 100644 --- a/services/s3_gate/docker-compose.yml +++ b/services/s3_gate/docker-compose.yml @@ -8,8 +8,6 @@ services: hostname: s3_gate container_name: s3_gate restart: on-failure - dns: - - ${IPV4_PREFIX}.53 networks: s3_gate_int: internet: @@ -24,6 +22,7 @@ services: environment: - S3_GW_WALLET_PATH=/wallet.json - S3_GW_WALLET_PASSPHRASE=s3 + - S3_GW_RPC_ENDPOINT=http://morph-chain.${LOCAL_DOMAIN}:30333 - S3_GW_SERVER_0_TLS_KEY_FILE=/tls.key - S3_GW_SERVER_0_TLS_CERT_FILE=/tls.crt - S3_GW_SERVER_0_ADDRESS=s3.${LOCAL_DOMAIN}:8080 From 36f775668d3a6e3ea977f471d937b05f7b9e0dbe Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 20 Jan 2023 11:11:40 +0300 Subject: [PATCH 029/149] [TrueCloudLab#1] Use latest frostfs contracts Signed-off-by: Denis Kirillov --- .env | 12 ++++++------ services/morph_chain/artifacts.mk | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.env b/.env index 47987c9..5b35d5a 100644 --- a/.env +++ b/.env @@ -12,11 +12,11 @@ NEOGO_VERSION=0.99.4 NEOGO_IMAGE=nspccdev/neo-go # FrostFS InnerRing nodes -IR_VERSION=e3554425 +IR_VERSION=d4d2a8c8 IR_IMAGE=truecloudlab/frostfs-ir # FrostFS Storage nodes -NODE_VERSION=e3554425 +NODE_VERSION=d4d2a8c8 NODE_IMAGE=truecloudlab/frostfs-storage # NATS Server @@ -40,15 +40,15 @@ LOCODE_DB_URL=https://github.com/nspcc-dev/neofs-locode-db/releases/download/v0. #LOCODE_DB_PATH=/path/to/locode_db # FrostFS CLI binary -FROSTFS_CLI_URL=https://http.t5.fs.neo.org/AQgse8bPCZx4zScMuAKxowJdZPbKHp8NDcp15o6VUNmk/DjKaXEXVmVunGgu2pMvpqZmzZ2omNxTQHx3sUYEz1s1b +FROSTFS_CLI_URL=https://http.t5.fs.neo.org/AQgse8bPCZx4zScMuAKxowJdZPbKHp8NDcp15o6VUNmk/C6BNLpYg5gWLHp3DrXozSxxGLDahBuSBCyJoYSSR1M3Q #FROSTFS_CLI_PATH=/path/to/frostfs-cli-binary # FrostFS ADM tool binary FROSTFS_ADM_VERSION=e3554425 -FROSTFS_ADM_URL=https://http.t5.fs.neo.org/AQgse8bPCZx4zScMuAKxowJdZPbKHp8NDcp15o6VUNmk/JAqZZR6Rh7QL6NrJc9cbALNrUJenGD2VK6Mjdjsa7tzF +FROSTFS_ADM_URL=https://http.t5.fs.neo.org/AQgse8bPCZx4zScMuAKxowJdZPbKHp8NDcp15o6VUNmk/sXZxy9vbFyJiLhN9qTSXozXK7SN9H8ZC6dpvAt59Zaj #FROSTFS_ADM_PATH=/path/to/frostfs-adm-binary # Compiled FrostFS Smart Contracts -FROSTFS_CONTRACTS_VERSION=v0.16.0 -FROSTFS_CONTRACTS_URL=https://github.com/nspcc-dev/neofs-contract/releases/download/${FROSTFS_CONTRACTS_VERSION}/neofs-contract-${FROSTFS_CONTRACTS_VERSION}.tar.gz +FROSTFS_CONTRACTS_VERSION=4f3c08f5 +FROSTFS_CONTRACTS_URL=https://http.t5.fs.neo.org/AQgse8bPCZx4zScMuAKxowJdZPbKHp8NDcp15o6VUNmk/c1nGtturFrSeygYP3AyNHDDLNbs7HhJiH2BQkgZxEmZ #FROSTFS_CONTRACTS_PATH=/path/to/unpacked/frostfs-contracts-dir diff --git a/services/morph_chain/artifacts.mk b/services/morph_chain/artifacts.mk index 0f24735..3572e69 100644 --- a/services/morph_chain/artifacts.mk +++ b/services/morph_chain/artifacts.mk @@ -10,7 +10,7 @@ get.contracts: ifeq (${FROSTFS_CONTRACTS_PATH},) @echo "⇒ Download compiled FrostFS contracts from ${FROSTFS_CONTRACTS_URL}" - @curl -sSL ${FROSTFS_CONTRACTS_URL} -o ${FROSTFS_CONTRACTS_ARCHIVE} + @curl -ksSL ${FROSTFS_CONTRACTS_URL} -o ${FROSTFS_CONTRACTS_ARCHIVE} @tar -xf ${FROSTFS_CONTRACTS_ARCHIVE} -C ${FROSTFS_CONTRACTS_DEST} --strip-components 1 @rm ${FROSTFS_CONTRACTS_ARCHIVE} else From b5fb2a9543904ede5ac40525ab385e50d4d6f482 Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Thu, 24 Nov 2022 15:24:52 +0300 Subject: [PATCH 030/149] [#241] node: Provide common params with `.yml` file Signed-off-by: Pavel Karpy --- services/storage/.storage.env | 89 ----------------------------- services/storage/cfg/config.yml | 81 ++++++++++++++++++++++++++ services/storage/docker-compose.yml | 8 +++ 3 files changed, 89 insertions(+), 89 deletions(-) create mode 100644 services/storage/cfg/config.yml diff --git a/services/storage/.storage.env b/services/storage/.storage.env index 50c0d6f..e69de29 100644 --- a/services/storage/.storage.env +++ b/services/storage/.storage.env @@ -1,89 +0,0 @@ -# Logger section -## Minimum enabled logging level -## Default: info -NEOFS_LOGGER_LEVEL=debug - -# Profiler section -NEOFS_PPROF_ENABLED=true -## Server address. Empty value disables profiler -NEOFS_PPROF_ADDRESS=:6060 -## Timeout to shut down the HTTP server -## Default: 30s -#NEOFS_PROFILER_SHUTDOWN_TIMEOUT=1m - -# Application metrics section -NEOFS_PROMETHEUS_ENABLED=true -## Server address. Empty value disables metrics gathering -NEOFS_PROMETHEUS_ADDRESS=:9090 -## Timeout to shut down the HTTP server -## Default: 30s -#NEOFS_METRICS_SHUTDOWN_TIMEOUT=1m - -# Morph section -## Endpoints of sidechain RPC nodes (comma-separated) -## Default: empty list (invalid) -NEOFS_MORPH_RPC_ENDPOINT_0_ADDRESS=ws://morph-chain:30333/ws -## Dial timeout to connect to notification endpoint -## Default: 5s -#NEOFS_MORPH_DIAL_TIMEOUT=1m -# Configure side chain caches (negative value to disable) -#NEOFS_MORPH_CACHE_TTL=15s - -# Cache of API clients section -## Dial timeout to connect to remote API server -## Default: 5s -#NEOFS_APICLIENT_DIAL_TIMEOUT=1m - -# Common storage node attribute -NEOFS_NODE_ATTRIBUTE_0=User-Agent:FrostFS\/0.33 - -NEOFS_NODE_NOTIFICATION_ENABLED=true -NEOFS_NODE_NOTIFICATION_ENDPOINT=tls://nats.frostfs.devenv:4222 -NEOFS_NODE_NOTIFICATION_DEFAULT_TOPIC=test -NEOFS_NODE_NOTIFICATION_CERTIFICATE=/etc/frostfs-node/nats.tls.cert -NEOFS_NODE_NOTIFICATION_KEY=/etc/frostfs-node/nats.tls.key -NEOFS_NODE_NOTIFICATION_CA=/etc/frostfs-node/nats.ca.crt - -# Tree section -NEOFS_TREE_ENABLED=True - -# Object section -## Worker pool capacity of API Object.Put client calls -## Default: 10 -#NEOFS_OBJECT_PUT_POOL_SIZE_REMOTE=100 -#NEOFS_OBJECT_PUT_POOL_SIZE_LOCAL=100 - -# Policer section -## Timeout of Object.Head calls by Policer -## Default: 5s -#NEOFS_POLICER_HEAD_TIMEOUT=1m - -# Replicator section -## Timeout of Object.Put calls by Replicator -## Default: 5s -#NEOFS_REPLICATOR_PUT_TIMEOUT=1m - -# Storage engine configuration -NEOFS_STORAGE_SHARD_0_WRITECACHE_ENABLED=false -NEOFS_STORAGE_SHARD_0_WRITECACHE_PATH=/storage/wc0 -NEOFS_STORAGE_SHARD_0_METABASE_PATH=/storage/meta0 -NEOFS_STORAGE_SHARD_0_BLOBSTOR_0_TYPE=blobovnicza -NEOFS_STORAGE_SHARD_0_BLOBSTOR_0_PATH=/storage/blobovnicza0 -NEOFS_STORAGE_SHARD_0_BLOBSTOR_0_DEPTH=2 -NEOFS_STORAGE_SHARD_0_BLOBSTOR_0_WIDTH=4 -NEOFS_STORAGE_SHARD_0_BLOBSTOR_1_TYPE=fstree -NEOFS_STORAGE_SHARD_0_BLOBSTOR_1_PATH=/storage/fstree0 -NEOFS_STORAGE_SHARD_0_BLOBSTOR_1_DEPTH=2 -NEOFS_STORAGE_SHARD_0_PILORAMA_PATH=/storage/pilorama0 - -NEOFS_STORAGE_SHARD_1_WRITECACHE_ENABLED=false -NEOFS_STORAGE_SHARD_1_WRITECACHE_PATH=/storage/wc1 -NEOFS_STORAGE_SHARD_1_METABASE_PATH=/storage/meta1 -NEOFS_STORAGE_SHARD_1_BLOBSTOR_0_TYPE=blobovnicza -NEOFS_STORAGE_SHARD_1_BLOBSTOR_0_PATH=/storage/blobovnicza1 -NEOFS_STORAGE_SHARD_1_BLOBSTOR_0_DEPTH=2 -NEOFS_STORAGE_SHARD_1_BLOBSTOR_0_WIDTH=4 -NEOFS_STORAGE_SHARD_1_BLOBSTOR_1_TYPE=fstree -NEOFS_STORAGE_SHARD_1_BLOBSTOR_1_PATH=/storage/fstree1 -NEOFS_STORAGE_SHARD_1_BLOBSTOR_1_DEPTH=2 -NEOFS_STORAGE_SHARD_1_PILORAMA_PATH=/storage/pilorama1 diff --git a/services/storage/cfg/config.yml b/services/storage/cfg/config.yml new file mode 100644 index 0000000..49b587f --- /dev/null +++ b/services/storage/cfg/config.yml @@ -0,0 +1,81 @@ +# Logger section +logger: + level: debug # Minimum enabled logging level + +# Profiler section +pprof: + enabled: true + address: :6060 # Server address + shutdown_timeout: 15s # Timeout for profiling HTTP server graceful shutdown + +# Application metrics section +prometheus: + enabled: true + address: :9090 # Server address + shutdown_timeout: 15s # Timeout for metrics HTTP server graceful shutdown + +# Morph section +morph: + dial_timeout: 30s # Timeout for side chain NEO RPC client connection + rpc_endpoint: # Side chain NEO RPC endpoints + - address: ws://morph-chain:30333/ws + priority: 1 + +# Common storage node settings +node: + attribute_0: "User-Agent:FrostFS/0.34" + notification: + enabled: true # Turn on object notification service + endpoint: "tls://nats.frostfs.devenv:4222" # Notification server endpoint + timeout: "6s" # Timeout for object notification client connection + default_topic: "test" # Default topic for object notifications if not found in object's meta + certificate: "/etc/frostfs-node/nats.tls.cert" # Path to TLS certificate + key: "/etc/frostfs-node/nats.tls.key" # Path to TLS key + ca: "/etc/frostfs-node/nats.ca.crt" # Path to optional CA certificate + +# Tree section +tree: + enabled: true + +# Storage engine configuration +storage: + shard: + 0: + writecache: + enabled: false + path: /storage/wc0 # Write-cache root directory + + metabase: + path: /storage/meta0 # Path to the metabase + + blobstor: + - type: blobovnicza + path: /storage/blobovnicza0 # Blobovnicza root directory + depth: 2 + width: 4 + - type: fstree + path: /storage/fstree0 # FSTree root directory + depth: 2 + + pilorama: + path: /storage/pilorama0 # Path to the pilorama database + + 1: + writecache: + enabled: false + path: /storage/wc1 # Write-cache root directory + + metabase: + path: /storage/meta1 # Path to the metabase + + blobstor: + - type: blobovnicza + path: /storage/blobovnicza1 # Blobovnicza root directory + depth: 2 + width: 4 + - type: fstree + path: /storage/fstree1 # FSTree root directory + depth: 2 + + pilorama: + path: /storage/pilorama1 # Path to the pilorama database diff --git a/services/storage/docker-compose.yml b/services/storage/docker-compose.yml index 3e94644..27eceb7 100644 --- a/services/storage/docker-compose.yml +++ b/services/storage/docker-compose.yml @@ -23,8 +23,10 @@ services: - ../nats/client-cert.pem:/etc/frostfs-node/nats.tls.cert - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key - ../nats/ca-cert.pem:/etc/frostfs-node/nats.ca.crt + - ./cfg:/etc/frostfs/storage stop_signal: SIGKILL env_file: [ ".env", ".storage.env", ".int_test.env" ] + command: [ "frostfs-node", "--config", "/etc/frostfs/storage/config.yml" ] environment: - NEOFS_NODE_WALLET_PATH=./wallet.json - NEOFS_NODE_WALLET_PASSWORD= @@ -61,8 +63,10 @@ services: - ../nats/client-cert.pem:/etc/frostfs-node/nats.tls.cert - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key - ../nats/ca-cert.pem:/etc/frostfs-node/nats.ca.crt + - ./cfg:/etc/frostfs/storage stop_signal: SIGKILL env_file: [ ".env", ".storage.env", ".int_test.env" ] + command: [ "frostfs-node", "--config", "/etc/frostfs/storage/config.yml" ] environment: - NEOFS_NODE_WALLET_PATH=./wallet.json - NEOFS_NODE_WALLET_PASSWORD= @@ -99,8 +103,10 @@ services: - ../nats/client-cert.pem:/etc/frostfs-node/nats.tls.cert - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key - ../nats/ca-cert.pem:/etc/frostfs-node/nats.ca.crt + - ./cfg:/etc/frostfs/storage stop_signal: SIGKILL env_file: [ ".env", ".storage.env", ".int_test.env" ] + command: [ "frostfs-node", "--config", "/etc/frostfs/storage/config.yml" ] environment: - NEOFS_NODE_WALLET_PATH=./wallet.json - NEOFS_NODE_WALLET_PASSWORD= @@ -138,8 +144,10 @@ services: - ../nats/client-cert.pem:/etc/frostfs-node/nats.tls.cert - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key - ../nats/ca-cert.pem:/etc/frostfs-node/nats.ca.crt + - ./cfg:/etc/frostfs/storage stop_signal: SIGKILL env_file: [ ".env", ".storage.env", ".int_test.env" ] + command: [ "frostfs-node", "--config", "/etc/frostfs/storage/config.yml" ] environment: - NEOFS_NODE_WALLET_PATH=./wallet.json - NEOFS_NODE_WALLET_PASSWORD= From 1659a76b5dd5fb49ddd985264b7ba4a4f5750a6d Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Thu, 24 Nov 2022 20:27:29 +0300 Subject: [PATCH 031/149] [#241] s3: Provide common params with `.yml` file Signed-off-by: Pavel Karpy --- services/s3_gate/.s3.env | 10 ---------- services/s3_gate/cfg/config.yml | 31 +++++++++++++++++++++++++++++ services/s3_gate/docker-compose.yml | 6 ++---- 3 files changed, 33 insertions(+), 14 deletions(-) create mode 100644 services/s3_gate/cfg/config.yml diff --git a/services/s3_gate/.s3.env b/services/s3_gate/.s3.env index e89578a..e69de29 100644 --- a/services/s3_gate/.s3.env +++ b/services/s3_gate/.s3.env @@ -1,10 +0,0 @@ -S3_GW_REBALANCE_INTERVAL=30s -S3_GW_HEALTHCHECK_TIMEOUT=15s -S3_GW_CONNECT_TIMEOUT=10s - -S3_GW_LOGGER_LEVEL=debug - -S3_GW_MAX_CLIENTS_COUNT=100 -S3_GW_MAX_CLIENTS_DEADLINE=30s - -S3_GW_RESOLVE_ORDER=nns diff --git a/services/s3_gate/cfg/config.yml b/services/s3_gate/cfg/config.yml new file mode 100644 index 0000000..374e4be --- /dev/null +++ b/services/s3_gate/cfg/config.yml @@ -0,0 +1,31 @@ +logger: + level: debug + +# Interval to check node health +rebalance_interval: 30s + +# Timeout to check node health during rebalance +healthcheck_timeout: 15s + +# Timeout to connect to a node +connect_timeout: 10s + +# Limits for processing of clients' requests +max_clients_count: 100 + +# Deadline after which the gate sends error `RequestTimeout` to a client +max_clients_deadline: 30s + +resolve_order: + - nns + +server: + - tls: + enabled: true + cert_file: /tls.crt + key_file: /tls.key + +# Wallet configuration +wallet: + path: /wallet.json # Path to wallet + passphrase: "s3" # Passphrase to decrypt wallet diff --git a/services/s3_gate/docker-compose.yml b/services/s3_gate/docker-compose.yml index 8dd9606..f86caf5 100644 --- a/services/s3_gate/docker-compose.yml +++ b/services/s3_gate/docker-compose.yml @@ -17,14 +17,12 @@ services: - ./tls.key:/tls.key - ./tls.crt:/tls.crt - ./../../vendor/hosts:/etc/hosts + - ./cfg:/etc/frostfs/s3 stop_signal: SIGKILL env_file: [ ".env", ".s3.env", ".int_test.env" ] + command: [ "frostfs-s3-gw", "--config", "/etc/frostfs/s3/config.yml" ] environment: - - S3_GW_WALLET_PATH=/wallet.json - - S3_GW_WALLET_PASSPHRASE=s3 - S3_GW_RPC_ENDPOINT=http://morph-chain.${LOCAL_DOMAIN}:30333 - - S3_GW_SERVER_0_TLS_KEY_FILE=/tls.key - - S3_GW_SERVER_0_TLS_CERT_FILE=/tls.crt - S3_GW_SERVER_0_ADDRESS=s3.${LOCAL_DOMAIN}:8080 - S3_GW_LISTEN_DOMAINS=s3.${LOCAL_DOMAIN} - S3_GW_TREE_SERVICE=s01.${LOCAL_DOMAIN}:8080 From 94ca4371802cceb1cd8e33259f7dbbbd747334ff Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Thu, 24 Nov 2022 20:53:59 +0300 Subject: [PATCH 032/149] [#241] ir: Provide common params with `.yml` file Signed-off-by: Pavel Karpy --- services/ir/.ir.env | 39 ----------------- services/ir/cfg/config.yml | 76 ++++++++++++++++++++++++++++++++++ services/ir/docker-compose.yml | 6 +-- 3 files changed, 78 insertions(+), 43 deletions(-) create mode 100644 services/ir/cfg/config.yml diff --git a/services/ir/.ir.env b/services/ir/.ir.env index ddc40e6..ce4e331 100644 --- a/services/ir/.ir.env +++ b/services/ir/.ir.env @@ -1,42 +1,3 @@ -NEOFS_IR_LOGGER_LEVEL=debug - -# Profiler section -NEOFS_IR_PPROF_ENABLED=true -## Server address. Empty value disables profiler -NEOFS_IR_PPROF_ADDRESS=:6060 -## Timeout to shut down the HTTP server -## Default: 30s -#NEOFS_IR_PROFILER_SHUTDOWN_TIMEOUT=1m - -# Application metrics section -NEOFS_IR_PROMETHEUS_ENABLED=true -## Server address. Empty value disables metrics gathering -NEOFS_IR_PROMETHEUS_ADDRESS=:9090 -## Timeout to shut down the HTTP server -## Default: 30s -#NEOFS_IR_METRICS_SHUTDOWN_TIMEOUT=1m - -# Toggling the sidechain-only mode -NEOFS_IR_WITHOUT_MAINNET=true - -NEOFS_IR_MORPH_ENDPOINT_CLIENT_0_ADDRESS=ws://morph-chain:30333/ws -NEOFS_IR_TIMERS_EMIT=50 -NEOFS_IR_TIMERS_STOP_ESTIMATION_DIV=4 -NEOFS_IR_TIMERS_COLLECT_BASIC_INCOME_DIV=2 -NEOFS_IR_TIMERS_DISTRIBUTE_BASIC_INCOME_MUL=3 -NEOFS_IR_TIMERS_DISTRIBUTE_BASIC_INCOME_DIV=4 - -NEOFS_IR_EMIT_STORAGE_AMOUNT=1000000000 -NEOFS_IR_NETMAP_CLEANER_ENABLED=true - NEOFS_IR_CONTRACTS_FROSTFSID=1943e9bb78a0fe2fe0c95fd2677eec2da6aa4aa5 -NEOFS_IR_MORPH_VALIDATORS=02b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc2 -NEOFS_IR_AUDIT_PDP_MAX_SLEEP_INTERVAL=100ms - -NEOFS_IR_SETTLEMENT_BASIC_INCOME_RATE=100000000 -NEOFS_IR_SETTLEMENT_AUDIT_FEE=100000 - -NEOFS_IR_LOCODE_DB_PATH=/locode/db - NEOFS_IR_CONTROL_GRPC_ENDPOINT=127.0.0.1:16512 diff --git a/services/ir/cfg/config.yml b/services/ir/cfg/config.yml new file mode 100644 index 0000000..bb15b3b --- /dev/null +++ b/services/ir/cfg/config.yml @@ -0,0 +1,76 @@ +# Logger section +logger: + level: debug # Minimum enabled logging level + +# Wallet settings +wallet: + path: /wallet.json # Path to NEP-6 NEO wallet file + address: Nhfg3TbpwogLvDGVvAvqyThbsHgoSUKwtn # Account address in the wallet; ignore to use default address + password: one # Account password in the wallet + +# Profiler section +pprof: + enabled: true + address: :6060 # Endpoint for application pprof profiling; disabled by default + shutdown_timeout: 30s # Timeout for profiling HTTP server graceful shutdown + +# Application metrics section +prometheus: + enabled: true + address: :9090 # Endpoint for application prometheus metrics; disabled by default + shutdown_timeout: 30s # Timeout for metrics HTTP server graceful shutdown + +# Toggling the sidechain-only mode +without_mainnet: true + +# Neo main chain RPC settings +mainnet: + endpoint: + client: # List of websocket RPC endpoints in mainchain; ignore if mainchain is disabled + - address: ws://main-chain:30333/ws + +# Neo side chain RPC settings +morph: + endpoint: + client: # List of websocket RPC endpoints in sidechain + - address: ws://morph-chain:30333/ws + validators: # List of hex-encoded 33-byte public keys of sidechain validators to vote for at application startup + - 02b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc2 + +# Network time settings +timers: + emit: 50 # Number of sidechain blocks between GAS emission cycles; disabled by default + stop_estimation: + mul: 1 # Multiplier in x/y relation of when to stop basic income estimation within the epoch + div: 4 # Divider in x/y relation of when to stop basic income estimation within the epoch + collect_basic_income: + mul: 1 # Multiplier in x/y relation of when to start basic income asset collection within the epoch + div: 2 # Divider in x/y relation of when to start basic income asset collecting within the epoch + distribute_basic_income: + mul: 3 # Multiplier in x/y relation of when to start basic income asset distribution within the epoch + div: 4 # Divider in x/y relation of when to start basic income asset distribution within the epoch + +# Storage node GAS emission settings +emit: + storage: + amount: 1000000000 # Fixed8 value of sidechain GAS emitted to all storage nodes once per GAS emission cycle; disabled by default + +# Storage node removal settings +netmap_cleaner: + enabled: true # Enable voting for removing stale storage nodes from network map + threshold: 3 # Number of FrostFS epoch without bootstrap request from storage node before it considered stale + +# Audit settings +audit: + pdp: + max_sleep_interval: 100ms # Maximum timeout between object.RangeHash requests to the storage node + +# Settlement settings +settlement: + basic_income_rate: 100000000 # Optional: override basic income rate value from network config; applied only in debug mode + audit_fee: 100000 # Optional: override audit fee value from network config; applied only in debug mode + +# LOCODE database +locode: + db: + path: /locode/db # Path to UN/LOCODE database file diff --git a/services/ir/docker-compose.yml b/services/ir/docker-compose.yml index cf23e22..77ec50d 100644 --- a/services/ir/docker-compose.yml +++ b/services/ir/docker-compose.yml @@ -21,11 +21,9 @@ services: - ./../../vendor/locode_db:/locode/db - ./../../vendor/frostfs-cli:/frostfs-cli - ./healthcheck.sh:/healthcheck.sh + - ./cfg:/etc/frostfs/ir env_file: [ ".env", ".ir.env", ".int_test.env" ] - environment: - - NEOFS_IR_WALLET_PATH=/wallet.json - - NEOFS_IR_WALLET_ADDRESS=Nhfg3TbpwogLvDGVvAvqyThbsHgoSUKwtn - - NEOFS_IR_WALLET_PASSWORD=one + command: [ "frostfs-ir", "--config", "/etc/frostfs/ir/config.yml" ] healthcheck: test: ["CMD-SHELL", "/healthcheck.sh"] interval: 2s From f549ef7d393ca748d2f3fbf78ebad9e55c4c1050 Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Thu, 24 Nov 2022 21:12:50 +0300 Subject: [PATCH 033/149] [#241] http: Provide common params with `.yml` file Signed-off-by: Pavel Karpy --- services/http_gate/.http.env | 9 --------- services/http_gate/cfg/config.yml | 20 ++++++++++++++++++++ services/http_gate/docker-compose.yml | 4 ++-- 3 files changed, 22 insertions(+), 11 deletions(-) create mode 100644 services/http_gate/cfg/config.yml diff --git a/services/http_gate/.http.env b/services/http_gate/.http.env index efef237..e69de29 100644 --- a/services/http_gate/.http.env +++ b/services/http_gate/.http.env @@ -1,9 +0,0 @@ -HTTP_GW_SERVER_0_ADDRESS=0.0.0.0:80 - -HTTP_GW_LOGGER_LEVEL=info - -HTTP_GW_REBALANCE_TIMER=5m -HTTP_GW_CONNECT_TIMEOUT=60s -HTTP_GW_REQUEST_TIMEOUT=300s - -HTTP_GW_RESOLVE_ORDER=nns diff --git a/services/http_gate/cfg/config.yml b/services/http_gate/cfg/config.yml new file mode 100644 index 0000000..7cfdd40 --- /dev/null +++ b/services/http_gate/cfg/config.yml @@ -0,0 +1,20 @@ +logger: + level: debug + +rebalance_timer: 5m # Interval to check nodes health + +connect_timeout: 60s # Timeout to dial node + +request_timeout: 300s # Timeout to check node health during rebalance + +# The order in which resolvers are used to find an container id by name +resolve_order: + - nns + +server: + - address: 0.0.0.0:80 + +# Wallet settings +wallet: + path: /wallet.json # Path to wallet + passphrase: one # Passphrase to decrypt wallet diff --git a/services/http_gate/docker-compose.yml b/services/http_gate/docker-compose.yml index d7f9c76..0dd3ad5 100644 --- a/services/http_gate/docker-compose.yml +++ b/services/http_gate/docker-compose.yml @@ -15,11 +15,11 @@ services: volumes: - ./wallet.json:/wallet.json - ./../../vendor/hosts:/etc/hosts + - ./cfg:/etc/frostfs/http stop_signal: SIGKILL env_file: [ ".env", ".http.env", ".int_test.env" ] + command: [ "frostfs-http-gw", "--config", "/etc/frostfs/http/config.yml" ] environment: - - HTTP_GW_WALLET_PATH=/wallet.json - - HTTP_GW_WALLET_PASSPHRASE=one - HTTP_GW_RPC_ENDPOINT=http://morph-chain.${LOCAL_DOMAIN}:30333 - HTTP_GW_PEERS_0_ADDRESS=s01.${LOCAL_DOMAIN}:8080 - HTTP_GW_PEERS_0_WEIGHT=0.2 From 9645cf779337fca618c743afbaf04806f5324327 Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Fri, 25 Nov 2022 12:32:20 +0300 Subject: [PATCH 034/149] [#241] rest: Provide common params with `.yml` file Signed-off-by: Pavel Karpy --- services/rest_gate/cfg/config.yml | 8 ++++++++ services/rest_gate/docker-compose.yml | 5 ++--- 2 files changed, 10 insertions(+), 3 deletions(-) create mode 100644 services/rest_gate/cfg/config.yml diff --git a/services/rest_gate/cfg/config.yml b/services/rest_gate/cfg/config.yml new file mode 100644 index 0000000..f733832 --- /dev/null +++ b/services/rest_gate/cfg/config.yml @@ -0,0 +1,8 @@ +server: + # The IP and port to listen on. + listen-address: 0.0.0.0:8090 + +# Wallet settings +wallet: + path: /wallet.json # Path to wallet + passphrase: one # Password to decrypt wallet diff --git a/services/rest_gate/docker-compose.yml b/services/rest_gate/docker-compose.yml index 478d25b..f5bac6b 100644 --- a/services/rest_gate/docker-compose.yml +++ b/services/rest_gate/docker-compose.yml @@ -15,12 +15,11 @@ services: volumes: - ./wallet.json:/wallet.json - ./../../vendor/hosts:/etc/hosts + - ./cfg:/etc/frostfs/rest stop_signal: SIGKILL env_file: [ ".env", ".int_test.env" ] + command: [ "frostfs-rest-gw", "--config", "/etc/frostfs/rest/config.yml" ] environment: - - REST_GW_WALLET_PATH=/wallet.json - - REST_GW_WALLET_PASSPHRASE=one - - REST_GW_SERVER_LISTEN_ADDRESS=0.0.0.0:8090 - REST_GW_POOL_PEERS_0_ADDRESS=s01.${LOCAL_DOMAIN}:8080 - REST_GW_POOL_PEERS_1_ADDRESS=s02.${LOCAL_DOMAIN}:8080 - REST_GW_POOL_PEERS_2_ADDRESS=s03.${LOCAL_DOMAIN}:8080 From 69bd34f2f6b6029495050aa5fc490df9626ec466 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Tue, 7 Feb 2023 11:57:26 +0300 Subject: [PATCH 035/149] [#1] Update env prefixes in builds Signed-off-by: Alex Vanin --- .env | 4 +- services/ir/.ir.env | 4 +- services/ir/healthcheck.sh | 2 +- services/storage/docker-compose.yml | 66 ++++++++++++++--------------- services/storage/healthcheck.sh | 2 +- 5 files changed, 39 insertions(+), 39 deletions(-) diff --git a/.env b/.env index 5b35d5a..c7955fc 100644 --- a/.env +++ b/.env @@ -12,11 +12,11 @@ NEOGO_VERSION=0.99.4 NEOGO_IMAGE=nspccdev/neo-go # FrostFS InnerRing nodes -IR_VERSION=d4d2a8c8 +IR_VERSION=5ffa8268 IR_IMAGE=truecloudlab/frostfs-ir # FrostFS Storage nodes -NODE_VERSION=d4d2a8c8 +NODE_VERSION=5ffa8268 NODE_IMAGE=truecloudlab/frostfs-storage # NATS Server diff --git a/services/ir/.ir.env b/services/ir/.ir.env index ce4e331..33df607 100644 --- a/services/ir/.ir.env +++ b/services/ir/.ir.env @@ -1,3 +1,3 @@ -NEOFS_IR_CONTRACTS_FROSTFSID=1943e9bb78a0fe2fe0c95fd2677eec2da6aa4aa5 +FROSTFS_IR_CONTRACTS_FROSTFSID=1943e9bb78a0fe2fe0c95fd2677eec2da6aa4aa5 -NEOFS_IR_CONTROL_GRPC_ENDPOINT=127.0.0.1:16512 +FROSTFS_IR_CONTROL_GRPC_ENDPOINT=127.0.0.1:16512 diff --git a/services/ir/healthcheck.sh b/services/ir/healthcheck.sh index 6e8ac3f..747bc8b 100755 --- a/services/ir/healthcheck.sh +++ b/services/ir/healthcheck.sh @@ -1,6 +1,6 @@ #!/bin/sh /frostfs-cli control healthcheck \ - --endpoint "$NEOFS_IR_CONTROL_GRPC_ENDPOINT" \ + --endpoint "$FROSTFS_IR_CONTROL_GRPC_ENDPOINT" \ --wallet /wallet01.key --ir | grep "Health status: READY" diff --git a/services/storage/docker-compose.yml b/services/storage/docker-compose.yml index 27eceb7..b32ba21 100644 --- a/services/storage/docker-compose.yml +++ b/services/storage/docker-compose.yml @@ -28,13 +28,13 @@ services: env_file: [ ".env", ".storage.env", ".int_test.env" ] command: [ "frostfs-node", "--config", "/etc/frostfs/storage/config.yml" ] environment: - - NEOFS_NODE_WALLET_PATH=./wallet.json - - NEOFS_NODE_WALLET_PASSWORD= - - NEOFS_NODE_ADDRESSES=s01.${LOCAL_DOMAIN}:8080 - - NEOFS_GRPC_0_ENDPOINT=s01.${LOCAL_DOMAIN}:8080 - - NEOFS_CONTROL_GRPC_ENDPOINT=s01.${LOCAL_DOMAIN}:8081 - - NEOFS_NODE_ATTRIBUTE_1=UN-LOCODE:RU MOW - - NEOFS_NODE_ATTRIBUTE_2=Price:22 + - FROSTFS_NODE_WALLET_PATH=./wallet.json + - FROSTFS_NODE_WALLET_PASSWORD= + - FROSTFS_NODE_ADDRESSES=s01.${LOCAL_DOMAIN}:8080 + - FROSTFS_GRPC_0_ENDPOINT=s01.${LOCAL_DOMAIN}:8080 + - FROSTFS_CONTROL_GRPC_ENDPOINT=s01.${LOCAL_DOMAIN}:8081 + - FROSTFS_NODE_ATTRIBUTE_1=UN-LOCODE:RU MOW + - FROSTFS_NODE_ATTRIBUTE_2=Price:22 healthcheck: test: ["CMD-SHELL", "/healthcheck.sh"] interval: 2s @@ -68,13 +68,13 @@ services: env_file: [ ".env", ".storage.env", ".int_test.env" ] command: [ "frostfs-node", "--config", "/etc/frostfs/storage/config.yml" ] environment: - - NEOFS_NODE_WALLET_PATH=./wallet.json - - NEOFS_NODE_WALLET_PASSWORD= - - NEOFS_NODE_ADDRESSES=s02.${LOCAL_DOMAIN}:8080 - - NEOFS_GRPC_0_ENDPOINT=s02.${LOCAL_DOMAIN}:8080 - - NEOFS_CONTROL_GRPC_ENDPOINT=s02.${LOCAL_DOMAIN}:8081 - - NEOFS_NODE_ATTRIBUTE_1=UN-LOCODE:RU LED - - NEOFS_NODE_ATTRIBUTE_2=Price:33 + - FROSTFS_NODE_WALLET_PATH=./wallet.json + - FROSTFS_NODE_WALLET_PASSWORD= + - FROSTFS_NODE_ADDRESSES=s02.${LOCAL_DOMAIN}:8080 + - FROSTFS_GRPC_0_ENDPOINT=s02.${LOCAL_DOMAIN}:8080 + - FROSTFS_CONTROL_GRPC_ENDPOINT=s02.${LOCAL_DOMAIN}:8081 + - FROSTFS_NODE_ATTRIBUTE_1=UN-LOCODE:RU LED + - FROSTFS_NODE_ATTRIBUTE_2=Price:33 healthcheck: test: ["CMD-SHELL", "/healthcheck.sh"] interval: 2s @@ -108,13 +108,13 @@ services: env_file: [ ".env", ".storage.env", ".int_test.env" ] command: [ "frostfs-node", "--config", "/etc/frostfs/storage/config.yml" ] environment: - - NEOFS_NODE_WALLET_PATH=./wallet.json - - NEOFS_NODE_WALLET_PASSWORD= - - NEOFS_NODE_ADDRESSES=s03.${LOCAL_DOMAIN}:8080 - - NEOFS_GRPC_0_ENDPOINT=s03.${LOCAL_DOMAIN}:8080 - - NEOFS_CONTROL_GRPC_ENDPOINT=s03.${LOCAL_DOMAIN}:8081 - - NEOFS_NODE_ATTRIBUTE_1=UN-LOCODE:SE STO - - NEOFS_NODE_ATTRIBUTE_2=Price:11 + - FROSTFS_NODE_WALLET_PATH=./wallet.json + - FROSTFS_NODE_WALLET_PASSWORD= + - FROSTFS_NODE_ADDRESSES=s03.${LOCAL_DOMAIN}:8080 + - FROSTFS_GRPC_0_ENDPOINT=s03.${LOCAL_DOMAIN}:8080 + - FROSTFS_CONTROL_GRPC_ENDPOINT=s03.${LOCAL_DOMAIN}:8081 + - FROSTFS_NODE_ATTRIBUTE_1=UN-LOCODE:SE STO + - FROSTFS_NODE_ATTRIBUTE_2=Price:11 healthcheck: test: ["CMD-SHELL", "/healthcheck.sh"] interval: 2s @@ -149,18 +149,18 @@ services: env_file: [ ".env", ".storage.env", ".int_test.env" ] command: [ "frostfs-node", "--config", "/etc/frostfs/storage/config.yml" ] environment: - - NEOFS_NODE_WALLET_PATH=./wallet.json - - NEOFS_NODE_WALLET_PASSWORD= - - NEOFS_NODE_ADDRESSES=s04.${LOCAL_DOMAIN}:8080 grpcs://s04.${LOCAL_DOMAIN}:8082 - - NEOFS_CONTROL_GRPC_ENDPOINT=s04.${LOCAL_DOMAIN}:8081 - - NEOFS_GRPC_NUM=2 - - NEOFS_GRPC_0_ENDPOINT=s04.${LOCAL_DOMAIN}:8080 - - NEOFS_GRPC_1_ENDPOINT=s04.${LOCAL_DOMAIN}:8082 - - NEOFS_GRPC_1_TLS_ENABLED=true - - NEOFS_GRPC_1_TLS_CERTIFICATE=/tls.crt - - NEOFS_GRPC_1_TLS_KEY=/tls.key - - NEOFS_NODE_ATTRIBUTE_1=UN-LOCODE:FI HEL - - NEOFS_NODE_ATTRIBUTE_2=Price:44 + - FROSTFS_NODE_WALLET_PATH=./wallet.json + - FROSTFS_NODE_WALLET_PASSWORD= + - FROSTFS_NODE_ADDRESSES=s04.${LOCAL_DOMAIN}:8080 grpcs://s04.${LOCAL_DOMAIN}:8082 + - FROSTFS_CONTROL_GRPC_ENDPOINT=s04.${LOCAL_DOMAIN}:8081 + - FROSTFS_GRPC_NUM=2 + - FROSTFS_GRPC_0_ENDPOINT=s04.${LOCAL_DOMAIN}:8080 + - FROSTFS_GRPC_1_ENDPOINT=s04.${LOCAL_DOMAIN}:8082 + - FROSTFS_GRPC_1_TLS_ENABLED=true + - FROSTFS_GRPC_1_TLS_CERTIFICATE=/tls.crt + - FROSTFS_GRPC_1_TLS_KEY=/tls.key + - FROSTFS_NODE_ATTRIBUTE_1=UN-LOCODE:FI HEL + - FROSTFS_NODE_ATTRIBUTE_2=Price:44 healthcheck: test: ["CMD-SHELL", "/healthcheck.sh"] interval: 2s diff --git a/services/storage/healthcheck.sh b/services/storage/healthcheck.sh index bdc5baf..792d2ad 100755 --- a/services/storage/healthcheck.sh +++ b/services/storage/healthcheck.sh @@ -1,5 +1,5 @@ #!/bin/sh /frostfs-cli control healthcheck -c /cli-cfg.yml \ - --endpoint "$NEOFS_CONTROL_GRPC_ENDPOINT" | + --endpoint "$FROSTFS_CONTROL_GRPC_ENDPOINT" | grep "Health status: READY" From 37507199e2909ee0ca0cd10703f9f9622e1315b0 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Tue, 7 Feb 2023 11:58:13 +0300 Subject: [PATCH 036/149] [#1] Deploy free containers because main chain is not available Signed-off-by: Alex Vanin --- frostfs-adm.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frostfs-adm.yml b/frostfs-adm.yml index ba2039c..dcc4ba2 100644 --- a/frostfs-adm.yml +++ b/frostfs-adm.yml @@ -7,8 +7,8 @@ network: fee: audit: 10000 candidate: 10000000000 - container: 1000 - container_alias: 500 + container: 0 + container_alias: 0 withdraw: 100000000 credentials: az: "one" From 93435bfb63d8b40aff45b37937c50cc001492f4d Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 17 Mar 2023 18:39:40 +0300 Subject: [PATCH 037/149] [#13] dev-env: Add tracing support Add Jaeger service as tracing collector. Signed-off-by: Dmitrii Stepanov --- .bootstrap_services | 1 + .env | 4 ++++ services/jaeger/.env | 1 + services/jaeger/.hosts | 1 + services/jaeger/.int_test.env | 1 + services/jaeger/.jaeger.env | 0 services/jaeger/docker-compose.yml | 29 +++++++++++++++++++++++++++++ 7 files changed, 37 insertions(+) create mode 120000 services/jaeger/.env create mode 100644 services/jaeger/.hosts create mode 120000 services/jaeger/.int_test.env create mode 100644 services/jaeger/.jaeger.env create mode 100644 services/jaeger/docker-compose.yml diff --git a/.bootstrap_services b/.bootstrap_services index 3071b48..b4cf622 100644 --- a/.bootstrap_services +++ b/.bootstrap_services @@ -2,3 +2,4 @@ # Will start from top to bottom and stop in reverse basenet morph_chain +jaeger diff --git a/.env b/.env index c7955fc..6fec3eb 100644 --- a/.env +++ b/.env @@ -52,3 +52,7 @@ FROSTFS_ADM_URL=https://http.t5.fs.neo.org/AQgse8bPCZx4zScMuAKxowJdZPbKHp8NDcp15 FROSTFS_CONTRACTS_VERSION=4f3c08f5 FROSTFS_CONTRACTS_URL=https://http.t5.fs.neo.org/AQgse8bPCZx4zScMuAKxowJdZPbKHp8NDcp15o6VUNmk/c1nGtturFrSeygYP3AyNHDDLNbs7HhJiH2BQkgZxEmZ #FROSTFS_CONTRACTS_PATH=/path/to/unpacked/frostfs-contracts-dir + +# Jaeger tracing +JAEGER_VERSION=1.42.0 +JAEGER_IMAGE=jaegertracing/all-in-one \ No newline at end of file diff --git a/services/jaeger/.env b/services/jaeger/.env new file mode 120000 index 0000000..c7360fb --- /dev/null +++ b/services/jaeger/.env @@ -0,0 +1 @@ +../../.env \ No newline at end of file diff --git a/services/jaeger/.hosts b/services/jaeger/.hosts new file mode 100644 index 0000000..06f6ef2 --- /dev/null +++ b/services/jaeger/.hosts @@ -0,0 +1 @@ +IPV4_PREFIX.120 jaeger.LOCAL_DOMAIN diff --git a/services/jaeger/.int_test.env b/services/jaeger/.int_test.env new file mode 120000 index 0000000..582b6a2 --- /dev/null +++ b/services/jaeger/.int_test.env @@ -0,0 +1 @@ +../../.int_test.env \ No newline at end of file diff --git a/services/jaeger/.jaeger.env b/services/jaeger/.jaeger.env new file mode 100644 index 0000000..e69de29 diff --git a/services/jaeger/docker-compose.yml b/services/jaeger/docker-compose.yml new file mode 100644 index 0000000..be045f8 --- /dev/null +++ b/services/jaeger/docker-compose.yml @@ -0,0 +1,29 @@ +version: '2.4' +services: + jaeger: + image: ${JAEGER_IMAGE}:${JAEGER_VERSION} + domainname: ${LOCAL_DOMAIN} + hostname: jaeger + container_name: jaeger + restart: on-failure + networks: + jaeger_int: + internet: + ipv4_address: ${IPV4_PREFIX}.120 + volumes: + - ./../../vendor/hosts:/etc/hosts + ports: + - '4317:4317' #OTLP over gRPC + - '4318:4318' #OTLP over HTTP + - '16686:16686' #frontend + stop_signal: SIGKILL + env_file: [ ".env", ".jaeger.env", ".int_test.env" ] + environment: + - COLLECTOR_OTLP_ENABLED=true + - MEMORY_MAX_TRACES=100000 + +networks: + jaeger_int: + internet: + external: true + name: basenet_internet \ No newline at end of file From b3a81ca14a4964f8b2c2dc1350aba3151544d56f Mon Sep 17 00:00:00 2001 From: Liza Date: Wed, 22 Mar 2023 21:00:33 +0300 Subject: [PATCH 038/149] [#14] Add Issue Template Add bug report and feature request templates Signed-off-by: Liza --- .github/ISSUE_TEMPLATE/bug_report.md | 45 +++++++++++++++++++++++ .github/ISSUE_TEMPLATE/config.yml | 1 + .github/ISSUE_TEMPLATE/feature_request.md | 20 ++++++++++ 3 files changed, 66 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..7e778d2 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,45 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: community, triage, bug +assignees: '' + +--- + + + +## Expected Behavior + + + +## Current Behavior + + + +## Possible Solution + + + + + + +## Steps to Reproduce (for bugs) + + + +1. + +## Context + + + +## Regression + + + +## Your Environment + +* Version used: +* Server setup and configuration: +* Operating System and version (`uname -a`): diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..3ba13e0 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1 @@ +blank_issues_enabled: false diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..d6d1162 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: community, triage +assignees: '' + +--- + +## Is your feature request related to a problem? Please describe. + + +## Describe the solution you'd like + + +## Describe alternatives you've considered + + +## Additional context + From 69cbe189cb517fb56be6cdbaec0d9d89e07c27fe Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 27 Mar 2023 15:35:36 +0300 Subject: [PATCH 039/149] .env: Update neo-go to v0.100.1 Signed-off-by: Evgenii Stratonikov --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 6fec3eb..b2ac4ef 100644 --- a/.env +++ b/.env @@ -8,7 +8,7 @@ BASTION_VERSION=10 BASTION_IMAGE=debian # NeoGo privnet -NEOGO_VERSION=0.99.4 +NEOGO_VERSION=0.100.1 NEOGO_IMAGE=nspccdev/neo-go # FrostFS InnerRing nodes From dca6ff620a45d0ff2b951456e1db15857e4d6aaf Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 10 Apr 2023 09:40:33 +0300 Subject: [PATCH 040/149] [#16] dev-env: Add prometheus Signed-off-by: Dmitrii Stepanov --- .bootstrap_services | 1 + .env | 6 +++++- services/prometheus/.env | 1 + services/prometheus/.hosts | 1 + services/prometheus/.int_test.env | 1 + services/prometheus/.prometheus.env | 0 services/prometheus/docker-compose.yml | 27 ++++++++++++++++++++++++++ services/prometheus/prometheus.yml | 19 ++++++++++++++++++ 8 files changed, 55 insertions(+), 1 deletion(-) create mode 120000 services/prometheus/.env create mode 100644 services/prometheus/.hosts create mode 120000 services/prometheus/.int_test.env create mode 100644 services/prometheus/.prometheus.env create mode 100644 services/prometheus/docker-compose.yml create mode 100644 services/prometheus/prometheus.yml diff --git a/.bootstrap_services b/.bootstrap_services index b4cf622..02237d1 100644 --- a/.bootstrap_services +++ b/.bootstrap_services @@ -3,3 +3,4 @@ basenet morph_chain jaeger +prometheus diff --git a/.env b/.env index b2ac4ef..c33accc 100644 --- a/.env +++ b/.env @@ -55,4 +55,8 @@ FROSTFS_CONTRACTS_URL=https://http.t5.fs.neo.org/AQgse8bPCZx4zScMuAKxowJdZPbKHp8 # Jaeger tracing JAEGER_VERSION=1.42.0 -JAEGER_IMAGE=jaegertracing/all-in-one \ No newline at end of file +JAEGER_IMAGE=jaegertracing/all-in-one + +# Prometheus monitoring +PROMETHEUS_VERSION=v2.43.0 +PROMETHEUS_IMAGE=prom/prometheus \ No newline at end of file diff --git a/services/prometheus/.env b/services/prometheus/.env new file mode 120000 index 0000000..c7360fb --- /dev/null +++ b/services/prometheus/.env @@ -0,0 +1 @@ +../../.env \ No newline at end of file diff --git a/services/prometheus/.hosts b/services/prometheus/.hosts new file mode 100644 index 0000000..89305b2 --- /dev/null +++ b/services/prometheus/.hosts @@ -0,0 +1 @@ +IPV4_PREFIX.121 prometheus.LOCAL_DOMAIN diff --git a/services/prometheus/.int_test.env b/services/prometheus/.int_test.env new file mode 120000 index 0000000..582b6a2 --- /dev/null +++ b/services/prometheus/.int_test.env @@ -0,0 +1 @@ +../../.int_test.env \ No newline at end of file diff --git a/services/prometheus/.prometheus.env b/services/prometheus/.prometheus.env new file mode 100644 index 0000000..e69de29 diff --git a/services/prometheus/docker-compose.yml b/services/prometheus/docker-compose.yml new file mode 100644 index 0000000..aef2c67 --- /dev/null +++ b/services/prometheus/docker-compose.yml @@ -0,0 +1,27 @@ +version: '2.4' +services: + prometheus: + image: ${PROMETHEUS_IMAGE}:${PROMETHEUS_VERSION} + domainname: ${LOCAL_DOMAIN} + hostname: prometheus + container_name: prometheus + restart: on-failure + networks: + prometheus_int: + internet: + ipv4_address: ${IPV4_PREFIX}.121 + volumes: + - ./../../vendor/hosts:/etc/hosts + - ./prometheus.yml:/etc/prometheus/prometheus.yml + command: + - --config.file=/etc/prometheus/prometheus.yml + ports: + - '9090:9090' + stop_signal: SIGKILL + env_file: [ ".env", ".prometheus.env", ".int_test.env" ] + +networks: + prometheus_int: + internet: + external: true + name: basenet_internet \ No newline at end of file diff --git a/services/prometheus/prometheus.yml b/services/prometheus/prometheus.yml new file mode 100644 index 0000000..1ea7300 --- /dev/null +++ b/services/prometheus/prometheus.yml @@ -0,0 +1,19 @@ +global: + scrape_interval: 15s + +scrape_configs: + - job_name: 'node' + static_configs: + - targets: ['s01.frostfs.devenv:9090', 's02.frostfs.devenv:9090', 's03.frostfs.devenv:9090', 's04.frostfs.devenv:9090'] + - job_name: 'http-gw' + static_configs: + - targets: ['http.frostfs.devenv:9090'] + - job_name: 'rest-gw' + static_configs: + - targets: ['rest.frostfs.devenv:9090'] + - job_name: 's3-gw' + static_configs: + - targets: ['s3.frostfs.devenv:9090'] + - job_name: 'neo-go' + static_configs: + - targets: ['morph-chain.frostfs.devenv:20001'] \ No newline at end of file From 11ff8a81cc05285578aa04b3262dbe8c4b17905b Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 10 Apr 2023 09:40:41 +0300 Subject: [PATCH 041/149] [#16] dev-env: Enable metrics for services. Signed-off-by: Dmitrii Stepanov --- services/http_gate/cfg/config.yml | 4 ++++ services/rest_gate/cfg/config.yml | 4 ++++ services/s3_gate/cfg/config.yml | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/services/http_gate/cfg/config.yml b/services/http_gate/cfg/config.yml index 7cfdd40..1684c58 100644 --- a/services/http_gate/cfg/config.yml +++ b/services/http_gate/cfg/config.yml @@ -1,5 +1,9 @@ logger: level: debug + +prometheus: + enabled: true + address: :9090 rebalance_timer: 5m # Interval to check nodes health diff --git a/services/rest_gate/cfg/config.yml b/services/rest_gate/cfg/config.yml index f733832..0acdad3 100644 --- a/services/rest_gate/cfg/config.yml +++ b/services/rest_gate/cfg/config.yml @@ -1,3 +1,7 @@ +prometheus: + enabled: true + address: :9090 + server: # The IP and port to listen on. listen-address: 0.0.0.0:8090 diff --git a/services/s3_gate/cfg/config.yml b/services/s3_gate/cfg/config.yml index 374e4be..7fa041a 100644 --- a/services/s3_gate/cfg/config.yml +++ b/services/s3_gate/cfg/config.yml @@ -1,6 +1,10 @@ logger: level: debug +prometheus: + enabled: true + address: :9090 + # Interval to check node health rebalance_interval: 30s From c7b4a8eb015dca3aac3ee31d91f916508da4519b Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 11 Apr 2023 15:44:58 +0300 Subject: [PATCH 042/149] [#17] storage: Enable tracing Signed-off-by: Dmitrii Stepanov --- services/storage/cfg/config.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/services/storage/cfg/config.yml b/services/storage/cfg/config.yml index 49b587f..fa50f4e 100644 --- a/services/storage/cfg/config.yml +++ b/services/storage/cfg/config.yml @@ -14,6 +14,12 @@ prometheus: address: :9090 # Server address shutdown_timeout: 15s # Timeout for metrics HTTP server graceful shutdown + # Application tracing section +tracing: + enabled: true + exporter: otlp_grpc + endpoint: "jaeger.frostfs.devenv:4317" + # Morph section morph: dial_timeout: 30s # Timeout for side chain NEO RPC client connection From 976f16803d5b901e171b39d7cc03fb19286af22e Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 28 Apr 2023 11:14:14 +0300 Subject: [PATCH 043/149] [#18] frostfs-adm.yml: Add alphabet-wallets Signed-off-by: Evgenii Stratonikov --- Makefile | 4 ++-- frostfs-adm.yml | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 61782ba..1f00ce8 100644 --- a/Makefile +++ b/Makefile @@ -75,8 +75,8 @@ up/basic: up/bootstrap up/bootstrap: get vendor/hosts @$(foreach SVC, $(START_BOOTSTRAP), $(shell docker-compose -f services/$(SVC)/docker-compose.yml up -d)) @source ./bin/helper.sh - @./vendor/frostfs-adm --config frostfs-adm.yml morph init --alphabet-wallets ./services/ir --contracts vendor/contracts || die "Failed to initialize Alphabet wallets" - @for f in ./services/storage/wallet*.json; do echo "Transfer GAS to wallet $${f}" && ./vendor/frostfs-adm -c frostfs-adm.yml morph refill-gas --storage-wallet $${f} --gas 10.0 --alphabet-wallets services/ir || die "Failed to transfer GAS to alphabet wallets"; done + @./vendor/frostfs-adm --config frostfs-adm.yml morph init --contracts vendor/contracts || die "Failed to initialize Alphabet wallets" + @for f in ./services/storage/wallet*.json; do echo "Transfer GAS to wallet $${f}" && ./vendor/frostfs-adm -c frostfs-adm.yml morph refill-gas --storage-wallet $${f} --gas 10.0 || die "Failed to transfer GAS to alphabet wallets"; done @echo "FrostFS sidechain environment is deployed" # Build up certain service diff --git a/frostfs-adm.yml b/frostfs-adm.yml index dcc4ba2..c44046d 100644 --- a/frostfs-adm.yml +++ b/frostfs-adm.yml @@ -1,4 +1,5 @@ rpc-endpoint: http://morph-chain.frostfs.devenv:30333 +alphabet-wallets: ./services/ir network: max_object_size: 67108864 epoch_duration: 240 From db0ebb7349745a11b241c994fc287cf6469f144b Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 28 Apr 2023 11:22:37 +0300 Subject: [PATCH 044/149] [#18] Makefile: Split long `for` line Signed-off-by: Evgenii Stratonikov --- Makefile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1f00ce8..d0dca8a 100644 --- a/Makefile +++ b/Makefile @@ -76,7 +76,11 @@ up/bootstrap: get vendor/hosts @$(foreach SVC, $(START_BOOTSTRAP), $(shell docker-compose -f services/$(SVC)/docker-compose.yml up -d)) @source ./bin/helper.sh @./vendor/frostfs-adm --config frostfs-adm.yml morph init --contracts vendor/contracts || die "Failed to initialize Alphabet wallets" - @for f in ./services/storage/wallet*.json; do echo "Transfer GAS to wallet $${f}" && ./vendor/frostfs-adm -c frostfs-adm.yml morph refill-gas --storage-wallet $${f} --gas 10.0 || die "Failed to transfer GAS to alphabet wallets"; done + @for f in ./services/storage/wallet*.json; do \ + echo "Transfer GAS to wallet $${f}" \ + && ./vendor/frostfs-adm -c frostfs-adm.yml morph refill-gas --storage-wallet $${f} --gas 10.0 \ + || die "Failed to transfer GAS to alphabet wallets"; \ + done @echo "FrostFS sidechain environment is deployed" # Build up certain service From 6d6aaefaec3af2440ffbc62246dc7a04bd19a67a Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 28 Apr 2023 11:24:32 +0300 Subject: [PATCH 045/149] [#18] bin: Replace some commands with frostfs-adm Signed-off-by: Evgenii Stratonikov --- Makefile | 6 ++--- bin/config.sh | 50 ------------------------------------------ bin/resolve.sh | 22 ------------------- bin/tick.sh | 49 ----------------------------------------- frostfs_config.mk | 24 +++++++++----------- services/ir/prepare.mk | 3 +-- 6 files changed, 14 insertions(+), 140 deletions(-) delete mode 100755 bin/config.sh delete mode 100755 bin/resolve.sh delete mode 100755 bin/tick.sh diff --git a/Makefile b/Makefile index d0dca8a..0322582 100644 --- a/Makefile +++ b/Makefile @@ -66,8 +66,8 @@ up: up/basic .PHONY: up/basic up/basic: up/bootstrap @$(foreach SVC, $(START_BASIC), $(shell docker-compose -f services/$(SVC)/docker-compose.yml up -d)) - @./bin/tick.sh - @./bin/config.sh string SystemDNS container + @./vendor/frostfs-adm -c ./frostfs-adm.yml morph force-new-epoch + @./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config SystemDNS=container --force @echo "Basic FrostFS Developer Environment is ready" # Start bootstrap services @@ -75,7 +75,7 @@ up/basic: up/bootstrap up/bootstrap: get vendor/hosts @$(foreach SVC, $(START_BOOTSTRAP), $(shell docker-compose -f services/$(SVC)/docker-compose.yml up -d)) @source ./bin/helper.sh - @./vendor/frostfs-adm --config frostfs-adm.yml morph init --contracts vendor/contracts || die "Failed to initialize Alphabet wallets" + @./vendor/frostfs-adm --config frostfs-adm.yml morph init --contracts vendor/contracts @for f in ./services/storage/wallet*.json; do \ echo "Transfer GAS to wallet $${f}" \ && ./vendor/frostfs-adm -c frostfs-adm.yml morph refill-gas --storage-wallet $${f} --gas 10.0 \ diff --git a/bin/config.sh b/bin/config.sh deleted file mode 100755 index 31b613f..0000000 --- a/bin/config.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env bash - -echo "Running bin/config.sh" - -# Source env settings -. .env -. services/ir/.ir.env -source bin/helper.sh - -# NeoGo binary path. -NEOGO="${NEOGO:-docker exec morph_chain neo-go}" - -# Wallet files to change config value -WALLET="${WALLET:-services/morph_chain/node-wallet.json}" -CONFIG_IMG="${CONFIG_IMG:-/wallets/config.yml}" - -NETMAP_ADDR=$(bin/resolve.sh netmap.frostfs) || die "Failed to resolve 'netmap.frostfs' domain name" - -# FrostFS configuration record: variable type [string|int|etc], -# key is a string and value is a constant of given type -TYPE=${1} -KEY=${2} -VALUE="${3}" - -[ -z "$TYPE" ] && echo "Empty config value type" && exit 1 -[ -z "$KEY" ] && echo "Empty config key" && exit 1 -[ -z "$VALUE" ] && echo "Empty config value" && exit 1 - -# Internal variables -if [[ -z "${FROSTFS_NOTARY_DISABLED}" ]]; then - ADDR=$(jq -r .accounts[2].address < "${WALLET}" || die "Cannot get address from ${WALLET}") -else - ADDR=$(jq -r .accounts[0].address < "${WALLET}" || die "Cannot get address from ${WALLET}") -fi - -# Change config value in side chain -echo "Changing ${KEY} configration value to ${VALUE}" - -# shellcheck disable=SC2086 -${NEOGO} contract invokefunction \ - --wallet-config ${CONFIG_IMG} \ - -a ${ADDR} --force \ - -r http://morph-chain.${LOCAL_DOMAIN}:30333 \ - ${NETMAP_ADDR} \ - setConfig bytes:beefcafe \ - string:${KEY} \ - ${TYPE}:${VALUE} -- ${ADDR} || exit 1 - -# Update epoch to apply new configuration value -./bin/tick.sh diff --git a/bin/resolve.sh b/bin/resolve.sh deleted file mode 100755 index 9aba65b..0000000 --- a/bin/resolve.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash - -# Source env settings -. .env -source bin/helper.sh - -# NeoGo binary path. -NEOGO="${NEOGO:-docker exec morph_chain neo-go}" -# NNS contract script hash -output=$(curl -s --data '{ "id": 1, "jsonrpc": "2.0", "method": "getcontractstate", "params": [1] }' \ - "http://morph-chain.${LOCAL_DOMAIN}:30333/") \ - || die "Cannot fetch NNS contract state" - -NNS_ADDR=$(jq -r '.result.hash' <<< "$output") \ - || die "Cannot parse NNS contract hash: $NNS_ADDR" - -${NEOGO} contract testinvokefunction \ - -r "http://morph-chain.${LOCAL_DOMAIN}:30333" \ - "${NNS_ADDR}" resolve string:"${1}" int:16 \ - | jq -r '.stack[0].value | if type=="array" then .[0].value else . end' \ - | base64 -d \ - || die "Cannot invoke 'NNS.resolve' $output" diff --git a/bin/tick.sh b/bin/tick.sh deleted file mode 100755 index 75482f0..0000000 --- a/bin/tick.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env bash - -echo "Running bin/tick.sh" - -# Source env settings -. .env -. services/ir/.ir.env -source bin/helper.sh - -# NeoGo binary path. -NEOGO="${NEOGO:-docker exec morph_chain neo-go}" - -# Wallet files to change config value -WALLET="${WALLET:-services/morph_chain/node-wallet.json}" -CONFIG_IMG="${CONFIG_IMG:-/wallets/config.yml}" - -# Internal variables -if [[ -z "${FROSTFS_NOTARY_DISABLED}" ]]; then - ADDR=$(jq -r .accounts[2].address < "${WALLET}" || die "Cannot get address from ${WALLET}") -else - ADDR=$(jq -r .accounts[0].address < "${WALLET}" || die "Cannot get address from ${WALLET}") -fi - -# Grep Morph block time -SIDECHAIN_PROTO="${SIDECHAIN_PROTO:-services/morph_chain/protocol.privnet.yml}" -BLOCK_DURATION=$(grep SecondsPerBlock < "$SIDECHAIN_PROTO" | awk '{print $2}') \ - || die "Cannot fetch block duration" -NETMAP_ADDR=$(bin/resolve.sh netmap.frostfs) || die "Cannot resolve netmap.frostfs" - -# Fetch current epoch value -EPOCH=$(${NEOGO} contract testinvokefunction \ - -r "http://morph-chain.${LOCAL_DOMAIN}:30333" "${NETMAP_ADDR}" epoch \ - | grep 'value' | awk -F'"' '{ print $4 }') \ - || die "Cannot fetch epoch from netmap contract" - -echo "Updating FrostFS epoch to $((EPOCH+1))" - -# shellcheck disable=SC2086 -${NEOGO} contract invokefunction \ - --wallet-config ${CONFIG_IMG} \ - -a ${ADDR} --force \ - -r http://morph-chain.${LOCAL_DOMAIN}:30333 \ - ${NETMAP_ADDR} \ - newEpoch int:$((EPOCH+1)) -- ${ADDR}:Global \ - || die "Cannot increment an epoch" - -# Wait one Morph block to ensure the transaction broadcasted -# shellcheck disable=SC2086 -sleep $BLOCK_DURATION diff --git a/frostfs_config.mk b/frostfs_config.mk index 8f164d1..71eb8f1 100644 --- a/frostfs_config.mk +++ b/frostfs_config.mk @@ -1,44 +1,40 @@ # Update epoch duration in side chain blocks (make update.epoch_duration val=30) update.epoch_duration: - @./bin/config.sh int EpochDuration $(val) + @./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config EpochDuration=$(val) # Update max object size in bytes (make update.max_object_size val=1000) update.max_object_size: - @./bin/config.sh int MaxObjectSize $(val) + @./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config MaxObjectSize=$(val) # Update audit fee per result in fixed 12 (make update.audit_fee val=100) update.audit_fee: - @./bin/config.sh int AuditFee $(val) + @./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config AuditFee=$(val) # Update container fee per alphabet node in fixed 12 (make update.container_fee val=500) update.container_fee: - @./bin/config.sh int ContainerFee $(val) + @./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config ContainerFee=$(val) # Update container alias fee per alphabet node in fixed 12 (make update.container_alias_fee val=100) update.container_alias_fee: - @./bin/config.sh int ContainerAliasFee $(val) + @./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config ContainerAliasFee=$(val) # Update amount of EigenTrust iterations (make update.eigen_trust_iterations val=2) update.eigen_trust_iterations: - @./bin/config.sh int EigenTrustIterations $(val) - + @./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config EigenTrustIterations=$(val) # Update system dns to resolve container names (make update.system_dns val=container) update.system_dns: - @./bin/config.sh string SystemDNS $(val) + @./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config SystemDNS=$(val) --force # Update alpha parameter of EigenTrust algorithm in 0 <= f <= 1.0 (make update.eigen_trust_alpha val=0.2) update.eigen_trust_alpha: - @./bin/config.sh string EigenTrustAlpha $(val) + @./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config EigenTrustAlpha=$(val) # Update basic income rate in fixed 12 (make update.basic_income_rate val=1000) update.basic_income_rate: - @./bin/config.sh int BasicIncomeRate $(val) + @./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config BasicIncomeRate=$(val) # Update homomorphic hashing disabled flag (make update.homomorphic_hashing_disable val=true) update.homomorphic_hashing_disable: - @./bin/config.sh bool HomomorphicHashingDisabled $(val) + @./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config HomomorphicHashingDisabled=$(val) -# Tick new epoch in side chain -tick.epoch: - @./bin/tick.sh diff --git a/services/ir/prepare.mk b/services/ir/prepare.mk index fd7f253..af13d12 100644 --- a/services/ir/prepare.mk +++ b/services/ir/prepare.mk @@ -1,4 +1,3 @@ # Deposit GAS from default wallet to FrostFS privnet contract prepare.ir: - @./bin/config.sh int ContainerFee 0 - @./bin/config.sh int ContainerAliasFee 0 + @./vendor/frostfs-adm -c ./frostfs-adm.yml ContainerFee=0 ContainerAliasFee=0 From 045b55250ab7193fd992320cc61480681598e1e6 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 28 Apr 2023 12:08:41 +0300 Subject: [PATCH 046/149] [#18] bin: Remove passwd.exp It was here from older times Signed-off-by: Evgenii Stratonikov --- bin/passwd.exp | 23 ----------------------- 1 file changed, 23 deletions(-) delete mode 100755 bin/passwd.exp diff --git a/bin/passwd.exp b/bin/passwd.exp deleted file mode 100755 index 08fd3c7..0000000 --- a/bin/passwd.exp +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/expect - -set passwd [lindex $argv 0] -set args [lrange $argv 1 end] - -spawn -noecho {*}$args -expect -re {^.*assword.*$} - -if { $passwd == "-"} { - send -- "\r" -} else { - send -- "$passwd\r" -} - -expect { - "Relay transaction" { - send "y\r" - exp_continue - } - EOF -} -lassign [wait] pid spawnid os_error_flag value -exit $value From 1b6b2d125993548efdaa17472f755239bbd7ccd3 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Tue, 2 May 2023 18:44:26 +0300 Subject: [PATCH 047/149] [#21] Update FrostFS Core components to the latest master Signed-off-by: Alex Vanin --- .env | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.env b/.env index c33accc..78c1aec 100644 --- a/.env +++ b/.env @@ -12,11 +12,11 @@ NEOGO_VERSION=0.100.1 NEOGO_IMAGE=nspccdev/neo-go # FrostFS InnerRing nodes -IR_VERSION=5ffa8268 +IR_VERSION=eca5c210 IR_IMAGE=truecloudlab/frostfs-ir # FrostFS Storage nodes -NODE_VERSION=5ffa8268 +NODE_VERSION=eca5c210 NODE_IMAGE=truecloudlab/frostfs-storage # NATS Server @@ -40,17 +40,17 @@ LOCODE_DB_URL=https://github.com/nspcc-dev/neofs-locode-db/releases/download/v0. #LOCODE_DB_PATH=/path/to/locode_db # FrostFS CLI binary -FROSTFS_CLI_URL=https://http.t5.fs.neo.org/AQgse8bPCZx4zScMuAKxowJdZPbKHp8NDcp15o6VUNmk/C6BNLpYg5gWLHp3DrXozSxxGLDahBuSBCyJoYSSR1M3Q +FROSTFS_CLI_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/BbngJDdRJEDJTJk7qptq3SxKqrJqtvVYWU6R5AaFGbtG #FROSTFS_CLI_PATH=/path/to/frostfs-cli-binary # FrostFS ADM tool binary -FROSTFS_ADM_VERSION=e3554425 -FROSTFS_ADM_URL=https://http.t5.fs.neo.org/AQgse8bPCZx4zScMuAKxowJdZPbKHp8NDcp15o6VUNmk/sXZxy9vbFyJiLhN9qTSXozXK7SN9H8ZC6dpvAt59Zaj +FROSTFS_ADM_VERSION=eca5c210 +FROSTFS_ADM_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/2GxarAjGUb3RevxvqFGYT3hDQxNNaHzK6aFxhJCAMehq #FROSTFS_ADM_PATH=/path/to/frostfs-adm-binary # Compiled FrostFS Smart Contracts -FROSTFS_CONTRACTS_VERSION=4f3c08f5 -FROSTFS_CONTRACTS_URL=https://http.t5.fs.neo.org/AQgse8bPCZx4zScMuAKxowJdZPbKHp8NDcp15o6VUNmk/c1nGtturFrSeygYP3AyNHDDLNbs7HhJiH2BQkgZxEmZ +FROSTFS_CONTRACTS_VERSION=8537293e +FROSTFS_CONTRACTS_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/6ccZoj4HxoN1G1qvJAX2Qw9p2D6qdyzAjNMaNkEKYQpA #FROSTFS_CONTRACTS_PATH=/path/to/unpacked/frostfs-contracts-dir # Jaeger tracing @@ -59,4 +59,4 @@ JAEGER_IMAGE=jaegertracing/all-in-one # Prometheus monitoring PROMETHEUS_VERSION=v2.43.0 -PROMETHEUS_IMAGE=prom/prometheus \ No newline at end of file +PROMETHEUS_IMAGE=prom/prometheus From f57a739c3f07ff8394d679be6cae213b5a41c271 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 3 May 2023 11:38:04 +0300 Subject: [PATCH 048/149] [#22] .env: Update neo-go to v0.101.1 Signed-off-by: Evgenii Stratonikov --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 78c1aec..be4f6ff 100644 --- a/.env +++ b/.env @@ -8,7 +8,7 @@ BASTION_VERSION=10 BASTION_IMAGE=debian # NeoGo privnet -NEOGO_VERSION=0.100.1 +NEOGO_VERSION=0.101.1 NEOGO_IMAGE=nspccdev/neo-go # FrostFS InnerRing nodes From 53007be0474333df3981a31cfb617d96d0522aa0 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 3 May 2023 11:46:38 +0300 Subject: [PATCH 049/149] [#22] Makefile: Remove unneeded escape Signed-off-by: Evgenii Stratonikov --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0322582..9f791cf 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ HOSTS_LINES = $(shell grep -Rl IPV4_PREFIX ./services/* | grep .hosts) MORPH_CHAIN_PROTOCOL = './services/morph_chain/protocol.privnet.yml' # List of grepped environment variables from *.env -GREP_DOTENV = $(shell find . -name '*.env' -exec grep -rhv -e '^\#' -e '^$$' {} + | sort -u ) +GREP_DOTENV = $(shell find . -name '*.env' -exec grep -rhv -e '^#' -e '^$$' {} + | sort -u ) # Pull all required Docker images .PHONY: pull From 7171e152e0f3a258409a0f22cce352aaddbc0d38 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 3 May 2023 11:56:23 +0300 Subject: [PATCH 050/149] [#22] morph_chain: Do not use deprecated config fields Signed-off-by: Evgenii Stratonikov --- Makefile | 2 +- services/morph_chain/protocol.privnet.yml | 32 +++++++++++++---------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index 9f791cf..4caa292 100644 --- a/Makefile +++ b/Makefile @@ -152,7 +152,7 @@ clean: .PHONY: env env: @$(foreach envvar,$(GREP_DOTENV),echo $(envvar);) - @echo MORPH_BLOCK_TIME=$(shell grep 'SecondsPerBlock' $(MORPH_CHAIN_PROTOCOL) | awk '{print $$2}')s + @echo MORPH_BLOCK_TIME=$(shell grep 'TimePerBlock' $(MORPH_CHAIN_PROTOCOL) | awk '{print $$2}')s @echo MORPH_MAGIC=$(shell grep 'Magic' $(MORPH_CHAIN_PROTOCOL) | awk '{print $$2}') # Restart storage nodes with clean volumes diff --git a/services/morph_chain/protocol.privnet.yml b/services/morph_chain/protocol.privnet.yml index 39155c8..13e03b5 100644 --- a/services/morph_chain/protocol.privnet.yml +++ b/services/morph_chain/protocol.privnet.yml @@ -1,50 +1,54 @@ ProtocolConfiguration: Magic: 15405 MaxTraceableBlocks: 200000 - SecondsPerBlock: 1 + TimePerBlock: 1s MemPoolSize: 50000 StandbyCommittee: - 02b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc2 ValidatorsCount: 1 SeedList: - 172.200.0.1:20333 - VerifyBlocks: true VerifyTransactions: true StateRootInHeader: true P2PSigExtensions: true ApplicationConfiguration: + SkipBlockVerification: false DBConfiguration: Type: "boltdb" BoltDBOptions: FilePath: "./db/morph.bolt" - NodePort: 20333 + P2P: + Addresses: + - ":20333" + DialTimeout: 3s + ProtoTickInterval: 2s + PingInterval: 30s + PingTimeout: 90s + MaxPeers: 10 + AttemptConnPeers: 5 + MinPeers: 0 Relay: true - DialTimeout: 3 - ProtoTickInterval: 2 - PingInterval: 30 - PingTimeout: 90 - MaxPeers: 10 - AttemptConnPeers: 5 - MinPeers: 0 RPC: - Address: 192.168.130.90 + Addresses: + - "192.168.130.90:30333" Enabled: true SessionEnabled: true EnableCORSWorkaround: false MaxGasInvoke: 100 - Port: 30333 P2PNotary: Enabled: true UnlockWallet: Path: "./wallets/node-wallet.json" Password: "one" Prometheus: + Addresses: + - ":20001" Enabled: true - Port: 20001 Pprof: + Addresses: + - ":20011" Enabled: true - Port: 20011 UnlockWallet: Path: "./wallets/node-wallet.json" Password: "one" From 45b98196ccb9fdbb1668c49b58a3fbf591c3815a Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 4 May 2023 17:42:48 +0300 Subject: [PATCH 051/149] [#24] Update frostfsid id Signed-off-by: Anton Nikiforov --- services/ir/.ir.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/ir/.ir.env b/services/ir/.ir.env index 33df607..08d7e17 100644 --- a/services/ir/.ir.env +++ b/services/ir/.ir.env @@ -1,3 +1,3 @@ -FROSTFS_IR_CONTRACTS_FROSTFSID=1943e9bb78a0fe2fe0c95fd2677eec2da6aa4aa5 +FROSTFS_IR_CONTRACTS_FROSTFSID=27407c76feabc407908f3d09a3d845d45e7c981a FROSTFS_IR_CONTROL_GRPC_ENDPOINT=127.0.0.1:16512 From cbed3ce798036a893b641db95e8b9ac21c9794da Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Wed, 10 May 2023 12:31:06 +0300 Subject: [PATCH 052/149] [#27] Fix step `make prepare.ir` Signed-off-by: Anton Nikiforov --- services/ir/prepare.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/ir/prepare.mk b/services/ir/prepare.mk index af13d12..6649502 100644 --- a/services/ir/prepare.mk +++ b/services/ir/prepare.mk @@ -1,3 +1,3 @@ # Deposit GAS from default wallet to FrostFS privnet contract prepare.ir: - @./vendor/frostfs-adm -c ./frostfs-adm.yml ContainerFee=0 ContainerAliasFee=0 + @./vendor/frostfs-adm morph set-config -c ./frostfs-adm.yml ContainerFee=0 ContainerAliasFee=0 From 2452e4469b86f116a1abd9e2b353f45a16457cd9 Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Tue, 16 May 2023 11:58:23 +0300 Subject: [PATCH 053/149] [#29] Remove prepare.ir make target Signed-off-by: Alejandro Lopez --- README.md | 11 ----------- services/ir/prepare.mk | 3 --- 2 files changed, 14 deletions(-) delete mode 100644 services/ir/prepare.mk diff --git a/README.md b/README.md index b9af689..197d680 100644 --- a/README.md +++ b/README.md @@ -57,17 +57,6 @@ Run all services with command: $ make up ``` -When all services are up, you need to make GAS deposit for test wallet to be -able to pay for FrostFS operations. Test wallet is located in -`wallets/wallet.json` with the corresponding key in `wallets/wallet.key`. The -password is empty. - -``` -$ make prepare.ir -password > -fa6ba62bffb04030d303dcc95bda7413e03aa3c7e6ca9c2f999d65db9ec9b82c -``` - Also, you should add self-signed node (`s04.frostfs.devenv`) certificate to trusted store (default location might be changed using `CA_CERTS_TRUSTED_STORE` variable). This step is required for client services (frostfs-http-gw, diff --git a/services/ir/prepare.mk b/services/ir/prepare.mk deleted file mode 100644 index 6649502..0000000 --- a/services/ir/prepare.mk +++ /dev/null @@ -1,3 +0,0 @@ -# Deposit GAS from default wallet to FrostFS privnet contract -prepare.ir: - @./vendor/frostfs-adm morph set-config -c ./frostfs-adm.yml ContainerFee=0 ContainerAliasFee=0 From 26e290efcbea7c842e179333ac6a59e34b23d2c8 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 23 May 2023 13:23:52 +0300 Subject: [PATCH 054/149] [#31] jaeger: Use badger as storage backend Signed-off-by: Dmitrii Stepanov --- services/jaeger/docker-compose.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/services/jaeger/docker-compose.yml b/services/jaeger/docker-compose.yml index be045f8..a460548 100644 --- a/services/jaeger/docker-compose.yml +++ b/services/jaeger/docker-compose.yml @@ -20,7 +20,10 @@ services: env_file: [ ".env", ".jaeger.env", ".int_test.env" ] environment: - COLLECTOR_OTLP_ENABLED=true - - MEMORY_MAX_TRACES=100000 + - SPAN_STORAGE_TYPE=badger + - BADGER_EPHEMERAL=false + - BADGER_DIRECTORY_VALUE=/badger/data + - BADGER_DIRECTORY_KEY=/badger/key networks: jaeger_int: From 5534204706097eacc4ee35eda96cdcf8bfc2e6b4 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 24 May 2023 11:45:25 +0300 Subject: [PATCH 055/149] [#33] s3: Use all storage nodes as tree service Signed-off-by: Dmitrii Stepanov --- services/s3_gate/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/s3_gate/docker-compose.yml b/services/s3_gate/docker-compose.yml index f86caf5..1445480 100644 --- a/services/s3_gate/docker-compose.yml +++ b/services/s3_gate/docker-compose.yml @@ -25,7 +25,7 @@ services: - S3_GW_RPC_ENDPOINT=http://morph-chain.${LOCAL_DOMAIN}:30333 - S3_GW_SERVER_0_ADDRESS=s3.${LOCAL_DOMAIN}:8080 - S3_GW_LISTEN_DOMAINS=s3.${LOCAL_DOMAIN} - - S3_GW_TREE_SERVICE=s01.${LOCAL_DOMAIN}:8080 + - S3_GW_TREE_SERVICE=s01.${LOCAL_DOMAIN}:8080 s02.${LOCAL_DOMAIN}:8080 s03.${LOCAL_DOMAIN}:8080 s04.${LOCAL_DOMAIN}:8080 - S3_GW_PEERS_0_ADDRESS=s01.${LOCAL_DOMAIN}:8080 - S3_GW_PEERS_0_WEIGHT=0.2 - S3_GW_PEERS_1_ADDRESS=s02.${LOCAL_DOMAIN}:8080 From ee4d5ee6a6c91a1a4e1f82679f8746d4db64fd61 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 23 May 2023 16:08:14 +0300 Subject: [PATCH 056/149] [#32] storage: Enable writecache by default Signed-off-by: Dmitrii Stepanov --- services/storage/cfg/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/storage/cfg/config.yml b/services/storage/cfg/config.yml index fa50f4e..0a85096 100644 --- a/services/storage/cfg/config.yml +++ b/services/storage/cfg/config.yml @@ -48,7 +48,7 @@ storage: shard: 0: writecache: - enabled: false + enabled: true path: /storage/wc0 # Write-cache root directory metabase: @@ -68,7 +68,7 @@ storage: 1: writecache: - enabled: false + enabled: true path: /storage/wc1 # Write-cache root directory metabase: From dc2a2862d410eab6aab07fe34fde744e2183d31b Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Tue, 30 May 2023 15:49:43 +0300 Subject: [PATCH 057/149] [#34] Bump FrostFS versions Signed-off-by: Alex Vanin --- .env | 8 ++++---- services/http_gate/docker-compose.yml | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.env b/.env index be4f6ff..8f69cc5 100644 --- a/.env +++ b/.env @@ -12,11 +12,11 @@ NEOGO_VERSION=0.101.1 NEOGO_IMAGE=nspccdev/neo-go # FrostFS InnerRing nodes -IR_VERSION=eca5c210 +IR_VERSION=365a7ca0 IR_IMAGE=truecloudlab/frostfs-ir # FrostFS Storage nodes -NODE_VERSION=eca5c210 +NODE_VERSION=365a7ca0 NODE_IMAGE=truecloudlab/frostfs-storage # NATS Server @@ -24,7 +24,7 @@ NATS_VERSION=2.7.2 NATS_IMAGE=nats # HTTP Gate -HTTP_GW_VERSION=6abd500b +HTTP_GW_VERSION=0.27.0-rc.1-15-g1776db28 HTTP_GW_IMAGE=truecloudlab/frostfs-http-gw # REST Gate @@ -32,7 +32,7 @@ REST_GW_VERSION=c9c85e90 REST_GW_IMAGE=truecloudlab/frostfs-rest-gw # S3 Gate -S3_GW_VERSION=000d9ed4 +S3_GW_VERSION=0.27.0-rc.1-30-gce929468 S3_GW_IMAGE=truecloudlab/frostfs-s3-gw # FrostFS LOCODE database diff --git a/services/http_gate/docker-compose.yml b/services/http_gate/docker-compose.yml index 0dd3ad5..7108ee5 100644 --- a/services/http_gate/docker-compose.yml +++ b/services/http_gate/docker-compose.yml @@ -21,6 +21,7 @@ services: command: [ "frostfs-http-gw", "--config", "/etc/frostfs/http/config.yml" ] environment: - HTTP_GW_RPC_ENDPOINT=http://morph-chain.${LOCAL_DOMAIN}:30333 + - HTTP_GW_TREE_SERVICE=s01.${LOCAL_DOMAIN}:8080 - HTTP_GW_PEERS_0_ADDRESS=s01.${LOCAL_DOMAIN}:8080 - HTTP_GW_PEERS_0_WEIGHT=0.2 - HTTP_GW_PEERS_1_ADDRESS=s02.${LOCAL_DOMAIN}:8080 From c50ea0ea5c7dc47f3653f0db24b029fcadb8f19a Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Wed, 7 Jun 2023 14:12:54 +0300 Subject: [PATCH 058/149] [#1] Add DCO check for forgejo runner Signed-off-by: Alex Vanin --- .dockerignore | 1 + .forgejo/workflows/dco.yml | 20 ++++++++++++++++++++ .github/workflows/dco.yml | 21 --------------------- 3 files changed, 21 insertions(+), 21 deletions(-) create mode 100644 .forgejo/workflows/dco.yml delete mode 100644 .github/workflows/dco.yml diff --git a/.dockerignore b/.dockerignore index 73b336a..2f9d4b4 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,6 @@ .docker .github +.forgejo vendor tmp .secrets diff --git a/.forgejo/workflows/dco.yml b/.forgejo/workflows/dco.yml new file mode 100644 index 0000000..880addd --- /dev/null +++ b/.forgejo/workflows/dco.yml @@ -0,0 +1,20 @@ +on: [pull_request] + +jobs: + dco: + name: DCO + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Setup Go + uses: actions/setup-go@v3 + with: + go-version: '1.20' + + - name: Run commit format checker + uses: https://git.alexvan.in/alexvanin/dco-go@v1 + with: + from: dca6ff62 diff --git a/.github/workflows/dco.yml b/.github/workflows/dco.yml deleted file mode 100644 index 40ed8fc..0000000 --- a/.github/workflows/dco.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: DCO check - -on: - pull_request: - branches: - - master - -jobs: - commits_check_job: - runs-on: ubuntu-latest - name: Commits Check - steps: - - name: Get PR Commits - id: 'get-pr-commits' - uses: tim-actions/get-pr-commits@master - with: - token: ${{ secrets.GITHUB_TOKEN }} - - name: DCO Check - uses: tim-actions/dco@master - with: - commits: ${{ steps.get-pr-commits.outputs.commits }} From 80dc0faae9a3cf8819cd21f305524b0a7ab0aa4c Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Wed, 7 Jun 2023 12:29:52 +0300 Subject: [PATCH 059/149] [#1] Use frostfs.info hosted locode URL Signed-off-by: Alex Vanin --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 8f69cc5..f921d46 100644 --- a/.env +++ b/.env @@ -36,7 +36,7 @@ S3_GW_VERSION=0.27.0-rc.1-30-gce929468 S3_GW_IMAGE=truecloudlab/frostfs-s3-gw # FrostFS LOCODE database -LOCODE_DB_URL=https://github.com/nspcc-dev/neofs-locode-db/releases/download/v0.3.0/locode_db.gz +LOCODE_DB_URL=https://git.frostfs.info/attachments/a2e8def7-52b6-49f1-89cd-a056712e8e54 #LOCODE_DB_PATH=/path/to/locode_db # FrostFS CLI binary From 6b0ce07808659bd67856f584c9d4b390462afc14 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Wed, 7 Jun 2023 12:30:48 +0300 Subject: [PATCH 060/149] [#1] Update cert generation scripts Signed-off-by: Alex Vanin --- services/nats/generate_cert.sh | 2 +- services/storage/generate_cert.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/services/nats/generate_cert.sh b/services/nats/generate_cert.sh index 071f9df..4c4dd0b 100755 --- a/services/nats/generate_cert.sh +++ b/services/nats/generate_cert.sh @@ -16,7 +16,7 @@ CLI_KEY=$WORKDIR/client-key.pem CLI_REQ=$WORKDIR/client-req.csr CLI_CRT=$WORKDIR/client-cert.pem -SUBJ="/O=NSPCC" +SUBJ="/O=TrueCloudLab" if [[ ! -f $CA_KEY || ! -f $CA_CRT ]]; then openssl req -newkey rsa:4096 -x509 -days 365 -nodes -keyout $CA_KEY -out $CA_CRT -subj $SUBJ 2>&1 || diff --git a/services/storage/generate_cert.sh b/services/storage/generate_cert.sh index ddc7aa1..87672d2 100755 --- a/services/storage/generate_cert.sh +++ b/services/storage/generate_cert.sh @@ -19,7 +19,7 @@ if [[ ! -f ${CERT} ]]; then ) > ${SSL_CONFIG} openssl req -new -newkey rsa:4096 -x509 -sha256 -days 365 -nodes \ - -subj "/C=RU/ST=SPB/L=St.Petersburg/O=NSPCC/OU=NSPCC/CN=s04.${LOCAL_DOMAIN}" \ + -subj "/C=RU/ST=SPB/L=St.Petersburg/O=TrueCloudLab/OU=TrueCloudLab/CN=s04.${LOCAL_DOMAIN}" \ -keyout "${KEY}" -out "${CERT}" -extensions san -config "${SSL_CONFIG}" &> /dev/null || { die "Failed to generate SSL certificate for s04" } From e551fac84d94cda36a0673fbb0815c168050456a Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Wed, 7 Jun 2023 12:27:13 +0300 Subject: [PATCH 061/149] [#1] Update notary service description Signed-off-by: Alex Vanin --- docs/notary.md | 66 +++----------------------------------------------- 1 file changed, 4 insertions(+), 62 deletions(-) diff --git a/docs/notary.md b/docs/notary.md index 66cd713..2b16689 100644 --- a/docs/notary.md +++ b/docs/notary.md @@ -7,66 +7,12 @@ create containers, approve balance changes, update network map, tick epochs, etc. With notary service, it takes up to seven times fewer transactions to do these operations. Notary service calculates the exact amount of GAS to execute transaction, therefore operations are cheaper (withdraw fee **with** -notary is less than 0.5 GAS; withdraw fee **without** notary is up to 7.0 GAS). +notary is less than 0.5 GAS; withdraw fee **without** notary is up to 7.0 GAS). -By default, main chain service is running without notary service, and side chain -running with notary service. However, you can change that in configuration. +Currently, frostfs-dev-env contains single chain (see morph service) and it +enables notary service from the genesis block. -# Disable notary service in side chain - -To disable notary service in side chain do these steps. - -1. Update `.env` and choose notary disabled chain dump for side chain. - -``` -MORPH_CHAIN_URL="https://github.com/nspcc-dev/neofs-contract/releases/download/v0.9.0/devenv_sidechain_notary_disabled.gz" -``` - -Make sure to update chain dump files with `make get` target. - -2. Update `service/morph_chain/protocol.privnet.yml` and disable notary settings -and state root in header. - -```yaml -ProtocolConfiguration: - StateRootInHeader: false - P2PSigExtensions: false -ApplicationConfiguration: - P2PNotary: - Enabled: false -``` - -Chain dump without notary service does not have predefined network map. -Therefore, you need to wait about 5 minutes until new epoch tick with updated -network map. - - -3. Enable helper commands - -To enable helper commands such as `make tick.epoch` or `make update.epoch_duration` -make sure to export non-empty `FROSTFS_NOTARY_DISABLED` environment variable. -``` -$ export FROSTFS_NOTARY_DISABLED=1 -``` - -Use `unset` command to return it back. -``` -$ unset FROSTFS_NOTARY_DISABLED -``` - -# Enable notary service in main chain - -To enable notary service in main chain do these steps. - -1. Update `.env` and choose notary enabled chain dump for main chain. - -``` -CHAIN_URL="https://github.com/nspcc-dev/neofs-contract/releases/download/v0.9.0/devenv_mainchain.gz" -``` - -Make sure to update chain dump files with `make get` target. - -2. Update `service/chain/protocol.privnet.yml` and enable notary settings. +To enable notary service, use neo-go configuration below. ```yaml ProtocolConfiguration: @@ -75,7 +21,3 @@ ApplicationConfiguration: P2PNotary: Enabled: true ``` - -Main chain generates a block once per 15 seconds, so Inner Ring takes about -15-30 seconds to make a notary deposit in main chain after startup. Then -frostfs-dev-env is ready to work. From 5050dff55aa556584cdfdbfddd81235fb4f2d34a Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Wed, 7 Jun 2023 12:37:51 +0300 Subject: [PATCH 062/149] [#1] Update morph service description Signed-off-by: Alex Vanin --- docs/morph.md | 69 ++++++++++++++------------------------------------- 1 file changed, 19 insertions(+), 50 deletions(-) diff --git a/docs/morph.md b/docs/morph.md index 4429298..56edfbc 100644 --- a/docs/morph.md +++ b/docs/morph.md @@ -3,52 +3,23 @@ A single-node N3 privnet deployment, running on [neo-go](https://github.com/nspcc-dev/neo-go). Represents N3 FrostFS SideChain. Contracts deployed: -- Alphabet (AZ) [contract](https://github.com/TrueCloudLab/frostfs-contract/tree/master/alphabet) -- Audit [contract](https://github.com/TrueCloudLab/frostfs-contract/tree/master/audit) -- Balance [contract](https://github.com/TrueCloudLab/frostfs-contract/tree/master/balance) -- Container [contract](https://github.com/TrueCloudLab/frostfs-contract/tree/master/container) -- Netmap [contract](https://github.com/TrueCloudLab/frostfs-contract/tree/master/netmap) -- NeoFSID [contract](https://github.com/TrueCloudLab/frostfs-contract/tree/master/neofsid) -- Proxy [contract](https://github.com/TrueCloudLab/frostfs-contract/tree/master/proxy) -- Reputation [contract](https://github.com/TrueCloudLab/frostfs-contract/tree/master/reputation) +- Alphabet (AZ) [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/alphabet) +- Audit [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/audit) +- Balance [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/balance) +- Container [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/container) +- Netmap [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/netmap) +- NeoFSID [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/neofsid) +- Proxy [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/proxy) +- Reputation [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/reputation) RPC available at `http://morph-chain.frostfs.devenv:30333`. ## .env settings -### MORPH_CHAIN_URL - -URL to get side chain dump. Used on artifact get stage. - -### MORPH_CHAIN_PATH - -Path to get side chain dump. If set, overrides `CHAIN_URL`. - ### NEOGO_VERSION Version of neo-go docker container for side chain deployment. -## Side chain wallets - -There is a wallet with GAS that used for contract deployment: -`wallets/wallet.json`. This wallet has one account with **empty password**. - -``` -$ neo-go wallet nep17 balance \ - -w wallets/wallet.json \ - -r http://morph-chain.frostfs.devenv:30333 - -Account NbUgTSFvPmsRxmGeWpuuGeJUoRoi6PErcM -GAS: GasToken (d2a4cff31913016155e38e474a2c06d08be276cf) - Amount : 189826.0515316 - Updated: 3909 -FROSTFS: FrostFS Balance (69550190e740b93f92dbd5dea52246f550391057) - Amount : 50 - Updated: 3909 -``` - -This way you can also monitor FrostFS internal balance of your account. - ## FrostFS global config FrostFS uses global configuration to store epoch duration, maximum object size, @@ -57,16 +28,10 @@ netmap contract and managed by Inner Ring (Alphabet) nodes. To change these parameters use `make update.*` commands. Command down below changes epoch duration from 300 blocks (about 300 seconds with 1bps) to 30. -Script enters passwords automatically with `expect` utility. ``` $ make update.epoch_duration val=30 -Changing EpochDuration configration value to 30 -Enter account NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP password > -Sent invocation transaction bdc0fa88cd6719ef6df2b9c82de423ddec6141ca24255c2d0072688083b1de9d -Updating FrostFS epoch to 20 -Enter account NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP password > -Sent invocation transaction 12296e1ce24dd6c04edb9c56d0a1d0e26d3226adefb0333c74a28788f44a8d0f +Waiting for transactions to persist... ``` Read more about available configuration in Makefile help. @@ -76,10 +41,14 @@ $ make help ... Targets: ... - update.audit_fee Update audit fee per result in fixed 12 (make update.audit_fee val=100) - update.basic_income_rate Update basic income rate in fixed 12 (make update.basic_income_rate val=1000) - update.container_fee Update container fee per alphabet node in fixed 12 (make update.container_fee val=500) - update.eigen_trust_iterations Update amount of EigenTrust iterations (make update.eigen_trust_iterations val=2) - update.epoch_duration Update epoch duration in side chain blocks (make update.epoch_duration val=30) - update.max_object_size Update max object size in bytes (make update.max_object_size val=1000) + update.audit_fee Update audit fee per result in fixed 12 (make update.audit_fee val=100) + update.basic_income_rate Update basic income rate in fixed 12 (make update.basic_income_rate val=1000) + update.container_alias_fee Update container alias fee per alphabet node in fixed 12 (make update.container_alias_fee val=100) + update.container_fee Update container fee per alphabet node in fixed 12 (make update.container_fee val=500) + update.eigen_trust_alpha Update alpha parameter of EigenTrust algorithm in 0 <= f <= 1.0 (make update.eigen_trust_alpha val=0.2) + update.eigen_trust_iterations Update amount of EigenTrust iterations (make update.eigen_trust_iterations val=2) + update.epoch_duration Update epoch duration in side chain blocks (make update.epoch_duration val=30) + update.homomorphic_hashing_disable Update homomorphic hashing disabled flag (make update.homomorphic_hashing_disable val=true) + update.max_object_size Update max object size in bytes (make update.max_object_size val=1000) + update.system_dns Update system dns to resolve container names (make update.system_dns val=container) ``` From aded88f09a0a2da3cd690b3a0d202219839d7b7c Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Wed, 7 Jun 2023 12:48:03 +0300 Subject: [PATCH 063/149] [#1] Update HTTP gateway description Signed-off-by: Alex Vanin --- docs/http_gate.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/http_gate.md b/docs/http_gate.md index 1aadd27..e2159ad 100644 --- a/docs/http_gate.md +++ b/docs/http_gate.md @@ -2,7 +2,7 @@ Protocol Gateway to access data in FrostFS using HTTP protocol. -Source code and more information can be found in [project's GitHub repository](https://github.com/TrueCloudLab/frostfs-http-gate) +Source code and more information can be found in [project's repository](https://git.frostfs.info/TrueCloudLab/frostfs-http-gw) ## .env settings From 656833d37e9780e9729d1e155756f88fb188884f Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Wed, 7 Jun 2023 12:49:23 +0300 Subject: [PATCH 064/149] [#1] Update REST gateway description Signed-off-by: Alex Vanin --- docs/rest_gate.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/rest_gate.md b/docs/rest_gate.md index e511280..ea2e95b 100644 --- a/docs/rest_gate.md +++ b/docs/rest_gate.md @@ -2,7 +2,7 @@ REST Gateway to access data in FrostFS using REST. -Source code and more information can be found in [project's GitHub repository](https://github.com/TrueCloudLab/frostfs-rest-gw) +Source code and more information can be found in [project's repository](https://git.frostfs.info/TrueCloudLab/frostfs-rest-gw) ## .env settings From 463521d511e04b195154985557f0bae153f62153 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Wed, 7 Jun 2023 12:52:57 +0300 Subject: [PATCH 065/149] [#1] Update S3 gateway description Signed-off-by: Alex Vanin --- docs/s3_gate.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/s3_gate.md b/docs/s3_gate.md index 0a110d6..717fded 100644 --- a/docs/s3_gate.md +++ b/docs/s3_gate.md @@ -2,7 +2,7 @@ Protocol Gateway to access data in FrostFS using AWS S3 protocol -Source code and more information can be found in [project's GitHub repository](https://github.com/TrueCloudLab/frostfs-s3-gw) +Source code and more information can be found in [project's repository](https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw) ## .env settings From 8eca14f331cf22e1f1b28e2542a0966f480845f9 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Wed, 7 Jun 2023 13:01:44 +0300 Subject: [PATCH 066/149] [#1] Update CONTRIBUTING Signed-off-by: Alex Vanin --- CONTRIBUTING.md | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f3ae1f1..fb57514 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3,8 +3,8 @@ First, thank you for contributing! We love and encourage pull requests from everyone. Please follow the guidelines: -- Check the open [issues](https://github.com/TrueCloudLab/frostfs-dev-env/issues) and - [pull requests](https://github.com/TrueCloudLab/frostfs-dev-env/pulls) for existing +- Check the open [issues](https://git.frostfs.info/TrueCloudLab/frostfs-dev-env/issues) and + [pull requests](https://git.frostfs.info/TrueCloudLab/frostfs-dev-env/pulls) for existing discussions. - Open an issue first, to discuss a new feature or enhancement. @@ -25,19 +25,19 @@ Start by forking the `frostfs-dev-env` repository, make changes in a branch and send a pull request. We encourage pull requests to discuss code changes. Here are the steps in details: -### Set up your GitHub Repository -Fork [FrostFS node upstream](https://github.com/TrueCloudLab/frostfs-dev-env/fork) source +### Set up your git repository +Fork [FrostFS node upstream](https://git.frostfs.info/repo/fork/24) source repository to your own personal repository. Copy the URL of your fork (you will need it for the `git clone` command below). ```sh -$ git clone https://github.com/TrueCloudLab/frostfs-dev-env +$ git clone https://git.frostfs.info//frostfs-dev-env.git ``` ### Set up git remote as ``upstream`` ```sh $ cd frostfs-dev-env -$ git remote add upstream https://github.com/TrueCloudLab/frostfs-dev-env +$ git remote add upstream https://git.frostfs.info/TrueCloudLab/frostfs-dev-env.git $ git fetch upstream $ git merge upstream/master ... @@ -55,8 +55,7 @@ $ git checkout -b feature/123-something_awesome ### Test your changes After your code changes, make sure -- To add test cases for the new code. -- To run `make lint` +- To run `make up` to check dev-env is not broken. - To squash your commits into a single commit or a series of logically separated commits run `git rebase -i`. It's okay to force update your pull request. @@ -86,8 +85,8 @@ $ git push origin feature/123-something_awesome ``` ### Create a Pull Request -Pull requests can be created via GitHub. Refer to [this -document](https://help.github.com/articles/creating-a-pull-request/) for +Pull requests can be created via Forgejo. Refer to [this +document](https://docs.codeberg.org/collaborating/pull-requests-and-git-flow/) for detailed steps on how to create a pull request. After a Pull Request gets peer reviewed and approved, it will be merged. From c31b7d9c84bc1f38f51252ee8eb7150d61ab845c Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Wed, 7 Jun 2023 12:44:09 +0300 Subject: [PATCH 067/149] [#1] Update README Signed-off-by: Alex Vanin --- README.md | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 197d680..aa07a08 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ Make sure you have installed all of the following prerequisites on your machine: Clone repo: ``` -$ git clone https://github.com/TrueCloudLab/frostfs-dev-env.git +$ git clone https://git.frostfs.info/TrueCloudLab/frostfs-dev-env.git ``` Run next commands from project's root: @@ -71,12 +71,7 @@ password of inner ring wallet is `one`. See examples in `make help`. ``` $ make update.epoch_duration val=30 -Changing EpochDuration configration value to 30 -Enter account NNudMSGzEoktFzdYGYoNb3bzHzbmM1genF password > -Sent invocation transaction dbb8c1145b6d10f150135630e13bb0dc282023163f5956c6945a60db0cb45cb0 -Updating FrostFS epoch to 2 -Enter account NNudMSGzEoktFzdYGYoNb3bzHzbmM1genF password > -Sent invocation transaction 0e6eb5e190f36332e5e5f4e866c7e100826e285fd949e11c085e15224f343ba6 +Waiting for transactions to persist... ``` For instructions on how to set up DevEnv on macOS, please refer [the @@ -112,7 +107,7 @@ Maybe you will find the answer for your question in [F.A.Q.](docs/faq.md) ## Using FrostFS Admin Tool in `dev-env` -Devenv supports FrostFS network management via [frostfs-adm](https://github.com/TrueCloudLab/frostfs-node/tree/master/cmd/frostfs-adm). +Devenv supports FrostFS network management via [frostfs-adm](https://git.frostfs.info/TrueCloudLab/frostfs-node/src/branch/master/cmd/frostfs-adm). `services/ir` contains the Alphabet wallet in a proper format, specify it with `--alphabet-wallets` flag. From d08b338e06c43f0889c26940a086488b01f673ee Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 14 Jun 2023 12:38:24 +0300 Subject: [PATCH 068/149] [#36] prometheus: Add IR metrics Signed-off-by: Dmitrii Stepanov --- services/prometheus/prometheus.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/services/prometheus/prometheus.yml b/services/prometheus/prometheus.yml index 1ea7300..9ea7aed 100644 --- a/services/prometheus/prometheus.yml +++ b/services/prometheus/prometheus.yml @@ -16,4 +16,7 @@ scrape_configs: - targets: ['s3.frostfs.devenv:9090'] - job_name: 'neo-go' static_configs: - - targets: ['morph-chain.frostfs.devenv:20001'] \ No newline at end of file + - targets: ['morph-chain.frostfs.devenv:20001'] + - job_name: 'inner-ring' + static_configs: + - targets: ['ir01.frostfs.devenv:9090'] \ No newline at end of file From 3400bb5736f769f5cf9720a3dbfff99809337c7a Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Thu, 13 Jul 2023 12:13:27 +0300 Subject: [PATCH 069/149] [#37] Add grafana service to visualize prometheus metrics Signed-off-by: Alex Vanin --- .env | 4 + .services | 1 + services/grafana/.env | 1 + services/grafana/.hosts | 1 + services/grafana/.int_test.env | 1 + services/grafana/docker-compose.yml | 26 + services/grafana/grafana.ini | 3 + .../provisioning/dashboards/14061_rev1.json | 1162 +++++++++++++++++ .../provisioning/dashboards/dashboard.yml | 10 + .../provisioning/datasources/datasource.yml | 8 + 10 files changed, 1217 insertions(+) create mode 120000 services/grafana/.env create mode 100644 services/grafana/.hosts create mode 120000 services/grafana/.int_test.env create mode 100644 services/grafana/docker-compose.yml create mode 100644 services/grafana/grafana.ini create mode 100644 services/grafana/provisioning/dashboards/14061_rev1.json create mode 100644 services/grafana/provisioning/dashboards/dashboard.yml create mode 100644 services/grafana/provisioning/datasources/datasource.yml diff --git a/.env b/.env index f921d46..e04d5ca 100644 --- a/.env +++ b/.env @@ -60,3 +60,7 @@ JAEGER_IMAGE=jaegertracing/all-in-one # Prometheus monitoring PROMETHEUS_VERSION=v2.43.0 PROMETHEUS_IMAGE=prom/prometheus + +# Grafana versions +GRAFANA_VERSION=9.5.6 +GRAFANA_IMAGE=grafana/grafana diff --git a/.services b/.services index d9f34c7..768e4e6 100644 --- a/.services +++ b/.services @@ -3,3 +3,4 @@ http_gate s3_gate rest_gate +grafana diff --git a/services/grafana/.env b/services/grafana/.env new file mode 120000 index 0000000..c7360fb --- /dev/null +++ b/services/grafana/.env @@ -0,0 +1 @@ +../../.env \ No newline at end of file diff --git a/services/grafana/.hosts b/services/grafana/.hosts new file mode 100644 index 0000000..8f36ea6 --- /dev/null +++ b/services/grafana/.hosts @@ -0,0 +1 @@ +IPV4_PREFIX.122 grafana.LOCAL_DOMAIN diff --git a/services/grafana/.int_test.env b/services/grafana/.int_test.env new file mode 120000 index 0000000..582b6a2 --- /dev/null +++ b/services/grafana/.int_test.env @@ -0,0 +1 @@ +../../.int_test.env \ No newline at end of file diff --git a/services/grafana/docker-compose.yml b/services/grafana/docker-compose.yml new file mode 100644 index 0000000..795b87c --- /dev/null +++ b/services/grafana/docker-compose.yml @@ -0,0 +1,26 @@ +version: '2.4' +services: + grafana: + image: ${GRAFANA_IMAGE}:${GRAFANA_VERSION} + domainname: ${LOCAL_DOMAIN} + hostname: grafana + container_name: grafana + restart: on-failure + networks: + grafana_int: + internet: + ipv4_address: ${IPV4_PREFIX}.122 + volumes: + - ./../../vendor/hosts:/etc/hosts + - ./grafana.ini:/etc/grafana/grafana.ini + - ./provisioning:/etc/grafana/provisioning + ports: + - '3000:3000' + stop_signal: SIGKILL + env_file: [ ".env", ".int_test.env" ] + +networks: + grafana_int: + internet: + external: true + name: basenet_internet diff --git a/services/grafana/grafana.ini b/services/grafana/grafana.ini new file mode 100644 index 0000000..256888f --- /dev/null +++ b/services/grafana/grafana.ini @@ -0,0 +1,3 @@ +[auth.anonymous] +# enable anonymous access +enabled = true diff --git a/services/grafana/provisioning/dashboards/14061_rev1.json b/services/grafana/provisioning/dashboards/14061_rev1.json new file mode 100644 index 0000000..deec62b --- /dev/null +++ b/services/grafana/provisioning/dashboards/14061_rev1.json @@ -0,0 +1,1162 @@ +{ + "__inputs": [], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "7.2.0" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "A quickstart to setup the Prometheus Go runtime exporter with preconfigured dashboards, alerting rules, and recording rules.", + "editable": true, + "graphTooltip": 0, + "id": null, + "iteration": 1602794777869, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Average total bytes of memory reserved across all process instances of a job.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg by(job)(go_memstats_sys_bytes{job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{job}} (avg)", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total Reserved Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Average stack memory usage across all instances of a job.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg by (job) (go_memstats_stack_sys_bytes{job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{job}}: stack inuse (avg)", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Stack Memory Use", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Average memory reservations by the runtime, not for stack or heap, across all instances of a job.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 26, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg by (job)(go_memstats_mspan_sys_bytes{job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{instance}}: mspan (avg)", + "refId": "B" + }, + { + "expr": "avg by (job)(go_memstats_mcache_sys_bytes{job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{instance}}: mcache (avg)", + "refId": "D" + }, + { + "expr": "avg by (job)(go_memstats_buck_hash_sys_bytes{job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{instance}}: buck hash (avg)", + "refId": "E" + }, + { + "expr": "avg by (job)(go_memstats_gc_sys_bytes{job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{job}}: gc (avg)", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Other Memory Reservations", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Average memory reserved, and actually in use, by the heap, across all instances of a job.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg by (job)(go_memstats_heap_sys_bytes{job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{job}}: heap reserved (avg)", + "refId": "B" + }, + { + "expr": "avg by (job)(go_memstats_heap_inuse_bytes{job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{job}}: heap in use (avg)", + "refId": "A" + }, + { + "expr": "avg by (job)(go_memstats_heap_alloc_bytes{job=~\"tns_app\",instance=~\".*\"})", + "interval": "", + "legendFormat": "{{job}}: heap alloc (avg)", + "refId": "C" + }, + { + "expr": "avg by (job)(go_memstats_heap_idle_bytes{job=~\"tns_app\",instance=~\".*\"})", + "interval": "", + "legendFormat": "{{job}}: heap idle (avg)", + "refId": "D" + }, + { + "expr": "avg by (job)(go_memstats_heap_released_bytes{job=~\"tns_app\",instance=~\".*\"})", + "interval": "", + "legendFormat": "{{job}}: heap released (avg)", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Heap Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Average allocation rate in bytes per second, across all instances of a job.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 1, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg by (job)(rate(go_memstats_alloc_bytes_total{job=\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "{{job}}: bytes malloced/s (avg)", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Allocation Rate, Bytes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Average rate of heap object allocation, across all instances of a job.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(go_memstats_mallocs_total{job=\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "interval": "", + "legendFormat": "{{job}}: obj mallocs/s (avg)", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Heap Object Allocation Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Average number of live memory objects across all instances of a job.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg by(job)(go_memstats_mallocs_total{job=\"$job\", instance=~\"$instance\"} - go_memstats_frees_total{job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{job}}: object count (avg)", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of Live Objects", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Average number of goroutines across instances of a job.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg by (job)(go_goroutines{job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{job}}: goroutine count (avg)", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg by (job)(go_gc_duration_seconds{quantile=\"0\", job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{job}}: min gc time (avg)", + "refId": "A" + }, + { + "expr": "avg by (job)(go_gc_duration_seconds{quantile=\"1\", job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{job}}: max gc time (avg)", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "GC min & max duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "The number used bytes at which the runtime plans to perform the next GC, averaged across all instances of a job.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "hiddenSeries": false, + "id": 27, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg by (job)(go_memstats_next_gc_bytes{job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{job}} next gc bytes (avg)", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Next GC, Bytes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "5s", + "schemaVersion": 25, + "style": "dark", + "tags": [ + "go", + "golang" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": null, + "current": {}, + "datasource": "$datasource", + "definition": "label_values(go_info, job)", + "hide": 0, + "includeAll": false, + "label": "job", + "multi": false, + "name": "job", + "options": [], + "query": "label_values(go_info, job)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": "", + "current": {}, + "datasource": "$datasource", + "definition": "label_values(go_info{job=\"$job\"}, instance)", + "hide": 0, + "includeAll": true, + "label": "instance", + "multi": true, + "name": "instance", + "options": [], + "query": "label_values(go_info{job=\"$job\"}, instance)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Go Runtime Exporter Quickstart and Dashboardby Grafana Labs", + "uid": "CgCw8jKZz3", + "version": 3, + "gnetId": 14061 +} \ No newline at end of file diff --git a/services/grafana/provisioning/dashboards/dashboard.yml b/services/grafana/provisioning/dashboards/dashboard.yml new file mode 100644 index 0000000..bc9f26e --- /dev/null +++ b/services/grafana/provisioning/dashboards/dashboard.yml @@ -0,0 +1,10 @@ +apiVersion: 1 + +providers: + - name: Prometheus + orgId: 1 + type: file + disableDeletion: false + editable: true + options: + path: /etc/grafana/provisioning/dashboards diff --git a/services/grafana/provisioning/datasources/datasource.yml b/services/grafana/provisioning/datasources/datasource.yml new file mode 100644 index 0000000..81b6a7b --- /dev/null +++ b/services/grafana/provisioning/datasources/datasource.yml @@ -0,0 +1,8 @@ +apiVersion: 1 + +datasources: +- name: Prometheus + type: prometheus + access: proxy + orgId: 1 + url: http://prometheus:9090 From 2744f675aa6efb52b046e9ee9a57d6fd9d9f0a1a Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 28 Jul 2023 18:40:52 +0300 Subject: [PATCH 070/149] [#38] grafana: Add storage dashboard Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 489 ++++++++++++++++++ 1 file changed, 489 insertions(+) create mode 100644 services/grafana/provisioning/dashboards/storage-node.json diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json new file mode 100644 index 0000000..8f38c0d --- /dev/null +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -0,0 +1,489 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 2, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [], + "title": "Server", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by gRPC method", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, grpc_service, grpc_method) (\n rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed requests", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(grpc_server_handled_total{instance=\"$instance\",grpc_code!=\"OK\"}[$__rate_interval])) by (grpc_service, grpc_method)\n / \nsum(rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])) by (grpc_service, grpc_method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "95%-tile latency of the requests.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile(0.95, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency, 0.95q", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "99%-tile latency of the requests.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile(0.99, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency, 0.99q", + "transformations": [], + "type": "timeseries" + } + ], + "refresh": "1m", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "s01.frostfs.devenv:9090", + "value": "s01.frostfs.devenv:9090" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_state_health,instance)", + "hide": 0, + "includeAll": false, + "label": "", + "multi": false, + "name": "instance", + "options": [], + "query": { + "query": "label_values(frostfs_node_state_health,instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Storage node", + "uid": "aa6318b6-4780-4a9a-9fe7-a4a81917265f", + "version": 1, + "weekStart": "" +} \ No newline at end of file From 584fa43ca7c0617d97eb874b4e38a806ad8a9e37 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 31 Jul 2023 12:05:40 +0300 Subject: [PATCH 071/149] [#38] grafana: Add overview board Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/overview.json | 327 ++++++++++++++++++ 1 file changed, 327 insertions(+) create mode 100644 services/grafana/provisioning/dashboards/overview.json diff --git a/services/grafana/provisioning/dashboards/overview.json b/services/grafana/provisioning/dashboards/overview.json new file mode 100644 index 0000000..46a5987 --- /dev/null +++ b/services/grafana/provisioning/dashboards/overview.json @@ -0,0 +1,327 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Overview panel for all services", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 3, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by service instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance) (\n rate(grpc_server_started_total[$__rate_interval])\n)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Error rate by service instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(rate(grpc_server_handled_total{grpc_code!=\"OK\"}[$__rate_interval])) by (instance)\n / \nsum(rate(grpc_server_started_total{}[$__rate_interval])) by (instance)\n * 100.0", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Resident memory size by service instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance) (\n process_resident_memory_bytes\n)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Memory", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Overview", + "uid": "da5f53f7-c069-44b5-8372-462333c9fe4a", + "version": 1, + "weekStart": "" + } \ No newline at end of file From 3abb217d3028937a4f50c506b7a16137dec9d4c6 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 31 Jul 2023 12:06:15 +0300 Subject: [PATCH 072/149] [#38] grafana: Change start order Start grafana right after prometheus. Signed-off-by: Dmitrii Stepanov --- .bootstrap_services | 1 + .services | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.bootstrap_services b/.bootstrap_services index 02237d1..a56729d 100644 --- a/.bootstrap_services +++ b/.bootstrap_services @@ -4,3 +4,4 @@ basenet morph_chain jaeger prometheus +grafana diff --git a/.services b/.services index 768e4e6..d9f34c7 100644 --- a/.services +++ b/.services @@ -3,4 +3,3 @@ http_gate s3_gate rest_gate -grafana From 7de23fe789d7817d553287d10ba312e2f867c2df Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 31 Jul 2023 12:06:51 +0300 Subject: [PATCH 073/149] [#38] grafana: Fix config Allow to acces grafana anonymous. Change home page. Signed-off-by: Dmitrii Stepanov --- services/grafana/grafana.ini | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/services/grafana/grafana.ini b/services/grafana/grafana.ini index 256888f..3c14906 100644 --- a/services/grafana/grafana.ini +++ b/services/grafana/grafana.ini @@ -1,3 +1,7 @@ [auth.anonymous] -# enable anonymous access enabled = true +org_name = Main Org. +org_role = Editor + +[dashboards] +default_home_dashboard_path= /etc/grafana/provisioning/dashboards/overview.json \ No newline at end of file From 2efc0442f18120cc05ea8fc287a70f284b4a6a37 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 1 Aug 2023 16:14:18 +0300 Subject: [PATCH 074/149] [#38] grafana: Add instance to panels Signed-off-by: Dmitrii Stepanov --- services/grafana/provisioning/dashboards/storage-node.json | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index 8f38c0d..62de402 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -19,7 +19,6 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 2, "links": [], "liveNow": false, "panels": [ @@ -327,7 +326,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "histogram_quantile(0.95, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", + "expr": "histogram_quantile(0.95, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", "format": "time_series", "instant": false, "interval": "", @@ -427,7 +426,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "histogram_quantile(0.99, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", + "expr": "histogram_quantile(0.99, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", "format": "time_series", "instant": false, "interval": "", From f749581c4ebd4e43932965fdf93f16b070dc2380 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 1 Aug 2023 17:18:54 +0300 Subject: [PATCH 075/149] [#38] grafana: Add tree service row Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 666 +++++++++++++++--- 1 file changed, 575 insertions(+), 91 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index 62de402..df05dc0 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -244,7 +244,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "95%-tile latency of the requests.", + "description": "Latency of the requests for selected quantile", "fieldConfig": { "defaults": { "color": { @@ -326,7 +326,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "histogram_quantile(0.95, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", + "expr": "histogram_quantile($quantile, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", "format": "time_series", "instant": false, "interval": "", @@ -335,109 +335,560 @@ "refId": "A" } ], - "title": "Latency, 0.95q", + "title": "Latency", "transformations": [], "type": "timeseries" }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "99%-tile latency of the requests.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, + "collapsed": true, "gridPos": { - "h": 10, + "h": 1, "w": 24, "x": 0, "y": 31 }, - "id": 5, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ + "id": 6, + "panels": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile(0.99, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{grpc_service}}.{{grpc_method}}", - "range": true, - "refId": "A" + "description": "Requests per second by Pilorama method for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id, method) (\n rate(frostfs_node_pilorama_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Pilorama RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed pilorama requests for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 42 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(frostfs_node_pilorama_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (shard_id, method)\n / \nsum(rate(frostfs_node_pilorama_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (shard_id, method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Pilorama error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the pilorama requests for selected quantile for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 52 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_pilorama_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Pilorama latency", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Duration of waiting time for replication, replication task duration and synchronization for selected quantile", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 62 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_treeservice_replicate_wait_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (success,le)\n)", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "replicate wait success:{{success}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile,\n sum(rate(frostfs_node_treeservice_replicate_task_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (success,le)\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "replicate task success:{{success}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_treeservice_sync_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (success,le)\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "sync success:{{success}}", + "range": true, + "refId": "C" + } + ], + "title": "Replication and synchronization", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Pilorama mode for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1.5, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 72 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(frostfs_node_pilorama_mode{job=\"node\", instance=\"$instance\"}) by (shard_id, mode)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, mode {{mode}}", + "range": true, + "refId": "A" + } + ], + "title": "Pilorama mode", + "transformations": [], + "type": "timeseries" } ], - "title": "Latency, 0.99q", - "transformations": [], - "type": "timeseries" + "title": "Tree service", + "type": "row" } ], "refresh": "1m", @@ -459,7 +910,7 @@ "definition": "label_values(frostfs_node_state_health,instance)", "hide": 0, "includeAll": false, - "label": "", + "label": "Instance", "multi": false, "name": "instance", "options": [], @@ -472,11 +923,44 @@ "skipUrlSync": false, "sort": 1, "type": "query" + }, + { + "current": { + "selected": false, + "text": "0.95", + "value": "0.95" + }, + "hide": 0, + "includeAll": false, + "label": "Quantile", + "multi": false, + "name": "quantile", + "options": [ + { + "selected": true, + "text": "0.95", + "value": "0.95" + }, + { + "selected": false, + "text": "0.99", + "value": "0.99" + }, + { + "selected": false, + "text": "1.00", + "value": "1.00" + } + ], + "query": "0.95, 0.99, 1.00", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" } ] }, "time": { - "from": "now-30m", + "from": "now-1h", "to": "now" }, "timepicker": {}, From 4f9285251ffcf271861f44c6c05dc2359c3931c3 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 2 Aug 2023 15:06:03 +0300 Subject: [PATCH 076/149] [#38] grafana: Add writecache boards Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 727 +++++++++++++++++- 1 file changed, 724 insertions(+), 3 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index df05dc0..ce27576 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -354,7 +354,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Requests per second by Pilorama method for each shard", + "description": "Requests per second by pilorama method for each shard", "fieldConfig": { "defaults": { "color": { @@ -640,7 +640,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_pilorama_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method,le)\n)\n", + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_pilorama_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method, le)\n)\n", "format": "time_series", "instant": false, "interval": "", @@ -833,7 +833,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -889,6 +890,726 @@ ], "title": "Tree service", "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 12, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by writecache method for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id, method) (\n rate(frostfs_node_writecache_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed writecache requests for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 43 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(frostfs_node_writecache_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (shard_id, method)\n / \nsum(rate(frostfs_node_writecache_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (shard_id, method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the Writecache requests for selected quantile for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 53 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_writecache_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Actual count of the objects, that writecache stores", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 63 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_writecache_actual_objects_total{job=\"node\", instance=\"$instance\"}", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, storage {{storage}}", + "range": true, + "refId": "A" + } + ], + "title": "Actual count", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Estimated size of the objects, that writecache stores", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 73 + }, + "id": 17, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_writecache_estimated_size_bytes{job=\"node\", instance=\"$instance\"}", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, storage {{storage}}", + "range": true, + "refId": "A" + } + ], + "title": "Estimated size", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Rate of the writecache background operations", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 83 + }, + "id": 18, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (shard_id, storage, operation) (rate(frostfs_node_writecache_operations_total{job=\"node\", instance=\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, storage {{storage}}, operation {{operation}}", + "range": true, + "refId": "A" + } + ], + "title": "Operations rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Writecache mode for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1.5, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 93 + }, + "id": 19, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(frostfs_node_writecache_mode_info{job=\"node\", instance=\"$instance\"}) by (shard_id, mode)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, mode {{mode}}", + "range": true, + "refId": "A" + } + ], + "title": "Mode", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Writecache", + "type": "row" } ], "refresh": "1m", From d5ee2907403686eb8df604bec19c9d6f05f94c4b Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 2 Aug 2023 16:14:29 +0300 Subject: [PATCH 077/149] [#38] grafana: Add node instance state Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 746 +++++++++++------- 1 file changed, 446 insertions(+), 300 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index ce27576..4bd7a0e 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -23,93 +23,102 @@ "liveNow": false, "panels": [ { - "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Instance state", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [ + { + "options": { + "0": { + "index": 0, + "text": "Undefined" + }, + "1": { + "index": 1, + "text": "Starting" + }, + "2": { + "index": 2, + "text": "Ready" + }, + "3": { + "index": 3, + "text": "Shutting down" + } + }, + "type": "value" + } + ], + "max": 3, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "string" + }, + "overrides": [] + }, "gridPos": { - "h": 1, + "h": 6, "w": 24, "x": 0, "y": 0 }, - "id": 2, - "panels": [], - "title": "Server", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Requests per second by gRPC method", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 1 - }, - "id": 1, + "id": 21, "options": { "legend": { "calcs": [], "displayMode": "table", "placement": "right", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "multi", @@ -125,217 +134,16 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum by (instance, grpc_service, grpc_method) (\n rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])\n)\n", + "expr": "frostfs_node_state_health{job=\"node\", instance=\"$instance\"}\n", "format": "time_series", "instant": false, "interval": "", - "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "legendFormat": "state", "range": true, "refId": "A" } ], - "title": "RPS", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Percentage of the failed requests", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 11 - }, - "id": 3, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(rate(grpc_server_handled_total{instance=\"$instance\",grpc_code!=\"OK\"}[$__rate_interval])) by (grpc_service, grpc_method)\n / \nsum(rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])) by (grpc_service, grpc_method)\n * 100.0\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{grpc_service}}.{{grpc_method}}", - "range": true, - "refId": "A" - } - ], - "title": "Error rate", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Latency of the requests for selected quantile", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 21 - }, - "id": 4, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile($quantile, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{grpc_service}}.{{grpc_method}}", - "range": true, - "refId": "A" - } - ], - "title": "Latency", + "title": "State", "transformations": [], "type": "timeseries" }, @@ -345,7 +153,350 @@ "h": 1, "w": 24, "x": 0, - "y": 31 + "y": 6 + }, + "id": 2, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by gRPC method", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, grpc_service, grpc_method) (\n rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed requests", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "neo.fs.v2.object.ObjectService.Search" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(grpc_server_handled_total{instance=\"$instance\",grpc_code!=\"OK\"}[$__rate_interval])) by (grpc_service, grpc_method)\n / \nsum(rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])) by (grpc_service, grpc_method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the requests for selected quantile", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Server", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 7 }, "id": 6, "panels": [ @@ -398,8 +549,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -499,8 +649,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -599,8 +748,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -699,8 +847,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -833,8 +980,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -897,7 +1043,7 @@ "h": 1, "w": 24, "x": 0, - "y": 32 + "y": 8 }, "id": 12, "panels": [ @@ -967,7 +1113,7 @@ "h": 10, "w": 24, "x": 0, - "y": 33 + "y": 39 }, "id": 13, "options": { @@ -1068,7 +1214,7 @@ "h": 10, "w": 24, "x": 0, - "y": 43 + "y": 49 }, "id": 15, "options": { @@ -1168,7 +1314,7 @@ "h": 10, "w": 24, "x": 0, - "y": 53 + "y": 59 }, "id": 14, "options": { @@ -1268,7 +1414,7 @@ "h": 10, "w": 24, "x": 0, - "y": 63 + "y": 69 }, "id": 16, "options": { @@ -1368,7 +1514,7 @@ "h": 10, "w": 24, "x": 0, - "y": 73 + "y": 79 }, "id": 17, "options": { @@ -1468,7 +1614,7 @@ "h": 10, "w": 24, "x": 0, - "y": 83 + "y": 89 }, "id": 18, "options": { @@ -1570,7 +1716,7 @@ "h": 10, "w": 24, "x": 0, - "y": 93 + "y": 99 }, "id": 19, "options": { From d76cc2e48a0b61534e024707d038184881ba5e5b Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 3 Aug 2023 15:31:19 +0300 Subject: [PATCH 078/149] [#38] grafana: Add replicator dashbords Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 982 ++++++++++++------ 1 file changed, 654 insertions(+), 328 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index 4bd7a0e..0b97983 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -22,6 +22,19 @@ "links": [], "liveNow": false, "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 22, + "panels": [], + "title": "Health", + "type": "row" + }, { "datasource": { "type": "prometheus", @@ -110,7 +123,7 @@ "h": 6, "w": 24, "x": 0, - "y": 0 + "y": 1 }, "id": 21, "options": { @@ -148,347 +161,346 @@ "type": "timeseries" }, { - "collapsed": true, + "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 6 + "y": 7 }, "id": 2, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "panels": [], + "title": "Server", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by gRPC method", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "description": "Requests per second by gRPC method", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 7 - }, - "id": 1, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" }, - "tooltip": { - "mode": "multi", - "sort": "desc" + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" } }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum by (instance, grpc_service, grpc_method) (\n rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])\n)\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{grpc_service}}.{{grpc_method}}", - "range": true, - "refId": "A" - } - ], - "title": "RPS", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Percentage of the failed requests", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [ + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "neo.fs.v2.object.ObjectService.Search" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 } ] }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 17 - }, - "id": 3, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(rate(grpc_server_handled_total{instance=\"$instance\",grpc_code!=\"OK\"}[$__rate_interval])) by (grpc_service, grpc_method)\n / \nsum(rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])) by (grpc_service, grpc_method)\n * 100.0\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{grpc_service}}.{{grpc_method}}", - "range": true, - "refId": "A" - } - ], - "title": "Error rate", - "transformations": [], - "type": "timeseries" + "unit": "reqps" }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Latency of the requests for selected quantile", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 27 - }, - "id": 4, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile($quantile, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{grpc_service}}.{{grpc_method}}", - "range": true, - "refId": "A" - } - ], - "title": "Latency", - "transformations": [], - "type": "timeseries" + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, grpc_service, grpc_method) (\n rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" } ], - "title": "Server", - "type": "row" + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed requests", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "neo.fs.v2.object.ObjectService.Search" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(grpc_server_handled_total{instance=\"$instance\",grpc_code!=\"OK\"}[$__rate_interval])) by (grpc_service, grpc_method)\n / \nsum(rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])) by (grpc_service, grpc_method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the requests for selected quantile", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" }, { "collapsed": true, @@ -496,7 +508,7 @@ "h": 1, "w": 24, "x": 0, - "y": 7 + "y": 38 }, "id": 6, "panels": [ @@ -1043,7 +1055,7 @@ "h": 1, "w": 24, "x": 0, - "y": 8 + "y": 39 }, "id": 12, "panels": [ @@ -1113,7 +1125,7 @@ "h": 10, "w": 24, "x": 0, - "y": 39 + "y": 4 }, "id": 13, "options": { @@ -1214,7 +1226,7 @@ "h": 10, "w": 24, "x": 0, - "y": 49 + "y": 14 }, "id": 15, "options": { @@ -1314,7 +1326,7 @@ "h": 10, "w": 24, "x": 0, - "y": 59 + "y": 24 }, "id": 14, "options": { @@ -1414,7 +1426,7 @@ "h": 10, "w": 24, "x": 0, - "y": 69 + "y": 34 }, "id": 16, "options": { @@ -1514,7 +1526,7 @@ "h": 10, "w": 24, "x": 0, - "y": 79 + "y": 44 }, "id": 17, "options": { @@ -1614,7 +1626,7 @@ "h": 10, "w": 24, "x": 0, - "y": 89 + "y": 54 }, "id": 18, "options": { @@ -1716,7 +1728,7 @@ "h": 10, "w": 24, "x": 0, - "y": 99 + "y": 64 }, "id": 19, "options": { @@ -1756,6 +1768,320 @@ ], "title": "Writecache", "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 23, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Count of in-flight requests", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 75 + }, + "id": 24, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_replicator_in_flight_requests_total{job=\"node\", instance=\"$instance\"}", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "in-flight requests", + "range": true, + "refId": "A" + } + ], + "title": "In-flight request", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Rate of the objects replicated", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 85 + }, + "id": 25, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "rate(frostfs_node_replicator_processed_objects_total{job=\"node\", instance=\"$instance\"}[$__rate_interval])", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "object processing rate", + "range": true, + "refId": "A" + } + ], + "title": "Objects processing rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Rate of the objects payload replicated", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 95 + }, + "id": 26, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "rate(frostfs_node_replicator_total_replicated_payload_size_bytes{job=\"node\", instance=\"$instance\"}[$__rate_interval])", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "payload processing rate", + "range": true, + "refId": "A" + } + ], + "title": "Payload processing rate", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Replicator", + "type": "row" } ], "refresh": "1m", From 74d43f48f2c31ea20d4238efd9a24218239077c4 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 3 Aug 2023 15:48:53 +0300 Subject: [PATCH 079/149] [#38] grafana: Add object service dashboards Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 435 +++++++++++++++++- 1 file changed, 428 insertions(+), 7 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index 0b97983..264ad20 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -1125,7 +1125,7 @@ "h": 10, "w": 24, "x": 0, - "y": 4 + "y": 40 }, "id": 13, "options": { @@ -1226,7 +1226,7 @@ "h": 10, "w": 24, "x": 0, - "y": 14 + "y": 50 }, "id": 15, "options": { @@ -1326,7 +1326,7 @@ "h": 10, "w": 24, "x": 0, - "y": 24 + "y": 60 }, "id": 14, "options": { @@ -1426,7 +1426,7 @@ "h": 10, "w": 24, "x": 0, - "y": 34 + "y": 70 }, "id": 16, "options": { @@ -1526,7 +1526,7 @@ "h": 10, "w": 24, "x": 0, - "y": 44 + "y": 80 }, "id": 17, "options": { @@ -1626,7 +1626,7 @@ "h": 10, "w": 24, "x": 0, - "y": 54 + "y": 90 }, "id": 18, "options": { @@ -1728,7 +1728,7 @@ "h": 10, "w": 24, "x": 0, - "y": 64 + "y": 100 }, "id": 19, "options": { @@ -2082,6 +2082,427 @@ ], "title": "Replicator", "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 27, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by object service method", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 112 + }, + "id": 28, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, method) (\n rate(frostfs_node_object_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed object service requests", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 122 + }, + "id": 29, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(frostfs_node_object_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (method)\n / \nsum(rate(frostfs_node_object_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the object service requests for selected quantile", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 132 + }, + "id": 30, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_object_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Bytes per second by object service method", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 142 + }, + "id": 31, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, method) (\n rate(frostfs_node_object_request_payload_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Payload rate", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Object service", + "type": "row" } ], "refresh": "1m", From 201855e72958978118c8ff9e2aede2c757acf5bc Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 3 Aug 2023 15:57:07 +0300 Subject: [PATCH 080/149] [#38] grafana: Add epoch dashboard Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 108 ++++++++++++++---- 1 file changed, 87 insertions(+), 21 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index 264ad20..4aaaebf 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -32,7 +32,7 @@ }, "id": 22, "panels": [], - "title": "Health", + "title": "General", "type": "row" }, { @@ -121,7 +121,7 @@ }, "gridPos": { "h": 6, - "w": 24, + "w": 12, "x": 0, "y": 1 }, @@ -160,6 +160,72 @@ "transformations": [], "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Current epoch on the instance", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 32, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "frostfs_node_ir_epoch{job=\"node\", instance=\"$instance\"}", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Epoch", + "type": "gauge" + }, { "collapsed": false, "gridPos": { @@ -577,7 +643,7 @@ "h": 10, "w": 24, "x": 0, - "y": 32 + "y": 40 }, "id": 8, "options": { @@ -677,7 +743,7 @@ "h": 10, "w": 24, "x": 0, - "y": 42 + "y": 50 }, "id": 9, "options": { @@ -776,7 +842,7 @@ "h": 10, "w": 24, "x": 0, - "y": 52 + "y": 60 }, "id": 10, "options": { @@ -875,7 +941,7 @@ "h": 10, "w": 24, "x": 0, - "y": 62 + "y": 70 }, "id": 7, "options": { @@ -1008,7 +1074,7 @@ "h": 10, "w": 24, "x": 0, - "y": 72 + "y": 80 }, "id": 11, "options": { @@ -1125,7 +1191,7 @@ "h": 10, "w": 24, "x": 0, - "y": 40 + "y": 48 }, "id": 13, "options": { @@ -1226,7 +1292,7 @@ "h": 10, "w": 24, "x": 0, - "y": 50 + "y": 58 }, "id": 15, "options": { @@ -1326,7 +1392,7 @@ "h": 10, "w": 24, "x": 0, - "y": 60 + "y": 68 }, "id": 14, "options": { @@ -1426,7 +1492,7 @@ "h": 10, "w": 24, "x": 0, - "y": 70 + "y": 78 }, "id": 16, "options": { @@ -1526,7 +1592,7 @@ "h": 10, "w": 24, "x": 0, - "y": 80 + "y": 88 }, "id": 17, "options": { @@ -1626,7 +1692,7 @@ "h": 10, "w": 24, "x": 0, - "y": 90 + "y": 98 }, "id": 18, "options": { @@ -1728,7 +1794,7 @@ "h": 10, "w": 24, "x": 0, - "y": 100 + "y": 108 }, "id": 19, "options": { @@ -1842,7 +1908,7 @@ "h": 10, "w": 24, "x": 0, - "y": 75 + "y": 83 }, "id": 24, "options": { @@ -1942,7 +2008,7 @@ "h": 10, "w": 24, "x": 0, - "y": 85 + "y": 93 }, "id": 25, "options": { @@ -2042,7 +2108,7 @@ "h": 10, "w": 24, "x": 0, - "y": 95 + "y": 103 }, "id": 26, "options": { @@ -2159,7 +2225,7 @@ "h": 10, "w": 24, "x": 0, - "y": 112 + "y": 120 }, "id": 28, "options": { @@ -2260,7 +2326,7 @@ "h": 10, "w": 24, "x": 0, - "y": 122 + "y": 130 }, "id": 29, "options": { @@ -2360,7 +2426,7 @@ "h": 10, "w": 24, "x": 0, - "y": 132 + "y": 140 }, "id": 30, "options": { @@ -2463,7 +2529,7 @@ "h": 10, "w": 24, "x": 0, - "y": 142 + "y": 150 }, "id": 31, "options": { From 8218440525c4129ef97e18534522a879a7a909dd Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 3 Aug 2023 16:11:35 +0300 Subject: [PATCH 081/149] [#38] grafana: Add metabase dashboards Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 3332 ++++++++++------- 1 file changed, 1876 insertions(+), 1456 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index 4aaaebf..20a3ed2 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -576,6 +576,1881 @@ "x": 0, "y": 38 }, + "id": 27, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by object service method", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 120 + }, + "id": 28, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, method) (\n rate(frostfs_node_object_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed object service requests", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 130 + }, + "id": 29, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(frostfs_node_object_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (method)\n / \nsum(rate(frostfs_node_object_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the object service requests for selected quantile", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 140 + }, + "id": 30, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_object_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Bytes per second by object service method", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 150 + }, + "id": 31, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, method) (\n rate(frostfs_node_object_request_payload_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Payload rate", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Object service", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 33, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by metabase method for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 35, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id, method) (\n rate(frostfs_node_metabase_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed metabase requests for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 37, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(frostfs_node_metabase_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (shard_id, method)\n / \nsum(rate(frostfs_node_metabase_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (shard_id, method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the metabase requests for selected quantile for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 36, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_metabase_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Metabase mode for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1.5, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 34, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(frostfs_node_metabase_mode{job=\"node\", instance=\"$instance\"}) by (shard_id, mode)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, mode {{mode}}", + "range": true, + "refId": "A" + } + ], + "title": "Mode", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Metabase", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 12, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by writecache method for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id, method) (\n rate(frostfs_node_writecache_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed writecache requests for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 51 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(frostfs_node_writecache_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (shard_id, method)\n / \nsum(rate(frostfs_node_writecache_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (shard_id, method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the writecache requests for selected quantile for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 61 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_writecache_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Actual count of the objects, that writecache stores", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 71 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_writecache_actual_objects_total{job=\"node\", instance=\"$instance\"}", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, storage {{storage}}", + "range": true, + "refId": "A" + } + ], + "title": "Actual count", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Estimated size of the objects, that writecache stores", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 81 + }, + "id": 17, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_writecache_estimated_size_bytes{job=\"node\", instance=\"$instance\"}", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, storage {{storage}}", + "range": true, + "refId": "A" + } + ], + "title": "Estimated size", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Rate of the writecache background operations", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 91 + }, + "id": 18, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (shard_id, storage, operation) (rate(frostfs_node_writecache_operations_total{job=\"node\", instance=\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, storage {{storage}}, operation {{operation}}", + "range": true, + "refId": "A" + } + ], + "title": "Operations rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Writecache mode for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1.5, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 101 + }, + "id": 19, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(frostfs_node_writecache_mode_info{job=\"node\", instance=\"$instance\"}) by (shard_id, mode)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, mode {{mode}}", + "range": true, + "refId": "A" + } + ], + "title": "Mode", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Writecache", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 23, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Count of in-flight requests", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 83 + }, + "id": 24, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_replicator_in_flight_requests_total{job=\"node\", instance=\"$instance\"}", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "in-flight requests", + "range": true, + "refId": "A" + } + ], + "title": "In-flight request", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Rate of the objects replicated", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 93 + }, + "id": 25, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "rate(frostfs_node_replicator_processed_objects_total{job=\"node\", instance=\"$instance\"}[$__rate_interval])", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "object processing rate", + "range": true, + "refId": "A" + } + ], + "title": "Objects processing rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Rate of the objects payload replicated", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 103 + }, + "id": 26, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "rate(frostfs_node_replicator_total_replicated_payload_size_bytes{job=\"node\", instance=\"$instance\"}[$__rate_interval])", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "payload processing rate", + "range": true, + "refId": "A" + } + ], + "title": "Payload processing rate", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Replicator", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 42 + }, "id": 6, "panels": [ { @@ -1114,1461 +2989,6 @@ ], "title": "Tree service", "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 39 - }, - "id": 12, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Requests per second by writecache method for each shard", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 48 - }, - "id": 13, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum by (instance, shard_id, method) (\n rate(frostfs_node_writecache_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{method}}, shard {{shard_id}}", - "range": true, - "refId": "A" - } - ], - "title": "RPS", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Percentage of the failed writecache requests for each shard", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 58 - }, - "id": 15, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(rate(frostfs_node_writecache_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (shard_id, method)\n / \nsum(rate(frostfs_node_writecache_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (shard_id, method)\n * 100.0\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{method}}, shard {{shard_id}}", - "range": true, - "refId": "A" - } - ], - "title": "Error rate", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Latency of the Writecache requests for selected quantile for each shard", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 68 - }, - "id": 14, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_writecache_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method,le)\n)\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{method}}, shard {{shard_id}}", - "range": true, - "refId": "A" - } - ], - "title": "Latency", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Actual count of the objects, that writecache stores", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 78 - }, - "id": 16, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "frostfs_node_writecache_actual_objects_total{job=\"node\", instance=\"$instance\"}", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "Shard {{shard_id}}, storage {{storage}}", - "range": true, - "refId": "A" - } - ], - "title": "Actual count", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Estimated size of the objects, that writecache stores", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 88 - }, - "id": 17, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "frostfs_node_writecache_estimated_size_bytes{job=\"node\", instance=\"$instance\"}", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "Shard {{shard_id}}, storage {{storage}}", - "range": true, - "refId": "A" - } - ], - "title": "Estimated size", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Rate of the writecache background operations", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 98 - }, - "id": 18, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum by (shard_id, storage, operation) (rate(frostfs_node_writecache_operations_total{job=\"node\", instance=\"$instance\"}[$__rate_interval]))", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "Shard {{shard_id}}, storage {{storage}}, operation {{operation}}", - "range": true, - "refId": "A" - } - ], - "title": "Operations rate", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Writecache mode for each shard", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 1.5, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 108 - }, - "id": 19, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(frostfs_node_writecache_mode_info{job=\"node\", instance=\"$instance\"}) by (shard_id, mode)\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "Shard {{shard_id}}, mode {{mode}}", - "range": true, - "refId": "A" - } - ], - "title": "Mode", - "transformations": [], - "type": "timeseries" - } - ], - "title": "Writecache", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 40 - }, - "id": 23, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Count of in-flight requests", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 83 - }, - "id": 24, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "frostfs_node_replicator_in_flight_requests_total{job=\"node\", instance=\"$instance\"}", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "in-flight requests", - "range": true, - "refId": "A" - } - ], - "title": "In-flight request", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Rate of the objects replicated", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 93 - }, - "id": 25, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "rate(frostfs_node_replicator_processed_objects_total{job=\"node\", instance=\"$instance\"}[$__rate_interval])", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "object processing rate", - "range": true, - "refId": "A" - } - ], - "title": "Objects processing rate", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Rate of the objects payload replicated", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 103 - }, - "id": 26, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "rate(frostfs_node_replicator_total_replicated_payload_size_bytes{job=\"node\", instance=\"$instance\"}[$__rate_interval])", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "payload processing rate", - "range": true, - "refId": "A" - } - ], - "title": "Payload processing rate", - "transformations": [], - "type": "timeseries" - } - ], - "title": "Replicator", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 41 - }, - "id": 27, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Requests per second by object service method", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 120 - }, - "id": 28, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum by (instance, method) (\n rate(frostfs_node_object_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{method}}", - "range": true, - "refId": "A" - } - ], - "title": "RPS", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Percentage of the failed object service requests", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 130 - }, - "id": 29, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(rate(frostfs_node_object_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (method)\n / \nsum(rate(frostfs_node_object_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (method)\n * 100.0\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{method}}, shard {{shard_id}}", - "range": true, - "refId": "A" - } - ], - "title": "Error rate", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Latency of the object service requests for selected quantile", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 140 - }, - "id": 30, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_object_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (method,le)\n)\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{method}}", - "range": true, - "refId": "A" - } - ], - "title": "Latency", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Bytes per second by object service method", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 150 - }, - "id": 31, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum by (instance, method) (\n rate(frostfs_node_object_request_payload_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{method}}", - "range": true, - "refId": "A" - } - ], - "title": "Payload rate", - "transformations": [], - "type": "timeseries" - } - ], - "title": "Object service", - "type": "row" } ], "refresh": "1m", @@ -2579,7 +2999,7 @@ "list": [ { "current": { - "selected": false, + "selected": true, "text": "s01.frostfs.devenv:9090", "value": "s01.frostfs.devenv:9090" }, From 1408558631b490387e17474775a704cdd1c9e4c1 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 4 Aug 2023 13:59:12 +0300 Subject: [PATCH 082/149] [#38] grafana: Fix units Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 681 +++++++++--------- 1 file changed, 341 insertions(+), 340 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index 20a3ed2..a2a37aa 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -227,7 +227,7 @@ "type": "gauge" }, { - "collapsed": false, + "collapsed": true, "gridPos": { "h": 1, "w": 24, @@ -235,338 +235,339 @@ "y": 7 }, "id": 2, - "panels": [], - "title": "Server", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Requests per second by gRPC method", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 8 - }, - "id": 1, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ + "panels": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "editorMode": "code", - "exemplar": false, - "expr": "sum by (instance, grpc_service, grpc_method) (\n rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])\n)\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{grpc_service}}.{{grpc_method}}", - "range": true, - "refId": "A" - } - ], - "title": "RPS", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Percentage of the failed requests", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "description": "Requests per second by gRPC method", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "neo.fs.v2.object.ObjectService.Search" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, - "viz": true + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 18 - }, - "id": 3, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(rate(grpc_server_handled_total{instance=\"$instance\",grpc_code!=\"OK\"}[$__rate_interval])) by (grpc_service, grpc_method)\n / \nsum(rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])) by (grpc_service, grpc_method)\n * 100.0\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{grpc_service}}.{{grpc_method}}", - "range": true, - "refId": "A" - } - ], - "title": "Error rate", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Latency of the requests for selected quantile", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 8 }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "multi", + "sort": "desc" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, grpc_service, grpc_method) (\n rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 28 - }, - "id": 4, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "9.5.6", - "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile($quantile, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{grpc_service}}.{{grpc_method}}", - "range": true, - "refId": "A" + "description": "Percentage of the failed requests", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "neo.fs.v2.object.ObjectService.Search" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(grpc_server_handled_total{instance=\"$instance\",grpc_code!=\"OK\"}[$__rate_interval])) by (grpc_service, grpc_method)\n / \nsum(rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])) by (grpc_service, grpc_method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the requests for selected quantile", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" } ], - "title": "Latency", - "transformations": [], - "type": "timeseries" + "title": "Server", + "type": "row" }, { "collapsed": true, @@ -574,7 +575,7 @@ "h": 1, "w": 24, "x": 0, - "y": 38 + "y": 8 }, "id": 27, "panels": [ @@ -644,7 +645,7 @@ "h": 10, "w": 24, "x": 0, - "y": 120 + "y": 9 }, "id": 28, "options": { @@ -745,7 +746,7 @@ "h": 10, "w": 24, "x": 0, - "y": 130 + "y": 19 }, "id": 29, "options": { @@ -837,7 +838,7 @@ } ] }, - "unit": "ms" + "unit": "s" }, "overrides": [] }, @@ -845,7 +846,7 @@ "h": 10, "w": 24, "x": 0, - "y": 140 + "y": 29 }, "id": 30, "options": { @@ -940,7 +941,7 @@ } ] }, - "unit": "Bps" + "unit": "binBps" }, "overrides": [] }, @@ -948,7 +949,7 @@ "h": 10, "w": 24, "x": 0, - "y": 150 + "y": 39 }, "id": 31, "options": { @@ -995,7 +996,7 @@ "h": 1, "w": 24, "x": 0, - "y": 39 + "y": 9 }, "id": 33, "panels": [ @@ -1065,7 +1066,7 @@ "h": 10, "w": 24, "x": 0, - "y": 1 + "y": 10 }, "id": 35, "options": { @@ -1166,7 +1167,7 @@ "h": 10, "w": 24, "x": 0, - "y": 11 + "y": 20 }, "id": 37, "options": { @@ -1258,7 +1259,7 @@ } ] }, - "unit": "ms" + "unit": "s" }, "overrides": [] }, @@ -1266,7 +1267,7 @@ "h": 10, "w": 24, "x": 0, - "y": 21 + "y": 30 }, "id": 36, "options": { @@ -1368,7 +1369,7 @@ "h": 10, "w": 24, "x": 0, - "y": 31 + "y": 40 }, "id": 34, "options": { @@ -1415,7 +1416,7 @@ "h": 1, "w": 24, "x": 0, - "y": 40 + "y": 10 }, "id": 12, "panels": [ @@ -1485,7 +1486,7 @@ "h": 10, "w": 24, "x": 0, - "y": 41 + "y": 11 }, "id": 13, "options": { @@ -1586,7 +1587,7 @@ "h": 10, "w": 24, "x": 0, - "y": 51 + "y": 21 }, "id": 15, "options": { @@ -1678,7 +1679,7 @@ } ] }, - "unit": "ms" + "unit": "s" }, "overrides": [] }, @@ -1686,7 +1687,7 @@ "h": 10, "w": 24, "x": 0, - "y": 61 + "y": 31 }, "id": 14, "options": { @@ -1786,7 +1787,7 @@ "h": 10, "w": 24, "x": 0, - "y": 71 + "y": 41 }, "id": 16, "options": { @@ -1878,7 +1879,7 @@ } ] }, - "unit": "decbytes" + "unit": "bytes" }, "overrides": [] }, @@ -1886,7 +1887,7 @@ "h": 10, "w": 24, "x": 0, - "y": 81 + "y": 51 }, "id": 17, "options": { @@ -1986,7 +1987,7 @@ "h": 10, "w": 24, "x": 0, - "y": 91 + "y": 61 }, "id": 18, "options": { @@ -2088,7 +2089,7 @@ "h": 10, "w": 24, "x": 0, - "y": 101 + "y": 71 }, "id": 19, "options": { @@ -2135,7 +2136,7 @@ "h": 1, "w": 24, "x": 0, - "y": 41 + "y": 11 }, "id": 23, "panels": [ @@ -2202,7 +2203,7 @@ "h": 10, "w": 24, "x": 0, - "y": 83 + "y": 12 }, "id": 24, "options": { @@ -2302,7 +2303,7 @@ "h": 10, "w": 24, "x": 0, - "y": 93 + "y": 22 }, "id": 25, "options": { @@ -2394,7 +2395,7 @@ } ] }, - "unit": "Bps" + "unit": "binBps" }, "overrides": [] }, @@ -2402,7 +2403,7 @@ "h": 10, "w": 24, "x": 0, - "y": 103 + "y": 32 }, "id": 26, "options": { @@ -2449,7 +2450,7 @@ "h": 1, "w": 24, "x": 0, - "y": 42 + "y": 12 }, "id": 6, "panels": [ @@ -2999,7 +3000,7 @@ "list": [ { "current": { - "selected": true, + "selected": false, "text": "s01.frostfs.devenv:9090", "value": "s01.frostfs.devenv:9090" }, From 0231b2bbf0b0a2cf95b255839c79bbbbc28527e7 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 4 Aug 2023 14:34:54 +0300 Subject: [PATCH 083/149] [#38] grafana: Add blobstore dashboard Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 565 +++++++++++++++++- 1 file changed, 551 insertions(+), 14 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index a2a37aa..0d8213f 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -168,10 +168,6 @@ "description": "Current epoch on the instance", "fieldConfig": { "defaults": { - "color": { - "fixedColor": "green", - "mode": "fixed" - }, "mappings": [], "thresholds": { "mode": "absolute", @@ -186,7 +182,7 @@ } ] }, - "unit": "none" + "unit": "short" }, "overrides": [] }, @@ -198,6 +194,9 @@ }, "id": 32, "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ @@ -206,8 +205,7 @@ "fields": "", "values": false }, - "showThresholdLabels": false, - "showThresholdMarkers": true + "textMode": "auto" }, "pluginVersion": "9.5.6", "targets": [ @@ -224,7 +222,7 @@ } ], "title": "Epoch", - "type": "gauge" + "type": "stat" }, { "collapsed": true, @@ -1066,7 +1064,7 @@ "h": 10, "w": 24, "x": 0, - "y": 10 + "y": 51 }, "id": 35, "options": { @@ -1167,7 +1165,7 @@ "h": 10, "w": 24, "x": 0, - "y": 20 + "y": 61 }, "id": 37, "options": { @@ -1267,7 +1265,7 @@ "h": 10, "w": 24, "x": 0, - "y": 30 + "y": 71 }, "id": 36, "options": { @@ -1369,7 +1367,7 @@ "h": 10, "w": 24, "x": 0, - "y": 40 + "y": 81 }, "id": 34, "options": { @@ -2138,6 +2136,545 @@ "x": 0, "y": 11 }, + "id": 38, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by blobstore method for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 39, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id, method) (\n rate(frostfs_node_blobstore_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed blobstore requests for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 40, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(frostfs_node_blobstore_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (shard_id, method)\n / \nsum(rate(frostfs_node_blobstore_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (shard_id, method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the metabase requests for selected quantile for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 41, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_blobstore_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Bytes per second by blobstore shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 42 + }, + "id": 43, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id) (\n rate(frostfs_node_blobstore_put_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Write rate, shard {{shard_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id) (\n rate(frostfs_node_blobstore_get_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Read rate, shard {{shard_id}}", + "range": true, + "refId": "B" + } + ], + "title": "Payload rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Blobstore mode for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1.5, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 52 + }, + "id": 42, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(frostfs_node_blobstore_mode{job=\"node\", instance=\"$instance\"}) by (shard_id, mode)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, mode {{mode}}", + "range": true, + "refId": "A" + } + ], + "title": "Mode", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Blobstore", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 12 + }, "id": 23, "panels": [ { @@ -2450,7 +2987,7 @@ "h": 1, "w": 24, "x": 0, - "y": 12 + "y": 13 }, "id": 6, "panels": [ @@ -3000,7 +3537,7 @@ "list": [ { "current": { - "selected": false, + "selected": true, "text": "s01.frostfs.devenv:9090", "value": "s01.frostfs.devenv:9090" }, From acd32cb87720840754556a255b15f81c6faf6fb8 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 4 Aug 2023 14:50:04 +0300 Subject: [PATCH 084/149] [#38] grafana: Add blobovnizca dashboards Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 757 +++++++++++++++++- 1 file changed, 751 insertions(+), 6 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index 0d8213f..cd8f32c 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -2204,7 +2204,7 @@ "h": 10, "w": 24, "x": 0, - "y": 12 + "y": 13 }, "id": 39, "options": { @@ -2305,7 +2305,7 @@ "h": 10, "w": 24, "x": 0, - "y": 22 + "y": 23 }, "id": 40, "options": { @@ -2405,7 +2405,7 @@ "h": 10, "w": 24, "x": 0, - "y": 32 + "y": 33 }, "id": 41, "options": { @@ -2508,7 +2508,7 @@ "h": 10, "w": 24, "x": 0, - "y": 42 + "y": 43 }, "id": 43, "options": { @@ -2626,7 +2626,7 @@ "h": 10, "w": 24, "x": 0, - "y": 52 + "y": 53 }, "id": 42, "options": { @@ -2675,6 +2675,751 @@ "x": 0, "y": 12 }, + "id": 44, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by blobovnizca method for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 45, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id, method) (\n rate(frostfs_node_blobovniza_tree_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed blobovnizca requests for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 46, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(frostfs_node_blobsovniza_tree_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (shard_id, method)\n / \nsum(rate(frostfs_node_blobsovniza_tree_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (shard_id, method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the blobovnizca requests for selected quantile for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 47, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_blobovniza_tree_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Bytes per second by blobovnizca shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 48, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id) (\n rate(frostfs_node_blobovniza_tree_put_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Write rate, shard {{shard_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id) (\n rate(frostfs_node_blobovniza_tree_get_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Read rate, shard {{shard_id}}", + "range": true, + "refId": "B" + } + ], + "title": "Payload rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Blobovnizca size for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 50, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_blobovniza_tree_size_bytes{instance=\"$instance\"}\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Size", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Count of open DB's of blobovnizca for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 51 + }, + "id": 51, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_blobovniza_tree_open_blobovnizca_count{instance=\"$instance\"}\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Open DB count", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Blobovnizca mode for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1.5, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 61 + }, + "id": 49, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(frostfs_node_blobovniza_tree_mode{job=\"node\", instance=\"$instance\"}) by (shard_id, mode)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, mode {{mode}}", + "range": true, + "refId": "A" + } + ], + "title": "Mode", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Blobovnizca", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 13 + }, "id": 23, "panels": [ { @@ -2987,7 +3732,7 @@ "h": 1, "w": 24, "x": 0, - "y": 13 + "y": 14 }, "id": 6, "panels": [ From cc963b78b5ca1f4b16c67d516a18467954dcce63 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 4 Aug 2023 15:03:48 +0300 Subject: [PATCH 085/149] [#38] grafana: Add fstree dashboards Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 567 +++++++++++++++++- 1 file changed, 553 insertions(+), 14 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index cd8f32c..6fd6bf5 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -2204,7 +2204,7 @@ "h": 10, "w": 24, "x": 0, - "y": 13 + "y": 12 }, "id": 39, "options": { @@ -2305,7 +2305,7 @@ "h": 10, "w": 24, "x": 0, - "y": 23 + "y": 22 }, "id": 40, "options": { @@ -2405,7 +2405,7 @@ "h": 10, "w": 24, "x": 0, - "y": 33 + "y": 32 }, "id": 41, "options": { @@ -2508,7 +2508,7 @@ "h": 10, "w": 24, "x": 0, - "y": 43 + "y": 42 }, "id": 43, "options": { @@ -2626,7 +2626,7 @@ "h": 10, "w": 24, "x": 0, - "y": 53 + "y": 52 }, "id": 42, "options": { @@ -2675,6 +2675,545 @@ "x": 0, "y": 12 }, + "id": 52, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by fstree method for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 63 + }, + "id": 53, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id, method) (\n rate(frostfs_node_fstree_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed fstree requests for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 73 + }, + "id": 54, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(frostfs_node_fstree_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (shard_id, method)\n / \nsum(rate(frostfs_node_fstree_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (shard_id, method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the fstree requests for selected quantile for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 83 + }, + "id": 55, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_fstree_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Bytes per second by fstree shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 93 + }, + "id": 56, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id) (\n rate(frostfs_node_fstree_put_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Write rate, shard {{shard_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id) (\n rate(frostfs_node_fstree_get_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Read rate, shard {{shard_id}}", + "range": true, + "refId": "B" + } + ], + "title": "Payload rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "FSTree mode for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1.5, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 103 + }, + "id": 57, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(frostfs_node_fstree_mode{job=\"node\", instance=\"$instance\"}) by (shard_id, mode)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, mode {{mode}}", + "range": true, + "refId": "A" + } + ], + "title": "Mode", + "transformations": [], + "type": "timeseries" + } + ], + "title": "FSTree", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 13 + }, "id": 44, "panels": [ { @@ -2743,7 +3282,7 @@ "h": 10, "w": 24, "x": 0, - "y": 1 + "y": 64 }, "id": 45, "options": { @@ -2844,7 +3383,7 @@ "h": 10, "w": 24, "x": 0, - "y": 11 + "y": 74 }, "id": 46, "options": { @@ -2944,7 +3483,7 @@ "h": 10, "w": 24, "x": 0, - "y": 21 + "y": 84 }, "id": 47, "options": { @@ -3047,7 +3586,7 @@ "h": 10, "w": 24, "x": 0, - "y": 31 + "y": 94 }, "id": 48, "options": { @@ -3166,7 +3705,7 @@ "h": 10, "w": 24, "x": 0, - "y": 41 + "y": 104 }, "id": 50, "options": { @@ -3269,7 +3808,7 @@ "h": 10, "w": 24, "x": 0, - "y": 51 + "y": 114 }, "id": 51, "options": { @@ -3371,7 +3910,7 @@ "h": 10, "w": 24, "x": 0, - "y": 61 + "y": 124 }, "id": 49, "options": { @@ -3418,7 +3957,7 @@ "h": 1, "w": 24, "x": 0, - "y": 13 + "y": 14 }, "id": 23, "panels": [ @@ -3732,7 +4271,7 @@ "h": 1, "w": 24, "x": 0, - "y": 14 + "y": 15 }, "id": 6, "panels": [ From d8df46b4d19ae673d44a0e55585c1a3fa68f8787 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Sat, 5 Aug 2023 21:16:47 +0300 Subject: [PATCH 086/149] [#38] grafana: Add engine dashboards Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 743 ++++++++++++++++-- 1 file changed, 672 insertions(+), 71 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index 6fd6bf5..0d08ded 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -403,7 +403,7 @@ "options": { "mode": "exclude", "names": [ - "neo.fs.v2.object.ObjectService.Search" + "neo.fs.v2.object.ObjectService.Put" ], "prefix": "All except:", "readOnly": true @@ -643,7 +643,7 @@ "h": 10, "w": 24, "x": 0, - "y": 9 + "y": 20 }, "id": 28, "options": { @@ -744,7 +744,7 @@ "h": 10, "w": 24, "x": 0, - "y": 19 + "y": 30 }, "id": 29, "options": { @@ -844,7 +844,7 @@ "h": 10, "w": 24, "x": 0, - "y": 29 + "y": 40 }, "id": 30, "options": { @@ -930,8 +930,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -947,7 +946,7 @@ "h": 10, "w": 24, "x": 0, - "y": 39 + "y": 50 }, "id": 31, "options": { @@ -996,6 +995,634 @@ "x": 0, "y": 9 }, + "id": 58, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by storage engine service method", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 59, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, method) (\n rate(frostfs_node_engine_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the engine requests for selected quantile for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 60, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_engine_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Payload size by shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 61, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_engine_payload_size_bytes{instance=\"$instance\"}\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Payload", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Objects count by shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 62, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_engine_objects_total{instance=\"$instance\"}\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, type {{type}}", + "range": true, + "refId": "A" + } + ], + "title": "Objects count", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Errors count by shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 50 + }, + "id": 63, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_engine_errors_total{instance=\"$instance\"}\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Errors count", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Mode for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1.5, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 60 + }, + "id": 64, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(frostfs_node_engine_mode_info{job=\"node\", instance=\"$instance\"}) by (shard_id, mode)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, mode {{mode}}", + "range": true, + "refId": "A" + } + ], + "title": "Shard mode", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Engine", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, "id": 33, "panels": [ { @@ -1047,8 +1674,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1148,8 +1774,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1248,8 +1873,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1350,8 +1974,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1414,7 +2037,7 @@ "h": 1, "w": 24, "x": 0, - "y": 10 + "y": 11 }, "id": 12, "panels": [ @@ -1467,8 +2090,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1568,8 +2190,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1668,8 +2289,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1768,8 +2388,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1868,8 +2487,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1968,8 +2586,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2070,8 +2687,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2134,7 +2750,7 @@ "h": 1, "w": 24, "x": 0, - "y": 11 + "y": 12 }, "id": 38, "panels": [ @@ -2204,7 +2820,7 @@ "h": 10, "w": 24, "x": 0, - "y": 12 + "y": 13 }, "id": 39, "options": { @@ -2305,7 +2921,7 @@ "h": 10, "w": 24, "x": 0, - "y": 22 + "y": 23 }, "id": 40, "options": { @@ -2405,7 +3021,7 @@ "h": 10, "w": 24, "x": 0, - "y": 32 + "y": 33 }, "id": 41, "options": { @@ -2508,7 +3124,7 @@ "h": 10, "w": 24, "x": 0, - "y": 42 + "y": 43 }, "id": 43, "options": { @@ -2626,7 +3242,7 @@ "h": 10, "w": 24, "x": 0, - "y": 52 + "y": 53 }, "id": 42, "options": { @@ -2673,7 +3289,7 @@ "h": 1, "w": 24, "x": 0, - "y": 12 + "y": 13 }, "id": 52, "panels": [ @@ -2726,8 +3342,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2827,8 +3442,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2927,8 +3541,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3030,8 +3643,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3148,8 +3760,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3212,7 +3823,7 @@ "h": 1, "w": 24, "x": 0, - "y": 13 + "y": 14 }, "id": 44, "panels": [ @@ -3265,8 +3876,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3366,8 +3976,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3466,8 +4075,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3569,8 +4177,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3688,8 +4295,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3791,8 +4397,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3893,8 +4498,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3957,7 +4561,7 @@ "h": 1, "w": 24, "x": 0, - "y": 14 + "y": 15 }, "id": 23, "panels": [ @@ -4007,8 +4611,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4107,8 +4710,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4207,8 +4809,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4271,7 +4872,7 @@ "h": 1, "w": 24, "x": 0, - "y": 15 + "y": 16 }, "id": 6, "panels": [ @@ -4821,7 +5422,7 @@ "list": [ { "current": { - "selected": true, + "selected": false, "text": "s01.frostfs.devenv:9090", "value": "s01.frostfs.devenv:9090" }, From b6f47cb2c260b7c12377f1cecddee42cb105e05a Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Sun, 6 Aug 2023 13:44:37 +0300 Subject: [PATCH 087/149] [#38] grafana: Add GC dashboard Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 180 +++++++++++++++--- 1 file changed, 152 insertions(+), 28 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index 0d08ded..0d9b437 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -283,8 +283,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -384,8 +383,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -509,8 +507,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -643,7 +640,7 @@ "h": 10, "w": 24, "x": 0, - "y": 20 + "y": 9 }, "id": 28, "options": { @@ -744,7 +741,7 @@ "h": 10, "w": 24, "x": 0, - "y": 30 + "y": 19 }, "id": 29, "options": { @@ -827,8 +824,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -844,7 +840,7 @@ "h": 10, "w": 24, "x": 0, - "y": 40 + "y": 29 }, "id": 30, "options": { @@ -946,7 +942,7 @@ "h": 10, "w": 24, "x": 0, - "y": 50 + "y": 39 }, "id": 31, "options": { @@ -1063,7 +1059,7 @@ "h": 10, "w": 24, "x": 0, - "y": 10 + "y": 11 }, "id": 59, "options": { @@ -1163,7 +1159,7 @@ "h": 10, "w": 24, "x": 0, - "y": 20 + "y": 21 }, "id": 60, "options": { @@ -1266,7 +1262,7 @@ "h": 10, "w": 24, "x": 0, - "y": 30 + "y": 31 }, "id": 61, "options": { @@ -1369,7 +1365,7 @@ "h": 10, "w": 24, "x": 0, - "y": 40 + "y": 41 }, "id": 62, "options": { @@ -1472,7 +1468,7 @@ "h": 10, "w": 24, "x": 0, - "y": 50 + "y": 51 }, "id": 63, "options": { @@ -1574,7 +1570,7 @@ "h": 10, "w": 24, "x": 0, - "y": 60 + "y": 61 }, "id": 64, "options": { @@ -2803,8 +2799,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2904,8 +2899,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3004,8 +2998,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3107,8 +3100,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3225,8 +3217,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5412,6 +5403,139 @@ ], "title": "Tree service", "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 66, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Count of objects GC marked to remove, deleted or failed to delete from disk", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 65, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_garbage_collector_deleted_objects_total{instance=\"$instance\"}\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, {{status}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_garbage_collector_marked_for_removal_objects_total{instance=\"$instance\"}\n", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, marked for remove, {{object_type}}", + "range": true, + "refId": "B" + } + ], + "title": "Objects count", + "transformations": [], + "type": "timeseries" + } + ], + "title": "GC", + "type": "row" } ], "refresh": "1m", From 9bb0385b85e20a8149c599c1fbdefcdf7143e737 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Sun, 6 Aug 2023 15:15:54 +0300 Subject: [PATCH 088/149] [#38] grafana: Add morph dashboards Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 999 ++++++++++++++++-- 1 file changed, 904 insertions(+), 95 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index 0d9b437..f7d827c 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -182,7 +182,7 @@ } ] }, - "unit": "short" + "unit": "none" }, "overrides": [] }, @@ -283,7 +283,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -299,7 +300,7 @@ "h": 10, "w": 24, "x": 0, - "y": 8 + "y": 2 }, "id": 1, "options": { @@ -383,7 +384,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -424,7 +426,7 @@ "h": 10, "w": 24, "x": 0, - "y": 18 + "y": 12 }, "id": 3, "options": { @@ -507,7 +509,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -523,7 +526,7 @@ "h": 10, "w": 24, "x": 0, - "y": 28 + "y": 22 }, "id": 4, "options": { @@ -640,7 +643,7 @@ "h": 10, "w": 24, "x": 0, - "y": 9 + "y": 3 }, "id": 28, "options": { @@ -741,7 +744,7 @@ "h": 10, "w": 24, "x": 0, - "y": 19 + "y": 13 }, "id": 29, "options": { @@ -824,7 +827,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -840,7 +844,7 @@ "h": 10, "w": 24, "x": 0, - "y": 29 + "y": 23 }, "id": 30, "options": { @@ -926,7 +930,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -942,7 +947,7 @@ "h": 10, "w": 24, "x": 0, - "y": 39 + "y": 33 }, "id": 31, "options": { @@ -1059,7 +1064,7 @@ "h": 10, "w": 24, "x": 0, - "y": 11 + "y": 4 }, "id": 59, "options": { @@ -1159,7 +1164,7 @@ "h": 10, "w": 24, "x": 0, - "y": 21 + "y": 14 }, "id": 60, "options": { @@ -1262,7 +1267,7 @@ "h": 10, "w": 24, "x": 0, - "y": 31 + "y": 24 }, "id": 61, "options": { @@ -1365,7 +1370,7 @@ "h": 10, "w": 24, "x": 0, - "y": 41 + "y": 34 }, "id": 62, "options": { @@ -1468,7 +1473,7 @@ "h": 10, "w": 24, "x": 0, - "y": 51 + "y": 44 }, "id": 63, "options": { @@ -1570,7 +1575,7 @@ "h": 10, "w": 24, "x": 0, - "y": 61 + "y": 54 }, "id": 64, "options": { @@ -1670,7 +1675,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1686,7 +1692,7 @@ "h": 10, "w": 24, "x": 0, - "y": 51 + "y": 5 }, "id": 35, "options": { @@ -1770,7 +1776,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1786,7 +1793,7 @@ "h": 10, "w": 24, "x": 0, - "y": 61 + "y": 15 }, "id": 37, "options": { @@ -1869,7 +1876,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1885,7 +1893,7 @@ "h": 10, "w": 24, "x": 0, - "y": 71 + "y": 25 }, "id": 36, "options": { @@ -1970,7 +1978,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1986,7 +1995,7 @@ "h": 10, "w": 24, "x": 0, - "y": 81 + "y": 35 }, "id": 34, "options": { @@ -2086,7 +2095,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2102,7 +2112,7 @@ "h": 10, "w": 24, "x": 0, - "y": 11 + "y": 6 }, "id": 13, "options": { @@ -2186,7 +2196,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2202,7 +2213,7 @@ "h": 10, "w": 24, "x": 0, - "y": 21 + "y": 16 }, "id": 15, "options": { @@ -2285,7 +2296,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2301,7 +2313,7 @@ "h": 10, "w": 24, "x": 0, - "y": 31 + "y": 26 }, "id": 14, "options": { @@ -2384,7 +2396,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2400,7 +2413,7 @@ "h": 10, "w": 24, "x": 0, - "y": 41 + "y": 36 }, "id": 16, "options": { @@ -2483,7 +2496,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2499,7 +2513,7 @@ "h": 10, "w": 24, "x": 0, - "y": 51 + "y": 46 }, "id": 17, "options": { @@ -2582,7 +2596,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2590,7 +2605,7 @@ } ] }, - "unit": "none" + "unit": "ops" }, "overrides": [] }, @@ -2598,7 +2613,7 @@ "h": 10, "w": 24, "x": 0, - "y": 61 + "y": 56 }, "id": 18, "options": { @@ -2683,7 +2698,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2699,7 +2715,7 @@ "h": 10, "w": 24, "x": 0, - "y": 71 + "y": 66 }, "id": 19, "options": { @@ -2799,7 +2815,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2815,7 +2832,7 @@ "h": 10, "w": 24, "x": 0, - "y": 13 + "y": 7 }, "id": 39, "options": { @@ -2899,7 +2916,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2915,7 +2933,7 @@ "h": 10, "w": 24, "x": 0, - "y": 23 + "y": 17 }, "id": 40, "options": { @@ -2998,7 +3016,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3014,7 +3033,7 @@ "h": 10, "w": 24, "x": 0, - "y": 33 + "y": 27 }, "id": 41, "options": { @@ -3100,7 +3119,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3116,7 +3136,7 @@ "h": 10, "w": 24, "x": 0, - "y": 43 + "y": 37 }, "id": 43, "options": { @@ -3217,7 +3237,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3233,7 +3254,7 @@ "h": 10, "w": 24, "x": 0, - "y": 53 + "y": 47 }, "id": 42, "options": { @@ -3333,7 +3354,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3349,7 +3371,7 @@ "h": 10, "w": 24, "x": 0, - "y": 63 + "y": 8 }, "id": 53, "options": { @@ -3433,7 +3455,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3449,7 +3472,7 @@ "h": 10, "w": 24, "x": 0, - "y": 73 + "y": 18 }, "id": 54, "options": { @@ -3532,7 +3555,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3548,7 +3572,7 @@ "h": 10, "w": 24, "x": 0, - "y": 83 + "y": 28 }, "id": 55, "options": { @@ -3634,7 +3658,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3650,7 +3675,7 @@ "h": 10, "w": 24, "x": 0, - "y": 93 + "y": 38 }, "id": 56, "options": { @@ -3751,7 +3776,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3767,7 +3793,7 @@ "h": 10, "w": 24, "x": 0, - "y": 103 + "y": 48 }, "id": 57, "options": { @@ -3867,7 +3893,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3883,7 +3910,7 @@ "h": 10, "w": 24, "x": 0, - "y": 64 + "y": 9 }, "id": 45, "options": { @@ -3967,7 +3994,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3983,7 +4011,7 @@ "h": 10, "w": 24, "x": 0, - "y": 74 + "y": 19 }, "id": 46, "options": { @@ -4066,7 +4094,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4082,7 +4111,7 @@ "h": 10, "w": 24, "x": 0, - "y": 84 + "y": 29 }, "id": 47, "options": { @@ -4168,7 +4197,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4184,7 +4214,7 @@ "h": 10, "w": 24, "x": 0, - "y": 94 + "y": 39 }, "id": 48, "options": { @@ -4286,7 +4316,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4302,7 +4333,7 @@ "h": 10, "w": 24, "x": 0, - "y": 104 + "y": 49 }, "id": 50, "options": { @@ -4388,7 +4419,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4404,7 +4436,7 @@ "h": 10, "w": 24, "x": 0, - "y": 114 + "y": 59 }, "id": 51, "options": { @@ -4489,7 +4521,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4505,7 +4538,7 @@ "h": 10, "w": 24, "x": 0, - "y": 124 + "y": 69 }, "id": 49, "options": { @@ -4602,7 +4635,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4618,7 +4652,7 @@ "h": 10, "w": 24, "x": 0, - "y": 12 + "y": 10 }, "id": 24, "options": { @@ -4701,7 +4735,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4709,7 +4744,7 @@ } ] }, - "unit": "none" + "unit": "cps" }, "overrides": [] }, @@ -4717,7 +4752,7 @@ "h": 10, "w": 24, "x": 0, - "y": 22 + "y": 20 }, "id": 25, "options": { @@ -4800,7 +4835,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4816,7 +4852,7 @@ "h": 10, "w": 24, "x": 0, - "y": 32 + "y": 30 }, "id": 26, "options": { @@ -4916,7 +4952,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4932,7 +4969,7 @@ "h": 10, "w": 24, "x": 0, - "y": 40 + "y": 11 }, "id": 8, "options": { @@ -5016,7 +5053,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -5032,7 +5070,7 @@ "h": 10, "w": 24, "x": 0, - "y": 50 + "y": 21 }, "id": 9, "options": { @@ -5115,7 +5153,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -5131,7 +5170,7 @@ "h": 10, "w": 24, "x": 0, - "y": 60 + "y": 31 }, "id": 10, "options": { @@ -5214,7 +5253,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -5230,7 +5270,7 @@ "h": 10, "w": 24, "x": 0, - "y": 70 + "y": 41 }, "id": 7, "options": { @@ -5347,7 +5387,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -5363,7 +5404,7 @@ "h": 10, "w": 24, "x": 0, - "y": 80 + "y": 51 }, "id": 11, "options": { @@ -5480,7 +5521,7 @@ "h": 10, "w": 24, "x": 0, - "y": 18 + "y": 12 }, "id": 65, "options": { @@ -5536,6 +5577,774 @@ ], "title": "GC", "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 67, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Number of endpoint switches", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 68, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "frostfs_node_morph_switches_total{job=\"node\", instance=\"$instance\"}", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Endpoint switches", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Index of the last received block", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 69, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "frostfs_node_morph_last_block{job=\"node\", instance=\"$instance\"}", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Last block", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Rate of notifications received by notification type", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 70, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "rate(frostfs_node_morph_notifications_total{job=\"node\", instance=\"$instance\"}[$__rate_interval])", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{notification_type}}", + "range": true, + "refId": "A" + } + ], + "title": "Notifications rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Rate of of contract invocations", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 71, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, contract, invoke_type, method) (\n rate(frostfs_node_morph_invoke_duration_seconds_count{job=\"node\", instance=\"$instance\"}[$__rate_interval])\n)", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Contract {{contract}}, type {{invoke_type}}, method {{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Contract method invoke rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed contract invocations", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "neo.fs.v2.object.ObjectService.Put" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 72, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, contract, invoke_type, method) (\n rate(frostfs_node_morph_invoke_duration_seconds_count{job=\"node\", instance=\"$instance\", success!=\"true\"}[$__rate_interval])\n)\n / \nsum by (instance, contract, invoke_type, method) (\n rate(frostfs_node_morph_invoke_duration_seconds_count{job=\"node\", instance=\"$instance\"}[$__rate_interval])\n)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Contract {{contract}}, type {{invoke_type}}, method {{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Contract invocations error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the contract invocations for selected quantile", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 73, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_morph_invoke_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (contract, invoke_type, method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Contract {{contract}}, type {{invoke_type}}, method {{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Contract invocations latency", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Rate of of morph cache methods invocations", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 47 + }, + "id": 75, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, method) (\n rate(frostfs_node_morphcache_request_duration_seconds_count{job=\"node\", instance=\"$instance\"}[$__rate_interval])\n)", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache method rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the morph cache for selected quantile", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 57 + }, + "id": 74, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_morphcache_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache invocations latency", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Morph", + "type": "row" } ], "refresh": "1m", @@ -5546,7 +6355,7 @@ "list": [ { "current": { - "selected": false, + "selected": true, "text": "s01.frostfs.devenv:9090", "value": "s01.frostfs.devenv:9090" }, From 04260ad0d8eba6dd33cf181371be3fa83996ed9c Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 7 Aug 2023 12:17:06 +0300 Subject: [PATCH 089/149] [#38] grafana: Fix writecache boards Stack count and size dashborads. Signed-off-by: Dmitrii Stepanov --- .gitattributes | 1 + .../provisioning/dashboards/storage-node.json | 161 +++++++----------- 2 files changed, 66 insertions(+), 96 deletions(-) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..5436e42 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +/services/grafana/provisioning/dashboards/* -diff -merge \ No newline at end of file diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index f7d827c..e0cd8a8 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -300,7 +300,7 @@ "h": 10, "w": 24, "x": 0, - "y": 2 + "y": 8 }, "id": 1, "options": { @@ -426,7 +426,7 @@ "h": 10, "w": 24, "x": 0, - "y": 12 + "y": 18 }, "id": 3, "options": { @@ -526,7 +526,7 @@ "h": 10, "w": 24, "x": 0, - "y": 22 + "y": 28 }, "id": 4, "options": { @@ -626,8 +626,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -727,8 +726,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -827,8 +825,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -930,8 +927,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1047,8 +1043,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1147,8 +1142,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1250,8 +1244,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1353,8 +1346,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1456,8 +1448,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1558,8 +1549,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1675,8 +1665,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1776,8 +1765,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1876,8 +1864,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1978,8 +1965,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2112,7 +2098,7 @@ "h": 10, "w": 24, "x": 0, - "y": 6 + "y": 12 }, "id": 13, "options": { @@ -2213,7 +2199,7 @@ "h": 10, "w": 24, "x": 0, - "y": 16 + "y": 22 }, "id": 15, "options": { @@ -2313,7 +2299,7 @@ "h": 10, "w": 24, "x": 0, - "y": 26 + "y": 32 }, "id": 14, "options": { @@ -2368,7 +2354,7 @@ "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 100, "gradientMode": "none", "hideFrom": { "legend": false, @@ -2385,7 +2371,7 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "none" + "mode": "normal" }, "thresholdsStyle": { "mode": "off" @@ -2413,7 +2399,7 @@ "h": 10, "w": 24, "x": 0, - "y": 36 + "y": 42 }, "id": 16, "options": { @@ -2468,7 +2454,7 @@ "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 100, "gradientMode": "none", "hideFrom": { "legend": false, @@ -2485,7 +2471,7 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "none" + "mode": "normal" }, "thresholdsStyle": { "mode": "off" @@ -2513,7 +2499,7 @@ "h": 10, "w": 24, "x": 0, - "y": 46 + "y": 52 }, "id": 17, "options": { @@ -2568,7 +2554,7 @@ "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 100, "gradientMode": "none", "hideFrom": { "legend": false, @@ -2585,7 +2571,7 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "none" + "mode": "normal" }, "thresholdsStyle": { "mode": "off" @@ -2613,7 +2599,7 @@ "h": 10, "w": 24, "x": 0, - "y": 56 + "y": 62 }, "id": 18, "options": { @@ -2715,7 +2701,7 @@ "h": 10, "w": 24, "x": 0, - "y": 66 + "y": 72 }, "id": 19, "options": { @@ -2832,7 +2818,7 @@ "h": 10, "w": 24, "x": 0, - "y": 7 + "y": 83 }, "id": 39, "options": { @@ -2933,7 +2919,7 @@ "h": 10, "w": 24, "x": 0, - "y": 17 + "y": 93 }, "id": 40, "options": { @@ -3033,7 +3019,7 @@ "h": 10, "w": 24, "x": 0, - "y": 27 + "y": 103 }, "id": 41, "options": { @@ -3136,7 +3122,7 @@ "h": 10, "w": 24, "x": 0, - "y": 37 + "y": 113 }, "id": 43, "options": { @@ -3254,7 +3240,7 @@ "h": 10, "w": 24, "x": 0, - "y": 47 + "y": 123 }, "id": 42, "options": { @@ -3371,7 +3357,7 @@ "h": 10, "w": 24, "x": 0, - "y": 8 + "y": 84 }, "id": 53, "options": { @@ -3472,7 +3458,7 @@ "h": 10, "w": 24, "x": 0, - "y": 18 + "y": 94 }, "id": 54, "options": { @@ -3572,7 +3558,7 @@ "h": 10, "w": 24, "x": 0, - "y": 28 + "y": 104 }, "id": 55, "options": { @@ -3675,7 +3661,7 @@ "h": 10, "w": 24, "x": 0, - "y": 38 + "y": 114 }, "id": 56, "options": { @@ -3793,7 +3779,7 @@ "h": 10, "w": 24, "x": 0, - "y": 48 + "y": 124 }, "id": 57, "options": { @@ -3910,7 +3896,7 @@ "h": 10, "w": 24, "x": 0, - "y": 9 + "y": 85 }, "id": 45, "options": { @@ -4011,7 +3997,7 @@ "h": 10, "w": 24, "x": 0, - "y": 19 + "y": 95 }, "id": 46, "options": { @@ -4111,7 +4097,7 @@ "h": 10, "w": 24, "x": 0, - "y": 29 + "y": 105 }, "id": 47, "options": { @@ -4214,7 +4200,7 @@ "h": 10, "w": 24, "x": 0, - "y": 39 + "y": 115 }, "id": 48, "options": { @@ -4333,7 +4319,7 @@ "h": 10, "w": 24, "x": 0, - "y": 49 + "y": 125 }, "id": 50, "options": { @@ -4436,7 +4422,7 @@ "h": 10, "w": 24, "x": 0, - "y": 59 + "y": 135 }, "id": 51, "options": { @@ -4538,7 +4524,7 @@ "h": 10, "w": 24, "x": 0, - "y": 69 + "y": 145 }, "id": 49, "options": { @@ -4635,8 +4621,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4735,8 +4720,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4835,8 +4819,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4952,8 +4935,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5053,8 +5035,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5153,8 +5134,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5253,8 +5233,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5387,8 +5366,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5504,8 +5482,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5601,8 +5578,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5665,8 +5641,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5762,8 +5737,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5862,8 +5836,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5963,8 +5936,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6088,8 +6060,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6188,8 +6159,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6288,8 +6258,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6355,7 +6324,7 @@ "list": [ { "current": { - "selected": true, + "selected": false, "text": "s01.frostfs.devenv:9090", "value": "s01.frostfs.devenv:9090" }, From 51053e331720b36b3b52adb8420b3883f530ce6f Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 11 Aug 2023 15:57:45 +0300 Subject: [PATCH 090/149] [#46] .forgejo: Update DCO action Signed-off-by: Evgenii Stratonikov --- .forgejo/workflows/dco.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.forgejo/workflows/dco.yml b/.forgejo/workflows/dco.yml index 880addd..6746408 100644 --- a/.forgejo/workflows/dco.yml +++ b/.forgejo/workflows/dco.yml @@ -1,3 +1,4 @@ +name: DCO action on: [pull_request] jobs: @@ -12,9 +13,9 @@ jobs: - name: Setup Go uses: actions/setup-go@v3 with: - go-version: '1.20' + go-version: '1.21' - name: Run commit format checker - uses: https://git.alexvan.in/alexvanin/dco-go@v1 + uses: https://git.frostfs.info/TrueCloudLab/dco-go@v2 with: - from: dca6ff62 + from: 'origin/${{ github.event.pull_request.base.ref }}' From 731976cc57de38e699032428695d0a276b6ecc13 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 11 Aug 2023 15:48:57 +0300 Subject: [PATCH 091/149] [#44] grafana: Add Client dashboard Client dashboard shows outgoing requests. Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 410 +++++++++++++++--- 1 file changed, 352 insertions(+), 58 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index e0cd8a8..f78962c 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -575,6 +575,324 @@ "x": 0, "y": 8 }, + "id": 76, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by gRPC method", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 77, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, grpc_service, grpc_method) (\n rate(grpc_client_started_total{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed requests", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 78, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(grpc_client_handled_total{instance=\"$instance\",grpc_code!=\"OK\"}[$__rate_interval])) by (grpc_service, grpc_method)\n / \nsum(rate(grpc_client_started_total{instance=\"$instance\"}[$__rate_interval])) by (grpc_service, grpc_method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the requests for selected quantile", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 79, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(grpc_client_handling_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Client", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, "id": 27, "panels": [ { @@ -990,7 +1308,7 @@ "h": 1, "w": 24, "x": 0, - "y": 9 + "y": 10 }, "id": 58, "panels": [ @@ -1612,7 +1930,7 @@ "h": 1, "w": 24, "x": 0, - "y": 10 + "y": 11 }, "id": 33, "panels": [ @@ -2028,7 +2346,7 @@ "h": 1, "w": 24, "x": 0, - "y": 11 + "y": 12 }, "id": 12, "panels": [ @@ -2081,8 +2399,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2182,8 +2499,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2282,8 +2598,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2382,8 +2697,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2482,8 +2796,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2582,8 +2895,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2684,8 +2996,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2748,7 +3059,7 @@ "h": 1, "w": 24, "x": 0, - "y": 12 + "y": 13 }, "id": 38, "panels": [ @@ -2801,8 +3112,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2902,8 +3212,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3002,8 +3311,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3105,8 +3413,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3223,8 +3530,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3287,7 +3593,7 @@ "h": 1, "w": 24, "x": 0, - "y": 13 + "y": 14 }, "id": 52, "panels": [ @@ -3340,8 +3646,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3441,8 +3746,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3541,8 +3845,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3644,8 +3947,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3762,8 +4064,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3826,7 +4127,7 @@ "h": 1, "w": 24, "x": 0, - "y": 14 + "y": 15 }, "id": 44, "panels": [ @@ -3879,8 +4180,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3980,8 +4280,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4080,8 +4379,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4183,8 +4481,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4302,8 +4599,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4405,8 +4701,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4507,8 +4802,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4571,7 +4865,7 @@ "h": 1, "w": 24, "x": 0, - "y": 15 + "y": 16 }, "id": 23, "panels": [ @@ -4882,7 +5176,7 @@ "h": 1, "w": 24, "x": 0, - "y": 16 + "y": 17 }, "id": 6, "panels": [ @@ -5429,7 +5723,7 @@ "h": 1, "w": 24, "x": 0, - "y": 17 + "y": 18 }, "id": 66, "panels": [ @@ -5561,7 +5855,7 @@ "h": 1, "w": 24, "x": 0, - "y": 18 + "y": 19 }, "id": 67, "panels": [ From 6eedab3d83db60fc68d4a7288ba463c7683b46b7 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 18 Aug 2023 13:40:03 +0300 Subject: [PATCH 092/149] [#48] grafana: Fix blobovnicza board Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/storage-node.json | 199 +++++++++++++----- 1 file changed, 152 insertions(+), 47 deletions(-) diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json index f78962c..318f8c7 100644 --- a/services/grafana/provisioning/dashboards/storage-node.json +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -283,8 +283,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -384,8 +383,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -509,8 +507,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -626,8 +623,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -727,8 +723,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -827,8 +822,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4136,7 +4130,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Requests per second by blobovnizca method for each shard", + "description": "Requests per second by blobovnicza method for each shard", "fieldConfig": { "defaults": { "color": { @@ -4180,7 +4174,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4196,7 +4191,7 @@ "h": 10, "w": 24, "x": 0, - "y": 85 + "y": 16 }, "id": 45, "options": { @@ -4220,7 +4215,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum by (instance, shard_id, method) (\n rate(frostfs_node_blobovniza_tree_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", + "expr": "sum by (instance, shard_id, method) (\n rate(frostfs_node_blobovnicza_tree_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", "format": "time_series", "instant": false, "interval": "", @@ -4238,7 +4233,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Percentage of the failed blobovnizca requests for each shard", + "description": "Percentage of the failed blobovnicza requests for each shard", "fieldConfig": { "defaults": { "color": { @@ -4280,7 +4275,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4296,7 +4292,7 @@ "h": 10, "w": 24, "x": 0, - "y": 95 + "y": 26 }, "id": 46, "options": { @@ -4320,7 +4316,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum(rate(frostfs_node_blobsovniza_tree_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (shard_id, method)\n / \nsum(rate(frostfs_node_blobsovniza_tree_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (shard_id, method)\n * 100.0\n", + "expr": "sum(rate(frostfs_node_blobsovnicza_tree_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (shard_id, method)\n / \nsum(rate(frostfs_node_blobsovnicza_tree_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (shard_id, method)\n * 100.0\n", "format": "time_series", "instant": false, "interval": "", @@ -4338,7 +4334,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Latency of the blobovnizca requests for selected quantile for each shard", + "description": "Latency of the blobovnicza requests for selected quantile for each shard", "fieldConfig": { "defaults": { "color": { @@ -4379,7 +4375,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4395,7 +4392,7 @@ "h": 10, "w": 24, "x": 0, - "y": 105 + "y": 36 }, "id": 47, "options": { @@ -4419,7 +4416,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_blobovniza_tree_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method,le)\n)\n", + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method,le)\n)\n", "format": "time_series", "instant": false, "interval": "", @@ -4437,7 +4434,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Bytes per second by blobovnizca shard", + "description": "Bytes per second by blobovnicza shard", "fieldConfig": { "defaults": { "color": { @@ -4481,7 +4478,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4497,7 +4495,7 @@ "h": 10, "w": 24, "x": 0, - "y": 115 + "y": 46 }, "id": 48, "options": { @@ -4521,7 +4519,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum by (instance, shard_id) (\n rate(frostfs_node_blobovniza_tree_put_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n", + "expr": "sum by (instance, shard_id) (\n rate(frostfs_node_blobovnicza_tree_put_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n", "format": "time_series", "instant": false, "interval": "", @@ -4536,7 +4534,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum by (instance, shard_id) (\n rate(frostfs_node_blobovniza_tree_get_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n", + "expr": "sum by (instance, shard_id) (\n rate(frostfs_node_blobovnicza_tree_get_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n", "format": "time_series", "hide": false, "instant": false, @@ -4555,7 +4553,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Blobovnizca size for each shard", + "description": "Open blobovnicza size for each shard", "fieldConfig": { "defaults": { "color": { @@ -4599,7 +4597,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4615,7 +4614,7 @@ "h": 10, "w": 24, "x": 0, - "y": 125 + "y": 56 }, "id": 50, "options": { @@ -4639,7 +4638,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "frostfs_node_blobovniza_tree_size_bytes{instance=\"$instance\"}\n", + "expr": "frostfs_node_blobovnicza_tree_open_blobovnicza_size_bytes{instance=\"$instance\"}\n", "format": "time_series", "instant": false, "interval": "", @@ -4648,7 +4647,7 @@ "refId": "A" } ], - "title": "Size", + "title": "Open blobovnicza size", "transformations": [], "type": "timeseries" }, @@ -4657,7 +4656,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Count of open DB's of blobovnizca for each shard", + "description": "Items count of open blobovnicza for each shard", "fieldConfig": { "defaults": { "color": { @@ -4701,7 +4700,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4717,7 +4717,110 @@ "h": 10, "w": 24, "x": 0, - "y": 135 + "y": 66 + }, + "id": 80, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_blobovnicza_tree_open_blobovnicza_items_total{instance=\"$instance\"}\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Open blobovnicza items", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Count of open blobovniczas for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 76 }, "id": 51, "options": { @@ -4741,7 +4844,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "frostfs_node_blobovniza_tree_open_blobovnizca_count{instance=\"$instance\"}\n", + "expr": "frostfs_node_blobovnicza_tree_open_blobovnicza_count{instance=\"$instance\"}\n", "format": "time_series", "instant": false, "interval": "", @@ -4750,7 +4853,7 @@ "refId": "A" } ], - "title": "Open DB count", + "title": "Open blobovnicza count", "transformations": [], "type": "timeseries" }, @@ -4759,7 +4862,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Blobovnizca mode for each shard", + "description": "Blobovnicza mode for each shard", "fieldConfig": { "defaults": { "color": { @@ -4802,7 +4905,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4818,7 +4922,7 @@ "h": 10, "w": 24, "x": 0, - "y": 145 + "y": 86 }, "id": 49, "options": { @@ -4842,7 +4946,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum(frostfs_node_blobovniza_tree_mode{job=\"node\", instance=\"$instance\"}) by (shard_id, mode)\n", + "expr": "sum(frostfs_node_blobovnicza_tree_mode{job=\"node\", instance=\"$instance\"}) by (shard_id, mode)\n", "format": "time_series", "instant": false, "interval": "", @@ -4856,7 +4960,7 @@ "type": "timeseries" } ], - "title": "Blobovnizca", + "title": "Blobovnicza", "type": "row" }, { @@ -5776,7 +5880,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -5792,7 +5897,7 @@ "h": 10, "w": 24, "x": 0, - "y": 12 + "y": 19 }, "id": 65, "options": { @@ -6618,7 +6723,7 @@ "list": [ { "current": { - "selected": false, + "selected": true, "text": "s01.frostfs.devenv:9090", "value": "s01.frostfs.devenv:9090" }, From b7ac6f30cf476556afa8f0bbb6aafecaeeb3d54f Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 21 Aug 2023 10:19:37 +0300 Subject: [PATCH 093/149] [#50] services/ir: Take all contract hashes from NNS Signed-off-by: Evgenii Stratonikov --- services/ir/.ir.env | 2 -- 1 file changed, 2 deletions(-) diff --git a/services/ir/.ir.env b/services/ir/.ir.env index 08d7e17..810cf6d 100644 --- a/services/ir/.ir.env +++ b/services/ir/.ir.env @@ -1,3 +1 @@ -FROSTFS_IR_CONTRACTS_FROSTFSID=27407c76feabc407908f3d09a3d845d45e7c981a - FROSTFS_IR_CONTROL_GRPC_ENDPOINT=127.0.0.1:16512 From eba763ff79829617896ed3e3a2471e82d7538fdd Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 25 Aug 2023 09:54:40 +0300 Subject: [PATCH 094/149] [#49] grafana: Server bucket distribution Signed-off-by: Dmitrii Stepanov --- .../provisioning/dashboards/grpc-buckets.json | 362 ++++++++++++++++++ 1 file changed, 362 insertions(+) create mode 100644 services/grafana/provisioning/dashboards/grpc-buckets.json diff --git a/services/grafana/provisioning/dashboards/grpc-buckets.json b/services/grafana/provisioning/dashboards/grpc-buckets.json new file mode 100644 index 0000000..ba7cd5f --- /dev/null +++ b/services/grafana/provisioning/dashboards/grpc-buckets.json @@ -0,0 +1,362 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 4, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "$service", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 20, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(grpc_server_handling_seconds_bucket{le=\"0.005\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])", + "hide": false, + "legendFormat": "< 0.005s", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"0.01\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"0.005\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.005s - 0.01s", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"0.025\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"0.01\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.01s - 0.025s", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"0.05\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"0.025\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.025s - 0.05s", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"0.1\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"0.05\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.05s - 0.1s", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"0.25\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"0.1\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.1s - 0.25s", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"0.5\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"0.25\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.25s - 0.5s", + "range": true, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"1\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"0.5\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.5s - 1s", + "range": true, + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"2.5\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"1\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "1s - 2.5s", + "range": true, + "refId": "I" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"5\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"2.5\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "2.5s - 5s", + "range": true, + "refId": "J" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"10\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"5\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "5s - 10s", + "range": true, + "refId": "K" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"+Inf\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"10\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "> 10s", + "range": true, + "refId": "L" + } + ], + "title": "$method", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "s01.frostfs.devenv:9090", + "value": "s01.frostfs.devenv:9090" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(grpc_server_handling_seconds_bucket,instance)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "instance", + "options": [], + "query": { + "query": "label_values(grpc_server_handling_seconds_bucket,instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": "", + "current": { + "selected": false, + "text": "neo.fs.v2.object.ObjectService", + "value": "neo.fs.v2.object.ObjectService" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(grpc_server_handling_seconds_bucket{instance=\"$instance\"},grpc_service)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "service", + "options": [], + "query": { + "query": "label_values(grpc_server_handling_seconds_bucket{instance=\"$instance\"},grpc_service)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "Put", + "value": "Put" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(grpc_server_handling_seconds_bucket{grpc_service=\"$service\", instance=\"$instance\"},grpc_method)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "method", + "options": [], + "query": { + "query": "label_values(grpc_server_handling_seconds_bucket{grpc_service=\"$service\", instance=\"$instance\"},grpc_method)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "GRPC server bucket distribution", + "uid": "a6830bc6-cffa-4e29-9622-6a10921748b0", + "version": 8, + "weekStart": "" + } \ No newline at end of file From 42e634927653e641430bac1dd07ea3571003aceb Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 25 Aug 2023 10:18:39 +0300 Subject: [PATCH 095/149] [#49] grafana: Storage engine bucket distribution Signed-off-by: Dmitrii Stepanov --- .../dashboards/engine-buckets.json | 335 ++++++++++++++++++ 1 file changed, 335 insertions(+) create mode 100644 services/grafana/provisioning/dashboards/engine-buckets.json diff --git a/services/grafana/provisioning/dashboards/engine-buckets.json b/services/grafana/provisioning/dashboards/engine-buckets.json new file mode 100644 index 0000000..8bc704b --- /dev/null +++ b/services/grafana/provisioning/dashboards/engine-buckets.json @@ -0,0 +1,335 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 5, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "$method", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 20, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])", + "hide": false, + "legendFormat": "< 0.005s", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.005s - 0.01s", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.01s - 0.025s", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.025s - 0.05s", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.05s - 0.1s", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.1s - 0.25s", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.25s - 0.5s", + "range": true, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.5s - 1s", + "range": true, + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "1s - 2.5s", + "range": true, + "refId": "I" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "2.5s - 5s", + "range": true, + "refId": "J" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "5s - 10s", + "range": true, + "refId": "K" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"+Inf\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "> 10s", + "range": true, + "refId": "L" + } + ], + "title": "$method", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "s01.frostfs.devenv:9090", + "value": "s01.frostfs.devenv:9090" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_engine_request_duration_seconds_bucket,instance)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "instance", + "options": [], + "query": { + "query": "label_values(frostfs_node_engine_request_duration_seconds_bucket,instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": "Put", + "value": "Put" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_engine_request_duration_seconds_bucket{instance=\"$instance\"},method)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "method", + "options": [], + "query": { + "query": "label_values(frostfs_node_engine_request_duration_seconds_bucket{instance=\"$instance\"},method)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Storage node engine bucket distribution", + "uid": "f886d27c-b877-495a-a1cb-d02839115cea", + "version": 2, + "weekStart": "" + } \ No newline at end of file From 4a6b481618f3eaa991a0f8db1101791136ce57ec Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 25 Aug 2023 10:43:51 +0300 Subject: [PATCH 096/149] [#49] grafana: Writecache bucket distribution Signed-off-by: Dmitrii Stepanov --- .../dashboards/writecache-buckets.json | 361 ++++++++++++++++++ 1 file changed, 361 insertions(+) create mode 100644 services/grafana/provisioning/dashboards/writecache-buckets.json diff --git a/services/grafana/provisioning/dashboards/writecache-buckets.json b/services/grafana/provisioning/dashboards/writecache-buckets.json new file mode 100644 index 0000000..cd37893 --- /dev/null +++ b/services/grafana/provisioning/dashboards/writecache-buckets.json @@ -0,0 +1,361 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 6, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "$method", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 20, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "< 0.005s", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.005s - 0.01s", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.01s - 0.025s", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.025s - 0.05s", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.05s - 0.1s", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.1s - 0.25s", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.25s - 0.5s", + "range": true, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.5s - 1s", + "range": true, + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "1s - 2.5s", + "range": true, + "refId": "I" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "2.5s - 5s", + "range": true, + "refId": "J" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "5s - 10s", + "range": true, + "refId": "K" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"+Inf\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "> 10s", + "range": true, + "refId": "L" + } + ], + "title": "$method", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "s01.frostfs.devenv:9090", + "value": "s01.frostfs.devenv:9090" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_writecache_request_duration_seconds_bucket,instance)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "instance", + "options": [], + "query": { + "query": "label_values(frostfs_node_writecache_request_duration_seconds_bucket,instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": "BGH3NNzxkiW7JB1BKLHb5z", + "value": "BGH3NNzxkiW7JB1BKLHb5z" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_writecache_request_duration_seconds_bucket{instance=\"$instance\"},shard_id)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "shard_id", + "options": [], + "query": { + "query": "label_values(frostfs_node_writecache_request_duration_seconds_bucket{instance=\"$instance\"},shard_id)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": "Delete", + "value": "Delete" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_writecache_request_duration_seconds_bucket{instance=\"$instance\", shard_id=\"$shard_id\"},method)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "method", + "options": [], + "query": { + "query": "label_values(frostfs_node_writecache_request_duration_seconds_bucket{instance=\"$instance\", shard_id=\"$shard_id\"},method)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Storage node writecache bucket distribution", + "uid": "d2fef1d1-9ef3-437e-ad21-670a20e0e9b4", + "version": 3, + "weekStart": "" + } \ No newline at end of file From f0c3c02943fd24e14b0a63d20f8cce0ee78f7edc Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 25 Aug 2023 10:54:38 +0300 Subject: [PATCH 097/149] [#49] grafana: Blobovnicza bucket distribution Signed-off-by: Dmitrii Stepanov --- .../dashboards/blobovnicza-buckets.json | 361 ++++++++++++++++++ 1 file changed, 361 insertions(+) create mode 100644 services/grafana/provisioning/dashboards/blobovnicza-buckets.json diff --git a/services/grafana/provisioning/dashboards/blobovnicza-buckets.json b/services/grafana/provisioning/dashboards/blobovnicza-buckets.json new file mode 100644 index 0000000..e35272a --- /dev/null +++ b/services/grafana/provisioning/dashboards/blobovnicza-buckets.json @@ -0,0 +1,361 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 7, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "$method", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 20, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "< 0.005s", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.005s - 0.01s", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.01s - 0.025s", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.025s - 0.05s", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.05s - 0.1s", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.1s - 0.25s", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.25s - 0.5s", + "range": true, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.5s - 1s", + "range": true, + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "1s - 2.5s", + "range": true, + "refId": "I" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "2.5s - 5s", + "range": true, + "refId": "J" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "5s - 10s", + "range": true, + "refId": "K" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"+Inf\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "> 10s", + "range": true, + "refId": "L" + } + ], + "title": "$method", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "s01.frostfs.devenv:9090", + "value": "s01.frostfs.devenv:9090" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket,instance)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "instance", + "options": [], + "query": { + "query": "label_values(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket,instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "3Xd2VDqghZnMnaRoP4bKeP", + "value": "3Xd2VDqghZnMnaRoP4bKeP" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{instance=\"$instance\"},shard_id)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "shard_id", + "options": [], + "query": { + "query": "label_values(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{instance=\"$instance\"},shard_id)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": "Put", + "value": "Put" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{instance=\"$instance\", shard_id=\"$shard_id\"},method)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "method", + "options": [], + "query": { + "query": "label_values(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{instance=\"$instance\", shard_id=\"$shard_id\"},method)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Storage node blobovnicza bucket distribution", + "uid": "e4097619-1ce9-4039-bd62-4f5b4bba8d26", + "version": 2, + "weekStart": "" + } \ No newline at end of file From fdcf71d5b6fe4d18b3060b8aec3244c19303a174 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 25 Aug 2023 11:12:11 +0300 Subject: [PATCH 098/149] [#49] grafana: Add metabase bucket distribution Signed-off-by: Dmitrii Stepanov --- .../dashboards/metabase-buckets.json | 361 ++++++++++++++++++ 1 file changed, 361 insertions(+) create mode 100644 services/grafana/provisioning/dashboards/metabase-buckets.json diff --git a/services/grafana/provisioning/dashboards/metabase-buckets.json b/services/grafana/provisioning/dashboards/metabase-buckets.json new file mode 100644 index 0000000..2fba386 --- /dev/null +++ b/services/grafana/provisioning/dashboards/metabase-buckets.json @@ -0,0 +1,361 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 8, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "$method", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 20, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "< 0.005s", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.005s - 0.01s", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.01s - 0.025s", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.025s - 0.05s", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.05s - 0.1s", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.1s - 0.25s", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.25s - 0.5s", + "range": true, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.5s - 1s", + "range": true, + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "1s - 2.5s", + "range": true, + "refId": "I" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "2.5s - 5s", + "range": true, + "refId": "J" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "5s - 10s", + "range": true, + "refId": "K" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"+Inf\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "> 10s", + "range": true, + "refId": "L" + } + ], + "title": "$method", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "s01.frostfs.devenv:9090", + "value": "s01.frostfs.devenv:9090" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_metabase_request_duration_seconds_bucket,instance)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "instance", + "options": [], + "query": { + "query": "label_values(frostfs_node_metabase_request_duration_seconds_bucket,instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "3Xd2VDqghZnMnaRoP4bKeP", + "value": "3Xd2VDqghZnMnaRoP4bKeP" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_metabase_request_duration_seconds_bucket{instance=\"$instance\"},shard_id)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "shard_id", + "options": [], + "query": { + "query": "label_values(frostfs_node_metabase_request_duration_seconds_bucket{instance=\"$instance\"},shard_id)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": "Delete", + "value": "Delete" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_metabase_request_duration_seconds_bucket{instance=\"$instance\", shard_id=\"$shard_id\"},method)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "method", + "options": [], + "query": { + "query": "label_values(frostfs_node_metabase_request_duration_seconds_bucket{instance=\"$instance\", shard_id=\"$shard_id\"},method)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Storage node metabase bucket distribution", + "uid": "e110ed66-d086-461d-958a-efc6c55607d2", + "version": 1, + "weekStart": "" + } \ No newline at end of file From 90bd39d7171fb7d43c71a008e78762aaf61d8e1a Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 25 Aug 2023 11:23:07 +0300 Subject: [PATCH 099/149] [#43] grafana: FSTree bucket distribution Signed-off-by: Dmitrii Stepanov --- .../dashboards/fstree-buckets.json | 361 ++++++++++++++++++ 1 file changed, 361 insertions(+) create mode 100644 services/grafana/provisioning/dashboards/fstree-buckets.json diff --git a/services/grafana/provisioning/dashboards/fstree-buckets.json b/services/grafana/provisioning/dashboards/fstree-buckets.json new file mode 100644 index 0000000..151376b --- /dev/null +++ b/services/grafana/provisioning/dashboards/fstree-buckets.json @@ -0,0 +1,361 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 9, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "$method", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 20, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "< 0.005s", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.005s - 0.01s", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.01s - 0.025s", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.025s - 0.05s", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.05s - 0.1s", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.1s - 0.25s", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.25s - 0.5s", + "range": true, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.5s - 1s", + "range": true, + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "1s - 2.5s", + "range": true, + "refId": "I" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "2.5s - 5s", + "range": true, + "refId": "J" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "5s - 10s", + "range": true, + "refId": "K" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"+Inf\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "> 10s", + "range": true, + "refId": "L" + } + ], + "title": "$method", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "s01.frostfs.devenv:9090", + "value": "s01.frostfs.devenv:9090" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_fstree_request_duration_seconds_bucket,instance)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "instance", + "options": [], + "query": { + "query": "label_values(frostfs_node_fstree_request_duration_seconds_bucket,instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "3Xd2VDqghZnMnaRoP4bKeP", + "value": "3Xd2VDqghZnMnaRoP4bKeP" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_fstree_request_duration_seconds_bucket{instance=\"$instance\"},shard_id)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "shard_id", + "options": [], + "query": { + "query": "label_values(frostfs_node_fstree_request_duration_seconds_bucket{instance=\"$instance\"},shard_id)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "Get", + "value": "Get" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_fstree_request_duration_seconds_bucket{instance=\"$instance\", shard_id=\"$shard_id\"},method)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "method", + "options": [], + "query": { + "query": "label_values(frostfs_node_fstree_request_duration_seconds_bucket{instance=\"$instance\", shard_id=\"$shard_id\"},method)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Storage node fstree bucket distribution", + "uid": "e6407e43-9419-445b-b715-284d69201519", + "version": 2, + "weekStart": "" + } \ No newline at end of file From e254eba6a86e5e0bef92e30ae33a94bdf9be7811 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 25 Aug 2023 14:09:45 +0300 Subject: [PATCH 100/149] [#52] frostfs-adm.yml: Allow maintenance mode by default It is a DEV-env after all. Signed-off-by: Evgenii Stratonikov --- frostfs-adm.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/frostfs-adm.yml b/frostfs-adm.yml index c44046d..5077431 100644 --- a/frostfs-adm.yml +++ b/frostfs-adm.yml @@ -5,6 +5,7 @@ network: epoch_duration: 240 basic_income_rate: 100000000 homomorphic_hash_disabled: false + maintenance_mode_allowed: true fee: audit: 10000 candidate: 10000000000 From 4413251994bc8480c038d1acc16810eeb005de0f Mon Sep 17 00:00:00 2001 From: Marina Biryukova Date: Wed, 13 Sep 2023 16:20:46 +0300 Subject: [PATCH 101/149] [#20] Add frostfs-cli configurations Signed-off-by: Marina Biryukova --- configs/s01-cli.yml | 4 ++++ configs/s02-cli.yml | 4 ++++ configs/s03-cli.yml | 4 ++++ configs/s04-cli.yml | 4 ++++ 4 files changed, 16 insertions(+) create mode 100644 configs/s01-cli.yml create mode 100644 configs/s02-cli.yml create mode 100644 configs/s03-cli.yml create mode 100644 configs/s04-cli.yml diff --git a/configs/s01-cli.yml b/configs/s01-cli.yml new file mode 100644 index 0000000..3bd7c43 --- /dev/null +++ b/configs/s01-cli.yml @@ -0,0 +1,4 @@ +wallet: services/storage/wallet01.json +password: "" +rpc-endpoint: s01.frostfs.devenv:8080 +endpoint: s01.frostfs.devenv:8081 \ No newline at end of file diff --git a/configs/s02-cli.yml b/configs/s02-cli.yml new file mode 100644 index 0000000..ac5a8bb --- /dev/null +++ b/configs/s02-cli.yml @@ -0,0 +1,4 @@ +wallet: services/storage/wallet02.json +password: "" +rpc-endpoint: s02.frostfs.devenv:8080 +endpoint: s02.frostfs.devenv:8081 \ No newline at end of file diff --git a/configs/s03-cli.yml b/configs/s03-cli.yml new file mode 100644 index 0000000..ed288dc --- /dev/null +++ b/configs/s03-cli.yml @@ -0,0 +1,4 @@ +wallet: services/storage/wallet03.json +password: "" +rpc-endpoint: s03.frostfs.devenv:8080 +endpoint: s03.frostfs.devenv:8081 \ No newline at end of file diff --git a/configs/s04-cli.yml b/configs/s04-cli.yml new file mode 100644 index 0000000..f4353d5 --- /dev/null +++ b/configs/s04-cli.yml @@ -0,0 +1,4 @@ +wallet: services/storage/wallet04.json +password: "" +rpc-endpoint: s04.frostfs.devenv:8080 +endpoint: s04.frostfs.devenv:8081 \ No newline at end of file From c2e2b6442c970022b5dc0a7dba100840c3daa652 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 29 Aug 2023 12:45:13 +0300 Subject: [PATCH 102/149] [#47] grafana: Don't bind port to localhost Signed-off-by: Denis Kirillov --- services/grafana/docker-compose.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/services/grafana/docker-compose.yml b/services/grafana/docker-compose.yml index 795b87c..6d27299 100644 --- a/services/grafana/docker-compose.yml +++ b/services/grafana/docker-compose.yml @@ -14,8 +14,6 @@ services: - ./../../vendor/hosts:/etc/hosts - ./grafana.ini:/etc/grafana/grafana.ini - ./provisioning:/etc/grafana/provisioning - ports: - - '3000:3000' stop_signal: SIGKILL env_file: [ ".env", ".int_test.env" ] From de8b58911e498b6a522297093ae221d93e2bd997 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 29 Aug 2023 14:06:09 +0300 Subject: [PATCH 103/149] [#47] prometheus: Don't bind port to localhost Signed-off-by: Denis Kirillov --- services/prometheus/docker-compose.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/services/prometheus/docker-compose.yml b/services/prometheus/docker-compose.yml index aef2c67..e9eb4f9 100644 --- a/services/prometheus/docker-compose.yml +++ b/services/prometheus/docker-compose.yml @@ -15,8 +15,6 @@ services: - ./prometheus.yml:/etc/prometheus/prometheus.yml command: - --config.file=/etc/prometheus/prometheus.yml - ports: - - '9090:9090' stop_signal: SIGKILL env_file: [ ".env", ".prometheus.env", ".int_test.env" ] @@ -24,4 +22,4 @@ networks: prometheus_int: internet: external: true - name: basenet_internet \ No newline at end of file + name: basenet_internet From 9c9ec639f05f1479cc84be713149df0937f33ac4 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 25 Oct 2023 10:56:08 +0300 Subject: [PATCH 104/149] [#58] Update neo-go to v0.103.0 Signed-off-by: Evgenii Stratonikov --- .env | 2 +- services/morph_chain/protocol.privnet.yml | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.env b/.env index e04d5ca..de7b691 100644 --- a/.env +++ b/.env @@ -8,7 +8,7 @@ BASTION_VERSION=10 BASTION_IMAGE=debian # NeoGo privnet -NEOGO_VERSION=0.101.1 +NEOGO_VERSION=0.103.0 NEOGO_IMAGE=nspccdev/neo-go # FrostFS InnerRing nodes diff --git a/services/morph_chain/protocol.privnet.yml b/services/morph_chain/protocol.privnet.yml index 13e03b5..560730a 100644 --- a/services/morph_chain/protocol.privnet.yml +++ b/services/morph_chain/protocol.privnet.yml @@ -29,6 +29,11 @@ ApplicationConfiguration: AttemptConnPeers: 5 MinPeers: 0 Relay: true + Consensus: + Enabled: true + UnlockWallet: + Path: "./wallets/node-wallet.json" + Password: "one" RPC: Addresses: - "192.168.130.90:30333" @@ -49,6 +54,3 @@ ApplicationConfiguration: Addresses: - ":20011" Enabled: true - UnlockWallet: - Path: "./wallets/node-wallet.json" - Password: "one" From 40454b55076c9590d9e9ded5a85069820cf88cbd Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Wed, 15 Nov 2023 16:08:15 +0300 Subject: [PATCH 105/149] [#56] Add loki Signed-off-by: Alexander Chuprov --- .env | 8 +- services/grafana/.hosts | 1 + services/grafana/docker-compose.yml | 8 + .../dashboards/storage-node-logs.json | 358 ++++++++++++++++++ .../provisioning/datasources/datasource.yml | 5 + services/storage/cfg/config.yml | 7 +- 6 files changed, 384 insertions(+), 3 deletions(-) create mode 100644 services/grafana/provisioning/dashboards/storage-node-logs.json diff --git a/.env b/.env index de7b691..0969ff5 100644 --- a/.env +++ b/.env @@ -12,11 +12,11 @@ NEOGO_VERSION=0.103.0 NEOGO_IMAGE=nspccdev/neo-go # FrostFS InnerRing nodes -IR_VERSION=365a7ca0 +IR_VERSION=0.37.0-rc.1-93-g364f835b IR_IMAGE=truecloudlab/frostfs-ir # FrostFS Storage nodes -NODE_VERSION=365a7ca0 +NODE_VERSION=0.37.0-rc.1-93-g364f835b NODE_IMAGE=truecloudlab/frostfs-storage # NATS Server @@ -64,3 +64,7 @@ PROMETHEUS_IMAGE=prom/prometheus # Grafana versions GRAFANA_VERSION=9.5.6 GRAFANA_IMAGE=grafana/grafana + +# Loki versions +LOKI_VERSION=2.9.1 +LOKI_IMAGE=grafana/loki \ No newline at end of file diff --git a/services/grafana/.hosts b/services/grafana/.hosts index 8f36ea6..537a644 100644 --- a/services/grafana/.hosts +++ b/services/grafana/.hosts @@ -1 +1,2 @@ IPV4_PREFIX.122 grafana.LOCAL_DOMAIN +IPV4_PREFIX.123 loki.LOCAL_DOMAIN diff --git a/services/grafana/docker-compose.yml b/services/grafana/docker-compose.yml index 6d27299..90b67f2 100644 --- a/services/grafana/docker-compose.yml +++ b/services/grafana/docker-compose.yml @@ -17,6 +17,14 @@ services: stop_signal: SIGKILL env_file: [ ".env", ".int_test.env" ] + loki: + image: ${LOKI_IMAGE}:${LOKI_VERSION} + command: -config.file=/etc/loki/local-config.yaml + networks: + grafana_int: + internet: + ipv4_address: ${IPV4_PREFIX}.123 + networks: grafana_int: internet: diff --git a/services/grafana/provisioning/dashboards/storage-node-logs.json b/services/grafana/provisioning/dashboards/storage-node-logs.json new file mode 100644 index 0000000..3c91d52 --- /dev/null +++ b/services/grafana/provisioning/dashboards/storage-node-logs.json @@ -0,0 +1,358 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Overview panel for all services", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "count_over_time({hostname=~\"$hostname\"} |~ \"$level\" |~ `$filter` [1s])", + "queryType": "range", + "refId": "A" + } + ], + "title": "Log intensity", + "type": "timeseries" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.width", + "value": 226 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "labels" + }, + "properties": [ + { + "id": "custom.width", + "value": 424 + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 4, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "{hostname=~\"$hostname\"} |~ `$filter` | json level |~ \"$level\"", + "queryType": "range", + "refId": "A" + } + ], + "title": "Log", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "id": true, + "tsNs": true + }, + "indexByName": {}, + "renameByName": {} + } + } + ], + "type": "table" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "label": "Hostname", + "multi": true, + "name": "hostname", + "options": [], + "query": { + "label": "hostname", + "refId": "LokiVariableQueryEditor-VariableQuery", + "stream": "", + "type": 1 + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "hide": 0, + "includeAll": true, + "label": "Level", + "multi": true, + "name": "level", + "options": [ + { + "selected": true, + "text": "All", + "value": "$__all" + }, + { + "selected": false, + "text": "debug", + "value": "debug" + }, + { + "selected": false, + "text": "info", + "value": "info" + }, + { + "selected": false, + "text": "warn", + "value": "warn" + }, + { + "selected": false, + "text": "error", + "value": "error" + }, + { + "selected": false, + "text": "dpanic", + "value": "dpanic" + }, + { + "selected": false, + "text": "panic", + "value": "panic" + }, + { + "selected": false, + "text": "fatal", + "value": "fatal" + } + ], + "query": "debug,info,warn,error,dpanic,panic,fatal", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + }, + { + "current": { + "selected": false, + "text": "", + "value": "" + }, + "hide": 0, + "label": "Filter", + "name": "filter", + "options": [ + { + "selected": true, + "text": "", + "value": "" + } + ], + "query": "", + "skipUrlSync": false, + "type": "textbox" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Storage node logs", + "uid": "bef4b42d-c74e-4241-96cb-d239113f44bc", + "version": 1, + "weekStart": "" + } \ No newline at end of file diff --git a/services/grafana/provisioning/datasources/datasource.yml b/services/grafana/provisioning/datasources/datasource.yml index 81b6a7b..7d7427f 100644 --- a/services/grafana/provisioning/datasources/datasource.yml +++ b/services/grafana/provisioning/datasources/datasource.yml @@ -6,3 +6,8 @@ datasources: access: proxy orgId: 1 url: http://prometheus:9090 +- name: Loki + type: loki + access: proxy + orgId: 1 + url: http://loki:3100 \ No newline at end of file diff --git a/services/storage/cfg/config.yml b/services/storage/cfg/config.yml index 0a85096..6b3e7a8 100644 --- a/services/storage/cfg/config.yml +++ b/services/storage/cfg/config.yml @@ -1,6 +1,11 @@ # Logger section logger: level: debug # Minimum enabled logging level + loki: + enabled: true + endpoint: "loki.frostfs.devenv:3100/api/prom/push" + max_batch_delay: 1s + max_batch_size: 200 # Profiler section pprof: @@ -14,7 +19,7 @@ prometheus: address: :9090 # Server address shutdown_timeout: 15s # Timeout for metrics HTTP server graceful shutdown - # Application tracing section +# Application tracing section tracing: enabled: true exporter: otlp_grpc From 1077c9d35898ec5dd4ba71436d9f4e6489ab26d3 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 6 Dec 2023 14:08:47 +0300 Subject: [PATCH 106/149] [#59] .env: Update neo-go to v0.104.0 Signed-off-by: Evgenii Stratonikov --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 0969ff5..16d2da5 100644 --- a/.env +++ b/.env @@ -8,7 +8,7 @@ BASTION_VERSION=10 BASTION_IMAGE=debian # NeoGo privnet -NEOGO_VERSION=0.103.0 +NEOGO_VERSION=0.104.0 NEOGO_IMAGE=nspccdev/neo-go # FrostFS InnerRing nodes From dfad34fdeae470937d797b065ead0e29fe4a01dc Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Thu, 14 Dec 2023 16:50:16 +0300 Subject: [PATCH 107/149] [#61] Update frostfs-node components with contracts Signed-off-by: Alex Vanin --- .env | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.env b/.env index 16d2da5..7752b67 100644 --- a/.env +++ b/.env @@ -12,11 +12,11 @@ NEOGO_VERSION=0.104.0 NEOGO_IMAGE=nspccdev/neo-go # FrostFS InnerRing nodes -IR_VERSION=0.37.0-rc.1-93-g364f835b +IR_VERSION=0.37.0-rc.1-164-gbdd43f62 IR_IMAGE=truecloudlab/frostfs-ir # FrostFS Storage nodes -NODE_VERSION=0.37.0-rc.1-93-g364f835b +NODE_VERSION=0.37.0-rc.1-164-gbdd43f62 NODE_IMAGE=truecloudlab/frostfs-storage # NATS Server @@ -40,17 +40,17 @@ LOCODE_DB_URL=https://git.frostfs.info/attachments/a2e8def7-52b6-49f1-89cd-a0567 #LOCODE_DB_PATH=/path/to/locode_db # FrostFS CLI binary -FROSTFS_CLI_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/BbngJDdRJEDJTJk7qptq3SxKqrJqtvVYWU6R5AaFGbtG +FROSTFS_CLI_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/HUu3vMojqiopmAzWgUPPPSN7ThosapsLj5WrBvZR67tC #FROSTFS_CLI_PATH=/path/to/frostfs-cli-binary # FrostFS ADM tool binary FROSTFS_ADM_VERSION=eca5c210 -FROSTFS_ADM_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/2GxarAjGUb3RevxvqFGYT3hDQxNNaHzK6aFxhJCAMehq +FROSTFS_ADM_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/NMw4AktoK2zo5Kftj23wGEVzE1d52ULufjtcLjAVqZc #FROSTFS_ADM_PATH=/path/to/frostfs-adm-binary # Compiled FrostFS Smart Contracts FROSTFS_CONTRACTS_VERSION=8537293e -FROSTFS_CONTRACTS_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/6ccZoj4HxoN1G1qvJAX2Qw9p2D6qdyzAjNMaNkEKYQpA +FROSTFS_CONTRACTS_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/BLG2NrQW5UAF1RSMSBK8Z4mGanhvzD9aTRk4NsRGVDV9 #FROSTFS_CONTRACTS_PATH=/path/to/unpacked/frostfs-contracts-dir # Jaeger tracing @@ -67,4 +67,4 @@ GRAFANA_IMAGE=grafana/grafana # Loki versions LOKI_VERSION=2.9.1 -LOKI_IMAGE=grafana/loki \ No newline at end of file +LOKI_IMAGE=grafana/loki From 9654b77236aa20f4badcc706a77bb7ed2c930e83 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Thu, 14 Dec 2023 16:53:12 +0300 Subject: [PATCH 108/149] [#61] Update frostfs-http-gw Signed-off-by: Alex Vanin --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 7752b67..99c5ed0 100644 --- a/.env +++ b/.env @@ -24,7 +24,7 @@ NATS_VERSION=2.7.2 NATS_IMAGE=nats # HTTP Gate -HTTP_GW_VERSION=0.27.0-rc.1-15-g1776db28 +HTTP_GW_VERSION=0.28.0-rc.1-15-g5ae75eb9 HTTP_GW_IMAGE=truecloudlab/frostfs-http-gw # REST Gate From 0f6f2722c2fe672a1246c6f6732c50711d1af271 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Thu, 14 Dec 2023 16:57:17 +0300 Subject: [PATCH 109/149] [#61] Update frostfs-s3-gw Signed-off-by: Alex Vanin --- .env | 2 +- services/s3_gate/cfg/config.yml | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/.env b/.env index 99c5ed0..793cffc 100644 --- a/.env +++ b/.env @@ -32,7 +32,7 @@ REST_GW_VERSION=c9c85e90 REST_GW_IMAGE=truecloudlab/frostfs-rest-gw # S3 Gate -S3_GW_VERSION=0.27.0-rc.1-30-gce929468 +S3_GW_VERSION=0.28.0-rc.1-51-g9272f4e1 S3_GW_IMAGE=truecloudlab/frostfs-s3-gw # FrostFS LOCODE database diff --git a/services/s3_gate/cfg/config.yml b/services/s3_gate/cfg/config.yml index 7fa041a..03e84e0 100644 --- a/services/s3_gate/cfg/config.yml +++ b/services/s3_gate/cfg/config.yml @@ -33,3 +33,17 @@ server: wallet: path: /wallet.json # Path to wallet passphrase: "s3" # Passphrase to decrypt wallet + +features: + md5: + enabled: true + +control: + grpc: + endpoint: localhost:16515 + +frostfsid: + enabled: false + +policy: + enabled: false From 19e5cec49fdd5fbf244ee234b5bc8cdb81873964 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 20 Feb 2024 15:47:55 +0300 Subject: [PATCH 110/149] [#63] Use SIGTERM to stop Go services Signed-off-by: Evgenii Stratonikov --- services/ir/docker-compose.yml | 3 ++- services/rest_gate/docker-compose.yml | 3 ++- services/s3_gate/docker-compose.yml | 3 ++- services/storage/docker-compose.yml | 12 ++++++++---- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/services/ir/docker-compose.yml b/services/ir/docker-compose.yml index 77ec50d..0f8e0f9 100644 --- a/services/ir/docker-compose.yml +++ b/services/ir/docker-compose.yml @@ -13,7 +13,8 @@ services: ir_int: internet: ipv4_address: ${IPV4_PREFIX}.61 - stop_signal: SIGKILL + stop_signal: SIGTERM + stop_grace_period: 15s volumes: - ./az.json:/wallet.json - ./az.key:/wallet01.key diff --git a/services/rest_gate/docker-compose.yml b/services/rest_gate/docker-compose.yml index f5bac6b..ee1ab5e 100644 --- a/services/rest_gate/docker-compose.yml +++ b/services/rest_gate/docker-compose.yml @@ -16,7 +16,8 @@ services: - ./wallet.json:/wallet.json - ./../../vendor/hosts:/etc/hosts - ./cfg:/etc/frostfs/rest - stop_signal: SIGKILL + stop_signal: SIGTERM + stop_grace_period: 15s env_file: [ ".env", ".int_test.env" ] command: [ "frostfs-rest-gw", "--config", "/etc/frostfs/rest/config.yml" ] environment: diff --git a/services/s3_gate/docker-compose.yml b/services/s3_gate/docker-compose.yml index 1445480..d1b7601 100644 --- a/services/s3_gate/docker-compose.yml +++ b/services/s3_gate/docker-compose.yml @@ -18,7 +18,8 @@ services: - ./tls.crt:/tls.crt - ./../../vendor/hosts:/etc/hosts - ./cfg:/etc/frostfs/s3 - stop_signal: SIGKILL + stop_signal: SIGTERM + stop_grace_period: 15s env_file: [ ".env", ".s3.env", ".int_test.env" ] command: [ "frostfs-s3-gw", "--config", "/etc/frostfs/s3/config.yml" ] environment: diff --git a/services/storage/docker-compose.yml b/services/storage/docker-compose.yml index b32ba21..1aabc5e 100644 --- a/services/storage/docker-compose.yml +++ b/services/storage/docker-compose.yml @@ -24,7 +24,8 @@ services: - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key - ../nats/ca-cert.pem:/etc/frostfs-node/nats.ca.crt - ./cfg:/etc/frostfs/storage - stop_signal: SIGKILL + stop_signal: SIGTERM + stop_grace_period: 15s env_file: [ ".env", ".storage.env", ".int_test.env" ] command: [ "frostfs-node", "--config", "/etc/frostfs/storage/config.yml" ] environment: @@ -64,7 +65,8 @@ services: - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key - ../nats/ca-cert.pem:/etc/frostfs-node/nats.ca.crt - ./cfg:/etc/frostfs/storage - stop_signal: SIGKILL + stop_signal: SIGTERM + stop_grace_period: 15s env_file: [ ".env", ".storage.env", ".int_test.env" ] command: [ "frostfs-node", "--config", "/etc/frostfs/storage/config.yml" ] environment: @@ -104,7 +106,8 @@ services: - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key - ../nats/ca-cert.pem:/etc/frostfs-node/nats.ca.crt - ./cfg:/etc/frostfs/storage - stop_signal: SIGKILL + stop_signal: SIGTERM + stop_grace_period: 15s env_file: [ ".env", ".storage.env", ".int_test.env" ] command: [ "frostfs-node", "--config", "/etc/frostfs/storage/config.yml" ] environment: @@ -145,7 +148,8 @@ services: - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key - ../nats/ca-cert.pem:/etc/frostfs-node/nats.ca.crt - ./cfg:/etc/frostfs/storage - stop_signal: SIGKILL + stop_signal: SIGTERM + stop_grace_period: 15s env_file: [ ".env", ".storage.env", ".int_test.env" ] command: [ "frostfs-node", "--config", "/etc/frostfs/storage/config.yml" ] environment: From 8edfcb364d9a104e15f976117d20485a864a0d41 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Thu, 22 Feb 2024 12:41:55 +0300 Subject: [PATCH 111/149] [#64] adm: Create default Allow policy for root namespace Signed-off-by: Airat Arifullin --- Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile b/Makefile index 4caa292..051aa7d 100644 --- a/Makefile +++ b/Makefile @@ -76,6 +76,10 @@ up/bootstrap: get vendor/hosts @$(foreach SVC, $(START_BOOTSTRAP), $(shell docker-compose -f services/$(SVC)/docker-compose.yml up -d)) @source ./bin/helper.sh @./vendor/frostfs-adm --config frostfs-adm.yml morph init --contracts vendor/contracts + echo "Set rule chain to policy contract" + @./vendor/frostfs-adm --config frostfs-adm.yml morph \ + ape add-rule-chain --target-type namespace --target-name "" \ + --rule 'allow Container.* *' --chain-id "allow_container_ops" @for f in ./services/storage/wallet*.json; do \ echo "Transfer GAS to wallet $${f}" \ && ./vendor/frostfs-adm -c frostfs-adm.yml morph refill-gas --storage-wallet $${f} --gas 10.0 \ From 47b4917e7bed1eae19b72e2d4dadc521b9b9c870 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 4 Apr 2024 11:38:41 +0300 Subject: [PATCH 112/149] [#65] services: Fix docker-compose warnings There were multiple warning like this one. ``` WARN[0000] /secret/services/rest_gate/docker-compose.yml: `version` is obsolete ``` According to docker-compose spec, the parameter is indeed purely informative: https://github.com/compose-spec/compose-spec/blob/master/spec.md#version-and-name-top-level-elements Signed-off-by: Evgenii Stratonikov --- services/basenet/docker-compose.yml | 1 - services/grafana/docker-compose.yml | 1 - services/http_gate/docker-compose.yml | 1 - services/ir/docker-compose.yml | 1 - services/jaeger/docker-compose.yml | 1 - services/morph_chain/docker-compose.yml | 1 - services/nats/docker-compose.yml | 1 - services/prometheus/docker-compose.yml | 1 - services/rest_gate/docker-compose.yml | 1 - services/s3_gate/docker-compose.yml | 1 - services/storage/docker-compose.yml | 1 - 11 files changed, 11 deletions(-) diff --git a/services/basenet/docker-compose.yml b/services/basenet/docker-compose.yml index abac92a..b2905a3 100644 --- a/services/basenet/docker-compose.yml +++ b/services/basenet/docker-compose.yml @@ -1,6 +1,5 @@ --- -version: "2.4" services: basenet: diff --git a/services/grafana/docker-compose.yml b/services/grafana/docker-compose.yml index 90b67f2..3953400 100644 --- a/services/grafana/docker-compose.yml +++ b/services/grafana/docker-compose.yml @@ -1,4 +1,3 @@ -version: '2.4' services: grafana: image: ${GRAFANA_IMAGE}:${GRAFANA_VERSION} diff --git a/services/http_gate/docker-compose.yml b/services/http_gate/docker-compose.yml index 7108ee5..c2e68ae 100644 --- a/services/http_gate/docker-compose.yml +++ b/services/http_gate/docker-compose.yml @@ -1,6 +1,5 @@ --- -version: "2.4" services: http_gate: image: ${HTTP_GW_IMAGE}:${HTTP_GW_VERSION} diff --git a/services/ir/docker-compose.yml b/services/ir/docker-compose.yml index 0f8e0f9..048651b 100644 --- a/services/ir/docker-compose.yml +++ b/services/ir/docker-compose.yml @@ -1,6 +1,5 @@ --- -version: "2.4" services: ir01: diff --git a/services/jaeger/docker-compose.yml b/services/jaeger/docker-compose.yml index a460548..1fb367b 100644 --- a/services/jaeger/docker-compose.yml +++ b/services/jaeger/docker-compose.yml @@ -1,4 +1,3 @@ -version: '2.4' services: jaeger: image: ${JAEGER_IMAGE}:${JAEGER_VERSION} diff --git a/services/morph_chain/docker-compose.yml b/services/morph_chain/docker-compose.yml index 4cba4b3..dd44b2c 100644 --- a/services/morph_chain/docker-compose.yml +++ b/services/morph_chain/docker-compose.yml @@ -1,6 +1,5 @@ --- -version: "2.4" services: frostfs_morph_chain: image: ${NEOGO_IMAGE}:${NEOGO_VERSION} diff --git a/services/nats/docker-compose.yml b/services/nats/docker-compose.yml index d05dd04..cb89948 100644 --- a/services/nats/docker-compose.yml +++ b/services/nats/docker-compose.yml @@ -1,6 +1,5 @@ --- -version: "2.4" services: nats: image: ${NATS_IMAGE}:${NATS_VERSION} diff --git a/services/prometheus/docker-compose.yml b/services/prometheus/docker-compose.yml index e9eb4f9..31f5fbe 100644 --- a/services/prometheus/docker-compose.yml +++ b/services/prometheus/docker-compose.yml @@ -1,4 +1,3 @@ -version: '2.4' services: prometheus: image: ${PROMETHEUS_IMAGE}:${PROMETHEUS_VERSION} diff --git a/services/rest_gate/docker-compose.yml b/services/rest_gate/docker-compose.yml index ee1ab5e..09cfe38 100644 --- a/services/rest_gate/docker-compose.yml +++ b/services/rest_gate/docker-compose.yml @@ -1,6 +1,5 @@ --- -version: "2.4" services: rest_gate: image: ${REST_GW_IMAGE}:${REST_GW_VERSION} diff --git a/services/s3_gate/docker-compose.yml b/services/s3_gate/docker-compose.yml index d1b7601..6b43a0b 100644 --- a/services/s3_gate/docker-compose.yml +++ b/services/s3_gate/docker-compose.yml @@ -1,6 +1,5 @@ --- -version: "2.4" services: s3_gate: image: ${S3_GW_IMAGE}:${S3_GW_VERSION} diff --git a/services/storage/docker-compose.yml b/services/storage/docker-compose.yml index 1aabc5e..8a8698e 100644 --- a/services/storage/docker-compose.yml +++ b/services/storage/docker-compose.yml @@ -1,6 +1,5 @@ --- -version: "2.4" services: storage01: image: ${NODE_IMAGE}:${NODE_VERSION} From 773ea2339bf1eebc4946a0adbad904d7a8c8debf Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 15 Apr 2024 20:16:51 +0300 Subject: [PATCH 113/149] [#66] Update frostfs-* to v0.38.5 Signed-off-by: Alexander Chuprov --- .env | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.env b/.env index 793cffc..ad96218 100644 --- a/.env +++ b/.env @@ -12,11 +12,11 @@ NEOGO_VERSION=0.104.0 NEOGO_IMAGE=nspccdev/neo-go # FrostFS InnerRing nodes -IR_VERSION=0.37.0-rc.1-164-gbdd43f62 +IR_VERSION=0.38.5 IR_IMAGE=truecloudlab/frostfs-ir # FrostFS Storage nodes -NODE_VERSION=0.37.0-rc.1-164-gbdd43f62 +NODE_VERSION=0.38.5 NODE_IMAGE=truecloudlab/frostfs-storage # NATS Server @@ -40,17 +40,17 @@ LOCODE_DB_URL=https://git.frostfs.info/attachments/a2e8def7-52b6-49f1-89cd-a0567 #LOCODE_DB_PATH=/path/to/locode_db # FrostFS CLI binary -FROSTFS_CLI_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/HUu3vMojqiopmAzWgUPPPSN7ThosapsLj5WrBvZR67tC +FROSTFS_CLI_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/BtA6zWLtoDuDnzxSNkKPjyN5hGuSZFxGuJD1gh8kBwkP #FROSTFS_CLI_PATH=/path/to/frostfs-cli-binary # FrostFS ADM tool binary -FROSTFS_ADM_VERSION=eca5c210 -FROSTFS_ADM_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/NMw4AktoK2zo5Kftj23wGEVzE1d52ULufjtcLjAVqZc +FROSTFS_ADM_VERSION=498f9955ea +FROSTFS_ADM_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/CjV4S6ENzo2FLK4KxXZHHNW4veR1ATtynGY6Mc1xQ6RB #FROSTFS_ADM_PATH=/path/to/frostfs-adm-binary # Compiled FrostFS Smart Contracts -FROSTFS_CONTRACTS_VERSION=8537293e -FROSTFS_CONTRACTS_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/BLG2NrQW5UAF1RSMSBK8Z4mGanhvzD9aTRk4NsRGVDV9 +FROSTFS_CONTRACTS_VERSION=694daebb19 +FROSTFS_CONTRACTS_URL=https://git.frostfs.info/TrueCloudLab/frostfs-contract/releases/download/v0.19.2/frostfs-contract-v0.19.2.tar.gz #FROSTFS_CONTRACTS_PATH=/path/to/unpacked/frostfs-contracts-dir # Jaeger tracing From 2b6122192a4c26a84e6f8faa5533ab52fb2a88ef Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 8 May 2024 11:33:19 +0300 Subject: [PATCH 114/149] [#67] services/ir: Remove deprecated flag in healthcheck Signed-off-by: Evgenii Stratonikov --- services/ir/healthcheck.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/ir/healthcheck.sh b/services/ir/healthcheck.sh index 747bc8b..88fcdaa 100755 --- a/services/ir/healthcheck.sh +++ b/services/ir/healthcheck.sh @@ -1,6 +1,6 @@ #!/bin/sh -/frostfs-cli control healthcheck \ +/frostfs-cli control ir healthcheck \ --endpoint "$FROSTFS_IR_CONTROL_GRPC_ENDPOINT" \ - --wallet /wallet01.key --ir | + --wallet /wallet01.key | grep "Health status: READY" From 0be22a937536ae5ef425b338caa676ea0711bbde Mon Sep 17 00:00:00 2001 From: Alexey Savchuk Date: Mon, 8 Jul 2024 10:34:28 +0300 Subject: [PATCH 115/149] [#73] Update HTTP gate docs Signed-off-by: Aleksey Savchuk --- docs/http_gate.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/http_gate.md b/docs/http_gate.md index e2159ad..b6b2591 100644 --- a/docs/http_gate.md +++ b/docs/http_gate.md @@ -22,8 +22,8 @@ Image label prefix to use for containers. - Create a new container ``` $ frostfs-cli --rpc-endpoint s01.frostfs.devenv:8080 \ - --key wallets/wallet.key \ - container create --basic-acl readonly --await \ + --wallet wallets/wallet.key \ + container create --basic-acl private --await \ --policy "REP 1 SELECT 1 FROM *" container ID: 4LfREK1cetL4PUji5fqj9SgRTSmaC5jExEDK9HKCDjdP awaiting... @@ -33,7 +33,7 @@ container has been persisted on sidechain - Put an object into the newly created container ``` $ frostfs-cli --rpc-endpoint s01.frostfs.devenv:8080 \ - --key wallets/wallet.key \ + --wallet wallets/wallet.key \ object put --file /tmp/backup.jpeg \ --cid 4LfREK1cetL4PUji5fqj9SgRTSmaC5jExEDK9HKCDjdP [/tmp/backup.jpeg] Object successfully stored From d03be14312da6b1bd2f5bcbb113c4594eeaff258 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Thu, 8 Aug 2024 13:18:13 +0300 Subject: [PATCH 116/149] [#75] Makefile: Add subjects for storage and client wallets to FrostfsID Signed-off-by: Airat Arifullin --- Makefile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 051aa7d..9483dc7 100644 --- a/Makefile +++ b/Makefile @@ -85,7 +85,12 @@ up/bootstrap: get vendor/hosts && ./vendor/frostfs-adm -c frostfs-adm.yml morph refill-gas --storage-wallet $${f} --gas 10.0 \ || die "Failed to transfer GAS to alphabet wallets"; \ done - @echo "FrostFS sidechain environment is deployed" + @echo "Create frostfsid subject for ./wallets/wallet.json" \ + && subj_key=`docker container exec -it morph_chain neo-go wallet dump-keys -w /wallets/wallet.json | tail -1 | tr -d ' \r\n'` \ + && echo "Subject key: $${subj_key}" \ + && ./vendor/frostfs-adm -c frostfs-adm.yml morph frostfsid create-subject --namespace "" --subject-key $${subj_key} --subject-name walletsubject \ + || die "Failed to create subject for the wallet"; \ + echo "FrostFS sidechain environment is deployed" # Build up certain service .PHONY: up/% From f94fa284ece87feb5c6dd9bf18b664da916b2656 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Sat, 17 Aug 2024 05:50:27 +0300 Subject: [PATCH 117/149] [#76] Update frostfs-core components to v0.42.9 Signed-off-by: Alexander Chuprov --- .env | 12 ++++++------ services/ir/artifacts.mk | 7 ++----- services/morph_chain/artifacts.mk | 7 ++----- 3 files changed, 10 insertions(+), 16 deletions(-) diff --git a/.env b/.env index ad96218..b414b0d 100644 --- a/.env +++ b/.env @@ -12,12 +12,12 @@ NEOGO_VERSION=0.104.0 NEOGO_IMAGE=nspccdev/neo-go # FrostFS InnerRing nodes -IR_VERSION=0.38.5 -IR_IMAGE=truecloudlab/frostfs-ir +IR_VERSION=v0.42.9 +IR_IMAGE=git.frostfs.info/truecloudlab/frostfs-ir # FrostFS Storage nodes -NODE_VERSION=0.38.5 -NODE_IMAGE=truecloudlab/frostfs-storage +NODE_VERSION=v0.42.9 +NODE_IMAGE=git.frostfs.info/truecloudlab/frostfs-storage # NATS Server NATS_VERSION=2.7.2 @@ -40,12 +40,12 @@ LOCODE_DB_URL=https://git.frostfs.info/attachments/a2e8def7-52b6-49f1-89cd-a0567 #LOCODE_DB_PATH=/path/to/locode_db # FrostFS CLI binary -FROSTFS_CLI_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/BtA6zWLtoDuDnzxSNkKPjyN5hGuSZFxGuJD1gh8kBwkP +FROSTFS_CLI_URL=https://git.frostfs.info/TrueCloudLab/frostfs-node/releases/download/${NODE_VERSION}//frostfs-cli #FROSTFS_CLI_PATH=/path/to/frostfs-cli-binary # FrostFS ADM tool binary FROSTFS_ADM_VERSION=498f9955ea -FROSTFS_ADM_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/CjV4S6ENzo2FLK4KxXZHHNW4veR1ATtynGY6Mc1xQ6RB +FROSTFS_ADM_URL=https://git.frostfs.info/TrueCloudLab/frostfs-node/releases/download/${NODE_VERSION}/frostfs-adm #FROSTFS_ADM_PATH=/path/to/frostfs-adm-binary # Compiled FrostFS Smart Contracts diff --git a/services/ir/artifacts.mk b/services/ir/artifacts.mk index 95b2c22..0cdbdbb 100644 --- a/services/ir/artifacts.mk +++ b/services/ir/artifacts.mk @@ -25,7 +25,6 @@ endif # Download FrostFS CLI .ONESHELL: get.cli: FROSTFS_CLI_FILE=./vendor/frostfs-cli -get.cli: FROSTFS_CLI_ARCHIVE_FILE=${FROSTFS_CLI_FILE}.tar.gz get.cli: FROSTFS_CLI_PATH?= get.cli: @mkdir -p ./vendor @@ -34,10 +33,8 @@ ifeq (${FROSTFS_CLI_PATH},) @echo "⇒ Download FrostFS CLI binary from ${FROSTFS_CLI_URL}" @curl \ -ksSL "${FROSTFS_CLI_URL}" \ - -o ${FROSTFS_CLI_ARCHIVE_FILE} - @tar -xvf ${FROSTFS_CLI_ARCHIVE_FILE} -C ./vendor | xargs -I {} \ - mv ./vendor/{} ${FROSTFS_CLI_FILE} - @rm ${FROSTFS_CLI_ARCHIVE_FILE} + -o ${FROSTFS_CLI_FILE} + @chmod +x ${FROSTFS_CLI_FILE} else @echo "⇒ Copy local binary from ${FROSTFS_CLI_PATH}" @cp ${FROSTFS_CLI_PATH} ${FROSTFS_CLI_FILE} diff --git a/services/morph_chain/artifacts.mk b/services/morph_chain/artifacts.mk index 3572e69..77a7ae3 100644 --- a/services/morph_chain/artifacts.mk +++ b/services/morph_chain/artifacts.mk @@ -20,15 +20,12 @@ endif # Download FrostFS ADM tool get.adm: FROSTFS_ADM_DEST=./vendor/frostfs-adm -get.adm: FROSTFS_ADM_ARCHIVE=frostfs-adm.tar.gz get.adm: ifeq (${FROSTFS_ADM_PATH},) @echo "⇒ Download FrostFS ADM binary from ${FROSTFS_ADM_URL}" - @curl -skSL ${FROSTFS_ADM_URL} -o ${FROSTFS_ADM_ARCHIVE} - @tar -xvf ${FROSTFS_ADM_ARCHIVE} -C ./vendor | xargs -I {} \ - mv ./vendor/{} ${FROSTFS_ADM_DEST} - @rm ${FROSTFS_ADM_ARCHIVE} + @curl -skSL ${FROSTFS_ADM_URL} -o ${FROSTFS_ADM_DEST} + @chmod +x ${FROSTFS_ADM_DEST} else @echo "⇒ Copy frostfs-adm binary from ${FROSTFS_ADM_PATH}" @cp ${FROSTFS_ADM_PATH} ${FROSTFS_ADM_DEST} From 155042343b40c94507af61d0cbc7598dfed995a0 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Sat, 17 Aug 2024 06:25:11 +0300 Subject: [PATCH 118/149] [#69] service/storage: Add support -q flag in healthcheck Signed-off-by: Alexander Chuprov --- services/storage/docker-compose.yml | 12 ++++-------- services/storage/healthcheck.sh | 5 ----- 2 files changed, 4 insertions(+), 13 deletions(-) delete mode 100755 services/storage/healthcheck.sh diff --git a/services/storage/docker-compose.yml b/services/storage/docker-compose.yml index 8a8698e..fb10974 100644 --- a/services/storage/docker-compose.yml +++ b/services/storage/docker-compose.yml @@ -17,7 +17,6 @@ services: - storage_s01:/storage - ./../../vendor/frostfs-cli:/frostfs-cli - ./cli-cfg.yml:/cli-cfg.yml - - ./healthcheck.sh:/healthcheck.sh - ./s04tls.crt:/etc/ssl/certs/s04tls.crt - ../nats/client-cert.pem:/etc/frostfs-node/nats.tls.cert - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key @@ -36,7 +35,7 @@ services: - FROSTFS_NODE_ATTRIBUTE_1=UN-LOCODE:RU MOW - FROSTFS_NODE_ATTRIBUTE_2=Price:22 healthcheck: - test: ["CMD-SHELL", "/healthcheck.sh"] + test: ["CMD-SHELL", "/frostfs-cli control healthcheck -q -c /cli-cfg.yml --endpoint \"$$FROSTFS_CONTROL_GRPC_ENDPOINT\""] interval: 2s timeout: 1s retries: 5 @@ -58,7 +57,6 @@ services: - storage_s02:/storage - ./../../vendor/frostfs-cli:/frostfs-cli - ./cli-cfg.yml:/cli-cfg.yml - - ./healthcheck.sh:/healthcheck.sh - ./s04tls.crt:/etc/ssl/certs/s04tls.crt - ../nats/client-cert.pem:/etc/frostfs-node/nats.tls.cert - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key @@ -77,7 +75,7 @@ services: - FROSTFS_NODE_ATTRIBUTE_1=UN-LOCODE:RU LED - FROSTFS_NODE_ATTRIBUTE_2=Price:33 healthcheck: - test: ["CMD-SHELL", "/healthcheck.sh"] + test: ["CMD-SHELL", "/frostfs-cli control healthcheck -q -c /cli-cfg.yml --endpoint \"$$FROSTFS_CONTROL_GRPC_ENDPOINT\""] interval: 2s timeout: 1s retries: 5 @@ -99,7 +97,6 @@ services: - storage_s03:/storage - ./../../vendor/frostfs-cli:/frostfs-cli - ./cli-cfg.yml:/cli-cfg.yml - - ./healthcheck.sh:/healthcheck.sh - ./s04tls.crt:/etc/ssl/certs/s04tls.crt - ../nats/client-cert.pem:/etc/frostfs-node/nats.tls.cert - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key @@ -118,7 +115,7 @@ services: - FROSTFS_NODE_ATTRIBUTE_1=UN-LOCODE:SE STO - FROSTFS_NODE_ATTRIBUTE_2=Price:11 healthcheck: - test: ["CMD-SHELL", "/healthcheck.sh"] + test: ["CMD-SHELL", "/frostfs-cli control healthcheck -q -c /cli-cfg.yml --endpoint \"$$FROSTFS_CONTROL_GRPC_ENDPOINT\""] interval: 2s timeout: 1s retries: 5 @@ -140,7 +137,6 @@ services: - storage_s04:/storage - ./../../vendor/frostfs-cli:/frostfs-cli - ./cli-cfg.yml:/cli-cfg.yml - - ./healthcheck.sh:/healthcheck.sh - ./s04tls.crt:/tls.crt - ./s04tls.key:/tls.key - ../nats/client-cert.pem:/etc/frostfs-node/nats.tls.cert @@ -165,7 +161,7 @@ services: - FROSTFS_NODE_ATTRIBUTE_1=UN-LOCODE:FI HEL - FROSTFS_NODE_ATTRIBUTE_2=Price:44 healthcheck: - test: ["CMD-SHELL", "/healthcheck.sh"] + test: ["CMD-SHELL", "/frostfs-cli control healthcheck -q -c /cli-cfg.yml --endpoint \"$$FROSTFS_CONTROL_GRPC_ENDPOINT\""] interval: 2s timeout: 1s retries: 5 diff --git a/services/storage/healthcheck.sh b/services/storage/healthcheck.sh deleted file mode 100755 index 792d2ad..0000000 --- a/services/storage/healthcheck.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh - -/frostfs-cli control healthcheck -c /cli-cfg.yml \ - --endpoint "$FROSTFS_CONTROL_GRPC_ENDPOINT" | - grep "Health status: READY" From ae658469a5d44191e51df539054b3f0367bfdb73 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Sat, 17 Aug 2024 06:25:24 +0300 Subject: [PATCH 119/149] [#69] service/ir: Add support -q flag in healthcheck Signed-off-by: Alexander Chuprov --- services/ir/docker-compose.yml | 3 +-- services/ir/healthcheck.sh | 6 ------ 2 files changed, 1 insertion(+), 8 deletions(-) delete mode 100755 services/ir/healthcheck.sh diff --git a/services/ir/docker-compose.yml b/services/ir/docker-compose.yml index 048651b..4e671b2 100644 --- a/services/ir/docker-compose.yml +++ b/services/ir/docker-compose.yml @@ -20,12 +20,11 @@ services: - ./../../vendor/hosts:/etc/hosts - ./../../vendor/locode_db:/locode/db - ./../../vendor/frostfs-cli:/frostfs-cli - - ./healthcheck.sh:/healthcheck.sh - ./cfg:/etc/frostfs/ir env_file: [ ".env", ".ir.env", ".int_test.env" ] command: [ "frostfs-ir", "--config", "/etc/frostfs/ir/config.yml" ] healthcheck: - test: ["CMD-SHELL", "/healthcheck.sh"] + test: ["CMD-SHELL", "/frostfs-cli control ir healthcheck -q --wallet /wallet01.key --endpoint \"$$FROSTFS_IR_CONTROL_GRPC_ENDPOINT\""] interval: 2s timeout: 1s retries: 5 diff --git a/services/ir/healthcheck.sh b/services/ir/healthcheck.sh deleted file mode 100755 index 88fcdaa..0000000 --- a/services/ir/healthcheck.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh - -/frostfs-cli control ir healthcheck \ - --endpoint "$FROSTFS_IR_CONTROL_GRPC_ENDPOINT" \ - --wallet /wallet01.key | - grep "Health status: READY" From 044cf99e8dae647abb3706b498b27d00b58b6edc Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Sat, 17 Aug 2024 08:08:24 +0300 Subject: [PATCH 120/149] [#70] Makefile: Make bootstrap idempotent Signed-off-by: Alexander Chuprov --- Makefile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 9483dc7..2f505a9 100644 --- a/Makefile +++ b/Makefile @@ -85,11 +85,15 @@ up/bootstrap: get vendor/hosts && ./vendor/frostfs-adm -c frostfs-adm.yml morph refill-gas --storage-wallet $${f} --gas 10.0 \ || die "Failed to transfer GAS to alphabet wallets"; \ done - @echo "Create frostfsid subject for ./wallets/wallet.json" \ - && subj_key=`docker container exec -it morph_chain neo-go wallet dump-keys -w /wallets/wallet.json | tail -1 | tr -d ' \r\n'` \ + @echo "Create frostfsid subject for ./wallets/wallet.json"; \ + if [ -n "$$(./vendor/frostfs-adm -c frostfs-adm.yml morph frostfsid list-subjects --namespace '')" ]; then \ + echo "Subject already exists"; \ + else \ + subj_key=`docker container exec -it morph_chain neo-go wallet dump-keys -w /wallets/wallet.json | tail -1 | tr -d ' \r\n'` \ && echo "Subject key: $${subj_key}" \ && ./vendor/frostfs-adm -c frostfs-adm.yml morph frostfsid create-subject --namespace "" --subject-key $${subj_key} --subject-name walletsubject \ || die "Failed to create subject for the wallet"; \ + fi echo "FrostFS sidechain environment is deployed" # Build up certain service From dd382f8ce048d9e0880f43471d7896de000c0843 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Sat, 17 Aug 2024 08:08:55 +0300 Subject: [PATCH 121/149] [#68] service/morph: Add volume for morph_chain Signed-off-by: Alexander Chuprov --- services/morph_chain/docker-compose.yml | 4 ++++ services/morph_chain/protocol.privnet.yml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/services/morph_chain/docker-compose.yml b/services/morph_chain/docker-compose.yml index dd44b2c..68535ea 100644 --- a/services/morph_chain/docker-compose.yml +++ b/services/morph_chain/docker-compose.yml @@ -19,9 +19,13 @@ services: - ./config.yml:/wallets/config.yml - ./../../vendor/hosts:/etc/hosts - ./../../wallets/wallet.json:/wallets/wallet.json + - chains:/chains networks: chain_int: internet: external: true name: basenet_internet + +volumes: + chains: diff --git a/services/morph_chain/protocol.privnet.yml b/services/morph_chain/protocol.privnet.yml index 560730a..2066029 100644 --- a/services/morph_chain/protocol.privnet.yml +++ b/services/morph_chain/protocol.privnet.yml @@ -17,7 +17,7 @@ ApplicationConfiguration: DBConfiguration: Type: "boltdb" BoltDBOptions: - FilePath: "./db/morph.bolt" + FilePath: "/chains/morph.bolt" P2P: Addresses: - ":20333" From 439a9e71cfaf3837f16d9c501385c8ed2ff67dbc Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Fri, 6 Sep 2024 14:09:08 +0300 Subject: [PATCH 122/149] [#81] env: Fix typo Signed-off-by: Alexander Chuprov --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index b414b0d..c62cf17 100644 --- a/.env +++ b/.env @@ -40,7 +40,7 @@ LOCODE_DB_URL=https://git.frostfs.info/attachments/a2e8def7-52b6-49f1-89cd-a0567 #LOCODE_DB_PATH=/path/to/locode_db # FrostFS CLI binary -FROSTFS_CLI_URL=https://git.frostfs.info/TrueCloudLab/frostfs-node/releases/download/${NODE_VERSION}//frostfs-cli +FROSTFS_CLI_URL=https://git.frostfs.info/TrueCloudLab/frostfs-node/releases/download/${NODE_VERSION}/frostfs-cli #FROSTFS_CLI_PATH=/path/to/frostfs-cli-binary # FrostFS ADM tool binary From 2e67acbcb2af952933b87856083a6a926b840723 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Fri, 6 Sep 2024 14:37:48 +0300 Subject: [PATCH 123/149] [#78] env: Restore version format Signed-off-by: Alexander Chuprov --- .env | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.env b/.env index c62cf17..dea5726 100644 --- a/.env +++ b/.env @@ -12,11 +12,11 @@ NEOGO_VERSION=0.104.0 NEOGO_IMAGE=nspccdev/neo-go # FrostFS InnerRing nodes -IR_VERSION=v0.42.9 +IR_VERSION=0.42.9 IR_IMAGE=git.frostfs.info/truecloudlab/frostfs-ir # FrostFS Storage nodes -NODE_VERSION=v0.42.9 +NODE_VERSION=0.42.9 NODE_IMAGE=git.frostfs.info/truecloudlab/frostfs-storage # NATS Server @@ -40,12 +40,12 @@ LOCODE_DB_URL=https://git.frostfs.info/attachments/a2e8def7-52b6-49f1-89cd-a0567 #LOCODE_DB_PATH=/path/to/locode_db # FrostFS CLI binary -FROSTFS_CLI_URL=https://git.frostfs.info/TrueCloudLab/frostfs-node/releases/download/${NODE_VERSION}/frostfs-cli +FROSTFS_CLI_URL=https://git.frostfs.info/TrueCloudLab/frostfs-node/releases/download/v${NODE_VERSION}/frostfs-cli #FROSTFS_CLI_PATH=/path/to/frostfs-cli-binary # FrostFS ADM tool binary FROSTFS_ADM_VERSION=498f9955ea -FROSTFS_ADM_URL=https://git.frostfs.info/TrueCloudLab/frostfs-node/releases/download/${NODE_VERSION}/frostfs-adm +FROSTFS_ADM_URL=https://git.frostfs.info/TrueCloudLab/frostfs-node/releases/download/v${NODE_VERSION}/frostfs-adm #FROSTFS_ADM_PATH=/path/to/frostfs-adm-binary # Compiled FrostFS Smart Contracts From 7538bd9b171409fc5eeb5c08d074efe5792db133 Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Tue, 10 Sep 2024 16:45:06 +0300 Subject: [PATCH 124/149] [#83] Honor IPV4_PREFIX in morph_chain config Signed-off-by: Vitaliy Potyarkin --- services/morph_chain/protocol.privnet.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/morph_chain/protocol.privnet.yml b/services/morph_chain/protocol.privnet.yml index 2066029..f1cc754 100644 --- a/services/morph_chain/protocol.privnet.yml +++ b/services/morph_chain/protocol.privnet.yml @@ -36,7 +36,7 @@ ApplicationConfiguration: Password: "one" RPC: Addresses: - - "192.168.130.90:30333" + - ":30333" Enabled: true SessionEnabled: true EnableCORSWorkaround: false From d0c32731f2c468dafd979337ad04af1075d2bca2 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 9 Sep 2024 18:59:33 +0300 Subject: [PATCH 125/149] [#80] Update frostfs-service components to v0.30.* Signed-off-by: Alexander Chuprov --- .env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.env b/.env index dea5726..17697f6 100644 --- a/.env +++ b/.env @@ -24,7 +24,7 @@ NATS_VERSION=2.7.2 NATS_IMAGE=nats # HTTP Gate -HTTP_GW_VERSION=0.28.0-rc.1-15-g5ae75eb9 +HTTP_GW_VERSION=0.30.2 HTTP_GW_IMAGE=truecloudlab/frostfs-http-gw # REST Gate @@ -32,7 +32,7 @@ REST_GW_VERSION=c9c85e90 REST_GW_IMAGE=truecloudlab/frostfs-rest-gw # S3 Gate -S3_GW_VERSION=0.28.0-rc.1-51-g9272f4e1 +S3_GW_VERSION=0.30.4 S3_GW_IMAGE=truecloudlab/frostfs-s3-gw # FrostFS LOCODE database From 636be7352ebf5221ed66b02a8df85cead7c3633e Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Mon, 30 Sep 2024 17:04:11 +0300 Subject: [PATCH 126/149] [#84] Make targets for issuing credentials Signed-off-by: Nikita Zinkevich --- Makefile | 1 + README.md | 59 +++++++++++++++++++++++++ services/morph_chain/docker-compose.yml | 1 + services/s3_gate/docker-compose.yml | 8 ++++ services/s3_gate/issue-creds.sh | 41 +++++++++++++++++ services/s3_gate/prepare.mk | 14 ++++++ 6 files changed, 124 insertions(+) create mode 100755 services/s3_gate/issue-creds.sh create mode 100644 services/s3_gate/prepare.mk diff --git a/Makefile b/Makefile index 2f505a9..dff81df 100644 --- a/Makefile +++ b/Makefile @@ -60,6 +60,7 @@ get: $(foreach SVC, $(GET_SVCS), get.$(SVC)) .PHONY: up up: up/basic @$(foreach SVC, $(START_SVCS), $(shell docker-compose -f services/$(SVC)/docker-compose.yml up -d)) + ./vendor/frostfs-adm morph proxy-add-account --config frostfs-adm.yml --account=`docker container exec morph_chain neo-go wallet dump-keys -w /wallets/s3-wallet.json | head -1 | awk '{print $1}'` || die "Couldn't set s3-gw wallet as proxy wallet" @echo "Full FrostFS Developer Environment is ready" # Build up FrostFS diff --git a/README.md b/README.md index aa07a08..ad57655 100644 --- a/README.md +++ b/README.md @@ -137,6 +137,65 @@ Display addresses and host names for each running service, if available. Clean up `vendor` directory. +### s3cred + +Registers user wallet and issues s3 credentials. + +Usage and default parameter values: +```sh +make s3cred [password=""] [contract_password=s3] [wallet=/user_wallet.json] [gate_public_key=0313b1ac3a8076e155a7e797b24f0b650cccad5941ea59d7cfd51a024a8b2a06bf] +``` + +As soon as the storage node is in the network map (see above) you can generate S3 +credentials: + +``` sh +$ make s3cred +{ + "access_key_id": "EXArWh8x1zeHG3851s1RtoCo7dowxF6rhLGA15nbMffT0AKRSjJ5fmcqf3Ht2VCAkfmPQUVARghRB77xHCA1BoN2p", + "secret_access_key": "d70c1dba83f0f90bb231f06f1ce0e0dfbcfb122f4b4345a3c18d3869c359b79f", + "owner_private_key": "140947599afd9ca89af4b358c3176eb046e554d942a0dc99a8e06f3e43c8f4ad", + "wallet_public_key": "0324e76288fcb900100d01802a14ef977cca45ad073561230446df14b344c858b6", + "container_id": "EXArWh8x1zeHG3851s1RtoCo7dowxF6rhLGA15nbMffT" +} +``` +Running without any parameters will result in defaults which are based on the private key from +`/user-wallet.json` file and `/wallet.json` contract wallet. + +Now let's configure an S3 client (AWS CLI will be used as example): + +``` sh +$ aws configure +AWS Access Key ID []: EXArWh8x1zeHG3851s1RtoCo7dowxF6rhLGA15nbMffT0AKRSjJ5fmcqf3Ht2VCAkfmPQUVARghRB77xHCA1BoN2p +AWS Secret Access Key []: d70c1dba83f0f90bb231f06f1ce0e0dfbcfb122f4b4345a3c18d3869c359b79f +Default region name []: us-east-1 +Default output format []: json +``` + +If you need to create credentials for different users, put user wallets to `wallets` dir and specify them via `wallet` parameter. +Pass wallet password in `password` parameter if it's not default. The same is for `contract_wallet` and `gate_public_key` params. + +```sh +$ make s3cred wallet=custom_wallet.json password=test +{ + "access_key_id": "jHhL5B33o16R4jQsb8wm9A3RRdS6KrTB5N4bja9Jys904W7xXFNKqem2ACvTRWRYJsZMCUikYFSokN7pPJziWyDi", + "secret_access_key": "21bb64fafa32c82417fd8b97ac56cc8a085998a3852632d52fe7042453daa440", + "owner_private_key": "10f6f9d7a47bb0bf68363ad8a99fe69f1493f8b6e1665b3e4e83feb2d5c7ee39", + "wallet_public_key": "03e38759973a6bb722baabc2dd84036a39f0b2f53d32fec45a4dacde8a50fe4b70", + "container_id": "jHhL5B33o16R4jQsb8wm9A3RRdS6KrTB5N4bja9Jys9" +} +``` + +To get credentials from custom wallet, place it in `wallets` dir before start. + +### cred + +Usage and default parameter values: +```sh +make cred [password=""] [contract_password=s3] [wallet=/user_wallet.json] +``` +The same as `s3cred`, but it doesn't issues s3 credentials. + ## Contributing Feel free to contribute to this project after reading the [contributing diff --git a/services/morph_chain/docker-compose.yml b/services/morph_chain/docker-compose.yml index 68535ea..2c2facb 100644 --- a/services/morph_chain/docker-compose.yml +++ b/services/morph_chain/docker-compose.yml @@ -19,6 +19,7 @@ services: - ./config.yml:/wallets/config.yml - ./../../vendor/hosts:/etc/hosts - ./../../wallets/wallet.json:/wallets/wallet.json + - ./../s3_gate/wallet.json:/wallets/s3-wallet.json - chains:/chains networks: diff --git a/services/s3_gate/docker-compose.yml b/services/s3_gate/docker-compose.yml index 6b43a0b..f7fc28d 100644 --- a/services/s3_gate/docker-compose.yml +++ b/services/s3_gate/docker-compose.yml @@ -12,11 +12,17 @@ services: internet: ipv4_address: ${IPV4_PREFIX}.82 volumes: + # Gate wallet - ./wallet.json:/wallet.json + # Custom user wallets + - ./wallets:/wallets + # Default user wallet + - ./../../wallets/wallet.json:/wallets/wallet.json - ./tls.key:/tls.key - ./tls.crt:/tls.crt - ./../../vendor/hosts:/etc/hosts - ./cfg:/etc/frostfs/s3 + - ./issue-creds.sh:/usr/bin/issue-creds.sh stop_signal: SIGTERM stop_grace_period: 15s env_file: [ ".env", ".s3.env", ".int_test.env" ] @@ -34,6 +40,8 @@ services: - S3_GW_PEERS_2_WEIGHT=0.2 - S3_GW_PEERS_3_ADDRESS=s04.${LOCAL_DOMAIN}:8080 - S3_GW_PEERS_3_WEIGHT=0.2 + - AUTHMATE_WALLET_PASSPHRASE= + - AUTHMATE_WALLET_CONTRACT_PASSPHRASE=s3 networks: s3_gate_int: diff --git a/services/s3_gate/issue-creds.sh b/services/s3_gate/issue-creds.sh new file mode 100755 index 0000000..e2355f8 --- /dev/null +++ b/services/s3_gate/issue-creds.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +initUser() { + /bin/frostfs-s3-authmate register-user \ + --wallet $WALLET_PATH \ + --rpc-endpoint http://morph-chain.frostfs.devenv:30333 \ + --username $USERNAME \ + --contract-wallet /wallet.json 1> /dev/null && touch $WALLET_CACHE/$USERNAME +} + +issueCreds() { + /bin/frostfs-s3-authmate issue-secret \ + --wallet $WALLET_PATH \ + --peer s01.frostfs.devenv:8080 \ + --gate-public-key $S3_GATE_PUBLIC_KEY \ + --container-placement-policy "REP 3" +} + +set -e + +WALLET_PATH=/wallets/$2 +if [[ -z "$2" ]]; then + WALLET_PATH=/wallets/wallet.json +fi + +S3_GATE_PUBLIC_KEY=$3 +if [[ -z "$3" ]]; then + S3_GATE_PUBLIC_KEY=0313b1ac3a8076e155a7e797b24f0b650cccad5941ea59d7cfd51a024a8b2a06bf +fi + +WALLET_CACHE=/data/wallets +mkdir -p $WALLET_CACHE + +USERNAME=$(echo $WALLET_PATH | md5sum | cut -d' ' -f1) +if [ ! -e $WALLET_CACHE/$USERNAME ]; then + initUser +fi + +if [ $1 == "s3" ]; then + issueCreds +fi diff --git a/services/s3_gate/prepare.mk b/services/s3_gate/prepare.mk new file mode 100644 index 0000000..c97add8 --- /dev/null +++ b/services/s3_gate/prepare.mk @@ -0,0 +1,14 @@ +.PHONY: s3cred register + +password?= +contract_password?=s3 +gate_public_key?= +wallet?= + +# Register wallet & generate S3 credentials +s3cred: + @docker exec -e AUTHMATE_WALLET_PASSPHRASE="$(password)" -e AUTHMATE_WALLET_CONTRACT_PASSPHRASE="$(contract_password)" s3_gate /usr/bin/issue-creds.sh s3 "$(wallet)" "$(gate_public_key)" + +# Only registers user wallet +register: + @docker exec -e AUTHMATE_WALLET_PASSPHRASE="$(password)" -e AUTHMATE_WALLET_CONTRACT_PASSPHRASE="$(contract_password)" s3_gate /usr/bin/issue-creds.sh native "$(wallet)" From 7152f59232ac5ac763cbb6433331d161f64c56d0 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 31 Oct 2024 10:24:30 +0300 Subject: [PATCH 127/149] [#88] Remove nats service It was removed from node in TrueCloudLab/frostfs-node#1161. Signed-off-by: Evgenii Stratonikov --- .basic_services | 1 - .env | 4 --- .gitignore | 1 - Makefile | 2 +- services/nats/.env | 1 - services/nats/.hosts | 1 - services/nats/.int_test.env | 1 - services/nats/artifacts.mk | 7 ----- services/nats/docker-compose.yml | 30 ------------------ services/nats/generate_cert.sh | 49 ----------------------------- services/nats/nats.conf | 15 --------- services/storage/cfg/config.yml | 8 ----- services/storage/docker-compose.yml | 12 ------- 13 files changed, 1 insertion(+), 131 deletions(-) delete mode 120000 services/nats/.env delete mode 100644 services/nats/.hosts delete mode 120000 services/nats/.int_test.env delete mode 100644 services/nats/artifacts.mk delete mode 100644 services/nats/docker-compose.yml delete mode 100755 services/nats/generate_cert.sh delete mode 100644 services/nats/nats.conf diff --git a/.basic_services b/.basic_services index 201ea70..845c731 100644 --- a/.basic_services +++ b/.basic_services @@ -1,5 +1,4 @@ # Services start/stop order # Will start from top to bottom and stop in reverse -nats ir storage diff --git a/.env b/.env index 17697f6..6bb3728 100644 --- a/.env +++ b/.env @@ -19,10 +19,6 @@ IR_IMAGE=git.frostfs.info/truecloudlab/frostfs-ir NODE_VERSION=0.42.9 NODE_IMAGE=git.frostfs.info/truecloudlab/frostfs-storage -# NATS Server -NATS_VERSION=2.7.2 -NATS_IMAGE=nats - # HTTP Gate HTTP_GW_VERSION=0.30.2 HTTP_GW_IMAGE=truecloudlab/frostfs-http-gw diff --git a/.gitignore b/.gitignore index 36e495d..a672692 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,3 @@ sites/* # Runtime generation keys services/storage/*tls.crt services/storage/*tls.key -services/nats/*.pem diff --git a/Makefile b/Makefile index dff81df..c4a6c48 100644 --- a/Makefile +++ b/Makefile @@ -150,7 +150,7 @@ hosts: vendor/hosts .PHONY: clean .ONESHELL: clean: - @rm -rf vendor/* services/storage/s04tls.* services/nats/*.pem + @rm -rf vendor/* services/storage/s04tls.* @> .int_test.env @for svc in $(PULL_SVCS) do diff --git a/services/nats/.env b/services/nats/.env deleted file mode 120000 index c7360fb..0000000 --- a/services/nats/.env +++ /dev/null @@ -1 +0,0 @@ -../../.env \ No newline at end of file diff --git a/services/nats/.hosts b/services/nats/.hosts deleted file mode 100644 index 6ed5417..0000000 --- a/services/nats/.hosts +++ /dev/null @@ -1 +0,0 @@ -IPV4_PREFIX.101 nats.LOCAL_DOMAIN diff --git a/services/nats/.int_test.env b/services/nats/.int_test.env deleted file mode 120000 index 582b6a2..0000000 --- a/services/nats/.int_test.env +++ /dev/null @@ -1 +0,0 @@ -../../.int_test.env \ No newline at end of file diff --git a/services/nats/artifacts.mk b/services/nats/artifacts.mk deleted file mode 100644 index b66cebf..0000000 --- a/services/nats/artifacts.mk +++ /dev/null @@ -1,7 +0,0 @@ -# Create new TLS certs for NATS server and clients - -NATS_DIR=$(abspath services/nats) - -get.nats: - @echo "⇒ Creating certs for NATS server and clients" - ${NATS_DIR}/generate_cert.sh ${LOCAL_DOMAIN} > /dev/null diff --git a/services/nats/docker-compose.yml b/services/nats/docker-compose.yml deleted file mode 100644 index cb89948..0000000 --- a/services/nats/docker-compose.yml +++ /dev/null @@ -1,30 +0,0 @@ ---- - -services: - nats: - image: ${NATS_IMAGE}:${NATS_VERSION} - domainname: ${LOCAL_DOMAIN} - hostname: nats - container_name: nats - restart: on-failure - dns: - - ${IPV4_PREFIX}.101 - networks: - nats_int: - internet: - ipv4_address: ${IPV4_PREFIX}.101 - volumes: - - ./../../vendor/hosts:/etc/hosts - - ./nats.conf:/etc/nats/frostfs-nats-server.conf - - ./server-cert.pem:/certs/server-cert.pem - - ./server-key.pem:/certs/server-key.pem - - ./ca-cert.pem:/certs/ca-cert.pem - stop_signal: SIGKILL - env_file: [ ".env", ".int_test.env" ] - command: ["-c", "/etc/nats/frostfs-nats-server.conf"] - -networks: - nats_int: - internet: - external: true - name: basenet_internet diff --git a/services/nats/generate_cert.sh b/services/nats/generate_cert.sh deleted file mode 100755 index 4c4dd0b..0000000 --- a/services/nats/generate_cert.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash - -source bin/helper.sh - -WORKDIR=$(dirname "$0") -LOCAL_DOMAIN=$1 - -CA_KEY=$WORKDIR/ca-key.pem -CA_CRT=$WORKDIR/ca-cert.pem - -SRV_KEY=$WORKDIR/server-key.pem -SRV_REQ=$WORKDIR/server-req.csr -SRV_CRT=$WORKDIR/server-cert.pem - -CLI_KEY=$WORKDIR/client-key.pem -CLI_REQ=$WORKDIR/client-req.csr -CLI_CRT=$WORKDIR/client-cert.pem - -SUBJ="/O=TrueCloudLab" - -if [[ ! -f $CA_KEY || ! -f $CA_CRT ]]; then - openssl req -newkey rsa:4096 -x509 -days 365 -nodes -keyout $CA_KEY -out $CA_CRT -subj $SUBJ 2>&1 || - die "CA certificate was not created" -fi - -if [[ ! -f $SRV_KEY || ! -f $SRV_CRT ]]; then - openssl req -newkey rsa:4096 -nodes -keyout $SRV_KEY -out $SRV_REQ -subj $SUBJ 2>&1 || - die "Server certificate was not created" - - openssl x509 -req -days 365 -set_serial 01 -in $SRV_REQ -out $SRV_CRT -CA $CA_CRT -CAkey $CA_KEY \ - -extensions san -extfile <(printf "[san]\nsubjectAltName=DNS:nats.$LOCAL_DOMAIN") 2>&1 || { - rm $SRV_REQ - die "Server certificate was not signed by CA" - } - - rm $SRV_REQ -fi - -if [[ ! -f $CLI_KEY || ! -f $CLI_CRT ]]; then - openssl req -newkey rsa:4096 -nodes -keyout $CLI_KEY -out $CLI_REQ -subj $SUBJ 2>&1 || - die "Client certificate was not created" - - openssl x509 -req -days 365 -set_serial 01 -in $CLI_REQ -out $CLI_CRT -CA $CA_CRT -CAkey $CA_KEY 2>&1 || { - rm $CLI_REQ - die "Client certificate was not signed by CA" - } - - rm $CLI_REQ -fi diff --git a/services/nats/nats.conf b/services/nats/nats.conf deleted file mode 100644 index 0c7af0a..0000000 --- a/services/nats/nats.conf +++ /dev/null @@ -1,15 +0,0 @@ -port: 4222 -monitor_port: 8222 - -jetstream { - store_dir=nats - max_memory_store: 1GB - max_file_store: 2GB -} - -tls { - cert_file: /certs/server-cert.pem - key_file: /certs/server-key.pem - ca_file: /certs/ca-cert.pem - verify: true -} diff --git a/services/storage/cfg/config.yml b/services/storage/cfg/config.yml index 6b3e7a8..8672069 100644 --- a/services/storage/cfg/config.yml +++ b/services/storage/cfg/config.yml @@ -35,14 +35,6 @@ morph: # Common storage node settings node: attribute_0: "User-Agent:FrostFS/0.34" - notification: - enabled: true # Turn on object notification service - endpoint: "tls://nats.frostfs.devenv:4222" # Notification server endpoint - timeout: "6s" # Timeout for object notification client connection - default_topic: "test" # Default topic for object notifications if not found in object's meta - certificate: "/etc/frostfs-node/nats.tls.cert" # Path to TLS certificate - key: "/etc/frostfs-node/nats.tls.key" # Path to TLS key - ca: "/etc/frostfs-node/nats.ca.crt" # Path to optional CA certificate # Tree section tree: diff --git a/services/storage/docker-compose.yml b/services/storage/docker-compose.yml index fb10974..6a070be 100644 --- a/services/storage/docker-compose.yml +++ b/services/storage/docker-compose.yml @@ -18,9 +18,6 @@ services: - ./../../vendor/frostfs-cli:/frostfs-cli - ./cli-cfg.yml:/cli-cfg.yml - ./s04tls.crt:/etc/ssl/certs/s04tls.crt - - ../nats/client-cert.pem:/etc/frostfs-node/nats.tls.cert - - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key - - ../nats/ca-cert.pem:/etc/frostfs-node/nats.ca.crt - ./cfg:/etc/frostfs/storage stop_signal: SIGTERM stop_grace_period: 15s @@ -58,9 +55,6 @@ services: - ./../../vendor/frostfs-cli:/frostfs-cli - ./cli-cfg.yml:/cli-cfg.yml - ./s04tls.crt:/etc/ssl/certs/s04tls.crt - - ../nats/client-cert.pem:/etc/frostfs-node/nats.tls.cert - - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key - - ../nats/ca-cert.pem:/etc/frostfs-node/nats.ca.crt - ./cfg:/etc/frostfs/storage stop_signal: SIGTERM stop_grace_period: 15s @@ -98,9 +92,6 @@ services: - ./../../vendor/frostfs-cli:/frostfs-cli - ./cli-cfg.yml:/cli-cfg.yml - ./s04tls.crt:/etc/ssl/certs/s04tls.crt - - ../nats/client-cert.pem:/etc/frostfs-node/nats.tls.cert - - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key - - ../nats/ca-cert.pem:/etc/frostfs-node/nats.ca.crt - ./cfg:/etc/frostfs/storage stop_signal: SIGTERM stop_grace_period: 15s @@ -139,9 +130,6 @@ services: - ./cli-cfg.yml:/cli-cfg.yml - ./s04tls.crt:/tls.crt - ./s04tls.key:/tls.key - - ../nats/client-cert.pem:/etc/frostfs-node/nats.tls.cert - - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key - - ../nats/ca-cert.pem:/etc/frostfs-node/nats.ca.crt - ./cfg:/etc/frostfs/storage stop_signal: SIGTERM stop_grace_period: 15s From 10e5bed2afec13cd9803f012360c1bd7dcd184c7 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 31 Oct 2024 10:38:09 +0300 Subject: [PATCH 128/149] [#79] storage: Take User-Agent from NODE_VERSION Signed-off-by: Evgenii Stratonikov --- services/storage/cfg/config.yml | 4 ---- services/storage/docker-compose.yml | 4 ++++ 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/services/storage/cfg/config.yml b/services/storage/cfg/config.yml index 8672069..0e2526d 100644 --- a/services/storage/cfg/config.yml +++ b/services/storage/cfg/config.yml @@ -32,10 +32,6 @@ morph: - address: ws://morph-chain:30333/ws priority: 1 -# Common storage node settings -node: - attribute_0: "User-Agent:FrostFS/0.34" - # Tree section tree: enabled: true diff --git a/services/storage/docker-compose.yml b/services/storage/docker-compose.yml index 6a070be..785ce79 100644 --- a/services/storage/docker-compose.yml +++ b/services/storage/docker-compose.yml @@ -29,6 +29,7 @@ services: - FROSTFS_NODE_ADDRESSES=s01.${LOCAL_DOMAIN}:8080 - FROSTFS_GRPC_0_ENDPOINT=s01.${LOCAL_DOMAIN}:8080 - FROSTFS_CONTROL_GRPC_ENDPOINT=s01.${LOCAL_DOMAIN}:8081 + - FROSTFS_NODE_ATTRIBUTE_0=User-Agent:FrostFS/${NODE_VERSION} - FROSTFS_NODE_ATTRIBUTE_1=UN-LOCODE:RU MOW - FROSTFS_NODE_ATTRIBUTE_2=Price:22 healthcheck: @@ -66,6 +67,7 @@ services: - FROSTFS_NODE_ADDRESSES=s02.${LOCAL_DOMAIN}:8080 - FROSTFS_GRPC_0_ENDPOINT=s02.${LOCAL_DOMAIN}:8080 - FROSTFS_CONTROL_GRPC_ENDPOINT=s02.${LOCAL_DOMAIN}:8081 + - FROSTFS_NODE_ATTRIBUTE_0=User-Agent:FrostFS/${NODE_VERSION} - FROSTFS_NODE_ATTRIBUTE_1=UN-LOCODE:RU LED - FROSTFS_NODE_ATTRIBUTE_2=Price:33 healthcheck: @@ -103,6 +105,7 @@ services: - FROSTFS_NODE_ADDRESSES=s03.${LOCAL_DOMAIN}:8080 - FROSTFS_GRPC_0_ENDPOINT=s03.${LOCAL_DOMAIN}:8080 - FROSTFS_CONTROL_GRPC_ENDPOINT=s03.${LOCAL_DOMAIN}:8081 + - FROSTFS_NODE_ATTRIBUTE_0=User-Agent:FrostFS/${NODE_VERSION} - FROSTFS_NODE_ATTRIBUTE_1=UN-LOCODE:SE STO - FROSTFS_NODE_ATTRIBUTE_2=Price:11 healthcheck: @@ -146,6 +149,7 @@ services: - FROSTFS_GRPC_1_TLS_ENABLED=true - FROSTFS_GRPC_1_TLS_CERTIFICATE=/tls.crt - FROSTFS_GRPC_1_TLS_KEY=/tls.key + - FROSTFS_NODE_ATTRIBUTE_0=User-Agent:FrostFS/${NODE_VERSION} - FROSTFS_NODE_ATTRIBUTE_1=UN-LOCODE:FI HEL - FROSTFS_NODE_ATTRIBUTE_2=Price:44 healthcheck: From a0fdaebbf41d847badeac4e39961ef4cf95df3d0 Mon Sep 17 00:00:00 2001 From: Pavel Pogodaev Date: Fri, 25 Oct 2024 16:24:26 +0300 Subject: [PATCH 129/149] [#85] Add s3 lifecycler Signed-off-by: Pavel Pogodaev --- .env | 6 +++- .services | 1 + services/s3_lifecycler/.env | 1 + services/s3_lifecycler/.hosts | 1 + services/s3_lifecycler/.int_test.env | 1 + services/s3_lifecycler/cfg/config.yml | 42 +++++++++++++++++++++++ services/s3_lifecycler/docker-compose.yml | 38 ++++++++++++++++++++ services/s3_lifecycler/wallet.json | 30 ++++++++++++++++ 8 files changed, 119 insertions(+), 1 deletion(-) create mode 120000 services/s3_lifecycler/.env create mode 100644 services/s3_lifecycler/.hosts create mode 120000 services/s3_lifecycler/.int_test.env create mode 100644 services/s3_lifecycler/cfg/config.yml create mode 100644 services/s3_lifecycler/docker-compose.yml create mode 100644 services/s3_lifecycler/wallet.json diff --git a/.env b/.env index 6bb3728..a1a1586 100644 --- a/.env +++ b/.env @@ -28,9 +28,13 @@ REST_GW_VERSION=c9c85e90 REST_GW_IMAGE=truecloudlab/frostfs-rest-gw # S3 Gate -S3_GW_VERSION=0.30.4 +S3_GW_VERSION=0.31.0-rc.4 S3_GW_IMAGE=truecloudlab/frostfs-s3-gw +# Lifecycler +S3_LIFECYCLER_VERSION=0.1.3 +S3_LIFECYCLER_IMAGE=truecloudlab/frostfs-s3-lifecycler + # FrostFS LOCODE database LOCODE_DB_URL=https://git.frostfs.info/attachments/a2e8def7-52b6-49f1-89cd-a056712e8e54 #LOCODE_DB_PATH=/path/to/locode_db diff --git a/.services b/.services index d9f34c7..d165241 100644 --- a/.services +++ b/.services @@ -3,3 +3,4 @@ http_gate s3_gate rest_gate +s3_lifecycler diff --git a/services/s3_lifecycler/.env b/services/s3_lifecycler/.env new file mode 120000 index 0000000..c7360fb --- /dev/null +++ b/services/s3_lifecycler/.env @@ -0,0 +1 @@ +../../.env \ No newline at end of file diff --git a/services/s3_lifecycler/.hosts b/services/s3_lifecycler/.hosts new file mode 100644 index 0000000..8eb1796 --- /dev/null +++ b/services/s3_lifecycler/.hosts @@ -0,0 +1 @@ +IPV4_PREFIX.84 lifecycler.LOCAL_DOMAIN diff --git a/services/s3_lifecycler/.int_test.env b/services/s3_lifecycler/.int_test.env new file mode 120000 index 0000000..582b6a2 --- /dev/null +++ b/services/s3_lifecycler/.int_test.env @@ -0,0 +1 @@ +../../.int_test.env \ No newline at end of file diff --git a/services/s3_lifecycler/cfg/config.yml b/services/s3_lifecycler/cfg/config.yml new file mode 100644 index 0000000..2555980 --- /dev/null +++ b/services/s3_lifecycler/cfg/config.yml @@ -0,0 +1,42 @@ +logger: + level: debug + +prometheus: + enabled: true + address: :9090 + +lifecycle: + job_fetcher_buffer: 1000 + executor_pool_size: 100 + +frostfs: + stream_timeout: 10s + connect_timeout: 10s + healthcheck_timeout: 15s + rebalance_interval: 60s + pool_error_threshold: 100 + tree_pool_max_attempts: 4 + +credential: + use: wallets + source: + wallets: + - path: /wallet.json + address: NTt1rxvmEDxEuuogLxs2xgxA71qhVaUcN7 + passphrase: "cycle" + - path: /user-wallet.json + address: NbUgTSFvPmsRxmGeWpuuGeJUoRoi6PErcM + passphrase: "" + +morph: + reconnect_clients_interval: 30s + dial_timeout: 5s + contract: + netmap: netmap.frostfs + frostfsid: frostfsid.frostfs + container: container.frostfs + +# Wallet configuration +wallet: + path: /wallet.json # Path to wallet + passphrase: "cycle" # Passphrase to decrypt wallet diff --git a/services/s3_lifecycler/docker-compose.yml b/services/s3_lifecycler/docker-compose.yml new file mode 100644 index 0000000..3456d0c --- /dev/null +++ b/services/s3_lifecycler/docker-compose.yml @@ -0,0 +1,38 @@ +--- + +version: "2.4" +services: + s3_lifecycler: + image: ${S3_LIFECYCLER_IMAGE}:${S3_LIFECYCLER_VERSION} + domainname: ${LOCAL_DOMAIN} + hostname: s3_lifecycler + container_name: s3_lifecycler + restart: on-failure + networks: + s3_lifecycler_int: + internet: + ipv4_address: ${IPV4_PREFIX}.84 + volumes: + - ./wallet.json:/wallet.json + - ./../../vendor/hosts:/etc/hosts + - ./cfg:/etc/frostfs/s3-lifecycler + - ./../../wallets/wallet.json:/user-wallet.json + stop_signal: SIGKILL + env_file: [ ".env", ".int_test.env" ] + command: [ "frostfs-s3-lifecycler", "--config", "/etc/frostfs/s3-lifecycler/config.yml" ] + environment: + - S3_LIFECYCLER_MORPH_RPC_ENDPOINT_0_ADDRESS=ws://morph-chain:30333/ws + - S3_LIFECYCLER_FROSTFS_PEERS_0_ADDRESS=s01.${LOCAL_DOMAIN}:8080 + - S3_LIFECYCLER_FROSTFS_PEERS_0_WEIGHT=0.2 + - S3_LIFECYCLER_FROSTFS_PEERS_1_ADDRESS=s02.${LOCAL_DOMAIN}:8080 + - S3_LIFECYCLER_FROSTFS_PEERS_1_WEIGHT=0.2 + - S3_LIFECYCLER_FROSTFS_PEERS_2_ADDRESS=s03.${LOCAL_DOMAIN}:8080 + - S3_LIFECYCLER_FROSTFS_PEERS_2_WEIGHT=0.2 + - S3_LIFECYCLER_FROSTFS_PEERS_3_ADDRESS=s04.${LOCAL_DOMAIN}:8080 + - S3_LIFECYCLER_FROSTFS_PEERS_3_WEIGHT=0.2 + +networks: + s3_lifecycler_int: + internet: + external: true + name: basenet_internet diff --git a/services/s3_lifecycler/wallet.json b/services/s3_lifecycler/wallet.json new file mode 100644 index 0000000..ff8f34e --- /dev/null +++ b/services/s3_lifecycler/wallet.json @@ -0,0 +1,30 @@ +{ + "version": "1.0", + "accounts": [ + { + "address": "NTt1rxvmEDxEuuogLxs2xgxA71qhVaUcN7", + "key": "6PYR3XurAyTzVeDG5WV2Z8vnGdySw3mTLuKjr6Nwo7tae64SJ7XjZSMMPQ", + "label": "lifecycler", + "contract": { + "script": "DCED9z0M+WSGfXZGxYLj1yYwmgxJXE/kNA4+oWNi0q1uKCdBVuezJw==", + "parameters": [ + { + "name": "parameter0", + "type": "Signature" + } + ], + "deployed": false + }, + "lock": false, + "isDefault": false + } + ], + "scrypt": { + "n": 16384, + "r": 8, + "p": 8 + }, + "extra": { + "Tokens": null + } +} From 326578f0ab2ead1423e137b012ed7dd0e3707f33 Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Wed, 6 Nov 2024 11:31:27 +0300 Subject: [PATCH 130/149] [#90] Stop using obsolete .github directory This commit is a part of multi-repo cleanup effort: https://git.frostfs.info/TrueCloudLab/frostfs-infra/issues/136 Signed-off-by: Vitaliy Potyarkin --- .dockerignore | 1 - {.github => .forgejo}/ISSUE_TEMPLATE/bug_report.md | 0 {.github => .forgejo}/ISSUE_TEMPLATE/config.yml | 0 {.github => .forgejo}/ISSUE_TEMPLATE/feature_request.md | 0 {.github => .forgejo}/logo.svg | 0 .github/CODEOWNERS | 1 - CODEOWNERS | 1 + README.md | 2 +- 8 files changed, 2 insertions(+), 3 deletions(-) rename {.github => .forgejo}/ISSUE_TEMPLATE/bug_report.md (100%) rename {.github => .forgejo}/ISSUE_TEMPLATE/config.yml (100%) rename {.github => .forgejo}/ISSUE_TEMPLATE/feature_request.md (100%) rename {.github => .forgejo}/logo.svg (100%) delete mode 100644 .github/CODEOWNERS create mode 100644 CODEOWNERS diff --git a/.dockerignore b/.dockerignore index 2f9d4b4..3867906 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,4 @@ .docker -.github .forgejo vendor tmp diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.forgejo/ISSUE_TEMPLATE/bug_report.md similarity index 100% rename from .github/ISSUE_TEMPLATE/bug_report.md rename to .forgejo/ISSUE_TEMPLATE/bug_report.md diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.forgejo/ISSUE_TEMPLATE/config.yml similarity index 100% rename from .github/ISSUE_TEMPLATE/config.yml rename to .forgejo/ISSUE_TEMPLATE/config.yml diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.forgejo/ISSUE_TEMPLATE/feature_request.md similarity index 100% rename from .github/ISSUE_TEMPLATE/feature_request.md rename to .forgejo/ISSUE_TEMPLATE/feature_request.md diff --git a/.github/logo.svg b/.forgejo/logo.svg similarity index 100% rename from .github/logo.svg rename to .forgejo/logo.svg diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS deleted file mode 100644 index a4413b0..0000000 --- a/.github/CODEOWNERS +++ /dev/null @@ -1 +0,0 @@ -* @alexvanin @fyrchik diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 0000000..d28053e --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1 @@ +.* @alexvanin @fyrchik diff --git a/README.md b/README.md index ad57655..8e4b714 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@

-FrostFS logo +FrostFS logo

FrostFS local Development and Testing environment From df6859d2463d768691d0d65d85961748e2309802 Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Fri, 13 Dec 2024 18:10:57 +0300 Subject: [PATCH 131/149] [#92] docs: Update contract list Contract list got outdated: it was mentioning NeoFS, Audit and Reputation Signed-off-by: Vitaliy Potyarkin --- docs/morph.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/morph.md b/docs/morph.md index 56edfbc..d713bf9 100644 --- a/docs/morph.md +++ b/docs/morph.md @@ -4,14 +4,16 @@ A single-node N3 privnet deployment, running on Contracts deployed: - Alphabet (AZ) [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/alphabet) -- Audit [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/audit) - Balance [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/balance) - Container [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/container) +- FrostFS [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/frostfs) +- FrostFSID [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/frostfsid) +- NNS [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/nns) - Netmap [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/netmap) -- NeoFSID [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/neofsid) +- Policy [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/policy) +- Processing [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/processing) - Proxy [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/proxy) -- Reputation [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/reputation) - + RPC available at `http://morph-chain.frostfs.devenv:30333`. ## .env settings From 0f9000bce6aedc9f1f376275ced6bc6a38cf6994 Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Tue, 10 Dec 2024 15:42:13 +0300 Subject: [PATCH 132/149] [#91] Refine CODEOWNERS settings Signed-off-by: Vitaliy Potyarkin --- CODEOWNERS | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index d28053e..3b080ce 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1 +1,3 @@ -.* @alexvanin @fyrchik +.* @alexvanin @fyrchik +.forgejo/.* @potyarkin +Makefile @potyarkin From 98484b97396d71d5cad90cc4ba504a743e9ba843 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 20 Dec 2024 14:58:00 +0300 Subject: [PATCH 133/149] [#93] Update neo-go to the latest version used by frostfs-node Signed-off-by: Alex Vanin --- .env | 2 +- services/morph_chain/protocol.privnet.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.env b/.env index a1a1586..4727fb1 100644 --- a/.env +++ b/.env @@ -8,7 +8,7 @@ BASTION_VERSION=10 BASTION_IMAGE=debian # NeoGo privnet -NEOGO_VERSION=0.104.0 +NEOGO_VERSION=0.106.3 NEOGO_IMAGE=nspccdev/neo-go # FrostFS InnerRing nodes diff --git a/services/morph_chain/protocol.privnet.yml b/services/morph_chain/protocol.privnet.yml index f1cc754..7cea631 100644 --- a/services/morph_chain/protocol.privnet.yml +++ b/services/morph_chain/protocol.privnet.yml @@ -11,6 +11,7 @@ ProtocolConfiguration: VerifyTransactions: true StateRootInHeader: true P2PSigExtensions: true + Hardforks: {} ApplicationConfiguration: SkipBlockVerification: false From e1b8fe791902f87b1347b3f2e948e4fc0fc583f1 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 20 Dec 2024 15:17:21 +0300 Subject: [PATCH 134/149] [#93] Update frostfs-node to the latest version Signed-off-by: Alex Vanin --- .env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.env b/.env index 4727fb1..a94341c 100644 --- a/.env +++ b/.env @@ -12,11 +12,11 @@ NEOGO_VERSION=0.106.3 NEOGO_IMAGE=nspccdev/neo-go # FrostFS InnerRing nodes -IR_VERSION=0.42.9 +IR_VERSION=0.44.4 IR_IMAGE=git.frostfs.info/truecloudlab/frostfs-ir # FrostFS Storage nodes -NODE_VERSION=0.42.9 +NODE_VERSION=0.44.4 NODE_IMAGE=git.frostfs.info/truecloudlab/frostfs-storage # HTTP Gate From 4db8ca356d5ebe8490ddfad69892da46b80e0ab6 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 20 Dec 2024 15:17:54 +0300 Subject: [PATCH 135/149] [#93] Remove unused env variables These variables were used when binaries were available in public storage network Signed-off-by: Alex Vanin --- .env | 2 -- 1 file changed, 2 deletions(-) diff --git a/.env b/.env index a94341c..aa96d95 100644 --- a/.env +++ b/.env @@ -44,12 +44,10 @@ FROSTFS_CLI_URL=https://git.frostfs.info/TrueCloudLab/frostfs-node/releases/down #FROSTFS_CLI_PATH=/path/to/frostfs-cli-binary # FrostFS ADM tool binary -FROSTFS_ADM_VERSION=498f9955ea FROSTFS_ADM_URL=https://git.frostfs.info/TrueCloudLab/frostfs-node/releases/download/v${NODE_VERSION}/frostfs-adm #FROSTFS_ADM_PATH=/path/to/frostfs-adm-binary # Compiled FrostFS Smart Contracts -FROSTFS_CONTRACTS_VERSION=694daebb19 FROSTFS_CONTRACTS_URL=https://git.frostfs.info/TrueCloudLab/frostfs-contract/releases/download/v0.19.2/frostfs-contract-v0.19.2.tar.gz #FROSTFS_CONTRACTS_PATH=/path/to/unpacked/frostfs-contracts-dir From 6a5817e15c1031b1aa06ebcb77dd8086be3079ee Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 20 Dec 2024 16:08:25 +0300 Subject: [PATCH 136/149] [#93] Register storage nodes in proxy contract during bootstrap This allows to send APE requests to the nodes right after `make up` command Signed-off-by: Alex Vanin --- Makefile | 2 ++ services/morph_chain/docker-compose.yml | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/Makefile b/Makefile index c4a6c48..58a12e0 100644 --- a/Makefile +++ b/Makefile @@ -85,6 +85,8 @@ up/bootstrap: get vendor/hosts echo "Transfer GAS to wallet $${f}" \ && ./vendor/frostfs-adm -c frostfs-adm.yml morph refill-gas --storage-wallet $${f} --gas 10.0 \ || die "Failed to transfer GAS to alphabet wallets"; \ + echo "Register storage wallet $${f} in proxy contract" \ + && ./vendor/frostfs-adm morph proxy-add-account --config frostfs-adm.yml --account=`docker container exec morph_chain neo-go wallet dump-keys -w /wallets/storage/$${f##*/} | head -1 | awk '{print $1}'` || die "Couldn't set storage allet as proxy wallet" done @echo "Create frostfsid subject for ./wallets/wallet.json"; \ if [ -n "$$(./vendor/frostfs-adm -c frostfs-adm.yml morph frostfsid list-subjects --namespace '')" ]; then \ diff --git a/services/morph_chain/docker-compose.yml b/services/morph_chain/docker-compose.yml index 2c2facb..6b83716 100644 --- a/services/morph_chain/docker-compose.yml +++ b/services/morph_chain/docker-compose.yml @@ -20,6 +20,10 @@ services: - ./../../vendor/hosts:/etc/hosts - ./../../wallets/wallet.json:/wallets/wallet.json - ./../s3_gate/wallet.json:/wallets/s3-wallet.json + - ./../storage/wallet01.json:/wallets/storage/wallet01.json + - ./../storage/wallet02.json:/wallets/storage/wallet02.json + - ./../storage/wallet03.json:/wallets/storage/wallet03.json + - ./../storage/wallet04.json:/wallets/storage/wallet04.json - chains:/chains networks: From d29d50a002c672a9ddfc6f6950dcba97e45344bc Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 20 Dec 2024 16:12:06 +0300 Subject: [PATCH 137/149] [#93] Update frostfs-contract to the latest version Signed-off-by: Alex Vanin --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index aa96d95..fe484c6 100644 --- a/.env +++ b/.env @@ -48,7 +48,7 @@ FROSTFS_ADM_URL=https://git.frostfs.info/TrueCloudLab/frostfs-node/releases/down #FROSTFS_ADM_PATH=/path/to/frostfs-adm-binary # Compiled FrostFS Smart Contracts -FROSTFS_CONTRACTS_URL=https://git.frostfs.info/TrueCloudLab/frostfs-contract/releases/download/v0.19.2/frostfs-contract-v0.19.2.tar.gz +FROSTFS_CONTRACTS_URL=https://git.frostfs.info/TrueCloudLab/frostfs-contract/releases/download/v0.21.1/frostfs-contract-v0.21.1.tar.gz #FROSTFS_CONTRACTS_PATH=/path/to/unpacked/frostfs-contracts-dir # Jaeger tracing From b08bb663f93b0aca97905e6bf8d4796cd8570328 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 20 Dec 2024 16:28:55 +0300 Subject: [PATCH 138/149] [#93] Update gateway components Signed-off-by: Alex Vanin --- .env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.env b/.env index fe484c6..7a58e35 100644 --- a/.env +++ b/.env @@ -20,7 +20,7 @@ NODE_VERSION=0.44.4 NODE_IMAGE=git.frostfs.info/truecloudlab/frostfs-storage # HTTP Gate -HTTP_GW_VERSION=0.30.2 +HTTP_GW_VERSION=0.32.0 HTTP_GW_IMAGE=truecloudlab/frostfs-http-gw # REST Gate @@ -28,7 +28,7 @@ REST_GW_VERSION=c9c85e90 REST_GW_IMAGE=truecloudlab/frostfs-rest-gw # S3 Gate -S3_GW_VERSION=0.31.0-rc.4 +S3_GW_VERSION=0.32.0 S3_GW_IMAGE=truecloudlab/frostfs-s3-gw # Lifecycler From d34d842700821a4eb8bcfb650734acb196957c64 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 20 Dec 2024 16:38:39 +0300 Subject: [PATCH 139/149] [#93] Remove frostfs-rest-gw from dev-env This repo is being archived and not maintained at the moment. Signed-off-by: Alex Vanin --- .env | 4 ---- .services | 1 - services/rest_gate/.env | 1 - services/rest_gate/.hosts | 1 - services/rest_gate/.int_test.env | 1 - services/rest_gate/cfg/config.yml | 12 ---------- services/rest_gate/docker-compose.yml | 32 --------------------------- services/rest_gate/wallet.json | 30 ------------------------- 8 files changed, 82 deletions(-) delete mode 120000 services/rest_gate/.env delete mode 100644 services/rest_gate/.hosts delete mode 120000 services/rest_gate/.int_test.env delete mode 100644 services/rest_gate/cfg/config.yml delete mode 100644 services/rest_gate/docker-compose.yml delete mode 100644 services/rest_gate/wallet.json diff --git a/.env b/.env index 7a58e35..824a4db 100644 --- a/.env +++ b/.env @@ -23,10 +23,6 @@ NODE_IMAGE=git.frostfs.info/truecloudlab/frostfs-storage HTTP_GW_VERSION=0.32.0 HTTP_GW_IMAGE=truecloudlab/frostfs-http-gw -# REST Gate -REST_GW_VERSION=c9c85e90 -REST_GW_IMAGE=truecloudlab/frostfs-rest-gw - # S3 Gate S3_GW_VERSION=0.32.0 S3_GW_IMAGE=truecloudlab/frostfs-s3-gw diff --git a/.services b/.services index d165241..ccc96f8 100644 --- a/.services +++ b/.services @@ -2,5 +2,4 @@ # Will start from top to bottom and stop in reverse http_gate s3_gate -rest_gate s3_lifecycler diff --git a/services/rest_gate/.env b/services/rest_gate/.env deleted file mode 120000 index c7360fb..0000000 --- a/services/rest_gate/.env +++ /dev/null @@ -1 +0,0 @@ -../../.env \ No newline at end of file diff --git a/services/rest_gate/.hosts b/services/rest_gate/.hosts deleted file mode 100644 index ee7578e..0000000 --- a/services/rest_gate/.hosts +++ /dev/null @@ -1 +0,0 @@ -IPV4_PREFIX.83 rest.LOCAL_DOMAIN diff --git a/services/rest_gate/.int_test.env b/services/rest_gate/.int_test.env deleted file mode 120000 index 582b6a2..0000000 --- a/services/rest_gate/.int_test.env +++ /dev/null @@ -1 +0,0 @@ -../../.int_test.env \ No newline at end of file diff --git a/services/rest_gate/cfg/config.yml b/services/rest_gate/cfg/config.yml deleted file mode 100644 index 0acdad3..0000000 --- a/services/rest_gate/cfg/config.yml +++ /dev/null @@ -1,12 +0,0 @@ -prometheus: - enabled: true - address: :9090 - -server: - # The IP and port to listen on. - listen-address: 0.0.0.0:8090 - -# Wallet settings -wallet: - path: /wallet.json # Path to wallet - passphrase: one # Password to decrypt wallet diff --git a/services/rest_gate/docker-compose.yml b/services/rest_gate/docker-compose.yml deleted file mode 100644 index 09cfe38..0000000 --- a/services/rest_gate/docker-compose.yml +++ /dev/null @@ -1,32 +0,0 @@ ---- - -services: - rest_gate: - image: ${REST_GW_IMAGE}:${REST_GW_VERSION} - domainname: ${LOCAL_DOMAIN} - hostname: rest - container_name: rest_gate - restart: on-failure - networks: - rest_gate_int: - internet: - ipv4_address: ${IPV4_PREFIX}.83 - volumes: - - ./wallet.json:/wallet.json - - ./../../vendor/hosts:/etc/hosts - - ./cfg:/etc/frostfs/rest - stop_signal: SIGTERM - stop_grace_period: 15s - env_file: [ ".env", ".int_test.env" ] - command: [ "frostfs-rest-gw", "--config", "/etc/frostfs/rest/config.yml" ] - environment: - - REST_GW_POOL_PEERS_0_ADDRESS=s01.${LOCAL_DOMAIN}:8080 - - REST_GW_POOL_PEERS_1_ADDRESS=s02.${LOCAL_DOMAIN}:8080 - - REST_GW_POOL_PEERS_2_ADDRESS=s03.${LOCAL_DOMAIN}:8080 - - REST_GW_POOL_PEERS_3_ADDRESS=s04.${LOCAL_DOMAIN}:8080 - -networks: - rest_gate_int: - internet: - external: true - name: basenet_internet diff --git a/services/rest_gate/wallet.json b/services/rest_gate/wallet.json deleted file mode 100644 index 2b60501..0000000 --- a/services/rest_gate/wallet.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "version": "3.0", - "accounts": [ - { - "address": "NPFCqWHfi9ixCJRu7DABRbVfXRbkSEr9Vo", - "key": "6PYTAGjdaeicUDPqGv9mmgwb9kTwimWJJmmfNqJSDGH9qM79zSRcL9oHiB", - "label": "REST Gateway", - "contract": { - "script": "DCECcuPzZCZ2VyDsm2jKEOMnU6xEWO2bF1dvOvBWTDFYB1ZBVuezJw==", - "parameters": [ - { - "name": "parameter0", - "type": "Signature" - } - ], - "deployed": false - }, - "lock": false, - "isDefault": false - } - ], - "scrypt": { - "n": 16384, - "r": 8, - "p": 8 - }, - "extra": { - "Tokens": null - } -} From c4d4fecb89ee1fbd2f7f02254f2ebf8264b47553 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 27 Dec 2024 10:42:19 +0300 Subject: [PATCH 140/149] [#96] s3_lifecycler: Fix docker-compose warnings Signed-off-by: Evgenii Stratonikov --- services/s3_lifecycler/docker-compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/services/s3_lifecycler/docker-compose.yml b/services/s3_lifecycler/docker-compose.yml index 3456d0c..2fffa50 100644 --- a/services/s3_lifecycler/docker-compose.yml +++ b/services/s3_lifecycler/docker-compose.yml @@ -1,6 +1,5 @@ --- -version: "2.4" services: s3_lifecycler: image: ${S3_LIFECYCLER_IMAGE}:${S3_LIFECYCLER_VERSION} From 90147c71081361e4348f0a041848e7ccf08d1235 Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Fri, 27 Dec 2024 09:33:04 +0300 Subject: [PATCH 141/149] [#95] Output errors during `make clean` It happens that a volume may not be deleted during `make clean`. For example, if the volume is being used by a container. If this happens, there are no errors printed to stdout. And old volumes may cause errors during subsequent `make up` Signed-off-by: Nikita Zinkevich --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 58a12e0..f24ec9d 100644 --- a/Makefile +++ b/Makefile @@ -159,7 +159,7 @@ clean: vols=`docker-compose -f services/$${svc}/docker-compose.yml config --volumes` if [[ ! -z "$${vols}" ]]; then for vol in $${vols}; do - docker volume rm -f "$${svc}_$${vol}" 2> /dev/null + docker volume rm -f "$${svc}_$${vol}" done fi done From 5471dbfc0e57babdc5f95c1c788fbd3a61c9e9d7 Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Thu, 9 Jan 2025 11:23:42 +0300 Subject: [PATCH 142/149] [#98] s3_gate: Fix custom user wallets folder creation during compose up Make custom wallets volume to point to the `wallets` directory in the project's root. Signed-off-by: Nikita Zinkevich --- README.md | 9 +++++---- services/s3_gate/docker-compose.yml | 6 ++---- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 8e4b714..415885e 100644 --- a/README.md +++ b/README.md @@ -143,7 +143,7 @@ Registers user wallet and issues s3 credentials. Usage and default parameter values: ```sh -make s3cred [password=""] [contract_password=s3] [wallet=/user_wallet.json] [gate_public_key=0313b1ac3a8076e155a7e797b24f0b650cccad5941ea59d7cfd51a024a8b2a06bf] +make s3cred [password=""] [contract_password=s3] [wallet=""] [gate_public_key=0313b1ac3a8076e155a7e797b24f0b650cccad5941ea59d7cfd51a024a8b2a06bf] ``` As soon as the storage node is in the network map (see above) you can generate S3 @@ -159,8 +159,9 @@ $ make s3cred "container_id": "EXArWh8x1zeHG3851s1RtoCo7dowxF6rhLGA15nbMffT" } ``` -Running without any parameters will result in defaults which are based on the private key from -`/user-wallet.json` file and `/wallet.json` contract wallet. +Running without any parameters results in defaults which are based on the private key from +`/wallets/wallet.json` user wallet and `/wallet.json` contract wallet. +If `wallet` parameter is set, gate searches custom user wallet file in `/wallets` directory. Now let's configure an S3 client (AWS CLI will be used as example): @@ -172,7 +173,7 @@ Default region name []: us-east-1 Default output format []: json ``` -If you need to create credentials for different users, put user wallets to `wallets` dir and specify them via `wallet` parameter. +If you need to create credentials for different users, put user wallet to `wallets` dir and specify it via `wallet` parameter. Pass wallet password in `password` parameter if it's not default. The same is for `contract_wallet` and `gate_public_key` params. ```sh diff --git a/services/s3_gate/docker-compose.yml b/services/s3_gate/docker-compose.yml index f7fc28d..3c1f9f6 100644 --- a/services/s3_gate/docker-compose.yml +++ b/services/s3_gate/docker-compose.yml @@ -14,10 +14,8 @@ services: volumes: # Gate wallet - ./wallet.json:/wallet.json - # Custom user wallets - - ./wallets:/wallets - # Default user wallet - - ./../../wallets/wallet.json:/wallets/wallet.json + # Folder for custom user wallets + - ./../../wallets/:/wallets/ - ./tls.key:/tls.key - ./tls.crt:/tls.crt - ./../../vendor/hosts:/etc/hosts From 0981202d6bcc0c454d55cfb1cb5c9b355e217170 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 21 Mar 2025 18:47:17 +0300 Subject: [PATCH 143/149] [#100] Use docker images as vendor binary source Signed-off-by: Alex Vanin --- .env | 6 ++++-- services/ir/artifacts.mk | 8 +++----- services/morph_chain/artifacts.mk | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.env b/.env index 824a4db..053f179 100644 --- a/.env +++ b/.env @@ -36,11 +36,13 @@ LOCODE_DB_URL=https://git.frostfs.info/attachments/a2e8def7-52b6-49f1-89cd-a0567 #LOCODE_DB_PATH=/path/to/locode_db # FrostFS CLI binary -FROSTFS_CLI_URL=https://git.frostfs.info/TrueCloudLab/frostfs-node/releases/download/v${NODE_VERSION}/frostfs-cli +FROSTFS_CLI_VERSION=0.44.4 +FROSTFS_CLI_IMAGE=git.frostfs.info/truecloudlab/frostfs-cli #FROSTFS_CLI_PATH=/path/to/frostfs-cli-binary # FrostFS ADM tool binary -FROSTFS_ADM_URL=https://git.frostfs.info/TrueCloudLab/frostfs-node/releases/download/v${NODE_VERSION}/frostfs-adm +FROSTFS_ADM_VERSION=0.44.4 +FROSTFS_ADM_IMAGE=git.frostfs.info/truecloudlab/frostfs-adm #FROSTFS_ADM_PATH=/path/to/frostfs-adm-binary # Compiled FrostFS Smart Contracts diff --git a/services/ir/artifacts.mk b/services/ir/artifacts.mk index 0cdbdbb..1c2cf5d 100644 --- a/services/ir/artifacts.mk +++ b/services/ir/artifacts.mk @@ -30,11 +30,9 @@ get.cli: @mkdir -p ./vendor ifeq (${FROSTFS_CLI_PATH},) - @echo "⇒ Download FrostFS CLI binary from ${FROSTFS_CLI_URL}" - @curl \ - -ksSL "${FROSTFS_CLI_URL}" \ - -o ${FROSTFS_CLI_FILE} - @chmod +x ${FROSTFS_CLI_FILE} + echo "⇒ Download FrostFS CLI binary from ${FROSTFS_CLI_IMAGE}:${FROSTFS_CLI_VERSION}" + $(shell docker cp `docker create --name tmp ${FROSTFS_CLI_IMAGE}:${FROSTFS_CLI_VERSION}`:/bin/frostfs-cli ${FROSTFS_CLI_FILE} && docker rm tmp >/dev/null) + chmod +x ${FROSTFS_CLI_FILE} else @echo "⇒ Copy local binary from ${FROSTFS_CLI_PATH}" @cp ${FROSTFS_CLI_PATH} ${FROSTFS_CLI_FILE} diff --git a/services/morph_chain/artifacts.mk b/services/morph_chain/artifacts.mk index 77a7ae3..4a4504a 100644 --- a/services/morph_chain/artifacts.mk +++ b/services/morph_chain/artifacts.mk @@ -23,8 +23,8 @@ get.adm: FROSTFS_ADM_DEST=./vendor/frostfs-adm get.adm: ifeq (${FROSTFS_ADM_PATH},) - @echo "⇒ Download FrostFS ADM binary from ${FROSTFS_ADM_URL}" - @curl -skSL ${FROSTFS_ADM_URL} -o ${FROSTFS_ADM_DEST} + @echo "⇒ Download FrostFS ADM binary from ${FROSTFS_ADM_IMAGE}:${FROSTFS_ADM_VERSION}" + $(shell docker cp `docker create --name tmp ${FROSTFS_ADM_IMAGE}:${FROSTFS_ADM_VERSION}`:/bin/frostfs-adm ${FROSTFS_ADM_DEST} && docker rm tmp >/dev/null) @chmod +x ${FROSTFS_ADM_DEST} else @echo "⇒ Copy frostfs-adm binary from ${FROSTFS_ADM_PATH}" From ea91d8823274fa5eac9bd86a49369c6267a3fc6f Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 21 Mar 2025 18:54:24 +0300 Subject: [PATCH 144/149] [#100] Add target to prepare storage for service components Signed-off-by: Alex Vanin --- Makefile | 22 ++++++++++++++++++++-- cli-cfg.yml | 3 +++ services/morph_chain/docker-compose.yml | 1 + wallets/system-wallet.json | 1 + 4 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 cli-cfg.yml create mode 100644 wallets/system-wallet.json diff --git a/Makefile b/Makefile index f24ec9d..b3b342d 100644 --- a/Makefile +++ b/Makefile @@ -58,9 +58,8 @@ get: $(foreach SVC, $(GET_SVCS), get.$(SVC)) # Start environment .PHONY: up -up: up/basic +up: up/basic up/pre-services @$(foreach SVC, $(START_SVCS), $(shell docker-compose -f services/$(SVC)/docker-compose.yml up -d)) - ./vendor/frostfs-adm morph proxy-add-account --config frostfs-adm.yml --account=`docker container exec morph_chain neo-go wallet dump-keys -w /wallets/s3-wallet.json | head -1 | awk '{print $1}'` || die "Couldn't set s3-gw wallet as proxy wallet" @echo "Full FrostFS Developer Environment is ready" # Build up FrostFS @@ -98,6 +97,25 @@ up/bootstrap: get vendor/hosts || die "Failed to create subject for the wallet"; \ fi echo "FrostFS sidechain environment is deployed" + +# Prepare to start services +.PHONY: up/pre-services +up/pre-services: + @source ./bin/helper.sh + @echo "Prepare storage for services"; \ + if [ -z "$$(./vendor/frostfs-cli -c cli-cfg.yml container list)" ]; then \ + subj_key=`docker container exec -it morph_chain neo-go wallet dump-keys -w /wallets/system-wallet.json | tail -1 | tr -d ' \r\n'` \ + && echo "Subject key: $${subj_key}" \ + && ./vendor/frostfs-adm -c frostfs-adm.yml morph frostfsid create-subject --namespace "" --subject-key $${subj_key} --subject-name system \ + || die "Failed to create subject for system wallet"; \ + proxy_acc=`docker container exec -it morph_chain neo-go wallet dump-keys -w /wallets/system-wallet.json | head -1 | cut -d" " -f1` \ + && echo "Proxy acc: $${proxy_acc}" \ + && ./vendor/frostfs-adm morph proxy-add-account --config frostfs-adm.yml --account=$${proxy_acc} || die "Failed to register S3 gateway as proxy acc"; \ + cid=`./vendor/frostfs-cli -c cli-cfg.yml container create -p "REP 4" --nns-name "cors" --nns-zone "container" --await | grep CID | cut -d" " -f2` \ + && echo "CORS Container: $${cid}" \ + && ./vendor/frostfs-cli -c cli-cfg.yml ape-manager add --target-type container --target-name $${cid} --rule "allow Object.* *" || die "Failed to create CORS container" + fi + @echo "Storage is prepared"; # Build up certain service .PHONY: up/% diff --git a/cli-cfg.yml b/cli-cfg.yml new file mode 100644 index 0000000..9a5c7eb --- /dev/null +++ b/cli-cfg.yml @@ -0,0 +1,3 @@ +wallet: ./wallets/system-wallet.json +password: "" +rpc-endpoint: s01.frostfs.devenv:8080 diff --git a/services/morph_chain/docker-compose.yml b/services/morph_chain/docker-compose.yml index 6b83716..c3a1481 100644 --- a/services/morph_chain/docker-compose.yml +++ b/services/morph_chain/docker-compose.yml @@ -19,6 +19,7 @@ services: - ./config.yml:/wallets/config.yml - ./../../vendor/hosts:/etc/hosts - ./../../wallets/wallet.json:/wallets/wallet.json + - ./../../wallets/system-wallet.json:/wallets/system-wallet.json - ./../s3_gate/wallet.json:/wallets/s3-wallet.json - ./../storage/wallet01.json:/wallets/storage/wallet01.json - ./../storage/wallet02.json:/wallets/storage/wallet02.json diff --git a/wallets/system-wallet.json b/wallets/system-wallet.json new file mode 100644 index 0000000..2f57db1 --- /dev/null +++ b/wallets/system-wallet.json @@ -0,0 +1 @@ +{"version":"1.0","accounts":[{"address":"NQijiVKHbL22PfF2AJQukv1CX75itxgzht","key":"6PYQKrpme57VqaucxuF7dDoSZRRA8d94oatHcScqhiFBauCXQvFDaYwEWa","label":"","contract":{"script":"DCEDRdLtpFIWeYyI7doTKRhIl4qYjaybGDveTyGpbqjsLZNBVuezJw==","parameters":[{"name":"parameter0","type":"Signature"}],"deployed":false},"lock":false,"isDefault":false}],"scrypt":{"n":16384,"r":8,"p":8},"extra":{"Tokens":null}} \ No newline at end of file From e0141c11545ecf107bf50fe5da293a1390ba2695 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 21 Mar 2025 18:55:43 +0300 Subject: [PATCH 145/149] [#100] Update components to the latest versions Signed-off-by: Alex Vanin --- .env | 16 ++++++++-------- services/http_gate/cfg/config.yml | 3 +++ services/s3_gate/cfg/config.yml | 3 +++ 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/.env b/.env index 053f179..dec29ce 100644 --- a/.env +++ b/.env @@ -12,20 +12,20 @@ NEOGO_VERSION=0.106.3 NEOGO_IMAGE=nspccdev/neo-go # FrostFS InnerRing nodes -IR_VERSION=0.44.4 +IR_VERSION=0.45.0-rc.3 IR_IMAGE=git.frostfs.info/truecloudlab/frostfs-ir # FrostFS Storage nodes -NODE_VERSION=0.44.4 +NODE_VERSION=0.45.0-rc.3 NODE_IMAGE=git.frostfs.info/truecloudlab/frostfs-storage # HTTP Gate -HTTP_GW_VERSION=0.32.0 -HTTP_GW_IMAGE=truecloudlab/frostfs-http-gw +HTTP_GW_VERSION=0.33.0-rc.2 +HTTP_GW_IMAGE=git.frostfs.info/truecloudlab/frostfs-http-gw # S3 Gate -S3_GW_VERSION=0.32.0 -S3_GW_IMAGE=truecloudlab/frostfs-s3-gw +S3_GW_VERSION=0.33.0-rc.2 +S3_GW_IMAGE=git.frostfs.info/truecloudlab/frostfs-s3-gw # Lifecycler S3_LIFECYCLER_VERSION=0.1.3 @@ -36,12 +36,12 @@ LOCODE_DB_URL=https://git.frostfs.info/attachments/a2e8def7-52b6-49f1-89cd-a0567 #LOCODE_DB_PATH=/path/to/locode_db # FrostFS CLI binary -FROSTFS_CLI_VERSION=0.44.4 +FROSTFS_CLI_VERSION=0.45.0-rc.3 FROSTFS_CLI_IMAGE=git.frostfs.info/truecloudlab/frostfs-cli #FROSTFS_CLI_PATH=/path/to/frostfs-cli-binary # FrostFS ADM tool binary -FROSTFS_ADM_VERSION=0.44.4 +FROSTFS_ADM_VERSION=0.45.0-rc.3 FROSTFS_ADM_IMAGE=git.frostfs.info/truecloudlab/frostfs-adm #FROSTFS_ADM_PATH=/path/to/frostfs-adm-binary diff --git a/services/http_gate/cfg/config.yml b/services/http_gate/cfg/config.yml index 1684c58..2f15e44 100644 --- a/services/http_gate/cfg/config.yml +++ b/services/http_gate/cfg/config.yml @@ -22,3 +22,6 @@ server: wallet: path: /wallet.json # Path to wallet passphrase: one # Passphrase to decrypt wallet + +containers: + cors: cors.container diff --git a/services/s3_gate/cfg/config.yml b/services/s3_gate/cfg/config.yml index 03e84e0..3d8e400 100644 --- a/services/s3_gate/cfg/config.yml +++ b/services/s3_gate/cfg/config.yml @@ -47,3 +47,6 @@ frostfsid: policy: enabled: false + +containers: + cors: cors.container From 018e33c1f9048df8e6c580bde535df535a297838 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Thu, 27 Mar 2025 17:34:37 +0300 Subject: [PATCH 146/149] [#101] Fix s3 gateway initialization Was broken in #100 Signed-off-by: Alex Vanin --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b3b342d..fcc09a3 100644 --- a/Makefile +++ b/Makefile @@ -108,7 +108,7 @@ up/pre-services: && echo "Subject key: $${subj_key}" \ && ./vendor/frostfs-adm -c frostfs-adm.yml morph frostfsid create-subject --namespace "" --subject-key $${subj_key} --subject-name system \ || die "Failed to create subject for system wallet"; \ - proxy_acc=`docker container exec -it morph_chain neo-go wallet dump-keys -w /wallets/system-wallet.json | head -1 | cut -d" " -f1` \ + proxy_acc=`docker container exec -it morph_chain neo-go wallet dump-keys -w /wallets/s3-wallet.json | head -1 | cut -d" " -f1` \ && echo "Proxy acc: $${proxy_acc}" \ && ./vendor/frostfs-adm morph proxy-add-account --config frostfs-adm.yml --account=$${proxy_acc} || die "Failed to register S3 gateway as proxy acc"; \ cid=`./vendor/frostfs-cli -c cli-cfg.yml container create -p "REP 4" --nns-name "cors" --nns-zone "container" --await | grep CID | cut -d" " -f2` \ From 6ef4d4aa0ad7f99f948c0cf19bda4f0faa191c37 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 28 Mar 2025 14:07:50 +0300 Subject: [PATCH 147/149] [#102] Makefile: Parallelize GAS transfer The last version of the `frostfs-adm` allows to work with multiple wallets in `refill-gas` and `proxy-add-account` commands. This makes `up/bootstrap` target take ~6s less time. The Makefile stuff is not obvious, but it seem to work as expected. Refs TrueCloudLab/frostfs-node#1590, TrueCloudLab/frostfs-node#1578 Signed-off-by: Evgenii Stratonikov --- Makefile | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index fcc09a3..911d168 100644 --- a/Makefile +++ b/Makefile @@ -72,6 +72,8 @@ up/basic: up/bootstrap # Start bootstrap services .PHONY: up/bootstrap +up/bootstrap: STORAGE_WALLETS = $(wildcard ./services/storage/wallet*.json) +up/bootstrap: STORAGE_ACCOUNTS = $(foreach wallet,$(STORAGE_WALLETS),$(shell docker container exec morph_chain neo-go wallet dump-keys -w /wallets/storage/$(notdir $(wallet)) | head -1 | awk '{print $$1}' )) up/bootstrap: get vendor/hosts @$(foreach SVC, $(START_BOOTSTRAP), $(shell docker-compose -f services/$(SVC)/docker-compose.yml up -d)) @source ./bin/helper.sh @@ -80,13 +82,16 @@ up/bootstrap: get vendor/hosts @./vendor/frostfs-adm --config frostfs-adm.yml morph \ ape add-rule-chain --target-type namespace --target-name "" \ --rule 'allow Container.* *' --chain-id "allow_container_ops" - @for f in ./services/storage/wallet*.json; do \ - echo "Transfer GAS to wallet $${f}" \ - && ./vendor/frostfs-adm -c frostfs-adm.yml morph refill-gas --storage-wallet $${f} --gas 10.0 \ - || die "Failed to transfer GAS to alphabet wallets"; \ - echo "Register storage wallet $${f} in proxy contract" \ - && ./vendor/frostfs-adm morph proxy-add-account --config frostfs-adm.yml --account=`docker container exec morph_chain neo-go wallet dump-keys -w /wallets/storage/$${f##*/} | head -1 | awk '{print $1}'` || die "Couldn't set storage allet as proxy wallet" - done + + echo -e "Transfer GAS to storage wallets: $(foreach wallet,$(STORAGE_WALLETS),\n\t$(wallet))" + ./vendor/frostfs-adm -c frostfs-adm.yml morph refill-gas \ + $(foreach wallet,$(STORAGE_WALLETS),--storage-wallet $(wallet)) \ + --gas 10.0 \ + || die "Failed to transfer GAS to alphabet wallets" + echo -e "Register storage accounts in proxy contract: $(foreach account,$(STORAGE_ACCOUNTS),\n\t$(account))" + ./vendor/frostfs-adm morph proxy-add-account --config frostfs-adm.yml \ + $(foreach account,$(STORAGE_ACCOUNTS),--account=$(account)) \ + || die "Couldn't set storage allet as proxy wallet" @echo "Create frostfsid subject for ./wallets/wallet.json"; \ if [ -n "$$(./vendor/frostfs-adm -c frostfs-adm.yml morph frostfsid list-subjects --namespace '')" ]; then \ echo "Subject already exists"; \ From 8608ba46ff2b6bc07a3d662ce0ace00bd137a31e Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Tue, 8 Apr 2025 14:51:37 +0300 Subject: [PATCH 148/149] [#104] Update components to the latest versions New S3 gateway version supports MFADelete feature that requires container with MFA objects. Signed-off-by: Alex Vanin --- .env | 12 ++++++------ Makefile | 5 ++++- services/s3_gate/cfg/config.yml | 1 + 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.env b/.env index dec29ce..9c4531d 100644 --- a/.env +++ b/.env @@ -12,19 +12,19 @@ NEOGO_VERSION=0.106.3 NEOGO_IMAGE=nspccdev/neo-go # FrostFS InnerRing nodes -IR_VERSION=0.45.0-rc.3 +IR_VERSION=0.45.0-rc.6 IR_IMAGE=git.frostfs.info/truecloudlab/frostfs-ir # FrostFS Storage nodes -NODE_VERSION=0.45.0-rc.3 +NODE_VERSION=0.45.0-rc.6 NODE_IMAGE=git.frostfs.info/truecloudlab/frostfs-storage # HTTP Gate -HTTP_GW_VERSION=0.33.0-rc.2 +HTTP_GW_VERSION=0.33.0-rc.3 HTTP_GW_IMAGE=git.frostfs.info/truecloudlab/frostfs-http-gw # S3 Gate -S3_GW_VERSION=0.33.0-rc.2 +S3_GW_VERSION=0.33.0-rc.3 S3_GW_IMAGE=git.frostfs.info/truecloudlab/frostfs-s3-gw # Lifecycler @@ -36,12 +36,12 @@ LOCODE_DB_URL=https://git.frostfs.info/attachments/a2e8def7-52b6-49f1-89cd-a0567 #LOCODE_DB_PATH=/path/to/locode_db # FrostFS CLI binary -FROSTFS_CLI_VERSION=0.45.0-rc.3 +FROSTFS_CLI_VERSION=0.45.0-rc.6 FROSTFS_CLI_IMAGE=git.frostfs.info/truecloudlab/frostfs-cli #FROSTFS_CLI_PATH=/path/to/frostfs-cli-binary # FrostFS ADM tool binary -FROSTFS_ADM_VERSION=0.45.0-rc.3 +FROSTFS_ADM_VERSION=0.45.0-rc.6 FROSTFS_ADM_IMAGE=git.frostfs.info/truecloudlab/frostfs-adm #FROSTFS_ADM_PATH=/path/to/frostfs-adm-binary diff --git a/Makefile b/Makefile index 911d168..56e5f7c 100644 --- a/Makefile +++ b/Makefile @@ -118,7 +118,10 @@ up/pre-services: && ./vendor/frostfs-adm morph proxy-add-account --config frostfs-adm.yml --account=$${proxy_acc} || die "Failed to register S3 gateway as proxy acc"; \ cid=`./vendor/frostfs-cli -c cli-cfg.yml container create -p "REP 4" --nns-name "cors" --nns-zone "container" --await | grep CID | cut -d" " -f2` \ && echo "CORS Container: $${cid}" \ - && ./vendor/frostfs-cli -c cli-cfg.yml ape-manager add --target-type container --target-name $${cid} --rule "allow Object.* *" || die "Failed to create CORS container" + && ./vendor/frostfs-cli -c cli-cfg.yml ape-manager add --target-type container --target-name $${cid} --rule "allow Object.* *" || die "Failed to create CORS container"; \ + cid=`./vendor/frostfs-cli -c cli-cfg.yml container create -p "REP 4" --nns-name "mfa" --nns-zone "container" --await | grep CID | cut -d" " -f2` \ + && echo "MFA Container: $${cid}" \ + && ./vendor/frostfs-cli -c cli-cfg.yml ape-manager add --target-type container --target-name $${cid} --rule "allow Object.* *" || die "Failed to create MFA container" fi @echo "Storage is prepared"; diff --git a/services/s3_gate/cfg/config.yml b/services/s3_gate/cfg/config.yml index 3d8e400..369325c 100644 --- a/services/s3_gate/cfg/config.yml +++ b/services/s3_gate/cfg/config.yml @@ -50,3 +50,4 @@ policy: containers: cors: cors.container + mfa: mfa.container From df8249f2aeabbbe8caab4229e42321027269cb74 Mon Sep 17 00:00:00 2001 From: Nikita Zinkevich Date: Fri, 11 Apr 2025 15:49:20 +0300 Subject: [PATCH 149/149] [#105] Add website container creation Signed-off-by: Nikita Zinkevich --- Makefile | 3 +++ services/s3_gate/cfg/config.yml | 1 + 2 files changed, 4 insertions(+) diff --git a/Makefile b/Makefile index 56e5f7c..1f97c5f 100644 --- a/Makefile +++ b/Makefile @@ -122,6 +122,9 @@ up/pre-services: cid=`./vendor/frostfs-cli -c cli-cfg.yml container create -p "REP 4" --nns-name "mfa" --nns-zone "container" --await | grep CID | cut -d" " -f2` \ && echo "MFA Container: $${cid}" \ && ./vendor/frostfs-cli -c cli-cfg.yml ape-manager add --target-type container --target-name $${cid} --rule "allow Object.* *" || die "Failed to create MFA container" + cid=`./vendor/frostfs-cli -c cli-cfg.yml container create -p "REP 4" --nns-name "website" --nns-zone "container" --await | grep CID | cut -d" " -f2` \ + && echo "Website configuration Container: $${cid}" \ + && ./vendor/frostfs-cli -c cli-cfg.yml ape-manager add --target-type container --target-name $${cid} --rule "allow Object.* *" || die "Failed to create Website configuration container" fi @echo "Storage is prepared"; diff --git a/services/s3_gate/cfg/config.yml b/services/s3_gate/cfg/config.yml index 369325c..e5c0b53 100644 --- a/services/s3_gate/cfg/config.yml +++ b/services/s3_gate/cfg/config.yml @@ -51,3 +51,4 @@ policy: containers: cors: cors.container mfa: mfa.container + website: website.container