diff --git a/.basic_services b/.basic_services index 201ea70..845c731 100644 --- a/.basic_services +++ b/.basic_services @@ -1,5 +1,4 @@ # Services start/stop order # Will start from top to bottom and stop in reverse -nats ir storage diff --git a/.bootstrap_services b/.bootstrap_services index 3d2562c..a56729d 100644 --- a/.bootstrap_services +++ b/.bootstrap_services @@ -1,5 +1,7 @@ # Services start/stop order # Will start from top to bottom and stop in reverse basenet -chain morph_chain +jaeger +prometheus +grafana diff --git a/.dockerignore b/.dockerignore index 73b336a..3867906 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,5 @@ .docker -.github +.forgejo vendor tmp .secrets diff --git a/.env b/.env index 906d3e8..9c4531d 100644 --- a/.env +++ b/.env @@ -1,5 +1,5 @@ # Basenet settings -LOCAL_DOMAIN=neofs.devenv +LOCAL_DOMAIN=frostfs.devenv IPV4_PREFIX=192.168.130 CA_CERTS_TRUSTED_STORE=/etc/ssl/certs @@ -8,53 +8,59 @@ BASTION_VERSION=10 BASTION_IMAGE=debian # NeoGo privnet -#CHAIN_PATH="/path/to/devenv.dump.gz" -CHAIN_URL="https://github.com/nspcc-dev/neofs-contract/releases/download/v0.15.5/devenv_mainchain_notary_disabled.gz" -NEOGO_VERSION=0.99.3-2-gaa57779d +NEOGO_VERSION=0.106.3 NEOGO_IMAGE=nspccdev/neo-go -# NeoFS InnerRing nodes -IR_VERSION=0.32.0 -IR_IMAGE=nspccdev/neofs-ir +# FrostFS InnerRing nodes +IR_VERSION=0.45.0-rc.6 +IR_IMAGE=git.frostfs.info/truecloudlab/frostfs-ir -# NeoFS Storage nodes -NODE_VERSION=0.32.0 -NODE_IMAGE=nspccdev/neofs-storage - -# NATS Server -NATS_VERSION=2.7.2 -NATS_IMAGE=nats +# FrostFS Storage nodes +NODE_VERSION=0.45.0-rc.6 +NODE_IMAGE=git.frostfs.info/truecloudlab/frostfs-storage # HTTP Gate -HTTP_GW_VERSION=0.24.0 -HTTP_GW_IMAGE=nspccdev/neofs-http-gw - -# REST Gate -REST_GW_VERSION=0.4.0 -REST_GW_IMAGE=nspccdev/neofs-rest-gw +HTTP_GW_VERSION=0.33.0-rc.3 +HTTP_GW_IMAGE=git.frostfs.info/truecloudlab/frostfs-http-gw # S3 Gate -S3_GW_VERSION=0.24.0 -S3_GW_IMAGE=nspccdev/neofs-s3-gw +S3_GW_VERSION=0.33.0-rc.3 +S3_GW_IMAGE=git.frostfs.info/truecloudlab/frostfs-s3-gw -# Coredns -COREDNS_VERSION=v016 -COREDNS_IMAGE=nspccdev/coredns +# Lifecycler +S3_LIFECYCLER_VERSION=0.1.3 +S3_LIFECYCLER_IMAGE=truecloudlab/frostfs-s3-lifecycler -# NeoFS LOCODE database -LOCODE_DB_URL=https://github.com/nspcc-dev/neofs-locode-db/releases/download/v0.2.1/locode_db.gz +# FrostFS LOCODE database +LOCODE_DB_URL=https://git.frostfs.info/attachments/a2e8def7-52b6-49f1-89cd-a056712e8e54 #LOCODE_DB_PATH=/path/to/locode_db -# NeoFS CLI binary -NEOFS_CLI_URL=https://github.com/nspcc-dev/neofs-node/releases/download/v0.32.0/neofs-cli-amd64.tar.gz -#NEOFS_CLI_PATH=/path/to/neofs-cli-binary +# FrostFS CLI binary +FROSTFS_CLI_VERSION=0.45.0-rc.6 +FROSTFS_CLI_IMAGE=git.frostfs.info/truecloudlab/frostfs-cli +#FROSTFS_CLI_PATH=/path/to/frostfs-cli-binary -# NeoFS ADM tool binary -NEOFS_ADM_VERSION=v0.32.0 -NEOFS_ADM_URL=https://github.com/nspcc-dev/neofs-node/releases/download/${NEOFS_ADM_VERSION}/neofs-adm-amd64.tar.gz -#NEOFS_ADM_PATH=/path/to/neofs-adm-binary +# FrostFS ADM tool binary +FROSTFS_ADM_VERSION=0.45.0-rc.6 +FROSTFS_ADM_IMAGE=git.frostfs.info/truecloudlab/frostfs-adm +#FROSTFS_ADM_PATH=/path/to/frostfs-adm-binary -# Compiled NeoFS Smart Contracts -NEOFS_CONTRACTS_VERSION=v0.15.5 -NEOFS_CONTRACTS_URL=https://github.com/nspcc-dev/neofs-contract/releases/download/${NEOFS_CONTRACTS_VERSION}/neofs-contract-${NEOFS_CONTRACTS_VERSION}.tar.gz -#NEOFS_CONTRACTS_PATH=/path/to/unpacked/neofs-contracts-dir +# Compiled FrostFS Smart Contracts +FROSTFS_CONTRACTS_URL=https://git.frostfs.info/TrueCloudLab/frostfs-contract/releases/download/v0.21.1/frostfs-contract-v0.21.1.tar.gz +#FROSTFS_CONTRACTS_PATH=/path/to/unpacked/frostfs-contracts-dir + +# Jaeger tracing +JAEGER_VERSION=1.42.0 +JAEGER_IMAGE=jaegertracing/all-in-one + +# Prometheus monitoring +PROMETHEUS_VERSION=v2.43.0 +PROMETHEUS_IMAGE=prom/prometheus + +# Grafana versions +GRAFANA_VERSION=9.5.6 +GRAFANA_IMAGE=grafana/grafana + +# Loki versions +LOKI_VERSION=2.9.1 +LOKI_IMAGE=grafana/loki diff --git a/.forgejo/ISSUE_TEMPLATE/bug_report.md b/.forgejo/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..7e778d2 --- /dev/null +++ b/.forgejo/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,45 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: community, triage, bug +assignees: '' + +--- + + + +## Expected Behavior + + + +## Current Behavior + + + +## Possible Solution + + + + + + +## Steps to Reproduce (for bugs) + + + +1. + +## Context + + + +## Regression + + + +## Your Environment + +* Version used: +* Server setup and configuration: +* Operating System and version (`uname -a`): diff --git a/.forgejo/ISSUE_TEMPLATE/config.yml b/.forgejo/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..3ba13e0 --- /dev/null +++ b/.forgejo/ISSUE_TEMPLATE/config.yml @@ -0,0 +1 @@ +blank_issues_enabled: false diff --git a/.forgejo/ISSUE_TEMPLATE/feature_request.md b/.forgejo/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..d6d1162 --- /dev/null +++ b/.forgejo/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: community, triage +assignees: '' + +--- + +## Is your feature request related to a problem? Please describe. + + +## Describe the solution you'd like + + +## Describe alternatives you've considered + + +## Additional context + diff --git a/.forgejo/logo.svg b/.forgejo/logo.svg new file mode 100644 index 0000000..148c359 --- /dev/null +++ b/.forgejo/logo.svg @@ -0,0 +1,70 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/.forgejo/workflows/dco.yml b/.forgejo/workflows/dco.yml new file mode 100644 index 0000000..6746408 --- /dev/null +++ b/.forgejo/workflows/dco.yml @@ -0,0 +1,21 @@ +name: DCO action +on: [pull_request] + +jobs: + dco: + name: DCO + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Setup Go + uses: actions/setup-go@v3 + with: + go-version: '1.21' + + - name: Run commit format checker + uses: https://git.frostfs.info/TrueCloudLab/dco-go@v2 + with: + from: 'origin/${{ github.event.pull_request.base.ref }}' diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..5436e42 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +/services/grafana/provisioning/dashboards/* -diff -merge \ No newline at end of file diff --git a/.github/logo.svg b/.github/logo.svg deleted file mode 100644 index b4da076..0000000 --- a/.github/logo.svg +++ /dev/null @@ -1,129 +0,0 @@ - - - - - - - - - - - - image/svg+xml - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/.github/workflows/dco.yml b/.github/workflows/dco.yml deleted file mode 100644 index 40ed8fc..0000000 --- a/.github/workflows/dco.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: DCO check - -on: - pull_request: - branches: - - master - -jobs: - commits_check_job: - runs-on: ubuntu-latest - name: Commits Check - steps: - - name: Get PR Commits - id: 'get-pr-commits' - uses: tim-actions/get-pr-commits@master - with: - token: ${{ secrets.GITHUB_TOKEN }} - - name: DCO Check - uses: tim-actions/dco@master - with: - commits: ${{ steps.get-pr-commits.outputs.commits }} diff --git a/.gitignore b/.gitignore index d5835aa..a672692 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,17 @@ +# IDE .idea +.vscode + +# Vendoring vendor + +temp tmp + .secrets sites/* !sites/.gitkeep + +# Runtime generation keys services/storage/*tls.crt services/storage/*tls.key -services/nats/*.pem diff --git a/.services b/.services index 19b229e..ccc96f8 100644 --- a/.services +++ b/.services @@ -2,5 +2,4 @@ # Will start from top to bottom and stop in reverse http_gate s3_gate -rest_gate -coredns +s3_lifecycler diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 0000000..3b080ce --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1,3 @@ +.* @alexvanin @fyrchik +.forgejo/.* @potyarkin +Makefile @potyarkin diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7f6b65f..fb57514 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3,8 +3,8 @@ First, thank you for contributing! We love and encourage pull requests from everyone. Please follow the guidelines: -- Check the open [issues](https://github.com/nspcc-dev/neofs-dev-env/issues) and - [pull requests](https://github.com/nspcc-dev/neofs-dev-env/pulls) for existing +- Check the open [issues](https://git.frostfs.info/TrueCloudLab/frostfs-dev-env/issues) and + [pull requests](https://git.frostfs.info/TrueCloudLab/frostfs-dev-env/pulls) for existing discussions. - Open an issue first, to discuss a new feature or enhancement. @@ -21,23 +21,23 @@ everyone. Please follow the guidelines: ## Development Workflow -Start by forking the `neofs-dev-env` repository, make changes in a branch and then +Start by forking the `frostfs-dev-env` repository, make changes in a branch and then send a pull request. We encourage pull requests to discuss code changes. Here are the steps in details: -### Setup your GitHub Repository -Fork [NeoFS node upstream](https://github.com/nspcc-dev/neofs-dev-env/fork) source +### Set up your git repository +Fork [FrostFS node upstream](https://git.frostfs.info/repo/fork/24) source repository to your own personal repository. Copy the URL of your fork (you will need it for the `git clone` command below). ```sh -$ git clone https://github.com/nspcc-dev/neofs-dev-env +$ git clone https://git.frostfs.info//frostfs-dev-env.git ``` ### Set up git remote as ``upstream`` ```sh -$ cd neofs-dev-env -$ git remote add upstream https://github.com/nspcc-dev/neofs-dev-env +$ cd frostfs-dev-env +$ git remote add upstream https://git.frostfs.info/TrueCloudLab/frostfs-dev-env.git $ git fetch upstream $ git merge upstream/master ... @@ -55,8 +55,7 @@ $ git checkout -b feature/123-something_awesome ### Test your changes After your code changes, make sure -- To add test cases for the new code. -- To run `make lint` +- To run `make up` to check dev-env is not broken. - To squash your commits into a single commit or a series of logically separated commits run `git rebase -i`. It's okay to force update your pull request. @@ -86,8 +85,8 @@ $ git push origin feature/123-something_awesome ``` ### Create a Pull Request -Pull requests can be created via GitHub. Refer to [this -document](https://help.github.com/articles/creating-a-pull-request/) for +Pull requests can be created via Forgejo. Refer to [this +document](https://docs.codeberg.org/collaborating/pull-requests-and-git-flow/) for detailed steps on how to create a pull request. After a Pull Request gets peer reviewed and approved, it will be merged. @@ -103,7 +102,7 @@ contributors". To sign your work, just add a line like this at the end of your commit message: ``` -Signed-off-by: Samii Sakisaka +Signed-off-by: Samii Sakisaka ``` This can easily be done with the `--signoff` option to `git commit`. diff --git a/Makefile b/Makefile index 7e7815a..1f97c5f 100644 --- a/Makefile +++ b/Makefile @@ -10,8 +10,8 @@ include .env # help target include help.mk -# update NeoFS global config targets -include neofs_config.mk +# update FrostFS global config targets +include frostfs_config.mk # Targets to get required artifacts and external resources for each service include services/*/artifacts.mk @@ -19,29 +19,31 @@ include services/*/artifacts.mk # Targets helpful to prepare service environment include services/*/prepare.mk +# List of services to run +START_SVCS = $(shell cat .services | grep -v '#') +START_BASIC = $(shell cat .basic_services | grep -ve '#') +START_BOOTSTRAP = $(shell cat .bootstrap_services | grep -v '#') +STOP_SVCS = $(shell tac .services | grep -v '#') +STOP_BASIC = $(shell tac .basic_services | grep -v '#') +STOP_BOOTSTRAP = $(shell tac .bootstrap_services | grep -v '#') + +# Enabled services dirs +ENABLED_SVCS_DIRS = $(shell echo "${START_BOOTSTRAP} ${START_BASIC} ${START_SVCS}" | sed 's|[^ ]* *|./services/&|g') + # Services that require artifacts GET_SVCS = $(shell grep -Rl "get.*:" ./services/* | sort -u | grep artifacts.mk | xargs -I {} dirname {} | xargs basename -a) # Services that require pulling images -PULL_SVCS = $(shell find ./services -type f -name 'docker-compose.yml' | sort -u | xargs -I {} dirname {} | xargs basename -a) - -# List of services to run -START_SVCS = $(shell cat .services | grep -v \\\#) -START_BASIC = $(shell cat .basic_services | grep -v \\\#) -START_BOOTSTRAP = $(shell cat .bootstrap_services | grep -v \\\#) -STOP_SVCS = $(shell tac .services | grep -v \\\#) -STOP_BASIC = $(shell tac .basic_services | grep -v \\\#) -STOP_BOOTSTRAP = $(shell tac .bootstrap_services | grep -v \\\#) +PULL_SVCS = $(shell find ${ENABLED_SVCS_DIRS} -type f -name 'docker-compose.yml' | sort -u | xargs -I {} dirname {} | xargs basename -a) # List of hosts available in devenv HOSTS_LINES = $(shell grep -Rl IPV4_PREFIX ./services/* | grep .hosts) # Paths to protocol.privnet.yml MORPH_CHAIN_PROTOCOL = './services/morph_chain/protocol.privnet.yml' -CHAIN_PROTOCOL = './services/chain/protocol.privnet.yml' # List of grepped environment variables from *.env -GREP_DOTENV = $(shell find . -name '*.env' -exec grep -rhv -e '^\#' -e '^$$' {} + | sort -u ) +GREP_DOTENV = $(shell find . -name '*.env' -exec grep -rhv -e '^#' -e '^$$' {} + | sort -u ) # Pull all required Docker images .PHONY: pull @@ -49,32 +51,82 @@ pull: $(foreach SVC, $(PULL_SVCS), $(shell cd services/$(SVC) && docker-compose pull)) @: -# Get all services artifacs +# Get all services artifacts .PHONY: get get: $(foreach SVC, $(GET_SVCS), get.$(SVC)) @: # Start environment .PHONY: up -up: up/basic +up: up/basic up/pre-services @$(foreach SVC, $(START_SVCS), $(shell docker-compose -f services/$(SVC)/docker-compose.yml up -d)) - @echo "Full NeoFS Developer Environment is ready" + @echo "Full FrostFS Developer Environment is ready" -# Build up NeoFS +# Build up FrostFS .PHONY: up/basic up/basic: up/bootstrap @$(foreach SVC, $(START_BASIC), $(shell docker-compose -f services/$(SVC)/docker-compose.yml up -d)) - @./bin/tick.sh - @./bin/config.sh string SystemDNS container - @echo "Basic NeoFS Developer Environment is ready" + @./vendor/frostfs-adm -c ./frostfs-adm.yml morph force-new-epoch + @./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config SystemDNS=container --force + @echo "Basic FrostFS Developer Environment is ready" # Start bootstrap services .PHONY: up/bootstrap +up/bootstrap: STORAGE_WALLETS = $(wildcard ./services/storage/wallet*.json) +up/bootstrap: STORAGE_ACCOUNTS = $(foreach wallet,$(STORAGE_WALLETS),$(shell docker container exec morph_chain neo-go wallet dump-keys -w /wallets/storage/$(notdir $(wallet)) | head -1 | awk '{print $$1}' )) up/bootstrap: get vendor/hosts @$(foreach SVC, $(START_BOOTSTRAP), $(shell docker-compose -f services/$(SVC)/docker-compose.yml up -d)) - @./vendor/neofs-adm --config neofs-adm.yml morph init --alphabet-wallets ./services/ir --contracts vendor/contracts || exit 1 - @for f in ./services/storage/wallet*.json; do echo "Transfer GAS to wallet $${f}" && ./vendor/neofs-adm -c neofs-adm.yml morph refill-gas --storage-wallet $${f} --gas 10.0 --alphabet-wallets services/ir || exit 1; done - @echo "NeoFS sidechain environment is deployed" + @source ./bin/helper.sh + @./vendor/frostfs-adm --config frostfs-adm.yml morph init --contracts vendor/contracts + echo "Set rule chain to policy contract" + @./vendor/frostfs-adm --config frostfs-adm.yml morph \ + ape add-rule-chain --target-type namespace --target-name "" \ + --rule 'allow Container.* *' --chain-id "allow_container_ops" + + echo -e "Transfer GAS to storage wallets: $(foreach wallet,$(STORAGE_WALLETS),\n\t$(wallet))" + ./vendor/frostfs-adm -c frostfs-adm.yml morph refill-gas \ + $(foreach wallet,$(STORAGE_WALLETS),--storage-wallet $(wallet)) \ + --gas 10.0 \ + || die "Failed to transfer GAS to alphabet wallets" + echo -e "Register storage accounts in proxy contract: $(foreach account,$(STORAGE_ACCOUNTS),\n\t$(account))" + ./vendor/frostfs-adm morph proxy-add-account --config frostfs-adm.yml \ + $(foreach account,$(STORAGE_ACCOUNTS),--account=$(account)) \ + || die "Couldn't set storage allet as proxy wallet" + @echo "Create frostfsid subject for ./wallets/wallet.json"; \ + if [ -n "$$(./vendor/frostfs-adm -c frostfs-adm.yml morph frostfsid list-subjects --namespace '')" ]; then \ + echo "Subject already exists"; \ + else \ + subj_key=`docker container exec -it morph_chain neo-go wallet dump-keys -w /wallets/wallet.json | tail -1 | tr -d ' \r\n'` \ + && echo "Subject key: $${subj_key}" \ + && ./vendor/frostfs-adm -c frostfs-adm.yml morph frostfsid create-subject --namespace "" --subject-key $${subj_key} --subject-name walletsubject \ + || die "Failed to create subject for the wallet"; \ + fi + echo "FrostFS sidechain environment is deployed" + +# Prepare to start services +.PHONY: up/pre-services +up/pre-services: + @source ./bin/helper.sh + @echo "Prepare storage for services"; \ + if [ -z "$$(./vendor/frostfs-cli -c cli-cfg.yml container list)" ]; then \ + subj_key=`docker container exec -it morph_chain neo-go wallet dump-keys -w /wallets/system-wallet.json | tail -1 | tr -d ' \r\n'` \ + && echo "Subject key: $${subj_key}" \ + && ./vendor/frostfs-adm -c frostfs-adm.yml morph frostfsid create-subject --namespace "" --subject-key $${subj_key} --subject-name system \ + || die "Failed to create subject for system wallet"; \ + proxy_acc=`docker container exec -it morph_chain neo-go wallet dump-keys -w /wallets/s3-wallet.json | head -1 | cut -d" " -f1` \ + && echo "Proxy acc: $${proxy_acc}" \ + && ./vendor/frostfs-adm morph proxy-add-account --config frostfs-adm.yml --account=$${proxy_acc} || die "Failed to register S3 gateway as proxy acc"; \ + cid=`./vendor/frostfs-cli -c cli-cfg.yml container create -p "REP 4" --nns-name "cors" --nns-zone "container" --await | grep CID | cut -d" " -f2` \ + && echo "CORS Container: $${cid}" \ + && ./vendor/frostfs-cli -c cli-cfg.yml ape-manager add --target-type container --target-name $${cid} --rule "allow Object.* *" || die "Failed to create CORS container"; \ + cid=`./vendor/frostfs-cli -c cli-cfg.yml container create -p "REP 4" --nns-name "mfa" --nns-zone "container" --await | grep CID | cut -d" " -f2` \ + && echo "MFA Container: $${cid}" \ + && ./vendor/frostfs-cli -c cli-cfg.yml ape-manager add --target-type container --target-name $${cid} --rule "allow Object.* *" || die "Failed to create MFA container" + cid=`./vendor/frostfs-cli -c cli-cfg.yml container create -p "REP 4" --nns-name "website" --nns-zone "container" --await | grep CID | cut -d" " -f2` \ + && echo "Website configuration Container: $${cid}" \ + && ./vendor/frostfs-cli -c cli-cfg.yml ape-manager add --target-type container --target-name $${cid} --rule "allow Object.* *" || die "Failed to create Website configuration container" + fi + @echo "Storage is prepared"; # Build up certain service .PHONY: up/% @@ -85,7 +137,7 @@ up/%: get vendor/hosts # Stop environment .PHONY: down down: down/add down/basic down/bootstrap - @echo "Full NeoFS Developer Environment is down" + @echo "Full FrostFS Developer Environment is down" .PHONY: down/add down/add: @@ -106,6 +158,7 @@ down/bootstrap: down/%: @docker-compose -f services/$*/docker-compose.yml down +# Generate changes for /etc/hosts .PHONY: vendor/hosts .ONESHELL: vendor/hosts: @@ -119,7 +172,7 @@ vendor/hosts: done < $${file}; done > $@ -# Display changes for /etc/hosts +# Generate and display changes for /etc/hosts .PHONY: hosts hosts: vendor/hosts @cat vendor/hosts @@ -128,14 +181,14 @@ hosts: vendor/hosts .PHONY: clean .ONESHELL: clean: - @rm -rf vendor/* services/storage/s04tls.* services/nats/*.pem + @rm -rf vendor/* services/storage/s04tls.* @> .int_test.env @for svc in $(PULL_SVCS) do vols=`docker-compose -f services/$${svc}/docker-compose.yml config --volumes` if [[ ! -z "$${vols}" ]]; then for vol in $${vols}; do - docker volume rm -f "$${svc}_$${vol}" 2> /dev/null + docker volume rm -f "$${svc}_$${vol}" done fi done @@ -144,8 +197,7 @@ clean: .PHONY: env env: @$(foreach envvar,$(GREP_DOTENV),echo $(envvar);) - @echo MORPH_BLOCK_TIME=$(shell grep 'SecondsPerBlock' $(MORPH_CHAIN_PROTOCOL) | awk '{print $$2}')s - @echo MAINNET_BLOCK_TIME=$(shell grep 'SecondsPerBlock' $(CHAIN_PROTOCOL) | awk '{print $$2}')s + @echo MORPH_BLOCK_TIME=$(shell grep 'TimePerBlock' $(MORPH_CHAIN_PROTOCOL) | awk '{print $$2}')s @echo MORPH_MAGIC=$(shell grep 'Magic' $(MORPH_CHAIN_PROTOCOL) | awk '{print $$2}') # Restart storage nodes with clean volumes diff --git a/README.md b/README.md index b620273..415885e 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,14 @@

-NeoFS +FrostFS logo

- NeoFS local Development and Testing environment + FrostFS local Development and Testing environment

--- ## Overview -Tools to set up local NeoFS network and N3 privnets. Devenv, for short. +Tools to set up local FrostFS network and N3 privnets. Devenv, for short. ## Prerequisites @@ -27,7 +27,7 @@ Make sure you have installed all of the following prerequisites on your machine: Clone repo: ``` -$ git clone https://github.com/nspcc-dev/neofs-dev-env.git +$ git clone https://git.frostfs.info/TrueCloudLab/frostfs-dev-env.git ``` Run next commands from project's root: @@ -42,11 +42,11 @@ been added already, there is no need to run it separately. ``` $ make hosts -192.168.130.10 bastion.neofs.devenv -192.168.130.50 main-chain.neofs.devenv -192.168.130.61 ir01.neofs.devenv +192.168.130.10 bastion.frostfs.devenv +192.168.130.50 main-chain.frostfs.devenv +192.168.130.61 ir01.frostfs.devenv ... -192.168.130.74 s04.neofs.devenv +192.168.130.74 s04.frostfs.devenv ``` This command shows addresses and hostnames of components. Add `make hosts` @@ -57,37 +57,21 @@ Run all services with command: $ make up ``` -When all services are up, you need to make GAS deposit for test wallet to be -able to pay for NeoFS operations. Test wallet is located in -`wallets/wallet.json` with the corresponding key in `wallets/wallet.key`. The -password is empty. - -``` -$ make prepare.ir -password > -fa6ba62bffb04030d303dcc95bda7413e03aa3c7e6ca9c2f999d65db9ec9b82c -``` - -Also you should add self-signed node (`s04.neofs.devenv`) certificate to trusted +Also, you should add self-signed node (`s04.frostfs.devenv`) certificate to trusted store (default location might be changed using `CA_CERTS_TRUSTED_STORE` -variable). This step is required for client services (neofs-http-gw, -neofs-s3-gw) to interact with the node: +variable). This step is required for client services (frostfs-http-gw, +frostfs-s3-gw) to interact with the node: ``` $ sudo make prepare.storage ``` -Change NeoFS global configuration values with `make update.*` commands. The +Change FrostFS global configuration values with `make update.*` commands. The password of inner ring wallet is `one`. See examples in `make help`. ``` $ make update.epoch_duration val=30 -Changing EpochDuration configration value to 30 -Enter account NNudMSGzEoktFzdYGYoNb3bzHzbmM1genF password > -Sent invocation transaction dbb8c1145b6d10f150135630e13bb0dc282023163f5956c6945a60db0cb45cb0 -Updating NeoFS epoch to 2 -Enter account NNudMSGzEoktFzdYGYoNb3bzHzbmM1genF password > -Sent invocation transaction 0e6eb5e190f36332e5e5f4e866c7e100826e285fd949e11c085e15224f343ba6 +Waiting for transactions to persist... ``` For instructions on how to set up DevEnv on macOS, please refer [the @@ -121,9 +105,9 @@ You can find more information on each service in `docs` directory. Maybe you will find the answer for your question in [F.A.Q.](docs/faq.md) -## Using NeoFS Admin Tool in `dev-env` +## Using FrostFS Admin Tool in `dev-env` -Devenv supports NeoFS network management via [neofs-adm](https://github.com/nspcc-dev/neofs-node/tree/master/cmd/neofs-adm). +Devenv supports FrostFS network management via [frostfs-adm](https://git.frostfs.info/TrueCloudLab/frostfs-node/src/branch/master/cmd/frostfs-adm). `services/ir` contains the Alphabet wallet in a proper format, specify it with `--alphabet-wallets` flag. @@ -153,6 +137,66 @@ Display addresses and host names for each running service, if available. Clean up `vendor` directory. +### s3cred + +Registers user wallet and issues s3 credentials. + +Usage and default parameter values: +```sh +make s3cred [password=""] [contract_password=s3] [wallet=""] [gate_public_key=0313b1ac3a8076e155a7e797b24f0b650cccad5941ea59d7cfd51a024a8b2a06bf] +``` + +As soon as the storage node is in the network map (see above) you can generate S3 +credentials: + +``` sh +$ make s3cred +{ + "access_key_id": "EXArWh8x1zeHG3851s1RtoCo7dowxF6rhLGA15nbMffT0AKRSjJ5fmcqf3Ht2VCAkfmPQUVARghRB77xHCA1BoN2p", + "secret_access_key": "d70c1dba83f0f90bb231f06f1ce0e0dfbcfb122f4b4345a3c18d3869c359b79f", + "owner_private_key": "140947599afd9ca89af4b358c3176eb046e554d942a0dc99a8e06f3e43c8f4ad", + "wallet_public_key": "0324e76288fcb900100d01802a14ef977cca45ad073561230446df14b344c858b6", + "container_id": "EXArWh8x1zeHG3851s1RtoCo7dowxF6rhLGA15nbMffT" +} +``` +Running without any parameters results in defaults which are based on the private key from +`/wallets/wallet.json` user wallet and `/wallet.json` contract wallet. +If `wallet` parameter is set, gate searches custom user wallet file in `/wallets` directory. + +Now let's configure an S3 client (AWS CLI will be used as example): + +``` sh +$ aws configure +AWS Access Key ID []: EXArWh8x1zeHG3851s1RtoCo7dowxF6rhLGA15nbMffT0AKRSjJ5fmcqf3Ht2VCAkfmPQUVARghRB77xHCA1BoN2p +AWS Secret Access Key []: d70c1dba83f0f90bb231f06f1ce0e0dfbcfb122f4b4345a3c18d3869c359b79f +Default region name []: us-east-1 +Default output format []: json +``` + +If you need to create credentials for different users, put user wallet to `wallets` dir and specify it via `wallet` parameter. +Pass wallet password in `password` parameter if it's not default. The same is for `contract_wallet` and `gate_public_key` params. + +```sh +$ make s3cred wallet=custom_wallet.json password=test +{ + "access_key_id": "jHhL5B33o16R4jQsb8wm9A3RRdS6KrTB5N4bja9Jys904W7xXFNKqem2ACvTRWRYJsZMCUikYFSokN7pPJziWyDi", + "secret_access_key": "21bb64fafa32c82417fd8b97ac56cc8a085998a3852632d52fe7042453daa440", + "owner_private_key": "10f6f9d7a47bb0bf68363ad8a99fe69f1493f8b6e1665b3e4e83feb2d5c7ee39", + "wallet_public_key": "03e38759973a6bb722baabc2dd84036a39f0b2f53d32fec45a4dacde8a50fe4b70", + "container_id": "jHhL5B33o16R4jQsb8wm9A3RRdS6KrTB5N4bja9Jys9" +} +``` + +To get credentials from custom wallet, place it in `wallets` dir before start. + +### cred + +Usage and default parameter values: +```sh +make cred [password=""] [contract_password=s3] [wallet=/user_wallet.json] +``` +The same as `s3cred`, but it doesn't issues s3 credentials. + ## Contributing Feel free to contribute to this project after reading the [contributing diff --git a/bin/config.sh b/bin/config.sh deleted file mode 100755 index 0f88192..0000000 --- a/bin/config.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env bash - -# Source env settings -. .env -. services/ir/.ir.env -source bin/helper.sh - -# NeoGo binary path. -NEOGO="${NEOGO:-docker exec -it main_chain neo-go}" - -# Wallet files to change config value -WALLET="${WALLET:-services/chain/node-wallet.json}" -WALLET_IMG="${WALLET_IMG:-wallets/node-wallet.json}" -# Wallet password that would be entered automatically; '-' means no password -PASSWD="one" -NETMAP_ADDR=$(bin/resolve.sh netmap.neofs) - -# NeoFS configuration record: variable type [string|int|etc], -# key is a string and value is a constant of given type -TYPE=${1} -KEY=${2} -VALUE="${3}" - -[ -z "$TYPE" ] && echo "Empty config value type" && exit 1 -[ -z "$KEY" ] && echo "Empty config key" && exit 1 -[ -z "$VALUE" ] && echo "Empty config value" && exit 1 - -# Internal variables -if [[ -z "${NEOFS_NOTARY_DISABLED}" ]]; then - ADDR=$(jq -r .accounts[2].address < "${WALLET}" || die "Cannot get address from ${WALLET}") -else - ADDR=$(jq -r .accounts[0].address < "${WALLET}" || die "Cannot get address from ${WALLET}") -fi - -# Change config value in side chain -echo "Changing ${KEY} configration value to ${VALUE}" - -# shellcheck disable=SC2086 -./bin/passwd.exp ${PASSWD} ${NEOGO} contract invokefunction \ - -w ${WALLET_IMG} \ - -a ${ADDR} \ - -r http://morph-chain.${LOCAL_DOMAIN}:30333 \ - ${NETMAP_ADDR} \ - setConfig bytes:beefcafe \ - string:${KEY} \ - ${TYPE}:${VALUE} -- ${ADDR} || exit 1 - -# Update epoch to apply new configuration value -./bin/tick.sh diff --git a/bin/deposit.sh b/bin/deposit.sh deleted file mode 100755 index 95fbc3e..0000000 --- a/bin/deposit.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash - -# Source env settings -. .env -. services/ir/.ir.env -source bin/helper.sh - -# NeoGo binary path. -NEOGO="${NEOGO:-docker exec -it main_chain neo-go}" -# Wallet file to use for deposit GAS from -WALLET="${WALLET:-wallets/wallet.json}" -# Wallet password that would be entered automatically; '-' means no password -PASSWD="-" -# How much GAS to deposit. First cli argument or 50 by default -DEPOSIT="${1:-50}" - -# Internal variables -ADDR=$(jq -r .accounts[0].address < "${WALLET}" \ - || die "Cannot get address from wallet: ${WALLET}") -CONTRACT_ADDR=$(${NEOGO} util convert "${NEOFS_IR_CONTRACTS_NEOFS}" \ - | grep 'LE ScriptHash to Address' \ - | awk '{print $5}' \ - | grep -oP "[A-z0-9]+" \ - || die "Cannot parse contract address: ${NEOFS_IR_CONTRACTS_NEOFS}") - -# Make deposit -# shellcheck disable=SC2086 -./bin/passwd.exp ${PASSWD} ${NEOGO} wallet nep17 transfer \ - -w ${WALLET} \ - -r http://main-chain.${LOCAL_DOMAIN}:30333 \ - --from ${ADDR} \ - --to ${CONTRACT_ADDR} \ - --token GAS \ - --amount ${DEPOSIT} diff --git a/bin/passwd.exp b/bin/passwd.exp deleted file mode 100755 index 08fd3c7..0000000 --- a/bin/passwd.exp +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/expect - -set passwd [lindex $argv 0] -set args [lrange $argv 1 end] - -spawn -noecho {*}$args -expect -re {^.*assword.*$} - -if { $passwd == "-"} { - send -- "\r" -} else { - send -- "$passwd\r" -} - -expect { - "Relay transaction" { - send "y\r" - exp_continue - } - EOF -} -lassign [wait] pid spawnid os_error_flag value -exit $value diff --git a/bin/resolve.sh b/bin/resolve.sh deleted file mode 100755 index 36a1ac2..0000000 --- a/bin/resolve.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash - -# Source env settings -. .env -source bin/helper.sh - -# NeoGo binary path. -NEOGO="${NEOGO:-docker exec -t morph_chain neo-go}" -# NNS contract script hash -output=$(curl -s --data '{ "id": 1, "jsonrpc": "2.0", "method": "getcontractstate", "params": [1] }' \ - "http://morph-chain.${LOCAL_DOMAIN}:30333/") \ - || die "Cannot fetch NNS contract state" - -NNS_ADDR=$(jq -r '.result.hash' <<< "$output") \ - || die "Cannot parse NNS contract hash: $NNS_ADDR" - -${NEOGO} contract testinvokefunction \ - -r "http://morph-chain.${LOCAL_DOMAIN}:30333" \ - "${NNS_ADDR}" resolve string:"${1}" int:16 \ - | jq -r '.stack[0].value | if type=="array" then .[0].value else . end' \ - | base64 -d \ - || die "Cannot invoke 'NNS.resolve' $output" diff --git a/bin/tick.sh b/bin/tick.sh deleted file mode 100755 index ede13c2..0000000 --- a/bin/tick.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env bash - -# Source env settings -. .env -. services/ir/.ir.env -source bin/helper.sh - -# NeoGo binary path. -NEOGO="${NEOGO:-docker exec -it main_chain neo-go}" -NEOGO_NONINTERACTIVE="${NEOGO_NONINTERACTIVE:-docker exec -t main_chain neo-go}" - -# Wallet files to change config value -WALLET="${WALLET:-services/chain/node-wallet.json}" -WALLET_IMG="${WALLET_IMG:-wallets/node-wallet.json}" - -# Wallet password that would be entered automatically; '-' means no password -PASSWD="one" - -# Internal variables -if [[ -z "${NEOFS_NOTARY_DISABLED}" ]]; then - ADDR=$(jq -r .accounts[2].address < "${WALLET}" || die "Cannot get address from ${WALLET}") -else - ADDR=$(jq -r .accounts[0].address < "${WALLET}" || die "Cannot get address from ${WALLET}") -fi - -# Grep Morph block time -SIDECHAIN_PROTO="${SIDECHAIN_PROTO:-services/morph_chain/protocol.privnet.yml}" -BLOCK_DURATION=$(grep SecondsPerBlock < "$SIDECHAIN_PROTO" | awk '{print $2}') \ - || die "Cannot fetch block duration" -NETMAP_ADDR=$(bin/resolve.sh netmap.neofs) || die "Cannot resolve netmap.neofs" - -# Fetch current epoch value -EPOCH=$(${NEOGO_NONINTERACTIVE} contract testinvokefunction \ - -r "http://morph-chain.${LOCAL_DOMAIN}:30333" "${NETMAP_ADDR}" epoch \ - | grep 'value' | awk -F'"' '{ print $4 }') \ - || die "Cannot fetch epoch from netmap contract" - -echo "Updating NeoFS epoch to $((EPOCH+1))" - -# shellcheck disable=SC2086 -./bin/passwd.exp ${PASSWD} ${NEOGO} contract invokefunction \ - -w ${WALLET_IMG} \ - -a ${ADDR} \ - -r http://morph-chain.${LOCAL_DOMAIN}:30333 \ - ${NETMAP_ADDR} \ - newEpoch int:$((EPOCH+1)) -- ${ADDR}:Global - -# Wait one Morph block to ensure the transaction broadcasted -# shellcheck disable=SC2086 -sleep $BLOCK_DURATION diff --git a/cli-cfg.yml b/cli-cfg.yml new file mode 100644 index 0000000..9a5c7eb --- /dev/null +++ b/cli-cfg.yml @@ -0,0 +1,3 @@ +wallet: ./wallets/system-wallet.json +password: "" +rpc-endpoint: s01.frostfs.devenv:8080 diff --git a/configs/s01-cli.yml b/configs/s01-cli.yml new file mode 100644 index 0000000..3bd7c43 --- /dev/null +++ b/configs/s01-cli.yml @@ -0,0 +1,4 @@ +wallet: services/storage/wallet01.json +password: "" +rpc-endpoint: s01.frostfs.devenv:8080 +endpoint: s01.frostfs.devenv:8081 \ No newline at end of file diff --git a/configs/s02-cli.yml b/configs/s02-cli.yml new file mode 100644 index 0000000..ac5a8bb --- /dev/null +++ b/configs/s02-cli.yml @@ -0,0 +1,4 @@ +wallet: services/storage/wallet02.json +password: "" +rpc-endpoint: s02.frostfs.devenv:8080 +endpoint: s02.frostfs.devenv:8081 \ No newline at end of file diff --git a/configs/s03-cli.yml b/configs/s03-cli.yml new file mode 100644 index 0000000..ed288dc --- /dev/null +++ b/configs/s03-cli.yml @@ -0,0 +1,4 @@ +wallet: services/storage/wallet03.json +password: "" +rpc-endpoint: s03.frostfs.devenv:8080 +endpoint: s03.frostfs.devenv:8081 \ No newline at end of file diff --git a/configs/s04-cli.yml b/configs/s04-cli.yml new file mode 100644 index 0000000..f4353d5 --- /dev/null +++ b/configs/s04-cli.yml @@ -0,0 +1,4 @@ +wallet: services/storage/wallet04.json +password: "" +rpc-endpoint: s04.frostfs.devenv:8080 +endpoint: s04.frostfs.devenv:8081 \ No newline at end of file diff --git a/docs/basenet.md b/docs/basenet.md index 7883889..0ce1f5e 100644 --- a/docs/basenet.md +++ b/docs/basenet.md @@ -7,7 +7,7 @@ host machine, so all programs running on host can connect to services exposed to ## .env settings -### LOCAL_DOMAIN=neofs.devenv +### LOCAL_DOMAIN=frostfs.devenv Domain to use for all containers exposed to `basenet_internet`. @@ -28,7 +28,7 @@ devenv services. Run shell in bastion: ``` -neofs-dev-env$ docker exec -ti bastion /bin/bash +frostfs-dev-env$ docker exec -ti bastion /bin/bash root@bastion:/# ip a sh 1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 diff --git a/docs/chain.md b/docs/chain.md deleted file mode 100644 index 0ed13ce..0000000 --- a/docs/chain.md +++ /dev/null @@ -1,158 +0,0 @@ -# N3 main chain privnet service - -A single-node N3 privnet deployment, running on -[neo-go](https://github.com/nspcc-dev/neo-go). Represents N3 MainNet. - -Contracts deployed: -- NeoFS [contract](https://github.com/nspcc-dev/neofs-contract/tree/master/neofs) -- Processing [contract](https://github.com/nspcc-dev/neofs-contract/tree/master/processing) - -RPC available at `http://main-chain.neofs.devenv:30333`. - -## .env settings - -### CHAIN_URL - -URL to get main chain dump. Used on artifact get stage. - -### CHAIN_PATH - -Path to get main chain dump. If set, overrides `CHAIN_URL`. - -### NEOGO_VERSION - -Version of neo-go docker container for main chain deployment. - -## Main chain wallets - -There is a wallet with GAS that used for contract deployment: -`wallets/wallet.json`. This wallet has one account with **empty password**. - -``` -$ neo-go wallet nep17 balance \ - -w wallets/wallet.json \ - -r http://main-chain.neofs.devenv:30333 - -Account NbUgTSFvPmsRxmGeWpuuGeJUoRoi6PErcM -GAS: GasToken (d2a4cff31913016155e38e474a2c06d08be276cf) - Amount : 9978.0074623 - Updated: 34 -``` - -If you want to operate in main chain with your personal wallet (e.g. to make -a deposit in NeoFS contract), you can transfer GAS from there. - -1. Create new wallet. - -``` -$ neo-go wallet init -a -w wallets/neofs1.json - -Enter the name of the account > neofs1 -Enter passphrase > -Confirm passphrase > - -{ - "version": "3.0", - "accounts": [ - { - "address": "NXnzw3J9VvKXjM1BPAJK4QUpTtEQu4TpU6", -... -wallet successfully created, file location is wallets/neofs1.json -``` - -2. Transfer GAS from `wallets/wallet.json`. The password is empty. - -``` -$ neo-go wallet nep17 transfer \ - -w wallets/wallet.json \ - -r http://main-chain.neofs.devenv:30333 \ - --from NbUgTSFvPmsRxmGeWpuuGeJUoRoi6PErcM \ - --to NXnzw3J9VvKXjM1BPAJK4QUpTtEQu4TpU6 \ - --amount 50 \ - --token GAS -``` - -3. Check it's there. - -``` -$ neo-go wallet nep17 balance \ - -w wallets/neofs1.json \ - -r http://main-chain.neofs.devenv:30333 - -Account NXnzw3J9VvKXjM1BPAJK4QUpTtEQu4TpU6 -GAS: GasToken (d2a4cff31913016155e38e474a2c06d08be276cf) - Amount : 50 - Updated: 14689 -``` - -## Claim GAS from consensus node - -If there is no enough GAS on `wallets/wallet.json` account, you can claim some -GAS to consensus node's wallet and then transfer it. - -Consensus node is running with `services/chain/node-wallet.json` wallet. It has -multiple accounts with the password `one`. - - -Claim GAS to consensus node's wallet. Use account that contains NEO tokens. -``` -$ neo-go wallet claim \ - -w services/chain/node-wallet.json \ - -r http://main-chain.neofs.devenv:30333 \ - -a NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP \ -Password > -70e09bbd55846dcc7cee23905b737c63e5a80d32e387bce108bc6db8e641fb90 -``` - -Then you can transfer GAS the same way as it was done in previous section. - -``` -$ neo-go wallet nep17 transfer \ - -w services/chain/node-wallet.json \ - -r http://main-chain.neofs.devenv:30333 \ - --from NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP \ - --to NXnzw3J9VvKXjM1BPAJK4QUpTtEQu4TpU6 \ - --amount 50 \ - --token GAS -``` - -## NeoFS GAS deposit - -NeoFS identifies users by their Neo wallet key pair. To start using NeoFS in -devenv you need to transfer some GAS to NeoFS contract in main chain. - -Invoke `bin/deposit.sh` script by running `make prepare.ir` command to transfer -50 GAS from account in `wallets/wallet.json` file. Script enters passwords -automatically with `expect` utility. - -``` -$ make prepare.ir -Password > -Can't find matching token in the wallet. Querying RPC-node for balances. -6713c776f4102300691d9c3c493bcd3402434f5e32e8147e0a5bc72209a1e410 -``` - -Script converts addresses and executes this command: -``` -$ neo-go wallet nep17 transfer \ - -w wallets/wallet.json \ - -r http://main-chain.neofs.devenv:30333 \ - --from NbUgTSFvPmsRxmGeWpuuGeJUoRoi6PErcM \ - --to NerhjaqJsJt4LxMqUbkkVMpsF2d9TtcpFv \ - --token GAS \ - --amount 50 -``` - -You can specify any wallet address scripthash in the transfer's data argument, -and NeoFS deposit will be transferred to that address. - -``` -$ neo-go wallet nep17 transfer \ - -w wallets/wallet.json \ - -r http://main-chain.neofs.devenv:30333 \ - --from NbUgTSFvPmsRxmGeWpuuGeJUoRoi6PErcM \ - --to NerhjaqJsJt4LxMqUbkkVMpsF2d9TtcpFv \ - --token GAS \ - --amount 50 \ - hash160:bd711de066e9c2f7b502c7f3f0e0a6f1c8341edd -``` diff --git a/docs/faq.md b/docs/faq.md index 14f0c4f..12e5df7 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -1,9 +1,9 @@ # F.A.Q, tips and tricks -### How to export private key from Neo wallet for NeoFS use? +### How to export private key from Neo wallet for FrostFS use? -Private key for usage with NeoFS tools can be extracted from Neo wallet in three +Private key for usage with FrostFS tools can be extracted from Neo wallet in three simple steps. 1. Get the key in WIF format @@ -17,7 +17,7 @@ KxDgvEKzgSBPPfuVfw67oPQBSjidEiqTHURKSDL1R7yGaGYAeYnr 2. Convert form WIF to HEX ``` -$ neofs-cli util keyer KxDgvEKzgSBPPfuVfw67oPQBSjidEiqTHURKSDL1R7yGaGYAeYnr +$ frostfs-cli util keyer KxDgvEKzgSBPPfuVfw67oPQBSjidEiqTHURKSDL1R7yGaGYAeYnr PrivateKey 1dd37fba80fec4e6a6f13fd708d8dcb3b29def768017052f6c930fa1c5d90bbb PublicKey 031a6c6fbbdf02ca351745fa86b9ba5a9452d785ac4f7fc2b7548ca2a46c4fcf4a WIF KxDgvEKzgSBPPfuVfw67oPQBSjidEiqTHURKSDL1R7yGaGYAeYnr @@ -35,16 +35,16 @@ $ xxd wallets/wallet.key 00000010: b29d ef76 8017 052f 6c93 0fa1 c5d9 0bbb ...v.../l....... ``` -Later you will be able to provide wallet file in neofs-node config. +Later you will be able to provide wallet file in frostfs-node config. -### How to create Neo wallet JSON file using a NeoFS key file? +### How to create Neo wallet JSON file using a FrostFS key file? -You will need `neo-go` and `neofs-cli`. +You will need `neo-go` and `frostfs-cli`. 1. Get the WIF format of the private key ``` -$ neofs-cli util keyer -key ./services/ir/01.key | grep WIF | awk '{print $NF}' > temp_WIF +$ frostfs-cli util keyer -key ./services/ir/01.key | grep WIF | awk '{print $NF}' > temp_WIF ``` 2. Init a new empty Neo wallet diff --git a/docs/http_gate.md b/docs/http_gate.md index 9141929..b6b2591 100644 --- a/docs/http_gate.md +++ b/docs/http_gate.md @@ -1,8 +1,8 @@ # HTTP Protocol gateway -Protocol Gateway to access data in NeoFS using HTTP protocol. +Protocol Gateway to access data in FrostFS using HTTP protocol. -Source code and more information can be found in [project's GitHub repository](https://github.com/nspcc-dev/neofs-http-gate) +Source code and more information can be found in [project's repository](https://git.frostfs.info/TrueCloudLab/frostfs-http-gw) ## .env settings @@ -10,10 +10,10 @@ Source code and more information can be found in [project's GitHub repository](h Image version label to use for containers. -If you want to use locally built image, just set it's label here. Instead of +If you want to use locally built image, just set its label here. Instead of pulling from DockerHub, the local image will be used. -### HTTP_GW_IMAGE=nspccdev/neofs-http-gw +### HTTP_GW_IMAGE=truecloudlab/frostfs-http-gw Image label prefix to use for containers. @@ -21,9 +21,9 @@ Image label prefix to use for containers. - Create a new container ``` -$ neofs-cli --rpc-endpoint s01.neofs.devenv:8080 \ - --key wallets/wallet.key \ - container create --basic-acl readonly --await \ +$ frostfs-cli --rpc-endpoint s01.frostfs.devenv:8080 \ + --wallet wallets/wallet.key \ + container create --basic-acl private --await \ --policy "REP 1 SELECT 1 FROM *" container ID: 4LfREK1cetL4PUji5fqj9SgRTSmaC5jExEDK9HKCDjdP awaiting... @@ -32,17 +32,17 @@ container has been persisted on sidechain ``` - Put an object into the newly created container ``` -$ neofs-cli --rpc-endpoint s01.neofs.devenv:8080 \ - --key wallets/wallet.key \ +$ frostfs-cli --rpc-endpoint s01.frostfs.devenv:8080 \ + --wallet wallets/wallet.key \ object put --file /tmp/backup.jpeg \ --cid 4LfREK1cetL4PUji5fqj9SgRTSmaC5jExEDK9HKCDjdP [/tmp/backup.jpeg] Object successfully stored ID: 6EPpYqSFMGWrNLvYE9mNnut1CPKuPBKyi1ixHakzqsSB CID: 4LfREK1cetL4PUji5fqj9SgRTSmaC5jExEDK9HKCDjdP ``` -- Call `curl -sSI -XGET http://http.neofs.devenv/get//` +- Call `curl -sSI -XGET http://http.frostfs.devenv/get//` ``` -$ curl -sSI -XGET http://http.neofs.devenv/get/4LfREK1cetL4PUji5fqj9SgRTSmaC5jExEDK9HKCDjdP/6EPpYqSFMGWrNLvYE9mNnut1CPKuPBKyi1ixHakzqsSB +$ curl -sSI -XGET http://http.frostfs.devenv/get/4LfREK1cetL4PUji5fqj9SgRTSmaC5jExEDK9HKCDjdP/6EPpYqSFMGWrNLvYE9mNnut1CPKuPBKyi1ixHakzqsSB HTTP/1.1 200 OK Date: Thu, 03 Dec 2020 10:34:26 GMT Content-Type: image/jpeg diff --git a/docs/ir.md b/docs/ir.md index 28313f0..a870063 100644 --- a/docs/ir.md +++ b/docs/ir.md @@ -1,6 +1,6 @@ -# NeoFS Inner Ring +# FrostFS Inner Ring -NeoFS Inner Ring (Alphabet) node. According to governance scheme, Inner Ring +FrostFS Inner Ring (Alphabet) node. According to governance scheme, Inner Ring should contain Alphabet nodes that share key with one of side chain consensus nodes. In basic setup there is a single consensus node and single Inner Ring (Alphabet) node. @@ -14,6 +14,6 @@ Image version label to use for Inner Ring docker containers. If you want to use locally built image, just set it's label here. Instead of pulling from DockerHub, the local image will be used. -### IR_IMAGE=nspccdev/neofs-ir +### IR_IMAGE=truecloudlab/frostfs-ir Image label prefix to use for Inner Ring docker containers. diff --git a/docs/morph.md b/docs/morph.md index a30bfb5..d713bf9 100644 --- a/docs/morph.md +++ b/docs/morph.md @@ -1,72 +1,39 @@ -# N3 NeoFS side chain privnet service +# N3 FrostFS side chain privnet service A single-node N3 privnet deployment, running on -[neo-go](https://github.com/nspcc-dev/neo-go). Represents N3 NeoFS SideChain. +[neo-go](https://github.com/nspcc-dev/neo-go). Represents N3 FrostFS SideChain. Contracts deployed: -- Alphabet (AZ) [contract](https://github.com/nspcc-dev/neofs-contract/tree/master/alphabet) -- Audit [contract](https://github.com/nspcc-dev/neofs-contract/tree/master/audit) -- Balance [contract](https://github.com/nspcc-dev/neofs-contract/tree/master/balance) -- Container [contract](https://github.com/nspcc-dev/neofs-contract/tree/master/container) -- Netmap [contract](https://github.com/nspcc-dev/neofs-contract/tree/master/netmap) -- NeoFSID [contract](https://github.com/nspcc-dev/neofs-contract/tree/master/neofsid) -- Proxy [contract](https://github.com/nspcc-dev/neofs-contract/tree/master/proxy) -- Reputation [contract](https://github.com/nspcc-dev/neofs-contract/tree/master/reputation) - -RPC available at `http://morph-chain.neofs.devenv:30333`. +- Alphabet (AZ) [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/alphabet) +- Balance [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/balance) +- Container [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/container) +- FrostFS [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/frostfs) +- FrostFSID [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/frostfsid) +- NNS [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/nns) +- Netmap [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/netmap) +- Policy [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/policy) +- Processing [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/processing) +- Proxy [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/proxy) + +RPC available at `http://morph-chain.frostfs.devenv:30333`. ## .env settings -### MORPH_CHAIN_URL - -URL to get side chain dump. Used on artifact get stage. - -### MORPH_CHAIN_PATH - -Path to get side chain dump. If set, overrides `CHAIN_URL`. - ### NEOGO_VERSION Version of neo-go docker container for side chain deployment. -## Side chain wallets +## FrostFS global config -There is a wallet with GAS that used for contract deployment: -`wallets/wallet.json`. This wallet has one account with **empty password**. - -``` -$ neo-go wallet nep17 balance \ - -w wallets/wallet.json \ - -r http://morph-chain.neofs.devenv:30333 - -Account NbUgTSFvPmsRxmGeWpuuGeJUoRoi6PErcM -GAS: GasToken (d2a4cff31913016155e38e474a2c06d08be276cf) - Amount : 189826.0515316 - Updated: 3909 -NEOFS: NeoFS Balance (69550190e740b93f92dbd5dea52246f550391057) - Amount : 50 - Updated: 3909 -``` - -This way you can also monitor NeoFS internal balance of your account. - -## NeoFS global config - -NeoFS uses global configuration to store epoch duration, maximum object size, +FrostFS uses global configuration to store epoch duration, maximum object size, container fee and other network parameters. Global configuration is stored in netmap contract and managed by Inner Ring (Alphabet) nodes. To change these parameters use `make update.*` commands. Command down below changes epoch duration from 300 blocks (about 300 seconds with 1bps) to 30. -Script enters passwords automatically with `expect` utility. ``` $ make update.epoch_duration val=30 -Changing EpochDuration configration value to 30 -Enter account NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP password > -Sent invocation transaction bdc0fa88cd6719ef6df2b9c82de423ddec6141ca24255c2d0072688083b1de9d -Updating NeoFS epoch to 20 -Enter account NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP password > -Sent invocation transaction 12296e1ce24dd6c04edb9c56d0a1d0e26d3226adefb0333c74a28788f44a8d0f +Waiting for transactions to persist... ``` Read more about available configuration in Makefile help. @@ -76,10 +43,14 @@ $ make help ... Targets: ... - update.audit_fee Update audit fee per result in fixed 12 (make update.audit_fee val=100) - update.basic_income_rate Update basic income rate in fixed 12 (make update.basic_income_rate val=1000) - update.container_fee Update container fee per alphabet node in fixed 12 (make update.container_fee val=500) - update.eigen_trust_iterations Update amount of EigenTrust iterations (make update.eigen_trust_iterations val=2) - update.epoch_duration Update epoch duration in side chain blocks (make update.epoch_duration val=30) - update.max_object_size Update max object size in bytes (make update.max_object_size val=1000) + update.audit_fee Update audit fee per result in fixed 12 (make update.audit_fee val=100) + update.basic_income_rate Update basic income rate in fixed 12 (make update.basic_income_rate val=1000) + update.container_alias_fee Update container alias fee per alphabet node in fixed 12 (make update.container_alias_fee val=100) + update.container_fee Update container fee per alphabet node in fixed 12 (make update.container_fee val=500) + update.eigen_trust_alpha Update alpha parameter of EigenTrust algorithm in 0 <= f <= 1.0 (make update.eigen_trust_alpha val=0.2) + update.eigen_trust_iterations Update amount of EigenTrust iterations (make update.eigen_trust_iterations val=2) + update.epoch_duration Update epoch duration in side chain blocks (make update.epoch_duration val=30) + update.homomorphic_hashing_disable Update homomorphic hashing disabled flag (make update.homomorphic_hashing_disable val=true) + update.max_object_size Update max object size in bytes (make update.max_object_size val=1000) + update.system_dns Update system dns to resolve container names (make update.system_dns val=container) ``` diff --git a/docs/notary.md b/docs/notary.md index b973ead..2b16689 100644 --- a/docs/notary.md +++ b/docs/notary.md @@ -7,66 +7,12 @@ create containers, approve balance changes, update network map, tick epochs, etc. With notary service, it takes up to seven times fewer transactions to do these operations. Notary service calculates the exact amount of GAS to execute transaction, therefore operations are cheaper (withdraw fee **with** -notary is less than 0.5 GAS; withdraw fee **without** notary is up to 7.0 GAS). +notary is less than 0.5 GAS; withdraw fee **without** notary is up to 7.0 GAS). -By default, main chain service is running without notary service, and side chain -running with notary service. However, you can change that in configuration. +Currently, frostfs-dev-env contains single chain (see morph service) and it +enables notary service from the genesis block. -# Disable notary service in side chain - -To disable notary service in side chain do these steps. - -1. Update `.env` and choose notary disabled chain dump for side chain. - -``` -MORPH_CHAIN_URL="https://github.com/nspcc-dev/neofs-contract/releases/download/v0.9.0/devenv_sidechain_notary_disabled.gz" -``` - -Make sure to update chain dump files with `make get` target. - -2. Update `service/morph_chain/protocol.privnet.yml` and disable notary settings -and state root in header. - -```yaml -ProtocolConfiguration: - StateRootInHeader: false - P2PSigExtensions: false -ApplicationConfiguration: - P2PNotary: - Enabled: false -``` - -Chain dump without notary service does not have predefined network map. -Therefore, you need to wait about 5 minutes until new epoch tick with updated -network map. - - -3. Enable helper commands - -To enable helper commands such as `make tick.epoch` or `make update.epoch_duration` -make sure to export non-empty `NEOFS_NOTARY_DISABLED` environment variable. -``` -$ export NEOFS_NOTARY_DISABLED=1 -``` - -Use `unset` command to return it back. -``` -$ unset NEOFS_NOTARY_DISABLED -``` - -# Enable notary service in main chain - -To enable notary service in main chain do these steps. - -1. Update `.env` and choose notary enabled chain dump for main chain. - -``` -CHAIN_URL="https://github.com/nspcc-dev/neofs-contract/releases/download/v0.9.0/devenv_mainchain.gz" -``` - -Make sure to update chain dump files with `make get` target. - -2. Update `service/chain/protocol.privnet.yml` and enable notary settings. +To enable notary service, use neo-go configuration below. ```yaml ProtocolConfiguration: @@ -75,7 +21,3 @@ ApplicationConfiguration: P2PNotary: Enabled: true ``` - -Main chain generates a block once per 15 seconds, so Inner Ring takes about -15-30 seconds to make a notary deposit in main chain after startup. Then -neofs-dev-env is ready to work. diff --git a/docs/rest_gate.md b/docs/rest_gate.md index 2597a2c..ea2e95b 100644 --- a/docs/rest_gate.md +++ b/docs/rest_gate.md @@ -1,8 +1,8 @@ # REST Gateway -REST Gateway to access data in NeoFS using REST. +REST Gateway to access data in FrostFS using REST. -Source code and more information can be found in [project's GitHub repository](https://github.com/nspcc-dev/neofs-rest-gw) +Source code and more information can be found in [project's repository](https://git.frostfs.info/TrueCloudLab/frostfs-rest-gw) ## .env settings @@ -13,7 +13,7 @@ Image version label to use for containers. If you want to use locally built image, just set its label here. Instead of pulling from DockerHub, the local image will be used. -### REST_GW_IMAGE=nspccdev/neofs-rest-gw +### REST_GW_IMAGE=truecloudlab/frostfs-rest-gw Image label prefix to use for containers. @@ -22,7 +22,7 @@ Image label prefix to use for containers. - List container for specific owner: ```shell -$ curl http://rest.neofs.devenv:8090/v1/containers?ownerId=NbUgTSFvPmsRxmGeWpuuGeJUoRoi6PErcM | jq +$ curl http://rest.frostfs.devenv:8090/v1/containers?ownerId=NbUgTSFvPmsRxmGeWpuuGeJUoRoi6PErcM | jq { "containers": [ { @@ -49,7 +49,7 @@ $ curl http://rest.neofs.devenv:8090/v1/containers?ownerId=NbUgTSFvPmsRxmGeWpuuG - Get container info: ```shell -$ curl http://rest.neofs.devenv:8090/v1/containers/BKcAvz8awKKy9NGsGKi1Hoxxu9AjTGvjKMNMQamvdLmX | jq +$ curl http://rest.frostfs.devenv:8090/v1/containers/BKcAvz8awKKy9NGsGKi1Hoxxu9AjTGvjKMNMQamvdLmX | jq { "attributes": [ { @@ -67,4 +67,4 @@ $ curl http://rest.neofs.devenv:8090/v1/containers/BKcAvz8awKKy9NGsGKi1Hoxxu9AjT } ``` -See all available routes http://rest.neofs.devenv:8090/v1/docs +See all available routes http://rest.frostfs.devenv:8090/v1/docs diff --git a/docs/s3_gate.md b/docs/s3_gate.md index 9c00573..717fded 100644 --- a/docs/s3_gate.md +++ b/docs/s3_gate.md @@ -1,8 +1,8 @@ # S3 Protocol gateway -Protocol Gateway to access data in NeoFS using AWS S3 protocol +Protocol Gateway to access data in FrostFS using AWS S3 protocol -Source code and more information can be found in [project's GitHub repository](https://github.com/nspcc-dev/neofs-s3-gw) +Source code and more information can be found in [project's repository](https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw) ## .env settings @@ -10,9 +10,9 @@ Source code and more information can be found in [project's GitHub repository](h Image version label to use for containers. -If you want to use locally built image, just set it's label here. Instead of +If you want to use locally built image, just set its label here. Instead of pulling from DockerHub, the local image will be used. -### S3_GW_IMAGE=nspccdev/neofs-s3-gw +### S3_GW_IMAGE=truecloudlab/frostfs-s3-gw Image label prefix to use for containers. diff --git a/docs/storage.md b/docs/storage.md index d46db7e..64ba5fe 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -6,9 +6,9 @@ Image version label to use for Storage docker containers. -If you want to use locally built image, just set it's label here. Instead of +If you want to use locally built image, just set its label here. Instead of pulling from DockerHub, the local image will be used. -### NODE_IMAGE=nspccdev/neofs-ir +### NODE_IMAGE=truecloudlab/frostfs-ir Image label prefix to use for Storage docker containers. diff --git a/neofs-adm.yml b/frostfs-adm.yml similarity index 60% rename from neofs-adm.yml rename to frostfs-adm.yml index dc10a6e..5077431 100644 --- a/neofs-adm.yml +++ b/frostfs-adm.yml @@ -1,14 +1,16 @@ -rpc-endpoint: http://morph-chain.neofs.devenv:30333 +rpc-endpoint: http://morph-chain.frostfs.devenv:30333 +alphabet-wallets: ./services/ir network: max_object_size: 67108864 epoch_duration: 240 basic_income_rate: 100000000 homomorphic_hash_disabled: false + maintenance_mode_allowed: true fee: audit: 10000 candidate: 10000000000 - container: 1000 - container_alias: 500 + container: 0 + container_alias: 0 withdraw: 100000000 credentials: az: "one" diff --git a/neofs_config.mk b/frostfs_config.mk similarity index 57% rename from neofs_config.mk rename to frostfs_config.mk index 8f164d1..71eb8f1 100644 --- a/neofs_config.mk +++ b/frostfs_config.mk @@ -1,44 +1,40 @@ # Update epoch duration in side chain blocks (make update.epoch_duration val=30) update.epoch_duration: - @./bin/config.sh int EpochDuration $(val) + @./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config EpochDuration=$(val) # Update max object size in bytes (make update.max_object_size val=1000) update.max_object_size: - @./bin/config.sh int MaxObjectSize $(val) + @./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config MaxObjectSize=$(val) # Update audit fee per result in fixed 12 (make update.audit_fee val=100) update.audit_fee: - @./bin/config.sh int AuditFee $(val) + @./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config AuditFee=$(val) # Update container fee per alphabet node in fixed 12 (make update.container_fee val=500) update.container_fee: - @./bin/config.sh int ContainerFee $(val) + @./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config ContainerFee=$(val) # Update container alias fee per alphabet node in fixed 12 (make update.container_alias_fee val=100) update.container_alias_fee: - @./bin/config.sh int ContainerAliasFee $(val) + @./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config ContainerAliasFee=$(val) # Update amount of EigenTrust iterations (make update.eigen_trust_iterations val=2) update.eigen_trust_iterations: - @./bin/config.sh int EigenTrustIterations $(val) - + @./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config EigenTrustIterations=$(val) # Update system dns to resolve container names (make update.system_dns val=container) update.system_dns: - @./bin/config.sh string SystemDNS $(val) + @./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config SystemDNS=$(val) --force # Update alpha parameter of EigenTrust algorithm in 0 <= f <= 1.0 (make update.eigen_trust_alpha val=0.2) update.eigen_trust_alpha: - @./bin/config.sh string EigenTrustAlpha $(val) + @./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config EigenTrustAlpha=$(val) # Update basic income rate in fixed 12 (make update.basic_income_rate val=1000) update.basic_income_rate: - @./bin/config.sh int BasicIncomeRate $(val) + @./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config BasicIncomeRate=$(val) # Update homomorphic hashing disabled flag (make update.homomorphic_hashing_disable val=true) update.homomorphic_hashing_disable: - @./bin/config.sh bool HomomorphicHashingDisabled $(val) + @./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config HomomorphicHashingDisabled=$(val) -# Tick new epoch in side chain -tick.epoch: - @./bin/tick.sh diff --git a/help.mk b/help.mk index ab30ca9..7964236 100644 --- a/help.mk +++ b/help.mk @@ -1,5 +1,7 @@ .PHONY: help +HELP_MAKEFILE_LIST=$(filter-out %/artifacts.mk, $(MAKEFILE_LIST)) + # Show this help prompt help: @echo ' Usage:' @@ -8,4 +10,4 @@ help: @echo '' @echo ' Targets:' @echo '' - @awk '/^#/{ comment = substr($$0,3) } comment && /^[a-zA-Z][a-zA-Z0-9._-]+ ?:/{ print " ", $$1, comment }' $(MAKEFILE_LIST) | column -t -s ':' | grep -v 'IGNORE' | sort -u + @awk '/^#/{ comment = substr($$0,3) } comment && /^[a-zA-Z][a-zA-Z0-9._/-]+ ?:/{ print " ", $$1, comment }' $(HELP_MAKEFILE_LIST) | column -t -s ':' | grep -v 'IGNORE' | sort -u diff --git a/services/basenet/docker-compose.yml b/services/basenet/docker-compose.yml index abac92a..b2905a3 100644 --- a/services/basenet/docker-compose.yml +++ b/services/basenet/docker-compose.yml @@ -1,6 +1,5 @@ --- -version: "2.4" services: basenet: diff --git a/services/chain/.hosts b/services/chain/.hosts deleted file mode 100644 index a576e59..0000000 --- a/services/chain/.hosts +++ /dev/null @@ -1 +0,0 @@ -IPV4_PREFIX.50 main-chain.LOCAL_DOMAIN diff --git a/services/chain/artifacts.mk b/services/chain/artifacts.mk deleted file mode 100644 index 535b867..0000000 --- a/services/chain/artifacts.mk +++ /dev/null @@ -1,17 +0,0 @@ -# Download privnet chain dump with pre-deployed NeoFS contracts -get.chain: CHAIN_DUMP_NAME=devenv.dump. -get.chain: CHAIN_PATH?= -get.chain: - @mkdir -p ./vendor - -ifeq (${CHAIN_PATH},) - @echo "⇒ Download blockchain dump from ${CHAIN_URL}" - @curl \ - -sSL "${CHAIN_URL}" \ - -o ./vendor/chain.gz -else - @echo "⇒ Copy local archive ${CHAIN_PATH}" - @cp ${CHAIN_PATH} ./vendor/chain.gz -endif - - diff --git a/services/chain/docker-compose.yml b/services/chain/docker-compose.yml deleted file mode 100644 index b3e0822..0000000 --- a/services/chain/docker-compose.yml +++ /dev/null @@ -1,30 +0,0 @@ ---- - -version: "2.4" -services: - neofs_main_chain: - image: ${NEOGO_IMAGE}:${NEOGO_VERSION} - container_name: main_chain - command: ["node", "--config-path", "/config", "--privnet", "--debug"] - domainname: ${LOCAL_DOMAIN} - hostname: main-chain - networks: - chain_int: - internet: - ipv4_address: ${IPV4_PREFIX}.50 - stop_signal: SIGKILL - env_file: [ ".int_test.env" ] - environment: - - ACC=/chain.gz - volumes: - - ./../../vendor/chain.gz:/chain.gz - - ./protocol.privnet.yml:/config/protocol.privnet.yml - - ./node-wallet.json:/wallets/node-wallet.json - - ./../../vendor/hosts:/etc/hosts - - ./../../wallets/wallet.json:/wallets/wallet.json - -networks: - chain_int: - internet: - external: true - name: basenet_internet diff --git a/services/chain/node-wallet.json b/services/chain/node-wallet.json deleted file mode 100644 index 00bfe8e..0000000 --- a/services/chain/node-wallet.json +++ /dev/null @@ -1,76 +0,0 @@ -{ - "version": "3.0", - "name":null, - "accounts": [ - { - "address": "Nhfg3TbpwogLvDGVvAvqyThbsHgoSUKwtn", - "key": "6PYM8VdX2BSm7BSXKzV4Fz6S3R9cDLLWNrD9nMjxW352jEv3fsC8N3wNLY", - "label": "", - "contract": { - "script": "DCECs2Ir9AF73+MXxYrtX0x1PyBrfbiWBG+n13S7xL9/jcJBVuezJw==", - "parameters": [ - { - "name": "parameter0", - "type": "Signature" - } - ], - "deployed": false - }, - "lock": false, - "extra":null, - "isDefault": false - }, - { - "address": "NVTiAjNgagDkTr5HTzDmQP9kPwPHN5BgVq", - "key": "6PYM8VdX2BSm7BSXKzV4Fz6S3R9cDLLWNrD9nMjxW352jEv3fsC8N3wNLY", - "label": "", - "contract": { - "script": "EwwhAhA6f33QFlWFl/eWDSfFFqQ5T9loueZRVetLAT5AQEBuDCECp7xV/oaE4BGXaNEEujB5W9zIZhnoZK3SYVZyPtGFzWIMIQKzYiv0AXvf4xfFiu1fTHU/IGt9uJYEb6fXdLvEv3+NwgwhA9kMB99j5pDOd5EuEKtRrMlEtmhgI3tgjE+PgwnnHuaZFEGe0Nw6", - "parameters": [ - { - "name": "parameter0", - "type": "Signature" - }, - { - "name": "parameter1", - "type": "Signature" - }, - { - "name": "parameter2", - "type": "Signature" - } - ], - "deployed": false - }, - "lock": false, - "extra":null, - "isDefault": false - }, - { - "address": "NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP", - "key": "6PYM8VdX2BSm7BSXKzV4Fz6S3R9cDLLWNrD9nMjxW352jEv3fsC8N3wNLY", - "label": "", - "contract": { - "script": "EQwhArNiK/QBe9/jF8WK7V9MdT8ga324lgRvp9d0u8S/f43CEUGe0Nw6", - "parameters": [ - { - "name": "parameter0", - "type": "Signature" - } - ], - "deployed": false - }, - "lock": false, - "extra":null, - "isDefault": false - } - ], - "scrypt": { - "n": 16384, - "r": 8, - "p": 8 - }, - "extra": { - "Tokens": null - } -} diff --git a/services/chain/protocol.privnet.yml b/services/chain/protocol.privnet.yml deleted file mode 100644 index d614613..0000000 --- a/services/chain/protocol.privnet.yml +++ /dev/null @@ -1,59 +0,0 @@ -ProtocolConfiguration: - Magic: 56753 - MaxTraceableBlocks: 200000 - SecondsPerBlock: 1 - MemPoolSize: 50000 - StandbyCommittee: - - 02b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc2 - ValidatorsCount: 1 - SeedList: - - 172.200.0.1:20333 - VerifyBlocks: true - VerifyTransactions: true - P2PSigExtensions: false - -ApplicationConfiguration: - DBConfiguration: - Type: "boltdb" - BoltDBOptions: - FilePath: "./db/privnet.bolt" - NodePort: 20333 - Relay: true - DialTimeout: 3 - ProtoTickInterval: 2 - PingInterval: 30 - PingTimeout: 90 - MaxPeers: 10 - AttemptConnPeers: 5 - MinPeers: 0 - RPC: - Enabled: true - SessionEnabled: true - EnableCORSWorkaround: false - MaxGasInvoke: 15 - Port: 30333 - Prometheus: - Enabled: true - Port: 20001 - Pprof: - Enabled: true - Port: 20011 - UnlockWallet: - Path: "./wallets/node-wallet.json" - Password: "one" - Oracle: - Enabled: true - NeoFS: - Nodes: - - s01.neofs.devenv:8080 - - s02.neofs.devenv:8080 - - s03.neofs.devenv:8080 - - s04.neofs.devenv:8080 - UnlockWallet: - Path: "./wallets/node-wallet.json" - Password: "one" - P2PNotary: - Enabled: false - UnlockWallet: - Path: "./wallets/node-wallet.json" - Password: "one" diff --git a/services/coredns/.hosts b/services/coredns/.hosts deleted file mode 100644 index 5f77ce9..0000000 --- a/services/coredns/.hosts +++ /dev/null @@ -1 +0,0 @@ -IPV4_PREFIX.53 coredns.LOCAL_DOMAIN diff --git a/services/coredns/Corefile b/services/coredns/Corefile deleted file mode 100644 index 852c758..0000000 --- a/services/coredns/Corefile +++ /dev/null @@ -1,8 +0,0 @@ -. { - nns http://192.168.130.90:30333 - transfer { - to * - } - log - debug -} diff --git a/services/coredns/docker-compose.yml b/services/coredns/docker-compose.yml deleted file mode 100644 index 0972a7c..0000000 --- a/services/coredns/docker-compose.yml +++ /dev/null @@ -1,25 +0,0 @@ ---- - -version: "2.4" -services: - neofs_coredns: - image: ${COREDNS_IMAGE}:${COREDNS_VERSION} - container_name: coredns - domainname: ${LOCAL_DOMAIN} - hostname: coredns - restart: on-failure - networks: - coredns: - internet: - ipv4_address: ${IPV4_PREFIX}.53 - stop_signal: SIGKILL - env_file: [ ".int_test.env" ] - volumes: - - ./Corefile:/Corefile - - ./../../vendor/hosts:/etc/hosts - -networks: - coredns: - internet: - external: true - name: basenet_internet diff --git a/services/chain/.env b/services/grafana/.env similarity index 100% rename from services/chain/.env rename to services/grafana/.env diff --git a/services/grafana/.hosts b/services/grafana/.hosts new file mode 100644 index 0000000..537a644 --- /dev/null +++ b/services/grafana/.hosts @@ -0,0 +1,2 @@ +IPV4_PREFIX.122 grafana.LOCAL_DOMAIN +IPV4_PREFIX.123 loki.LOCAL_DOMAIN diff --git a/services/chain/.int_test.env b/services/grafana/.int_test.env similarity index 100% rename from services/chain/.int_test.env rename to services/grafana/.int_test.env diff --git a/services/grafana/docker-compose.yml b/services/grafana/docker-compose.yml new file mode 100644 index 0000000..3953400 --- /dev/null +++ b/services/grafana/docker-compose.yml @@ -0,0 +1,31 @@ +services: + grafana: + image: ${GRAFANA_IMAGE}:${GRAFANA_VERSION} + domainname: ${LOCAL_DOMAIN} + hostname: grafana + container_name: grafana + restart: on-failure + networks: + grafana_int: + internet: + ipv4_address: ${IPV4_PREFIX}.122 + volumes: + - ./../../vendor/hosts:/etc/hosts + - ./grafana.ini:/etc/grafana/grafana.ini + - ./provisioning:/etc/grafana/provisioning + stop_signal: SIGKILL + env_file: [ ".env", ".int_test.env" ] + + loki: + image: ${LOKI_IMAGE}:${LOKI_VERSION} + command: -config.file=/etc/loki/local-config.yaml + networks: + grafana_int: + internet: + ipv4_address: ${IPV4_PREFIX}.123 + +networks: + grafana_int: + internet: + external: true + name: basenet_internet diff --git a/services/grafana/grafana.ini b/services/grafana/grafana.ini new file mode 100644 index 0000000..3c14906 --- /dev/null +++ b/services/grafana/grafana.ini @@ -0,0 +1,7 @@ +[auth.anonymous] +enabled = true +org_name = Main Org. +org_role = Editor + +[dashboards] +default_home_dashboard_path= /etc/grafana/provisioning/dashboards/overview.json \ No newline at end of file diff --git a/services/grafana/provisioning/dashboards/14061_rev1.json b/services/grafana/provisioning/dashboards/14061_rev1.json new file mode 100644 index 0000000..deec62b --- /dev/null +++ b/services/grafana/provisioning/dashboards/14061_rev1.json @@ -0,0 +1,1162 @@ +{ + "__inputs": [], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "7.2.0" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "A quickstart to setup the Prometheus Go runtime exporter with preconfigured dashboards, alerting rules, and recording rules.", + "editable": true, + "graphTooltip": 0, + "id": null, + "iteration": 1602794777869, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Average total bytes of memory reserved across all process instances of a job.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg by(job)(go_memstats_sys_bytes{job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{job}} (avg)", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total Reserved Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Average stack memory usage across all instances of a job.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg by (job) (go_memstats_stack_sys_bytes{job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{job}}: stack inuse (avg)", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Stack Memory Use", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Average memory reservations by the runtime, not for stack or heap, across all instances of a job.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 26, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg by (job)(go_memstats_mspan_sys_bytes{job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{instance}}: mspan (avg)", + "refId": "B" + }, + { + "expr": "avg by (job)(go_memstats_mcache_sys_bytes{job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{instance}}: mcache (avg)", + "refId": "D" + }, + { + "expr": "avg by (job)(go_memstats_buck_hash_sys_bytes{job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{instance}}: buck hash (avg)", + "refId": "E" + }, + { + "expr": "avg by (job)(go_memstats_gc_sys_bytes{job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{job}}: gc (avg)", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Other Memory Reservations", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Average memory reserved, and actually in use, by the heap, across all instances of a job.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg by (job)(go_memstats_heap_sys_bytes{job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{job}}: heap reserved (avg)", + "refId": "B" + }, + { + "expr": "avg by (job)(go_memstats_heap_inuse_bytes{job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{job}}: heap in use (avg)", + "refId": "A" + }, + { + "expr": "avg by (job)(go_memstats_heap_alloc_bytes{job=~\"tns_app\",instance=~\".*\"})", + "interval": "", + "legendFormat": "{{job}}: heap alloc (avg)", + "refId": "C" + }, + { + "expr": "avg by (job)(go_memstats_heap_idle_bytes{job=~\"tns_app\",instance=~\".*\"})", + "interval": "", + "legendFormat": "{{job}}: heap idle (avg)", + "refId": "D" + }, + { + "expr": "avg by (job)(go_memstats_heap_released_bytes{job=~\"tns_app\",instance=~\".*\"})", + "interval": "", + "legendFormat": "{{job}}: heap released (avg)", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Heap Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Average allocation rate in bytes per second, across all instances of a job.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 1, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg by (job)(rate(go_memstats_alloc_bytes_total{job=\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "{{job}}: bytes malloced/s (avg)", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Allocation Rate, Bytes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Average rate of heap object allocation, across all instances of a job.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(go_memstats_mallocs_total{job=\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "interval": "", + "legendFormat": "{{job}}: obj mallocs/s (avg)", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Heap Object Allocation Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Average number of live memory objects across all instances of a job.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg by(job)(go_memstats_mallocs_total{job=\"$job\", instance=~\"$instance\"} - go_memstats_frees_total{job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{job}}: object count (avg)", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of Live Objects", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Average number of goroutines across instances of a job.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg by (job)(go_goroutines{job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{job}}: goroutine count (avg)", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg by (job)(go_gc_duration_seconds{quantile=\"0\", job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{job}}: min gc time (avg)", + "refId": "A" + }, + { + "expr": "avg by (job)(go_gc_duration_seconds{quantile=\"1\", job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{job}}: max gc time (avg)", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "GC min & max duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "The number used bytes at which the runtime plans to perform the next GC, averaged across all instances of a job.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "hiddenSeries": false, + "id": 27, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg by (job)(go_memstats_next_gc_bytes{job=\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{job}} next gc bytes (avg)", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Next GC, Bytes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "5s", + "schemaVersion": 25, + "style": "dark", + "tags": [ + "go", + "golang" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": null, + "current": {}, + "datasource": "$datasource", + "definition": "label_values(go_info, job)", + "hide": 0, + "includeAll": false, + "label": "job", + "multi": false, + "name": "job", + "options": [], + "query": "label_values(go_info, job)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": "", + "current": {}, + "datasource": "$datasource", + "definition": "label_values(go_info{job=\"$job\"}, instance)", + "hide": 0, + "includeAll": true, + "label": "instance", + "multi": true, + "name": "instance", + "options": [], + "query": "label_values(go_info{job=\"$job\"}, instance)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Go Runtime Exporter Quickstart and Dashboardby Grafana Labs", + "uid": "CgCw8jKZz3", + "version": 3, + "gnetId": 14061 +} \ No newline at end of file diff --git a/services/grafana/provisioning/dashboards/blobovnicza-buckets.json b/services/grafana/provisioning/dashboards/blobovnicza-buckets.json new file mode 100644 index 0000000..e35272a --- /dev/null +++ b/services/grafana/provisioning/dashboards/blobovnicza-buckets.json @@ -0,0 +1,361 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 7, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "$method", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 20, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "< 0.005s", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.005s - 0.01s", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.01s - 0.025s", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.025s - 0.05s", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.05s - 0.1s", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.1s - 0.25s", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.25s - 0.5s", + "range": true, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.5s - 1s", + "range": true, + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "1s - 2.5s", + "range": true, + "refId": "I" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "2.5s - 5s", + "range": true, + "refId": "J" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "5s - 10s", + "range": true, + "refId": "K" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"+Inf\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "> 10s", + "range": true, + "refId": "L" + } + ], + "title": "$method", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "s01.frostfs.devenv:9090", + "value": "s01.frostfs.devenv:9090" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket,instance)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "instance", + "options": [], + "query": { + "query": "label_values(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket,instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "3Xd2VDqghZnMnaRoP4bKeP", + "value": "3Xd2VDqghZnMnaRoP4bKeP" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{instance=\"$instance\"},shard_id)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "shard_id", + "options": [], + "query": { + "query": "label_values(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{instance=\"$instance\"},shard_id)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": "Put", + "value": "Put" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{instance=\"$instance\", shard_id=\"$shard_id\"},method)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "method", + "options": [], + "query": { + "query": "label_values(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{instance=\"$instance\", shard_id=\"$shard_id\"},method)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Storage node blobovnicza bucket distribution", + "uid": "e4097619-1ce9-4039-bd62-4f5b4bba8d26", + "version": 2, + "weekStart": "" + } \ No newline at end of file diff --git a/services/grafana/provisioning/dashboards/dashboard.yml b/services/grafana/provisioning/dashboards/dashboard.yml new file mode 100644 index 0000000..bc9f26e --- /dev/null +++ b/services/grafana/provisioning/dashboards/dashboard.yml @@ -0,0 +1,10 @@ +apiVersion: 1 + +providers: + - name: Prometheus + orgId: 1 + type: file + disableDeletion: false + editable: true + options: + path: /etc/grafana/provisioning/dashboards diff --git a/services/grafana/provisioning/dashboards/engine-buckets.json b/services/grafana/provisioning/dashboards/engine-buckets.json new file mode 100644 index 0000000..8bc704b --- /dev/null +++ b/services/grafana/provisioning/dashboards/engine-buckets.json @@ -0,0 +1,335 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 5, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "$method", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 20, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])", + "hide": false, + "legendFormat": "< 0.005s", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.005s - 0.01s", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.01s - 0.025s", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.025s - 0.05s", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.05s - 0.1s", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.1s - 0.25s", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.25s - 0.5s", + "range": true, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.5s - 1s", + "range": true, + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "1s - 2.5s", + "range": true, + "refId": "I" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "2.5s - 5s", + "range": true, + "refId": "J" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "5s - 10s", + "range": true, + "refId": "K" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"+Inf\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "> 10s", + "range": true, + "refId": "L" + } + ], + "title": "$method", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "s01.frostfs.devenv:9090", + "value": "s01.frostfs.devenv:9090" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_engine_request_duration_seconds_bucket,instance)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "instance", + "options": [], + "query": { + "query": "label_values(frostfs_node_engine_request_duration_seconds_bucket,instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": "Put", + "value": "Put" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_engine_request_duration_seconds_bucket{instance=\"$instance\"},method)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "method", + "options": [], + "query": { + "query": "label_values(frostfs_node_engine_request_duration_seconds_bucket{instance=\"$instance\"},method)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Storage node engine bucket distribution", + "uid": "f886d27c-b877-495a-a1cb-d02839115cea", + "version": 2, + "weekStart": "" + } \ No newline at end of file diff --git a/services/grafana/provisioning/dashboards/fstree-buckets.json b/services/grafana/provisioning/dashboards/fstree-buckets.json new file mode 100644 index 0000000..151376b --- /dev/null +++ b/services/grafana/provisioning/dashboards/fstree-buckets.json @@ -0,0 +1,361 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 9, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "$method", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 20, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "< 0.005s", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.005s - 0.01s", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.01s - 0.025s", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.025s - 0.05s", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.05s - 0.1s", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.1s - 0.25s", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.25s - 0.5s", + "range": true, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.5s - 1s", + "range": true, + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "1s - 2.5s", + "range": true, + "refId": "I" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "2.5s - 5s", + "range": true, + "refId": "J" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "5s - 10s", + "range": true, + "refId": "K" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"+Inf\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "> 10s", + "range": true, + "refId": "L" + } + ], + "title": "$method", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "s01.frostfs.devenv:9090", + "value": "s01.frostfs.devenv:9090" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_fstree_request_duration_seconds_bucket,instance)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "instance", + "options": [], + "query": { + "query": "label_values(frostfs_node_fstree_request_duration_seconds_bucket,instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "3Xd2VDqghZnMnaRoP4bKeP", + "value": "3Xd2VDqghZnMnaRoP4bKeP" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_fstree_request_duration_seconds_bucket{instance=\"$instance\"},shard_id)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "shard_id", + "options": [], + "query": { + "query": "label_values(frostfs_node_fstree_request_duration_seconds_bucket{instance=\"$instance\"},shard_id)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "Get", + "value": "Get" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_fstree_request_duration_seconds_bucket{instance=\"$instance\", shard_id=\"$shard_id\"},method)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "method", + "options": [], + "query": { + "query": "label_values(frostfs_node_fstree_request_duration_seconds_bucket{instance=\"$instance\", shard_id=\"$shard_id\"},method)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Storage node fstree bucket distribution", + "uid": "e6407e43-9419-445b-b715-284d69201519", + "version": 2, + "weekStart": "" + } \ No newline at end of file diff --git a/services/grafana/provisioning/dashboards/grpc-buckets.json b/services/grafana/provisioning/dashboards/grpc-buckets.json new file mode 100644 index 0000000..ba7cd5f --- /dev/null +++ b/services/grafana/provisioning/dashboards/grpc-buckets.json @@ -0,0 +1,362 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 4, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "$service", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 20, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(grpc_server_handling_seconds_bucket{le=\"0.005\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])", + "hide": false, + "legendFormat": "< 0.005s", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"0.01\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"0.005\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.005s - 0.01s", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"0.025\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"0.01\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.01s - 0.025s", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"0.05\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"0.025\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.025s - 0.05s", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"0.1\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"0.05\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.05s - 0.1s", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"0.25\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"0.1\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.1s - 0.25s", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"0.5\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"0.25\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.25s - 0.5s", + "range": true, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"1\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"0.5\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "0.5s - 1s", + "range": true, + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"2.5\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"1\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "1s - 2.5s", + "range": true, + "refId": "I" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"5\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"2.5\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "2.5s - 5s", + "range": true, + "refId": "J" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"10\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"5\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "5s - 10s", + "range": true, + "refId": "K" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"+Inf\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"10\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n", + "hide": false, + "legendFormat": "> 10s", + "range": true, + "refId": "L" + } + ], + "title": "$method", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "s01.frostfs.devenv:9090", + "value": "s01.frostfs.devenv:9090" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(grpc_server_handling_seconds_bucket,instance)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "instance", + "options": [], + "query": { + "query": "label_values(grpc_server_handling_seconds_bucket,instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": "", + "current": { + "selected": false, + "text": "neo.fs.v2.object.ObjectService", + "value": "neo.fs.v2.object.ObjectService" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(grpc_server_handling_seconds_bucket{instance=\"$instance\"},grpc_service)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "service", + "options": [], + "query": { + "query": "label_values(grpc_server_handling_seconds_bucket{instance=\"$instance\"},grpc_service)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "Put", + "value": "Put" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(grpc_server_handling_seconds_bucket{grpc_service=\"$service\", instance=\"$instance\"},grpc_method)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "method", + "options": [], + "query": { + "query": "label_values(grpc_server_handling_seconds_bucket{grpc_service=\"$service\", instance=\"$instance\"},grpc_method)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "GRPC server bucket distribution", + "uid": "a6830bc6-cffa-4e29-9622-6a10921748b0", + "version": 8, + "weekStart": "" + } \ No newline at end of file diff --git a/services/grafana/provisioning/dashboards/metabase-buckets.json b/services/grafana/provisioning/dashboards/metabase-buckets.json new file mode 100644 index 0000000..2fba386 --- /dev/null +++ b/services/grafana/provisioning/dashboards/metabase-buckets.json @@ -0,0 +1,361 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 8, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "$method", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 20, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "< 0.005s", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.005s - 0.01s", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.01s - 0.025s", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.025s - 0.05s", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.05s - 0.1s", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.1s - 0.25s", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.25s - 0.5s", + "range": true, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.5s - 1s", + "range": true, + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "1s - 2.5s", + "range": true, + "refId": "I" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "2.5s - 5s", + "range": true, + "refId": "J" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "5s - 10s", + "range": true, + "refId": "K" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"+Inf\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "> 10s", + "range": true, + "refId": "L" + } + ], + "title": "$method", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "s01.frostfs.devenv:9090", + "value": "s01.frostfs.devenv:9090" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_metabase_request_duration_seconds_bucket,instance)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "instance", + "options": [], + "query": { + "query": "label_values(frostfs_node_metabase_request_duration_seconds_bucket,instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "3Xd2VDqghZnMnaRoP4bKeP", + "value": "3Xd2VDqghZnMnaRoP4bKeP" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_metabase_request_duration_seconds_bucket{instance=\"$instance\"},shard_id)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "shard_id", + "options": [], + "query": { + "query": "label_values(frostfs_node_metabase_request_duration_seconds_bucket{instance=\"$instance\"},shard_id)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": "Delete", + "value": "Delete" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_metabase_request_duration_seconds_bucket{instance=\"$instance\", shard_id=\"$shard_id\"},method)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "method", + "options": [], + "query": { + "query": "label_values(frostfs_node_metabase_request_duration_seconds_bucket{instance=\"$instance\", shard_id=\"$shard_id\"},method)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Storage node metabase bucket distribution", + "uid": "e110ed66-d086-461d-958a-efc6c55607d2", + "version": 1, + "weekStart": "" + } \ No newline at end of file diff --git a/services/grafana/provisioning/dashboards/overview.json b/services/grafana/provisioning/dashboards/overview.json new file mode 100644 index 0000000..46a5987 --- /dev/null +++ b/services/grafana/provisioning/dashboards/overview.json @@ -0,0 +1,327 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Overview panel for all services", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 3, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by service instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance) (\n rate(grpc_server_started_total[$__rate_interval])\n)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Error rate by service instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(rate(grpc_server_handled_total{grpc_code!=\"OK\"}[$__rate_interval])) by (instance)\n / \nsum(rate(grpc_server_started_total{}[$__rate_interval])) by (instance)\n * 100.0", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Resident memory size by service instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance) (\n process_resident_memory_bytes\n)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Memory", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Overview", + "uid": "da5f53f7-c069-44b5-8372-462333c9fe4a", + "version": 1, + "weekStart": "" + } \ No newline at end of file diff --git a/services/grafana/provisioning/dashboards/storage-node-logs.json b/services/grafana/provisioning/dashboards/storage-node-logs.json new file mode 100644 index 0000000..3c91d52 --- /dev/null +++ b/services/grafana/provisioning/dashboards/storage-node-logs.json @@ -0,0 +1,358 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Overview panel for all services", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "count_over_time({hostname=~\"$hostname\"} |~ \"$level\" |~ `$filter` [1s])", + "queryType": "range", + "refId": "A" + } + ], + "title": "Log intensity", + "type": "timeseries" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.width", + "value": 226 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "labels" + }, + "properties": [ + { + "id": "custom.width", + "value": 424 + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 4, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "{hostname=~\"$hostname\"} |~ `$filter` | json level |~ \"$level\"", + "queryType": "range", + "refId": "A" + } + ], + "title": "Log", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "id": true, + "tsNs": true + }, + "indexByName": {}, + "renameByName": {} + } + } + ], + "type": "table" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "label": "Hostname", + "multi": true, + "name": "hostname", + "options": [], + "query": { + "label": "hostname", + "refId": "LokiVariableQueryEditor-VariableQuery", + "stream": "", + "type": 1 + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "hide": 0, + "includeAll": true, + "label": "Level", + "multi": true, + "name": "level", + "options": [ + { + "selected": true, + "text": "All", + "value": "$__all" + }, + { + "selected": false, + "text": "debug", + "value": "debug" + }, + { + "selected": false, + "text": "info", + "value": "info" + }, + { + "selected": false, + "text": "warn", + "value": "warn" + }, + { + "selected": false, + "text": "error", + "value": "error" + }, + { + "selected": false, + "text": "dpanic", + "value": "dpanic" + }, + { + "selected": false, + "text": "panic", + "value": "panic" + }, + { + "selected": false, + "text": "fatal", + "value": "fatal" + } + ], + "query": "debug,info,warn,error,dpanic,panic,fatal", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + }, + { + "current": { + "selected": false, + "text": "", + "value": "" + }, + "hide": 0, + "label": "Filter", + "name": "filter", + "options": [ + { + "selected": true, + "text": "", + "value": "" + } + ], + "query": "", + "skipUrlSync": false, + "type": "textbox" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Storage node logs", + "uid": "bef4b42d-c74e-4241-96cb-d239113f44bc", + "version": 1, + "weekStart": "" + } \ No newline at end of file diff --git a/services/grafana/provisioning/dashboards/storage-node.json b/services/grafana/provisioning/dashboards/storage-node.json new file mode 100644 index 0000000..318f8c7 --- /dev/null +++ b/services/grafana/provisioning/dashboards/storage-node.json @@ -0,0 +1,6796 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 22, + "panels": [], + "title": "General", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Instance state", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [ + { + "options": { + "0": { + "index": 0, + "text": "Undefined" + }, + "1": { + "index": 1, + "text": "Starting" + }, + "2": { + "index": 2, + "text": "Ready" + }, + "3": { + "index": 3, + "text": "Shutting down" + } + }, + "type": "value" + } + ], + "max": 3, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "string" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 21, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_state_health{job=\"node\", instance=\"$instance\"}\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "state", + "range": true, + "refId": "A" + } + ], + "title": "State", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Current epoch on the instance", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 32, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "frostfs_node_ir_epoch{job=\"node\", instance=\"$instance\"}", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Epoch", + "type": "stat" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 2, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by gRPC method", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, grpc_service, grpc_method) (\n rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed requests", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "neo.fs.v2.object.ObjectService.Put" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(grpc_server_handled_total{instance=\"$instance\",grpc_code!=\"OK\"}[$__rate_interval])) by (grpc_service, grpc_method)\n / \nsum(rate(grpc_server_started_total{instance=\"$instance\"}[$__rate_interval])) by (grpc_service, grpc_method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the requests for selected quantile", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Server", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 76, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by gRPC method", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 77, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, grpc_service, grpc_method) (\n rate(grpc_client_started_total{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed requests", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 78, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(grpc_client_handled_total{instance=\"$instance\",grpc_code!=\"OK\"}[$__rate_interval])) by (grpc_service, grpc_method)\n / \nsum(rate(grpc_client_started_total{instance=\"$instance\"}[$__rate_interval])) by (grpc_service, grpc_method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the requests for selected quantile", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 79, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(grpc_client_handling_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (grpc_service,grpc_method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{grpc_service}}.{{grpc_method}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Client", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 27, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by object service method", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 28, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, method) (\n rate(frostfs_node_object_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed object service requests", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 29, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(frostfs_node_object_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (method)\n / \nsum(rate(frostfs_node_object_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the object service requests for selected quantile", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 30, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_object_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Bytes per second by object service method", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 31, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, method) (\n rate(frostfs_node_object_request_payload_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Payload rate", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Object service", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 58, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by storage engine service method", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 59, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, method) (\n rate(frostfs_node_engine_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the engine requests for selected quantile for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 60, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_engine_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Payload size by shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 61, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_engine_payload_size_bytes{instance=\"$instance\"}\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Payload", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Objects count by shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 62, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_engine_objects_total{instance=\"$instance\"}\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, type {{type}}", + "range": true, + "refId": "A" + } + ], + "title": "Objects count", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Errors count by shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 44 + }, + "id": 63, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_engine_errors_total{instance=\"$instance\"}\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Errors count", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Mode for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1.5, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 54 + }, + "id": 64, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(frostfs_node_engine_mode_info{job=\"node\", instance=\"$instance\"}) by (shard_id, mode)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, mode {{mode}}", + "range": true, + "refId": "A" + } + ], + "title": "Shard mode", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Engine", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 33, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by metabase method for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 35, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id, method) (\n rate(frostfs_node_metabase_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed metabase requests for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 37, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(frostfs_node_metabase_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (shard_id, method)\n / \nsum(rate(frostfs_node_metabase_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (shard_id, method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the metabase requests for selected quantile for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 36, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_metabase_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Metabase mode for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1.5, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 34, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(frostfs_node_metabase_mode{job=\"node\", instance=\"$instance\"}) by (shard_id, mode)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, mode {{mode}}", + "range": true, + "refId": "A" + } + ], + "title": "Mode", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Metabase", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 12, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by writecache method for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id, method) (\n rate(frostfs_node_writecache_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed writecache requests for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(frostfs_node_writecache_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (shard_id, method)\n / \nsum(rate(frostfs_node_writecache_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (shard_id, method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the writecache requests for selected quantile for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_writecache_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Actual count of the objects, that writecache stores", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 42 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_writecache_actual_objects_total{job=\"node\", instance=\"$instance\"}", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, storage {{storage}}", + "range": true, + "refId": "A" + } + ], + "title": "Actual count", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Estimated size of the objects, that writecache stores", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 52 + }, + "id": 17, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_writecache_estimated_size_bytes{job=\"node\", instance=\"$instance\"}", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, storage {{storage}}", + "range": true, + "refId": "A" + } + ], + "title": "Estimated size", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Rate of the writecache background operations", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 62 + }, + "id": 18, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (shard_id, storage, operation) (rate(frostfs_node_writecache_operations_total{job=\"node\", instance=\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, storage {{storage}}, operation {{operation}}", + "range": true, + "refId": "A" + } + ], + "title": "Operations rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Writecache mode for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1.5, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 72 + }, + "id": 19, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(frostfs_node_writecache_mode_info{job=\"node\", instance=\"$instance\"}) by (shard_id, mode)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, mode {{mode}}", + "range": true, + "refId": "A" + } + ], + "title": "Mode", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Writecache", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 38, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by blobstore method for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 83 + }, + "id": 39, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id, method) (\n rate(frostfs_node_blobstore_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed blobstore requests for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 93 + }, + "id": 40, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(frostfs_node_blobstore_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (shard_id, method)\n / \nsum(rate(frostfs_node_blobstore_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (shard_id, method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the metabase requests for selected quantile for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 103 + }, + "id": 41, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_blobstore_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Bytes per second by blobstore shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 113 + }, + "id": 43, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id) (\n rate(frostfs_node_blobstore_put_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Write rate, shard {{shard_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id) (\n rate(frostfs_node_blobstore_get_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Read rate, shard {{shard_id}}", + "range": true, + "refId": "B" + } + ], + "title": "Payload rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Blobstore mode for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1.5, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 123 + }, + "id": 42, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(frostfs_node_blobstore_mode{job=\"node\", instance=\"$instance\"}) by (shard_id, mode)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, mode {{mode}}", + "range": true, + "refId": "A" + } + ], + "title": "Mode", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Blobstore", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 52, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by fstree method for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 84 + }, + "id": 53, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id, method) (\n rate(frostfs_node_fstree_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed fstree requests for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 94 + }, + "id": 54, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(frostfs_node_fstree_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (shard_id, method)\n / \nsum(rate(frostfs_node_fstree_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (shard_id, method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the fstree requests for selected quantile for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 104 + }, + "id": 55, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_fstree_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Bytes per second by fstree shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 114 + }, + "id": 56, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id) (\n rate(frostfs_node_fstree_put_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Write rate, shard {{shard_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id) (\n rate(frostfs_node_fstree_get_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Read rate, shard {{shard_id}}", + "range": true, + "refId": "B" + } + ], + "title": "Payload rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "FSTree mode for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1.5, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 124 + }, + "id": 57, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(frostfs_node_fstree_mode{job=\"node\", instance=\"$instance\"}) by (shard_id, mode)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, mode {{mode}}", + "range": true, + "refId": "A" + } + ], + "title": "Mode", + "transformations": [], + "type": "timeseries" + } + ], + "title": "FSTree", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 44, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by blobovnicza method for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 45, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id, method) (\n rate(frostfs_node_blobovnicza_tree_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed blobovnicza requests for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 46, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(frostfs_node_blobsovnicza_tree_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (shard_id, method)\n / \nsum(rate(frostfs_node_blobsovnicza_tree_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (shard_id, method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the blobovnicza requests for selected quantile for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 47, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Bytes per second by blobovnicza shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 46 + }, + "id": 48, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id) (\n rate(frostfs_node_blobovnicza_tree_put_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Write rate, shard {{shard_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id) (\n rate(frostfs_node_blobovnicza_tree_get_bytes{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Read rate, shard {{shard_id}}", + "range": true, + "refId": "B" + } + ], + "title": "Payload rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Open blobovnicza size for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 56 + }, + "id": 50, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_blobovnicza_tree_open_blobovnicza_size_bytes{instance=\"$instance\"}\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Open blobovnicza size", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Items count of open blobovnicza for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 66 + }, + "id": 80, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_blobovnicza_tree_open_blobovnicza_items_total{instance=\"$instance\"}\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Open blobovnicza items", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Count of open blobovniczas for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 76 + }, + "id": 51, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_blobovnicza_tree_open_blobovnicza_count{instance=\"$instance\"}\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Open blobovnicza count", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Blobovnicza mode for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1.5, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 86 + }, + "id": 49, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(frostfs_node_blobovnicza_tree_mode{job=\"node\", instance=\"$instance\"}) by (shard_id, mode)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, mode {{mode}}", + "range": true, + "refId": "A" + } + ], + "title": "Mode", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Blobovnicza", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 23, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Count of in-flight requests", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 24, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_replicator_in_flight_requests_total{job=\"node\", instance=\"$instance\"}", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "in-flight requests", + "range": true, + "refId": "A" + } + ], + "title": "In-flight request", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Rate of the objects replicated", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 25, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "rate(frostfs_node_replicator_processed_objects_total{job=\"node\", instance=\"$instance\"}[$__rate_interval])", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "object processing rate", + "range": true, + "refId": "A" + } + ], + "title": "Objects processing rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Rate of the objects payload replicated", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 26, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "rate(frostfs_node_replicator_total_replicated_payload_size_bytes{job=\"node\", instance=\"$instance\"}[$__rate_interval])", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "payload processing rate", + "range": true, + "refId": "A" + } + ], + "title": "Payload processing rate", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Replicator", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 6, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Requests per second by pilorama method for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, shard_id, method) (\n rate(frostfs_node_pilorama_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Pilorama RPS", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed pilorama requests for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(frostfs_node_pilorama_request_duration_seconds_count{instance=\"$instance\",success!=\"true\"}[$__rate_interval])) by (shard_id, method)\n / \nsum(rate(frostfs_node_pilorama_request_duration_seconds_count{instance=\"$instance\"}[$__rate_interval])) by (shard_id, method)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Pilorama error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the pilorama requests for selected quantile for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_pilorama_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (shard_id, method, le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}, shard {{shard_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Pilorama latency", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Duration of waiting time for replication, replication task duration and synchronization for selected quantile", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_treeservice_replicate_wait_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (success,le)\n)", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "replicate wait success:{{success}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile,\n sum(rate(frostfs_node_treeservice_replicate_task_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (success,le)\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "replicate task success:{{success}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_treeservice_sync_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (success,le)\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "sync success:{{success}}", + "range": true, + "refId": "C" + } + ], + "title": "Replication and synchronization", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Pilorama mode for each shard", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1.5, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 51 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(frostfs_node_pilorama_mode{job=\"node\", instance=\"$instance\"}) by (shard_id, mode)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, mode {{mode}}", + "range": true, + "refId": "A" + } + ], + "title": "Pilorama mode", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Tree service", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 66, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Count of objects GC marked to remove, deleted or failed to delete from disk", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 65, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_garbage_collector_deleted_objects_total{instance=\"$instance\"}\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, {{status}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "frostfs_node_garbage_collector_marked_for_removal_objects_total{instance=\"$instance\"}\n", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Shard {{shard_id}}, marked for remove, {{object_type}}", + "range": true, + "refId": "B" + } + ], + "title": "Objects count", + "transformations": [], + "type": "timeseries" + } + ], + "title": "GC", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 67, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Number of endpoint switches", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 68, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "frostfs_node_morph_switches_total{job=\"node\", instance=\"$instance\"}", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Endpoint switches", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Index of the last received block", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 69, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "frostfs_node_morph_last_block{job=\"node\", instance=\"$instance\"}", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Last block", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Rate of notifications received by notification type", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 70, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "rate(frostfs_node_morph_notifications_total{job=\"node\", instance=\"$instance\"}[$__rate_interval])", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{notification_type}}", + "range": true, + "refId": "A" + } + ], + "title": "Notifications rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Rate of of contract invocations", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 71, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, contract, invoke_type, method) (\n rate(frostfs_node_morph_invoke_duration_seconds_count{job=\"node\", instance=\"$instance\"}[$__rate_interval])\n)", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Contract {{contract}}, type {{invoke_type}}, method {{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Contract method invoke rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Percentage of the failed contract invocations", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "neo.fs.v2.object.ObjectService.Put" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 72, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, contract, invoke_type, method) (\n rate(frostfs_node_morph_invoke_duration_seconds_count{job=\"node\", instance=\"$instance\", success!=\"true\"}[$__rate_interval])\n)\n / \nsum by (instance, contract, invoke_type, method) (\n rate(frostfs_node_morph_invoke_duration_seconds_count{job=\"node\", instance=\"$instance\"}[$__rate_interval])\n)\n * 100.0\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Contract {{contract}}, type {{invoke_type}}, method {{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Contract invocations error rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the contract invocations for selected quantile", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 73, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_morph_invoke_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (contract, invoke_type, method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Contract {{contract}}, type {{invoke_type}}, method {{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Contract invocations latency", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Rate of of morph cache methods invocations", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 47 + }, + "id": 75, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (instance, method) (\n rate(frostfs_node_morphcache_request_duration_seconds_count{job=\"node\", instance=\"$instance\"}[$__rate_interval])\n)", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache method rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Latency of the morph cache for selected quantile", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 57 + }, + "id": 74, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile($quantile, \n sum(rate(frostfs_node_morphcache_request_duration_seconds_bucket{job=\"node\", instance=\"$instance\"}[$__rate_interval])) by (method,le)\n)\n", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache invocations latency", + "transformations": [], + "type": "timeseries" + } + ], + "title": "Morph", + "type": "row" + } + ], + "refresh": "1m", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "s01.frostfs.devenv:9090", + "value": "s01.frostfs.devenv:9090" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_state_health,instance)", + "hide": 0, + "includeAll": false, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [], + "query": { + "query": "label_values(frostfs_node_state_health,instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "0.95", + "value": "0.95" + }, + "hide": 0, + "includeAll": false, + "label": "Quantile", + "multi": false, + "name": "quantile", + "options": [ + { + "selected": true, + "text": "0.95", + "value": "0.95" + }, + { + "selected": false, + "text": "0.99", + "value": "0.99" + }, + { + "selected": false, + "text": "1.00", + "value": "1.00" + } + ], + "query": "0.95, 0.99, 1.00", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Storage node", + "uid": "aa6318b6-4780-4a9a-9fe7-a4a81917265f", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/services/grafana/provisioning/dashboards/writecache-buckets.json b/services/grafana/provisioning/dashboards/writecache-buckets.json new file mode 100644 index 0000000..cd37893 --- /dev/null +++ b/services/grafana/provisioning/dashboards/writecache-buckets.json @@ -0,0 +1,361 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 6, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "$method", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 20, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "< 0.005s", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.005s - 0.01s", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.01s - 0.025s", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.025s - 0.05s", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.05s - 0.1s", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.1s - 0.25s", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.25s - 0.5s", + "range": true, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "0.5s - 1s", + "range": true, + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "1s - 2.5s", + "range": true, + "refId": "I" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "2.5s - 5s", + "range": true, + "refId": "J" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "5s - 10s", + "range": true, + "refId": "K" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"+Inf\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)", + "hide": false, + "legendFormat": "> 10s", + "range": true, + "refId": "L" + } + ], + "title": "$method", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "s01.frostfs.devenv:9090", + "value": "s01.frostfs.devenv:9090" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_writecache_request_duration_seconds_bucket,instance)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "instance", + "options": [], + "query": { + "query": "label_values(frostfs_node_writecache_request_duration_seconds_bucket,instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": "BGH3NNzxkiW7JB1BKLHb5z", + "value": "BGH3NNzxkiW7JB1BKLHb5z" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_writecache_request_duration_seconds_bucket{instance=\"$instance\"},shard_id)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "shard_id", + "options": [], + "query": { + "query": "label_values(frostfs_node_writecache_request_duration_seconds_bucket{instance=\"$instance\"},shard_id)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": "Delete", + "value": "Delete" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(frostfs_node_writecache_request_duration_seconds_bucket{instance=\"$instance\", shard_id=\"$shard_id\"},method)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "method", + "options": [], + "query": { + "query": "label_values(frostfs_node_writecache_request_duration_seconds_bucket{instance=\"$instance\", shard_id=\"$shard_id\"},method)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Storage node writecache bucket distribution", + "uid": "d2fef1d1-9ef3-437e-ad21-670a20e0e9b4", + "version": 3, + "weekStart": "" + } \ No newline at end of file diff --git a/services/grafana/provisioning/datasources/datasource.yml b/services/grafana/provisioning/datasources/datasource.yml new file mode 100644 index 0000000..7d7427f --- /dev/null +++ b/services/grafana/provisioning/datasources/datasource.yml @@ -0,0 +1,13 @@ +apiVersion: 1 + +datasources: +- name: Prometheus + type: prometheus + access: proxy + orgId: 1 + url: http://prometheus:9090 +- name: Loki + type: loki + access: proxy + orgId: 1 + url: http://loki:3100 \ No newline at end of file diff --git a/services/http_gate/.http.env b/services/http_gate/.http.env index 97f6312..e69de29 100644 --- a/services/http_gate/.http.env +++ b/services/http_gate/.http.env @@ -1,9 +0,0 @@ -HTTP_GW_LISTEN_ADDRESS=0.0.0.0:80 - -HTTP_GW_LOGGER_LEVEL=info - -HTTP_GW_REBALANCE_TIMER=5m -HTTP_GW_CONNECT_TIMEOUT=60s -HTTP_GW_REQUEST_TIMEOUT=300s - -HTTP_GW_RESOLVE_ORDER=dns diff --git a/services/http_gate/cfg/config.yml b/services/http_gate/cfg/config.yml new file mode 100644 index 0000000..2f15e44 --- /dev/null +++ b/services/http_gate/cfg/config.yml @@ -0,0 +1,27 @@ +logger: + level: debug + +prometheus: + enabled: true + address: :9090 + +rebalance_timer: 5m # Interval to check nodes health + +connect_timeout: 60s # Timeout to dial node + +request_timeout: 300s # Timeout to check node health during rebalance + +# The order in which resolvers are used to find an container id by name +resolve_order: + - nns + +server: + - address: 0.0.0.0:80 + +# Wallet settings +wallet: + path: /wallet.json # Path to wallet + passphrase: one # Passphrase to decrypt wallet + +containers: + cors: cors.container diff --git a/services/http_gate/docker-compose.yml b/services/http_gate/docker-compose.yml index 1e822c8..c2e68ae 100644 --- a/services/http_gate/docker-compose.yml +++ b/services/http_gate/docker-compose.yml @@ -1,6 +1,5 @@ --- -version: "2.4" services: http_gate: image: ${HTTP_GW_IMAGE}:${HTTP_GW_VERSION} @@ -8,8 +7,6 @@ services: hostname: http_gate container_name: http_gate restart: on-failure - dns: - - ${IPV4_PREFIX}.53 networks: http_gate_int: internet: @@ -17,11 +14,13 @@ services: volumes: - ./wallet.json:/wallet.json - ./../../vendor/hosts:/etc/hosts + - ./cfg:/etc/frostfs/http stop_signal: SIGKILL env_file: [ ".env", ".http.env", ".int_test.env" ] + command: [ "frostfs-http-gw", "--config", "/etc/frostfs/http/config.yml" ] environment: - - HTTP_GW_WALLET_PATH=/wallet.json - - HTTP_GW_WALLET_PASSPHRASE=one + - HTTP_GW_RPC_ENDPOINT=http://morph-chain.${LOCAL_DOMAIN}:30333 + - HTTP_GW_TREE_SERVICE=s01.${LOCAL_DOMAIN}:8080 - HTTP_GW_PEERS_0_ADDRESS=s01.${LOCAL_DOMAIN}:8080 - HTTP_GW_PEERS_0_WEIGHT=0.2 - HTTP_GW_PEERS_1_ADDRESS=s02.${LOCAL_DOMAIN}:8080 diff --git a/services/ir/.ir.env b/services/ir/.ir.env index 01dbaaf..810cf6d 100644 --- a/services/ir/.ir.env +++ b/services/ir/.ir.env @@ -1,44 +1 @@ -NEOFS_IR_LOGGER_LEVEL=debug - -# Profiler section -NEOFS_IR_PPROF_ENABLED=true -## Server address. Empty value disables profiler -NEOFS_IR_PPROF_ADDRESS=:6060 -## Timeout to shut down the HTTP server -## Default: 30s -#NEOFS_IR_PROFILER_SHUTDOWN_TIMEOUT=1m - -# Application metrics section -NEOFS_IR_PROMETHEUS_ENABLED=true -## Server address. Empty value disables metrics gathering -NEOFS_IR_PROMETHEUS_ADDRESS=:9090 -## Timeout to shut down the HTTP server -## Default: 30s -#NEOFS_IR_METRICS_SHUTDOWN_TIMEOUT=1m - -# Toggling the sidechain-only mode -NEOFS_IR_WITHOUT_MAINNET=false - -NEOFS_IR_MAINNET_ENDPOINT_CLIENT_0_ADDRESS=ws://main-chain:30333/ws -NEOFS_IR_MORPH_ENDPOINT_CLIENT_0_ADDRESS=ws://morph-chain:30333/ws -NEOFS_IR_TIMERS_EMIT=50 -NEOFS_IR_TIMERS_STOP_ESTIMATION_DIV=4 -NEOFS_IR_TIMERS_COLLECT_BASIC_INCOME_DIV=2 -NEOFS_IR_TIMERS_DISTRIBUTE_BASIC_INCOME_MUL=3 -NEOFS_IR_TIMERS_DISTRIBUTE_BASIC_INCOME_DIV=4 - -NEOFS_IR_EMIT_STORAGE_AMOUNT=1000000000 -NEOFS_IR_NETMAP_CLEANER_ENABLED=true - -NEOFS_IR_CONTRACTS_NEOFS=b797c965c186811455430a7b39b81beb1dc3772a -NEOFS_IR_CONTRACTS_PROCESSING=ab239aa64ff6419419390038682a58b3e7dd0e86 - -NEOFS_IR_MORPH_VALIDATORS=02b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc2 -NEOFS_IR_AUDIT_PDP_MAX_SLEEP_INTERVAL=100ms - -NEOFS_IR_SETTLEMENT_BASIC_INCOME_RATE=100000000 -NEOFS_IR_SETTLEMENT_AUDIT_FEE=100000 - -NEOFS_IR_LOCODE_DB_PATH=/locode/db - -NEOFS_IR_CONTROL_GRPC_ENDPOINT=127.0.0.1:16512 +FROSTFS_IR_CONTROL_GRPC_ENDPOINT=127.0.0.1:16512 diff --git a/services/ir/artifacts.mk b/services/ir/artifacts.mk index fa009b1..1c2cf5d 100644 --- a/services/ir/artifacts.mk +++ b/services/ir/artifacts.mk @@ -1,42 +1,39 @@ -# Get NeoFS LOCODE database +# Get FrostFS IR artifacts (LOCODE database and FrostFS CLI) LOCODE_DB_ARCHIVE_PATH=./vendor LOCODE_DB_ARCHIVE_FILE=locode_db.gz get.ir: get.locode get.cli +# Get FrostFS LOCODE database get.locode: LOCODE_DB_PATH?= get.locode: @mkdir -p ${LOCODE_DB_ARCHIVE_PATH} ifeq (${LOCODE_DB_PATH},) - @echo "⇒ Download NeoFS LOCODE database from ${LOCODE_DB_URL}" + @echo "⇒ Download FrostFS LOCODE database from ${LOCODE_DB_URL}" @curl \ -sSL "${LOCODE_DB_URL}" \ -o ${LOCODE_DB_ARCHIVE_PATH}/${LOCODE_DB_ARCHIVE_FILE} else - @echo "⇒ Copy local archive of NeoFS LOCODE database from ${LOCODE_DB_PATH}" + @echo "⇒ Copy local archive of FrostFS LOCODE database from ${LOCODE_DB_PATH}" @cp ${LOCODE_DB_PATH} ${LOCODE_DB_ARCHIVE_PATH}/${LOCODE_DB_ARCHIVE_FILE} endif gzip -dfk ${LOCODE_DB_ARCHIVE_PATH}/${LOCODE_DB_ARCHIVE_FILE} +# Download FrostFS CLI .ONESHELL: -get.cli: NEOFS_CLI_FILE=./vendor/neofs-cli -get.cli: NEOFS_CLI_ARCHIVE_FILE=${NEOFS_CLI_FILE}.tar.gz -get.cli: NEOFS_CLI_PATH?= +get.cli: FROSTFS_CLI_FILE=./vendor/frostfs-cli +get.cli: FROSTFS_CLI_PATH?= get.cli: @mkdir -p ./vendor -ifeq (${NEOFS_CLI_PATH},) - @echo "⇒ Download NeoFS CLI binary from ${NEOFS_CLI_URL}" - @curl \ - -sSL "${NEOFS_CLI_URL}" \ - -o ${NEOFS_CLI_ARCHIVE_FILE} - @tar -xvf ${NEOFS_CLI_ARCHIVE_FILE} -C ./vendor | xargs -I {} \ - mv ./vendor/{} ${NEOFS_CLI_FILE} - @rm ${NEOFS_CLI_ARCHIVE_FILE} +ifeq (${FROSTFS_CLI_PATH},) + echo "⇒ Download FrostFS CLI binary from ${FROSTFS_CLI_IMAGE}:${FROSTFS_CLI_VERSION}" + $(shell docker cp `docker create --name tmp ${FROSTFS_CLI_IMAGE}:${FROSTFS_CLI_VERSION}`:/bin/frostfs-cli ${FROSTFS_CLI_FILE} && docker rm tmp >/dev/null) + chmod +x ${FROSTFS_CLI_FILE} else - @echo "⇒ Copy local binary from ${NEOFS_CLI_PATH}" - @cp ${NEOFS_CLI_PATH} ${NEOFS_CLI_FILE} + @echo "⇒ Copy local binary from ${FROSTFS_CLI_PATH}" + @cp ${FROSTFS_CLI_PATH} ${FROSTFS_CLI_FILE} endif diff --git a/services/ir/cfg/config.yml b/services/ir/cfg/config.yml new file mode 100644 index 0000000..bb15b3b --- /dev/null +++ b/services/ir/cfg/config.yml @@ -0,0 +1,76 @@ +# Logger section +logger: + level: debug # Minimum enabled logging level + +# Wallet settings +wallet: + path: /wallet.json # Path to NEP-6 NEO wallet file + address: Nhfg3TbpwogLvDGVvAvqyThbsHgoSUKwtn # Account address in the wallet; ignore to use default address + password: one # Account password in the wallet + +# Profiler section +pprof: + enabled: true + address: :6060 # Endpoint for application pprof profiling; disabled by default + shutdown_timeout: 30s # Timeout for profiling HTTP server graceful shutdown + +# Application metrics section +prometheus: + enabled: true + address: :9090 # Endpoint for application prometheus metrics; disabled by default + shutdown_timeout: 30s # Timeout for metrics HTTP server graceful shutdown + +# Toggling the sidechain-only mode +without_mainnet: true + +# Neo main chain RPC settings +mainnet: + endpoint: + client: # List of websocket RPC endpoints in mainchain; ignore if mainchain is disabled + - address: ws://main-chain:30333/ws + +# Neo side chain RPC settings +morph: + endpoint: + client: # List of websocket RPC endpoints in sidechain + - address: ws://morph-chain:30333/ws + validators: # List of hex-encoded 33-byte public keys of sidechain validators to vote for at application startup + - 02b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc2 + +# Network time settings +timers: + emit: 50 # Number of sidechain blocks between GAS emission cycles; disabled by default + stop_estimation: + mul: 1 # Multiplier in x/y relation of when to stop basic income estimation within the epoch + div: 4 # Divider in x/y relation of when to stop basic income estimation within the epoch + collect_basic_income: + mul: 1 # Multiplier in x/y relation of when to start basic income asset collection within the epoch + div: 2 # Divider in x/y relation of when to start basic income asset collecting within the epoch + distribute_basic_income: + mul: 3 # Multiplier in x/y relation of when to start basic income asset distribution within the epoch + div: 4 # Divider in x/y relation of when to start basic income asset distribution within the epoch + +# Storage node GAS emission settings +emit: + storage: + amount: 1000000000 # Fixed8 value of sidechain GAS emitted to all storage nodes once per GAS emission cycle; disabled by default + +# Storage node removal settings +netmap_cleaner: + enabled: true # Enable voting for removing stale storage nodes from network map + threshold: 3 # Number of FrostFS epoch without bootstrap request from storage node before it considered stale + +# Audit settings +audit: + pdp: + max_sleep_interval: 100ms # Maximum timeout between object.RangeHash requests to the storage node + +# Settlement settings +settlement: + basic_income_rate: 100000000 # Optional: override basic income rate value from network config; applied only in debug mode + audit_fee: 100000 # Optional: override audit fee value from network config; applied only in debug mode + +# LOCODE database +locode: + db: + path: /locode/db # Path to UN/LOCODE database file diff --git a/services/ir/docker-compose.yml b/services/ir/docker-compose.yml index 929b99a..4e671b2 100644 --- a/services/ir/docker-compose.yml +++ b/services/ir/docker-compose.yml @@ -1,6 +1,5 @@ --- -version: "2.4" services: ir01: @@ -13,21 +12,19 @@ services: ir_int: internet: ipv4_address: ${IPV4_PREFIX}.61 - stop_signal: SIGKILL + stop_signal: SIGTERM + stop_grace_period: 15s volumes: - ./az.json:/wallet.json - ./az.key:/wallet01.key - ./../../vendor/hosts:/etc/hosts - ./../../vendor/locode_db:/locode/db - - ./../../vendor/neofs-cli:/neofs-cli - - ./healthcheck.sh:/healthcheck.sh + - ./../../vendor/frostfs-cli:/frostfs-cli + - ./cfg:/etc/frostfs/ir env_file: [ ".env", ".ir.env", ".int_test.env" ] - environment: - - NEOFS_IR_WALLET_PATH=/wallet.json - - NEOFS_IR_WALLET_ADDRESS=Nhfg3TbpwogLvDGVvAvqyThbsHgoSUKwtn - - NEOFS_IR_WALLET_PASSWORD=one + command: [ "frostfs-ir", "--config", "/etc/frostfs/ir/config.yml" ] healthcheck: - test: ["CMD-SHELL", "/healthcheck.sh"] + test: ["CMD-SHELL", "/frostfs-cli control ir healthcheck -q --wallet /wallet01.key --endpoint \"$$FROSTFS_IR_CONTROL_GRPC_ENDPOINT\""] interval: 2s timeout: 1s retries: 5 diff --git a/services/ir/healthcheck.sh b/services/ir/healthcheck.sh deleted file mode 100755 index 2934969..0000000 --- a/services/ir/healthcheck.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh - -/neofs-cli control healthcheck \ - --endpoint "$NEOFS_IR_CONTROL_GRPC_ENDPOINT" \ - --wallet /wallet01.key --ir | - grep "Health status: READY" diff --git a/services/ir/prepare.mk b/services/ir/prepare.mk deleted file mode 100644 index 00a9253..0000000 --- a/services/ir/prepare.mk +++ /dev/null @@ -1,3 +0,0 @@ -# Deposit GAS from default wallet to NeoFS privnet contract -prepare.ir: - @./bin/deposit.sh diff --git a/services/coredns/.env b/services/jaeger/.env similarity index 100% rename from services/coredns/.env rename to services/jaeger/.env diff --git a/services/jaeger/.hosts b/services/jaeger/.hosts new file mode 100644 index 0000000..06f6ef2 --- /dev/null +++ b/services/jaeger/.hosts @@ -0,0 +1 @@ +IPV4_PREFIX.120 jaeger.LOCAL_DOMAIN diff --git a/services/coredns/.int_test.env b/services/jaeger/.int_test.env similarity index 100% rename from services/coredns/.int_test.env rename to services/jaeger/.int_test.env diff --git a/services/jaeger/.jaeger.env b/services/jaeger/.jaeger.env new file mode 100644 index 0000000..e69de29 diff --git a/services/jaeger/docker-compose.yml b/services/jaeger/docker-compose.yml new file mode 100644 index 0000000..1fb367b --- /dev/null +++ b/services/jaeger/docker-compose.yml @@ -0,0 +1,31 @@ +services: + jaeger: + image: ${JAEGER_IMAGE}:${JAEGER_VERSION} + domainname: ${LOCAL_DOMAIN} + hostname: jaeger + container_name: jaeger + restart: on-failure + networks: + jaeger_int: + internet: + ipv4_address: ${IPV4_PREFIX}.120 + volumes: + - ./../../vendor/hosts:/etc/hosts + ports: + - '4317:4317' #OTLP over gRPC + - '4318:4318' #OTLP over HTTP + - '16686:16686' #frontend + stop_signal: SIGKILL + env_file: [ ".env", ".jaeger.env", ".int_test.env" ] + environment: + - COLLECTOR_OTLP_ENABLED=true + - SPAN_STORAGE_TYPE=badger + - BADGER_EPHEMERAL=false + - BADGER_DIRECTORY_VALUE=/badger/data + - BADGER_DIRECTORY_KEY=/badger/key + +networks: + jaeger_int: + internet: + external: true + name: basenet_internet \ No newline at end of file diff --git a/services/morph_chain/artifacts.mk b/services/morph_chain/artifacts.mk index 9c2a761..4a4504a 100644 --- a/services/morph_chain/artifacts.mk +++ b/services/morph_chain/artifacts.mk @@ -1,32 +1,32 @@ -# Download NeoFS Contracts and Adm tool to deploy environment +# Download FrostFS Contracts and Adm tool to deploy environment + get.morph_chain: get.contracts get.adm -get.contracts: NEOFS_CONTRACTS_DEST=./vendor/contracts -get.contracts: NEOFS_CONTRACTS_ARCHIVE=neofs-contracts.tar.gz +# Download FrostFS Contracts +get.contracts: FROSTFS_CONTRACTS_DEST=./vendor/contracts +get.contracts: FROSTFS_CONTRACTS_ARCHIVE=frostfs-contracts.tar.gz get.contracts: - @mkdir -p ${NEOFS_CONTRACTS_DEST} + @mkdir -p ${FROSTFS_CONTRACTS_DEST} -ifeq (${NEOFS_CONTRACTS_PATH},) - @echo "⇒ Download compiled NeoFS contracts from ${NEOFS_CONTRACTS_URL}" - @curl -sSL ${NEOFS_CONTRACTS_URL} -o ${NEOFS_CONTRACTS_ARCHIVE} - @tar -xf ${NEOFS_CONTRACTS_ARCHIVE} -C ${NEOFS_CONTRACTS_DEST} --strip-components 1 - @rm ${NEOFS_CONTRACTS_ARCHIVE} +ifeq (${FROSTFS_CONTRACTS_PATH},) + @echo "⇒ Download compiled FrostFS contracts from ${FROSTFS_CONTRACTS_URL}" + @curl -ksSL ${FROSTFS_CONTRACTS_URL} -o ${FROSTFS_CONTRACTS_ARCHIVE} + @tar -xf ${FROSTFS_CONTRACTS_ARCHIVE} -C ${FROSTFS_CONTRACTS_DEST} --strip-components 1 + @rm ${FROSTFS_CONTRACTS_ARCHIVE} else - @echo "⇒ Copy compiled contracts from ${NEOFS_CONTRACTS_PATH}" - @cp -r ${NEOFS_CONTRACTS_PATH} ${NEOFS_CONTRACTS_DEST} + @echo "⇒ Copy compiled contracts from ${FROSTFS_CONTRACTS_PATH}" + @cp -r ${FROSTFS_CONTRACTS_PATH}/* ${FROSTFS_CONTRACTS_DEST} endif -get.adm: NEOFS_ADM_DEST=./vendor/neofs-adm -get.adm: NEOFS_ADM_ARCHIVE=neofs-adm.tar.gz +# Download FrostFS ADM tool +get.adm: FROSTFS_ADM_DEST=./vendor/frostfs-adm get.adm: -ifeq (${NEOFS_ADM_PATH},) - @echo "⇒ Download NeoFS ADM binary from ${NEOFS_ADM_URL}" - @curl -sSL ${NEOFS_ADM_URL} -o ${NEOFS_ADM_ARCHIVE} - @tar -xvf ${NEOFS_ADM_ARCHIVE} -C ./vendor | xargs -I {} \ - mv ./vendor/{} ${NEOFS_ADM_DEST} - @rm ${NEOFS_ADM_ARCHIVE} +ifeq (${FROSTFS_ADM_PATH},) + @echo "⇒ Download FrostFS ADM binary from ${FROSTFS_ADM_IMAGE}:${FROSTFS_ADM_VERSION}" + $(shell docker cp `docker create --name tmp ${FROSTFS_ADM_IMAGE}:${FROSTFS_ADM_VERSION}`:/bin/frostfs-adm ${FROSTFS_ADM_DEST} && docker rm tmp >/dev/null) + @chmod +x ${FROSTFS_ADM_DEST} else - @echo "⇒ Copy neofs-adm binary from ${NEOFS_ADM_PATH}" - @cp ${NEOFS_ADM_PATH} ${NEOFS_ADM_DEST} + @echo "⇒ Copy frostfs-adm binary from ${FROSTFS_ADM_PATH}" + @cp ${FROSTFS_ADM_PATH} ${FROSTFS_ADM_DEST} endif diff --git a/services/morph_chain/config.yml b/services/morph_chain/config.yml new file mode 100644 index 0000000..7b4bb29 --- /dev/null +++ b/services/morph_chain/config.yml @@ -0,0 +1,4 @@ +--- + +Path: "/wallets/node-wallet.json" +Password: "one" diff --git a/services/morph_chain/docker-compose.yml b/services/morph_chain/docker-compose.yml index a4cebf4..c3a1481 100644 --- a/services/morph_chain/docker-compose.yml +++ b/services/morph_chain/docker-compose.yml @@ -1,8 +1,7 @@ --- -version: "2.4" services: - neofs_morph_chain: + frostfs_morph_chain: image: ${NEOGO_IMAGE}:${NEOGO_VERSION} container_name: morph_chain command: ["node", "--config-path", "/config", "--privnet", "--debug"] @@ -17,11 +16,22 @@ services: volumes: - ./protocol.privnet.yml:/config/protocol.privnet.yml - ./node-wallet.json:/wallets/node-wallet.json + - ./config.yml:/wallets/config.yml - ./../../vendor/hosts:/etc/hosts - ./../../wallets/wallet.json:/wallets/wallet.json + - ./../../wallets/system-wallet.json:/wallets/system-wallet.json + - ./../s3_gate/wallet.json:/wallets/s3-wallet.json + - ./../storage/wallet01.json:/wallets/storage/wallet01.json + - ./../storage/wallet02.json:/wallets/storage/wallet02.json + - ./../storage/wallet03.json:/wallets/storage/wallet03.json + - ./../storage/wallet04.json:/wallets/storage/wallet04.json + - chains:/chains networks: chain_int: internet: external: true name: basenet_internet + +volumes: + chains: diff --git a/services/morph_chain/protocol.privnet.yml b/services/morph_chain/protocol.privnet.yml index 6992ced..7cea631 100644 --- a/services/morph_chain/protocol.privnet.yml +++ b/services/morph_chain/protocol.privnet.yml @@ -1,49 +1,57 @@ ProtocolConfiguration: Magic: 15405 MaxTraceableBlocks: 200000 - SecondsPerBlock: 1 + TimePerBlock: 1s MemPoolSize: 50000 StandbyCommittee: - 02b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc2 ValidatorsCount: 1 SeedList: - 172.200.0.1:20333 - VerifyBlocks: true VerifyTransactions: true StateRootInHeader: true P2PSigExtensions: true + Hardforks: {} ApplicationConfiguration: + SkipBlockVerification: false DBConfiguration: Type: "boltdb" BoltDBOptions: - FilePath: "./db/morph.bolt" - NodePort: 20333 + FilePath: "/chains/morph.bolt" + P2P: + Addresses: + - ":20333" + DialTimeout: 3s + ProtoTickInterval: 2s + PingInterval: 30s + PingTimeout: 90s + MaxPeers: 10 + AttemptConnPeers: 5 + MinPeers: 0 Relay: true - DialTimeout: 3 - ProtoTickInterval: 2 - PingInterval: 30 - PingTimeout: 90 - MaxPeers: 10 - AttemptConnPeers: 5 - MinPeers: 0 + Consensus: + Enabled: true + UnlockWallet: + Path: "./wallets/node-wallet.json" + Password: "one" RPC: + Addresses: + - ":30333" Enabled: true SessionEnabled: true EnableCORSWorkaround: false MaxGasInvoke: 100 - Port: 30333 P2PNotary: Enabled: true UnlockWallet: Path: "./wallets/node-wallet.json" Password: "one" Prometheus: + Addresses: + - ":20001" Enabled: true - Port: 20001 Pprof: + Addresses: + - ":20011" Enabled: true - Port: 20011 - UnlockWallet: - Path: "./wallets/node-wallet.json" - Password: "one" diff --git a/services/nats/.hosts b/services/nats/.hosts deleted file mode 100644 index 6ed5417..0000000 --- a/services/nats/.hosts +++ /dev/null @@ -1 +0,0 @@ -IPV4_PREFIX.101 nats.LOCAL_DOMAIN diff --git a/services/nats/artifacts.mk b/services/nats/artifacts.mk deleted file mode 100644 index c68c57e..0000000 --- a/services/nats/artifacts.mk +++ /dev/null @@ -1,7 +0,0 @@ -# Create new tls certs - -NATS_DIR=$(abspath services/nats) - -get.nats: - @echo "⇒ Creating certs for NATS server and clients" - ${NATS_DIR}/generate_cert.sh ${LOCAL_DOMAIN} diff --git a/services/nats/docker-compose.yml b/services/nats/docker-compose.yml deleted file mode 100644 index d69d10d..0000000 --- a/services/nats/docker-compose.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- - -version: "2.4" -services: - nats: - image: ${NATS_IMAGE}:${NATS_VERSION} - domainname: ${LOCAL_DOMAIN} - hostname: nats - container_name: nats - restart: on-failure - dns: - - ${IPV4_PREFIX}.101 - networks: - nats_int: - internet: - ipv4_address: ${IPV4_PREFIX}.101 - volumes: - - ./../../vendor/hosts:/etc/hosts - - ./nats.conf:/etc/nats/neofs-nats-server.conf - - ./server-cert.pem:/certs/server-cert.pem - - ./server-key.pem:/certs/server-key.pem - - ./ca-cert.pem:/certs/ca-cert.pem - stop_signal: SIGKILL - env_file: [ ".env", ".int_test.env" ] - command: ["-c", "/etc/nats/neofs-nats-server.conf"] - -networks: - nats_int: - internet: - external: true - name: basenet_internet diff --git a/services/nats/generate_cert.sh b/services/nats/generate_cert.sh deleted file mode 100755 index 071f9df..0000000 --- a/services/nats/generate_cert.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash - -source bin/helper.sh - -WORKDIR=$(dirname "$0") -LOCAL_DOMAIN=$1 - -CA_KEY=$WORKDIR/ca-key.pem -CA_CRT=$WORKDIR/ca-cert.pem - -SRV_KEY=$WORKDIR/server-key.pem -SRV_REQ=$WORKDIR/server-req.csr -SRV_CRT=$WORKDIR/server-cert.pem - -CLI_KEY=$WORKDIR/client-key.pem -CLI_REQ=$WORKDIR/client-req.csr -CLI_CRT=$WORKDIR/client-cert.pem - -SUBJ="/O=NSPCC" - -if [[ ! -f $CA_KEY || ! -f $CA_CRT ]]; then - openssl req -newkey rsa:4096 -x509 -days 365 -nodes -keyout $CA_KEY -out $CA_CRT -subj $SUBJ 2>&1 || - die "CA certificate was not created" -fi - -if [[ ! -f $SRV_KEY || ! -f $SRV_CRT ]]; then - openssl req -newkey rsa:4096 -nodes -keyout $SRV_KEY -out $SRV_REQ -subj $SUBJ 2>&1 || - die "Server certificate was not created" - - openssl x509 -req -days 365 -set_serial 01 -in $SRV_REQ -out $SRV_CRT -CA $CA_CRT -CAkey $CA_KEY \ - -extensions san -extfile <(printf "[san]\nsubjectAltName=DNS:nats.$LOCAL_DOMAIN") 2>&1 || { - rm $SRV_REQ - die "Server certificate was not signed by CA" - } - - rm $SRV_REQ -fi - -if [[ ! -f $CLI_KEY || ! -f $CLI_CRT ]]; then - openssl req -newkey rsa:4096 -nodes -keyout $CLI_KEY -out $CLI_REQ -subj $SUBJ 2>&1 || - die "Client certificate was not created" - - openssl x509 -req -days 365 -set_serial 01 -in $CLI_REQ -out $CLI_CRT -CA $CA_CRT -CAkey $CA_KEY 2>&1 || { - rm $CLI_REQ - die "Client certificate was not signed by CA" - } - - rm $CLI_REQ -fi diff --git a/services/nats/nats.conf b/services/nats/nats.conf deleted file mode 100644 index 0c7af0a..0000000 --- a/services/nats/nats.conf +++ /dev/null @@ -1,15 +0,0 @@ -port: 4222 -monitor_port: 8222 - -jetstream { - store_dir=nats - max_memory_store: 1GB - max_file_store: 2GB -} - -tls { - cert_file: /certs/server-cert.pem - key_file: /certs/server-key.pem - ca_file: /certs/ca-cert.pem - verify: true -} diff --git a/services/nats/.env b/services/prometheus/.env similarity index 100% rename from services/nats/.env rename to services/prometheus/.env diff --git a/services/prometheus/.hosts b/services/prometheus/.hosts new file mode 100644 index 0000000..89305b2 --- /dev/null +++ b/services/prometheus/.hosts @@ -0,0 +1 @@ +IPV4_PREFIX.121 prometheus.LOCAL_DOMAIN diff --git a/services/nats/.int_test.env b/services/prometheus/.int_test.env similarity index 100% rename from services/nats/.int_test.env rename to services/prometheus/.int_test.env diff --git a/services/prometheus/.prometheus.env b/services/prometheus/.prometheus.env new file mode 100644 index 0000000..e69de29 diff --git a/services/prometheus/docker-compose.yml b/services/prometheus/docker-compose.yml new file mode 100644 index 0000000..31f5fbe --- /dev/null +++ b/services/prometheus/docker-compose.yml @@ -0,0 +1,24 @@ +services: + prometheus: + image: ${PROMETHEUS_IMAGE}:${PROMETHEUS_VERSION} + domainname: ${LOCAL_DOMAIN} + hostname: prometheus + container_name: prometheus + restart: on-failure + networks: + prometheus_int: + internet: + ipv4_address: ${IPV4_PREFIX}.121 + volumes: + - ./../../vendor/hosts:/etc/hosts + - ./prometheus.yml:/etc/prometheus/prometheus.yml + command: + - --config.file=/etc/prometheus/prometheus.yml + stop_signal: SIGKILL + env_file: [ ".env", ".prometheus.env", ".int_test.env" ] + +networks: + prometheus_int: + internet: + external: true + name: basenet_internet diff --git a/services/prometheus/prometheus.yml b/services/prometheus/prometheus.yml new file mode 100644 index 0000000..9ea7aed --- /dev/null +++ b/services/prometheus/prometheus.yml @@ -0,0 +1,22 @@ +global: + scrape_interval: 15s + +scrape_configs: + - job_name: 'node' + static_configs: + - targets: ['s01.frostfs.devenv:9090', 's02.frostfs.devenv:9090', 's03.frostfs.devenv:9090', 's04.frostfs.devenv:9090'] + - job_name: 'http-gw' + static_configs: + - targets: ['http.frostfs.devenv:9090'] + - job_name: 'rest-gw' + static_configs: + - targets: ['rest.frostfs.devenv:9090'] + - job_name: 's3-gw' + static_configs: + - targets: ['s3.frostfs.devenv:9090'] + - job_name: 'neo-go' + static_configs: + - targets: ['morph-chain.frostfs.devenv:20001'] + - job_name: 'inner-ring' + static_configs: + - targets: ['ir01.frostfs.devenv:9090'] \ No newline at end of file diff --git a/services/rest_gate/.hosts b/services/rest_gate/.hosts deleted file mode 100644 index ee7578e..0000000 --- a/services/rest_gate/.hosts +++ /dev/null @@ -1 +0,0 @@ -IPV4_PREFIX.83 rest.LOCAL_DOMAIN diff --git a/services/rest_gate/docker-compose.yml b/services/rest_gate/docker-compose.yml deleted file mode 100644 index 75b0b7a..0000000 --- a/services/rest_gate/docker-compose.yml +++ /dev/null @@ -1,33 +0,0 @@ ---- - -version: "2.4" -services: - rest_gate: - image: ${REST_GW_IMAGE}:${REST_GW_VERSION} - domainname: ${LOCAL_DOMAIN} - hostname: rest - container_name: rest_gate - restart: on-failure - networks: - rest_gate_int: - internet: - ipv4_address: ${IPV4_PREFIX}.83 - volumes: - - ./wallet.json:/wallet.json - - ./../../vendor/hosts:/etc/hosts - stop_signal: SIGKILL - env_file: [ ".env", ".int_test.env" ] - environment: - - REST_GW_WALLET_PATH=/wallet.json - - REST_GW_WALLET_PASSPHRASE=one - - REST_GW_LISTEN_ADDRESS=0.0.0.0:8090 - - REST_GW_PEERS_0_ADDRESS=s01.${LOCAL_DOMAIN}:8080 - - REST_GW_PEERS_1_ADDRESS=s02.${LOCAL_DOMAIN}:8080 - - REST_GW_PEERS_2_ADDRESS=s03.${LOCAL_DOMAIN}:8080 - - REST_GW_PEERS_3_ADDRESS=s04.${LOCAL_DOMAIN}:8080 - -networks: - rest_gate_int: - internet: - external: true - name: basenet_internet diff --git a/services/s3_gate/.s3.env b/services/s3_gate/.s3.env index 039939d..e69de29 100644 --- a/services/s3_gate/.s3.env +++ b/services/s3_gate/.s3.env @@ -1,8 +0,0 @@ -S3_GW_REBALANCE_INTERVAL=30s -S3_GW_HEALTHCHECK_TIMEOUT=15s -S3_GW_CONNECT_TIMEOUT=10s - -S3_GW_LOGGER_LEVEL=debug - -S3_GW_MAX_CLIENTS_COUNT=100 -S3_GW_MAX_CLIENTS_DEADLINE=30s diff --git a/services/s3_gate/cfg/config.yml b/services/s3_gate/cfg/config.yml new file mode 100644 index 0000000..e5c0b53 --- /dev/null +++ b/services/s3_gate/cfg/config.yml @@ -0,0 +1,54 @@ +logger: + level: debug + +prometheus: + enabled: true + address: :9090 + +# Interval to check node health +rebalance_interval: 30s + +# Timeout to check node health during rebalance +healthcheck_timeout: 15s + +# Timeout to connect to a node +connect_timeout: 10s + +# Limits for processing of clients' requests +max_clients_count: 100 + +# Deadline after which the gate sends error `RequestTimeout` to a client +max_clients_deadline: 30s + +resolve_order: + - nns + +server: + - tls: + enabled: true + cert_file: /tls.crt + key_file: /tls.key + +# Wallet configuration +wallet: + path: /wallet.json # Path to wallet + passphrase: "s3" # Passphrase to decrypt wallet + +features: + md5: + enabled: true + +control: + grpc: + endpoint: localhost:16515 + +frostfsid: + enabled: false + +policy: + enabled: false + +containers: + cors: cors.container + mfa: mfa.container + website: website.container diff --git a/services/s3_gate/docker-compose.yml b/services/s3_gate/docker-compose.yml index 82efcaa..3c1f9f6 100644 --- a/services/s3_gate/docker-compose.yml +++ b/services/s3_gate/docker-compose.yml @@ -1,6 +1,5 @@ --- -version: "2.4" services: s3_gate: image: ${S3_GW_IMAGE}:${S3_GW_VERSION} @@ -8,27 +7,29 @@ services: hostname: s3_gate container_name: s3_gate restart: on-failure - dns: - - ${IPV4_PREFIX}.53 networks: s3_gate_int: internet: ipv4_address: ${IPV4_PREFIX}.82 volumes: + # Gate wallet - ./wallet.json:/wallet.json + # Folder for custom user wallets + - ./../../wallets/:/wallets/ - ./tls.key:/tls.key - ./tls.crt:/tls.crt - ./../../vendor/hosts:/etc/hosts - stop_signal: SIGKILL + - ./cfg:/etc/frostfs/s3 + - ./issue-creds.sh:/usr/bin/issue-creds.sh + stop_signal: SIGTERM + stop_grace_period: 15s env_file: [ ".env", ".s3.env", ".int_test.env" ] + command: [ "frostfs-s3-gw", "--config", "/etc/frostfs/s3/config.yml" ] environment: - - S3_GW_WALLET_PATH=/wallet.json - - S3_GW_WALLET_PASSPHRASE=s3 - - S3_GW_TLS_KEY_FILE=/tls.key - - S3_GW_TLS_CERT_FILE=/tls.crt - - S3_GW_LISTEN_ADDRESS=s3.${LOCAL_DOMAIN}:8080 + - S3_GW_RPC_ENDPOINT=http://morph-chain.${LOCAL_DOMAIN}:30333 + - S3_GW_SERVER_0_ADDRESS=s3.${LOCAL_DOMAIN}:8080 - S3_GW_LISTEN_DOMAINS=s3.${LOCAL_DOMAIN} - - S3_GW_TREE_SERVICE=s01.${LOCAL_DOMAIN}:8080 + - S3_GW_TREE_SERVICE=s01.${LOCAL_DOMAIN}:8080 s02.${LOCAL_DOMAIN}:8080 s03.${LOCAL_DOMAIN}:8080 s04.${LOCAL_DOMAIN}:8080 - S3_GW_PEERS_0_ADDRESS=s01.${LOCAL_DOMAIN}:8080 - S3_GW_PEERS_0_WEIGHT=0.2 - S3_GW_PEERS_1_ADDRESS=s02.${LOCAL_DOMAIN}:8080 @@ -37,6 +38,8 @@ services: - S3_GW_PEERS_2_WEIGHT=0.2 - S3_GW_PEERS_3_ADDRESS=s04.${LOCAL_DOMAIN}:8080 - S3_GW_PEERS_3_WEIGHT=0.2 + - AUTHMATE_WALLET_PASSPHRASE= + - AUTHMATE_WALLET_CONTRACT_PASSPHRASE=s3 networks: s3_gate_int: diff --git a/services/s3_gate/issue-creds.sh b/services/s3_gate/issue-creds.sh new file mode 100755 index 0000000..e2355f8 --- /dev/null +++ b/services/s3_gate/issue-creds.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +initUser() { + /bin/frostfs-s3-authmate register-user \ + --wallet $WALLET_PATH \ + --rpc-endpoint http://morph-chain.frostfs.devenv:30333 \ + --username $USERNAME \ + --contract-wallet /wallet.json 1> /dev/null && touch $WALLET_CACHE/$USERNAME +} + +issueCreds() { + /bin/frostfs-s3-authmate issue-secret \ + --wallet $WALLET_PATH \ + --peer s01.frostfs.devenv:8080 \ + --gate-public-key $S3_GATE_PUBLIC_KEY \ + --container-placement-policy "REP 3" +} + +set -e + +WALLET_PATH=/wallets/$2 +if [[ -z "$2" ]]; then + WALLET_PATH=/wallets/wallet.json +fi + +S3_GATE_PUBLIC_KEY=$3 +if [[ -z "$3" ]]; then + S3_GATE_PUBLIC_KEY=0313b1ac3a8076e155a7e797b24f0b650cccad5941ea59d7cfd51a024a8b2a06bf +fi + +WALLET_CACHE=/data/wallets +mkdir -p $WALLET_CACHE + +USERNAME=$(echo $WALLET_PATH | md5sum | cut -d' ' -f1) +if [ ! -e $WALLET_CACHE/$USERNAME ]; then + initUser +fi + +if [ $1 == "s3" ]; then + issueCreds +fi diff --git a/services/s3_gate/prepare.mk b/services/s3_gate/prepare.mk new file mode 100644 index 0000000..c97add8 --- /dev/null +++ b/services/s3_gate/prepare.mk @@ -0,0 +1,14 @@ +.PHONY: s3cred register + +password?= +contract_password?=s3 +gate_public_key?= +wallet?= + +# Register wallet & generate S3 credentials +s3cred: + @docker exec -e AUTHMATE_WALLET_PASSPHRASE="$(password)" -e AUTHMATE_WALLET_CONTRACT_PASSPHRASE="$(contract_password)" s3_gate /usr/bin/issue-creds.sh s3 "$(wallet)" "$(gate_public_key)" + +# Only registers user wallet +register: + @docker exec -e AUTHMATE_WALLET_PASSPHRASE="$(password)" -e AUTHMATE_WALLET_CONTRACT_PASSPHRASE="$(contract_password)" s3_gate /usr/bin/issue-creds.sh native "$(wallet)" diff --git a/services/s3_gate/wallet.json b/services/s3_gate/wallet.json index 8d3d4c2..ff1adfd 100644 --- a/services/s3_gate/wallet.json +++ b/services/s3_gate/wallet.json @@ -5,7 +5,7 @@ { "address": "NUUb82KR2JrVByHs2YSKgtK29gKnF5q6Vt", "key": "6PYSPET41jKtqie2cfkqgy7q9ueeucH8bX9Gotm2HwdNEDptj6aBe8dDg8", - "label": "NeoFS S3 Gate", + "label": "FrostFS S3 Gate", "contract": { "script": "DCEDE7GsOoB24VWn55eyTwtlDMytWUHqWdfP1RoCSosqBr9BVuezJw==", "parameters": [ diff --git a/services/rest_gate/.env b/services/s3_lifecycler/.env similarity index 100% rename from services/rest_gate/.env rename to services/s3_lifecycler/.env diff --git a/services/s3_lifecycler/.hosts b/services/s3_lifecycler/.hosts new file mode 100644 index 0000000..8eb1796 --- /dev/null +++ b/services/s3_lifecycler/.hosts @@ -0,0 +1 @@ +IPV4_PREFIX.84 lifecycler.LOCAL_DOMAIN diff --git a/services/rest_gate/.int_test.env b/services/s3_lifecycler/.int_test.env similarity index 100% rename from services/rest_gate/.int_test.env rename to services/s3_lifecycler/.int_test.env diff --git a/services/s3_lifecycler/cfg/config.yml b/services/s3_lifecycler/cfg/config.yml new file mode 100644 index 0000000..2555980 --- /dev/null +++ b/services/s3_lifecycler/cfg/config.yml @@ -0,0 +1,42 @@ +logger: + level: debug + +prometheus: + enabled: true + address: :9090 + +lifecycle: + job_fetcher_buffer: 1000 + executor_pool_size: 100 + +frostfs: + stream_timeout: 10s + connect_timeout: 10s + healthcheck_timeout: 15s + rebalance_interval: 60s + pool_error_threshold: 100 + tree_pool_max_attempts: 4 + +credential: + use: wallets + source: + wallets: + - path: /wallet.json + address: NTt1rxvmEDxEuuogLxs2xgxA71qhVaUcN7 + passphrase: "cycle" + - path: /user-wallet.json + address: NbUgTSFvPmsRxmGeWpuuGeJUoRoi6PErcM + passphrase: "" + +morph: + reconnect_clients_interval: 30s + dial_timeout: 5s + contract: + netmap: netmap.frostfs + frostfsid: frostfsid.frostfs + container: container.frostfs + +# Wallet configuration +wallet: + path: /wallet.json # Path to wallet + passphrase: "cycle" # Passphrase to decrypt wallet diff --git a/services/s3_lifecycler/docker-compose.yml b/services/s3_lifecycler/docker-compose.yml new file mode 100644 index 0000000..2fffa50 --- /dev/null +++ b/services/s3_lifecycler/docker-compose.yml @@ -0,0 +1,37 @@ +--- + +services: + s3_lifecycler: + image: ${S3_LIFECYCLER_IMAGE}:${S3_LIFECYCLER_VERSION} + domainname: ${LOCAL_DOMAIN} + hostname: s3_lifecycler + container_name: s3_lifecycler + restart: on-failure + networks: + s3_lifecycler_int: + internet: + ipv4_address: ${IPV4_PREFIX}.84 + volumes: + - ./wallet.json:/wallet.json + - ./../../vendor/hosts:/etc/hosts + - ./cfg:/etc/frostfs/s3-lifecycler + - ./../../wallets/wallet.json:/user-wallet.json + stop_signal: SIGKILL + env_file: [ ".env", ".int_test.env" ] + command: [ "frostfs-s3-lifecycler", "--config", "/etc/frostfs/s3-lifecycler/config.yml" ] + environment: + - S3_LIFECYCLER_MORPH_RPC_ENDPOINT_0_ADDRESS=ws://morph-chain:30333/ws + - S3_LIFECYCLER_FROSTFS_PEERS_0_ADDRESS=s01.${LOCAL_DOMAIN}:8080 + - S3_LIFECYCLER_FROSTFS_PEERS_0_WEIGHT=0.2 + - S3_LIFECYCLER_FROSTFS_PEERS_1_ADDRESS=s02.${LOCAL_DOMAIN}:8080 + - S3_LIFECYCLER_FROSTFS_PEERS_1_WEIGHT=0.2 + - S3_LIFECYCLER_FROSTFS_PEERS_2_ADDRESS=s03.${LOCAL_DOMAIN}:8080 + - S3_LIFECYCLER_FROSTFS_PEERS_2_WEIGHT=0.2 + - S3_LIFECYCLER_FROSTFS_PEERS_3_ADDRESS=s04.${LOCAL_DOMAIN}:8080 + - S3_LIFECYCLER_FROSTFS_PEERS_3_WEIGHT=0.2 + +networks: + s3_lifecycler_int: + internet: + external: true + name: basenet_internet diff --git a/services/rest_gate/wallet.json b/services/s3_lifecycler/wallet.json similarity index 57% rename from services/rest_gate/wallet.json rename to services/s3_lifecycler/wallet.json index 2b60501..ff8f34e 100644 --- a/services/rest_gate/wallet.json +++ b/services/s3_lifecycler/wallet.json @@ -1,12 +1,12 @@ { - "version": "3.0", + "version": "1.0", "accounts": [ { - "address": "NPFCqWHfi9ixCJRu7DABRbVfXRbkSEr9Vo", - "key": "6PYTAGjdaeicUDPqGv9mmgwb9kTwimWJJmmfNqJSDGH9qM79zSRcL9oHiB", - "label": "REST Gateway", + "address": "NTt1rxvmEDxEuuogLxs2xgxA71qhVaUcN7", + "key": "6PYR3XurAyTzVeDG5WV2Z8vnGdySw3mTLuKjr6Nwo7tae64SJ7XjZSMMPQ", + "label": "lifecycler", "contract": { - "script": "DCECcuPzZCZ2VyDsm2jKEOMnU6xEWO2bF1dvOvBWTDFYB1ZBVuezJw==", + "script": "DCED9z0M+WSGfXZGxYLj1yYwmgxJXE/kNA4+oWNi0q1uKCdBVuezJw==", "parameters": [ { "name": "parameter0", diff --git a/services/storage/.storage.env b/services/storage/.storage.env index c3605fb..e69de29 100644 --- a/services/storage/.storage.env +++ b/services/storage/.storage.env @@ -1,89 +0,0 @@ -# Logger section -## Minimum enabled logging level -## Default: info -NEOFS_LOGGER_LEVEL=debug - -# Profiler section -NEOFS_PPROF_ENABLED=true -## Server address. Empty value disables profiler -NEOFS_PPROF_ADDRESS=:6060 -## Timeout to shut down the HTTP server -## Default: 30s -#NEOFS_PROFILER_SHUTDOWN_TIMEOUT=1m - -# Application metrics section -NEOFS_PROMETHEUS_ENABLED=true -## Server address. Empty value disables metrics gathering -NEOFS_PROMETHEUS_ADDRESS=:9090 -## Timeout to shut down the HTTP server -## Default: 30s -#NEOFS_METRICS_SHUTDOWN_TIMEOUT=1m - -# Morph section -## Endpoints of sidechain RPC nodes (comma-separated) -## Default: empty list (invalid) -NEOFS_MORPH_RPC_ENDPOINT_0_ADDRESS=ws://morph-chain:30333/ws -## Dial timeout to connect to notification endpoint -## Default: 5s -#NEOFS_MORPH_DIAL_TIMEOUT=1m -# Configure side chain caches (negative value to disable) -NEOFS_MORPH_CACHE_TTL=-1 - -# Cache of API clients section -## Dial timeout to connect to remote API server -## Default: 5s -#NEOFS_APICLIENT_DIAL_TIMEOUT=1m - -# Common storage node attribute -NEOFS_NODE_ATTRIBUTE_0=User-Agent:NeoFS\/0.26 - -NEOFS_NODE_NOTIFICATION_ENABLED=true -NEOFS_NODE_NOTIFICATION_ENDPOINT=tls://nats.neofs.devenv:4222 -NEOFS_NODE_NOTIFICATION_DEFAULT_TOPIC=test -NEOFS_NODE_NOTIFICATION_CERTIFICATE=/etc/neofs-node/nats.tls.cert -NEOFS_NODE_NOTIFICATION_KEY=/etc/neofs-node/nats.tls.key -NEOFS_NODE_NOTIFICATION_CA=/etc/neofs-node/nats.ca.crt - -# Tree section -NEOFS_TREE_ENABLED=True - -# Object section -## Worker pool capacity of API Object.Put client calls -## Default: 10 -#NEOFS_OBJECT_PUT_POOL_SIZE_REMOTE=100 -#NEOFS_OBJECT_PUT_POOL_SIZE_LOCAL=100 - -# Policer section -## Timeout of Object.Head calls by Policer -## Default: 5s -#NEOFS_POLICER_HEAD_TIMEOUT=1m - -# Replicator section -## Timeout of Object.Put calls by Replicator -## Default: 5s -#NEOFS_REPLICATOR_PUT_TIMEOUT=1m - -# Storage engine configuration -NEOFS_STORAGE_SHARD_0_WRITECACHE_ENABLED=false -NEOFS_STORAGE_SHARD_0_WRITECACHE_PATH=/storage/wc0 -NEOFS_STORAGE_SHARD_0_METABASE_PATH=/storage/meta0 -NEOFS_STORAGE_SHARD_0_BLOBSTOR_0_TYPE=blobovnicza -NEOFS_STORAGE_SHARD_0_BLOBSTOR_0_PATH=/storage/blobovnicza0 -NEOFS_STORAGE_SHARD_0_BLOBSTOR_0_DEPTH=2 -NEOFS_STORAGE_SHARD_0_BLOBSTOR_0_WIDTH=4 -NEOFS_STORAGE_SHARD_0_BLOBSTOR_1_TYPE=fstree -NEOFS_STORAGE_SHARD_0_BLOBSTOR_1_PATH=/storage/fstree0 -NEOFS_STORAGE_SHARD_0_BLOBSTOR_1_DEPTH=2 -NEOFS_STORAGE_SHARD_0_PILORAMA_PATH=/storage/pilorama0 - -NEOFS_STORAGE_SHARD_1_WRITECACHE_ENABLED=false -NEOFS_STORAGE_SHARD_1_WRITECACHE_PATH=/storage/wc1 -NEOFS_STORAGE_SHARD_1_METABASE_PATH=/storage/meta1 -NEOFS_STORAGE_SHARD_1_BLOBSTOR_0_TYPE=blobovnicza -NEOFS_STORAGE_SHARD_1_BLOBSTOR_0_PATH=/storage/blobovnicza1 -NEOFS_STORAGE_SHARD_1_BLOBSTOR_0_DEPTH=2 -NEOFS_STORAGE_SHARD_1_BLOBSTOR_0_WIDTH=4 -NEOFS_STORAGE_SHARD_1_BLOBSTOR_1_TYPE=fstree -NEOFS_STORAGE_SHARD_1_BLOBSTOR_1_PATH=/storage/fstree1 -NEOFS_STORAGE_SHARD_1_BLOBSTOR_1_DEPTH=2 -NEOFS_STORAGE_SHARD_1_PILORAMA_PATH=/storage/pilorama1 diff --git a/services/storage/artifacts.mk b/services/storage/artifacts.mk index 4866a32..30d60be 100644 --- a/services/storage/artifacts.mk +++ b/services/storage/artifacts.mk @@ -1,20 +1,7 @@ -# Create new tls certs +# Create new TLS certs to FrostFS node -CURRENT_DIR=$(dir $(abspath $(lastword $(MAKEFILE_LIST)))) -STORAGE_DIR=$(patsubst %/,%,$(CURRENT_DIR)) -SSL_CONFIG := $(shell mktemp) +STORAGE_DIR=$(abspath services/storage) get.storage: - @echo "⇒ Creating tls certs to NeoFS node" - @(echo "[req]"; \ - echo "distinguished_name=req"; \ - echo "req_extensions=san"; \ - echo "[san]"; \ - echo "subjectAltName=DNS:s04.${LOCAL_DOMAIN}") > ${SSL_CONFIG} - @echo $(test -e "${STORAGE_DIR}/s04tls.key" && echo true) - @if [ ! -e "${STORAGE_DIR}/s04tls.key" ]; then \ - openssl req -new -newkey rsa:4096 -x509 -sha256 -days 365 -nodes \ - -subj "/C=RU/ST=SPB/L=St.Petersburg/O=NSPCC/OU=NSPCC/CN=s04.${LOCAL_DOMAIN}" \ - -keyout "${STORAGE_DIR}/s04tls.key" -out "${STORAGE_DIR}/s04tls.crt" \ - -extensions san -config "${SSL_CONFIG}" ; \ - fi + @echo "⇒ Creating TLS certs to FrostFS node" + ${STORAGE_DIR}/generate_cert.sh ${LOCAL_DOMAIN} > /dev/null diff --git a/services/storage/cfg/config.yml b/services/storage/cfg/config.yml new file mode 100644 index 0000000..0e2526d --- /dev/null +++ b/services/storage/cfg/config.yml @@ -0,0 +1,80 @@ +# Logger section +logger: + level: debug # Minimum enabled logging level + loki: + enabled: true + endpoint: "loki.frostfs.devenv:3100/api/prom/push" + max_batch_delay: 1s + max_batch_size: 200 + +# Profiler section +pprof: + enabled: true + address: :6060 # Server address + shutdown_timeout: 15s # Timeout for profiling HTTP server graceful shutdown + +# Application metrics section +prometheus: + enabled: true + address: :9090 # Server address + shutdown_timeout: 15s # Timeout for metrics HTTP server graceful shutdown + +# Application tracing section +tracing: + enabled: true + exporter: otlp_grpc + endpoint: "jaeger.frostfs.devenv:4317" + +# Morph section +morph: + dial_timeout: 30s # Timeout for side chain NEO RPC client connection + rpc_endpoint: # Side chain NEO RPC endpoints + - address: ws://morph-chain:30333/ws + priority: 1 + +# Tree section +tree: + enabled: true + +# Storage engine configuration +storage: + shard: + 0: + writecache: + enabled: true + path: /storage/wc0 # Write-cache root directory + + metabase: + path: /storage/meta0 # Path to the metabase + + blobstor: + - type: blobovnicza + path: /storage/blobovnicza0 # Blobovnicza root directory + depth: 2 + width: 4 + - type: fstree + path: /storage/fstree0 # FSTree root directory + depth: 2 + + pilorama: + path: /storage/pilorama0 # Path to the pilorama database + + 1: + writecache: + enabled: true + path: /storage/wc1 # Write-cache root directory + + metabase: + path: /storage/meta1 # Path to the metabase + + blobstor: + - type: blobovnicza + path: /storage/blobovnicza1 # Blobovnicza root directory + depth: 2 + width: 4 + - type: fstree + path: /storage/fstree1 # FSTree root directory + depth: 2 + + pilorama: + path: /storage/pilorama1 # Path to the pilorama database diff --git a/services/storage/docker-compose.yml b/services/storage/docker-compose.yml index 6bf8939..785ce79 100644 --- a/services/storage/docker-compose.yml +++ b/services/storage/docker-compose.yml @@ -1,6 +1,5 @@ --- -version: "2.4" services: storage01: image: ${NODE_IMAGE}:${NODE_VERSION} @@ -16,25 +15,25 @@ services: - ./wallet01.json:/wallet.json - ./../../vendor/hosts:/etc/hosts - storage_s01:/storage - - ./../../vendor/neofs-cli:/neofs-cli + - ./../../vendor/frostfs-cli:/frostfs-cli - ./cli-cfg.yml:/cli-cfg.yml - - ./healthcheck.sh:/healthcheck.sh - ./s04tls.crt:/etc/ssl/certs/s04tls.crt - - ../nats/client-cert.pem:/etc/neofs-node/nats.tls.cert - - ../nats/client-key.pem:/etc/neofs-node/nats.tls.key - - ../nats/ca-cert.pem:/etc/neofs-node/nats.ca.crt - stop_signal: SIGKILL + - ./cfg:/etc/frostfs/storage + stop_signal: SIGTERM + stop_grace_period: 15s env_file: [ ".env", ".storage.env", ".int_test.env" ] + command: [ "frostfs-node", "--config", "/etc/frostfs/storage/config.yml" ] environment: - - NEOFS_NODE_WALLET_PATH=./wallet.json - - NEOFS_NODE_WALLET_PASSWORD= - - NEOFS_NODE_ADDRESSES=s01.${LOCAL_DOMAIN}:8080 - - NEOFS_GRPC_0_ENDPOINT=s01.${LOCAL_DOMAIN}:8080 - - NEOFS_CONTROL_GRPC_ENDPOINT=s01.${LOCAL_DOMAIN}:8081 - - NEOFS_NODE_ATTRIBUTE_1=UN-LOCODE:RU MOW - - NEOFS_NODE_ATTRIBUTE_2=Price:22 + - FROSTFS_NODE_WALLET_PATH=./wallet.json + - FROSTFS_NODE_WALLET_PASSWORD= + - FROSTFS_NODE_ADDRESSES=s01.${LOCAL_DOMAIN}:8080 + - FROSTFS_GRPC_0_ENDPOINT=s01.${LOCAL_DOMAIN}:8080 + - FROSTFS_CONTROL_GRPC_ENDPOINT=s01.${LOCAL_DOMAIN}:8081 + - FROSTFS_NODE_ATTRIBUTE_0=User-Agent:FrostFS/${NODE_VERSION} + - FROSTFS_NODE_ATTRIBUTE_1=UN-LOCODE:RU MOW + - FROSTFS_NODE_ATTRIBUTE_2=Price:22 healthcheck: - test: ["CMD-SHELL", "/healthcheck.sh"] + test: ["CMD-SHELL", "/frostfs-cli control healthcheck -q -c /cli-cfg.yml --endpoint \"$$FROSTFS_CONTROL_GRPC_ENDPOINT\""] interval: 2s timeout: 1s retries: 5 @@ -54,25 +53,25 @@ services: - ./wallet02.json:/wallet.json - ./../../vendor/hosts:/etc/hosts - storage_s02:/storage - - ./../../vendor/neofs-cli:/neofs-cli + - ./../../vendor/frostfs-cli:/frostfs-cli - ./cli-cfg.yml:/cli-cfg.yml - - ./healthcheck.sh:/healthcheck.sh - ./s04tls.crt:/etc/ssl/certs/s04tls.crt - - ../nats/client-cert.pem:/etc/neofs-node/nats.tls.cert - - ../nats/client-key.pem:/etc/neofs-node/nats.tls.key - - ../nats/ca-cert.pem:/etc/neofs-node/nats.ca.crt - stop_signal: SIGKILL + - ./cfg:/etc/frostfs/storage + stop_signal: SIGTERM + stop_grace_period: 15s env_file: [ ".env", ".storage.env", ".int_test.env" ] + command: [ "frostfs-node", "--config", "/etc/frostfs/storage/config.yml" ] environment: - - NEOFS_NODE_WALLET_PATH=./wallet.json - - NEOFS_NODE_WALLET_PASSWORD= - - NEOFS_NODE_ADDRESSES=s02.${LOCAL_DOMAIN}:8080 - - NEOFS_GRPC_0_ENDPOINT=s02.${LOCAL_DOMAIN}:8080 - - NEOFS_CONTROL_GRPC_ENDPOINT=s02.${LOCAL_DOMAIN}:8081 - - NEOFS_NODE_ATTRIBUTE_1=UN-LOCODE:RU LED - - NEOFS_NODE_ATTRIBUTE_2=Price:33 + - FROSTFS_NODE_WALLET_PATH=./wallet.json + - FROSTFS_NODE_WALLET_PASSWORD= + - FROSTFS_NODE_ADDRESSES=s02.${LOCAL_DOMAIN}:8080 + - FROSTFS_GRPC_0_ENDPOINT=s02.${LOCAL_DOMAIN}:8080 + - FROSTFS_CONTROL_GRPC_ENDPOINT=s02.${LOCAL_DOMAIN}:8081 + - FROSTFS_NODE_ATTRIBUTE_0=User-Agent:FrostFS/${NODE_VERSION} + - FROSTFS_NODE_ATTRIBUTE_1=UN-LOCODE:RU LED + - FROSTFS_NODE_ATTRIBUTE_2=Price:33 healthcheck: - test: ["CMD-SHELL", "/healthcheck.sh"] + test: ["CMD-SHELL", "/frostfs-cli control healthcheck -q -c /cli-cfg.yml --endpoint \"$$FROSTFS_CONTROL_GRPC_ENDPOINT\""] interval: 2s timeout: 1s retries: 5 @@ -92,25 +91,25 @@ services: - ./wallet03.json:/wallet.json - ./../../vendor/hosts:/etc/hosts - storage_s03:/storage - - ./../../vendor/neofs-cli:/neofs-cli + - ./../../vendor/frostfs-cli:/frostfs-cli - ./cli-cfg.yml:/cli-cfg.yml - - ./healthcheck.sh:/healthcheck.sh - ./s04tls.crt:/etc/ssl/certs/s04tls.crt - - ../nats/client-cert.pem:/etc/neofs-node/nats.tls.cert - - ../nats/client-key.pem:/etc/neofs-node/nats.tls.key - - ../nats/ca-cert.pem:/etc/neofs-node/nats.ca.crt - stop_signal: SIGKILL + - ./cfg:/etc/frostfs/storage + stop_signal: SIGTERM + stop_grace_period: 15s env_file: [ ".env", ".storage.env", ".int_test.env" ] + command: [ "frostfs-node", "--config", "/etc/frostfs/storage/config.yml" ] environment: - - NEOFS_NODE_WALLET_PATH=./wallet.json - - NEOFS_NODE_WALLET_PASSWORD= - - NEOFS_NODE_ADDRESSES=s03.${LOCAL_DOMAIN}:8080 - - NEOFS_GRPC_0_ENDPOINT=s03.${LOCAL_DOMAIN}:8080 - - NEOFS_CONTROL_GRPC_ENDPOINT=s03.${LOCAL_DOMAIN}:8081 - - NEOFS_NODE_ATTRIBUTE_1=UN-LOCODE:SE STO - - NEOFS_NODE_ATTRIBUTE_2=Price:11 + - FROSTFS_NODE_WALLET_PATH=./wallet.json + - FROSTFS_NODE_WALLET_PASSWORD= + - FROSTFS_NODE_ADDRESSES=s03.${LOCAL_DOMAIN}:8080 + - FROSTFS_GRPC_0_ENDPOINT=s03.${LOCAL_DOMAIN}:8080 + - FROSTFS_CONTROL_GRPC_ENDPOINT=s03.${LOCAL_DOMAIN}:8081 + - FROSTFS_NODE_ATTRIBUTE_0=User-Agent:FrostFS/${NODE_VERSION} + - FROSTFS_NODE_ATTRIBUTE_1=UN-LOCODE:SE STO + - FROSTFS_NODE_ATTRIBUTE_2=Price:11 healthcheck: - test: ["CMD-SHELL", "/healthcheck.sh"] + test: ["CMD-SHELL", "/frostfs-cli control healthcheck -q -c /cli-cfg.yml --endpoint \"$$FROSTFS_CONTROL_GRPC_ENDPOINT\""] interval: 2s timeout: 1s retries: 5 @@ -130,31 +129,31 @@ services: - ./wallet04.json:/wallet.json - ./../../vendor/hosts:/etc/hosts - storage_s04:/storage - - ./../../vendor/neofs-cli:/neofs-cli + - ./../../vendor/frostfs-cli:/frostfs-cli - ./cli-cfg.yml:/cli-cfg.yml - - ./healthcheck.sh:/healthcheck.sh - ./s04tls.crt:/tls.crt - ./s04tls.key:/tls.key - - ../nats/client-cert.pem:/etc/neofs-node/nats.tls.cert - - ../nats/client-key.pem:/etc/neofs-node/nats.tls.key - - ../nats/ca-cert.pem:/etc/neofs-node/nats.ca.crt - stop_signal: SIGKILL + - ./cfg:/etc/frostfs/storage + stop_signal: SIGTERM + stop_grace_period: 15s env_file: [ ".env", ".storage.env", ".int_test.env" ] + command: [ "frostfs-node", "--config", "/etc/frostfs/storage/config.yml" ] environment: - - NEOFS_NODE_WALLET_PATH=./wallet.json - - NEOFS_NODE_WALLET_PASSWORD= - - NEOFS_NODE_ADDRESSES=s04.${LOCAL_DOMAIN}:8080 grpcs://s04.${LOCAL_DOMAIN}:8082 - - NEOFS_CONTROL_GRPC_ENDPOINT=s04.${LOCAL_DOMAIN}:8081 - - NEOFS_GRPC_NUM=2 - - NEOFS_GRPC_0_ENDPOINT=s04.${LOCAL_DOMAIN}:8080 - - NEOFS_GRPC_1_ENDPOINT=s04.${LOCAL_DOMAIN}:8082 - - NEOFS_GRPC_1_TLS_ENABLED=true - - NEOFS_GRPC_1_TLS_CERTIFICATE=/tls.crt - - NEOFS_GRPC_1_TLS_KEY=/tls.key - - NEOFS_NODE_ATTRIBUTE_1=UN-LOCODE:FI HEL - - NEOFS_NODE_ATTRIBUTE_2=Price:44 + - FROSTFS_NODE_WALLET_PATH=./wallet.json + - FROSTFS_NODE_WALLET_PASSWORD= + - FROSTFS_NODE_ADDRESSES=s04.${LOCAL_DOMAIN}:8080 grpcs://s04.${LOCAL_DOMAIN}:8082 + - FROSTFS_CONTROL_GRPC_ENDPOINT=s04.${LOCAL_DOMAIN}:8081 + - FROSTFS_GRPC_NUM=2 + - FROSTFS_GRPC_0_ENDPOINT=s04.${LOCAL_DOMAIN}:8080 + - FROSTFS_GRPC_1_ENDPOINT=s04.${LOCAL_DOMAIN}:8082 + - FROSTFS_GRPC_1_TLS_ENABLED=true + - FROSTFS_GRPC_1_TLS_CERTIFICATE=/tls.crt + - FROSTFS_GRPC_1_TLS_KEY=/tls.key + - FROSTFS_NODE_ATTRIBUTE_0=User-Agent:FrostFS/${NODE_VERSION} + - FROSTFS_NODE_ATTRIBUTE_1=UN-LOCODE:FI HEL + - FROSTFS_NODE_ATTRIBUTE_2=Price:44 healthcheck: - test: ["CMD-SHELL", "/healthcheck.sh"] + test: ["CMD-SHELL", "/frostfs-cli control healthcheck -q -c /cli-cfg.yml --endpoint \"$$FROSTFS_CONTROL_GRPC_ENDPOINT\""] interval: 2s timeout: 1s retries: 5 diff --git a/services/storage/generate_cert.sh b/services/storage/generate_cert.sh new file mode 100755 index 0000000..87672d2 --- /dev/null +++ b/services/storage/generate_cert.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +source bin/helper.sh + +WORKDIR=$(dirname "$0") +LOCAL_DOMAIN=$1 +SSL_CONFIG=$(mktemp) +CERT="${WORKDIR}/s04tls.crt" +KEY="${WORKDIR}/s04tls.key" + + +if [[ ! -f ${CERT} ]]; then + ( + echo "[req]"; \ + echo "distinguished_name=req"; \ + echo "req_extensions=san"; \ + echo "[san]"; \ + echo "subjectAltName=DNS:s04.${LOCAL_DOMAIN}" + ) > ${SSL_CONFIG} + + openssl req -new -newkey rsa:4096 -x509 -sha256 -days 365 -nodes \ + -subj "/C=RU/ST=SPB/L=St.Petersburg/O=TrueCloudLab/OU=TrueCloudLab/CN=s04.${LOCAL_DOMAIN}" \ + -keyout "${KEY}" -out "${CERT}" -extensions san -config "${SSL_CONFIG}" &> /dev/null || { + die "Failed to generate SSL certificate for s04" + } +fi diff --git a/services/storage/healthcheck.sh b/services/storage/healthcheck.sh deleted file mode 100755 index fdbf390..0000000 --- a/services/storage/healthcheck.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh - -/neofs-cli control healthcheck -c /cli-cfg.yml \ - --endpoint "$NEOFS_CONTROL_GRPC_ENDPOINT" | - grep "Health status: READY" diff --git a/services/storage/prepare.mk b/services/storage/prepare.mk index 22c276b..fad47b1 100644 --- a/services/storage/prepare.mk +++ b/services/storage/prepare.mk @@ -1,3 +1,5 @@ +# Add self-signed node TLS certificate to trusted store + prepare.storage: - @echo "Adding self-signed tls certs to trusted store" + @echo "Adding self-signed TLS certs to trusted store" @./bin/addCert.sh diff --git a/wallets/system-wallet.json b/wallets/system-wallet.json new file mode 100644 index 0000000..2f57db1 --- /dev/null +++ b/wallets/system-wallet.json @@ -0,0 +1 @@ +{"version":"1.0","accounts":[{"address":"NQijiVKHbL22PfF2AJQukv1CX75itxgzht","key":"6PYQKrpme57VqaucxuF7dDoSZRRA8d94oatHcScqhiFBauCXQvFDaYwEWa","label":"","contract":{"script":"DCEDRdLtpFIWeYyI7doTKRhIl4qYjaybGDveTyGpbqjsLZNBVuezJw==","parameters":[{"name":"parameter0","type":"Signature"}],"deployed":false},"lock":false,"isDefault":false}],"scrypt":{"n":16384,"r":8,"p":8},"extra":{"Tokens":null}} \ No newline at end of file