WIP

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>

.env

@@ -8,23 +8,23 @@ BASTION_VERSION=10
 BASTION_IMAGE=debian
 
 # NeoGo privnet
-NEOGO_VERSION=0.101.1
+NEOGO_VERSION=0.106.1-pre-4-g39c65ab5
 NEOGO_IMAGE=nspccdev/neo-go
 
 # FrostFS InnerRing nodes
-IR_VERSION=365a7ca0
-IR_IMAGE=truecloudlab/frostfs-ir
+IR_VERSION=0.40.0-17-ge15b545f-dirty
+IR_IMAGE=truecloudlab/frostfs-dirty-ir
 
 # FrostFS Storage nodes
-NODE_VERSION=365a7ca0
-NODE_IMAGE=truecloudlab/frostfs-storage
+NODE_VERSION=0.40.0-17-ge15b545f-dirty
+NODE_IMAGE=truecloudlab/frostfs-dirty-storage
 
 # NATS Server
 NATS_VERSION=2.7.2
 NATS_IMAGE=nats
 
 # HTTP Gate
-HTTP_GW_VERSION=0.27.0-rc.1-15-g1776db28
+HTTP_GW_VERSION=0.28.0-rc.1-15-g5ae75eb9
 HTTP_GW_IMAGE=truecloudlab/frostfs-http-gw
 
 # REST Gate
@@ -32,7 +32,7 @@ REST_GW_VERSION=c9c85e90
 REST_GW_IMAGE=truecloudlab/frostfs-rest-gw
 
 # S3 Gate
-S3_GW_VERSION=0.27.0-rc.1-30-gce929468
+S3_GW_VERSION=0.28.0-rc.1-51-g9272f4e1
 S3_GW_IMAGE=truecloudlab/frostfs-s3-gw
 
 # FrostFS LOCODE database
@@ -40,17 +40,17 @@ LOCODE_DB_URL=https://git.frostfs.info/attachments/a2e8def7-52b6-49f1-89cd-a0567
 #LOCODE_DB_PATH=/path/to/locode_db
 
 # FrostFS CLI binary
-FROSTFS_CLI_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/BbngJDdRJEDJTJk7qptq3SxKqrJqtvVYWU6R5AaFGbtG
-#FROSTFS_CLI_PATH=/path/to/frostfs-cli-binary
+FROSTFS_CLI_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/BtA6zWLtoDuDnzxSNkKPjyN5hGuSZFxGuJD1gh8kBwkP
+FROSTFS_CLI_PATH=/repo/frostfs/node/bin/frostfs-cli
 
 # FrostFS ADM tool binary
-FROSTFS_ADM_VERSION=eca5c210
-FROSTFS_ADM_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/2GxarAjGUb3RevxvqFGYT3hDQxNNaHzK6aFxhJCAMehq
-#FROSTFS_ADM_PATH=/path/to/frostfs-adm-binary
+FROSTFS_ADM_VERSION=498f9955ea
+FROSTFS_ADM_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/CjV4S6ENzo2FLK4KxXZHHNW4veR1ATtynGY6Mc1xQ6RB
+FROSTFS_ADM_PATH=/repo/frostfs/node/bin/frostfs-adm
 
 # Compiled FrostFS Smart Contracts
-FROSTFS_CONTRACTS_VERSION=8537293e
-FROSTFS_CONTRACTS_URL=https://http.t5.fs.neo.org/7sm9csjtRLpr4c9QD55q9JJM73v79ohuAhTzP4fYRHFz/6ccZoj4HxoN1G1qvJAX2Qw9p2D6qdyzAjNMaNkEKYQpA
+FROSTFS_CONTRACTS_VERSION=694daebb19
+FROSTFS_CONTRACTS_URL=https://git.frostfs.info/TrueCloudLab/frostfs-contract/releases/download/v0.19.2/frostfs-contract-v0.19.2.tar.gz
 #FROSTFS_CONTRACTS_PATH=/path/to/unpacked/frostfs-contracts-dir
 
 # Jaeger tracing
@@ -64,3 +64,7 @@ PROMETHEUS_IMAGE=prom/prometheus
 # Grafana versions
 GRAFANA_VERSION=9.5.6
 GRAFANA_IMAGE=grafana/grafana
+
+# Loki versions
+LOKI_VERSION=2.9.1
+LOKI_IMAGE=grafana/loki

.forgejo/workflows/dco.yml

@@ -1,3 +1,4 @@
+name: DCO action
 on: [pull_request]
 
 jobs:
@@ -12,9 +13,9 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v3
         with:
-          go-version: '1.20'
+          go-version: '1.21'
 
       - name: Run commit format checker
-        uses: https://git.alexvan.in/alexvanin/dco-go@v1
+        uses: https://git.frostfs.info/TrueCloudLab/dco-go@v2
         with:
-          from: dca6ff62
+          from: 'origin/${{ github.event.pull_request.base.ref }}'

.gitattributes

@@ -0,0 +1 @@
+/services/grafana/provisioning/dashboards/* -diff -merge

Makefile

@@ -76,6 +76,10 @@ up/bootstrap: get vendor/hosts
 	@$(foreach SVC, $(START_BOOTSTRAP), $(shell docker-compose -f services/$(SVC)/docker-compose.yml up -d))
 	@source ./bin/helper.sh
 	@./vendor/frostfs-adm --config frostfs-adm.yml morph init --contracts vendor/contracts
+	echo "Set rule chain to policy contract" 
+	@./vendor/frostfs-adm --config frostfs-adm.yml morph \
+		ape add-rule-chain --target-type namespace --target-name "" \
+		--rule 'allow Container.* *' --chain-id "allow_container_ops"
 	@for f in ./services/storage/wallet*.json; do \
 		echo "Transfer GAS to wallet $${f}" \
 		&& ./vendor/frostfs-adm -c frostfs-adm.yml morph refill-gas --storage-wallet $${f} --gas 10.0 \

configs/s01-cli.yml

@@ -0,0 +1,4 @@
+wallet: services/storage/wallet01.json
+password: ""
+rpc-endpoint: s01.frostfs.devenv:8080
+endpoint: s01.frostfs.devenv:8081

configs/s02-cli.yml

@@ -0,0 +1,4 @@
+wallet: services/storage/wallet02.json
+password: ""
+rpc-endpoint: s02.frostfs.devenv:8080
+endpoint: s02.frostfs.devenv:8081

configs/s03-cli.yml

@@ -0,0 +1,4 @@
+wallet: services/storage/wallet03.json
+password: ""
+rpc-endpoint: s03.frostfs.devenv:8080
+endpoint: s03.frostfs.devenv:8081

configs/s04-cli.yml

@@ -0,0 +1,4 @@
+wallet: services/storage/wallet04.json
+password: ""
+rpc-endpoint: s04.frostfs.devenv:8080
+endpoint: s04.frostfs.devenv:8081

frostfs-adm.yml

@@ -1,10 +1,16 @@
-rpc-endpoint: http://morph-chain.frostfs.devenv:30333
+rpc-endpoint: https://192.168.130.90:30333
+tls:
+  root_cas:
+    - ./mtls/CA1_cert.pem
+  certificate: ./mtls/peerCA2_IR_cert.pem
+  key: ./mtls/peerCA2_IR_key.pem
 alphabet-wallets: ./services/ir
 network:
   max_object_size: 67108864
   epoch_duration: 240
   basic_income_rate: 100000000
   homomorphic_hash_disabled: false
+  maintenance_mode_allowed: true
   fee:
     audit: 10000
     candidate: 10000000000

mtls/CA.cnf

@@ -0,0 +1,12 @@
+[ req ]
+prompt = no
+distinguished_name = req_distinguished_name
+
+[ req_distinguished_name ]
+C = US
+ST = Localzone
+L = localhost
+O = Neo Go Testing Certificate Authority
+OU = Develop
+CN = localdomain
+emailAddress = root@localhost.localdomain

mtls/CA1_cert.pem

@@ -0,0 +1,25 @@
+-----BEGIN CERTIFICATE-----
+MIIELTCCAxWgAwIBAgIUQXbR0CjHcyf5y4N9aSw++jijH6gwDQYJKoZIhvcNAQEL
+BQAwgbcxCzAJBgNVBAYTAlVTMRIwEAYDVQQIDAlMb2NhbHpvbmUxEjAQBgNVBAcM
+CWxvY2FsaG9zdDEtMCsGA1UECgwkTmVvIEdvIFRlc3RpbmcgQ2VydGlmaWNhdGUg
+QXV0aG9yaXR5MRAwDgYDVQQLDAdEZXZlbG9wMRQwEgYDVQQDDAtsb2NhbGRvbWFp
+bjEpMCcGCSqGSIb3DQEJARYacm9vdEBsb2NhbGhvc3QubG9jYWxkb21haW4wHhcN
+MjQwNjExMDgyNzQ4WhcNMjkwNjEwMDgyNzQ4WjCBtzELMAkGA1UEBhMCVVMxEjAQ
+BgNVBAgMCUxvY2Fsem9uZTESMBAGA1UEBwwJbG9jYWxob3N0MS0wKwYDVQQKDCRO
+ZW8gR28gVGVzdGluZyBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkxEDAOBgNVBAsMB0Rl
+dmVsb3AxFDASBgNVBAMMC2xvY2FsZG9tYWluMSkwJwYJKoZIhvcNAQkBFhpyb290
+QGxvY2FsaG9zdC5sb2NhbGRvbWFpbjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
+AQoCggEBAJ5Kzcp7qw74MqSktDnl3ITFCOJjoFw/lwqv5ktG8pw0pV46cYhIB39e
+lPJX7Po8Wi4v+P83LeMgIa0L7rvJmJyQpgdgsqKkFGYuTJJxlDsgYvcJOUK6tA4G
+X0/uj4Yq+iaO9HBiU3rD9SvnbXLNRAefxBWqucbjEoP5TYL2E20+gLtb5XCG2TH7
+rR0fvDoIQJ8BFpDd/D3GqEupzZ0pEC3x2zRFZ4MRFV64EFoO/CeymTUUBM3+vPTe
+D1kqP6FE+lww8aEg1Y6Q5+Il9nDY/BUvwUYugRxYj4Cgh+ZlmrVYShexz+0NpVHh
+VFn/B7vBuQrSWBVlMAVkLeU4t7Oy0C8CAwEAAaMvMC0wDAYDVR0TBAUwAwEB/zAd
+BgNVHQ4EFgQUGCmiSSj3ZXOraieVqDfYH7thN3IwDQYJKoZIhvcNAQELBQADggEB
+ABO9eWnrdihGe2Pe0AmO8jCCC1u8dD0h2dX4y+A1bL+0Wocvl9aZqxs1OkhHKese
+Eg16Uz2Vx95O5h9zMcH+0sazVADmLSUUzAlVE3cWVjPx5wOBBE2ym/nMqwoS1G06
+IUz561WTLInh1zcAVyifdnFpdPKBBNB7iH9zmiZwxXa5XP6JL4qdUNrw7rGfADON
+zCWorz0JrURC5nWj47rxUWoyFlT728Whzo5kl6Ynd4/yuwvcRKuCK7+eNIIks2Dd
+VbgbnROjSeXG0MKyWa41H4/4gY0TW8QTbXX458gzTfsp9sy82Ih6cbOnDUGaab3K
++OkGsYmEUC5xVYX5e1e4CPg=
+-----END CERTIFICATE-----

mtls/CA1_cert.srl

@@ -0,0 +1 @@
+28484A1BD594A9F1C28B5F1E4D45010D923E9D1B

mtls/CA1_key.pem

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEuwIBADANBgkqhkiG9w0BAQEFAASCBKUwggShAgEAAoIBAQCeSs3Ke6sO+DKk
+pLQ55dyExQjiY6BcP5cKr+ZLRvKcNKVeOnGISAd/XpTyV+z6PFouL/j/Ny3jICGt
+C+67yZickKYHYLKipBRmLkyScZQ7IGL3CTlCurQOBl9P7o+GKvomjvRwYlN6w/Ur
+521yzUQHn8QVqrnG4xKD+U2C9hNtPoC7W+Vwhtkx+60dH7w6CECfARaQ3fw9xqhL
+qc2dKRAt8ds0RWeDERVeuBBaDvwnspk1FATN/rz03g9ZKj+hRPpcMPGhINWOkOfi
+JfZw2PwVL8FGLoEcWI+AoIfmZZq1WEoXsc/tDaVR4VRZ/we7wbkK0lgVZTAFZC3l
+OLezstAvAgMBAAECgf8nUOn+VHq38jJ8gHjAJyW3lg069ZzMOh5i5tDcIBqGQXqG
+lZmp4i2jJAaWHmxgN9V/feGAJNCL3f7Wt7XfLRppluGZolgy4a1nmtOoeUjOq3TW
+iWE47RovpkZRHYvyobPXoXhA+xhjUveMG9ATgbAyDx8PI2s/tl71rVP9a2xVGYI8
+0KgXRYu/RIk3J2IhffIDGeDZe8oqV6E1Ch3x0yAy4xM6t2raqW4lxygZIT83AgaA
+fzu5bRdWDHc2SPXMZq8Gg9HT1FEE57VH4EsaiK7mcg5NKAeeyAriqzKPKb9YVboj
+y9Yx8dL5F+v14lx+5r5uWVzZ05Q+1U7xzRKQNrECgYEAyxHfrKmrUVoApaQIsv/N
+Szwo9pkU/GC2ogDMduN0L90aaGlPb+vt5szkO5bw5vCt+Pk+Xni8S5q+D7iW1wtK
+BqTy0YLrGPh7WVfS8axVtAvN8xXwUl2cVM3+4m++JUHpeEGmHzP/tCarqZciwY8G
+mHbtzYx4tIYgnFS5klGE22cCgYEAx40UQ6mWyC9wacl8UlGDnL8oseQpaJSNdi9s
+rZh0w7558THsyLYksCvxvB9Zej8f11Tuk91bcLzv/hYjwKCapLefFKAQoLgNU3Z+
+LCC6WCkrreskM6uo3hB99iXCzKqVOAOQcv3jJiJPn0BaJNRHLH7+zfoSIN29Kl7C
+1VvSr/kCgYAP+E3g5E8GzM80N5w/h4J7ojWOfyd5WZnDT9DEoMqoYPkMsQuAP8ur
+1P01+n7jipYD+IYBI/ma8IZ18gxayRZ/Q0WdWHBovFHnCtMaEnWC/VKrBVSrBZWf
+zbt3z1Xi1DYqMO+BkLNRVaE2ax5enoLkRqYY/i0X9VxnLvqRGTwbjQKBgQCdcyVw
+FdmQ7k1mSsI7OXH9NZ9obTx29e+ANv8rIcXPxPT2aIwJ+h2Mm7/KzXmOSUxYXP05
+0kiZ9ZnvA40GKjKakP+GCJ3WEiMBndcD/nbJpQJ/4a7YzjMZtC2PEFBZqKbAYZ+R
++91Ec4Ibh20Y7h2ScEmgQBYZtJvTWJ2mNyNQEQKBgBvDhoxafVAcVaZsIBVNfJ9e
+E5Oc3NSRplP8Q4L3CamLlvf28KYg4nrqvYeHkZ6JWq6VwBXv9F3g1fxn47tMWnBG
+GIY0UJnKtny58Jb6MrPp7YKboMxpigIFbsFZJI7ruh0g69W6oIqBIXDgcvlkAZT6
+Ianf60mgQzKmhNHHTZoW
+-----END PRIVATE KEY-----

mtls/CA2_cert.pem

@@ -0,0 +1,25 @@
+-----BEGIN CERTIFICATE-----
+MIIELTCCAxWgAwIBAgIUFU/P6uIwPnuxXA2smt/gcj2kF20wDQYJKoZIhvcNAQEL
+BQAwgbcxCzAJBgNVBAYTAlVTMRIwEAYDVQQIDAlMb2NhbHpvbmUxEjAQBgNVBAcM
+CWxvY2FsaG9zdDEtMCsGA1UECgwkTmVvIEdvIFRlc3RpbmcgQ2VydGlmaWNhdGUg
+QXV0aG9yaXR5MRAwDgYDVQQLDAdEZXZlbG9wMRQwEgYDVQQDDAtsb2NhbGRvbWFp
+bjEpMCcGCSqGSIb3DQEJARYacm9vdEBsb2NhbGhvc3QubG9jYWxkb21haW4wHhcN
+MjQwNjExMDgyNzQ4WhcNMjkwNjEwMDgyNzQ4WjCBtzELMAkGA1UEBhMCVVMxEjAQ
+BgNVBAgMCUxvY2Fsem9uZTESMBAGA1UEBwwJbG9jYWxob3N0MS0wKwYDVQQKDCRO
+ZW8gR28gVGVzdGluZyBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkxEDAOBgNVBAsMB0Rl
+dmVsb3AxFDASBgNVBAMMC2xvY2FsZG9tYWluMSkwJwYJKoZIhvcNAQkBFhpyb290
+QGxvY2FsaG9zdC5sb2NhbGRvbWFpbjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
+AQoCggEBALt25FN+ON6BC+Zoq/YKHtZNUpGDdbrubku+dVvJZqBKgonhvd9f5x5T
+4urQTyebHHZt0J7v9iVZ0e8C4+ddReG2R/D0L1b5EKSJps7fnXRiTZ6zyheT5tXF
+PjobCqdz1+tFDjs1ARCgOIpgT1/LCnGJAp95uNa63DFJVcLSwCk8XwrHWIKJpBla
+LdtlBXAv6O3AqIoO1PB816aYO4kvepnoG2gj18VWzxrUA1T4d73kKE4qpEiD9+sP
+xrf/ouBKXepgvvVzyH2ST+tfbc80mWdH1L9bmCOhuIr7Td/3x8CgyL6mqpp5Juul
+9oW/4zR60uwO8jt2+mSAvxCnQ/rHyOkCAwEAAaMvMC0wDAYDVR0TBAUwAwEB/zAd
+BgNVHQ4EFgQUdQ8u1e/E+11t2azBZbzCKYSCF3AwDQYJKoZIhvcNAQELBQADggEB
+AECiiXbIIGOLYVN99VXjnIkV48GkNTpzzgM9k4tWn81rDmkJROrrevWLv+LyD/N7
+hBWFwDE6C6gdlYuUc3w8SV6SrtjSqEJnKxW4ACLi7gceuToKXSJ3h9BR+0Xij1OV
+7biasJs63K2ASaGl/1gxrtHMMMaRYs58/6EOvPrY7f4CdcJOaeypOllSs0Xls323
+HTemYYyQJMHX0J1puUjipCw39ZpTcdB6OPscidvjJp9f3nfwbO4h9+eNlnvOv8L6
+p3gyITdDnSNZzVAXxbrEy5dLPZBwuZ1iFiLw5jeTTModRry+bXPzaTlLt711ULu/
+f9s6LUwv0bh6hwr0Ttq0saM=
+-----END CERTIFICATE-----

mtls/CA2_cert.srl

@@ -0,0 +1 @@
+2A340296D6CDD0E6A8F678126AC75B1DA2019AE2

mtls/CA2_key.pem

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQC7duRTfjjegQvm
+aKv2Ch7WTVKRg3W67m5LvnVbyWagSoKJ4b3fX+ceU+Lq0E8nmxx2bdCe7/YlWdHv
+AuPnXUXhtkfw9C9W+RCkiabO3510Yk2es8oXk+bVxT46Gwqnc9frRQ47NQEQoDiK
+YE9fywpxiQKfebjWutwxSVXC0sApPF8Kx1iCiaQZWi3bZQVwL+jtwKiKDtTwfNem
+mDuJL3qZ6BtoI9fFVs8a1ANU+He95ChOKqRIg/frD8a3/6LgSl3qYL71c8h9kk/r
+X23PNJlnR9S/W5gjobiK+03f98fAoMi+pqqaeSbrpfaFv+M0etLsDvI7dvpkgL8Q
+p0P6x8jpAgMBAAECggEADiuHFML5iRZ+D/uHf0CPYvZTSgm2A3Xaw0G+EAl7bWnu
+AuToyKxKHwiLCK7LdoK+AwgUxeUwWO04j8JLK2wce43SU39Rq/EfQrpFvP6gIaT9
+ZjdqDKAyyaPWkU1r+Vf2hHBWCnNDBYuzpzOieR4y4UqDw6Q1+ZBqua40zyL5DX5c
+9Mp0bXFsN+RVDh+K/9vbL+7vZaqZfe0wQDX1HbwP+u0vcDZjWS6QN09BGoJJr/PL
+SoovlVuEsDxoBIadGYtzGlRcLa6k03CI/jYFMKHOnyGFu87bNA8lPy9OCmJOaLgL
+Sb5aMkVoVFDZaOPtnxOTk9FRltXn1gWwyzVMJs7dRQKBgQDo/f2S58Mk4Rjudr4U
+ds6uamTOazMKlsTuMn0KeiOw3iWIYtrj7DxTRFhK0GR8dAksQCRHkDcQzbofzt0B
+/Qtx0bqEp7nLOLxjz/xf5i/h8394imR9IKpr02OvDjpGJ2itl2A3ADpsvrFR38ND
+z+CuyZ6RnPw0uYQqHmLWmgxNTwKBgQDN+fd21SsZe0MxhSL98fIZUOenUkxrJMac
+NL/JysKwfqXlJYpra8dyzQMoD3zUjIktRp45xKgJ8pUBfphmcUytJNJAW6hZ9A4t
+PnxQco2WPLfPSTPjVApSZRttfmMfZk3CMkfJt3k/L+cBD6umBzNHi1Q5ISV62luo
+Qh//J5YoRwKBgGjyf1W8r1ndSq2fLt9o5JALvHx+LO+IGgxxEYG2onz4k09rt80S
+i9m1309Tru+gFMGp4q2ZeDRJdpz9QO+0W4NZRPqb0kSHudP7y2Im/Up5s+FH9Run
+qxwGi9A+SGFxPuVyWhRsNeTlwFlwwTO6XypcBuDwJWpaZf3S0hJ4MTJhAoGACxNZ
+3yXvP1uFbXA/RXBxGDNlgXXHbit4EoYzmeIxPdRRj1TJcKVHAEFN3iWXDMbRyYF2
+mJyefIpywLXR9BA7k0PbmmJTVyLscDpf7ON8AYwaRiclhx/Jvkx5gRW+tr2FiN+z
+RIF9H4yv2zrAfBj/BOXVDY1eupBrRUzo84tpY5kCgYACptkwcJoaBvJdsql0iH7a
+0iCc+xO6rie1CRmcD/HzlWEYrsxDjmI1fdF/k52NPmgqBda38jq3ikkWe42/de0p
+GfR+fasq+K5h5FNEqudPW1POi3WG5m4MXljuRt/BXLHvXsnPHwK1vkpUNzViAkjf
+XCskokPNc8SDzPWgXd+VlA==
+-----END PRIVATE KEY-----

mtls/generate

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+outdir="${1:-./out}"
+
+genca() {
+    local name="$1"
+
+    echo "Generating $name ..."
+    openssl req -nodes -new -x509 \
+        -keyout "${name}_key.pem" \
+        -out "${name}_cert.pem" \
+        -addext basicConstraints=CA:TRUE \
+        -days 1825 -config CA.cnf
+}
+
+gencert() {
+    local ca="$1"
+    local i="$2"
+
+    openssl req -sha256 -nodes -newkey rsa:2048 \
+        -keyout "peer${ca}_${i}_key.pem" \
+        -out "peer${i}.csr" -config "peer${i}.cnf"
+    openssl x509 -req -days 398 -in "peer${i}.csr" \
+        -CA "${ca}_cert.pem" \
+        -CAkey "${ca}_key.pem" \
+        -CAcreateserial -out "peer${ca}_${i}_cert.pem" \
+        -extensions req_ext \
+        -extfile "peer${i}.cnf"
+    rm "peer${i}.csr"
+}
+
+genca CA1
+genca CA2
+gencert CA1 X
+gencert CA2 1
+gencert CA2 2
+gencert CA2 3
+gencert CA2 4
+gencert CA2 IR

mtls/peer1.cnf

@@ -0,0 +1,23 @@
+[req]
+default_bits  = 2048
+distinguished_name = req_distinguished_name
+req_extensions = req_ext
+x509_extensions = v3_req
+prompt = no
+
+[req_distinguished_name]
+countryName = US
+stateOrProvinceName = Localzone
+localityName = Localhost
+organizationName = Certificate signed by my CA
+commonName = peer1.localdomain
+
+[req_ext]
+subjectAltName = @alt_names
+
+[v3_req]
+subjectAltName = @alt_names
+
+[alt_names]
+DNS.1 = node_one
+IP.1 = 192.168.130.71

mtls/peer2.cnf

@@ -0,0 +1,23 @@
+[req]
+default_bits  = 2048
+distinguished_name = req_distinguished_name
+req_extensions = req_ext
+x509_extensions = v3_req
+prompt = no
+
+[req_distinguished_name]
+countryName = US
+stateOrProvinceName = Localzone
+localityName = Localhost
+organizationName = Certificate signed by my CA
+commonName = peer2.localdomain
+
+[req_ext]
+subjectAltName = @alt_names
+
+[v3_req]
+subjectAltName = @alt_names
+
+[alt_names]
+DNS.1 = node_two
+IP.1 = 192.168.130.72

mtls/peer3.cnf

@@ -0,0 +1,23 @@
+[req]
+default_bits  = 2048
+distinguished_name = req_distinguished_name
+req_extensions = req_ext
+x509_extensions = v3_req
+prompt = no
+
+[req_distinguished_name]
+countryName = US
+stateOrProvinceName = Localzone
+localityName = Localhost
+organizationName = Certificate signed by my CA
+commonName = peer3.localdomain
+
+[req_ext]
+subjectAltName = @alt_names
+
+[v3_req]
+subjectAltName = @alt_names
+
+[alt_names]
+DNS.1 = node_three
+IP.1 = 192.168.130.73

mtls/peer4.cnf

@@ -0,0 +1,23 @@
+[req]
+default_bits  = 2048
+distinguished_name = req_distinguished_name
+req_extensions = req_ext
+x509_extensions = v3_req
+prompt = no
+
+[req_distinguished_name]
+countryName = US
+stateOrProvinceName = Localzone
+localityName = Localhost
+organizationName = Certificate signed by my CA
+commonName = peer4.localdomain
+
+[req_ext]
+subjectAltName = @alt_names
+
+[v3_req]
+subjectAltName = @alt_names
+
+[alt_names]
+DNS.1 = node_four
+IP.1 = 192.168.130.74

mtls/peerCA1_1_cert.pem

@@ -0,0 +1,24 @@
+-----BEGIN CERTIFICATE-----
+MIIEGjCCAwKgAwIBAgIUKEhKG9WUqfHCi18eTUUBDZI+nRYwDQYJKoZIhvcNAQEL
+BQAwgbcxCzAJBgNVBAYTAlVTMRIwEAYDVQQIDAlMb2NhbHpvbmUxEjAQBgNVBAcM
+CWxvY2FsaG9zdDEtMCsGA1UECgwkTmVvIEdvIFRlc3RpbmcgQ2VydGlmaWNhdGUg
+QXV0aG9yaXR5MRAwDgYDVQQLDAdEZXZlbG9wMRQwEgYDVQQDDAtsb2NhbGRvbWFp
+bjEpMCcGCSqGSIb3DQEJARYacm9vdEBsb2NhbGhvc3QubG9jYWxkb21haW4wHhcN
+MjQwNjEwMTI0NjE2WhcNMjUwNzEzMTI0NjE2WjB3MQswCQYDVQQGEwJVUzESMBAG
+A1UECAwJTG9jYWx6b25lMRIwEAYDVQQHDAlMb2NhbGhvc3QxJDAiBgNVBAoMG0Nl
+cnRpZmljYXRlIHNpZ25lZCBieSBteSBDQTEaMBgGA1UEAwwRcGVlcjEubG9jYWxk
+b21haW4wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDDbW864Uy2nmSe
+pIcZw5TuPmA9t144D9t6h32E4PXdah1xon+HPszrHtPr4VHaILK+rJEdzWNa6069
+0gQgezUaI9Br39JbfB89dh/o7ncGB1tz/Q0C86Cj9EGGsPeL45xd+RSD0sbB5xYi
+fQqkvBHqNPmwJ1fIjIicEtQNtQmrBsBeCxLqhCzvVvqcHTclii5lG5nTRH8PzrFH
++K6S+l+GG3F4MCz/A4p7HvpkV0hXeo+VndL6TRUhIOUaB25PNW7HimAdIZaBpDpK
+IPx0KVCpiAk5wIS03sDxjiqnzwdyfj6qOEs1vLFsAq6RCCBtGhNMx3P6WpnTzhTq
+T/d/oDtRAgMBAAGjXTBbMBkGA1UdEQQSMBCCCG5vZGVfb25lhwTAqIJHMB0GA1Ud
+DgQWBBRp+mk7AmDNXGTHWPzJ4LHviCIjcjAfBgNVHSMEGDAWgBTpBvlB4aszHPUK
+jjsgWQNvcqAOeTANBgkqhkiG9w0BAQsFAAOCAQEAgBmp8qGsD5sDj8uZpDY1y4R/
+ne9xgesQejcESgxK2PoBk+iB4NSCIahMKI29q5M36C6xq4Zzj9mAETDdQgv8eJJU
+DHkFY1PKsULs3W5ODF4aufNogtm68mzqVQHW+qt/GdZj0BlIui6+G6uxFaRUjzry
+wl/7GXNFms8NiGRDmx0XdZHhA4aD3pprJVLNYEuZUH9N2Q4sUqxwjizh9qj5e5Vu
+XpnO6TqZvFJQMs61IYzlSWXpmBpDSjbTnm7+2C1RLRpidEhTtC1N8SRghc4n6yXg
+DnME009cIO9XwyAiFPrTheBZlgFJ+Z+aqTBGh0nG2AAbV6/2xh3rZMGA6QWHRw==
+-----END CERTIFICATE-----

mtls/peerCA1_1_key.pem

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDDbW864Uy2nmSe
+pIcZw5TuPmA9t144D9t6h32E4PXdah1xon+HPszrHtPr4VHaILK+rJEdzWNa6069
+0gQgezUaI9Br39JbfB89dh/o7ncGB1tz/Q0C86Cj9EGGsPeL45xd+RSD0sbB5xYi
+fQqkvBHqNPmwJ1fIjIicEtQNtQmrBsBeCxLqhCzvVvqcHTclii5lG5nTRH8PzrFH
++K6S+l+GG3F4MCz/A4p7HvpkV0hXeo+VndL6TRUhIOUaB25PNW7HimAdIZaBpDpK
+IPx0KVCpiAk5wIS03sDxjiqnzwdyfj6qOEs1vLFsAq6RCCBtGhNMx3P6WpnTzhTq
+T/d/oDtRAgMBAAECggEAJnFDwNmaYU8BnIphUBJe3Mm03VcRh+Yopq755DZrbvFl
+ltmgC/7wyOyAxAUH+KhXePYRusLRAUzJ6cGjEa+i18BFSr8Vzmy5aXnv1353N96h
+IQ0dbt/1UOMms6Pioig1NXAelrrS4cg/HKwruJGNVZyzpDDtDfSFF+yFPagH3ZYn
+laZJ/ZTiGu30ezu48QDEIzC4CKKaFIoBCpcG5kyQUx4aZInXjTDnDzjzm8+OnaKE
+9pQ6ILBDprjjFhbH+gphfo32Dmt9lbZ26s7ZsmjMChgIwIyUHliu+xDPJeFAcWMl
+/yNchDupx/jKVoj4I2YrICJwZLtJy6tPxdUGPsPPpQKBgQDo5v8nuS0NJm20aA1Q
+YNJWkG9ziRJ+XdYJw7fe2lSkZYhsG1J+MC0FLS0xAsZzo1i7kBETEFBm0Tf+D1DZ
+81Heo0Brh11qdMDhe+ENaLoswML1adbNJnDq6MW+clZcpgPKchSSEDeX9MFUL+x4
+yHsLZyT1lWd+PcC2rPE7TD61vQKBgQDWzwN3bNYdeuYytFGWTPt0pIFuaSkghO61
+g6+VJkwSOCbYnFddJIZSEbiPapZ3iaEMoQL7qE9Z/egkTFmxqtxW0aWruD5ETWyr
+gyrmQeimewKW7PkZMWjmYI7Uy6CGaCVL9QiMxxfaLoe3rJDlYM+m3xXceLlpKTNI
+Uu34zjXDJQKBgQCw5hNTprDzVgbpikbikEfy/A3LIYfsd3WFQSU9nXPA0mXGRxJn
+SkxHrEKxATyexKk2/XwakL+g+LSgwh1VDh1XD4R0qAJqQ3flTGRp/Ou75kWuhc9j
+TxlhM6BOwSWbhXVHpVK+hpbTUlItkaD5NL1Qse0+o1ZSarEfj0hDE+xluQKBgQCH
+jSLW3etPtxpErN0/R1z2gkja76lwR/KKwIqwR/aBGMYKsr2+nSnV6DDa75v72vGy
+EnjbbZ6FX2B6Z2PCZdudPbaZ4i4I1ALbSG5t1vGE/OLpcqjOaQuVNSFszwKBTn8a
+q4m1rH00xWAHW/IjjS/8UZJuChCWd9VQsmRROW9bsQKBgQDnEaDlfDB9C42Ep3G5
+tHfSaEsrrwyX619yDiJ52VKlzC48nDHn15Rq7iw9GTwAd5ILH+ii/uvzusAHOYyx
+AIlvtD/3Qkoc6PjQY2nCfCtWggVWcg2BK7sPpjK0hEjqbERbJgYshoNOsCL+K+zc
+oqDoG/GSG3nY9OSYNN6zq3m6lw==
+-----END PRIVATE KEY-----

mtls/peerCA1_X_cert.pem

@@ -0,0 +1,25 @@
+-----BEGIN CERTIFICATE-----
+MIIEOTCCAyGgAwIBAgIUKEhKG9WUqfHCi18eTUUBDZI+nRswDQYJKoZIhvcNAQEL
+BQAwgbcxCzAJBgNVBAYTAlVTMRIwEAYDVQQIDAlMb2NhbHpvbmUxEjAQBgNVBAcM
+CWxvY2FsaG9zdDEtMCsGA1UECgwkTmVvIEdvIFRlc3RpbmcgQ2VydGlmaWNhdGUg
+QXV0aG9yaXR5MRAwDgYDVQQLDAdEZXZlbG9wMRQwEgYDVQQDDAtsb2NhbGRvbWFp
+bjEpMCcGCSqGSIb3DQEJARYacm9vdEBsb2NhbGhvc3QubG9jYWxkb21haW4wHhcN
+MjQwNjExMDgyNzQ4WhcNMjUwNzE0MDgyNzQ4WjB3MQswCQYDVQQGEwJVUzESMBAG
+A1UECAwJTG9jYWx6b25lMRIwEAYDVQQHDAlMb2NhbGhvc3QxJDAiBgNVBAoMG0Nl
+cnRpZmljYXRlIHNpZ25lZCBieSBteSBDQTEaMBgGA1UEAwwRcGVlcjEubG9jYWxk
+b21haW4wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC2JVCd0TRhpsNP
+pHxo9DyIGMiuHPdj18meSe0HWbHk3aMLonBmB+1RqJa/7MnwYTFVsb4hajlTCVmd
+2xOGA73NWZt3VFLY0tWp3AHqmrqqC7jE1DIXnJPQNesYifKNK5e2uJQf3xkwBYz3
+Gr664U9dAIDJtE3nFX7QWG5k1ozDaeeqX3kylevcgLxyZBrXPozkfggJRrvVFgct
+GuLAKdApVsIiCSPRMnTYM3dLgos0gaio5KHWXE6QfpbqC12EIw0fqucmXqjtpa9B
+/tIRkRMQfqJ8sMdI9XOwc3G2yGaI5HTVeiyDHnDnb10izdpN8gi3tH3r8sEN8ENs
+taaKu+njAgMBAAGjfDB6MDgGA1UdEQQxMC+CGm1vcnBoLWNoYWluLmZyb3N0ZnMu
+ZGV2ZW52ggttb3JwaC1jaGFpbocEwKiCWjAdBgNVHQ4EFgQUY203z9j1OaMLZSEa
+FGBD6IMQVFowHwYDVR0jBBgwFoAUGCmiSSj3ZXOraieVqDfYH7thN3IwDQYJKoZI
+hvcNAQELBQADggEBAACEPUGv/sd/QQDLt/AF8tBiF6O8eDgx6FIrA0GTbb7+5oGK
+DxA4F5CTrLz8LnsEqujq6pSaa33WUdEnrh/FlGmdIEwHtCc8dukiff8bWvpTLIaY
+TPNUh4Y0Y2dZJ8ienO4OSU9psUDCyDaNmOja/DlfYYeIdSXtrMr10fEQgeIKEYGL
+yH1kHrz1aMJvLdk08U1hRlFWApn1JoSAVyWR/uyrUJ7czXEtgbKdVqnuzAGvnn/l
+FHHNiQmelK/6aEAa1Cg2Q7Rz+9lszRVZPpBJvHUjeA5N6b7eEFgLdhJSmJV40OeV
+6pw9IVP7CMezPBNXjMcj+COme4pK6HY9zHQCiFw=
+-----END CERTIFICATE-----

mtls/peerCA1_X_key.pem

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC2JVCd0TRhpsNP
+pHxo9DyIGMiuHPdj18meSe0HWbHk3aMLonBmB+1RqJa/7MnwYTFVsb4hajlTCVmd
+2xOGA73NWZt3VFLY0tWp3AHqmrqqC7jE1DIXnJPQNesYifKNK5e2uJQf3xkwBYz3
+Gr664U9dAIDJtE3nFX7QWG5k1ozDaeeqX3kylevcgLxyZBrXPozkfggJRrvVFgct
+GuLAKdApVsIiCSPRMnTYM3dLgos0gaio5KHWXE6QfpbqC12EIw0fqucmXqjtpa9B
+/tIRkRMQfqJ8sMdI9XOwc3G2yGaI5HTVeiyDHnDnb10izdpN8gi3tH3r8sEN8ENs
+taaKu+njAgMBAAECggEAOxjlTKcVfz97rCn6oZTo1OFxryg/zhmz1VM4AAQGl3Kf
+1YUJQdl9ktqHutKpfIAUYTDvtui/m6Tzeieb8IXjPa6+pdtqm+XgJCH5TzSPrt9p
+vhb4/7cyMEJmOkp7s3j62vKNQMACbJXIdF4tlvXGaa7hmlARXJIZWKDwhHAmMLpt
+z9t5bW2N/oZOC9lMwmz48FLIszrskVKX3kQTr1rb0cux3SSmDSnd9l8Zu7TMqKTt
+Dja2JDYFBTuOpa7WcWLyMHM/1omnRXok1cFKgCURsl487/B6r9FZaOtEQqtElZXB
+w+7cB1iG+1q0JXk/qaFtf+5Bu637AJwdELGnJm8gzQKBgQDn7gxvT2vrkQUuQfB1
+URqzJl5yuFsVSeu6xFqmWJORwBhdVO7nqAMX2IGrbnmBslGUys5LlpjN0wK/xMrw
+D8bRnqTlZgOG/lTrcZqsqqDOGn6k6oKaufIlt4jVwWzr1R4qFQNe5lXXqJ5Ka7LO
+Gt4ej5yJz7PN//Hlrgk3bVjjLQKBgQDJDJf8Ii210W9/QjAJuCGCsbVDUMS92t7F
+4GGVLAoY0PHsa8s2RRTQmrgfLa/Zzx/oAZs3tkihxGriZLk/Xvk+VaJY3wIoWCAm
+6uskFBtINlt/lqdtbe8nZRONlxUYB20izYHcoQ6RVj/JmpLR8Tib9lyYItKnGltQ
+eUOEuwJrTwKBgQDMK648qwEJPf/+icPBsyU7ApLwkMR2CD4Bi2t1ttIp455PQz6R
+JkGa01jLbpR+jVjuEF7tAeUAhDqOURKJpInU7MCixVxjkS92mOqeCBNJfN9yFJPm
+27yEYX6PFhjpvxIv35G5EDIPiHvYbdjZQaO56ZjXZMYXmbH8DGpGpfcy/QKBgQCP
+wm+THKVu0zlU1/EdogpkN8oTPlfQIgcVmyJr1jfKEu5KTbHbNn5xw5BQywxpRI/e
+B/gDWFaLRbXf+IH17GVwyy7xOVWDn7At4+ELxUzUz/mRPBqwRuGxW7clkZjFu9mv
+/x8Ssh+GkRO/hsXhVQlTjFAdgRj73/7Xhj5fVJGgRQKBgHZzvs8er7jt0bPF5qZm
+MFSy8rqgSzaDXbZQsEccIdZmiz2D8mfaj3MnjzRdV9eO+1A/rwYo7+Co740Bxx7D
+24AoLhBEEStKWwenLkBq9jDUhlR9+o1E2Zg71w++QV/dmOHSNWZOjWEItFA+IZ/i
+lZVCsTptAVe0khSUBYFaEAsA
+-----END PRIVATE KEY-----

mtls/peerCA2_1_cert.pem

@@ -0,0 +1,24 @@
+-----BEGIN CERTIFICATE-----
+MIIEGjCCAwKgAwIBAgIUKjQCltbN0Oao9ngSasdbHaIBmt4wDQYJKoZIhvcNAQEL
+BQAwgbcxCzAJBgNVBAYTAlVTMRIwEAYDVQQIDAlMb2NhbHpvbmUxEjAQBgNVBAcM
+CWxvY2FsaG9zdDEtMCsGA1UECgwkTmVvIEdvIFRlc3RpbmcgQ2VydGlmaWNhdGUg
+QXV0aG9yaXR5MRAwDgYDVQQLDAdEZXZlbG9wMRQwEgYDVQQDDAtsb2NhbGRvbWFp
+bjEpMCcGCSqGSIb3DQEJARYacm9vdEBsb2NhbGhvc3QubG9jYWxkb21haW4wHhcN
+MjQwNjExMDgyNzQ4WhcNMjUwNzE0MDgyNzQ4WjB3MQswCQYDVQQGEwJVUzESMBAG
+A1UECAwJTG9jYWx6b25lMRIwEAYDVQQHDAlMb2NhbGhvc3QxJDAiBgNVBAoMG0Nl
+cnRpZmljYXRlIHNpZ25lZCBieSBteSBDQTEaMBgGA1UEAwwRcGVlcjEubG9jYWxk
+b21haW4wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC6LtYIP1mXV7pn
+UIHkLXubW8Z1i/+9mxVFBUBEz8LE71AMtCjjox63sxeK9iklr3gjhgqqRDNjnexo
+CEU61iS8814WiTn5gLVbLleaQ7j1u8bGFxxCwsz1trYv0Xr39j41q/j1NpRgb6+q
+MhG2HkSZx+h3Fg8s2tNkoMaa5xOD547E6KJPOuQJe/JHeQW5WNmTFf99DdJ1YBum
+DP2toblkOtbP/EJ970B6EdEJGmrAjMRaJcvRsgF3fWQpA94NrsfL5rxhS8mZMtXv
+l/gaRA9ntsl3uUHcSBjrgBLnhTBTpeBbyEE07jiGVWg/A61f39ULgFCZ8OAA40Fd
+oNAKHrDBAgMBAAGjXTBbMBkGA1UdEQQSMBCCCG5vZGVfb25lhwTAqIJHMB0GA1Ud
+DgQWBBSIu7uWx5liJGDXuaemGYD9HK100TAfBgNVHSMEGDAWgBR1Dy7V78T7XW3Z
+rMFlvMIphIIXcDANBgkqhkiG9w0BAQsFAAOCAQEAAZzhN4+6EohtnusOiklPC2rY
+063JBG+vbaPI4QpmCdg2Hu+gHquJQ7y2gXOL0VxmQzh+NpOS10c6+U1oVsEEcpE2
+MYwRdVy8aUczn5lpwU1HQqoHI5/J1J9G++Zh4RbS33DZQpA34sqyMjdka951YQhu
+Nsgc/j7G5QOra1/CaGQKdjF9rlQKED44V6G/Y67AKHAFfkExubo+Dc/78uVe+UL0
+PPopHf+edQ3xWVB9iDUBDAdhCX0LOs0BowSlYl9rNG99zfI1NLcJC1bHdFJ7e3tW
+nIiivTi870w/WCuT5Zf7lYAz3KqjUqxl4PPVyDrxBL1z+h7RoUzDKoSBfpEFhA==
+-----END CERTIFICATE-----

mtls/peerCA2_1_key.pem

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC6LtYIP1mXV7pn
+UIHkLXubW8Z1i/+9mxVFBUBEz8LE71AMtCjjox63sxeK9iklr3gjhgqqRDNjnexo
+CEU61iS8814WiTn5gLVbLleaQ7j1u8bGFxxCwsz1trYv0Xr39j41q/j1NpRgb6+q
+MhG2HkSZx+h3Fg8s2tNkoMaa5xOD547E6KJPOuQJe/JHeQW5WNmTFf99DdJ1YBum
+DP2toblkOtbP/EJ970B6EdEJGmrAjMRaJcvRsgF3fWQpA94NrsfL5rxhS8mZMtXv
+l/gaRA9ntsl3uUHcSBjrgBLnhTBTpeBbyEE07jiGVWg/A61f39ULgFCZ8OAA40Fd
+oNAKHrDBAgMBAAECggEAAN9YQd/zhzybt2JZ3qdm+2zEpHbU6oYaW+c1KiPmwn8+
+63WBSxCZVF7o7LZNwMeFE96AA0aE9xc/9Wl4SYFb4oJNU9+A1rOSIm518sujPetS
+Z8TzAoSFqxw99AjWFe3Mqale9qmkixyWbKuuHYxY7atdUc9j2NU65XQDYQ00NPT0
+g2GYXqi1fDam4mR/GqgnbwTYnCMRfbTp61E62n3g8UeTDX5gwLjiRiI2f7MaKQM/
+OzYa3nLi2YrNbk9nm/QfWSImSX+htD7vzF11GJ90SxwnscjbJyDZ9x/GrJTsT4MU
+kC26vuj86MXRD0+JnS1B89d0zp1TTqDiH1mqh13DnQKBgQD1tfnio4gJNBbxP4cG
+ltljlEOTPkTnMABYAXUmK8K5cXxdzMbmBaPY6Ih27X7PsMNtN4OcWr1fJfzUSlOp
+d/NbBJBWn4wUArcENmlpCD4MlhnLT54APr12R5+5UxzMRAzXomYNJLCNP3R49N5F
+jiV0Qs1OaeZueRrkyLBP/4u3ZQKBgQDB+rhXDuKsnjdkFZM/M6ovQ/CIji7rqMKG
+lwAB6dDWTl04Y/DO6+WF+vOYUJicq/STqPbSS8YkwGX+f4Jmx90ej+O6X/wB2zM0
+ifh9+fgYSNKHgvsRdwHU2qGG+5e9uCE/3ZA0/y5f91YWRxvX5g694RDB1ViJBIhb
+5KO5fnxkLQKBgQDmn8SI9mc0BEFYdQE+rQ4kSkBXDwZLXKGl+NBFYKOnvZ42yIMT
+2c1dRXz465ewR0BW3dlZQ04l5Zj2hUEbeqvfDVyRAbXP/nBbstqjrAnZdHWHgBZU
+Mo9QWo/PufP4whKvvw08Xuib2yhuc5/A91euVRkw9XNE08TrYnciz0ALhQKBgDnP
+y1jG6u+fR8rgUz4m27WyfeF03q8FYNRT9ICdeswkFsxTnNVOoN1JaYgfnRw0c3P+
+i3EpfCg/3yX2Qpr3PgN5xtko+iZjeeNSq2iAM1ypd3VDqCgQW89gSXu9SAAW/HHY
+Sr5Av0WCpXmGP9ocipA0n5XlS7mqTCl2/wStmc0RAoGBAIJmrLt0TCJ1kh8b7g0w
+d+Bt/4oCVDgyKt9MOnBmPXBalYvEICLQIQ3YcXD0Lx3yy5MKY5MfokMUek5NSdjK
+UIBYhOhgckIrfT924zPPQqp7OnVAPIgvCZASBGmof2ithiYl/g03S0ZuN2bIcDEM
+j2JMnyCeSFqln72tzeMm6M0O
+-----END PRIVATE KEY-----

mtls/peerCA2_2_cert.pem

@@ -0,0 +1,24 @@
+-----BEGIN CERTIFICATE-----
+MIIEGjCCAwKgAwIBAgIUKjQCltbN0Oao9ngSasdbHaIBmt8wDQYJKoZIhvcNAQEL
+BQAwgbcxCzAJBgNVBAYTAlVTMRIwEAYDVQQIDAlMb2NhbHpvbmUxEjAQBgNVBAcM
+CWxvY2FsaG9zdDEtMCsGA1UECgwkTmVvIEdvIFRlc3RpbmcgQ2VydGlmaWNhdGUg
+QXV0aG9yaXR5MRAwDgYDVQQLDAdEZXZlbG9wMRQwEgYDVQQDDAtsb2NhbGRvbWFp
+bjEpMCcGCSqGSIb3DQEJARYacm9vdEBsb2NhbGhvc3QubG9jYWxkb21haW4wHhcN
+MjQwNjExMDgyNzQ4WhcNMjUwNzE0MDgyNzQ4WjB3MQswCQYDVQQGEwJVUzESMBAG
+A1UECAwJTG9jYWx6b25lMRIwEAYDVQQHDAlMb2NhbGhvc3QxJDAiBgNVBAoMG0Nl
+cnRpZmljYXRlIHNpZ25lZCBieSBteSBDQTEaMBgGA1UEAwwRcGVlcjIubG9jYWxk
+b21haW4wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDLpXZ2amr7CS0J
+MPhf3u/xizep+xZcXOryubo+kGbkr9QWmq/lDvIUNv8Jpdb24oq4qjTOCUMMY2le
+6gEmJF0/z7bZMmPEpmsLvIKQap1JpnfAmgxx7qv05S2FgTHv/nhlb55MLItA7HVO
+HDfbvtgePrRIfjm+EfAFT8x4+Vhf6xPH5E+Wn8mXH+7pwloHG8/gGefo6FTpv5IX
+vaRkRxeDD8usD5DdiIWruJ+BiPpOS6BkwbPGBOWh747ZXk59/afve0MnTszoO86o
+zniwAfGcuNCSsJxaduReaVddE0NqRrOE60h42zBGr2yxRkC1IoKL5Ae7kDcNLfIf
+bgoCWk/zAgMBAAGjXTBbMBkGA1UdEQQSMBCCCG5vZGVfdHdvhwTAqIJIMB0GA1Ud
+DgQWBBTCguoTciQNYyd3tCacwGxeRLvcsDAfBgNVHSMEGDAWgBR1Dy7V78T7XW3Z
+rMFlvMIphIIXcDANBgkqhkiG9w0BAQsFAAOCAQEAEPHDVLCv7J+nFKkLrWcAuUlr
+vtG11Z5cFowlwU+L3W2jqAcnANCAfz4FOgj6Gq0PGzKhqtHwjztvRd8zyTxpdVw4
+D7cU6OQKBGw+Grcyhj/eN4zUpD30oR/vxZS1zfQl5jasw0TYQUbjbQZz1d0n9NAW
+eZZ//aprE9EVbzQIirbWGC6ORKqws3k8/Pf6o5aow31puJDmOn26ISeWHLrydBHU
+XjxK3w6/sY1ioTNUSuGkZzM/sDeG1CRrhc50tRtuL+p/v35CLc2cd7LXBUBxnnqH
+DtsGtlUsk6WHwYXqAeREZypVTCkTcQ5OOPur080TOIrOJilRGq6yre+c6wjhow==
+-----END CERTIFICATE-----

mtls/peerCA2_2_key.pem

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDLpXZ2amr7CS0J
+MPhf3u/xizep+xZcXOryubo+kGbkr9QWmq/lDvIUNv8Jpdb24oq4qjTOCUMMY2le
+6gEmJF0/z7bZMmPEpmsLvIKQap1JpnfAmgxx7qv05S2FgTHv/nhlb55MLItA7HVO
+HDfbvtgePrRIfjm+EfAFT8x4+Vhf6xPH5E+Wn8mXH+7pwloHG8/gGefo6FTpv5IX
+vaRkRxeDD8usD5DdiIWruJ+BiPpOS6BkwbPGBOWh747ZXk59/afve0MnTszoO86o
+zniwAfGcuNCSsJxaduReaVddE0NqRrOE60h42zBGr2yxRkC1IoKL5Ae7kDcNLfIf
+bgoCWk/zAgMBAAECggEAHUtmkJ87V3RY9acFO6lSw+Ded4XGkC/Pnm4INCLKomAG
+tt+2qNEvEqhcpLMmCqrJAMp/QRpE+l28lSo9UxLNFTZYm8LrZij4Yh5Se9rSWHtp
+Y5vlaWhHdTmYhEly+Q6WnCTj1RpdR9AcCcn2YRaBeUxU287ImKZ07BawF8DqrRVe
+YKOEdBDHIXkmGswn00L3vbnoexVryJeIBcNKsQOJQq0SeE9YHCb52U+iifWwEK6d
+hucft0cHatP2ogW2b7GfP4040M9cRn7P2Knl9ORL1VqqHRKS3QvxKfKoXaopaRFg
+C32qWo0pW050kYtsA9PJRFYHRjjGFV6AP/lLBw65kQKBgQD87IhxpKwOIW/62ceu
+cyEIFI7RLZf6sX9HG7aJMV6/vIaWSS1I0ICLpBOVon39kf4/8vNeEx6pY8gIeg9g
+dAcaDsg/nS1gZynTOVgFTC9Z+5UwX4Azm+bXxFzR2jcOOAMXxjMFx/rliS1Lgdrv
+QfWnr/DW/aKz3KEUO9imCdnU8QKBgQDOH4GVC91n5e/VG421hNU9OEse2eSC84ci
+2wbBJVnsUarH/0W1z1Vk+HBEhcsNEpTCsH0A5c69xizOueL9zgEQinv2I04M1DAn
+vSG9G1cSz+01dl1Pn4RFoSCl7M2Yua0kRLVb88/TP0/6fWvKsDzcKMagb4EFg6eV
+af9m68ljIwKBgQDhWX5YgvgpvvEe+FtJu5p6sJZvGciHvQ5CNRb2Nd7NxxPDY3/N
+mLhvsskgUZ8gBqwaHkVoVHDFS/o2U0rQDZEmTnVs+IYtlr1VTYp3beHisp3Cc9Su
+JwIOzQ5Gi77yFEXtMNLVG9EYIpkQ5apD8ukFd2WGqIYG61U5yfjGN5C2gQKBgDKP
+r+2rt2fhE2+nDp/UMaqOwH6U3GTy79cw+vst+lzpu4bLq4HplfzqpD9qwmA2ip8n
+Q8KQnKIYlq0vE+tGOSShk77Q8jhbZGSbFEebCUToFDvlCQabDos19xg8ekJYaYxX
+/lTL/dLH2QqM8YAKsyMeLXr1XyTHIriYA7pvZDsZAoGAGwU0vmfUidqcJrFuQ3hM
+z0lUUpvUAIqlpE//NFQay3qA68Kt454vMDYpbPT9ZxW/yF39kNHw8PSWSBaomF3Z
+6n82LiL8O+SNaHeGstNj9ATUn2qPFotqUozDKa3A3RxVzXGzCFQzrh7pWcjH7cOd
+scdlOJZhh1kr0eW71OqcKJU=
+-----END PRIVATE KEY-----

mtls/peerCA2_3_cert.pem

@@ -0,0 +1,24 @@
+-----BEGIN CERTIFICATE-----
+MIIEHDCCAwSgAwIBAgIUKjQCltbN0Oao9ngSasdbHaIBmuAwDQYJKoZIhvcNAQEL
+BQAwgbcxCzAJBgNVBAYTAlVTMRIwEAYDVQQIDAlMb2NhbHpvbmUxEjAQBgNVBAcM
+CWxvY2FsaG9zdDEtMCsGA1UECgwkTmVvIEdvIFRlc3RpbmcgQ2VydGlmaWNhdGUg
+QXV0aG9yaXR5MRAwDgYDVQQLDAdEZXZlbG9wMRQwEgYDVQQDDAtsb2NhbGRvbWFp
+bjEpMCcGCSqGSIb3DQEJARYacm9vdEBsb2NhbGhvc3QubG9jYWxkb21haW4wHhcN
+MjQwNjExMDgyNzQ4WhcNMjUwNzE0MDgyNzQ4WjB3MQswCQYDVQQGEwJVUzESMBAG
+A1UECAwJTG9jYWx6b25lMRIwEAYDVQQHDAlMb2NhbGhvc3QxJDAiBgNVBAoMG0Nl
+cnRpZmljYXRlIHNpZ25lZCBieSBteSBDQTEaMBgGA1UEAwwRcGVlcjMubG9jYWxk
+b21haW4wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDMsMLbfg9Mtgvr
+K9LvlingAgU5c1XfGMkQigbShtVyiknwx7HjAX6hQLRvcIG/S9LOG48YHbI9oEmP
+8goZW/umZPHnwPJpNmDaPPZMkoTNwVXrCq/fbjoiY038zGCLYzBzwBMFzu51sraF
+GZnAW3az5m7l8d6xXOrhbxISGkhCVJaVEBu0eGt7bAxo8OwJmt6t8pzVWFwNq9lS
+dMGqwudmzY4xMs6W2ie8BODOmXIAya5X0yGDwFA0nUIeTMjQK3PqezOTaARz1Eol
+L87I63XN+bl1nYLYdPpwREqXDiIYEq/t+JmuTja3Bs90Qp6n7GWxPz5O6SHjyair
+sgwxJb1BAgMBAAGjXzBdMBsGA1UdEQQUMBKCCm5vZGVfdGhyZWWHBMCogkkwHQYD
+VR0OBBYEFBsOHOXvrx6XnQpEIXw3XU5XsuDHMB8GA1UdIwQYMBaAFHUPLtXvxPtd
+bdmswWW8wimEghdwMA0GCSqGSIb3DQEBCwUAA4IBAQBEgeew/row9fTrpQjlDGTn
+4vI21j2qjIQx9EVbSllogiWRuVLDoMvWi3cOn77jnNEDDqjxK0ocuZRvulyx/65Z
+OMsXi2sHDsbM41RLAy/Jw1pbxzK2TkDkEja5kKb1YPrvZd+8h0EUJ8jtgzTEfBtH
+bt32vLkg4CsznRIcdW2oeolJwS4MH7XVRF7X1EBvSO8pbRxHSCQHzkcfN+8b1CVW
+xJTIcI4bbcaHTl/m4osEMGjOiQwgg6yloFQ7wqF1xP97+ArVL9FXIyVfTWFedY48
+AMSCDgESHgweU4YeH9Qo5KJZU9iUOVKSmArK8xD6QSWZjfANOcP8Fwcyh9o3n/tl
+-----END CERTIFICATE-----

mtls/peerCA2_3_key.pem

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDMsMLbfg9Mtgvr
+K9LvlingAgU5c1XfGMkQigbShtVyiknwx7HjAX6hQLRvcIG/S9LOG48YHbI9oEmP
+8goZW/umZPHnwPJpNmDaPPZMkoTNwVXrCq/fbjoiY038zGCLYzBzwBMFzu51sraF
+GZnAW3az5m7l8d6xXOrhbxISGkhCVJaVEBu0eGt7bAxo8OwJmt6t8pzVWFwNq9lS
+dMGqwudmzY4xMs6W2ie8BODOmXIAya5X0yGDwFA0nUIeTMjQK3PqezOTaARz1Eol
+L87I63XN+bl1nYLYdPpwREqXDiIYEq/t+JmuTja3Bs90Qp6n7GWxPz5O6SHjyair
+sgwxJb1BAgMBAAECggEAHjrOvwQk4KM+1bUBBTbr/p7IbxMnBU36R9yH+b3wYRGm
+WOPiftBLh3zFs8hDCTb6rkt9/HBKUMqkLUOa6o2IoqrkL7dxtZ3Se6NRjBPUKYpa
+P4kHr1uZRKyI1IMfO7Fi8v1UxzX9Rq7662Omrt9zLkojv7GoaLunHrMSJqVGCFwv
+yB7Rd/UYUyoJgZj7k6hev4rvqNgK4zQ2mkdRg5uyU0EHPZd6c3tFSt3vU1K3LhFW
+Ij3gbm2ZCW4vimAOA4ryAQLqcykcKwP1VzMl1iw+JCiUH5lIlBqJbL7C2ifYVJeD
+FaWUSzEu452sZmqIL9JztLNDEQ5cRMNE8bBEMnbTBwKBgQDp/nL/HFxvRmBf2r8D
+0DevasfnsBFfcYZEpBeel29RQ/kM8XRv2N+GBKJUPQ+FLDdZt1TNZ+2TGwiU/OMg
+1jiELVYk0uV+CgY1qdYVdrfScXy2l5MgMTNI+Vgb/tBYOefgXkgUy7FRo26FfnAB
+rnfT4Peaf7nztu4BkqZTNZKy1wKBgQDf8NAFKVZWphJ+VfoV12TAhU/j5F1ta1Kk
+f4PcMD5RheVjezojpW/W9bvLF7dBSOsAYiAWw6l01bbgUCM5Yld5XYp2JTXROyq1
+uPFadLz7Xr5y0+HMYjhvZkfsf+p1uZU9IOAJgSWuNmISTKP2YKPOvqbxiA8nmor/
+mC7NUPYlpwKBgEs4rSctqtlquliWLYnASeZuYU0t4KfaCtvOFHm7HOH3A/0RMeky
+wsiGfNivuGmLSU/iC7TskAcWTa/3i9xmgUycdnDoCzZ1aoGd02CbBYT1Tq40E+vp
+ZR80aQq246s5Ej2ikXF9+cbQzxrDBqbDu2m1jGNyZAjg6ao/xpBBTKOJAoGAUIdZ
+VrHRwWEA/3tNbjNBoNWPsAS78BAK7OqJ8VFL56b9oTN76buhJyzAtjy2An0FPOMM
+ZQNBWalMlQdcU1Ng31za4Ldqze234xc5e2zFVKNyWvqElSmw36ZpMMui9WiiGRRM
+XD6b3TTjOW9j+sIvxbXbdLmXsYcqetUv51c43LsCgYBtfAZ55PfOLON0vKeSpfEo
+MwtsWK+7uCe2ybVV0ddZ3maimlR1sgAoMooDGG+9IsQMw7EYGIGG6PHjQQMb542R
+Cq3rWoVsQo9VSfHI1iO+neDsK84+SyJWeytHNqBO0EHdzQl/7u9gudv4Fq1k0HF7
+E+TBh+K0H35VaB2rxKyD5w==
+-----END PRIVATE KEY-----

mtls/peerCA2_4_cert.pem

@@ -0,0 +1,24 @@
+-----BEGIN CERTIFICATE-----
+MIIEGzCCAwOgAwIBAgIUKjQCltbN0Oao9ngSasdbHaIBmuEwDQYJKoZIhvcNAQEL
+BQAwgbcxCzAJBgNVBAYTAlVTMRIwEAYDVQQIDAlMb2NhbHpvbmUxEjAQBgNVBAcM
+CWxvY2FsaG9zdDEtMCsGA1UECgwkTmVvIEdvIFRlc3RpbmcgQ2VydGlmaWNhdGUg
+QXV0aG9yaXR5MRAwDgYDVQQLDAdEZXZlbG9wMRQwEgYDVQQDDAtsb2NhbGRvbWFp
+bjEpMCcGCSqGSIb3DQEJARYacm9vdEBsb2NhbGhvc3QubG9jYWxkb21haW4wHhcN
+MjQwNjExMDgyNzQ4WhcNMjUwNzE0MDgyNzQ4WjB3MQswCQYDVQQGEwJVUzESMBAG
+A1UECAwJTG9jYWx6b25lMRIwEAYDVQQHDAlMb2NhbGhvc3QxJDAiBgNVBAoMG0Nl
+cnRpZmljYXRlIHNpZ25lZCBieSBteSBDQTEaMBgGA1UEAwwRcGVlcjQubG9jYWxk
+b21haW4wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCvSZ8q0s+y+Uzc
+XPafneBvlXFoosKBva3cA3OigqOnfG7lxajte6oqFZoUSIApHKriinwbKkKLyxox
+KW2ajJKFrPF5QvcJ+AqoHoMoaLU7iTcUnphxzbgSK1wR6mo2O7ocSH9IqntCTbJO
+bnLZqDOlqHWwaJppaS+9ELb06VT7Cr6oiRQCP2o5g5dh0yjTYn8CFe6+ZOOQrHSw
+EjaZqzHIAJcvgwqGpmFJzbVaFlmrNon5sNZZfSiUabzk+GtlEKzejNTqzjq+Y4sl
+B8Mcm0tQzpeX67+bRzrcnG1oBaqJ72QqzTSe/pZH15OWBhTBRsWAExtAbiqACYdW
+z4Zf3MprAgMBAAGjXjBcMBoGA1UdEQQTMBGCCW5vZGVfZm91cocEwKiCSjAdBgNV
+HQ4EFgQUXmuqD1Cj9H4JcSF3rnXVArC5R8gwHwYDVR0jBBgwFoAUdQ8u1e/E+11t
+2azBZbzCKYSCF3AwDQYJKoZIhvcNAQELBQADggEBAEWCIYIdiZDjbKzkjqo5aL7s
+zGfBlzv53xuKalJT3Om5dOkI9z/W+FPwwFJyBvhYJ/FAG1IMA8bV7NLcbnE10QSj
+AgQoucu/JJWuW4bv2HSYRFTwsGEfznz2XOdJs48OwfVZGP7cNztdLdHJUK82o628
+caL9VBEbzpvSn0TJazNCNU3Q3e7PxhbQMZU+65MgwDOES1v04xCZ+vdV0Te8CGtv
+bEVfmPk4XCbNRb5VDpEwuYesJ1SH8xuq4KyYdJlizwDR0K+6WaAmfIAoQd7LpRA4
+xKdKNexUw0PWtNKJrUCIXLaMyxuwzxPfXnHoJhx6+6v+KAUa1qVvQKurQS6ri4k=
+-----END CERTIFICATE-----

mtls/peerCA2_4_key.pem

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCvSZ8q0s+y+Uzc
+XPafneBvlXFoosKBva3cA3OigqOnfG7lxajte6oqFZoUSIApHKriinwbKkKLyxox
+KW2ajJKFrPF5QvcJ+AqoHoMoaLU7iTcUnphxzbgSK1wR6mo2O7ocSH9IqntCTbJO
+bnLZqDOlqHWwaJppaS+9ELb06VT7Cr6oiRQCP2o5g5dh0yjTYn8CFe6+ZOOQrHSw
+EjaZqzHIAJcvgwqGpmFJzbVaFlmrNon5sNZZfSiUabzk+GtlEKzejNTqzjq+Y4sl
+B8Mcm0tQzpeX67+bRzrcnG1oBaqJ72QqzTSe/pZH15OWBhTBRsWAExtAbiqACYdW
+z4Zf3MprAgMBAAECggEAAMPTe/5bha5s1XFaaivoNNR0ID7FlodbYOBq+sSTLkHu
+wfXw3MhzX6eken4ugP9tB3wPOpcenzmNrbU+kzQV15t/mkXeih6Y/U7Ux7JdSBs7
+ME3Y4HZDXWDvD+zz9bShBUqTs5961jk8k/cLbUO8pfZsPQ7Xzan12V9fTQZPpqDN
+hfE3xhgoWvryhY4Yu9Zjkudjg+rpEgErB0sJPBK5YqqTgg3y6rqSQW8Ol0hEB7NJ
+S0emJ0q9K8UAq5+RKZanmnaIX73pr+TUbi4btVuery5kHymq+ippx9D1fOkBQKzV
+Y7zJrRi2+w1tBySYA7tHg4GaDe5vNwlB2tA4nHusgQKBgQDhm8MTVzCy5SBavN67
+0NcIKFutFGGi3RkVYBBFdSDfDnxeDW8dcmLd65W1aTZkhweoKXqxN8TabQUkJxcJ
+IHdg9XY9RXh1epv0wQ6fIb7kPM5xIJQ+dlLoHVMV1dWTVWkILolWnqcJepcl4iZJ
+bFixwPCdYO0X8f1JblskrYdzSwKBgQDG5oQE3T6mMv9uUg068W2OwZfeCINDdn7B
+XHs0LYc6NRNIrjNHxfPc37qptQf1K/S0jSErc0VRR3Ya3eRCzNGEMl1zmIv87y1W
+1EY4Jm1fM0MDD7KuDG/lVsrcKeogcVCYV2gyPqZAB/b1P9ljNVaVuo5MKkFjlpOA
+FSRWUzJxYQKBgQDBRR2toODwMEcyVGB/Bx5fN6upry8hRUVqwVvPFNMvIwq5y8pn
+sIkv0dTzjwlwvxh5H+eij8MM154y5WXfhLegEUfUn4HdKkcWOpjbDB6rCchUs3Mr
+u1O00wJNT11ilSWgQ7WVxd4UybdYVc800SN+S2oVeGZEQCMa1/YuipXh1wKBgF2g
+sVCg3RbbNsY1taijcVEb7GNGWpPtOanH4O8k510NEvDJn7YPhsXYQ0QTxZUxhdVg
+Adc0QdMK8pufDA4t4Ap8suTxUCRWal9POWbBIAcKWGcegpvPmvS4MHIHAQEYljnA
+G3pbZ2MI3MWBstEykHmXAdj0oydAAFrwmNRSAZLBAoGAFJtkyR8GSJ8merCLAuqC
+CN1an5OcuGcYXoyt1FE6VCI/YVQ03c0NCv0FciRAqqybrKaxvJ/qTYD6//Lk2um5
+aSs+PQ/wiGkH0mLI8zAkoqPEBqu+RqP3c9d80mLOwOHT7oPhkB7bSJViEwO5XZKN
+ylGmE6Sq+AZ9jCztJ6Tina8=
+-----END PRIVATE KEY-----

mtls/peerCA2_IR_cert.pem

@@ -0,0 +1,24 @@
+-----BEGIN CERTIFICATE-----
+MIIEGjCCAwKgAwIBAgIUKjQCltbN0Oao9ngSasdbHaIBmuIwDQYJKoZIhvcNAQEL
+BQAwgbcxCzAJBgNVBAYTAlVTMRIwEAYDVQQIDAlMb2NhbHpvbmUxEjAQBgNVBAcM
+CWxvY2FsaG9zdDEtMCsGA1UECgwkTmVvIEdvIFRlc3RpbmcgQ2VydGlmaWNhdGUg
+QXV0aG9yaXR5MRAwDgYDVQQLDAdEZXZlbG9wMRQwEgYDVQQDDAtsb2NhbGRvbWFp
+bjEpMCcGCSqGSIb3DQEJARYacm9vdEBsb2NhbGhvc3QubG9jYWxkb21haW4wHhcN
+MjQwNjExMDgyNzQ4WhcNMjUwNzE0MDgyNzQ4WjB3MQswCQYDVQQGEwJVUzESMBAG
+A1UECAwJTG9jYWx6b25lMRIwEAYDVQQHDAlMb2NhbGhvc3QxJDAiBgNVBAoMG0Nl
+cnRpZmljYXRlIHNpZ25lZCBieSBteSBDQTEaMBgGA1UEAwwRcGVlcjEubG9jYWxk
+b21haW4wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDwLU9zl6iHlDLQ
+LvPKMngsxNKqK+hwoaXuiAXNIGN/MAjWE7RL2bo5Ah0x/359Pi2MWNBBmdVnuDK/
+5UqK+HXli3qZhUWjMwWDYJM9jwhyx00spKfa6+9mVgMesdTix/xRDAFK27UNlJya
+q4kUqinvGJaPKgbszUpEyMqOeVKU157tOzPVMWQBYfHJDbHHerd7jHPKlDvXSMkR
+TtPO+OojFhXrXlo0Ljif8A6AagdMks1ozvaFCs4fBTIiDJWqLxttilf6GxkEfFqt
+Oor3hraobe1OBKwHRI2r9hu7BVg8gjZ+Hcdw7tJ2HtuHfd9d1k+XW31oe6HMVdgZ
+QK0hRHofAgMBAAGjXTBbMBkGA1UdEQQSMBCCCG5vZGVfb25lhwTAqII9MB0GA1Ud
+DgQWBBRk8ONdkn0MuaBRHeUuM4iU8fc4pTAfBgNVHSMEGDAWgBR1Dy7V78T7XW3Z
+rMFlvMIphIIXcDANBgkqhkiG9w0BAQsFAAOCAQEAZdNGSxGjZSfaZPPujYNC6ZrY
+c07mRV3W+NGJZoWvDaEYy3MYZje+wghBJG9nXD4eQ/55q/k9C3AvcM31H2eBBFCW
+f6Jcp414NZ52lTCoeDmmRTsggJS7IPE6kaBe3GoUJR+bs/ktv1im+ep/8/eEVf3g
+FlteNDQ4kL24m8Ps1CnDLLBM3OQphD3MuQrMpZPSaSKhZr9RJrZ6jcuAcH6uhYZi
+c0mevVLDahoGwrZpMnVYHgVOAmygbQv4wONAvrhNgDrKmtMczmIb2q0BIG4Jdtsm
+Co1EiJ3/eHL+Vu6MhLWngLBg3Yl0bUZIw/xPu+jD8GmH+qrGcvq3hqSTFHIX5w==
+-----END CERTIFICATE-----

mtls/peerCA2_IR_key.pem

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDwLU9zl6iHlDLQ
+LvPKMngsxNKqK+hwoaXuiAXNIGN/MAjWE7RL2bo5Ah0x/359Pi2MWNBBmdVnuDK/
+5UqK+HXli3qZhUWjMwWDYJM9jwhyx00spKfa6+9mVgMesdTix/xRDAFK27UNlJya
+q4kUqinvGJaPKgbszUpEyMqOeVKU157tOzPVMWQBYfHJDbHHerd7jHPKlDvXSMkR
+TtPO+OojFhXrXlo0Ljif8A6AagdMks1ozvaFCs4fBTIiDJWqLxttilf6GxkEfFqt
+Oor3hraobe1OBKwHRI2r9hu7BVg8gjZ+Hcdw7tJ2HtuHfd9d1k+XW31oe6HMVdgZ
+QK0hRHofAgMBAAECggEAPQ731OYoVRdq95wIJE5CWPdMqzBwbjnaKlLwTp6+TLZU
+eMAXpPTcL+QGshBHAuYMxFJL13GZXD3qwPg0xDG4HzwfVeoYsw2kiCrEI/E575wV
+VZUiizhR601Qi7wf5+t5jM0lgvdqBuLSEUwslFFSSdORayH/ErJ6ABSf0iqrwOS2
+DmEAi02ZqumLrd41lY72QrROCP/B2GcedD8pq8CKHxgwUk6Gb+jLlyr7Kg2Ubiro
+/elKOVdTBClInKLBxlWPXsGbnwdw1WNseh9FoQP71hgJI+HWK4gAZkl7umvLjdFo
+I94vSgELmj3hpRT88ysB3dyUwG84bFlmFQAplgZ4mQKBgQD7lk27ysUTCMti1Lud
+4tes7Khvx02itGwjI3/UwGcizjC1sGprvfun5btXSfeOhg8kkj+FQ7n+yBY9ISyi
+8jUdmPR/DamviYcRUs3xy9XzNiR7JJ1gM/NdgU7FG+RYNMC8lHkxgePQDyYUHSAy
+28DFDcG+z7WzYVPqZb8a3mw42wKBgQD0Y8Wkt9UzI6dQEtKCOA/IoZDsrwgMev9U
+ZTulJBPY4mO7rS4nfaF2vs6ENwjzjCblr+mWjV13Ir+eylaD5zwc1SOKJtR1WEpN
+7HWLcfI8qVerqN+uknU4fXlZmpp+StNYvCQ44QpHJE9dHpMs0htdMl2EztQVupY2
+MC0B6tT1DQKBgFsWoC5Ny+yIUpsFyqfvaYcCaDmQP1uZV02hnLa6spy3aotdxCoe
+Lu8rDhkcfrTrdLAZA0aMrtrANs0LJc4ZQ4HjzyHxIG6drHlpMYdJ9byI7cxoBVK/
+fG1uU8apwpLtBptAZmC2VnUOBwthQDcpuTGfOXaMXY0EwA0tqXNg9G3hAoGBAIko
+Was5VRlPYD5rYeOdbRZPvtNm5GCEwzntWs0y80ScwhZ5elbFhlHrgmHntUlilg4A
+bVuGWTdctCh9LJL4ut7/q+OEKWb2NzWGiO3K9IWhEMgRjgAeyFT87pcgUqagff7m
+EHovqqIEudnsJ/NWs+7ZLm0z2wma32ToIspZrk0dAoGBAPhIcZgmTpJmZ4lmYwF/
+xZSSMW2L0P0VYiXD27l17fPL7no9VgBihhki+FxLq5UPvg5M3Nx7EUA20R3RY8tU
+hZdwIiLSB4WMzbob7+esPqgs4nTJTD6rFpagt4Bw2/WpJRaogi3SvoQ0lAHRaq+z
+yGlQNGg2J3DN7T6u3Ng8VNfr
+-----END PRIVATE KEY-----

mtls/peerIR.cnf

@@ -0,0 +1,23 @@
+[req]
+default_bits  = 2048
+distinguished_name = req_distinguished_name
+req_extensions = req_ext
+x509_extensions = v3_req
+prompt = no
+
+[req_distinguished_name]
+countryName = US
+stateOrProvinceName = Localzone
+localityName = Localhost
+organizationName = Certificate signed by my CA
+commonName = peer1.localdomain
+
+[req_ext]
+subjectAltName = @alt_names
+
+[v3_req]
+subjectAltName = @alt_names
+
+[alt_names]
+DNS.1 = node_one
+IP.1 = 192.168.130.61

mtls/peerX.cnf

@@ -0,0 +1,24 @@
+[req]
+default_bits  = 2048
+distinguished_name = req_distinguished_name
+req_extensions = req_ext
+x509_extensions = v3_req
+prompt = no
+
+[req_distinguished_name]
+countryName = US
+stateOrProvinceName = Localzone
+localityName = Localhost
+organizationName = Certificate signed by my CA
+commonName = peer1.localdomain
+
+[req_ext]
+subjectAltName = @alt_names
+
+[v3_req]
+subjectAltName = @alt_names
+
+[alt_names]
+DNS.1 = morph-chain.frostfs.devenv
+DNS.2 = morph-chain
+IP.1 = 192.168.130.90

services/basenet/docker-compose.yml

@@ -1,6 +1,5 @@
 ---
 
-version: "2.4"
 services:
 
   basenet:

services/grafana/.hosts

@@ -1 +1,2 @@
 IPV4_PREFIX.122 grafana.LOCAL_DOMAIN
+IPV4_PREFIX.123 loki.LOCAL_DOMAIN

services/grafana/docker-compose.yml

@@ -1,4 +1,3 @@
-version: '2.4'
 services:
   grafana:
     image: ${GRAFANA_IMAGE}:${GRAFANA_VERSION}
@@ -14,11 +13,17 @@ services:
       - ./../../vendor/hosts:/etc/hosts
       - ./grafana.ini:/etc/grafana/grafana.ini
       - ./provisioning:/etc/grafana/provisioning
-    ports:
-      - '3000:3000'
     stop_signal: SIGKILL
     env_file: [ ".env", ".int_test.env" ]
 
+  loki:
+    image: ${LOKI_IMAGE}:${LOKI_VERSION}
+    command: -config.file=/etc/loki/local-config.yaml
+    networks:
+      grafana_int:
+      internet:
+        ipv4_address: ${IPV4_PREFIX}.123
+
 networks:
   grafana_int:
   internet:

services/grafana/provisioning/dashboards/blobovnicza-buckets.json

@@ -0,0 +1,361 @@
+{
+    "annotations": {
+      "list": [
+        {
+          "builtIn": 1,
+          "datasource": {
+            "type": "grafana",
+            "uid": "-- Grafana --"
+          },
+          "enable": true,
+          "hide": true,
+          "iconColor": "rgba(0, 211, 255, 1)",
+          "name": "Annotations & Alerts",
+          "type": "dashboard"
+        }
+      ]
+    },
+    "editable": true,
+    "fiscalYearStartMonth": 0,
+    "graphTooltip": 0,
+    "id": 7,
+    "links": [],
+    "liveNow": false,
+    "panels": [
+      {
+        "collapsed": false,
+        "gridPos": {
+          "h": 1,
+          "w": 24,
+          "x": 0,
+          "y": 0
+        },
+        "id": 1,
+        "panels": [],
+        "title": "$method",
+        "type": "row"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "PBFA97CFB590B2093"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 50,
+              "gradientMode": "none",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "lineInterpolation": "smooth",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "never",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "normal"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            },
+            "unit": "reqps"
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 20,
+          "w": 24,
+          "x": 0,
+          "y": 1
+        },
+        "id": 7,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": true
+          },
+          "tooltip": {
+            "mode": "multi",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)",
+            "hide": false,
+            "legendFormat": "< 0.005s",
+            "range": true,
+            "refId": "A"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)",
+            "hide": false,
+            "legendFormat": "0.005s - 0.01s",
+            "range": true,
+            "refId": "B"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)",
+            "hide": false,
+            "legendFormat": "0.01s - 0.025s",
+            "range": true,
+            "refId": "C"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)",
+            "hide": false,
+            "legendFormat": "0.025s - 0.05s",
+            "range": true,
+            "refId": "D"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)",
+            "hide": false,
+            "legendFormat": "0.05s - 0.1s",
+            "range": true,
+            "refId": "E"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)",
+            "hide": false,
+            "legendFormat": "0.1s - 0.25s",
+            "range": true,
+            "refId": "F"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)",
+            "hide": false,
+            "legendFormat": "0.25s - 0.5s",
+            "range": true,
+            "refId": "G"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)",
+            "hide": false,
+            "legendFormat": "0.5s - 1s",
+            "range": true,
+            "refId": "H"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)",
+            "hide": false,
+            "legendFormat": "1s - 2.5s",
+            "range": true,
+            "refId": "I"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)",
+            "hide": false,
+            "legendFormat": "2.5s - 5s",
+            "range": true,
+            "refId": "J"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)",
+            "hide": false,
+            "legendFormat": "5s - 10s",
+            "range": true,
+            "refId": "K"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"+Inf\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)",
+            "hide": false,
+            "legendFormat": "> 10s",
+            "range": true,
+            "refId": "L"
+          }
+        ],
+        "title": "$method",
+        "type": "timeseries"
+      }
+    ],
+    "refresh": "",
+    "schemaVersion": 38,
+    "style": "dark",
+    "tags": [],
+    "templating": {
+      "list": [
+        {
+          "current": {
+            "selected": false,
+            "text": "s01.frostfs.devenv:9090",
+            "value": "s01.frostfs.devenv:9090"
+          },
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "definition": "label_values(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket,instance)",
+          "hide": 0,
+          "includeAll": false,
+          "multi": false,
+          "name": "instance",
+          "options": [],
+          "query": {
+            "query": "label_values(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket,instance)",
+            "refId": "PrometheusVariableQueryEditor-VariableQuery"
+          },
+          "refresh": 1,
+          "regex": "",
+          "skipUrlSync": false,
+          "sort": 1,
+          "type": "query"
+        },
+        {
+          "current": {
+            "selected": false,
+            "text": "3Xd2VDqghZnMnaRoP4bKeP",
+            "value": "3Xd2VDqghZnMnaRoP4bKeP"
+          },
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "definition": "label_values(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{instance=\"$instance\"},shard_id)",
+          "hide": 0,
+          "includeAll": false,
+          "multi": false,
+          "name": "shard_id",
+          "options": [],
+          "query": {
+            "query": "label_values(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{instance=\"$instance\"},shard_id)",
+            "refId": "PrometheusVariableQueryEditor-VariableQuery"
+          },
+          "refresh": 1,
+          "regex": "",
+          "skipUrlSync": false,
+          "sort": 1,
+          "type": "query"
+        },
+        {
+          "current": {
+            "selected": true,
+            "text": "Put",
+            "value": "Put"
+          },
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "definition": "label_values(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{instance=\"$instance\", shard_id=\"$shard_id\"},method)",
+          "hide": 0,
+          "includeAll": false,
+          "multi": false,
+          "name": "method",
+          "options": [],
+          "query": {
+            "query": "label_values(frostfs_node_blobovnicza_tree_request_duration_seconds_bucket{instance=\"$instance\", shard_id=\"$shard_id\"},method)",
+            "refId": "PrometheusVariableQueryEditor-VariableQuery"
+          },
+          "refresh": 1,
+          "regex": "",
+          "skipUrlSync": false,
+          "sort": 1,
+          "type": "query"
+        }
+      ]
+    },
+    "time": {
+      "from": "now-1h",
+      "to": "now"
+    },
+    "timepicker": {},
+    "timezone": "",
+    "title": "Storage node blobovnicza bucket distribution",
+    "uid": "e4097619-1ce9-4039-bd62-4f5b4bba8d26",
+    "version": 2,
+    "weekStart": ""
+  }

services/grafana/provisioning/dashboards/engine-buckets.json

@@ -0,0 +1,335 @@
+{
+    "annotations": {
+      "list": [
+        {
+          "builtIn": 1,
+          "datasource": {
+            "type": "grafana",
+            "uid": "-- Grafana --"
+          },
+          "enable": true,
+          "hide": true,
+          "iconColor": "rgba(0, 211, 255, 1)",
+          "name": "Annotations & Alerts",
+          "type": "dashboard"
+        }
+      ]
+    },
+    "editable": true,
+    "fiscalYearStartMonth": 0,
+    "graphTooltip": 0,
+    "id": 5,
+    "links": [],
+    "liveNow": false,
+    "panels": [
+      {
+        "collapsed": false,
+        "gridPos": {
+          "h": 1,
+          "w": 24,
+          "x": 0,
+          "y": 0
+        },
+        "id": 1,
+        "panels": [],
+        "title": "$method",
+        "type": "row"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "PBFA97CFB590B2093"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 50,
+              "gradientMode": "none",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "lineInterpolation": "smooth",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "never",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "normal"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            },
+            "unit": "reqps"
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 20,
+          "w": 24,
+          "x": 0,
+          "y": 1
+        },
+        "id": 7,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": true
+          },
+          "tooltip": {
+            "mode": "multi",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "rate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])",
+            "hide": false,
+            "legendFormat": "< 0.005s",
+            "range": true,
+            "refId": "A"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n",
+            "hide": false,
+            "legendFormat": "0.005s - 0.01s",
+            "range": true,
+            "refId": "B"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n",
+            "hide": false,
+            "legendFormat": "0.01s - 0.025s",
+            "range": true,
+            "refId": "C"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n",
+            "hide": false,
+            "legendFormat": "0.025s - 0.05s",
+            "range": true,
+            "refId": "D"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n",
+            "hide": false,
+            "legendFormat": "0.05s - 0.1s",
+            "range": true,
+            "refId": "E"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n",
+            "hide": false,
+            "legendFormat": "0.1s - 0.25s",
+            "range": true,
+            "refId": "F"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n",
+            "hide": false,
+            "legendFormat": "0.25s - 0.5s",
+            "range": true,
+            "refId": "G"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n",
+            "hide": false,
+            "legendFormat": "0.5s - 1s",
+            "range": true,
+            "refId": "H"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n",
+            "hide": false,
+            "legendFormat": "1s - 2.5s",
+            "range": true,
+            "refId": "I"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n",
+            "hide": false,
+            "legendFormat": "2.5s - 5s",
+            "range": true,
+            "refId": "J"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n",
+            "hide": false,
+            "legendFormat": "5s - 10s",
+            "range": true,
+            "refId": "K"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"+Inf\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_engine_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", method=\"$method\"}[$__rate_interval])\n",
+            "hide": false,
+            "legendFormat": "> 10s",
+            "range": true,
+            "refId": "L"
+          }
+        ],
+        "title": "$method",
+        "type": "timeseries"
+      }
+    ],
+    "refresh": "",
+    "schemaVersion": 38,
+    "style": "dark",
+    "tags": [],
+    "templating": {
+      "list": [
+        {
+          "current": {
+            "selected": false,
+            "text": "s01.frostfs.devenv:9090",
+            "value": "s01.frostfs.devenv:9090"
+          },
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "definition": "label_values(frostfs_node_engine_request_duration_seconds_bucket,instance)",
+          "hide": 0,
+          "includeAll": false,
+          "multi": false,
+          "name": "instance",
+          "options": [],
+          "query": {
+            "query": "label_values(frostfs_node_engine_request_duration_seconds_bucket,instance)",
+            "refId": "PrometheusVariableQueryEditor-VariableQuery"
+          },
+          "refresh": 1,
+          "regex": "",
+          "skipUrlSync": false,
+          "sort": 1,
+          "type": "query"
+        },
+        {
+          "current": {
+            "selected": true,
+            "text": "Put",
+            "value": "Put"
+          },
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "definition": "label_values(frostfs_node_engine_request_duration_seconds_bucket{instance=\"$instance\"},method)",
+          "hide": 0,
+          "includeAll": false,
+          "multi": false,
+          "name": "method",
+          "options": [],
+          "query": {
+            "query": "label_values(frostfs_node_engine_request_duration_seconds_bucket{instance=\"$instance\"},method)",
+            "refId": "PrometheusVariableQueryEditor-VariableQuery"
+          },
+          "refresh": 1,
+          "regex": "",
+          "skipUrlSync": false,
+          "sort": 1,
+          "type": "query"
+        }
+      ]
+    },
+    "time": {
+      "from": "now-1h",
+      "to": "now"
+    },
+    "timepicker": {},
+    "timezone": "",
+    "title": "Storage node engine bucket distribution",
+    "uid": "f886d27c-b877-495a-a1cb-d02839115cea",
+    "version": 2,
+    "weekStart": ""
+  }

services/grafana/provisioning/dashboards/fstree-buckets.json

@@ -0,0 +1,361 @@
+{
+    "annotations": {
+      "list": [
+        {
+          "builtIn": 1,
+          "datasource": {
+            "type": "grafana",
+            "uid": "-- Grafana --"
+          },
+          "enable": true,
+          "hide": true,
+          "iconColor": "rgba(0, 211, 255, 1)",
+          "name": "Annotations & Alerts",
+          "type": "dashboard"
+        }
+      ]
+    },
+    "editable": true,
+    "fiscalYearStartMonth": 0,
+    "graphTooltip": 0,
+    "id": 9,
+    "links": [],
+    "liveNow": false,
+    "panels": [
+      {
+        "collapsed": false,
+        "gridPos": {
+          "h": 1,
+          "w": 24,
+          "x": 0,
+          "y": 0
+        },
+        "id": 1,
+        "panels": [],
+        "title": "$method",
+        "type": "row"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "PBFA97CFB590B2093"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 50,
+              "gradientMode": "none",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "lineInterpolation": "smooth",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "never",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "normal"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            },
+            "unit": "reqps"
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 20,
+          "w": 24,
+          "x": 0,
+          "y": 1
+        },
+        "id": 7,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": true
+          },
+          "tooltip": {
+            "mode": "multi",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)",
+            "hide": false,
+            "legendFormat": "< 0.005s",
+            "range": true,
+            "refId": "A"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)",
+            "hide": false,
+            "legendFormat": "0.005s - 0.01s",
+            "range": true,
+            "refId": "B"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)",
+            "hide": false,
+            "legendFormat": "0.01s - 0.025s",
+            "range": true,
+            "refId": "C"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)",
+            "hide": false,
+            "legendFormat": "0.025s - 0.05s",
+            "range": true,
+            "refId": "D"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)",
+            "hide": false,
+            "legendFormat": "0.05s - 0.1s",
+            "range": true,
+            "refId": "E"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)",
+            "hide": false,
+            "legendFormat": "0.1s - 0.25s",
+            "range": true,
+            "refId": "F"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)",
+            "hide": false,
+            "legendFormat": "0.25s - 0.5s",
+            "range": true,
+            "refId": "G"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)",
+            "hide": false,
+            "legendFormat": "0.5s - 1s",
+            "range": true,
+            "refId": "H"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)",
+            "hide": false,
+            "legendFormat": "1s - 2.5s",
+            "range": true,
+            "refId": "I"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)",
+            "hide": false,
+            "legendFormat": "2.5s - 5s",
+            "range": true,
+            "refId": "J"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)",
+            "hide": false,
+            "legendFormat": "5s - 10s",
+            "range": true,
+            "refId": "K"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"+Inf\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_fstree_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)",
+            "hide": false,
+            "legendFormat": "> 10s",
+            "range": true,
+            "refId": "L"
+          }
+        ],
+        "title": "$method",
+        "type": "timeseries"
+      }
+    ],
+    "refresh": "",
+    "schemaVersion": 38,
+    "style": "dark",
+    "tags": [],
+    "templating": {
+      "list": [
+        {
+          "current": {
+            "selected": true,
+            "text": "s01.frostfs.devenv:9090",
+            "value": "s01.frostfs.devenv:9090"
+          },
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "definition": "label_values(frostfs_node_fstree_request_duration_seconds_bucket,instance)",
+          "hide": 0,
+          "includeAll": false,
+          "multi": false,
+          "name": "instance",
+          "options": [],
+          "query": {
+            "query": "label_values(frostfs_node_fstree_request_duration_seconds_bucket,instance)",
+            "refId": "PrometheusVariableQueryEditor-VariableQuery"
+          },
+          "refresh": 1,
+          "regex": "",
+          "skipUrlSync": false,
+          "sort": 1,
+          "type": "query"
+        },
+        {
+          "current": {
+            "selected": false,
+            "text": "3Xd2VDqghZnMnaRoP4bKeP",
+            "value": "3Xd2VDqghZnMnaRoP4bKeP"
+          },
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "definition": "label_values(frostfs_node_fstree_request_duration_seconds_bucket{instance=\"$instance\"},shard_id)",
+          "hide": 0,
+          "includeAll": false,
+          "multi": false,
+          "name": "shard_id",
+          "options": [],
+          "query": {
+            "query": "label_values(frostfs_node_fstree_request_duration_seconds_bucket{instance=\"$instance\"},shard_id)",
+            "refId": "PrometheusVariableQueryEditor-VariableQuery"
+          },
+          "refresh": 1,
+          "regex": "",
+          "skipUrlSync": false,
+          "sort": 1,
+          "type": "query"
+        },
+        {
+          "current": {
+            "selected": false,
+            "text": "Get",
+            "value": "Get"
+          },
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "definition": "label_values(frostfs_node_fstree_request_duration_seconds_bucket{instance=\"$instance\", shard_id=\"$shard_id\"},method)",
+          "hide": 0,
+          "includeAll": false,
+          "multi": false,
+          "name": "method",
+          "options": [],
+          "query": {
+            "query": "label_values(frostfs_node_fstree_request_duration_seconds_bucket{instance=\"$instance\", shard_id=\"$shard_id\"},method)",
+            "refId": "PrometheusVariableQueryEditor-VariableQuery"
+          },
+          "refresh": 1,
+          "regex": "",
+          "skipUrlSync": false,
+          "sort": 1,
+          "type": "query"
+        }
+      ]
+    },
+    "time": {
+      "from": "now-1h",
+      "to": "now"
+    },
+    "timepicker": {},
+    "timezone": "",
+    "title": "Storage node fstree bucket distribution",
+    "uid": "e6407e43-9419-445b-b715-284d69201519",
+    "version": 2,
+    "weekStart": ""
+  }

services/grafana/provisioning/dashboards/grpc-buckets.json

@@ -0,0 +1,362 @@
+{
+    "annotations": {
+      "list": [
+        {
+          "builtIn": 1,
+          "datasource": {
+            "type": "grafana",
+            "uid": "-- Grafana --"
+          },
+          "enable": true,
+          "hide": true,
+          "iconColor": "rgba(0, 211, 255, 1)",
+          "name": "Annotations & Alerts",
+          "type": "dashboard"
+        }
+      ]
+    },
+    "editable": true,
+    "fiscalYearStartMonth": 0,
+    "graphTooltip": 0,
+    "id": 4,
+    "links": [],
+    "liveNow": false,
+    "panels": [
+      {
+        "collapsed": false,
+        "gridPos": {
+          "h": 1,
+          "w": 24,
+          "x": 0,
+          "y": 0
+        },
+        "id": 1,
+        "panels": [],
+        "title": "$service",
+        "type": "row"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "PBFA97CFB590B2093"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 50,
+              "gradientMode": "none",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "lineInterpolation": "smooth",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "never",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "normal"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            },
+            "unit": "reqps"
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 20,
+          "w": 24,
+          "x": 0,
+          "y": 1
+        },
+        "id": 7,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": true
+          },
+          "tooltip": {
+            "mode": "multi",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "rate(grpc_server_handling_seconds_bucket{le=\"0.005\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])",
+            "hide": false,
+            "legendFormat": "< 0.005s",
+            "range": true,
+            "refId": "A"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"0.01\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"0.005\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n",
+            "hide": false,
+            "legendFormat": "0.005s - 0.01s",
+            "range": true,
+            "refId": "B"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"0.025\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"0.01\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n",
+            "hide": false,
+            "legendFormat": "0.01s - 0.025s",
+            "range": true,
+            "refId": "C"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"0.05\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"0.025\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n",
+            "hide": false,
+            "legendFormat": "0.025s - 0.05s",
+            "range": true,
+            "refId": "D"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"0.1\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"0.05\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n",
+            "hide": false,
+            "legendFormat": "0.05s - 0.1s",
+            "range": true,
+            "refId": "E"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"0.25\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"0.1\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n",
+            "hide": false,
+            "legendFormat": "0.1s - 0.25s",
+            "range": true,
+            "refId": "F"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"0.5\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"0.25\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n",
+            "hide": false,
+            "legendFormat": "0.25s - 0.5s",
+            "range": true,
+            "refId": "G"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"1\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"0.5\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n",
+            "hide": false,
+            "legendFormat": "0.5s - 1s",
+            "range": true,
+            "refId": "H"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"2.5\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"1\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n",
+            "hide": false,
+            "legendFormat": "1s - 2.5s",
+            "range": true,
+            "refId": "I"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"5\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"2.5\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n",
+            "hide": false,
+            "legendFormat": "2.5s - 5s",
+            "range": true,
+            "refId": "J"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"10\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"5\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n",
+            "hide": false,
+            "legendFormat": "5s - 10s",
+            "range": true,
+            "refId": "K"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "\nrate(grpc_server_handling_seconds_bucket{le=\"+Inf\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(grpc_server_handling_seconds_bucket{le=\"10\", instance=\"$instance\", grpc_service=\"$service\", grpc_method=\"$method\"}[$__rate_interval])\n",
+            "hide": false,
+            "legendFormat": "> 10s",
+            "range": true,
+            "refId": "L"
+          }
+        ],
+        "title": "$method",
+        "type": "timeseries"
+      }
+    ],
+    "refresh": "",
+    "schemaVersion": 38,
+    "style": "dark",
+    "tags": [],
+    "templating": {
+      "list": [
+        {
+          "current": {
+            "selected": false,
+            "text": "s01.frostfs.devenv:9090",
+            "value": "s01.frostfs.devenv:9090"
+          },
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "definition": "label_values(grpc_server_handling_seconds_bucket,instance)",
+          "hide": 0,
+          "includeAll": false,
+          "multi": false,
+          "name": "instance",
+          "options": [],
+          "query": {
+            "query": "label_values(grpc_server_handling_seconds_bucket,instance)",
+            "refId": "PrometheusVariableQueryEditor-VariableQuery"
+          },
+          "refresh": 1,
+          "regex": "",
+          "skipUrlSync": false,
+          "sort": 1,
+          "type": "query"
+        },
+        {
+          "allValue": "",
+          "current": {
+            "selected": false,
+            "text": "neo.fs.v2.object.ObjectService",
+            "value": "neo.fs.v2.object.ObjectService"
+          },
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "definition": "label_values(grpc_server_handling_seconds_bucket{instance=\"$instance\"},grpc_service)",
+          "hide": 0,
+          "includeAll": false,
+          "multi": false,
+          "name": "service",
+          "options": [],
+          "query": {
+            "query": "label_values(grpc_server_handling_seconds_bucket{instance=\"$instance\"},grpc_service)",
+            "refId": "PrometheusVariableQueryEditor-VariableQuery"
+          },
+          "refresh": 1,
+          "regex": "",
+          "skipUrlSync": false,
+          "sort": 1,
+          "type": "query"
+        },
+        {
+          "current": {
+            "selected": false,
+            "text": "Put",
+            "value": "Put"
+          },
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "definition": "label_values(grpc_server_handling_seconds_bucket{grpc_service=\"$service\", instance=\"$instance\"},grpc_method)",
+          "hide": 0,
+          "includeAll": false,
+          "multi": false,
+          "name": "method",
+          "options": [],
+          "query": {
+            "query": "label_values(grpc_server_handling_seconds_bucket{grpc_service=\"$service\", instance=\"$instance\"},grpc_method)",
+            "refId": "PrometheusVariableQueryEditor-VariableQuery"
+          },
+          "refresh": 1,
+          "regex": "",
+          "skipUrlSync": false,
+          "sort": 1,
+          "type": "query"
+        }
+      ]
+    },
+    "time": {
+      "from": "now-1h",
+      "to": "now"
+    },
+    "timepicker": {},
+    "timezone": "",
+    "title": "GRPC server bucket distribution",
+    "uid": "a6830bc6-cffa-4e29-9622-6a10921748b0",
+    "version": 8,
+    "weekStart": ""
+  }

services/grafana/provisioning/dashboards/metabase-buckets.json

@@ -0,0 +1,361 @@
+{
+    "annotations": {
+      "list": [
+        {
+          "builtIn": 1,
+          "datasource": {
+            "type": "grafana",
+            "uid": "-- Grafana --"
+          },
+          "enable": true,
+          "hide": true,
+          "iconColor": "rgba(0, 211, 255, 1)",
+          "name": "Annotations & Alerts",
+          "type": "dashboard"
+        }
+      ]
+    },
+    "editable": true,
+    "fiscalYearStartMonth": 0,
+    "graphTooltip": 0,
+    "id": 8,
+    "links": [],
+    "liveNow": false,
+    "panels": [
+      {
+        "collapsed": false,
+        "gridPos": {
+          "h": 1,
+          "w": 24,
+          "x": 0,
+          "y": 0
+        },
+        "id": 1,
+        "panels": [],
+        "title": "$method",
+        "type": "row"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "PBFA97CFB590B2093"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 50,
+              "gradientMode": "none",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "lineInterpolation": "smooth",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "never",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "normal"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            },
+            "unit": "reqps"
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 20,
+          "w": 24,
+          "x": 0,
+          "y": 1
+        },
+        "id": 7,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": true
+          },
+          "tooltip": {
+            "mode": "multi",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)",
+            "hide": false,
+            "legendFormat": "< 0.005s",
+            "range": true,
+            "refId": "A"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)",
+            "hide": false,
+            "legendFormat": "0.005s - 0.01s",
+            "range": true,
+            "refId": "B"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)",
+            "hide": false,
+            "legendFormat": "0.01s - 0.025s",
+            "range": true,
+            "refId": "C"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)",
+            "hide": false,
+            "legendFormat": "0.025s - 0.05s",
+            "range": true,
+            "refId": "D"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)",
+            "hide": false,
+            "legendFormat": "0.05s - 0.1s",
+            "range": true,
+            "refId": "E"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)",
+            "hide": false,
+            "legendFormat": "0.1s - 0.25s",
+            "range": true,
+            "refId": "F"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)",
+            "hide": false,
+            "legendFormat": "0.25s - 0.5s",
+            "range": true,
+            "refId": "G"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)",
+            "hide": false,
+            "legendFormat": "0.5s - 1s",
+            "range": true,
+            "refId": "H"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)",
+            "hide": false,
+            "legendFormat": "1s - 2.5s",
+            "range": true,
+            "refId": "I"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)",
+            "hide": false,
+            "legendFormat": "2.5s - 5s",
+            "range": true,
+            "refId": "J"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)",
+            "hide": false,
+            "legendFormat": "5s - 10s",
+            "range": true,
+            "refId": "K"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"+Inf\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_metabase_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)",
+            "hide": false,
+            "legendFormat": "> 10s",
+            "range": true,
+            "refId": "L"
+          }
+        ],
+        "title": "$method",
+        "type": "timeseries"
+      }
+    ],
+    "refresh": "",
+    "schemaVersion": 38,
+    "style": "dark",
+    "tags": [],
+    "templating": {
+      "list": [
+        {
+          "current": {
+            "selected": false,
+            "text": "s01.frostfs.devenv:9090",
+            "value": "s01.frostfs.devenv:9090"
+          },
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "definition": "label_values(frostfs_node_metabase_request_duration_seconds_bucket,instance)",
+          "hide": 0,
+          "includeAll": false,
+          "multi": false,
+          "name": "instance",
+          "options": [],
+          "query": {
+            "query": "label_values(frostfs_node_metabase_request_duration_seconds_bucket,instance)",
+            "refId": "PrometheusVariableQueryEditor-VariableQuery"
+          },
+          "refresh": 1,
+          "regex": "",
+          "skipUrlSync": false,
+          "sort": 1,
+          "type": "query"
+        },
+        {
+          "current": {
+            "selected": false,
+            "text": "3Xd2VDqghZnMnaRoP4bKeP",
+            "value": "3Xd2VDqghZnMnaRoP4bKeP"
+          },
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "definition": "label_values(frostfs_node_metabase_request_duration_seconds_bucket{instance=\"$instance\"},shard_id)",
+          "hide": 0,
+          "includeAll": false,
+          "multi": false,
+          "name": "shard_id",
+          "options": [],
+          "query": {
+            "query": "label_values(frostfs_node_metabase_request_duration_seconds_bucket{instance=\"$instance\"},shard_id)",
+            "refId": "PrometheusVariableQueryEditor-VariableQuery"
+          },
+          "refresh": 1,
+          "regex": "",
+          "skipUrlSync": false,
+          "sort": 1,
+          "type": "query"
+        },
+        {
+          "current": {
+            "selected": true,
+            "text": "Delete",
+            "value": "Delete"
+          },
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "definition": "label_values(frostfs_node_metabase_request_duration_seconds_bucket{instance=\"$instance\", shard_id=\"$shard_id\"},method)",
+          "hide": 0,
+          "includeAll": false,
+          "multi": false,
+          "name": "method",
+          "options": [],
+          "query": {
+            "query": "label_values(frostfs_node_metabase_request_duration_seconds_bucket{instance=\"$instance\", shard_id=\"$shard_id\"},method)",
+            "refId": "PrometheusVariableQueryEditor-VariableQuery"
+          },
+          "refresh": 1,
+          "regex": "",
+          "skipUrlSync": false,
+          "sort": 1,
+          "type": "query"
+        }
+      ]
+    },
+    "time": {
+      "from": "now-1h",
+      "to": "now"
+    },
+    "timepicker": {},
+    "timezone": "",
+    "title": "Storage node metabase bucket distribution",
+    "uid": "e110ed66-d086-461d-958a-efc6c55607d2",
+    "version": 1,
+    "weekStart": ""
+  }

services/grafana/provisioning/dashboards/storage-node-logs.json

@@ -0,0 +1,358 @@
+{
+    "annotations": {
+      "list": [
+        {
+          "builtIn": 1,
+          "datasource": {
+            "type": "grafana",
+            "uid": "-- Grafana --"
+          },
+          "enable": true,
+          "hide": true,
+          "iconColor": "rgba(0, 211, 255, 1)",
+          "name": "Annotations & Alerts",
+          "type": "dashboard"
+        }
+      ]
+    },
+    "description": "Overview panel for all services",
+    "editable": true,
+    "fiscalYearStartMonth": 0,
+    "graphTooltip": 0,
+    "links": [],
+    "liveNow": false,
+    "panels": [
+      {
+        "datasource": {
+          "type": "loki",
+          "uid": "P8E80F9AEF21F6940"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 0,
+              "gradientMode": "none",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "lineInterpolation": "linear",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "none"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 8,
+          "w": 24,
+          "x": 0,
+          "y": 0
+        },
+        "id": 5,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": true
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": {
+              "type": "loki",
+              "uid": "P8E80F9AEF21F6940"
+            },
+            "editorMode": "code",
+            "expr": "count_over_time({hostname=~\"$hostname\"}  |~ \"$level\" |~ `$filter` [1s])",
+            "queryType": "range",
+            "refId": "A"
+          }
+        ],
+        "title": "Log intensity",
+        "type": "timeseries"
+      },
+      {
+        "datasource": {
+          "type": "loki",
+          "uid": "P8E80F9AEF21F6940"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "thresholds"
+            },
+            "custom": {
+              "align": "auto",
+              "cellOptions": {
+                "type": "auto"
+              },
+              "inspect": false
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": [
+            {
+              "matcher": {
+                "id": "byName",
+                "options": "Time"
+              },
+              "properties": [
+                {
+                  "id": "custom.width",
+                  "value": 226
+                }
+              ]
+            },
+            {
+              "matcher": {
+                "id": "byName",
+                "options": "labels"
+              },
+              "properties": [
+                {
+                  "id": "custom.width",
+                  "value": 424
+                }
+              ]
+            }
+          ]
+        },
+        "gridPos": {
+          "h": 11,
+          "w": 24,
+          "x": 0,
+          "y": 8
+        },
+        "id": 4,
+        "options": {
+          "cellHeight": "sm",
+          "footer": {
+            "countRows": false,
+            "fields": "",
+            "reducer": [
+              "sum"
+            ],
+            "show": false
+          },
+          "showHeader": true,
+          "sortBy": []
+        },
+        "pluginVersion": "9.5.6",
+        "targets": [
+          {
+            "datasource": {
+              "type": "loki",
+              "uid": "P8E80F9AEF21F6940"
+            },
+            "editorMode": "code",
+            "expr": "{hostname=~\"$hostname\"} |~ `$filter` | json level |~ \"$level\"",
+            "queryType": "range",
+            "refId": "A"
+          }
+        ],
+        "title": "Log",
+        "transformations": [
+          {
+            "id": "organize",
+            "options": {
+              "excludeByName": {
+                "id": true,
+                "tsNs": true
+              },
+              "indexByName": {},
+              "renameByName": {}
+            }
+          }
+        ],
+        "type": "table"
+      }
+    ],
+    "refresh": "",
+    "schemaVersion": 38,
+    "style": "dark",
+    "tags": [],
+    "templating": {
+      "list": [
+        {
+          "current": {
+            "selected": true,
+            "text": [
+              "All"
+            ],
+            "value": [
+              "$__all"
+            ]
+          },
+          "datasource": {
+            "type": "loki",
+            "uid": "P8E80F9AEF21F6940"
+          },
+          "definition": "",
+          "hide": 0,
+          "includeAll": true,
+          "label": "Hostname",
+          "multi": true,
+          "name": "hostname",
+          "options": [],
+          "query": {
+            "label": "hostname",
+            "refId": "LokiVariableQueryEditor-VariableQuery",
+            "stream": "",
+            "type": 1
+          },
+          "refresh": 1,
+          "regex": "",
+          "skipUrlSync": false,
+          "sort": 0,
+          "type": "query"
+        },
+        {
+          "current": {
+            "selected": true,
+            "text": [
+              "All"
+            ],
+            "value": [
+              "$__all"
+            ]
+          },
+          "hide": 0,
+          "includeAll": true,
+          "label": "Level",
+          "multi": true,
+          "name": "level",
+          "options": [
+            {
+              "selected": true,
+              "text": "All",
+              "value": "$__all"
+            },
+            {
+              "selected": false,
+              "text": "debug",
+              "value": "debug"
+            },
+            {
+              "selected": false,
+              "text": "info",
+              "value": "info"
+            },
+            {
+              "selected": false,
+              "text": "warn",
+              "value": "warn"
+            },
+            {
+              "selected": false,
+              "text": "error",
+              "value": "error"
+            },
+            {
+              "selected": false,
+              "text": "dpanic",
+              "value": "dpanic"
+            },
+            {
+              "selected": false,
+              "text": "panic",
+              "value": "panic"
+            },
+            {
+              "selected": false,
+              "text": "fatal",
+              "value": "fatal"
+            }
+          ],
+          "query": "debug,info,warn,error,dpanic,panic,fatal",
+          "queryValue": "",
+          "skipUrlSync": false,
+          "type": "custom"
+        },
+        {
+          "current": {
+            "selected": false,
+            "text": "",
+            "value": ""
+          },
+          "hide": 0,
+          "label": "Filter",
+          "name": "filter",
+          "options": [
+            {
+              "selected": true,
+              "text": "",
+              "value": ""
+            }
+          ],
+          "query": "",
+          "skipUrlSync": false,
+          "type": "textbox"
+        }
+      ]
+    },
+    "time": {
+      "from": "now-5m",
+      "to": "now"
+    },
+    "timepicker": {},
+    "timezone": "",
+    "title": "Storage node logs",
+    "uid": "bef4b42d-c74e-4241-96cb-d239113f44bc",
+    "version": 1,
+    "weekStart": ""
+  }

services/grafana/provisioning/dashboards/writecache-buckets.json

@@ -0,0 +1,361 @@
+{
+    "annotations": {
+      "list": [
+        {
+          "builtIn": 1,
+          "datasource": {
+            "type": "grafana",
+            "uid": "-- Grafana --"
+          },
+          "enable": true,
+          "hide": true,
+          "iconColor": "rgba(0, 211, 255, 1)",
+          "name": "Annotations & Alerts",
+          "type": "dashboard"
+        }
+      ]
+    },
+    "editable": true,
+    "fiscalYearStartMonth": 0,
+    "graphTooltip": 0,
+    "id": 6,
+    "links": [],
+    "liveNow": false,
+    "panels": [
+      {
+        "collapsed": false,
+        "gridPos": {
+          "h": 1,
+          "w": 24,
+          "x": 0,
+          "y": 0
+        },
+        "id": 1,
+        "panels": [],
+        "title": "$method",
+        "type": "row"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "PBFA97CFB590B2093"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 50,
+              "gradientMode": "none",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "lineInterpolation": "smooth",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "never",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "normal"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            },
+            "unit": "reqps"
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 20,
+          "w": 24,
+          "x": 0,
+          "y": 1
+        },
+        "id": 7,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": true
+          },
+          "tooltip": {
+            "mode": "multi",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)",
+            "hide": false,
+            "legendFormat": "< 0.005s",
+            "range": true,
+            "refId": "A"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.005\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)",
+            "hide": false,
+            "legendFormat": "0.005s - 0.01s",
+            "range": true,
+            "refId": "B"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.01\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)",
+            "hide": false,
+            "legendFormat": "0.01s - 0.025s",
+            "range": true,
+            "refId": "C"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.025\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)",
+            "hide": false,
+            "legendFormat": "0.025s - 0.05s",
+            "range": true,
+            "refId": "D"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.05\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)",
+            "hide": false,
+            "legendFormat": "0.05s - 0.1s",
+            "range": true,
+            "refId": "E"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)",
+            "hide": false,
+            "legendFormat": "0.1s - 0.25s",
+            "range": true,
+            "refId": "F"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.25\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)",
+            "hide": false,
+            "legendFormat": "0.25s - 0.5s",
+            "range": true,
+            "refId": "G"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"0.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)",
+            "hide": false,
+            "legendFormat": "0.5s - 1s",
+            "range": true,
+            "refId": "H"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"1\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)",
+            "hide": false,
+            "legendFormat": "1s - 2.5s",
+            "range": true,
+            "refId": "I"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"2.5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)",
+            "hide": false,
+            "legendFormat": "2.5s - 5s",
+            "range": true,
+            "refId": "J"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"5\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)",
+            "hide": false,
+            "legendFormat": "5s - 10s",
+            "range": true,
+            "refId": "K"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "PBFA97CFB590B2093"
+            },
+            "editorMode": "code",
+            "expr": "sum by (instance, shard_id, method) (\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"+Inf\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n- ignoring(le)\nrate(frostfs_node_writecache_request_duration_seconds_bucket{le=\"10\", instance=\"$instance\", shard_id=\"$shard_id\", method=\"$method\"}[$__rate_interval])\n)",
+            "hide": false,
+            "legendFormat": "> 10s",
+            "range": true,
+            "refId": "L"
+          }
+        ],
+        "title": "$method",
+        "type": "timeseries"
+      }
+    ],
+    "refresh": "",
+    "schemaVersion": 38,
+    "style": "dark",
+    "tags": [],
+    "templating": {
+      "list": [
+        {
+          "current": {
+            "selected": true,
+            "text": "s01.frostfs.devenv:9090",
+            "value": "s01.frostfs.devenv:9090"
+          },
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "definition": "label_values(frostfs_node_writecache_request_duration_seconds_bucket,instance)",
+          "hide": 0,
+          "includeAll": false,
+          "multi": false,
+          "name": "instance",
+          "options": [],
+          "query": {
+            "query": "label_values(frostfs_node_writecache_request_duration_seconds_bucket,instance)",
+            "refId": "PrometheusVariableQueryEditor-VariableQuery"
+          },
+          "refresh": 1,
+          "regex": "",
+          "skipUrlSync": false,
+          "sort": 1,
+          "type": "query"
+        },
+        {
+          "current": {
+            "selected": true,
+            "text": "BGH3NNzxkiW7JB1BKLHb5z",
+            "value": "BGH3NNzxkiW7JB1BKLHb5z"
+          },
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "definition": "label_values(frostfs_node_writecache_request_duration_seconds_bucket{instance=\"$instance\"},shard_id)",
+          "hide": 0,
+          "includeAll": false,
+          "multi": false,
+          "name": "shard_id",
+          "options": [],
+          "query": {
+            "query": "label_values(frostfs_node_writecache_request_duration_seconds_bucket{instance=\"$instance\"},shard_id)",
+            "refId": "PrometheusVariableQueryEditor-VariableQuery"
+          },
+          "refresh": 1,
+          "regex": "",
+          "skipUrlSync": false,
+          "sort": 1,
+          "type": "query"
+        },
+        {
+          "current": {
+            "selected": true,
+            "text": "Delete",
+            "value": "Delete"
+          },
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "definition": "label_values(frostfs_node_writecache_request_duration_seconds_bucket{instance=\"$instance\", shard_id=\"$shard_id\"},method)",
+          "hide": 0,
+          "includeAll": false,
+          "multi": false,
+          "name": "method",
+          "options": [],
+          "query": {
+            "query": "label_values(frostfs_node_writecache_request_duration_seconds_bucket{instance=\"$instance\", shard_id=\"$shard_id\"},method)",
+            "refId": "PrometheusVariableQueryEditor-VariableQuery"
+          },
+          "refresh": 1,
+          "regex": "",
+          "skipUrlSync": false,
+          "sort": 1,
+          "type": "query"
+        }
+      ]
+    },
+    "time": {
+      "from": "now-1h",
+      "to": "now"
+    },
+    "timepicker": {},
+    "timezone": "",
+    "title": "Storage node writecache bucket distribution",
+    "uid": "d2fef1d1-9ef3-437e-ad21-670a20e0e9b4",
+    "version": 3,
+    "weekStart": ""
+  }

services/grafana/provisioning/datasources/datasource.yml

@@ -6,3 +6,8 @@ datasources:
   access: proxy
   orgId: 1
   url: http://prometheus:9090
+- name: Loki
+  type: loki
+  access: proxy
+  orgId: 1
+  url: http://loki:3100

services/http_gate/docker-compose.yml

@@ -1,6 +1,5 @@
 ---
 
-version: "2.4"
 services:
   http_gate:
     image: ${HTTP_GW_IMAGE}:${HTTP_GW_VERSION}

services/ir/.ir.env

@@ -1,3 +1 @@
-FROSTFS_IR_CONTRACTS_FROSTFSID=27407c76feabc407908f3d09a3d845d45e7c981a
-
 FROSTFS_IR_CONTROL_GRPC_ENDPOINT=127.0.0.1:16512

services/ir/cfg/config.yml

@@ -33,7 +33,11 @@ mainnet:
 morph:
   endpoint:
     client:  # List of websocket RPC endpoints in sidechain
-      - address: ws://morph-chain:30333/ws
+      - address: wss://morph-chain:30333/ws
+        root_cas:
+          - /wallets/mtls/CA1_cert.pem
+        certificate: /wallets/mtls/peerCA2_IR_cert.pem
+        key: /wallets/mtls/peerCA2_IR_key.pem
   validators:  # List of hex-encoded 33-byte public keys of sidechain validators to vote for at application startup
     - 02b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc2
 

services/ir/docker-compose.yml

@@ -1,6 +1,5 @@
 ---
 
-version: "2.4"
 services:
 
   ir01:
@@ -13,7 +12,8 @@ services:
       ir_int:
       internet:
         ipv4_address: ${IPV4_PREFIX}.61
-    stop_signal: SIGKILL
+    stop_signal: SIGTERM
+    stop_grace_period: 15s
     volumes:
       - ./az.json:/wallet.json
       - ./az.key:/wallet01.key
@@ -22,6 +22,7 @@ services:
       - ./../../vendor/frostfs-cli:/frostfs-cli
       - ./healthcheck.sh:/healthcheck.sh
       - ./cfg:/etc/frostfs/ir
+      - ./../../mtls:/wallets/mtls:ro
     env_file: [ ".env", ".ir.env", ".int_test.env" ]
     command: [ "frostfs-ir", "--config", "/etc/frostfs/ir/config.yml" ]
     healthcheck:

services/ir/healthcheck.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-/frostfs-cli control healthcheck \
+/frostfs-cli control ir healthcheck \
 	--endpoint "$FROSTFS_IR_CONTROL_GRPC_ENDPOINT" \
-	--wallet /wallet01.key --ir |
+	--wallet /wallet01.key |
 	grep "Health status: READY"

services/jaeger/docker-compose.yml

@@ -1,4 +1,3 @@
-version: '2.4'
 services:
   jaeger:
     image: ${JAEGER_IMAGE}:${JAEGER_VERSION}

services/morph_chain/docker-compose.yml

@@ -1,6 +1,5 @@
 ---
 
-version: "2.4"
 services:
   frostfs_morph_chain:
     image: ${NEOGO_IMAGE}:${NEOGO_VERSION}
@@ -20,6 +19,7 @@ services:
     - ./config.yml:/wallets/config.yml
     - ./../../vendor/hosts:/etc/hosts
     - ./../../wallets/wallet.json:/wallets/wallet.json
+    - ./../../mtls:/wallets/mtls:ro
 
 networks:
   chain_int:

services/morph_chain/protocol.privnet.yml

@@ -29,13 +29,26 @@ ApplicationConfiguration:
     AttemptConnPeers: 5
     MinPeers: 0
   Relay: true
+  Consensus:
+    Enabled: true
+    UnlockWallet:
+      Path: "./wallets/node-wallet.json"
+      Password: "one"
   RPC:
-    Addresses:
-      - "192.168.130.90:30333"
+    # Addresses:
+    #   - "192.168.130.90:30333"
     Enabled: true
     SessionEnabled: true
     EnableCORSWorkaround: false
     MaxGasInvoke: 100
+    TLSConfig:
+      Enabled: true
+      Addresses:
+        - "192.168.130.90:30333"
+      RootCAs:
+        - "/wallets/mtls/CA2_cert.pem"
+      CertFile: "/wallets/mtls/peerCA1_X_cert.pem"
+      KeyFile: "/wallets/mtls/peerCA1_X_key.pem"
   P2PNotary:
     Enabled: true
     UnlockWallet:
@@ -49,6 +62,3 @@ ApplicationConfiguration:
     Addresses:
       - ":20011"
     Enabled: true
-  UnlockWallet:
-    Path: "./wallets/node-wallet.json"
-    Password: "one"

services/nats/docker-compose.yml

@@ -1,6 +1,5 @@
 ---
 
-version: "2.4"
 services:
   nats:
     image: ${NATS_IMAGE}:${NATS_VERSION}

services/prometheus/docker-compose.yml

@@ -1,4 +1,3 @@
-version: '2.4'
 services:
   prometheus:
     image: ${PROMETHEUS_IMAGE}:${PROMETHEUS_VERSION}
@@ -15,8 +14,6 @@ services:
       - ./prometheus.yml:/etc/prometheus/prometheus.yml
     command:
       - --config.file=/etc/prometheus/prometheus.yml
-    ports:
-      - '9090:9090'
     stop_signal: SIGKILL
     env_file: [ ".env", ".prometheus.env", ".int_test.env" ]
 
@@ -24,4 +21,4 @@ networks:
   prometheus_int:
   internet:
     external: true
-    name: basenet_internet
+    name: basenet_internet

services/rest_gate/docker-compose.yml

@@ -1,6 +1,5 @@
 ---
 
-version: "2.4"
 services:
   rest_gate:
     image: ${REST_GW_IMAGE}:${REST_GW_VERSION}
@@ -16,7 +15,8 @@ services:
       - ./wallet.json:/wallet.json
       - ./../../vendor/hosts:/etc/hosts
       - ./cfg:/etc/frostfs/rest
-    stop_signal: SIGKILL
+    stop_signal: SIGTERM
+    stop_grace_period: 15s
     env_file: [ ".env", ".int_test.env" ]
     command: [ "frostfs-rest-gw", "--config", "/etc/frostfs/rest/config.yml" ]
     environment:

services/s3_gate/cfg/config.yml

@@ -33,3 +33,17 @@ server:
 wallet:
   path: /wallet.json # Path to wallet
   passphrase: "s3" # Passphrase to decrypt wallet
+
+features:
+  md5:
+    enabled: true
+
+control:
+  grpc:
+    endpoint: localhost:16515
+
+frostfsid:
+  enabled: false
+
+policy:
+  enabled: false

services/s3_gate/docker-compose.yml

@@ -1,6 +1,5 @@
 ---
 
-version: "2.4"
 services:
   s3_gate:
     image: ${S3_GW_IMAGE}:${S3_GW_VERSION}
@@ -18,7 +17,8 @@ services:
       - ./tls.crt:/tls.crt
       - ./../../vendor/hosts:/etc/hosts
       - ./cfg:/etc/frostfs/s3
-    stop_signal: SIGKILL
+    stop_signal: SIGTERM
+    stop_grace_period: 15s
     env_file: [ ".env", ".s3.env", ".int_test.env" ]
     command: [ "frostfs-s3-gw", "--config", "/etc/frostfs/s3/config.yml" ]
     environment:

services/storage/cfg/config.yml

@@ -1,6 +1,11 @@
 # Logger section
 logger:
   level: debug # Minimum enabled logging level
+  loki:
+    enabled: true
+    endpoint: "loki.frostfs.devenv:3100/api/prom/push"
+    max_batch_delay: 1s
+    max_batch_size: 200
 
 # Profiler section
 pprof:
@@ -14,7 +19,7 @@ prometheus:
   address: :9090  # Server address
   shutdown_timeout: 15s  # Timeout for metrics HTTP server graceful shutdown
 
-  # Application tracing section
+# Application tracing section
 tracing:
   enabled: true
   exporter: otlp_grpc
@@ -24,7 +29,7 @@ tracing:
 morph:
   dial_timeout: 30s  # Timeout for side chain NEO RPC client connection
   rpc_endpoint:  # Side chain NEO RPC endpoints
-    - address: ws://morph-chain:30333/ws
+    - address: wss://morph-chain:30333/ws
       priority: 1
 
 # Common storage node settings

services/storage/docker-compose.yml

@@ -1,6 +1,5 @@
 ---
 
-version: "2.4"
 services:
   storage01:
     image: ${NODE_IMAGE}:${NODE_VERSION}
@@ -24,7 +23,9 @@ services:
       - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key
       - ../nats/ca-cert.pem:/etc/frostfs-node/nats.ca.crt
       - ./cfg:/etc/frostfs/storage
-    stop_signal: SIGKILL
+      - ./../../mtls:/wallets/mtls:ro
+    stop_signal: SIGTERM
+    stop_grace_period: 15s
     env_file: [ ".env", ".storage.env", ".int_test.env" ]
     command: [ "frostfs-node", "--config", "/etc/frostfs/storage/config.yml" ]
     environment:
@@ -35,6 +36,9 @@ services:
       - FROSTFS_CONTROL_GRPC_ENDPOINT=s01.${LOCAL_DOMAIN}:8081
       - FROSTFS_NODE_ATTRIBUTE_1=UN-LOCODE:RU MOW
       - FROSTFS_NODE_ATTRIBUTE_2=Price:22
+      - FROSTFS_MORPH_RPC_ENDPOINT_0_ROOT_CAS=/wallets/mtls/CA1_cert.pem
+      - FROSTFS_MORPH_RPC_ENDPOINT_0_CERTIFICATE=/wallets/mtls/peerCA2_1_cert.pem
+      - FROSTFS_MORPH_RPC_ENDPOINT_0_KEY=/wallets/mtls/peerCA2_1_key.pem
     healthcheck:
       test: ["CMD-SHELL", "/healthcheck.sh"]
       interval: 2s
@@ -64,7 +68,9 @@ services:
       - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key
       - ../nats/ca-cert.pem:/etc/frostfs-node/nats.ca.crt
       - ./cfg:/etc/frostfs/storage
-    stop_signal: SIGKILL
+      - ./../../mtls:/wallets/mtls:ro
+    stop_signal: SIGTERM
+    stop_grace_period: 15s
     env_file: [ ".env", ".storage.env", ".int_test.env" ]
     command: [ "frostfs-node", "--config", "/etc/frostfs/storage/config.yml" ]
     environment:
@@ -75,6 +81,9 @@ services:
       - FROSTFS_CONTROL_GRPC_ENDPOINT=s02.${LOCAL_DOMAIN}:8081
       - FROSTFS_NODE_ATTRIBUTE_1=UN-LOCODE:RU LED
       - FROSTFS_NODE_ATTRIBUTE_2=Price:33
+      - FROSTFS_MORPH_RPC_ENDPOINT_0_ROOT_CAS=/wallets/mtls/CA1_cert.pem
+      - FROSTFS_MORPH_RPC_ENDPOINT_0_CERTIFICATE=/wallets/mtls/peerCA2_2_cert.pem
+      - FROSTFS_MORPH_RPC_ENDPOINT_0_KEY=/wallets/mtls/peerCA2_2_key.pem
     healthcheck:
       test: ["CMD-SHELL", "/healthcheck.sh"]
       interval: 2s
@@ -104,7 +113,9 @@ services:
       - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key
       - ../nats/ca-cert.pem:/etc/frostfs-node/nats.ca.crt
       - ./cfg:/etc/frostfs/storage
-    stop_signal: SIGKILL
+      - ./../../mtls:/wallets/mtls:ro
+    stop_signal: SIGTERM
+    stop_grace_period: 15s
     env_file: [ ".env", ".storage.env", ".int_test.env" ]
     command: [ "frostfs-node", "--config", "/etc/frostfs/storage/config.yml" ]
     environment:
@@ -115,6 +126,9 @@ services:
       - FROSTFS_CONTROL_GRPC_ENDPOINT=s03.${LOCAL_DOMAIN}:8081
       - FROSTFS_NODE_ATTRIBUTE_1=UN-LOCODE:SE STO
       - FROSTFS_NODE_ATTRIBUTE_2=Price:11
+      - FROSTFS_MORPH_RPC_ENDPOINT_0_ROOT_CAS=/wallets/mtls/CA1_cert.pem
+      - FROSTFS_MORPH_RPC_ENDPOINT_0_CERTIFICATE=/wallets/mtls/peerCA2_3_cert.pem
+      - FROSTFS_MORPH_RPC_ENDPOINT_0_KEY=/wallets/mtls/peerCA2_3_key.pem
     healthcheck:
       test: ["CMD-SHELL", "/healthcheck.sh"]
       interval: 2s
@@ -145,7 +159,9 @@ services:
       - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key
       - ../nats/ca-cert.pem:/etc/frostfs-node/nats.ca.crt
       - ./cfg:/etc/frostfs/storage
-    stop_signal: SIGKILL
+      - ./../../mtls:/wallets/mtls:ro
+    stop_signal: SIGTERM
+    stop_grace_period: 15s
     env_file: [ ".env", ".storage.env", ".int_test.env" ]
     command: [ "frostfs-node", "--config", "/etc/frostfs/storage/config.yml" ]
     environment:
@@ -161,6 +177,9 @@ services:
       - FROSTFS_GRPC_1_TLS_KEY=/tls.key
       - FROSTFS_NODE_ATTRIBUTE_1=UN-LOCODE:FI HEL
       - FROSTFS_NODE_ATTRIBUTE_2=Price:44
+      - FROSTFS_MORPH_RPC_ENDPOINT_0_ROOT_CAS=/wallets/mtls/CA1_cert.pem
+      - FROSTFS_MORPH_RPC_ENDPOINT_0_CERTIFICATE=/wallets/mtls/peerCA2_4_cert.pem
+      - FROSTFS_MORPH_RPC_ENDPOINT_0_KEY=/wallets/mtls/peerCA2_4_key.pem
     healthcheck:
       test: ["CMD-SHELL", "/healthcheck.sh"]
       interval: 2s