[#95] Output errors during make clean

It happens that a volume may not be deleted during `make clean`.
For example, if the volume is being used by a container.
If this happens, there are no errors printed to stdout.
And old volumes may cause errors during subsequent `make up`

Signed-off-by: Nikita Zinkevich <n.zinkevich@yadro.com>

.basic_services

@@ -1,5 +1,4 @@
 # Services start/stop order
 # Will start from top to bottom and stop in reverse
-nats
 ir
 storage

.bootstrap_services

@@ -4,3 +4,4 @@ basenet
 morph_chain
 jaeger
 prometheus
+grafana

.dockerignore

@@ -1,5 +1,5 @@
 .docker
-.github
+.forgejo
 vendor
 tmp
 .secrets

.env

@@ -8,49 +8,43 @@ BASTION_VERSION=10
 BASTION_IMAGE=debian
 
 # NeoGo privnet
-NEOGO_VERSION=0.100.1
+NEOGO_VERSION=0.106.3
 NEOGO_IMAGE=nspccdev/neo-go
 
 # FrostFS InnerRing nodes
-IR_VERSION=5ffa8268
-IR_IMAGE=truecloudlab/frostfs-ir
+IR_VERSION=0.44.4
+IR_IMAGE=git.frostfs.info/truecloudlab/frostfs-ir
 
 # FrostFS Storage nodes
-NODE_VERSION=5ffa8268
-NODE_IMAGE=truecloudlab/frostfs-storage
-
-# NATS Server
-NATS_VERSION=2.7.2
-NATS_IMAGE=nats
+NODE_VERSION=0.44.4
+NODE_IMAGE=git.frostfs.info/truecloudlab/frostfs-storage
 
 # HTTP Gate
-HTTP_GW_VERSION=6abd500b
+HTTP_GW_VERSION=0.32.0
 HTTP_GW_IMAGE=truecloudlab/frostfs-http-gw
 
-# REST Gate
-REST_GW_VERSION=c9c85e90
-REST_GW_IMAGE=truecloudlab/frostfs-rest-gw
-
 # S3 Gate
-S3_GW_VERSION=000d9ed4
+S3_GW_VERSION=0.32.0
 S3_GW_IMAGE=truecloudlab/frostfs-s3-gw
 
+# Lifecycler
+S3_LIFECYCLER_VERSION=0.1.3
+S3_LIFECYCLER_IMAGE=truecloudlab/frostfs-s3-lifecycler
+
 # FrostFS LOCODE database
-LOCODE_DB_URL=https://github.com/nspcc-dev/neofs-locode-db/releases/download/v0.3.0/locode_db.gz
+LOCODE_DB_URL=https://git.frostfs.info/attachments/a2e8def7-52b6-49f1-89cd-a056712e8e54
 #LOCODE_DB_PATH=/path/to/locode_db
 
 # FrostFS CLI binary
-FROSTFS_CLI_URL=https://http.t5.fs.neo.org/AQgse8bPCZx4zScMuAKxowJdZPbKHp8NDcp15o6VUNmk/C6BNLpYg5gWLHp3DrXozSxxGLDahBuSBCyJoYSSR1M3Q
+FROSTFS_CLI_URL=https://git.frostfs.info/TrueCloudLab/frostfs-node/releases/download/v${NODE_VERSION}/frostfs-cli
 #FROSTFS_CLI_PATH=/path/to/frostfs-cli-binary
 
 # FrostFS ADM tool binary
-FROSTFS_ADM_VERSION=e3554425
-FROSTFS_ADM_URL=https://http.t5.fs.neo.org/AQgse8bPCZx4zScMuAKxowJdZPbKHp8NDcp15o6VUNmk/sXZxy9vbFyJiLhN9qTSXozXK7SN9H8ZC6dpvAt59Zaj
+FROSTFS_ADM_URL=https://git.frostfs.info/TrueCloudLab/frostfs-node/releases/download/v${NODE_VERSION}/frostfs-adm
 #FROSTFS_ADM_PATH=/path/to/frostfs-adm-binary
 
 # Compiled FrostFS Smart Contracts
-FROSTFS_CONTRACTS_VERSION=4f3c08f5
-FROSTFS_CONTRACTS_URL=https://http.t5.fs.neo.org/AQgse8bPCZx4zScMuAKxowJdZPbKHp8NDcp15o6VUNmk/c1nGtturFrSeygYP3AyNHDDLNbs7HhJiH2BQkgZxEmZ
+FROSTFS_CONTRACTS_URL=https://git.frostfs.info/TrueCloudLab/frostfs-contract/releases/download/v0.21.1/frostfs-contract-v0.21.1.tar.gz
 #FROSTFS_CONTRACTS_PATH=/path/to/unpacked/frostfs-contracts-dir
 
 # Jaeger tracing
@@ -59,4 +53,12 @@ JAEGER_IMAGE=jaegertracing/all-in-one
 
 # Prometheus monitoring
 PROMETHEUS_VERSION=v2.43.0
-PROMETHEUS_IMAGE=prom/prometheus
+PROMETHEUS_IMAGE=prom/prometheus
+
+# Grafana versions
+GRAFANA_VERSION=9.5.6
+GRAFANA_IMAGE=grafana/grafana
+
+# Loki versions
+LOKI_VERSION=2.9.1
+LOKI_IMAGE=grafana/loki

.forgejo/workflows/dco.yml

@@ -0,0 +1,21 @@
+name: DCO action
+on: [pull_request]
+
+jobs:
+  dco:
+    name: DCO
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Setup Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: '1.21'
+
+      - name: Run commit format checker
+        uses: https://git.frostfs.info/TrueCloudLab/dco-go@v2
+        with:
+          from: 'origin/${{ github.event.pull_request.base.ref }}'

.gitattributes

@@ -0,0 +1 @@
+/services/grafana/provisioning/dashboards/* -diff -merge

.github/CODEOWNERS

@@ -1 +0,0 @@
-*       @alexvanin @fyrchik

.github/workflows/dco.yml

@@ -1,21 +0,0 @@
-name: DCO check
-
-on:
-  pull_request:
-    branches:
-      - master
-
-jobs:
-  commits_check_job:
-    runs-on: ubuntu-latest
-    name: Commits Check
-    steps:
-    - name: Get PR Commits
-      id: 'get-pr-commits'
-      uses: tim-actions/get-pr-commits@master
-      with:
-        token: ${{ secrets.GITHUB_TOKEN }}
-    - name: DCO Check
-      uses: tim-actions/dco@master
-      with:
-        commits: ${{ steps.get-pr-commits.outputs.commits }}

.gitignore

@@ -15,4 +15,3 @@ sites/*
 # Runtime generation keys
 services/storage/*tls.crt
 services/storage/*tls.key
-services/nats/*.pem

.services

@@ -2,4 +2,4 @@
 # Will start from top to bottom and stop in reverse
 http_gate
 s3_gate
-rest_gate
+s3_lifecycler

CODEOWNERS

@@ -0,0 +1,3 @@
+.*          @alexvanin @fyrchik
+.forgejo/.* @potyarkin
+Makefile    @potyarkin

CONTRIBUTING.md

@@ -3,8 +3,8 @@
 First, thank you for contributing! We love and encourage pull requests from
 everyone. Please follow the guidelines:
 
-- Check the open [issues](https://github.com/TrueCloudLab/frostfs-dev-env/issues) and
-  [pull requests](https://github.com/TrueCloudLab/frostfs-dev-env/pulls) for existing
+- Check the open [issues](https://git.frostfs.info/TrueCloudLab/frostfs-dev-env/issues) and
+  [pull requests](https://git.frostfs.info/TrueCloudLab/frostfs-dev-env/pulls) for existing
   discussions.
 
 - Open an issue first, to discuss a new feature or enhancement.
@@ -25,19 +25,19 @@ Start by forking the `frostfs-dev-env` repository, make changes in a branch and
 send a pull request. We encourage pull requests to discuss code changes. Here
 are the steps in details:
 
-### Set up your GitHub Repository
-Fork [FrostFS node upstream](https://github.com/TrueCloudLab/frostfs-dev-env/fork) source
+### Set up your git repository
+Fork [FrostFS node upstream](https://git.frostfs.info/repo/fork/24) source
 repository to your own personal repository. Copy the URL of your fork (you will
 need it for the `git clone` command below).
 
 ```sh
-$ git clone https://github.com/TrueCloudLab/frostfs-dev-env
+$ git clone https://git.frostfs.info/<username>/frostfs-dev-env.git
 ```
 
 ### Set up git remote as ``upstream``
 ```sh
 $ cd frostfs-dev-env
-$ git remote add upstream https://github.com/TrueCloudLab/frostfs-dev-env
+$ git remote add upstream https://git.frostfs.info/TrueCloudLab/frostfs-dev-env.git
 $ git fetch upstream
 $ git merge upstream/master
 ...
@@ -55,8 +55,7 @@ $ git checkout -b feature/123-something_awesome
 ### Test your changes
 After your code changes, make sure
 
-- To add test cases for the new code.
-- To run `make lint`
+- To run `make up` to check dev-env is not broken.
 - To squash your commits into a single commit or a series of logically separated
   commits run `git rebase -i`. It's okay to force update your pull request.
 
@@ -86,8 +85,8 @@ $ git push origin feature/123-something_awesome
 ```
 
 ### Create a Pull Request
-Pull requests can be created via GitHub. Refer to [this
-document](https://help.github.com/articles/creating-a-pull-request/) for
+Pull requests can be created via Forgejo. Refer to [this
+document](https://docs.codeberg.org/collaborating/pull-requests-and-git-flow/) for
 detailed steps on how to create a pull request. After a Pull Request gets peer
 reviewed and approved, it will be merged.
 

Makefile

@@ -43,7 +43,7 @@ HOSTS_LINES = $(shell grep -Rl IPV4_PREFIX ./services/* | grep .hosts)
 MORPH_CHAIN_PROTOCOL = './services/morph_chain/protocol.privnet.yml'
 
 # List of grepped environment variables from *.env
-GREP_DOTENV = $(shell find . -name '*.env' -exec grep -rhv -e '^\#' -e '^$$' {} + | sort -u )
+GREP_DOTENV = $(shell find . -name '*.env' -exec grep -rhv -e '^#' -e '^$$' {} + | sort -u )
 
 # Pull all required Docker images
 .PHONY: pull
@@ -60,14 +60,15 @@ get: $(foreach SVC, $(GET_SVCS), get.$(SVC))
 .PHONY: up
 up: up/basic
 	@$(foreach SVC, $(START_SVCS), $(shell docker-compose -f services/$(SVC)/docker-compose.yml up -d))
+	./vendor/frostfs-adm morph proxy-add-account --config frostfs-adm.yml --account=`docker container exec morph_chain neo-go wallet dump-keys -w /wallets/s3-wallet.json | head -1 | awk '{print $1}'` || die "Couldn't set s3-gw wallet as proxy wallet"
 	@echo "Full FrostFS Developer Environment is ready"
 
 # Build up FrostFS
 .PHONY: up/basic
 up/basic: up/bootstrap
 	@$(foreach SVC, $(START_BASIC), $(shell docker-compose -f services/$(SVC)/docker-compose.yml up -d))
-	@./bin/tick.sh
-	@./bin/config.sh string SystemDNS container
+	@./vendor/frostfs-adm -c ./frostfs-adm.yml morph force-new-epoch
+	@./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config SystemDNS=container --force
 	@echo "Basic FrostFS Developer Environment is ready"
 
 # Start bootstrap services
@@ -75,9 +76,28 @@ up/basic: up/bootstrap
 up/bootstrap: get vendor/hosts
 	@$(foreach SVC, $(START_BOOTSTRAP), $(shell docker-compose -f services/$(SVC)/docker-compose.yml up -d))
 	@source ./bin/helper.sh
-	@./vendor/frostfs-adm --config frostfs-adm.yml morph init --alphabet-wallets ./services/ir --contracts vendor/contracts || die "Failed to initialize Alphabet wallets"
-	@for f in ./services/storage/wallet*.json; do echo "Transfer GAS to wallet $${f}" && ./vendor/frostfs-adm -c frostfs-adm.yml morph refill-gas --storage-wallet $${f} --gas 10.0 --alphabet-wallets services/ir || die "Failed to transfer GAS to alphabet wallets"; done
-	@echo "FrostFS sidechain environment is deployed"
+	@./vendor/frostfs-adm --config frostfs-adm.yml morph init --contracts vendor/contracts
+	echo "Set rule chain to policy contract" 
+	@./vendor/frostfs-adm --config frostfs-adm.yml morph \
+		ape add-rule-chain --target-type namespace --target-name "" \
+		--rule 'allow Container.* *' --chain-id "allow_container_ops"
+	@for f in ./services/storage/wallet*.json; do \
+		echo "Transfer GAS to wallet $${f}" \
+		&& ./vendor/frostfs-adm -c frostfs-adm.yml morph refill-gas --storage-wallet $${f} --gas 10.0 \
+		|| die "Failed to transfer GAS to alphabet wallets"; \
+		echo "Register storage wallet $${f} in proxy contract" \
+		&& ./vendor/frostfs-adm morph proxy-add-account --config frostfs-adm.yml --account=`docker container exec morph_chain neo-go wallet dump-keys -w /wallets/storage/$${f##*/} | head -1 | awk '{print $1}'` || die "Couldn't set storage allet as proxy wallet"
+		done
+	@echo "Create frostfsid subject for ./wallets/wallet.json"; \
+	if [ -n "$$(./vendor/frostfs-adm -c frostfs-adm.yml morph frostfsid list-subjects --namespace '')" ]; then \
+		echo "Subject already exists"; \
+	else \
+		subj_key=`docker container exec -it morph_chain neo-go wallet dump-keys -w /wallets/wallet.json | tail -1 | tr -d ' \r\n'` \
+		&& echo "Subject key: $${subj_key}" \
+		&& ./vendor/frostfs-adm -c frostfs-adm.yml morph frostfsid create-subject --namespace "" --subject-key $${subj_key} --subject-name walletsubject \
+		|| die "Failed to create subject for the wallet"; \
+	fi
+	echo "FrostFS sidechain environment is deployed"
 
 # Build up certain service
 .PHONY: up/%
@@ -132,14 +152,14 @@ hosts: vendor/hosts
 .PHONY: clean
 .ONESHELL:
 clean:
-	@rm -rf vendor/* services/storage/s04tls.* services/nats/*.pem
+	@rm -rf vendor/* services/storage/s04tls.*
 	@> .int_test.env
 	@for svc in $(PULL_SVCS)
 	do
 		vols=`docker-compose -f services/$${svc}/docker-compose.yml config --volumes`
 		if [[ ! -z "$${vols}" ]]; then
 			for vol in $${vols}; do
-				docker volume rm -f "$${svc}_$${vol}" 2> /dev/null
+				docker volume rm -f "$${svc}_$${vol}"
 			done
 		fi
 	done
@@ -148,7 +168,7 @@ clean:
 .PHONY: env
 env:
 	@$(foreach envvar,$(GREP_DOTENV),echo $(envvar);)
-	@echo MORPH_BLOCK_TIME=$(shell grep 'SecondsPerBlock' $(MORPH_CHAIN_PROTOCOL) | awk '{print $$2}')s
+	@echo MORPH_BLOCK_TIME=$(shell grep 'TimePerBlock' $(MORPH_CHAIN_PROTOCOL) | awk '{print $$2}')s
 	@echo MORPH_MAGIC=$(shell grep 'Magic' $(MORPH_CHAIN_PROTOCOL) | awk '{print $$2}')
 
 # Restart storage nodes with clean volumes

README.md

@@ -1,5 +1,5 @@
 <p align="center">
-<img src="./.github/logo.svg" width="500px" alt="FrostFS logo">
+<img src="./.forgejo/logo.svg" width="500px" alt="FrostFS logo">
 </p>
 <p align="center">
   <a href="https://frostfs.info">FrostFS</a> local Development and Testing environment
@@ -27,7 +27,7 @@ Make sure you have installed all of the following prerequisites on your machine:
 Clone repo: 
 
 ```
-$ git clone https://github.com/TrueCloudLab/frostfs-dev-env.git
+$ git clone https://git.frostfs.info/TrueCloudLab/frostfs-dev-env.git
 ```
 
 Run next commands from project's root:
@@ -57,17 +57,6 @@ Run all services with command:
 $ make up
 ```
 
-When all services are up, you need to make GAS deposit for test wallet to be
-able to pay for FrostFS operations. Test wallet is located in
-`wallets/wallet.json` with the corresponding key in `wallets/wallet.key`. The
-password is empty.
-
-```
-$ make prepare.ir
-password >
-fa6ba62bffb04030d303dcc95bda7413e03aa3c7e6ca9c2f999d65db9ec9b82c
-```
-
 Also, you should add self-signed node (`s04.frostfs.devenv`) certificate to trusted
 store (default location might be changed using `CA_CERTS_TRUSTED_STORE`
 variable). This step is required for client services (frostfs-http-gw,
@@ -82,12 +71,7 @@ password of inner ring wallet is `one`. See examples in `make help`.
 
 ```
 $ make update.epoch_duration val=30
-Changing EpochDuration configration value to 30
-Enter account NNudMSGzEoktFzdYGYoNb3bzHzbmM1genF password > 
-Sent invocation transaction dbb8c1145b6d10f150135630e13bb0dc282023163f5956c6945a60db0cb45cb0
-Updating FrostFS epoch to 2
-Enter account NNudMSGzEoktFzdYGYoNb3bzHzbmM1genF password > 
-Sent invocation transaction 0e6eb5e190f36332e5e5f4e866c7e100826e285fd949e11c085e15224f343ba6
+Waiting for transactions to persist...
 ```
 
 For instructions on how to set up DevEnv on macOS, please refer [the
@@ -123,7 +107,7 @@ Maybe you will find the answer for your question in [F.A.Q.](docs/faq.md)
 
 ## Using FrostFS Admin Tool in `dev-env`
 
-Devenv supports FrostFS network management via [frostfs-adm](https://github.com/TrueCloudLab/frostfs-node/tree/master/cmd/frostfs-adm).
+Devenv supports FrostFS network management via [frostfs-adm](https://git.frostfs.info/TrueCloudLab/frostfs-node/src/branch/master/cmd/frostfs-adm).
 `services/ir` contains the Alphabet wallet in a proper format, specify it
 with `--alphabet-wallets` flag.
 
@@ -153,6 +137,65 @@ Display addresses and host names for each running service, if available.
 
 Clean up `vendor` directory.
 
+### s3cred
+
+Registers user wallet and issues s3 credentials.
+
+Usage and default parameter values:
+```sh
+make s3cred [password=""] [contract_password=s3] [wallet=/user_wallet.json] [gate_public_key=0313b1ac3a8076e155a7e797b24f0b650cccad5941ea59d7cfd51a024a8b2a06bf]
+```
+
+As soon as the storage node is in the network map (see above) you can generate S3
+credentials:
+
+``` sh
+$ make s3cred
+{
+  "access_key_id": "EXArWh8x1zeHG3851s1RtoCo7dowxF6rhLGA15nbMffT0AKRSjJ5fmcqf3Ht2VCAkfmPQUVARghRB77xHCA1BoN2p",
+  "secret_access_key": "d70c1dba83f0f90bb231f06f1ce0e0dfbcfb122f4b4345a3c18d3869c359b79f",
+  "owner_private_key": "140947599afd9ca89af4b358c3176eb046e554d942a0dc99a8e06f3e43c8f4ad",
+  "wallet_public_key": "0324e76288fcb900100d01802a14ef977cca45ad073561230446df14b344c858b6",
+  "container_id": "EXArWh8x1zeHG3851s1RtoCo7dowxF6rhLGA15nbMffT"
+}                   
+```
+Running without any parameters will result in defaults which are based on the private key from
+`/user-wallet.json` file and `/wallet.json` contract wallet.
+
+Now let's configure an S3 client (AWS CLI will be used as example):
+
+``` sh
+$ aws configure
+AWS Access Key ID []: EXArWh8x1zeHG3851s1RtoCo7dowxF6rhLGA15nbMffT0AKRSjJ5fmcqf3Ht2VCAkfmPQUVARghRB77xHCA1BoN2p
+AWS Secret Access Key []: d70c1dba83f0f90bb231f06f1ce0e0dfbcfb122f4b4345a3c18d3869c359b79f
+Default region name []: us-east-1
+Default output format []: json
+```
+
+If you need to create credentials for different users, put user wallets to `wallets` dir and specify them via `wallet` parameter.
+Pass wallet password in `password` parameter if it's not default. The same is for `contract_wallet` and `gate_public_key` params.
+
+```sh
+$ make s3cred wallet=custom_wallet.json password=test
+{
+  "access_key_id": "jHhL5B33o16R4jQsb8wm9A3RRdS6KrTB5N4bja9Jys904W7xXFNKqem2ACvTRWRYJsZMCUikYFSokN7pPJziWyDi",
+  "secret_access_key": "21bb64fafa32c82417fd8b97ac56cc8a085998a3852632d52fe7042453daa440",
+  "owner_private_key": "10f6f9d7a47bb0bf68363ad8a99fe69f1493f8b6e1665b3e4e83feb2d5c7ee39",
+  "wallet_public_key": "03e38759973a6bb722baabc2dd84036a39f0b2f53d32fec45a4dacde8a50fe4b70",
+  "container_id": "jHhL5B33o16R4jQsb8wm9A3RRdS6KrTB5N4bja9Jys9"
+}
+```
+
+To get credentials from custom wallet, place it in `wallets` dir before start.
+
+### cred
+
+Usage and default parameter values:
+```sh
+make cred [password=""] [contract_password=s3] [wallet=/user_wallet.json]
+```
+The same as `s3cred`, but it doesn't issues s3 credentials.
+
 ## Contributing
 
 Feel free to contribute to this project after reading the [contributing

bin/config.sh

@@ -1,50 +0,0 @@
-#!/usr/bin/env bash
-
-echo "Running bin/config.sh"
-
-# Source env settings
-. .env
-. services/ir/.ir.env
-source bin/helper.sh
-
-# NeoGo binary path.
-NEOGO="${NEOGO:-docker exec morph_chain neo-go}"
-
-# Wallet files to change config value
-WALLET="${WALLET:-services/morph_chain/node-wallet.json}"
-CONFIG_IMG="${CONFIG_IMG:-/wallets/config.yml}"
-
-NETMAP_ADDR=$(bin/resolve.sh netmap.frostfs) || die "Failed to resolve 'netmap.frostfs' domain name"
-
-# FrostFS configuration record: variable type [string|int|etc],
-# key is a string and value is a constant of given type
-TYPE=${1}
-KEY=${2}
-VALUE="${3}"
-
-[ -z "$TYPE" ] && echo "Empty config value type" && exit 1
-[ -z "$KEY" ] && echo "Empty config key" && exit 1
-[ -z "$VALUE" ] && echo "Empty config value" && exit 1
-
-# Internal variables
-if [[ -z "${FROSTFS_NOTARY_DISABLED}" ]]; then
-  ADDR=$(jq -r .accounts[2].address < "${WALLET}" || die "Cannot get address from ${WALLET}")
-else
-  ADDR=$(jq -r .accounts[0].address < "${WALLET}" || die "Cannot get address from ${WALLET}")
-fi
-
-# Change config value in side chain
-echo "Changing ${KEY} configration value to ${VALUE}"
-
-# shellcheck disable=SC2086
-${NEOGO} contract invokefunction \
-	--wallet-config ${CONFIG_IMG} \
-	-a ${ADDR} --force \
-	-r http://morph-chain.${LOCAL_DOMAIN}:30333 \
-	${NETMAP_ADDR} \
-	setConfig bytes:beefcafe \
-	string:${KEY} \
-	${TYPE}:${VALUE} -- ${ADDR} || exit 1
-
-# Update epoch to apply new configuration value
-./bin/tick.sh

bin/passwd.exp

@@ -1,23 +0,0 @@
-#!/usr/bin/expect
-
-set passwd [lindex $argv 0]
-set args [lrange $argv 1 end]
-
-spawn -noecho {*}$args
-expect -re {^.*assword.*$}
-
-if { $passwd == "-"} {
-	send -- "\r"
-} else {
-	send -- "$passwd\r"
-}
-
-expect {
-    "Relay transaction" {
-        send "y\r"
-        exp_continue
-    }
-    EOF
-}
-lassign [wait] pid spawnid os_error_flag value
-exit $value

bin/resolve.sh

@@ -1,22 +0,0 @@
-#!/usr/bin/env bash
-
-# Source env settings
-. .env
-source bin/helper.sh
-
-# NeoGo binary path.
-NEOGO="${NEOGO:-docker exec morph_chain neo-go}"
-# NNS contract script hash
-output=$(curl -s --data '{ "id": 1, "jsonrpc": "2.0", "method": "getcontractstate", "params": [1] }' \
-	"http://morph-chain.${LOCAL_DOMAIN}:30333/") \
-	|| die "Cannot fetch NNS contract state"
-
-NNS_ADDR=$(jq -r '.result.hash' <<< "$output") \
-	|| die "Cannot parse NNS contract hash: $NNS_ADDR"
-
-${NEOGO} contract testinvokefunction \
-	-r "http://morph-chain.${LOCAL_DOMAIN}:30333" \
-	"${NNS_ADDR}" resolve string:"${1}" int:16 \
-	| jq -r '.stack[0].value | if type=="array" then .[0].value else . end' \
-	| base64 -d \
-	|| die "Cannot invoke 'NNS.resolve' $output"

bin/tick.sh

@@ -1,49 +0,0 @@
-#!/usr/bin/env bash
-
-echo "Running bin/tick.sh"
-
-# Source env settings
-. .env
-. services/ir/.ir.env
-source bin/helper.sh
-
-# NeoGo binary path.
-NEOGO="${NEOGO:-docker exec morph_chain neo-go}"
-
-# Wallet files to change config value
-WALLET="${WALLET:-services/morph_chain/node-wallet.json}"
-CONFIG_IMG="${CONFIG_IMG:-/wallets/config.yml}"
-
-# Internal variables
-if [[ -z "${FROSTFS_NOTARY_DISABLED}" ]]; then
-  ADDR=$(jq -r .accounts[2].address < "${WALLET}" || die "Cannot get address from ${WALLET}")
-else
-  ADDR=$(jq -r .accounts[0].address < "${WALLET}" || die "Cannot get address from ${WALLET}")
-fi
-
-# Grep Morph block time
-SIDECHAIN_PROTO="${SIDECHAIN_PROTO:-services/morph_chain/protocol.privnet.yml}"
-BLOCK_DURATION=$(grep SecondsPerBlock < "$SIDECHAIN_PROTO" | awk '{print $2}') \
-	|| die "Cannot fetch block duration"
-NETMAP_ADDR=$(bin/resolve.sh netmap.frostfs) || die "Cannot resolve netmap.frostfs"
-
-# Fetch current epoch value
-EPOCH=$(${NEOGO} contract testinvokefunction \
-	-r "http://morph-chain.${LOCAL_DOMAIN}:30333" "${NETMAP_ADDR}" epoch \
-	| grep 'value' | awk -F'"' '{ print $4 }') \
-	|| die "Cannot fetch epoch from netmap contract"
-
-echo "Updating FrostFS epoch to $((EPOCH+1))"
-
-# shellcheck disable=SC2086
-${NEOGO} contract invokefunction \
-        --wallet-config ${CONFIG_IMG} \
-	-a ${ADDR} --force \
-	-r http://morph-chain.${LOCAL_DOMAIN}:30333 \
-	${NETMAP_ADDR} \
-	newEpoch int:$((EPOCH+1)) -- ${ADDR}:Global \
-        || die "Cannot increment an epoch"
-
-# Wait one Morph block to ensure the transaction broadcasted
-# shellcheck disable=SC2086
-sleep $BLOCK_DURATION

configs/s01-cli.yml

@@ -0,0 +1,4 @@
+wallet: services/storage/wallet01.json
+password: ""
+rpc-endpoint: s01.frostfs.devenv:8080
+endpoint: s01.frostfs.devenv:8081

configs/s02-cli.yml

@@ -0,0 +1,4 @@
+wallet: services/storage/wallet02.json
+password: ""
+rpc-endpoint: s02.frostfs.devenv:8080
+endpoint: s02.frostfs.devenv:8081

configs/s03-cli.yml

@@ -0,0 +1,4 @@
+wallet: services/storage/wallet03.json
+password: ""
+rpc-endpoint: s03.frostfs.devenv:8080
+endpoint: s03.frostfs.devenv:8081

configs/s04-cli.yml

@@ -0,0 +1,4 @@
+wallet: services/storage/wallet04.json
+password: ""
+rpc-endpoint: s04.frostfs.devenv:8080
+endpoint: s04.frostfs.devenv:8081

docs/http_gate.md

@@ -2,7 +2,7 @@
 
 Protocol Gateway to access data in FrostFS using HTTP protocol.
 
-Source code and more information can be found in [project's GitHub repository](https://github.com/TrueCloudLab/frostfs-http-gate)
+Source code and more information can be found in [project's repository](https://git.frostfs.info/TrueCloudLab/frostfs-http-gw)
 
 ## .env settings
 
@@ -22,8 +22,8 @@ Image label prefix to use for containers.
 - Create a new container
 ```
 $ frostfs-cli --rpc-endpoint s01.frostfs.devenv:8080 \
-            --key wallets/wallet.key \
-            container create --basic-acl readonly --await \
+            --wallet wallets/wallet.key \
+            container create --basic-acl private --await \
             --policy "REP 1 SELECT 1 FROM *"
 container ID: 4LfREK1cetL4PUji5fqj9SgRTSmaC5jExEDK9HKCDjdP
 awaiting...
@@ -33,7 +33,7 @@ container has been persisted on sidechain
 - Put an object into the newly created container
 ```
 $ frostfs-cli --rpc-endpoint s01.frostfs.devenv:8080 \
-            --key wallets/wallet.key \
+            --wallet wallets/wallet.key \
             object put --file /tmp/backup.jpeg \
             --cid 4LfREK1cetL4PUji5fqj9SgRTSmaC5jExEDK9HKCDjdP
 [/tmp/backup.jpeg] Object successfully stored

docs/morph.md

@@ -3,52 +3,25 @@ A single-node N3 privnet deployment, running on
 [neo-go](https://github.com/nspcc-dev/neo-go). Represents N3 FrostFS SideChain.
 
 Contracts deployed:
-- Alphabet (AZ) [contract](https://github.com/TrueCloudLab/frostfs-contract/tree/master/alphabet)
-- Audit [contract](https://github.com/TrueCloudLab/frostfs-contract/tree/master/audit)
-- Balance [contract](https://github.com/TrueCloudLab/frostfs-contract/tree/master/balance)
-- Container [contract](https://github.com/TrueCloudLab/frostfs-contract/tree/master/container)
-- Netmap [contract](https://github.com/TrueCloudLab/frostfs-contract/tree/master/netmap)
-- NeoFSID [contract](https://github.com/TrueCloudLab/frostfs-contract/tree/master/neofsid)
-- Proxy [contract](https://github.com/TrueCloudLab/frostfs-contract/tree/master/proxy)
-- Reputation [contract](https://github.com/TrueCloudLab/frostfs-contract/tree/master/reputation)
- 
+- Alphabet (AZ) [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/alphabet)
+- Balance [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/balance)
+- Container [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/container)
+- FrostFS [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/frostfs)
+- FrostFSID [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/frostfsid)
+- NNS [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/nns)
+- Netmap [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/netmap)
+- Policy [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/policy)
+- Processing [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/processing)
+- Proxy [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/proxy)
+
 RPC available at `http://morph-chain.frostfs.devenv:30333`.
 
 ## .env settings
 
-### MORPH_CHAIN_URL
-
-URL to get side chain dump. Used on artifact get stage.
-
-### MORPH_CHAIN_PATH
-
-Path to get side chain dump. If set, overrides `CHAIN_URL`.
-
 ### NEOGO_VERSION
 
 Version of neo-go docker container for side chain deployment.
 
-## Side chain wallets
-
-There is a wallet with GAS that used for contract deployment:
-`wallets/wallet.json`. This wallet has one account with **empty password**.
-
-```
-$ neo-go wallet nep17 balance \
-    -w wallets/wallet.json \
-    -r http://morph-chain.frostfs.devenv:30333
-
-Account NbUgTSFvPmsRxmGeWpuuGeJUoRoi6PErcM
-GAS: GasToken (d2a4cff31913016155e38e474a2c06d08be276cf)
-        Amount : 189826.0515316
-        Updated: 3909
-FROSTFS: FrostFS Balance (69550190e740b93f92dbd5dea52246f550391057)
-        Amount : 50
-        Updated: 3909
-```
-
-This way you can also monitor FrostFS internal balance of your account.
-
 ## FrostFS global config
 
 FrostFS uses global configuration to store epoch duration, maximum object size, 
@@ -57,16 +30,10 @@ netmap contract and managed by Inner Ring (Alphabet) nodes.
 
 To change these parameters use `make update.*` commands. Command down below
 changes epoch duration from 300 blocks (about 300 seconds with 1bps) to 30.
-Script enters passwords automatically with `expect` utility.
 
 ```
 $ make update.epoch_duration val=30
-Changing EpochDuration configration value to 30
-Enter account NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP password > 
-Sent invocation transaction bdc0fa88cd6719ef6df2b9c82de423ddec6141ca24255c2d0072688083b1de9d
-Updating FrostFS epoch to 20
-Enter account NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP password > 
-Sent invocation transaction 12296e1ce24dd6c04edb9c56d0a1d0e26d3226adefb0333c74a28788f44a8d0f
+Waiting for transactions to persist...
 ```
 
 Read more about available configuration in Makefile help.
@@ -76,10 +43,14 @@ $ make help
   ...
   Targets:
   ...
-    update.audit_fee                Update audit fee per result in fixed 12 (make update.audit_fee val=100)
-    update.basic_income_rate        Update basic income rate in fixed 12 (make update.basic_income_rate val=1000)
-    update.container_fee            Update container fee per alphabet node in fixed 12 (make update.container_fee val=500)
-    update.eigen_trust_iterations   Update amount of EigenTrust iterations (make update.eigen_trust_iterations val=2)
-    update.epoch_duration           Update epoch duration in side chain blocks (make update.epoch_duration val=30)
-    update.max_object_size          Update max object size in bytes (make update.max_object_size val=1000)
+    update.audit_fee                     Update audit fee per result in fixed 12 (make update.audit_fee val=100)
+    update.basic_income_rate             Update basic income rate in fixed 12 (make update.basic_income_rate val=1000)
+    update.container_alias_fee           Update container alias fee per alphabet node in fixed 12 (make update.container_alias_fee val=100)
+    update.container_fee                 Update container fee per alphabet node in fixed 12 (make update.container_fee val=500)
+    update.eigen_trust_alpha             Update alpha parameter of EigenTrust algorithm in 0 <= f <= 1.0 (make update.eigen_trust_alpha val=0.2)
+    update.eigen_trust_iterations        Update amount of EigenTrust iterations (make update.eigen_trust_iterations val=2)
+    update.epoch_duration                Update epoch duration in side chain blocks (make update.epoch_duration val=30)
+    update.homomorphic_hashing_disable   Update homomorphic hashing disabled flag (make update.homomorphic_hashing_disable val=true)
+    update.max_object_size               Update max object size in bytes (make update.max_object_size val=1000)
+    update.system_dns                    Update system dns to resolve container names (make update.system_dns val=container)
 ```

docs/notary.md

@@ -7,66 +7,12 @@ create containers, approve balance changes, update network map, tick epochs,
 etc. With notary service, it takes up to seven times fewer transactions 
 to do these operations. Notary service calculates the exact amount of GAS 
 to execute transaction, therefore operations are cheaper (withdraw fee **with**
-notary is less than 0.5 GAS; withdraw fee **without** notary is up to 7.0 GAS). 
+notary is less than 0.5 GAS; withdraw fee **without** notary is up to 7.0 GAS).
 
-By default, main chain service is running without notary service, and side chain
-running with notary service. However, you can change that in configuration.
+Currently, frostfs-dev-env contains single chain (see morph service) and it
+enables notary service from the genesis block.
 
-# Disable notary service in side chain
-
-To disable notary service in side chain do these steps.
-
-1. Update `.env` and choose notary disabled chain dump for side chain.
-
-```
-MORPH_CHAIN_URL="https://github.com/nspcc-dev/neofs-contract/releases/download/v0.9.0/devenv_sidechain_notary_disabled.gz"
-```
-
-Make sure to update chain dump files with `make get` target.
-
-2. Update `service/morph_chain/protocol.privnet.yml` and disable notary settings
-and state root in header.
-   
-```yaml
-ProtocolConfiguration:
-  StateRootInHeader: false
-  P2PSigExtensions: false
-ApplicationConfiguration:
-  P2PNotary:
-    Enabled: false
-```
-
-Chain dump without notary service does not have predefined network map.
-Therefore, you need to wait about 5 minutes until new epoch tick with updated
-network map.
-
-
-3. Enable helper commands
-
-To enable helper commands such as `make tick.epoch` or `make update.epoch_duration`
-make sure to export non-empty `FROSTFS_NOTARY_DISABLED` environment variable. 
-```
-$ export FROSTFS_NOTARY_DISABLED=1
-```
-
-Use `unset` command to return it back.
-```
-$ unset FROSTFS_NOTARY_DISABLED
-```
-
-# Enable notary service in main chain
-
-To enable notary service in main chain do these steps.
-
-1. Update `.env` and choose notary enabled chain dump for main chain.
-
-```
-CHAIN_URL="https://github.com/nspcc-dev/neofs-contract/releases/download/v0.9.0/devenv_mainchain.gz"
-```
-
-Make sure to update chain dump files with `make get` target.
-
-2. Update `service/chain/protocol.privnet.yml` and enable notary settings.
+To enable notary service, use neo-go configuration below.
 
 ```yaml
 ProtocolConfiguration:
@@ -75,7 +21,3 @@ ApplicationConfiguration:
   P2PNotary:
     Enabled: true
 ```
-
-Main chain generates a block once per 15 seconds, so Inner Ring takes about 
-15-30 seconds to make a notary deposit in main chain after startup. Then 
-frostfs-dev-env is ready to work.

docs/rest_gate.md

@@ -2,7 +2,7 @@
 
 REST Gateway to access data in FrostFS using REST.
 
-Source code and more information can be found in [project's GitHub repository](https://github.com/TrueCloudLab/frostfs-rest-gw)
+Source code and more information can be found in [project's repository](https://git.frostfs.info/TrueCloudLab/frostfs-rest-gw)
 
 ## .env settings
 

docs/s3_gate.md

@@ -2,7 +2,7 @@
 
 Protocol Gateway to access data in FrostFS using AWS S3 protocol
 
-Source code and more information can be found in [project's GitHub repository](https://github.com/TrueCloudLab/frostfs-s3-gw)
+Source code and more information can be found in [project's repository](https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw)
 
 ## .env settings
 

frostfs-adm.yml

@@ -1,9 +1,11 @@
 rpc-endpoint: http://morph-chain.frostfs.devenv:30333
+alphabet-wallets: ./services/ir
 network:
   max_object_size: 67108864
   epoch_duration: 240
   basic_income_rate: 100000000
   homomorphic_hash_disabled: false
+  maintenance_mode_allowed: true
   fee:
     audit: 10000
     candidate: 10000000000

frostfs_config.mk

@@ -1,44 +1,40 @@
 # Update epoch duration in side chain blocks (make update.epoch_duration val=30)
 update.epoch_duration:
-	@./bin/config.sh int EpochDuration $(val)
+	@./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config EpochDuration=$(val)
 
 # Update max object size in bytes (make update.max_object_size val=1000)
 update.max_object_size:
-	@./bin/config.sh int MaxObjectSize $(val)
+	@./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config MaxObjectSize=$(val)
 
 # Update audit fee per result in fixed 12 (make update.audit_fee val=100)
 update.audit_fee:
-	@./bin/config.sh int AuditFee $(val)
+	@./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config AuditFee=$(val)
 
 # Update container fee per alphabet node in fixed 12 (make update.container_fee val=500)
 update.container_fee:
-	@./bin/config.sh int ContainerFee $(val)
+	@./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config ContainerFee=$(val)
 
 # Update container alias fee per alphabet node in fixed 12 (make update.container_alias_fee val=100)
 update.container_alias_fee:
-	@./bin/config.sh int ContainerAliasFee $(val)
+	@./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config ContainerAliasFee=$(val)
 
 # Update amount of EigenTrust iterations (make update.eigen_trust_iterations val=2)
 update.eigen_trust_iterations:
-	@./bin/config.sh int EigenTrustIterations $(val)
-
+	@./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config EigenTrustIterations=$(val)
 
 # Update system dns to resolve container names (make update.system_dns val=container)
 update.system_dns:
-	@./bin/config.sh string SystemDNS $(val)
+	@./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config SystemDNS=$(val) --force
 
 # Update alpha parameter of EigenTrust algorithm in 0 <= f <= 1.0 (make update.eigen_trust_alpha val=0.2)
 update.eigen_trust_alpha:
-	@./bin/config.sh string EigenTrustAlpha $(val)
+	@./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config EigenTrustAlpha=$(val)
 
 # Update basic income rate in fixed 12 (make update.basic_income_rate val=1000)
 update.basic_income_rate:
-	@./bin/config.sh int BasicIncomeRate $(val)
+	@./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config BasicIncomeRate=$(val)
 
 # Update homomorphic hashing disabled flag (make update.homomorphic_hashing_disable val=true)
 update.homomorphic_hashing_disable:
-	@./bin/config.sh bool HomomorphicHashingDisabled $(val)
+	@./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config HomomorphicHashingDisabled=$(val)
 
-# Tick new epoch in side chain
-tick.epoch:
-	@./bin/tick.sh

services/basenet/docker-compose.yml

@@ -1,6 +1,5 @@
 ---
 
-version: "2.4"
 services:
 
   basenet:

services/grafana/.hosts

@@ -0,0 +1,2 @@
+IPV4_PREFIX.122 grafana.LOCAL_DOMAIN
+IPV4_PREFIX.123 loki.LOCAL_DOMAIN

services/grafana/docker-compose.yml

@@ -0,0 +1,31 @@
+services:
+  grafana:
+    image: ${GRAFANA_IMAGE}:${GRAFANA_VERSION}
+    domainname: ${LOCAL_DOMAIN}
+    hostname: grafana
+    container_name: grafana
+    restart: on-failure
+    networks:
+      grafana_int:
+      internet:
+        ipv4_address: ${IPV4_PREFIX}.122
+    volumes:
+      - ./../../vendor/hosts:/etc/hosts
+      - ./grafana.ini:/etc/grafana/grafana.ini
+      - ./provisioning:/etc/grafana/provisioning
+    stop_signal: SIGKILL
+    env_file: [ ".env", ".int_test.env" ]
+
+  loki:
+    image: ${LOKI_IMAGE}:${LOKI_VERSION}
+    command: -config.file=/etc/loki/local-config.yaml
+    networks:
+      grafana_int:
+      internet:
+        ipv4_address: ${IPV4_PREFIX}.123
+
+networks:
+  grafana_int:
+  internet:
+    external: true
+    name: basenet_internet

services/grafana/grafana.ini

@@ -0,0 +1,7 @@
+[auth.anonymous]
+enabled = true
+org_name = Main Org.
+org_role = Editor
+
+[dashboards]
+default_home_dashboard_path= /etc/grafana/provisioning/dashboards/overview.json

services/grafana/provisioning/datasources/datasource.yml

@@ -0,0 +1,13 @@
+apiVersion: 1
+
+datasources:
+- name: Prometheus
+  type: prometheus
+  access: proxy
+  orgId: 1
+  url: http://prometheus:9090
+- name: Loki
+  type: loki
+  access: proxy
+  orgId: 1
+  url: http://loki:3100

services/http_gate/docker-compose.yml

@@ -1,6 +1,5 @@
 ---
 
-version: "2.4"
 services:
   http_gate:
     image: ${HTTP_GW_IMAGE}:${HTTP_GW_VERSION}
@@ -21,6 +20,7 @@ services:
     command: [ "frostfs-http-gw", "--config", "/etc/frostfs/http/config.yml" ]
     environment:
       - HTTP_GW_RPC_ENDPOINT=http://morph-chain.${LOCAL_DOMAIN}:30333
+      - HTTP_GW_TREE_SERVICE=s01.${LOCAL_DOMAIN}:8080
       - HTTP_GW_PEERS_0_ADDRESS=s01.${LOCAL_DOMAIN}:8080
       - HTTP_GW_PEERS_0_WEIGHT=0.2
       - HTTP_GW_PEERS_1_ADDRESS=s02.${LOCAL_DOMAIN}:8080

services/ir/.ir.env

@@ -1,3 +1 @@
-FROSTFS_IR_CONTRACTS_FROSTFSID=1943e9bb78a0fe2fe0c95fd2677eec2da6aa4aa5
-
 FROSTFS_IR_CONTROL_GRPC_ENDPOINT=127.0.0.1:16512

services/ir/artifacts.mk

@@ -25,7 +25,6 @@ endif
 # Download FrostFS CLI 
 .ONESHELL:
 get.cli: FROSTFS_CLI_FILE=./vendor/frostfs-cli
-get.cli: FROSTFS_CLI_ARCHIVE_FILE=${FROSTFS_CLI_FILE}.tar.gz
 get.cli: FROSTFS_CLI_PATH?=
 get.cli:
 	@mkdir -p ./vendor
@@ -34,10 +33,8 @@ ifeq (${FROSTFS_CLI_PATH},)
 	@echo "⇒ Download FrostFS CLI binary from ${FROSTFS_CLI_URL}"
 	@curl \
 		-ksSL "${FROSTFS_CLI_URL}" \
-		-o ${FROSTFS_CLI_ARCHIVE_FILE} 
-	@tar -xvf ${FROSTFS_CLI_ARCHIVE_FILE} -C ./vendor | xargs -I {} \
-		mv ./vendor/{} ${FROSTFS_CLI_FILE}
-	@rm ${FROSTFS_CLI_ARCHIVE_FILE}
+		-o ${FROSTFS_CLI_FILE} 
+	@chmod +x ${FROSTFS_CLI_FILE}
 else
 	@echo "⇒ Copy local binary from ${FROSTFS_CLI_PATH}"
 	@cp ${FROSTFS_CLI_PATH} ${FROSTFS_CLI_FILE}

services/ir/docker-compose.yml

@@ -1,6 +1,5 @@
 ---
 
-version: "2.4"
 services:
 
   ir01:
@@ -13,19 +12,19 @@ services:
       ir_int:
       internet:
         ipv4_address: ${IPV4_PREFIX}.61
-    stop_signal: SIGKILL
+    stop_signal: SIGTERM
+    stop_grace_period: 15s
     volumes:
       - ./az.json:/wallet.json
       - ./az.key:/wallet01.key
       - ./../../vendor/hosts:/etc/hosts
       - ./../../vendor/locode_db:/locode/db
       - ./../../vendor/frostfs-cli:/frostfs-cli
-      - ./healthcheck.sh:/healthcheck.sh
       - ./cfg:/etc/frostfs/ir
     env_file: [ ".env", ".ir.env", ".int_test.env" ]
     command: [ "frostfs-ir", "--config", "/etc/frostfs/ir/config.yml" ]
     healthcheck:
-      test: ["CMD-SHELL", "/healthcheck.sh"]
+      test: ["CMD-SHELL", "/frostfs-cli control ir healthcheck -q --wallet /wallet01.key  --endpoint \"$$FROSTFS_IR_CONTROL_GRPC_ENDPOINT\""]
       interval: 2s
       timeout: 1s
       retries: 5

services/ir/healthcheck.sh

@@ -1,6 +0,0 @@
-#!/bin/sh
-
-/frostfs-cli control healthcheck \
-	--endpoint "$FROSTFS_IR_CONTROL_GRPC_ENDPOINT" \
-	--wallet /wallet01.key --ir |
-	grep "Health status: READY"

services/ir/prepare.mk

@@ -1,4 +0,0 @@
-# Deposit GAS from default wallet to FrostFS privnet contract 
-prepare.ir:
-	@./bin/config.sh int ContainerFee 0
-	@./bin/config.sh int ContainerAliasFee 0

services/jaeger/docker-compose.yml

@@ -1,4 +1,3 @@
-version: '2.4'
 services:
   jaeger:
     image: ${JAEGER_IMAGE}:${JAEGER_VERSION}
@@ -20,7 +19,10 @@ services:
     env_file: [ ".env", ".jaeger.env", ".int_test.env" ]
     environment:
       - COLLECTOR_OTLP_ENABLED=true
-      - MEMORY_MAX_TRACES=100000
+      - SPAN_STORAGE_TYPE=badger
+      - BADGER_EPHEMERAL=false
+      - BADGER_DIRECTORY_VALUE=/badger/data
+      - BADGER_DIRECTORY_KEY=/badger/key
 
 networks:
   jaeger_int:

services/morph_chain/artifacts.mk

@@ -20,15 +20,12 @@ endif
 
 # Download FrostFS ADM tool 
 get.adm: FROSTFS_ADM_DEST=./vendor/frostfs-adm
-get.adm: FROSTFS_ADM_ARCHIVE=frostfs-adm.tar.gz
 get.adm:
 
 ifeq (${FROSTFS_ADM_PATH},)
 	@echo "⇒ Download FrostFS ADM binary from ${FROSTFS_ADM_URL}"
-	@curl -skSL ${FROSTFS_ADM_URL} -o ${FROSTFS_ADM_ARCHIVE}
-	@tar -xvf ${FROSTFS_ADM_ARCHIVE} -C ./vendor | xargs -I {} \
-		mv ./vendor/{} ${FROSTFS_ADM_DEST}
-	@rm ${FROSTFS_ADM_ARCHIVE}
+	@curl -skSL ${FROSTFS_ADM_URL} -o ${FROSTFS_ADM_DEST}
+	@chmod +x ${FROSTFS_ADM_DEST}
 else
 	@echo "⇒ Copy frostfs-adm binary from ${FROSTFS_ADM_PATH}"
 	@cp ${FROSTFS_ADM_PATH} ${FROSTFS_ADM_DEST}

services/morph_chain/docker-compose.yml

@@ -1,6 +1,5 @@
 ---
 
-version: "2.4"
 services:
   frostfs_morph_chain:
     image: ${NEOGO_IMAGE}:${NEOGO_VERSION}
@@ -20,9 +19,18 @@ services:
     - ./config.yml:/wallets/config.yml
     - ./../../vendor/hosts:/etc/hosts
     - ./../../wallets/wallet.json:/wallets/wallet.json
+    - ./../s3_gate/wallet.json:/wallets/s3-wallet.json
+    - ./../storage/wallet01.json:/wallets/storage/wallet01.json
+    - ./../storage/wallet02.json:/wallets/storage/wallet02.json
+    - ./../storage/wallet03.json:/wallets/storage/wallet03.json
+    - ./../storage/wallet04.json:/wallets/storage/wallet04.json
+    - chains:/chains
 
 networks:
   chain_int:
   internet:
     external: true
     name: basenet_internet
+
+volumes:
+  chains:

services/morph_chain/protocol.privnet.yml

@@ -1,50 +1,57 @@
 ProtocolConfiguration:
   Magic: 15405
   MaxTraceableBlocks: 200000
-  SecondsPerBlock: 1
+  TimePerBlock: 1s
   MemPoolSize: 50000
   StandbyCommittee:
     - 02b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc2
   ValidatorsCount: 1
   SeedList:
     - 172.200.0.1:20333
-  VerifyBlocks: true
   VerifyTransactions: true
   StateRootInHeader: true
   P2PSigExtensions: true
+  Hardforks: {}
 
 ApplicationConfiguration:
+  SkipBlockVerification: false
   DBConfiguration:
     Type: "boltdb"
     BoltDBOptions:
-      FilePath: "./db/morph.bolt"
-  NodePort: 20333
+      FilePath: "/chains/morph.bolt"
+  P2P:
+    Addresses:
+      - ":20333"
+    DialTimeout: 3s
+    ProtoTickInterval: 2s
+    PingInterval: 30s
+    PingTimeout: 90s
+    MaxPeers: 10
+    AttemptConnPeers: 5
+    MinPeers: 0
   Relay: true
-  DialTimeout: 3
-  ProtoTickInterval: 2
-  PingInterval: 30
-  PingTimeout: 90
-  MaxPeers: 10
-  AttemptConnPeers: 5
-  MinPeers: 0
+  Consensus:
+    Enabled: true
+    UnlockWallet:
+      Path: "./wallets/node-wallet.json"
+      Password: "one"
   RPC:
-    Address: 192.168.130.90
+    Addresses:
+      - ":30333"
     Enabled: true
     SessionEnabled: true
     EnableCORSWorkaround: false
     MaxGasInvoke: 100
-    Port: 30333
   P2PNotary:
     Enabled: true
     UnlockWallet:
       Path: "./wallets/node-wallet.json"
       Password: "one"
   Prometheus:
+    Addresses:
+      - ":20001"
     Enabled: true
-    Port: 20001
   Pprof:
+    Addresses:
+      - ":20011"
     Enabled: true
-    Port: 20011
-  UnlockWallet:
-    Path: "./wallets/node-wallet.json"
-    Password: "one"

services/nats/.hosts

@@ -1 +0,0 @@
-IPV4_PREFIX.101 nats.LOCAL_DOMAIN

services/nats/artifacts.mk

@@ -1,7 +0,0 @@
-# Create new TLS certs for NATS server and clients
-
-NATS_DIR=$(abspath services/nats)
-
-get.nats:
-	@echo "⇒ Creating certs for NATS server and clients"
-	${NATS_DIR}/generate_cert.sh ${LOCAL_DOMAIN} > /dev/null

services/nats/docker-compose.yml

@@ -1,31 +0,0 @@
----
-
-version: "2.4"
-services:
-  nats:
-    image: ${NATS_IMAGE}:${NATS_VERSION}
-    domainname: ${LOCAL_DOMAIN}
-    hostname: nats
-    container_name: nats
-    restart: on-failure
-    dns: 
-      - ${IPV4_PREFIX}.101
-    networks:
-      nats_int:
-      internet:
-        ipv4_address: ${IPV4_PREFIX}.101
-    volumes:
-      - ./../../vendor/hosts:/etc/hosts
-      - ./nats.conf:/etc/nats/frostfs-nats-server.conf
-      - ./server-cert.pem:/certs/server-cert.pem
-      - ./server-key.pem:/certs/server-key.pem
-      - ./ca-cert.pem:/certs/ca-cert.pem
-    stop_signal: SIGKILL
-    env_file: [ ".env", ".int_test.env" ]
-    command: ["-c", "/etc/nats/frostfs-nats-server.conf"]
-
-networks:
-  nats_int:
-  internet:
-    external: true
-    name: basenet_internet

services/nats/generate_cert.sh

@@ -1,49 +0,0 @@
-#!/bin/bash
-
-source bin/helper.sh
-
-WORKDIR=$(dirname "$0")
-LOCAL_DOMAIN=$1
-
-CA_KEY=$WORKDIR/ca-key.pem
-CA_CRT=$WORKDIR/ca-cert.pem
-
-SRV_KEY=$WORKDIR/server-key.pem
-SRV_REQ=$WORKDIR/server-req.csr
-SRV_CRT=$WORKDIR/server-cert.pem
-
-CLI_KEY=$WORKDIR/client-key.pem
-CLI_REQ=$WORKDIR/client-req.csr
-CLI_CRT=$WORKDIR/client-cert.pem
-
-SUBJ="/O=NSPCC"
-
-if [[ ! -f $CA_KEY || ! -f $CA_CRT ]]; then
-	openssl req -newkey rsa:4096 -x509 -days 365 -nodes -keyout $CA_KEY -out $CA_CRT -subj $SUBJ 2>&1 ||
-		die "CA certificate was not created"
-fi
-
-if [[ ! -f $SRV_KEY || ! -f $SRV_CRT ]]; then
-	openssl req -newkey rsa:4096 -nodes -keyout $SRV_KEY -out $SRV_REQ -subj $SUBJ 2>&1 ||
-		die "Server certificate was not created"
-
-	openssl x509 -req -days 365 -set_serial 01 -in $SRV_REQ -out $SRV_CRT -CA $CA_CRT -CAkey $CA_KEY \
- 		-extensions san -extfile <(printf "[san]\nsubjectAltName=DNS:nats.$LOCAL_DOMAIN") 2>&1 || {
-		rm $SRV_REQ
-		die "Server certificate was not signed by CA"
-	}
-
-	rm $SRV_REQ
-fi
-
-if [[ ! -f $CLI_KEY || ! -f $CLI_CRT ]]; then
-	openssl req -newkey rsa:4096 -nodes -keyout $CLI_KEY -out $CLI_REQ -subj $SUBJ 2>&1 ||
-		die "Client certificate was not created"
-
-	openssl x509 -req -days 365 -set_serial 01 -in $CLI_REQ -out $CLI_CRT -CA $CA_CRT -CAkey $CA_KEY 2>&1 || {
-		rm $CLI_REQ
-		die "Client certificate was not signed by CA"
-	}
-
-	rm $CLI_REQ
-fi

services/nats/nats.conf

@@ -1,15 +0,0 @@
-port: 4222
-monitor_port: 8222
-
-jetstream {
-   store_dir=nats
-   max_memory_store: 1GB
-   max_file_store: 2GB
-}
-
-tls {
-  cert_file: /certs/server-cert.pem
-  key_file:  /certs/server-key.pem
-  ca_file:   /certs/ca-cert.pem 
-  verify:    true
-}

services/prometheus/docker-compose.yml

@@ -1,4 +1,3 @@
-version: '2.4'
 services:
   prometheus:
     image: ${PROMETHEUS_IMAGE}:${PROMETHEUS_VERSION}
@@ -15,8 +14,6 @@ services:
       - ./prometheus.yml:/etc/prometheus/prometheus.yml
     command:
       - --config.file=/etc/prometheus/prometheus.yml
-    ports:
-      - '9090:9090'
     stop_signal: SIGKILL
     env_file: [ ".env", ".prometheus.env", ".int_test.env" ]
 
@@ -24,4 +21,4 @@ networks:
   prometheus_int:
   internet:
     external: true
-    name: basenet_internet
+    name: basenet_internet

services/prometheus/prometheus.yml

@@ -16,4 +16,7 @@ scrape_configs:
       - targets: ['s3.frostfs.devenv:9090']
   - job_name: 'neo-go'
     static_configs:
-      - targets: ['morph-chain.frostfs.devenv:20001']
+      - targets: ['morph-chain.frostfs.devenv:20001']
+  - job_name: 'inner-ring'
+    static_configs:
+      - targets: ['ir01.frostfs.devenv:9090']

services/rest_gate/.hosts

@@ -1 +0,0 @@
-IPV4_PREFIX.83 rest.LOCAL_DOMAIN

services/rest_gate/cfg/config.yml

@@ -1,12 +0,0 @@
-prometheus:
-  enabled: true
-  address: :9090
-
-server:
-  # The IP and port to listen on.
-  listen-address: 0.0.0.0:8090
-
-# Wallet settings
-wallet:
-  path: /wallet.json # Path to wallet
-  passphrase: one # Password to decrypt wallet

services/rest_gate/docker-compose.yml

@@ -1,32 +0,0 @@
----
-
-version: "2.4"
-services:
-  rest_gate:
-    image: ${REST_GW_IMAGE}:${REST_GW_VERSION}
-    domainname: ${LOCAL_DOMAIN}
-    hostname: rest
-    container_name: rest_gate
-    restart: on-failure
-    networks:
-      rest_gate_int:
-      internet:
-        ipv4_address: ${IPV4_PREFIX}.83
-    volumes:
-      - ./wallet.json:/wallet.json
-      - ./../../vendor/hosts:/etc/hosts
-      - ./cfg:/etc/frostfs/rest
-    stop_signal: SIGKILL
-    env_file: [ ".env", ".int_test.env" ]
-    command: [ "frostfs-rest-gw", "--config", "/etc/frostfs/rest/config.yml" ]
-    environment:
-      - REST_GW_POOL_PEERS_0_ADDRESS=s01.${LOCAL_DOMAIN}:8080
-      - REST_GW_POOL_PEERS_1_ADDRESS=s02.${LOCAL_DOMAIN}:8080
-      - REST_GW_POOL_PEERS_2_ADDRESS=s03.${LOCAL_DOMAIN}:8080
-      - REST_GW_POOL_PEERS_3_ADDRESS=s04.${LOCAL_DOMAIN}:8080
-
-networks:
-  rest_gate_int:
-  internet:
-    external: true
-    name: basenet_internet

services/s3_gate/cfg/config.yml

@@ -33,3 +33,17 @@ server:
 wallet:
   path: /wallet.json # Path to wallet
   passphrase: "s3" # Passphrase to decrypt wallet
+
+features:
+  md5:
+    enabled: true
+
+control:
+  grpc:
+    endpoint: localhost:16515
+
+frostfsid:
+  enabled: false
+
+policy:
+  enabled: false

services/s3_gate/docker-compose.yml

@@ -1,6 +1,5 @@
 ---
 
-version: "2.4"
 services:
   s3_gate:
     image: ${S3_GW_IMAGE}:${S3_GW_VERSION}
@@ -13,19 +12,26 @@ services:
       internet:
         ipv4_address: ${IPV4_PREFIX}.82
     volumes:
+      # Gate wallet
       - ./wallet.json:/wallet.json
+      # Custom user wallets
+      - ./wallets:/wallets
+      # Default user wallet
+      - ./../../wallets/wallet.json:/wallets/wallet.json
       - ./tls.key:/tls.key
       - ./tls.crt:/tls.crt
       - ./../../vendor/hosts:/etc/hosts
       - ./cfg:/etc/frostfs/s3
-    stop_signal: SIGKILL
+      - ./issue-creds.sh:/usr/bin/issue-creds.sh
+    stop_signal: SIGTERM
+    stop_grace_period: 15s
     env_file: [ ".env", ".s3.env", ".int_test.env" ]
     command: [ "frostfs-s3-gw", "--config", "/etc/frostfs/s3/config.yml" ]
     environment:
       - S3_GW_RPC_ENDPOINT=http://morph-chain.${LOCAL_DOMAIN}:30333
       - S3_GW_SERVER_0_ADDRESS=s3.${LOCAL_DOMAIN}:8080
       - S3_GW_LISTEN_DOMAINS=s3.${LOCAL_DOMAIN}
-      - S3_GW_TREE_SERVICE=s01.${LOCAL_DOMAIN}:8080
+      - S3_GW_TREE_SERVICE=s01.${LOCAL_DOMAIN}:8080 s02.${LOCAL_DOMAIN}:8080 s03.${LOCAL_DOMAIN}:8080 s04.${LOCAL_DOMAIN}:8080
       - S3_GW_PEERS_0_ADDRESS=s01.${LOCAL_DOMAIN}:8080
       - S3_GW_PEERS_0_WEIGHT=0.2
       - S3_GW_PEERS_1_ADDRESS=s02.${LOCAL_DOMAIN}:8080
@@ -34,6 +40,8 @@ services:
       - S3_GW_PEERS_2_WEIGHT=0.2
       - S3_GW_PEERS_3_ADDRESS=s04.${LOCAL_DOMAIN}:8080
       - S3_GW_PEERS_3_WEIGHT=0.2
+      - AUTHMATE_WALLET_PASSPHRASE=
+      - AUTHMATE_WALLET_CONTRACT_PASSPHRASE=s3
 
 networks:
   s3_gate_int:

services/s3_gate/issue-creds.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+initUser() {
+  /bin/frostfs-s3-authmate register-user \
+    --wallet $WALLET_PATH \
+    --rpc-endpoint http://morph-chain.frostfs.devenv:30333 \
+    --username $USERNAME \
+    --contract-wallet /wallet.json 1> /dev/null && touch $WALLET_CACHE/$USERNAME
+}
+
+issueCreds() {
+  /bin/frostfs-s3-authmate issue-secret \
+    --wallet $WALLET_PATH \
+    --peer s01.frostfs.devenv:8080 \
+    --gate-public-key $S3_GATE_PUBLIC_KEY \
+    --container-placement-policy "REP 3"
+}
+
+set -e
+
+WALLET_PATH=/wallets/$2
+if [[ -z "$2" ]]; then
+  WALLET_PATH=/wallets/wallet.json
+fi
+
+S3_GATE_PUBLIC_KEY=$3
+if [[ -z "$3" ]]; then
+  S3_GATE_PUBLIC_KEY=0313b1ac3a8076e155a7e797b24f0b650cccad5941ea59d7cfd51a024a8b2a06bf
+fi 
+
+WALLET_CACHE=/data/wallets
+mkdir -p $WALLET_CACHE
+
+USERNAME=$(echo $WALLET_PATH | md5sum | cut -d' ' -f1)
+if [ ! -e $WALLET_CACHE/$USERNAME ]; then
+  initUser
+fi
+
+if [ $1 == "s3" ]; then
+  issueCreds
+fi

services/s3_gate/prepare.mk

@@ -0,0 +1,14 @@
+.PHONY: s3cred register
+
+password?=
+contract_password?=s3
+gate_public_key?=
+wallet?=
+
+# Register wallet & generate S3 credentials
+s3cred:
+	@docker exec -e AUTHMATE_WALLET_PASSPHRASE="$(password)" -e AUTHMATE_WALLET_CONTRACT_PASSPHRASE="$(contract_password)" s3_gate /usr/bin/issue-creds.sh s3 "$(wallet)" "$(gate_public_key)"
+
+# Only registers user wallet
+register:
+	@docker exec -e AUTHMATE_WALLET_PASSPHRASE="$(password)" -e AUTHMATE_WALLET_CONTRACT_PASSPHRASE="$(contract_password)" s3_gate /usr/bin/issue-creds.sh native "$(wallet)"

services/s3_lifecycler/.hosts

@@ -0,0 +1 @@
+IPV4_PREFIX.84 lifecycler.LOCAL_DOMAIN

services/s3_lifecycler/cfg/config.yml

@@ -0,0 +1,42 @@
+logger:
+  level: debug
+
+prometheus:
+  enabled: true
+  address: :9090
+  
+lifecycle:
+  job_fetcher_buffer: 1000
+  executor_pool_size: 100  
+
+frostfs:
+  stream_timeout: 10s
+  connect_timeout: 10s
+  healthcheck_timeout: 15s
+  rebalance_interval: 60s
+  pool_error_threshold: 100
+  tree_pool_max_attempts: 4
+
+credential:
+  use: wallets
+  source:
+    wallets:
+      - path: /wallet.json
+        address: NTt1rxvmEDxEuuogLxs2xgxA71qhVaUcN7
+        passphrase: "cycle"
+      - path: /user-wallet.json
+        address: NbUgTSFvPmsRxmGeWpuuGeJUoRoi6PErcM
+        passphrase: ""
+        
+morph:
+  reconnect_clients_interval: 30s
+  dial_timeout: 5s
+  contract:
+    netmap: netmap.frostfs
+    frostfsid: frostfsid.frostfs
+    container: container.frostfs
+
+# Wallet configuration
+wallet:
+  path: /wallet.json # Path to wallet
+  passphrase: "cycle" # Passphrase to decrypt wallet

services/s3_lifecycler/docker-compose.yml

@@ -0,0 +1,37 @@
+---
+
+services:
+  s3_lifecycler:
+    image: ${S3_LIFECYCLER_IMAGE}:${S3_LIFECYCLER_VERSION}
+    domainname: ${LOCAL_DOMAIN}
+    hostname: s3_lifecycler
+    container_name: s3_lifecycler
+    restart: on-failure
+    networks:
+      s3_lifecycler_int:
+      internet:
+        ipv4_address: ${IPV4_PREFIX}.84
+    volumes:
+      - ./wallet.json:/wallet.json
+      - ./../../vendor/hosts:/etc/hosts
+      - ./cfg:/etc/frostfs/s3-lifecycler
+      - ./../../wallets/wallet.json:/user-wallet.json
+    stop_signal: SIGKILL
+    env_file: [ ".env", ".int_test.env" ]
+    command: [ "frostfs-s3-lifecycler", "--config", "/etc/frostfs/s3-lifecycler/config.yml" ]
+    environment:
+      - S3_LIFECYCLER_MORPH_RPC_ENDPOINT_0_ADDRESS=ws://morph-chain:30333/ws
+      - S3_LIFECYCLER_FROSTFS_PEERS_0_ADDRESS=s01.${LOCAL_DOMAIN}:8080
+      - S3_LIFECYCLER_FROSTFS_PEERS_0_WEIGHT=0.2
+      - S3_LIFECYCLER_FROSTFS_PEERS_1_ADDRESS=s02.${LOCAL_DOMAIN}:8080
+      - S3_LIFECYCLER_FROSTFS_PEERS_1_WEIGHT=0.2
+      - S3_LIFECYCLER_FROSTFS_PEERS_2_ADDRESS=s03.${LOCAL_DOMAIN}:8080
+      - S3_LIFECYCLER_FROSTFS_PEERS_2_WEIGHT=0.2
+      - S3_LIFECYCLER_FROSTFS_PEERS_3_ADDRESS=s04.${LOCAL_DOMAIN}:8080
+      - S3_LIFECYCLER_FROSTFS_PEERS_3_WEIGHT=0.2
+
+networks:
+  s3_lifecycler_int:
+  internet:
+    external: true
+    name: basenet_internet

services/s3_lifecycler/wallet.json

@@ -1,12 +1,12 @@
 {
-  "version": "3.0",
+  "version": "1.0",
   "accounts": [
     {
-      "address": "NPFCqWHfi9ixCJRu7DABRbVfXRbkSEr9Vo",
-      "key": "6PYTAGjdaeicUDPqGv9mmgwb9kTwimWJJmmfNqJSDGH9qM79zSRcL9oHiB",
-      "label": "REST Gateway",
+      "address": "NTt1rxvmEDxEuuogLxs2xgxA71qhVaUcN7",
+      "key": "6PYR3XurAyTzVeDG5WV2Z8vnGdySw3mTLuKjr6Nwo7tae64SJ7XjZSMMPQ",
+      "label": "lifecycler",
       "contract": {
-        "script": "DCECcuPzZCZ2VyDsm2jKEOMnU6xEWO2bF1dvOvBWTDFYB1ZBVuezJw==",
+        "script": "DCED9z0M+WSGfXZGxYLj1yYwmgxJXE/kNA4+oWNi0q1uKCdBVuezJw==",
         "parameters": [
           {
             "name": "parameter0",

services/storage/cfg/config.yml

@@ -1,6 +1,11 @@
 # Logger section
 logger:
   level: debug # Minimum enabled logging level
+  loki:
+    enabled: true
+    endpoint: "loki.frostfs.devenv:3100/api/prom/push"
+    max_batch_delay: 1s
+    max_batch_size: 200
 
 # Profiler section
 pprof:
@@ -14,6 +19,12 @@ prometheus:
   address: :9090  # Server address
   shutdown_timeout: 15s  # Timeout for metrics HTTP server graceful shutdown
 
+# Application tracing section
+tracing:
+  enabled: true
+  exporter: otlp_grpc
+  endpoint: "jaeger.frostfs.devenv:4317"
+
 # Morph section
 morph:
   dial_timeout: 30s  # Timeout for side chain NEO RPC client connection
@@ -21,18 +32,6 @@ morph:
     - address: ws://morph-chain:30333/ws
       priority: 1
 
-# Common storage node settings
-node:
-  attribute_0: "User-Agent:FrostFS/0.34"
-  notification:
-    enabled: true  # Turn on object notification service
-    endpoint: "tls://nats.frostfs.devenv:4222"  # Notification server endpoint
-    timeout: "6s"  # Timeout for object notification client connection
-    default_topic: "test"  # Default topic for object notifications if not found in object's meta
-    certificate: "/etc/frostfs-node/nats.tls.cert"  # Path to TLS certificate
-    key: "/etc/frostfs-node/nats.tls.key"  # Path to TLS key
-    ca: "/etc/frostfs-node/nats.ca.crt"  # Path to optional CA certificate
-
 # Tree section
 tree:
   enabled: true
@@ -42,7 +41,7 @@ storage:
   shard:
     0:
       writecache:
-        enabled: false
+        enabled: true
         path: /storage/wc0  # Write-cache root directory
 
       metabase:
@@ -62,7 +61,7 @@ storage:
 
     1:
       writecache:
-        enabled: false
+        enabled: true
         path: /storage/wc1  # Write-cache root directory
 
       metabase:

services/storage/docker-compose.yml

@@ -1,6 +1,5 @@
 ---
 
-version: "2.4"
 services:
   storage01:
     image: ${NODE_IMAGE}:${NODE_VERSION}
@@ -18,13 +17,10 @@ services:
       - storage_s01:/storage
       - ./../../vendor/frostfs-cli:/frostfs-cli
       - ./cli-cfg.yml:/cli-cfg.yml
-      - ./healthcheck.sh:/healthcheck.sh
       - ./s04tls.crt:/etc/ssl/certs/s04tls.crt
-      - ../nats/client-cert.pem:/etc/frostfs-node/nats.tls.cert
-      - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key
-      - ../nats/ca-cert.pem:/etc/frostfs-node/nats.ca.crt
       - ./cfg:/etc/frostfs/storage
-    stop_signal: SIGKILL
+    stop_signal: SIGTERM
+    stop_grace_period: 15s
     env_file: [ ".env", ".storage.env", ".int_test.env" ]
     command: [ "frostfs-node", "--config", "/etc/frostfs/storage/config.yml" ]
     environment:
@@ -33,10 +29,11 @@ services:
       - FROSTFS_NODE_ADDRESSES=s01.${LOCAL_DOMAIN}:8080
       - FROSTFS_GRPC_0_ENDPOINT=s01.${LOCAL_DOMAIN}:8080
       - FROSTFS_CONTROL_GRPC_ENDPOINT=s01.${LOCAL_DOMAIN}:8081
+      - FROSTFS_NODE_ATTRIBUTE_0=User-Agent:FrostFS/${NODE_VERSION}
       - FROSTFS_NODE_ATTRIBUTE_1=UN-LOCODE:RU MOW
       - FROSTFS_NODE_ATTRIBUTE_2=Price:22
     healthcheck:
-      test: ["CMD-SHELL", "/healthcheck.sh"]
+      test: ["CMD-SHELL", "/frostfs-cli control healthcheck -q -c /cli-cfg.yml --endpoint \"$$FROSTFS_CONTROL_GRPC_ENDPOINT\""]
       interval: 2s
       timeout: 1s
       retries: 5
@@ -58,13 +55,10 @@ services:
       - storage_s02:/storage
       - ./../../vendor/frostfs-cli:/frostfs-cli
       - ./cli-cfg.yml:/cli-cfg.yml
-      - ./healthcheck.sh:/healthcheck.sh
       - ./s04tls.crt:/etc/ssl/certs/s04tls.crt
-      - ../nats/client-cert.pem:/etc/frostfs-node/nats.tls.cert
-      - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key
-      - ../nats/ca-cert.pem:/etc/frostfs-node/nats.ca.crt
       - ./cfg:/etc/frostfs/storage
-    stop_signal: SIGKILL
+    stop_signal: SIGTERM
+    stop_grace_period: 15s
     env_file: [ ".env", ".storage.env", ".int_test.env" ]
     command: [ "frostfs-node", "--config", "/etc/frostfs/storage/config.yml" ]
     environment:
@@ -73,10 +67,11 @@ services:
       - FROSTFS_NODE_ADDRESSES=s02.${LOCAL_DOMAIN}:8080
       - FROSTFS_GRPC_0_ENDPOINT=s02.${LOCAL_DOMAIN}:8080
       - FROSTFS_CONTROL_GRPC_ENDPOINT=s02.${LOCAL_DOMAIN}:8081
+      - FROSTFS_NODE_ATTRIBUTE_0=User-Agent:FrostFS/${NODE_VERSION}
       - FROSTFS_NODE_ATTRIBUTE_1=UN-LOCODE:RU LED
       - FROSTFS_NODE_ATTRIBUTE_2=Price:33
     healthcheck:
-      test: ["CMD-SHELL", "/healthcheck.sh"]
+      test: ["CMD-SHELL", "/frostfs-cli control healthcheck -q -c /cli-cfg.yml --endpoint \"$$FROSTFS_CONTROL_GRPC_ENDPOINT\""]
       interval: 2s
       timeout: 1s
       retries: 5
@@ -98,13 +93,10 @@ services:
       - storage_s03:/storage
       - ./../../vendor/frostfs-cli:/frostfs-cli
       - ./cli-cfg.yml:/cli-cfg.yml
-      - ./healthcheck.sh:/healthcheck.sh
       - ./s04tls.crt:/etc/ssl/certs/s04tls.crt
-      - ../nats/client-cert.pem:/etc/frostfs-node/nats.tls.cert
-      - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key
-      - ../nats/ca-cert.pem:/etc/frostfs-node/nats.ca.crt
       - ./cfg:/etc/frostfs/storage
-    stop_signal: SIGKILL
+    stop_signal: SIGTERM
+    stop_grace_period: 15s
     env_file: [ ".env", ".storage.env", ".int_test.env" ]
     command: [ "frostfs-node", "--config", "/etc/frostfs/storage/config.yml" ]
     environment:
@@ -113,10 +105,11 @@ services:
       - FROSTFS_NODE_ADDRESSES=s03.${LOCAL_DOMAIN}:8080
       - FROSTFS_GRPC_0_ENDPOINT=s03.${LOCAL_DOMAIN}:8080
       - FROSTFS_CONTROL_GRPC_ENDPOINT=s03.${LOCAL_DOMAIN}:8081
+      - FROSTFS_NODE_ATTRIBUTE_0=User-Agent:FrostFS/${NODE_VERSION}
       - FROSTFS_NODE_ATTRIBUTE_1=UN-LOCODE:SE STO
       - FROSTFS_NODE_ATTRIBUTE_2=Price:11
     healthcheck:
-      test: ["CMD-SHELL", "/healthcheck.sh"]
+      test: ["CMD-SHELL", "/frostfs-cli control healthcheck -q -c /cli-cfg.yml --endpoint \"$$FROSTFS_CONTROL_GRPC_ENDPOINT\""]
       interval: 2s
       timeout: 1s
       retries: 5
@@ -138,14 +131,11 @@ services:
       - storage_s04:/storage
       - ./../../vendor/frostfs-cli:/frostfs-cli
       - ./cli-cfg.yml:/cli-cfg.yml
-      - ./healthcheck.sh:/healthcheck.sh
       - ./s04tls.crt:/tls.crt
       - ./s04tls.key:/tls.key
-      - ../nats/client-cert.pem:/etc/frostfs-node/nats.tls.cert
-      - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key
-      - ../nats/ca-cert.pem:/etc/frostfs-node/nats.ca.crt
       - ./cfg:/etc/frostfs/storage
-    stop_signal: SIGKILL
+    stop_signal: SIGTERM
+    stop_grace_period: 15s
     env_file: [ ".env", ".storage.env", ".int_test.env" ]
     command: [ "frostfs-node", "--config", "/etc/frostfs/storage/config.yml" ]
     environment:
@@ -159,10 +149,11 @@ services:
       - FROSTFS_GRPC_1_TLS_ENABLED=true
       - FROSTFS_GRPC_1_TLS_CERTIFICATE=/tls.crt
       - FROSTFS_GRPC_1_TLS_KEY=/tls.key
+      - FROSTFS_NODE_ATTRIBUTE_0=User-Agent:FrostFS/${NODE_VERSION}
       - FROSTFS_NODE_ATTRIBUTE_1=UN-LOCODE:FI HEL
       - FROSTFS_NODE_ATTRIBUTE_2=Price:44
     healthcheck:
-      test: ["CMD-SHELL", "/healthcheck.sh"]
+      test: ["CMD-SHELL", "/frostfs-cli control healthcheck -q -c /cli-cfg.yml --endpoint \"$$FROSTFS_CONTROL_GRPC_ENDPOINT\""]
       interval: 2s
       timeout: 1s
       retries: 5

services/storage/generate_cert.sh

@@ -19,7 +19,7 @@ if [[ ! -f ${CERT} ]]; then
         ) > ${SSL_CONFIG}
 
         openssl req -new -newkey rsa:4096 -x509 -sha256 -days 365 -nodes \
-        -subj "/C=RU/ST=SPB/L=St.Petersburg/O=NSPCC/OU=NSPCC/CN=s04.${LOCAL_DOMAIN}" \
+        -subj "/C=RU/ST=SPB/L=St.Petersburg/O=TrueCloudLab/OU=TrueCloudLab/CN=s04.${LOCAL_DOMAIN}" \
         -keyout "${KEY}" -out "${CERT}" -extensions san -config "${SSL_CONFIG}" &> /dev/null || {
             die "Failed to generate SSL certificate for s04"
         }

services/storage/healthcheck.sh

@@ -1,5 +0,0 @@
-#!/bin/sh
-
-/frostfs-cli control healthcheck -c /cli-cfg.yml \
-	--endpoint "$FROSTFS_CONTROL_GRPC_ENDPOINT" |
-	grep "Health status: READY"