Compare commits

..

No commits in common. "master" and "prometheus" have entirely different histories.

77 changed files with 561 additions and 529 deletions

View file

@ -1,4 +1,5 @@
# Services start/stop order
# Will start from top to bottom and stop in reverse
nats
ir
storage

View file

@ -4,4 +4,3 @@ basenet
morph_chain
jaeger
prometheus
grafana

View file

@ -1,6 +1,5 @@
.docker
.github
.forgejo
vendor
tmp
.secrets

44
.env
View file

@ -8,19 +8,23 @@ BASTION_VERSION=10
BASTION_IMAGE=debian
# NeoGo privnet
NEOGO_VERSION=0.104.0
NEOGO_VERSION=0.100.1
NEOGO_IMAGE=nspccdev/neo-go
# FrostFS InnerRing nodes
IR_VERSION=0.42.9
IR_IMAGE=git.frostfs.info/truecloudlab/frostfs-ir
IR_VERSION=5ffa8268
IR_IMAGE=truecloudlab/frostfs-ir
# FrostFS Storage nodes
NODE_VERSION=0.42.9
NODE_IMAGE=git.frostfs.info/truecloudlab/frostfs-storage
NODE_VERSION=5ffa8268
NODE_IMAGE=truecloudlab/frostfs-storage
# NATS Server
NATS_VERSION=2.7.2
NATS_IMAGE=nats
# HTTP Gate
HTTP_GW_VERSION=0.30.2
HTTP_GW_VERSION=6abd500b
HTTP_GW_IMAGE=truecloudlab/frostfs-http-gw
# REST Gate
@ -28,29 +32,25 @@ REST_GW_VERSION=c9c85e90
REST_GW_IMAGE=truecloudlab/frostfs-rest-gw
# S3 Gate
S3_GW_VERSION=0.31.0-rc.4
S3_GW_VERSION=000d9ed4
S3_GW_IMAGE=truecloudlab/frostfs-s3-gw
# Lifecycler
S3_LIFECYCLER_VERSION=0.1.3
S3_LIFECYCLER_IMAGE=truecloudlab/frostfs-s3-lifecycler
# FrostFS LOCODE database
LOCODE_DB_URL=https://git.frostfs.info/attachments/a2e8def7-52b6-49f1-89cd-a056712e8e54
LOCODE_DB_URL=https://github.com/nspcc-dev/neofs-locode-db/releases/download/v0.3.0/locode_db.gz
#LOCODE_DB_PATH=/path/to/locode_db
# FrostFS CLI binary
FROSTFS_CLI_URL=https://git.frostfs.info/TrueCloudLab/frostfs-node/releases/download/v${NODE_VERSION}/frostfs-cli
FROSTFS_CLI_URL=https://http.t5.fs.neo.org/AQgse8bPCZx4zScMuAKxowJdZPbKHp8NDcp15o6VUNmk/C6BNLpYg5gWLHp3DrXozSxxGLDahBuSBCyJoYSSR1M3Q
#FROSTFS_CLI_PATH=/path/to/frostfs-cli-binary
# FrostFS ADM tool binary
FROSTFS_ADM_VERSION=498f9955ea
FROSTFS_ADM_URL=https://git.frostfs.info/TrueCloudLab/frostfs-node/releases/download/v${NODE_VERSION}/frostfs-adm
FROSTFS_ADM_VERSION=e3554425
FROSTFS_ADM_URL=https://http.t5.fs.neo.org/AQgse8bPCZx4zScMuAKxowJdZPbKHp8NDcp15o6VUNmk/sXZxy9vbFyJiLhN9qTSXozXK7SN9H8ZC6dpvAt59Zaj
#FROSTFS_ADM_PATH=/path/to/frostfs-adm-binary
# Compiled FrostFS Smart Contracts
FROSTFS_CONTRACTS_VERSION=694daebb19
FROSTFS_CONTRACTS_URL=https://git.frostfs.info/TrueCloudLab/frostfs-contract/releases/download/v0.19.2/frostfs-contract-v0.19.2.tar.gz
FROSTFS_CONTRACTS_VERSION=4f3c08f5
FROSTFS_CONTRACTS_URL=https://http.t5.fs.neo.org/AQgse8bPCZx4zScMuAKxowJdZPbKHp8NDcp15o6VUNmk/c1nGtturFrSeygYP3AyNHDDLNbs7HhJiH2BQkgZxEmZ
#FROSTFS_CONTRACTS_PATH=/path/to/unpacked/frostfs-contracts-dir
# Jaeger tracing
@ -59,12 +59,4 @@ JAEGER_IMAGE=jaegertracing/all-in-one
# Prometheus monitoring
PROMETHEUS_VERSION=v2.43.0
PROMETHEUS_IMAGE=prom/prometheus
# Grafana versions
GRAFANA_VERSION=9.5.6
GRAFANA_IMAGE=grafana/grafana
# Loki versions
LOKI_VERSION=2.9.1
LOKI_IMAGE=grafana/loki
PROMETHEUS_IMAGE=prom/prometheus

View file

@ -1,21 +0,0 @@
name: DCO action
on: [pull_request]
jobs:
dco:
name: DCO
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Setup Go
uses: actions/setup-go@v3
with:
go-version: '1.21'
- name: Run commit format checker
uses: https://git.frostfs.info/TrueCloudLab/dco-go@v2
with:
from: 'origin/${{ github.event.pull_request.base.ref }}'

1
.gitattributes vendored
View file

@ -1 +0,0 @@
/services/grafana/provisioning/dashboards/* -diff -merge

21
.github/workflows/dco.yml vendored Normal file
View file

@ -0,0 +1,21 @@
name: DCO check
on:
pull_request:
branches:
- master
jobs:
commits_check_job:
runs-on: ubuntu-latest
name: Commits Check
steps:
- name: Get PR Commits
id: 'get-pr-commits'
uses: tim-actions/get-pr-commits@master
with:
token: ${{ secrets.GITHUB_TOKEN }}
- name: DCO Check
uses: tim-actions/dco@master
with:
commits: ${{ steps.get-pr-commits.outputs.commits }}

1
.gitignore vendored
View file

@ -15,3 +15,4 @@ sites/*
# Runtime generation keys
services/storage/*tls.crt
services/storage/*tls.key
services/nats/*.pem

View file

@ -3,4 +3,3 @@
http_gate
s3_gate
rest_gate
s3_lifecycler

View file

@ -3,8 +3,8 @@
First, thank you for contributing! We love and encourage pull requests from
everyone. Please follow the guidelines:
- Check the open [issues](https://git.frostfs.info/TrueCloudLab/frostfs-dev-env/issues) and
[pull requests](https://git.frostfs.info/TrueCloudLab/frostfs-dev-env/pulls) for existing
- Check the open [issues](https://github.com/TrueCloudLab/frostfs-dev-env/issues) and
[pull requests](https://github.com/TrueCloudLab/frostfs-dev-env/pulls) for existing
discussions.
- Open an issue first, to discuss a new feature or enhancement.
@ -25,19 +25,19 @@ Start by forking the `frostfs-dev-env` repository, make changes in a branch and
send a pull request. We encourage pull requests to discuss code changes. Here
are the steps in details:
### Set up your git repository
Fork [FrostFS node upstream](https://git.frostfs.info/repo/fork/24) source
### Set up your GitHub Repository
Fork [FrostFS node upstream](https://github.com/TrueCloudLab/frostfs-dev-env/fork) source
repository to your own personal repository. Copy the URL of your fork (you will
need it for the `git clone` command below).
```sh
$ git clone https://git.frostfs.info/<username>/frostfs-dev-env.git
$ git clone https://github.com/TrueCloudLab/frostfs-dev-env
```
### Set up git remote as ``upstream``
```sh
$ cd frostfs-dev-env
$ git remote add upstream https://git.frostfs.info/TrueCloudLab/frostfs-dev-env.git
$ git remote add upstream https://github.com/TrueCloudLab/frostfs-dev-env
$ git fetch upstream
$ git merge upstream/master
...
@ -55,7 +55,8 @@ $ git checkout -b feature/123-something_awesome
### Test your changes
After your code changes, make sure
- To run `make up` to check dev-env is not broken.
- To add test cases for the new code.
- To run `make lint`
- To squash your commits into a single commit or a series of logically separated
commits run `git rebase -i`. It's okay to force update your pull request.
@ -85,8 +86,8 @@ $ git push origin feature/123-something_awesome
```
### Create a Pull Request
Pull requests can be created via Forgejo. Refer to [this
document](https://docs.codeberg.org/collaborating/pull-requests-and-git-flow/) for
Pull requests can be created via GitHub. Refer to [this
document](https://help.github.com/articles/creating-a-pull-request/) for
detailed steps on how to create a pull request. After a Pull Request gets peer
reviewed and approved, it will be merged.

View file

@ -43,7 +43,7 @@ HOSTS_LINES = $(shell grep -Rl IPV4_PREFIX ./services/* | grep .hosts)
MORPH_CHAIN_PROTOCOL = './services/morph_chain/protocol.privnet.yml'
# List of grepped environment variables from *.env
GREP_DOTENV = $(shell find . -name '*.env' -exec grep -rhv -e '^#' -e '^$$' {} + | sort -u )
GREP_DOTENV = $(shell find . -name '*.env' -exec grep -rhv -e '^\#' -e '^$$' {} + | sort -u )
# Pull all required Docker images
.PHONY: pull
@ -60,15 +60,14 @@ get: $(foreach SVC, $(GET_SVCS), get.$(SVC))
.PHONY: up
up: up/basic
@$(foreach SVC, $(START_SVCS), $(shell docker-compose -f services/$(SVC)/docker-compose.yml up -d))
./vendor/frostfs-adm morph proxy-add-account --config frostfs-adm.yml --account=`docker container exec morph_chain neo-go wallet dump-keys -w /wallets/s3-wallet.json | head -1 | awk '{print $1}'` || die "Couldn't set s3-gw wallet as proxy wallet"
@echo "Full FrostFS Developer Environment is ready"
# Build up FrostFS
.PHONY: up/basic
up/basic: up/bootstrap
@$(foreach SVC, $(START_BASIC), $(shell docker-compose -f services/$(SVC)/docker-compose.yml up -d))
@./vendor/frostfs-adm -c ./frostfs-adm.yml morph force-new-epoch
@./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config SystemDNS=container --force
@./bin/tick.sh
@./bin/config.sh string SystemDNS container
@echo "Basic FrostFS Developer Environment is ready"
# Start bootstrap services
@ -76,26 +75,9 @@ up/basic: up/bootstrap
up/bootstrap: get vendor/hosts
@$(foreach SVC, $(START_BOOTSTRAP), $(shell docker-compose -f services/$(SVC)/docker-compose.yml up -d))
@source ./bin/helper.sh
@./vendor/frostfs-adm --config frostfs-adm.yml morph init --contracts vendor/contracts
echo "Set rule chain to policy contract"
@./vendor/frostfs-adm --config frostfs-adm.yml morph \
ape add-rule-chain --target-type namespace --target-name "" \
--rule 'allow Container.* *' --chain-id "allow_container_ops"
@for f in ./services/storage/wallet*.json; do \
echo "Transfer GAS to wallet $${f}" \
&& ./vendor/frostfs-adm -c frostfs-adm.yml morph refill-gas --storage-wallet $${f} --gas 10.0 \
|| die "Failed to transfer GAS to alphabet wallets"; \
done
@echo "Create frostfsid subject for ./wallets/wallet.json"; \
if [ -n "$$(./vendor/frostfs-adm -c frostfs-adm.yml morph frostfsid list-subjects --namespace '')" ]; then \
echo "Subject already exists"; \
else \
subj_key=`docker container exec -it morph_chain neo-go wallet dump-keys -w /wallets/wallet.json | tail -1 | tr -d ' \r\n'` \
&& echo "Subject key: $${subj_key}" \
&& ./vendor/frostfs-adm -c frostfs-adm.yml morph frostfsid create-subject --namespace "" --subject-key $${subj_key} --subject-name walletsubject \
|| die "Failed to create subject for the wallet"; \
fi
echo "FrostFS sidechain environment is deployed"
@./vendor/frostfs-adm --config frostfs-adm.yml morph init --alphabet-wallets ./services/ir --contracts vendor/contracts || die "Failed to initialize Alphabet wallets"
@for f in ./services/storage/wallet*.json; do echo "Transfer GAS to wallet $${f}" && ./vendor/frostfs-adm -c frostfs-adm.yml morph refill-gas --storage-wallet $${f} --gas 10.0 --alphabet-wallets services/ir || die "Failed to transfer GAS to alphabet wallets"; done
@echo "FrostFS sidechain environment is deployed"
# Build up certain service
.PHONY: up/%
@ -150,7 +132,7 @@ hosts: vendor/hosts
.PHONY: clean
.ONESHELL:
clean:
@rm -rf vendor/* services/storage/s04tls.*
@rm -rf vendor/* services/storage/s04tls.* services/nats/*.pem
@> .int_test.env
@for svc in $(PULL_SVCS)
do
@ -166,7 +148,7 @@ clean:
.PHONY: env
env:
@$(foreach envvar,$(GREP_DOTENV),echo $(envvar);)
@echo MORPH_BLOCK_TIME=$(shell grep 'TimePerBlock' $(MORPH_CHAIN_PROTOCOL) | awk '{print $$2}')s
@echo MORPH_BLOCK_TIME=$(shell grep 'SecondsPerBlock' $(MORPH_CHAIN_PROTOCOL) | awk '{print $$2}')s
@echo MORPH_MAGIC=$(shell grep 'Magic' $(MORPH_CHAIN_PROTOCOL) | awk '{print $$2}')
# Restart storage nodes with clean volumes

View file

@ -27,7 +27,7 @@ Make sure you have installed all of the following prerequisites on your machine:
Clone repo:
```
$ git clone https://git.frostfs.info/TrueCloudLab/frostfs-dev-env.git
$ git clone https://github.com/TrueCloudLab/frostfs-dev-env.git
```
Run next commands from project's root:
@ -57,6 +57,17 @@ Run all services with command:
$ make up
```
When all services are up, you need to make GAS deposit for test wallet to be
able to pay for FrostFS operations. Test wallet is located in
`wallets/wallet.json` with the corresponding key in `wallets/wallet.key`. The
password is empty.
```
$ make prepare.ir
password >
fa6ba62bffb04030d303dcc95bda7413e03aa3c7e6ca9c2f999d65db9ec9b82c
```
Also, you should add self-signed node (`s04.frostfs.devenv`) certificate to trusted
store (default location might be changed using `CA_CERTS_TRUSTED_STORE`
variable). This step is required for client services (frostfs-http-gw,
@ -71,7 +82,12 @@ password of inner ring wallet is `one`. See examples in `make help`.
```
$ make update.epoch_duration val=30
Waiting for transactions to persist...
Changing EpochDuration configration value to 30
Enter account NNudMSGzEoktFzdYGYoNb3bzHzbmM1genF password >
Sent invocation transaction dbb8c1145b6d10f150135630e13bb0dc282023163f5956c6945a60db0cb45cb0
Updating FrostFS epoch to 2
Enter account NNudMSGzEoktFzdYGYoNb3bzHzbmM1genF password >
Sent invocation transaction 0e6eb5e190f36332e5e5f4e866c7e100826e285fd949e11c085e15224f343ba6
```
For instructions on how to set up DevEnv on macOS, please refer [the
@ -107,7 +123,7 @@ Maybe you will find the answer for your question in [F.A.Q.](docs/faq.md)
## Using FrostFS Admin Tool in `dev-env`
Devenv supports FrostFS network management via [frostfs-adm](https://git.frostfs.info/TrueCloudLab/frostfs-node/src/branch/master/cmd/frostfs-adm).
Devenv supports FrostFS network management via [frostfs-adm](https://github.com/TrueCloudLab/frostfs-node/tree/master/cmd/frostfs-adm).
`services/ir` contains the Alphabet wallet in a proper format, specify it
with `--alphabet-wallets` flag.
@ -137,65 +153,6 @@ Display addresses and host names for each running service, if available.
Clean up `vendor` directory.
### s3cred
Registers user wallet and issues s3 credentials.
Usage and default parameter values:
```sh
make s3cred [password=""] [contract_password=s3] [wallet=/user_wallet.json] [gate_public_key=0313b1ac3a8076e155a7e797b24f0b650cccad5941ea59d7cfd51a024a8b2a06bf]
```
As soon as the storage node is in the network map (see above) you can generate S3
credentials:
``` sh
$ make s3cred
{
"access_key_id": "EXArWh8x1zeHG3851s1RtoCo7dowxF6rhLGA15nbMffT0AKRSjJ5fmcqf3Ht2VCAkfmPQUVARghRB77xHCA1BoN2p",
"secret_access_key": "d70c1dba83f0f90bb231f06f1ce0e0dfbcfb122f4b4345a3c18d3869c359b79f",
"owner_private_key": "140947599afd9ca89af4b358c3176eb046e554d942a0dc99a8e06f3e43c8f4ad",
"wallet_public_key": "0324e76288fcb900100d01802a14ef977cca45ad073561230446df14b344c858b6",
"container_id": "EXArWh8x1zeHG3851s1RtoCo7dowxF6rhLGA15nbMffT"
}
```
Running without any parameters will result in defaults which are based on the private key from
`/user-wallet.json` file and `/wallet.json` contract wallet.
Now let's configure an S3 client (AWS CLI will be used as example):
``` sh
$ aws configure
AWS Access Key ID []: EXArWh8x1zeHG3851s1RtoCo7dowxF6rhLGA15nbMffT0AKRSjJ5fmcqf3Ht2VCAkfmPQUVARghRB77xHCA1BoN2p
AWS Secret Access Key []: d70c1dba83f0f90bb231f06f1ce0e0dfbcfb122f4b4345a3c18d3869c359b79f
Default region name []: us-east-1
Default output format []: json
```
If you need to create credentials for different users, put user wallets to `wallets` dir and specify them via `wallet` parameter.
Pass wallet password in `password` parameter if it's not default. The same is for `contract_wallet` and `gate_public_key` params.
```sh
$ make s3cred wallet=custom_wallet.json password=test
{
"access_key_id": "jHhL5B33o16R4jQsb8wm9A3RRdS6KrTB5N4bja9Jys904W7xXFNKqem2ACvTRWRYJsZMCUikYFSokN7pPJziWyDi",
"secret_access_key": "21bb64fafa32c82417fd8b97ac56cc8a085998a3852632d52fe7042453daa440",
"owner_private_key": "10f6f9d7a47bb0bf68363ad8a99fe69f1493f8b6e1665b3e4e83feb2d5c7ee39",
"wallet_public_key": "03e38759973a6bb722baabc2dd84036a39f0b2f53d32fec45a4dacde8a50fe4b70",
"container_id": "jHhL5B33o16R4jQsb8wm9A3RRdS6KrTB5N4bja9Jys9"
}
```
To get credentials from custom wallet, place it in `wallets` dir before start.
### cred
Usage and default parameter values:
```sh
make cred [password=""] [contract_password=s3] [wallet=/user_wallet.json]
```
The same as `s3cred`, but it doesn't issues s3 credentials.
## Contributing
Feel free to contribute to this project after reading the [contributing

50
bin/config.sh Executable file
View file

@ -0,0 +1,50 @@
#!/usr/bin/env bash
echo "Running bin/config.sh"
# Source env settings
. .env
. services/ir/.ir.env
source bin/helper.sh
# NeoGo binary path.
NEOGO="${NEOGO:-docker exec morph_chain neo-go}"
# Wallet files to change config value
WALLET="${WALLET:-services/morph_chain/node-wallet.json}"
CONFIG_IMG="${CONFIG_IMG:-/wallets/config.yml}"
NETMAP_ADDR=$(bin/resolve.sh netmap.frostfs) || die "Failed to resolve 'netmap.frostfs' domain name"
# FrostFS configuration record: variable type [string|int|etc],
# key is a string and value is a constant of given type
TYPE=${1}
KEY=${2}
VALUE="${3}"
[ -z "$TYPE" ] && echo "Empty config value type" && exit 1
[ -z "$KEY" ] && echo "Empty config key" && exit 1
[ -z "$VALUE" ] && echo "Empty config value" && exit 1
# Internal variables
if [[ -z "${FROSTFS_NOTARY_DISABLED}" ]]; then
ADDR=$(jq -r .accounts[2].address < "${WALLET}" || die "Cannot get address from ${WALLET}")
else
ADDR=$(jq -r .accounts[0].address < "${WALLET}" || die "Cannot get address from ${WALLET}")
fi
# Change config value in side chain
echo "Changing ${KEY} configration value to ${VALUE}"
# shellcheck disable=SC2086
${NEOGO} contract invokefunction \
--wallet-config ${CONFIG_IMG} \
-a ${ADDR} --force \
-r http://morph-chain.${LOCAL_DOMAIN}:30333 \
${NETMAP_ADDR} \
setConfig bytes:beefcafe \
string:${KEY} \
${TYPE}:${VALUE} -- ${ADDR} || exit 1
# Update epoch to apply new configuration value
./bin/tick.sh

23
bin/passwd.exp Executable file
View file

@ -0,0 +1,23 @@
#!/usr/bin/expect
set passwd [lindex $argv 0]
set args [lrange $argv 1 end]
spawn -noecho {*}$args
expect -re {^.*assword.*$}
if { $passwd == "-"} {
send -- "\r"
} else {
send -- "$passwd\r"
}
expect {
"Relay transaction" {
send "y\r"
exp_continue
}
EOF
}
lassign [wait] pid spawnid os_error_flag value
exit $value

22
bin/resolve.sh Executable file
View file

@ -0,0 +1,22 @@
#!/usr/bin/env bash
# Source env settings
. .env
source bin/helper.sh
# NeoGo binary path.
NEOGO="${NEOGO:-docker exec morph_chain neo-go}"
# NNS contract script hash
output=$(curl -s --data '{ "id": 1, "jsonrpc": "2.0", "method": "getcontractstate", "params": [1] }' \
"http://morph-chain.${LOCAL_DOMAIN}:30333/") \
|| die "Cannot fetch NNS contract state"
NNS_ADDR=$(jq -r '.result.hash' <<< "$output") \
|| die "Cannot parse NNS contract hash: $NNS_ADDR"
${NEOGO} contract testinvokefunction \
-r "http://morph-chain.${LOCAL_DOMAIN}:30333" \
"${NNS_ADDR}" resolve string:"${1}" int:16 \
| jq -r '.stack[0].value | if type=="array" then .[0].value else . end' \
| base64 -d \
|| die "Cannot invoke 'NNS.resolve' $output"

49
bin/tick.sh Executable file
View file

@ -0,0 +1,49 @@
#!/usr/bin/env bash
echo "Running bin/tick.sh"
# Source env settings
. .env
. services/ir/.ir.env
source bin/helper.sh
# NeoGo binary path.
NEOGO="${NEOGO:-docker exec morph_chain neo-go}"
# Wallet files to change config value
WALLET="${WALLET:-services/morph_chain/node-wallet.json}"
CONFIG_IMG="${CONFIG_IMG:-/wallets/config.yml}"
# Internal variables
if [[ -z "${FROSTFS_NOTARY_DISABLED}" ]]; then
ADDR=$(jq -r .accounts[2].address < "${WALLET}" || die "Cannot get address from ${WALLET}")
else
ADDR=$(jq -r .accounts[0].address < "${WALLET}" || die "Cannot get address from ${WALLET}")
fi
# Grep Morph block time
SIDECHAIN_PROTO="${SIDECHAIN_PROTO:-services/morph_chain/protocol.privnet.yml}"
BLOCK_DURATION=$(grep SecondsPerBlock < "$SIDECHAIN_PROTO" | awk '{print $2}') \
|| die "Cannot fetch block duration"
NETMAP_ADDR=$(bin/resolve.sh netmap.frostfs) || die "Cannot resolve netmap.frostfs"
# Fetch current epoch value
EPOCH=$(${NEOGO} contract testinvokefunction \
-r "http://morph-chain.${LOCAL_DOMAIN}:30333" "${NETMAP_ADDR}" epoch \
| grep 'value' | awk -F'"' '{ print $4 }') \
|| die "Cannot fetch epoch from netmap contract"
echo "Updating FrostFS epoch to $((EPOCH+1))"
# shellcheck disable=SC2086
${NEOGO} contract invokefunction \
--wallet-config ${CONFIG_IMG} \
-a ${ADDR} --force \
-r http://morph-chain.${LOCAL_DOMAIN}:30333 \
${NETMAP_ADDR} \
newEpoch int:$((EPOCH+1)) -- ${ADDR}:Global \
|| die "Cannot increment an epoch"
# Wait one Morph block to ensure the transaction broadcasted
# shellcheck disable=SC2086
sleep $BLOCK_DURATION

View file

@ -1,4 +0,0 @@
wallet: services/storage/wallet01.json
password: ""
rpc-endpoint: s01.frostfs.devenv:8080
endpoint: s01.frostfs.devenv:8081

View file

@ -1,4 +0,0 @@
wallet: services/storage/wallet02.json
password: ""
rpc-endpoint: s02.frostfs.devenv:8080
endpoint: s02.frostfs.devenv:8081

View file

@ -1,4 +0,0 @@
wallet: services/storage/wallet03.json
password: ""
rpc-endpoint: s03.frostfs.devenv:8080
endpoint: s03.frostfs.devenv:8081

View file

@ -1,4 +0,0 @@
wallet: services/storage/wallet04.json
password: ""
rpc-endpoint: s04.frostfs.devenv:8080
endpoint: s04.frostfs.devenv:8081

View file

@ -2,7 +2,7 @@
Protocol Gateway to access data in FrostFS using HTTP protocol.
Source code and more information can be found in [project's repository](https://git.frostfs.info/TrueCloudLab/frostfs-http-gw)
Source code and more information can be found in [project's GitHub repository](https://github.com/TrueCloudLab/frostfs-http-gate)
## .env settings
@ -22,8 +22,8 @@ Image label prefix to use for containers.
- Create a new container
```
$ frostfs-cli --rpc-endpoint s01.frostfs.devenv:8080 \
--wallet wallets/wallet.key \
container create --basic-acl private --await \
--key wallets/wallet.key \
container create --basic-acl readonly --await \
--policy "REP 1 SELECT 1 FROM *"
container ID: 4LfREK1cetL4PUji5fqj9SgRTSmaC5jExEDK9HKCDjdP
awaiting...
@ -33,7 +33,7 @@ container has been persisted on sidechain
- Put an object into the newly created container
```
$ frostfs-cli --rpc-endpoint s01.frostfs.devenv:8080 \
--wallet wallets/wallet.key \
--key wallets/wallet.key \
object put --file /tmp/backup.jpeg \
--cid 4LfREK1cetL4PUji5fqj9SgRTSmaC5jExEDK9HKCDjdP
[/tmp/backup.jpeg] Object successfully stored

View file

@ -3,23 +3,52 @@ A single-node N3 privnet deployment, running on
[neo-go](https://github.com/nspcc-dev/neo-go). Represents N3 FrostFS SideChain.
Contracts deployed:
- Alphabet (AZ) [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/alphabet)
- Audit [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/audit)
- Balance [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/balance)
- Container [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/container)
- Netmap [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/netmap)
- NeoFSID [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/neofsid)
- Proxy [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/proxy)
- Reputation [contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/branch/master/reputation)
- Alphabet (AZ) [contract](https://github.com/TrueCloudLab/frostfs-contract/tree/master/alphabet)
- Audit [contract](https://github.com/TrueCloudLab/frostfs-contract/tree/master/audit)
- Balance [contract](https://github.com/TrueCloudLab/frostfs-contract/tree/master/balance)
- Container [contract](https://github.com/TrueCloudLab/frostfs-contract/tree/master/container)
- Netmap [contract](https://github.com/TrueCloudLab/frostfs-contract/tree/master/netmap)
- NeoFSID [contract](https://github.com/TrueCloudLab/frostfs-contract/tree/master/neofsid)
- Proxy [contract](https://github.com/TrueCloudLab/frostfs-contract/tree/master/proxy)
- Reputation [contract](https://github.com/TrueCloudLab/frostfs-contract/tree/master/reputation)
RPC available at `http://morph-chain.frostfs.devenv:30333`.
## .env settings
### MORPH_CHAIN_URL
URL to get side chain dump. Used on artifact get stage.
### MORPH_CHAIN_PATH
Path to get side chain dump. If set, overrides `CHAIN_URL`.
### NEOGO_VERSION
Version of neo-go docker container for side chain deployment.
## Side chain wallets
There is a wallet with GAS that used for contract deployment:
`wallets/wallet.json`. This wallet has one account with **empty password**.
```
$ neo-go wallet nep17 balance \
-w wallets/wallet.json \
-r http://morph-chain.frostfs.devenv:30333
Account NbUgTSFvPmsRxmGeWpuuGeJUoRoi6PErcM
GAS: GasToken (d2a4cff31913016155e38e474a2c06d08be276cf)
Amount : 189826.0515316
Updated: 3909
FROSTFS: FrostFS Balance (69550190e740b93f92dbd5dea52246f550391057)
Amount : 50
Updated: 3909
```
This way you can also monitor FrostFS internal balance of your account.
## FrostFS global config
FrostFS uses global configuration to store epoch duration, maximum object size,
@ -28,10 +57,16 @@ netmap contract and managed by Inner Ring (Alphabet) nodes.
To change these parameters use `make update.*` commands. Command down below
changes epoch duration from 300 blocks (about 300 seconds with 1bps) to 30.
Script enters passwords automatically with `expect` utility.
```
$ make update.epoch_duration val=30
Waiting for transactions to persist...
Changing EpochDuration configration value to 30
Enter account NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP password >
Sent invocation transaction bdc0fa88cd6719ef6df2b9c82de423ddec6141ca24255c2d0072688083b1de9d
Updating FrostFS epoch to 20
Enter account NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP password >
Sent invocation transaction 12296e1ce24dd6c04edb9c56d0a1d0e26d3226adefb0333c74a28788f44a8d0f
```
Read more about available configuration in Makefile help.
@ -41,14 +76,10 @@ $ make help
...
Targets:
...
update.audit_fee Update audit fee per result in fixed 12 (make update.audit_fee val=100)
update.basic_income_rate Update basic income rate in fixed 12 (make update.basic_income_rate val=1000)
update.container_alias_fee Update container alias fee per alphabet node in fixed 12 (make update.container_alias_fee val=100)
update.container_fee Update container fee per alphabet node in fixed 12 (make update.container_fee val=500)
update.eigen_trust_alpha Update alpha parameter of EigenTrust algorithm in 0 <= f <= 1.0 (make update.eigen_trust_alpha val=0.2)
update.eigen_trust_iterations Update amount of EigenTrust iterations (make update.eigen_trust_iterations val=2)
update.epoch_duration Update epoch duration in side chain blocks (make update.epoch_duration val=30)
update.homomorphic_hashing_disable Update homomorphic hashing disabled flag (make update.homomorphic_hashing_disable val=true)
update.max_object_size Update max object size in bytes (make update.max_object_size val=1000)
update.system_dns Update system dns to resolve container names (make update.system_dns val=container)
update.audit_fee Update audit fee per result in fixed 12 (make update.audit_fee val=100)
update.basic_income_rate Update basic income rate in fixed 12 (make update.basic_income_rate val=1000)
update.container_fee Update container fee per alphabet node in fixed 12 (make update.container_fee val=500)
update.eigen_trust_iterations Update amount of EigenTrust iterations (make update.eigen_trust_iterations val=2)
update.epoch_duration Update epoch duration in side chain blocks (make update.epoch_duration val=30)
update.max_object_size Update max object size in bytes (make update.max_object_size val=1000)
```

View file

@ -7,12 +7,66 @@ create containers, approve balance changes, update network map, tick epochs,
etc. With notary service, it takes up to seven times fewer transactions
to do these operations. Notary service calculates the exact amount of GAS
to execute transaction, therefore operations are cheaper (withdraw fee **with**
notary is less than 0.5 GAS; withdraw fee **without** notary is up to 7.0 GAS).
notary is less than 0.5 GAS; withdraw fee **without** notary is up to 7.0 GAS).
Currently, frostfs-dev-env contains single chain (see morph service) and it
enables notary service from the genesis block.
By default, main chain service is running without notary service, and side chain
running with notary service. However, you can change that in configuration.
To enable notary service, use neo-go configuration below.
# Disable notary service in side chain
To disable notary service in side chain do these steps.
1. Update `.env` and choose notary disabled chain dump for side chain.
```
MORPH_CHAIN_URL="https://github.com/nspcc-dev/neofs-contract/releases/download/v0.9.0/devenv_sidechain_notary_disabled.gz"
```
Make sure to update chain dump files with `make get` target.
2. Update `service/morph_chain/protocol.privnet.yml` and disable notary settings
and state root in header.
```yaml
ProtocolConfiguration:
StateRootInHeader: false
P2PSigExtensions: false
ApplicationConfiguration:
P2PNotary:
Enabled: false
```
Chain dump without notary service does not have predefined network map.
Therefore, you need to wait about 5 minutes until new epoch tick with updated
network map.
3. Enable helper commands
To enable helper commands such as `make tick.epoch` or `make update.epoch_duration`
make sure to export non-empty `FROSTFS_NOTARY_DISABLED` environment variable.
```
$ export FROSTFS_NOTARY_DISABLED=1
```
Use `unset` command to return it back.
```
$ unset FROSTFS_NOTARY_DISABLED
```
# Enable notary service in main chain
To enable notary service in main chain do these steps.
1. Update `.env` and choose notary enabled chain dump for main chain.
```
CHAIN_URL="https://github.com/nspcc-dev/neofs-contract/releases/download/v0.9.0/devenv_mainchain.gz"
```
Make sure to update chain dump files with `make get` target.
2. Update `service/chain/protocol.privnet.yml` and enable notary settings.
```yaml
ProtocolConfiguration:
@ -21,3 +75,7 @@ ApplicationConfiguration:
P2PNotary:
Enabled: true
```
Main chain generates a block once per 15 seconds, so Inner Ring takes about
15-30 seconds to make a notary deposit in main chain after startup. Then
frostfs-dev-env is ready to work.

View file

@ -2,7 +2,7 @@
REST Gateway to access data in FrostFS using REST.
Source code and more information can be found in [project's repository](https://git.frostfs.info/TrueCloudLab/frostfs-rest-gw)
Source code and more information can be found in [project's GitHub repository](https://github.com/TrueCloudLab/frostfs-rest-gw)
## .env settings

View file

@ -2,7 +2,7 @@
Protocol Gateway to access data in FrostFS using AWS S3 protocol
Source code and more information can be found in [project's repository](https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw)
Source code and more information can be found in [project's GitHub repository](https://github.com/TrueCloudLab/frostfs-s3-gw)
## .env settings

View file

@ -1,11 +1,9 @@
rpc-endpoint: http://morph-chain.frostfs.devenv:30333
alphabet-wallets: ./services/ir
network:
max_object_size: 67108864
epoch_duration: 240
basic_income_rate: 100000000
homomorphic_hash_disabled: false
maintenance_mode_allowed: true
fee:
audit: 10000
candidate: 10000000000

View file

@ -1,40 +1,44 @@
# Update epoch duration in side chain blocks (make update.epoch_duration val=30)
update.epoch_duration:
@./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config EpochDuration=$(val)
@./bin/config.sh int EpochDuration $(val)
# Update max object size in bytes (make update.max_object_size val=1000)
update.max_object_size:
@./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config MaxObjectSize=$(val)
@./bin/config.sh int MaxObjectSize $(val)
# Update audit fee per result in fixed 12 (make update.audit_fee val=100)
update.audit_fee:
@./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config AuditFee=$(val)
@./bin/config.sh int AuditFee $(val)
# Update container fee per alphabet node in fixed 12 (make update.container_fee val=500)
update.container_fee:
@./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config ContainerFee=$(val)
@./bin/config.sh int ContainerFee $(val)
# Update container alias fee per alphabet node in fixed 12 (make update.container_alias_fee val=100)
update.container_alias_fee:
@./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config ContainerAliasFee=$(val)
@./bin/config.sh int ContainerAliasFee $(val)
# Update amount of EigenTrust iterations (make update.eigen_trust_iterations val=2)
update.eigen_trust_iterations:
@./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config EigenTrustIterations=$(val)
@./bin/config.sh int EigenTrustIterations $(val)
# Update system dns to resolve container names (make update.system_dns val=container)
update.system_dns:
@./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config SystemDNS=$(val) --force
@./bin/config.sh string SystemDNS $(val)
# Update alpha parameter of EigenTrust algorithm in 0 <= f <= 1.0 (make update.eigen_trust_alpha val=0.2)
update.eigen_trust_alpha:
@./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config EigenTrustAlpha=$(val)
@./bin/config.sh string EigenTrustAlpha $(val)
# Update basic income rate in fixed 12 (make update.basic_income_rate val=1000)
update.basic_income_rate:
@./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config BasicIncomeRate=$(val)
@./bin/config.sh int BasicIncomeRate $(val)
# Update homomorphic hashing disabled flag (make update.homomorphic_hashing_disable val=true)
update.homomorphic_hashing_disable:
@./vendor/frostfs-adm -c ./frostfs-adm.yml morph set-config HomomorphicHashingDisabled=$(val)
@./bin/config.sh bool HomomorphicHashingDisabled $(val)
# Tick new epoch in side chain
tick.epoch:
@./bin/tick.sh

View file

@ -1,5 +1,6 @@
---
version: "2.4"
services:
basenet:

View file

@ -1,2 +0,0 @@
IPV4_PREFIX.122 grafana.LOCAL_DOMAIN
IPV4_PREFIX.123 loki.LOCAL_DOMAIN

View file

@ -1,31 +0,0 @@
services:
grafana:
image: ${GRAFANA_IMAGE}:${GRAFANA_VERSION}
domainname: ${LOCAL_DOMAIN}
hostname: grafana
container_name: grafana
restart: on-failure
networks:
grafana_int:
internet:
ipv4_address: ${IPV4_PREFIX}.122
volumes:
- ./../../vendor/hosts:/etc/hosts
- ./grafana.ini:/etc/grafana/grafana.ini
- ./provisioning:/etc/grafana/provisioning
stop_signal: SIGKILL
env_file: [ ".env", ".int_test.env" ]
loki:
image: ${LOKI_IMAGE}:${LOKI_VERSION}
command: -config.file=/etc/loki/local-config.yaml
networks:
grafana_int:
internet:
ipv4_address: ${IPV4_PREFIX}.123
networks:
grafana_int:
internet:
external: true
name: basenet_internet

View file

@ -1,7 +0,0 @@
[auth.anonymous]
enabled = true
org_name = Main Org.
org_role = Editor
[dashboards]
default_home_dashboard_path= /etc/grafana/provisioning/dashboards/overview.json

View file

@ -1,13 +0,0 @@
apiVersion: 1
datasources:
- name: Prometheus
type: prometheus
access: proxy
orgId: 1
url: http://prometheus:9090
- name: Loki
type: loki
access: proxy
orgId: 1
url: http://loki:3100

View file

@ -1,5 +1,6 @@
---
version: "2.4"
services:
http_gate:
image: ${HTTP_GW_IMAGE}:${HTTP_GW_VERSION}
@ -20,7 +21,6 @@ services:
command: [ "frostfs-http-gw", "--config", "/etc/frostfs/http/config.yml" ]
environment:
- HTTP_GW_RPC_ENDPOINT=http://morph-chain.${LOCAL_DOMAIN}:30333
- HTTP_GW_TREE_SERVICE=s01.${LOCAL_DOMAIN}:8080
- HTTP_GW_PEERS_0_ADDRESS=s01.${LOCAL_DOMAIN}:8080
- HTTP_GW_PEERS_0_WEIGHT=0.2
- HTTP_GW_PEERS_1_ADDRESS=s02.${LOCAL_DOMAIN}:8080

View file

@ -1 +1,3 @@
FROSTFS_IR_CONTRACTS_FROSTFSID=1943e9bb78a0fe2fe0c95fd2677eec2da6aa4aa5
FROSTFS_IR_CONTROL_GRPC_ENDPOINT=127.0.0.1:16512

View file

@ -25,6 +25,7 @@ endif
# Download FrostFS CLI
.ONESHELL:
get.cli: FROSTFS_CLI_FILE=./vendor/frostfs-cli
get.cli: FROSTFS_CLI_ARCHIVE_FILE=${FROSTFS_CLI_FILE}.tar.gz
get.cli: FROSTFS_CLI_PATH?=
get.cli:
@mkdir -p ./vendor
@ -33,8 +34,10 @@ ifeq (${FROSTFS_CLI_PATH},)
@echo "⇒ Download FrostFS CLI binary from ${FROSTFS_CLI_URL}"
@curl \
-ksSL "${FROSTFS_CLI_URL}" \
-o ${FROSTFS_CLI_FILE}
@chmod +x ${FROSTFS_CLI_FILE}
-o ${FROSTFS_CLI_ARCHIVE_FILE}
@tar -xvf ${FROSTFS_CLI_ARCHIVE_FILE} -C ./vendor | xargs -I {} \
mv ./vendor/{} ${FROSTFS_CLI_FILE}
@rm ${FROSTFS_CLI_ARCHIVE_FILE}
else
@echo "⇒ Copy local binary from ${FROSTFS_CLI_PATH}"
@cp ${FROSTFS_CLI_PATH} ${FROSTFS_CLI_FILE}

View file

@ -1,5 +1,6 @@
---
version: "2.4"
services:
ir01:
@ -12,19 +13,19 @@ services:
ir_int:
internet:
ipv4_address: ${IPV4_PREFIX}.61
stop_signal: SIGTERM
stop_grace_period: 15s
stop_signal: SIGKILL
volumes:
- ./az.json:/wallet.json
- ./az.key:/wallet01.key
- ./../../vendor/hosts:/etc/hosts
- ./../../vendor/locode_db:/locode/db
- ./../../vendor/frostfs-cli:/frostfs-cli
- ./healthcheck.sh:/healthcheck.sh
- ./cfg:/etc/frostfs/ir
env_file: [ ".env", ".ir.env", ".int_test.env" ]
command: [ "frostfs-ir", "--config", "/etc/frostfs/ir/config.yml" ]
healthcheck:
test: ["CMD-SHELL", "/frostfs-cli control ir healthcheck -q --wallet /wallet01.key --endpoint \"$$FROSTFS_IR_CONTROL_GRPC_ENDPOINT\""]
test: ["CMD-SHELL", "/healthcheck.sh"]
interval: 2s
timeout: 1s
retries: 5

6
services/ir/healthcheck.sh Executable file
View file

@ -0,0 +1,6 @@
#!/bin/sh
/frostfs-cli control healthcheck \
--endpoint "$FROSTFS_IR_CONTROL_GRPC_ENDPOINT" \
--wallet /wallet01.key --ir |
grep "Health status: READY"

4
services/ir/prepare.mk Normal file
View file

@ -0,0 +1,4 @@
# Deposit GAS from default wallet to FrostFS privnet contract
prepare.ir:
@./bin/config.sh int ContainerFee 0
@./bin/config.sh int ContainerAliasFee 0

View file

@ -1,3 +1,4 @@
version: '2.4'
services:
jaeger:
image: ${JAEGER_IMAGE}:${JAEGER_VERSION}
@ -19,10 +20,7 @@ services:
env_file: [ ".env", ".jaeger.env", ".int_test.env" ]
environment:
- COLLECTOR_OTLP_ENABLED=true
- SPAN_STORAGE_TYPE=badger
- BADGER_EPHEMERAL=false
- BADGER_DIRECTORY_VALUE=/badger/data
- BADGER_DIRECTORY_KEY=/badger/key
- MEMORY_MAX_TRACES=100000
networks:
jaeger_int:

View file

@ -20,12 +20,15 @@ endif
# Download FrostFS ADM tool
get.adm: FROSTFS_ADM_DEST=./vendor/frostfs-adm
get.adm: FROSTFS_ADM_ARCHIVE=frostfs-adm.tar.gz
get.adm:
ifeq (${FROSTFS_ADM_PATH},)
@echo "⇒ Download FrostFS ADM binary from ${FROSTFS_ADM_URL}"
@curl -skSL ${FROSTFS_ADM_URL} -o ${FROSTFS_ADM_DEST}
@chmod +x ${FROSTFS_ADM_DEST}
@curl -skSL ${FROSTFS_ADM_URL} -o ${FROSTFS_ADM_ARCHIVE}
@tar -xvf ${FROSTFS_ADM_ARCHIVE} -C ./vendor | xargs -I {} \
mv ./vendor/{} ${FROSTFS_ADM_DEST}
@rm ${FROSTFS_ADM_ARCHIVE}
else
@echo "⇒ Copy frostfs-adm binary from ${FROSTFS_ADM_PATH}"
@cp ${FROSTFS_ADM_PATH} ${FROSTFS_ADM_DEST}

View file

@ -1,5 +1,6 @@
---
version: "2.4"
services:
frostfs_morph_chain:
image: ${NEOGO_IMAGE}:${NEOGO_VERSION}
@ -19,14 +20,9 @@ services:
- ./config.yml:/wallets/config.yml
- ./../../vendor/hosts:/etc/hosts
- ./../../wallets/wallet.json:/wallets/wallet.json
- ./../s3_gate/wallet.json:/wallets/s3-wallet.json
- chains:/chains
networks:
chain_int:
internet:
external: true
name: basenet_internet
volumes:
chains:

View file

@ -1,56 +1,50 @@
ProtocolConfiguration:
Magic: 15405
MaxTraceableBlocks: 200000
TimePerBlock: 1s
SecondsPerBlock: 1
MemPoolSize: 50000
StandbyCommittee:
- 02b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc2
ValidatorsCount: 1
SeedList:
- 172.200.0.1:20333
VerifyBlocks: true
VerifyTransactions: true
StateRootInHeader: true
P2PSigExtensions: true
ApplicationConfiguration:
SkipBlockVerification: false
DBConfiguration:
Type: "boltdb"
BoltDBOptions:
FilePath: "/chains/morph.bolt"
P2P:
Addresses:
- ":20333"
DialTimeout: 3s
ProtoTickInterval: 2s
PingInterval: 30s
PingTimeout: 90s
MaxPeers: 10
AttemptConnPeers: 5
MinPeers: 0
FilePath: "./db/morph.bolt"
NodePort: 20333
Relay: true
Consensus:
Enabled: true
UnlockWallet:
Path: "./wallets/node-wallet.json"
Password: "one"
DialTimeout: 3
ProtoTickInterval: 2
PingInterval: 30
PingTimeout: 90
MaxPeers: 10
AttemptConnPeers: 5
MinPeers: 0
RPC:
Addresses:
- ":30333"
Address: 192.168.130.90
Enabled: true
SessionEnabled: true
EnableCORSWorkaround: false
MaxGasInvoke: 100
Port: 30333
P2PNotary:
Enabled: true
UnlockWallet:
Path: "./wallets/node-wallet.json"
Password: "one"
Prometheus:
Addresses:
- ":20001"
Enabled: true
Port: 20001
Pprof:
Addresses:
- ":20011"
Enabled: true
Port: 20011
UnlockWallet:
Path: "./wallets/node-wallet.json"
Password: "one"

1
services/nats/.hosts Normal file
View file

@ -0,0 +1 @@
IPV4_PREFIX.101 nats.LOCAL_DOMAIN

View file

@ -0,0 +1,7 @@
# Create new TLS certs for NATS server and clients
NATS_DIR=$(abspath services/nats)
get.nats:
@echo "⇒ Creating certs for NATS server and clients"
${NATS_DIR}/generate_cert.sh ${LOCAL_DOMAIN} > /dev/null

View file

@ -0,0 +1,31 @@
---
version: "2.4"
services:
nats:
image: ${NATS_IMAGE}:${NATS_VERSION}
domainname: ${LOCAL_DOMAIN}
hostname: nats
container_name: nats
restart: on-failure
dns:
- ${IPV4_PREFIX}.101
networks:
nats_int:
internet:
ipv4_address: ${IPV4_PREFIX}.101
volumes:
- ./../../vendor/hosts:/etc/hosts
- ./nats.conf:/etc/nats/frostfs-nats-server.conf
- ./server-cert.pem:/certs/server-cert.pem
- ./server-key.pem:/certs/server-key.pem
- ./ca-cert.pem:/certs/ca-cert.pem
stop_signal: SIGKILL
env_file: [ ".env", ".int_test.env" ]
command: ["-c", "/etc/nats/frostfs-nats-server.conf"]
networks:
nats_int:
internet:
external: true
name: basenet_internet

49
services/nats/generate_cert.sh Executable file
View file

@ -0,0 +1,49 @@
#!/bin/bash
source bin/helper.sh
WORKDIR=$(dirname "$0")
LOCAL_DOMAIN=$1
CA_KEY=$WORKDIR/ca-key.pem
CA_CRT=$WORKDIR/ca-cert.pem
SRV_KEY=$WORKDIR/server-key.pem
SRV_REQ=$WORKDIR/server-req.csr
SRV_CRT=$WORKDIR/server-cert.pem
CLI_KEY=$WORKDIR/client-key.pem
CLI_REQ=$WORKDIR/client-req.csr
CLI_CRT=$WORKDIR/client-cert.pem
SUBJ="/O=NSPCC"
if [[ ! -f $CA_KEY || ! -f $CA_CRT ]]; then
openssl req -newkey rsa:4096 -x509 -days 365 -nodes -keyout $CA_KEY -out $CA_CRT -subj $SUBJ 2>&1 ||
die "CA certificate was not created"
fi
if [[ ! -f $SRV_KEY || ! -f $SRV_CRT ]]; then
openssl req -newkey rsa:4096 -nodes -keyout $SRV_KEY -out $SRV_REQ -subj $SUBJ 2>&1 ||
die "Server certificate was not created"
openssl x509 -req -days 365 -set_serial 01 -in $SRV_REQ -out $SRV_CRT -CA $CA_CRT -CAkey $CA_KEY \
-extensions san -extfile <(printf "[san]\nsubjectAltName=DNS:nats.$LOCAL_DOMAIN") 2>&1 || {
rm $SRV_REQ
die "Server certificate was not signed by CA"
}
rm $SRV_REQ
fi
if [[ ! -f $CLI_KEY || ! -f $CLI_CRT ]]; then
openssl req -newkey rsa:4096 -nodes -keyout $CLI_KEY -out $CLI_REQ -subj $SUBJ 2>&1 ||
die "Client certificate was not created"
openssl x509 -req -days 365 -set_serial 01 -in $CLI_REQ -out $CLI_CRT -CA $CA_CRT -CAkey $CA_KEY 2>&1 || {
rm $CLI_REQ
die "Client certificate was not signed by CA"
}
rm $CLI_REQ
fi

15
services/nats/nats.conf Normal file
View file

@ -0,0 +1,15 @@
port: 4222
monitor_port: 8222
jetstream {
store_dir=nats
max_memory_store: 1GB
max_file_store: 2GB
}
tls {
cert_file: /certs/server-cert.pem
key_file: /certs/server-key.pem
ca_file: /certs/ca-cert.pem
verify: true
}

View file

@ -1,3 +1,4 @@
version: '2.4'
services:
prometheus:
image: ${PROMETHEUS_IMAGE}:${PROMETHEUS_VERSION}
@ -14,6 +15,8 @@ services:
- ./prometheus.yml:/etc/prometheus/prometheus.yml
command:
- --config.file=/etc/prometheus/prometheus.yml
ports:
- '9090:9090'
stop_signal: SIGKILL
env_file: [ ".env", ".prometheus.env", ".int_test.env" ]
@ -21,4 +24,4 @@ networks:
prometheus_int:
internet:
external: true
name: basenet_internet
name: basenet_internet

View file

@ -16,7 +16,4 @@ scrape_configs:
- targets: ['s3.frostfs.devenv:9090']
- job_name: 'neo-go'
static_configs:
- targets: ['morph-chain.frostfs.devenv:20001']
- job_name: 'inner-ring'
static_configs:
- targets: ['ir01.frostfs.devenv:9090']
- targets: ['morph-chain.frostfs.devenv:20001']

View file

@ -1,5 +1,6 @@
---
version: "2.4"
services:
rest_gate:
image: ${REST_GW_IMAGE}:${REST_GW_VERSION}
@ -15,8 +16,7 @@ services:
- ./wallet.json:/wallet.json
- ./../../vendor/hosts:/etc/hosts
- ./cfg:/etc/frostfs/rest
stop_signal: SIGTERM
stop_grace_period: 15s
stop_signal: SIGKILL
env_file: [ ".env", ".int_test.env" ]
command: [ "frostfs-rest-gw", "--config", "/etc/frostfs/rest/config.yml" ]
environment:

View file

@ -33,17 +33,3 @@ server:
wallet:
path: /wallet.json # Path to wallet
passphrase: "s3" # Passphrase to decrypt wallet
features:
md5:
enabled: true
control:
grpc:
endpoint: localhost:16515
frostfsid:
enabled: false
policy:
enabled: false

View file

@ -1,5 +1,6 @@
---
version: "2.4"
services:
s3_gate:
image: ${S3_GW_IMAGE}:${S3_GW_VERSION}
@ -12,26 +13,19 @@ services:
internet:
ipv4_address: ${IPV4_PREFIX}.82
volumes:
# Gate wallet
- ./wallet.json:/wallet.json
# Custom user wallets
- ./wallets:/wallets
# Default user wallet
- ./../../wallets/wallet.json:/wallets/wallet.json
- ./tls.key:/tls.key
- ./tls.crt:/tls.crt
- ./../../vendor/hosts:/etc/hosts
- ./cfg:/etc/frostfs/s3
- ./issue-creds.sh:/usr/bin/issue-creds.sh
stop_signal: SIGTERM
stop_grace_period: 15s
stop_signal: SIGKILL
env_file: [ ".env", ".s3.env", ".int_test.env" ]
command: [ "frostfs-s3-gw", "--config", "/etc/frostfs/s3/config.yml" ]
environment:
- S3_GW_RPC_ENDPOINT=http://morph-chain.${LOCAL_DOMAIN}:30333
- S3_GW_SERVER_0_ADDRESS=s3.${LOCAL_DOMAIN}:8080
- S3_GW_LISTEN_DOMAINS=s3.${LOCAL_DOMAIN}
- S3_GW_TREE_SERVICE=s01.${LOCAL_DOMAIN}:8080 s02.${LOCAL_DOMAIN}:8080 s03.${LOCAL_DOMAIN}:8080 s04.${LOCAL_DOMAIN}:8080
- S3_GW_TREE_SERVICE=s01.${LOCAL_DOMAIN}:8080
- S3_GW_PEERS_0_ADDRESS=s01.${LOCAL_DOMAIN}:8080
- S3_GW_PEERS_0_WEIGHT=0.2
- S3_GW_PEERS_1_ADDRESS=s02.${LOCAL_DOMAIN}:8080
@ -40,8 +34,6 @@ services:
- S3_GW_PEERS_2_WEIGHT=0.2
- S3_GW_PEERS_3_ADDRESS=s04.${LOCAL_DOMAIN}:8080
- S3_GW_PEERS_3_WEIGHT=0.2
- AUTHMATE_WALLET_PASSPHRASE=
- AUTHMATE_WALLET_CONTRACT_PASSPHRASE=s3
networks:
s3_gate_int:

View file

@ -1,41 +0,0 @@
#!/bin/bash
initUser() {
/bin/frostfs-s3-authmate register-user \
--wallet $WALLET_PATH \
--rpc-endpoint http://morph-chain.frostfs.devenv:30333 \
--username $USERNAME \
--contract-wallet /wallet.json 1> /dev/null && touch $WALLET_CACHE/$USERNAME
}
issueCreds() {
/bin/frostfs-s3-authmate issue-secret \
--wallet $WALLET_PATH \
--peer s01.frostfs.devenv:8080 \
--gate-public-key $S3_GATE_PUBLIC_KEY \
--container-placement-policy "REP 3"
}
set -e
WALLET_PATH=/wallets/$2
if [[ -z "$2" ]]; then
WALLET_PATH=/wallets/wallet.json
fi
S3_GATE_PUBLIC_KEY=$3
if [[ -z "$3" ]]; then
S3_GATE_PUBLIC_KEY=0313b1ac3a8076e155a7e797b24f0b650cccad5941ea59d7cfd51a024a8b2a06bf
fi
WALLET_CACHE=/data/wallets
mkdir -p $WALLET_CACHE
USERNAME=$(echo $WALLET_PATH | md5sum | cut -d' ' -f1)
if [ ! -e $WALLET_CACHE/$USERNAME ]; then
initUser
fi
if [ $1 == "s3" ]; then
issueCreds
fi

View file

@ -1,14 +0,0 @@
.PHONY: s3cred register
password?=
contract_password?=s3
gate_public_key?=
wallet?=
# Register wallet & generate S3 credentials
s3cred:
@docker exec -e AUTHMATE_WALLET_PASSPHRASE="$(password)" -e AUTHMATE_WALLET_CONTRACT_PASSPHRASE="$(contract_password)" s3_gate /usr/bin/issue-creds.sh s3 "$(wallet)" "$(gate_public_key)"
# Only registers user wallet
register:
@docker exec -e AUTHMATE_WALLET_PASSPHRASE="$(password)" -e AUTHMATE_WALLET_CONTRACT_PASSPHRASE="$(contract_password)" s3_gate /usr/bin/issue-creds.sh native "$(wallet)"

View file

@ -1 +0,0 @@
../../.env

View file

@ -1 +0,0 @@
IPV4_PREFIX.84 lifecycler.LOCAL_DOMAIN

View file

@ -1 +0,0 @@
../../.int_test.env

View file

@ -1,42 +0,0 @@
logger:
level: debug
prometheus:
enabled: true
address: :9090
lifecycle:
job_fetcher_buffer: 1000
executor_pool_size: 100
frostfs:
stream_timeout: 10s
connect_timeout: 10s
healthcheck_timeout: 15s
rebalance_interval: 60s
pool_error_threshold: 100
tree_pool_max_attempts: 4
credential:
use: wallets
source:
wallets:
- path: /wallet.json
address: NTt1rxvmEDxEuuogLxs2xgxA71qhVaUcN7
passphrase: "cycle"
- path: /user-wallet.json
address: NbUgTSFvPmsRxmGeWpuuGeJUoRoi6PErcM
passphrase: ""
morph:
reconnect_clients_interval: 30s
dial_timeout: 5s
contract:
netmap: netmap.frostfs
frostfsid: frostfsid.frostfs
container: container.frostfs
# Wallet configuration
wallet:
path: /wallet.json # Path to wallet
passphrase: "cycle" # Passphrase to decrypt wallet

View file

@ -1,38 +0,0 @@
---
version: "2.4"
services:
s3_lifecycler:
image: ${S3_LIFECYCLER_IMAGE}:${S3_LIFECYCLER_VERSION}
domainname: ${LOCAL_DOMAIN}
hostname: s3_lifecycler
container_name: s3_lifecycler
restart: on-failure
networks:
s3_lifecycler_int:
internet:
ipv4_address: ${IPV4_PREFIX}.84
volumes:
- ./wallet.json:/wallet.json
- ./../../vendor/hosts:/etc/hosts
- ./cfg:/etc/frostfs/s3-lifecycler
- ./../../wallets/wallet.json:/user-wallet.json
stop_signal: SIGKILL
env_file: [ ".env", ".int_test.env" ]
command: [ "frostfs-s3-lifecycler", "--config", "/etc/frostfs/s3-lifecycler/config.yml" ]
environment:
- S3_LIFECYCLER_MORPH_RPC_ENDPOINT_0_ADDRESS=ws://morph-chain:30333/ws
- S3_LIFECYCLER_FROSTFS_PEERS_0_ADDRESS=s01.${LOCAL_DOMAIN}:8080
- S3_LIFECYCLER_FROSTFS_PEERS_0_WEIGHT=0.2
- S3_LIFECYCLER_FROSTFS_PEERS_1_ADDRESS=s02.${LOCAL_DOMAIN}:8080
- S3_LIFECYCLER_FROSTFS_PEERS_1_WEIGHT=0.2
- S3_LIFECYCLER_FROSTFS_PEERS_2_ADDRESS=s03.${LOCAL_DOMAIN}:8080
- S3_LIFECYCLER_FROSTFS_PEERS_2_WEIGHT=0.2
- S3_LIFECYCLER_FROSTFS_PEERS_3_ADDRESS=s04.${LOCAL_DOMAIN}:8080
- S3_LIFECYCLER_FROSTFS_PEERS_3_WEIGHT=0.2
networks:
s3_lifecycler_int:
internet:
external: true
name: basenet_internet

View file

@ -1,30 +0,0 @@
{
"version": "1.0",
"accounts": [
{
"address": "NTt1rxvmEDxEuuogLxs2xgxA71qhVaUcN7",
"key": "6PYR3XurAyTzVeDG5WV2Z8vnGdySw3mTLuKjr6Nwo7tae64SJ7XjZSMMPQ",
"label": "lifecycler",
"contract": {
"script": "DCED9z0M+WSGfXZGxYLj1yYwmgxJXE/kNA4+oWNi0q1uKCdBVuezJw==",
"parameters": [
{
"name": "parameter0",
"type": "Signature"
}
],
"deployed": false
},
"lock": false,
"isDefault": false
}
],
"scrypt": {
"n": 16384,
"r": 8,
"p": 8
},
"extra": {
"Tokens": null
}
}

View file

@ -1,11 +1,6 @@
# Logger section
logger:
level: debug # Minimum enabled logging level
loki:
enabled: true
endpoint: "loki.frostfs.devenv:3100/api/prom/push"
max_batch_delay: 1s
max_batch_size: 200
# Profiler section
pprof:
@ -19,12 +14,6 @@ prometheus:
address: :9090 # Server address
shutdown_timeout: 15s # Timeout for metrics HTTP server graceful shutdown
# Application tracing section
tracing:
enabled: true
exporter: otlp_grpc
endpoint: "jaeger.frostfs.devenv:4317"
# Morph section
morph:
dial_timeout: 30s # Timeout for side chain NEO RPC client connection
@ -32,6 +21,18 @@ morph:
- address: ws://morph-chain:30333/ws
priority: 1
# Common storage node settings
node:
attribute_0: "User-Agent:FrostFS/0.34"
notification:
enabled: true # Turn on object notification service
endpoint: "tls://nats.frostfs.devenv:4222" # Notification server endpoint
timeout: "6s" # Timeout for object notification client connection
default_topic: "test" # Default topic for object notifications if not found in object's meta
certificate: "/etc/frostfs-node/nats.tls.cert" # Path to TLS certificate
key: "/etc/frostfs-node/nats.tls.key" # Path to TLS key
ca: "/etc/frostfs-node/nats.ca.crt" # Path to optional CA certificate
# Tree section
tree:
enabled: true
@ -41,7 +42,7 @@ storage:
shard:
0:
writecache:
enabled: true
enabled: false
path: /storage/wc0 # Write-cache root directory
metabase:
@ -61,7 +62,7 @@ storage:
1:
writecache:
enabled: true
enabled: false
path: /storage/wc1 # Write-cache root directory
metabase:

View file

@ -1,5 +1,6 @@
---
version: "2.4"
services:
storage01:
image: ${NODE_IMAGE}:${NODE_VERSION}
@ -17,10 +18,13 @@ services:
- storage_s01:/storage
- ./../../vendor/frostfs-cli:/frostfs-cli
- ./cli-cfg.yml:/cli-cfg.yml
- ./healthcheck.sh:/healthcheck.sh
- ./s04tls.crt:/etc/ssl/certs/s04tls.crt
- ../nats/client-cert.pem:/etc/frostfs-node/nats.tls.cert
- ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key
- ../nats/ca-cert.pem:/etc/frostfs-node/nats.ca.crt
- ./cfg:/etc/frostfs/storage
stop_signal: SIGTERM
stop_grace_period: 15s
stop_signal: SIGKILL
env_file: [ ".env", ".storage.env", ".int_test.env" ]
command: [ "frostfs-node", "--config", "/etc/frostfs/storage/config.yml" ]
environment:
@ -29,11 +33,10 @@ services:
- FROSTFS_NODE_ADDRESSES=s01.${LOCAL_DOMAIN}:8080
- FROSTFS_GRPC_0_ENDPOINT=s01.${LOCAL_DOMAIN}:8080
- FROSTFS_CONTROL_GRPC_ENDPOINT=s01.${LOCAL_DOMAIN}:8081
- FROSTFS_NODE_ATTRIBUTE_0=User-Agent:FrostFS/${NODE_VERSION}
- FROSTFS_NODE_ATTRIBUTE_1=UN-LOCODE:RU MOW
- FROSTFS_NODE_ATTRIBUTE_2=Price:22
healthcheck:
test: ["CMD-SHELL", "/frostfs-cli control healthcheck -q -c /cli-cfg.yml --endpoint \"$$FROSTFS_CONTROL_GRPC_ENDPOINT\""]
test: ["CMD-SHELL", "/healthcheck.sh"]
interval: 2s
timeout: 1s
retries: 5
@ -55,10 +58,13 @@ services:
- storage_s02:/storage
- ./../../vendor/frostfs-cli:/frostfs-cli
- ./cli-cfg.yml:/cli-cfg.yml
- ./healthcheck.sh:/healthcheck.sh
- ./s04tls.crt:/etc/ssl/certs/s04tls.crt
- ../nats/client-cert.pem:/etc/frostfs-node/nats.tls.cert
- ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key
- ../nats/ca-cert.pem:/etc/frostfs-node/nats.ca.crt
- ./cfg:/etc/frostfs/storage
stop_signal: SIGTERM
stop_grace_period: 15s
stop_signal: SIGKILL
env_file: [ ".env", ".storage.env", ".int_test.env" ]
command: [ "frostfs-node", "--config", "/etc/frostfs/storage/config.yml" ]
environment:
@ -67,11 +73,10 @@ services:
- FROSTFS_NODE_ADDRESSES=s02.${LOCAL_DOMAIN}:8080
- FROSTFS_GRPC_0_ENDPOINT=s02.${LOCAL_DOMAIN}:8080
- FROSTFS_CONTROL_GRPC_ENDPOINT=s02.${LOCAL_DOMAIN}:8081
- FROSTFS_NODE_ATTRIBUTE_0=User-Agent:FrostFS/${NODE_VERSION}
- FROSTFS_NODE_ATTRIBUTE_1=UN-LOCODE:RU LED
- FROSTFS_NODE_ATTRIBUTE_2=Price:33
healthcheck:
test: ["CMD-SHELL", "/frostfs-cli control healthcheck -q -c /cli-cfg.yml --endpoint \"$$FROSTFS_CONTROL_GRPC_ENDPOINT\""]
test: ["CMD-SHELL", "/healthcheck.sh"]
interval: 2s
timeout: 1s
retries: 5
@ -93,10 +98,13 @@ services:
- storage_s03:/storage
- ./../../vendor/frostfs-cli:/frostfs-cli
- ./cli-cfg.yml:/cli-cfg.yml
- ./healthcheck.sh:/healthcheck.sh
- ./s04tls.crt:/etc/ssl/certs/s04tls.crt
- ../nats/client-cert.pem:/etc/frostfs-node/nats.tls.cert
- ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key
- ../nats/ca-cert.pem:/etc/frostfs-node/nats.ca.crt
- ./cfg:/etc/frostfs/storage
stop_signal: SIGTERM
stop_grace_period: 15s
stop_signal: SIGKILL
env_file: [ ".env", ".storage.env", ".int_test.env" ]
command: [ "frostfs-node", "--config", "/etc/frostfs/storage/config.yml" ]
environment:
@ -105,11 +113,10 @@ services:
- FROSTFS_NODE_ADDRESSES=s03.${LOCAL_DOMAIN}:8080
- FROSTFS_GRPC_0_ENDPOINT=s03.${LOCAL_DOMAIN}:8080
- FROSTFS_CONTROL_GRPC_ENDPOINT=s03.${LOCAL_DOMAIN}:8081
- FROSTFS_NODE_ATTRIBUTE_0=User-Agent:FrostFS/${NODE_VERSION}
- FROSTFS_NODE_ATTRIBUTE_1=UN-LOCODE:SE STO
- FROSTFS_NODE_ATTRIBUTE_2=Price:11
healthcheck:
test: ["CMD-SHELL", "/frostfs-cli control healthcheck -q -c /cli-cfg.yml --endpoint \"$$FROSTFS_CONTROL_GRPC_ENDPOINT\""]
test: ["CMD-SHELL", "/healthcheck.sh"]
interval: 2s
timeout: 1s
retries: 5
@ -131,11 +138,14 @@ services:
- storage_s04:/storage
- ./../../vendor/frostfs-cli:/frostfs-cli
- ./cli-cfg.yml:/cli-cfg.yml
- ./healthcheck.sh:/healthcheck.sh
- ./s04tls.crt:/tls.crt
- ./s04tls.key:/tls.key
- ../nats/client-cert.pem:/etc/frostfs-node/nats.tls.cert
- ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key
- ../nats/ca-cert.pem:/etc/frostfs-node/nats.ca.crt
- ./cfg:/etc/frostfs/storage
stop_signal: SIGTERM
stop_grace_period: 15s
stop_signal: SIGKILL
env_file: [ ".env", ".storage.env", ".int_test.env" ]
command: [ "frostfs-node", "--config", "/etc/frostfs/storage/config.yml" ]
environment:
@ -149,11 +159,10 @@ services:
- FROSTFS_GRPC_1_TLS_ENABLED=true
- FROSTFS_GRPC_1_TLS_CERTIFICATE=/tls.crt
- FROSTFS_GRPC_1_TLS_KEY=/tls.key
- FROSTFS_NODE_ATTRIBUTE_0=User-Agent:FrostFS/${NODE_VERSION}
- FROSTFS_NODE_ATTRIBUTE_1=UN-LOCODE:FI HEL
- FROSTFS_NODE_ATTRIBUTE_2=Price:44
healthcheck:
test: ["CMD-SHELL", "/frostfs-cli control healthcheck -q -c /cli-cfg.yml --endpoint \"$$FROSTFS_CONTROL_GRPC_ENDPOINT\""]
test: ["CMD-SHELL", "/healthcheck.sh"]
interval: 2s
timeout: 1s
retries: 5

View file

@ -19,7 +19,7 @@ if [[ ! -f ${CERT} ]]; then
) > ${SSL_CONFIG}
openssl req -new -newkey rsa:4096 -x509 -sha256 -days 365 -nodes \
-subj "/C=RU/ST=SPB/L=St.Petersburg/O=TrueCloudLab/OU=TrueCloudLab/CN=s04.${LOCAL_DOMAIN}" \
-subj "/C=RU/ST=SPB/L=St.Petersburg/O=NSPCC/OU=NSPCC/CN=s04.${LOCAL_DOMAIN}" \
-keyout "${KEY}" -out "${CERT}" -extensions san -config "${SSL_CONFIG}" &> /dev/null || {
die "Failed to generate SSL certificate for s04"
}

View file

@ -0,0 +1,5 @@
#!/bin/sh
/frostfs-cli control healthcheck -c /cli-cfg.yml \
--endpoint "$FROSTFS_CONTROL_GRPC_ENDPOINT" |
grep "Health status: READY"