Release v1.6.4

Update s3-gw to fix aws-chunked encoding

Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>

.env

@@ -1,3 +1,3 @@
 FROSTFS_HUB_IMAGE=truecloudlab/frostfs
 AIO_IMAGE=truecloudlab/frostfs-aio
-AIO_VERSION=1.5.0-dev.1
+AIO_VERSION=1.6.4

Dockerfile

@@ -1,18 +1,18 @@
 ARG FROSTFS_HUB_IMAGE=truecloudlab/frostfs
 ARG NEOGO_HUB_IMAGE=nspccdev/neo-go
 
-ARG FROSTFS_CORE_TAG=0.38.6
-ARG FROSTFS_HTTP_GATE_TAG=0.29.0
-ARG FROSTFS_S3_GATE_TAG=0.29.3-2-gff7f963f
-ARG NEOGO_TAG=0.104.0
+ARG FROSTFS_CORE_TAG=0.42.15
+ARG FROSTFS_HTTP_GATE_TAG=0.30.2
+ARG FROSTFS_S3_GATE_TAG=0.30.7
+ARG NEOGO_TAG=0.106.0
 
-FROM ${NEOGO_HUB_IMAGE}:${NEOGO_TAG} as neo-go
-FROM ${FROSTFS_HUB_IMAGE}-adm:${FROSTFS_CORE_TAG} as frostfs-adm
-FROM ${FROSTFS_HUB_IMAGE}-cli:${FROSTFS_CORE_TAG} as frostfs-cli
-FROM ${FROSTFS_HUB_IMAGE}-ir:${FROSTFS_CORE_TAG} as frostfs-ir
-FROM ${FROSTFS_HUB_IMAGE}-storage:${FROSTFS_CORE_TAG} as frostfs-storage
-FROM ${FROSTFS_HUB_IMAGE}-s3-gw:${FROSTFS_S3_GATE_TAG} as frostfs-s3-gw
-FROM ${FROSTFS_HUB_IMAGE}-http-gw:${FROSTFS_HTTP_GATE_TAG} as frostfs-http-gw
+FROM ${NEOGO_HUB_IMAGE}:${NEOGO_TAG} AS neo-go
+FROM ${FROSTFS_HUB_IMAGE}-adm:${FROSTFS_CORE_TAG} AS frostfs-adm
+FROM ${FROSTFS_HUB_IMAGE}-cli:${FROSTFS_CORE_TAG} AS frostfs-cli
+FROM ${FROSTFS_HUB_IMAGE}-ir:${FROSTFS_CORE_TAG} AS frostfs-ir
+FROM ${FROSTFS_HUB_IMAGE}-storage:${FROSTFS_CORE_TAG} AS frostfs-storage
+FROM ${FROSTFS_HUB_IMAGE}-s3-gw:${FROSTFS_S3_GATE_TAG} AS frostfs-s3-gw
+FROM ${FROSTFS_HUB_IMAGE}-http-gw:${FROSTFS_HTTP_GATE_TAG} AS frostfs-http-gw
 
 # Executable image
 FROM alpine AS frostfs-aio
@@ -46,6 +46,7 @@ COPY ./morph/node-config.yaml /config/node-config.yaml
 COPY ./http-gw/http-gw-config.yaml /config/http-gw-config.yaml
 COPY ./http-gw/http-gw-wallet.json /config/http-gw-wallet.json
 COPY ./s3-gw/rules.json /config/bearer-rules.json
+COPY ./s3-gw/regions.json /config/regions.json
 COPY ./s3-gw/s3-gw-config.yaml /config/s3-gw-config.yaml
 COPY ./s3-gw/s3-gw-wallet.json /config/s3-gw-wallet.json
 COPY ./s3-gw/user-wallet.json /config/user-wallet.json
@@ -56,7 +57,9 @@ COPY ./vendor/locode_db /config/locode.db
 COPY ./vendor/contracts/ /config/contracts
 
 COPY ./bin/init-aio.sh /usr/bin/init-aio.sh
+COPY ./bin/issue-creds.sh /usr/bin/issue-creds.sh
 
 ENV AUTHMATE_WALLET_PASSPHRASE=""
+ENV AUTHMATE_WALLET_CONTRACT_PASSPHRASE=""
 
 ENTRYPOINT ["/usr/bin/init-aio.sh"]

Dockerfile.custom

@@ -1,18 +1,18 @@
 ARG FROSTFS_HUB_IMAGE=truecloudlab/frostfs
 ARG NEOGO_HUB_IMAGE=nspccdev/neo-go
 
-ARG FROSTFS_CORE_TAG=0.38.6
-ARG FROSTFS_HTTP_GATE_TAG=0.29.0
-ARG FROSTFS_S3_GATE_TAG=0.29.3-2-gff7f963f
-ARG NEOGO_TAG=0.104.0
+ARG FROSTFS_CORE_TAG=0.42.15
+ARG FROSTFS_HTTP_GATE_TAG=0.30.2
+ARG FROSTFS_S3_GATE_TAG=0.30.7
+ARG NEOGO_TAG=0.106.0
 
-FROM ${NEOGO_HUB_IMAGE}:${NEOGO_TAG} as neo-go
-FROM ${FROSTFS_HUB_IMAGE}-adm:${FROSTFS_CORE_TAG} as frostfs-adm
-FROM ${FROSTFS_HUB_IMAGE}-cli:${FROSTFS_CORE_TAG} as frostfs-cli
-FROM ${FROSTFS_HUB_IMAGE}-ir:${FROSTFS_CORE_TAG} as frostfs-ir
-FROM ${FROSTFS_HUB_IMAGE}-storage:${FROSTFS_CORE_TAG} as frostfs-storage
-FROM ${FROSTFS_HUB_IMAGE}-s3-gw:${FROSTFS_S3_GATE_TAG} as frostfs-s3-gw
-FROM ${FROSTFS_HUB_IMAGE}-http-gw:${FROSTFS_HTTP_GATE_TAG} as frostfs-http-gw
+FROM ${NEOGO_HUB_IMAGE}:${NEOGO_TAG} AS neo-go
+FROM ${FROSTFS_HUB_IMAGE}-adm:${FROSTFS_CORE_TAG} AS frostfs-adm
+FROM ${FROSTFS_HUB_IMAGE}-cli:${FROSTFS_CORE_TAG} AS frostfs-cli
+FROM ${FROSTFS_HUB_IMAGE}-ir:${FROSTFS_CORE_TAG} AS frostfs-ir
+FROM ${FROSTFS_HUB_IMAGE}-storage:${FROSTFS_CORE_TAG} AS frostfs-storage
+FROM ${FROSTFS_HUB_IMAGE}-s3-gw:${FROSTFS_S3_GATE_TAG} AS frostfs-s3-gw
+FROM ${FROSTFS_HUB_IMAGE}-http-gw:${FROSTFS_HTTP_GATE_TAG} AS frostfs-http-gw
 
 # Executable image
 FROM alpine AS frostfs-aio
@@ -46,6 +46,7 @@ COPY ./morph/node-config.yaml /config/node-config.yaml
 COPY ./http-gw/http-gw-config.yaml /config/http-gw-config.yaml
 COPY ./http-gw/http-gw-wallet.json /config/http-gw-wallet.json
 COPY ./s3-gw/rules.json /config/bearer-rules.json
+COPY ./s3-gw/regions.json /config/regions.json
 COPY ./s3-gw/s3-gw-config.yaml /config/s3-gw-config.yaml
 COPY ./s3-gw/s3-gw-wallet.json /config/s3-gw-wallet.json
 COPY ./s3-gw/user-wallet.json /config/user-wallet.json
@@ -58,5 +59,6 @@ COPY ./vendor/contracts/ /config/contracts
 COPY ./bin/* /usr/bin/
 
 ENV AUTHMATE_WALLET_PASSPHRASE=""
+ENV AUTHMATE_WALLET_CONTRACT_PASSPHRASE=""
 
 ENTRYPOINT ["/usr/bin/init-aio.sh"]

Dockerfile.local

@@ -18,6 +18,7 @@ COPY ./bin/frostfs-node /usr/bin/frostfs-node
 COPY ./bin/frostfs-s3-gw /usr/bin/frostfs-s3-gw
 COPY ./bin/frostfs-s3-authmate /usr/bin/frostfs-s3-authmate
 COPY ./bin/frostfs-http-gw /usr/bin/frostfs-http-gw
+COPY ./bin/issue-creds.sh /usr/bin/issue-creds.sh
 
 COPY ./adm/frostfs-adm.yml /config/frostfs-adm.yml
 COPY ./ir/cli-cfg.yaml /config/cli-cfg-ir.yaml
@@ -30,6 +31,7 @@ COPY ./morph/node-config.yaml /config/node-config.yaml
 COPY ./http-gw/http-gw-config.yaml /config/http-gw-config.yaml
 COPY ./http-gw/http-gw-wallet.json /config/http-gw-wallet.json
 COPY ./s3-gw/rules.json /config/bearer-rules.json
+COPY ./s3-gw/regions.json /config/regions.json
 COPY ./s3-gw/s3-gw-config.yaml /config/s3-gw-config.yaml
 COPY ./s3-gw/s3-gw-wallet.json /config/s3-gw-wallet.json
 COPY ./s3-gw/user-wallet.json /config/user-wallet.json
@@ -42,5 +44,6 @@ COPY ./vendor/contracts/ /config/contracts
 COPY ./bin/init-aio.sh /usr/bin/init-aio.sh
 
 ENV AUTHMATE_WALLET_PASSPHRASE=""
+ENV AUTHMATE_WALLET_CONTRACT_PASSPHRASE=""
 
 ENTRYPOINT ["/usr/bin/init-aio.sh"]

Makefile

@@ -9,6 +9,7 @@ VERSION ?= "$(shell git describe --tags --match "v*" 2>/dev/null || git rev-pars
 
 # Variables for docker
 AIO_IMAGE ?= "truecloudlab/frostfs-aio"
+LOG_DIR ?= "/var/log"
 
 # Variables for compose
 COMPOSE_CMD_UP ?= docker-compose up -d
@@ -62,19 +63,19 @@ clean: down
 
 # Generate S3 credentials
 s3cred:
-	@docker exec aio /usr/bin/frostfs-s3-authmate issue-secret \
-		--wallet /config/user-wallet.json \
-		--peer localhost:8080 \
-		--gate-public-key $(S3_GATE_PUBLIC_KEY) \
-		--container-placement-policy "REP 1"
+	@docker exec aio /usr/bin/issue-creds.sh s3
 
 # Generate S3 credentials based on imported wallets
 s3cred-custom:
-	@docker exec aio /usr/bin/frostfs-s3-authmate issue-secret \
-		--wallet /wallets/$(wallet) \
-		--peer localhost:8080 \
-		--gate-public-key $(S3_GATE_PUBLIC_KEY) \
-		--container-placement-policy "REP 1"
+	@docker exec aio /usr/bin/issue-creds.sh s3 $(wallet)
+
+# Register user-wallet.json in FrostFS ID contract
+cred:
+	@docker exec aio /usr/bin/issue-creds.sh native
+
+# Register custom wallet in FrostFS ID contract
+cred-custom:
+	@docker exec aio /usr/bin/issue-creds.sh native $(wallet)
 
 enable-oracle:
 	@docker exec aio neo-go contract invokefunction -r http://localhost:30333 \
@@ -91,8 +92,37 @@ enable-oracle:
 tick.epoch:
 	@docker exec aio /usr/bin/frostfs-adm --config /config/frostfs-adm.yml morph force-new-epoch
 
+# Create new wallet
+WALLET?=wallets/wallet1.json
+.PHONY: wallet
+wallet:
+	@! test -f "$(WALLET)" || { echo "File exists: $(WALLET)"; exit 1; }
+	@docker exec -it aio /usr/bin/neo-go wallet init -a -w "$(WALLET)"
+
+# Refill GAS in arbitrary WALLET (must be under wallets/ directory)
+GAS?=50.0
+.PHONY: refill
+refill:
+	@docker exec aio /usr/bin/frostfs-adm --config /config/frostfs-adm.yml \
+		morph refill-gas --storage-wallet=/$(WALLET) --gas=$(GAS)
+	@$(MAKE) --no-print-directory balance
+
+# Show wallet balance
+.PHONY: balance
+balance:
+	@echo "Balance for $(WALLET)"
+	@docker exec aio /usr/bin/neo-go wallet nep17 balance -r http://localhost:30333 -w "$(WALLET)"
+
 # Show current version
 version:
 	@echo $(VERSION)
 	@echo "frostfs-node: $(AIO_VERSION)"
 	@echo "neo-go: $(NEOGO_VERSION)"
+
+# Show all logs of the FrostFS service
+logs/%:
+	@docker exec aio cat ${LOG_DIR}/$*
+
+# Streaming of the frostFS service logs
+logs-stream/%:
+	@docker exec aio tail -f ${LOG_DIR}/$*

README.md

@@ -62,7 +62,8 @@ Data and the system configuration is stored in container's volume.
 Next time container is started, it will take less time to initialize.
 
 Before updating image version, reset `frostfs-aio` by clearing its local volume
-before starting container.
+before starting container. Update may contain incompatible changes, e.g. neo-go
+database format update. This repository does not provide any migration tools.
 
 ```
 $ make clean
@@ -287,16 +288,55 @@ If everything is rigth, the stdout will show a pretty-printed contents of the wa
 wallet successfully created, file location is /wallets/new-wallet.json
 ```
 
+You can use `make wallet WALLET=wallets/new-wallet.json`
+to execute the steps listed above a little faster.
+
+
+### Transfering GAS tokens to wallet
+
+To add GAS to a new wallet use `make refill`
+
+```console
+$ make refill GAS=12.3 WALLET=wallets/new-wallet.json
+Waiting for transactions to persist...
+Balance for wallets/new-wallet.json
+Account NgB6jnyVsMg6E3gQQLXoccZ3JBMLsgiut3
+GAS: GasToken (d2a4cff31913016155e38e474a2c06d08be276cf)
+        Amount : 12.3
+        Updated: 25069
+```
+
+### Checking wallet balance
+
+```console
+$ make balance WALLET=wallets/new-wallet.json
+Balance for wallets/new-wallet.json
+Account NgB6jnyVsMg6E3gQQLXoccZ3JBMLsgiut3
+GAS: GasToken (d2a4cff31913016155e38e474a2c06d08be276cf)
+        Amount : 12.3
+        Updated: 25069
+```
+
 
 ## frostfs-cli interface
 
+### Native credentials
+
+To use wallet file, it should be register in FrostFS ID contract and have  
+policies to perform FrostFS operations.
+
+Use `make cred` to register `s3-gw/user-wallet.json`. For custom wallets from
+`wallet` dir, use `make cred-custom wallet=wallet1.json`.
+
+These commands are similar to `make s3cred` and `make s3cred-custom`.
+
 ### Create container with frostfs-cli
 
 ``` sh
-$ frostfs-cli -r localhost:8080 -w /config/user-wallet.json \
+$ frostfs-cli -r localhost:8080 -w s3-gw/user-wallet.json \
             --address NWeByJPgNC97F83hTUnSbnZSBKaFvk5HNw \
             container create \
-            --policy "REP 1" --basic-acl public-read-write --await
+            --policy "REP 1" --await
 container ID: GfWw35kHds7gKWmSvW7Zi4U39K7NMLK8EfXBQ5FPJA46
 awaiting...
 container has been persisted on sidechain
@@ -305,7 +345,7 @@ container has been persisted on sidechain
 ### Put an object with frostfs-cli
 
 ``` sh
-$ frostfs-cli -r localhost:8080 -w /config/user-wallet.json \
+$ frostfs-cli -r localhost:8080 -w s3-gw/user-wallet.json \
             --address NWeByJPgNC97F83hTUnSbnZSBKaFvk5HNw \
             object put \
             --cid GfWw35kHds7gKWmSvW7Zi4U39K7NMLK8EfXBQ5FPJA46 \
@@ -318,9 +358,25 @@ $ frostfs-cli -r localhost:8080 -w /config/user-wallet.json \
 ### Get and object with frostfs-cli
 
 ``` sh
-$ frostfs-cli -r localhost:8080 -w /config/user-wallet.json \
+$ frostfs-cli -r localhost:8080 -w s3-gw/user-wallet.json \
             --address NWeByJPgNC97F83hTUnSbnZSBKaFvk5HNw \
             object get \
             --cid GfWw35kHds7gKWmSvW7Zi4U39K7NMLK8EfXBQ5FPJA46 \
             --oid HByVC9A34i22BnzW3n83z9vEMxuYZoC7nNu11ZvGeCTe > new_cat.jpg
 ```
+
+## Viewing logs
+
+There are two targets for viewing logs.
+
+You can view all the logs by replacing the desired service name:
+
+``` sh
+make logs/frostfs-node
+```
+
+You can also make a continuous stream of logs by replacing the desired service name:
+
+``` sh
+make stream-logs/frostfs-node
+```

bin/init-aio.sh

@@ -14,7 +14,7 @@ die() {
 runBlockchain() {
   stage "Starting the blockchain"
 
-  /usr/bin/neo-go node --config-path /config --privnet >dev/null 2>&1 &
+  /usr/bin/neo-go node --config-path /config --privnet |& tee -a ${LOG_DIR}/neo-go &
 
   while [[ "$(curl -s -o /dev/null -w %{http_code} localhost:30333)" != "422" ]];
   do
@@ -31,13 +31,15 @@ configure() {
 
   /usr/bin/frostfs-adm morph refill-gas --config /config/frostfs-adm.yml --storage-wallet /config/wallet-sn.json --gas 10.0 || die "Failed to transfer GAS to alphabet wallets"
 
+  /usr/bin/frostfs-adm morph proxy-add-account --config /config/frostfs-adm.yml --account NejLbQpojKJWec4NQRMBhzsrmCyhXfGJJe || die "Failed to set storage wallet as proxy wallet"
+
   /usr/bin/frostfs-adm morph proxy-add-account --config /config/frostfs-adm.yml --account NN1RQ3qwnvDMVNsQw4WPkKi7BrjxdVTDZp || die "Failed to set s3 gateway wallet as proxy wallet"
 }
 
 runServices() {
   stage "Running services"
 
-  /usr/bin/frostfs-ir --config /config/config-ir.yaml >/dev/null 2>&1 &
+  /usr/bin/frostfs-ir --config /config/config-ir.yaml |& tee -a ${LOG_DIR}/frostfs-ir &
 
   while [[ -z "$(/usr/bin/frostfs-cli control ir healthcheck --endpoint localhost:16512 -c /config/cli-cfg-ir.yaml | grep 'Health status: READY')" ]];
   do
@@ -45,7 +47,7 @@ runServices() {
   done
 
   set -m
-  /usr/bin/frostfs-node --config /config/config-sn.yaml >/dev/null 2>&1 &
+  /usr/bin/frostfs-node --config /config/config-sn.yaml |& tee -a ${LOG_DIR}/frostfs-node &
 
   while [[ -z "$(/usr/bin/frostfs-cli control healthcheck --endpoint localhost:16513 -c /config/cli-cfg-sn.yaml | grep 'Health status: READY')" ]];
   do
@@ -63,8 +65,8 @@ runServices() {
     sleep 2
   done
 
-  /usr/bin/frostfs-s3-gw --config /config/s3-gw-config.yaml &
-  /usr/bin/frostfs-http-gw --config /config/http-gw-config.yaml &
+  /usr/bin/frostfs-s3-gw --config /config/s3-gw-config.yaml |& tee -a ${LOG_DIR}/frostfs-s3-gw &
+  /usr/bin/frostfs-http-gw --config /config/http-gw-config.yaml |& tee -a ${LOG_DIR}/frostfs-http-gw &
 }
 
 

bin/issue-creds.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+initUser() {
+  /usr/bin/frostfs-s3-authmate register-user \
+    --wallet $WALLET_PATH \
+    --rpc-endpoint http://localhost:30333 \
+    --username $USERNAME \
+    --contract-wallet /config/s3-gw-wallet.json >/dev/null 2>&1 && touch $WALLET_CACHE/$USERNAME
+}
+
+issueAWS() {
+  /usr/bin/frostfs-s3-authmate issue-secret \
+    --wallet $WALLET_PATH \
+    --peer localhost:8080 \
+    --gate-public-key $S3_GATE_PUBLIC_KEY \
+    --container-placement-policy "REP 1"
+}
+
+S3_GATE_PUBLIC_KEY=$(neo-go wallet dump-keys -w /config/s3-gw-wallet.json | tail -1)
+WALLET_PATH=/wallets/$2
+if [[ -z "$2" ]]; then
+  WALLET_PATH=/config/user-wallet.json
+fi
+
+WALLET_CACHE=/data/wallets
+mkdir -p $WALLET_CACHE
+
+USERNAME=$(echo $WALLET_PATH | md5sum | cut -d' ' -f1)
+if [ ! -e $WALLET_CACHE/$USERNAME ]; then
+  initUser
+fi
+
+if [ $1 == "s3" ]; then
+  issueAWS
+fi

docker-compose.yml

@@ -15,6 +15,8 @@ services:
       retries: 100
       start_period: 2s
       timeout: 2s
+    environment:
+      - LOG_DIR=/var/log
     volumes:
       - data:/data
       - ./wallets:/wallets

morph/protocol.privnet.yml

@@ -11,6 +11,7 @@ ProtocolConfiguration:
   VerifyTransactions: true
   StateRootInHeader: true
   P2PSigExtensions: true
+  Hardforks: {}
 
 ApplicationConfiguration:
   SkipBlockVerification: false

s3-gw/regions.json

@@ -0,0 +1,4 @@
+{
+  "us-east-1": "REP 1",
+  "us-west-1": "REP 1"
+}

s3-gw/s3-gw-config.yaml

@@ -32,7 +32,7 @@ stream_timeout: 10s
 healthcheck_timeout: 15s
 rebalance_interval: 60s
 pool_error_threshold: 100
-max_clients_count: 100
+max_clients_count: 1000
 max_clients_deadline: 30s
 
 cache:
@@ -60,6 +60,7 @@ cache:
 
 placement_policy:
   default: REP 1
+  region_mapping: /config/regions.json
 
 features:
   md5:
@@ -69,9 +70,6 @@ control:
   grpc:
     endpoint: localhost:16515
 
-frostfsid:
-  validation:
-    enabled: false
-
 kludge:
   use_default_xmlns: true
+  bypass_content_encoding_check_in_chunks: true

sn/config.yaml

@@ -2,7 +2,6 @@ morph:
   rpc_endpoint:
     - address: ws://localhost:30333/ws
   dial_timeout: 20s
-  ape_chain_cache_size: 0
 
 control:
   grpc:
@@ -38,6 +37,7 @@ pprof:
 
 storage:
   shard_num: 1
+  shard_pool_size: 1000
   shard:
     0:
       writecache:
@@ -66,6 +66,4 @@ tree:
 
 object:
   put:
-    pool_size_remote: 100
-    pool_size_local: 100
-
+    local_pool_size: 1000