Update debugging log output

Signed-off-by: Alex Vanin <a.vanin@yadro.com>

.env

@@ -1,3 +1,3 @@
 FROSTFS_HUB_IMAGE=truecloudlab/frostfs
 AIO_IMAGE=truecloudlab/frostfs-aio
-AIO_VERSION=1.6.4
+AIO_VERSION=1.5.0-log.2

Dockerfile

@@ -1,18 +1,18 @@
 ARG FROSTFS_HUB_IMAGE=truecloudlab/frostfs
 ARG NEOGO_HUB_IMAGE=nspccdev/neo-go
 
-ARG FROSTFS_CORE_TAG=0.42.15
-ARG FROSTFS_HTTP_GATE_TAG=0.30.2
-ARG FROSTFS_S3_GATE_TAG=0.30.7
-ARG NEOGO_TAG=0.106.0
+ARG FROSTFS_CORE_TAG=0.38.6
+ARG FROSTFS_HTTP_GATE_TAG=0.29.0
+ARG FROSTFS_S3_GATE_TAG=0.29.3-12-g4ff049f4
+ARG NEOGO_TAG=0.104.0
 
-FROM ${NEOGO_HUB_IMAGE}:${NEOGO_TAG} AS neo-go
-FROM ${FROSTFS_HUB_IMAGE}-adm:${FROSTFS_CORE_TAG} AS frostfs-adm
-FROM ${FROSTFS_HUB_IMAGE}-cli:${FROSTFS_CORE_TAG} AS frostfs-cli
-FROM ${FROSTFS_HUB_IMAGE}-ir:${FROSTFS_CORE_TAG} AS frostfs-ir
-FROM ${FROSTFS_HUB_IMAGE}-storage:${FROSTFS_CORE_TAG} AS frostfs-storage
-FROM ${FROSTFS_HUB_IMAGE}-s3-gw:${FROSTFS_S3_GATE_TAG} AS frostfs-s3-gw
-FROM ${FROSTFS_HUB_IMAGE}-http-gw:${FROSTFS_HTTP_GATE_TAG} AS frostfs-http-gw
+FROM ${NEOGO_HUB_IMAGE}:${NEOGO_TAG} as neo-go
+FROM ${FROSTFS_HUB_IMAGE}-adm:${FROSTFS_CORE_TAG} as frostfs-adm
+FROM ${FROSTFS_HUB_IMAGE}-cli:${FROSTFS_CORE_TAG} as frostfs-cli
+FROM ${FROSTFS_HUB_IMAGE}-ir:${FROSTFS_CORE_TAG} as frostfs-ir
+FROM ${FROSTFS_HUB_IMAGE}-storage:${FROSTFS_CORE_TAG} as frostfs-storage
+FROM ${FROSTFS_HUB_IMAGE}-s3-gw:${FROSTFS_S3_GATE_TAG} as frostfs-s3-gw
+FROM ${FROSTFS_HUB_IMAGE}-http-gw:${FROSTFS_HTTP_GATE_TAG} as frostfs-http-gw
 
 # Executable image
 FROM alpine AS frostfs-aio
@@ -46,7 +46,6 @@ COPY ./morph/node-config.yaml /config/node-config.yaml
 COPY ./http-gw/http-gw-config.yaml /config/http-gw-config.yaml
 COPY ./http-gw/http-gw-wallet.json /config/http-gw-wallet.json
 COPY ./s3-gw/rules.json /config/bearer-rules.json
-COPY ./s3-gw/regions.json /config/regions.json
 COPY ./s3-gw/s3-gw-config.yaml /config/s3-gw-config.yaml
 COPY ./s3-gw/s3-gw-wallet.json /config/s3-gw-wallet.json
 COPY ./s3-gw/user-wallet.json /config/user-wallet.json
@@ -57,9 +56,7 @@ COPY ./vendor/locode_db /config/locode.db
 COPY ./vendor/contracts/ /config/contracts
 
 COPY ./bin/init-aio.sh /usr/bin/init-aio.sh
-COPY ./bin/issue-creds.sh /usr/bin/issue-creds.sh
 
 ENV AUTHMATE_WALLET_PASSPHRASE=""
-ENV AUTHMATE_WALLET_CONTRACT_PASSPHRASE=""
 
 ENTRYPOINT ["/usr/bin/init-aio.sh"]

Dockerfile.custom

@@ -1,18 +1,18 @@
 ARG FROSTFS_HUB_IMAGE=truecloudlab/frostfs
 ARG NEOGO_HUB_IMAGE=nspccdev/neo-go
 
-ARG FROSTFS_CORE_TAG=0.42.15
-ARG FROSTFS_HTTP_GATE_TAG=0.30.2
-ARG FROSTFS_S3_GATE_TAG=0.30.7
-ARG NEOGO_TAG=0.106.0
+ARG FROSTFS_CORE_TAG=0.38.6
+ARG FROSTFS_HTTP_GATE_TAG=0.29.0
+ARG FROSTFS_S3_GATE_TAG=0.29.3-12-g4ff049f4
+ARG NEOGO_TAG=0.104.0
 
-FROM ${NEOGO_HUB_IMAGE}:${NEOGO_TAG} AS neo-go
-FROM ${FROSTFS_HUB_IMAGE}-adm:${FROSTFS_CORE_TAG} AS frostfs-adm
-FROM ${FROSTFS_HUB_IMAGE}-cli:${FROSTFS_CORE_TAG} AS frostfs-cli
-FROM ${FROSTFS_HUB_IMAGE}-ir:${FROSTFS_CORE_TAG} AS frostfs-ir
-FROM ${FROSTFS_HUB_IMAGE}-storage:${FROSTFS_CORE_TAG} AS frostfs-storage
-FROM ${FROSTFS_HUB_IMAGE}-s3-gw:${FROSTFS_S3_GATE_TAG} AS frostfs-s3-gw
-FROM ${FROSTFS_HUB_IMAGE}-http-gw:${FROSTFS_HTTP_GATE_TAG} AS frostfs-http-gw
+FROM ${NEOGO_HUB_IMAGE}:${NEOGO_TAG} as neo-go
+FROM ${FROSTFS_HUB_IMAGE}-adm:${FROSTFS_CORE_TAG} as frostfs-adm
+FROM ${FROSTFS_HUB_IMAGE}-cli:${FROSTFS_CORE_TAG} as frostfs-cli
+FROM ${FROSTFS_HUB_IMAGE}-ir:${FROSTFS_CORE_TAG} as frostfs-ir
+FROM ${FROSTFS_HUB_IMAGE}-storage:${FROSTFS_CORE_TAG} as frostfs-storage
+FROM ${FROSTFS_HUB_IMAGE}-s3-gw:${FROSTFS_S3_GATE_TAG} as frostfs-s3-gw
+FROM ${FROSTFS_HUB_IMAGE}-http-gw:${FROSTFS_HTTP_GATE_TAG} as frostfs-http-gw
 
 # Executable image
 FROM alpine AS frostfs-aio
@@ -46,7 +46,6 @@ COPY ./morph/node-config.yaml /config/node-config.yaml
 COPY ./http-gw/http-gw-config.yaml /config/http-gw-config.yaml
 COPY ./http-gw/http-gw-wallet.json /config/http-gw-wallet.json
 COPY ./s3-gw/rules.json /config/bearer-rules.json
-COPY ./s3-gw/regions.json /config/regions.json
 COPY ./s3-gw/s3-gw-config.yaml /config/s3-gw-config.yaml
 COPY ./s3-gw/s3-gw-wallet.json /config/s3-gw-wallet.json
 COPY ./s3-gw/user-wallet.json /config/user-wallet.json
@@ -59,6 +58,5 @@ COPY ./vendor/contracts/ /config/contracts
 COPY ./bin/* /usr/bin/
 
 ENV AUTHMATE_WALLET_PASSPHRASE=""
-ENV AUTHMATE_WALLET_CONTRACT_PASSPHRASE=""
 
 ENTRYPOINT ["/usr/bin/init-aio.sh"]

Dockerfile.local

@@ -18,7 +18,6 @@ COPY ./bin/frostfs-node /usr/bin/frostfs-node
 COPY ./bin/frostfs-s3-gw /usr/bin/frostfs-s3-gw
 COPY ./bin/frostfs-s3-authmate /usr/bin/frostfs-s3-authmate
 COPY ./bin/frostfs-http-gw /usr/bin/frostfs-http-gw
-COPY ./bin/issue-creds.sh /usr/bin/issue-creds.sh
 
 COPY ./adm/frostfs-adm.yml /config/frostfs-adm.yml
 COPY ./ir/cli-cfg.yaml /config/cli-cfg-ir.yaml
@@ -31,7 +30,6 @@ COPY ./morph/node-config.yaml /config/node-config.yaml
 COPY ./http-gw/http-gw-config.yaml /config/http-gw-config.yaml
 COPY ./http-gw/http-gw-wallet.json /config/http-gw-wallet.json
 COPY ./s3-gw/rules.json /config/bearer-rules.json
-COPY ./s3-gw/regions.json /config/regions.json
 COPY ./s3-gw/s3-gw-config.yaml /config/s3-gw-config.yaml
 COPY ./s3-gw/s3-gw-wallet.json /config/s3-gw-wallet.json
 COPY ./s3-gw/user-wallet.json /config/user-wallet.json
@@ -44,6 +42,5 @@ COPY ./vendor/contracts/ /config/contracts
 COPY ./bin/init-aio.sh /usr/bin/init-aio.sh
 
 ENV AUTHMATE_WALLET_PASSPHRASE=""
-ENV AUTHMATE_WALLET_CONTRACT_PASSPHRASE=""
 
 ENTRYPOINT ["/usr/bin/init-aio.sh"]

Makefile

@@ -9,7 +9,6 @@ VERSION ?= "$(shell git describe --tags --match "v*" 2>/dev/null || git rev-pars
 
 # Variables for docker
 AIO_IMAGE ?= "truecloudlab/frostfs-aio"
-LOG_DIR ?= "/var/log"
 
 # Variables for compose
 COMPOSE_CMD_UP ?= docker-compose up -d
@@ -63,19 +62,19 @@ clean: down
 
 # Generate S3 credentials
 s3cred:
-	@docker exec aio /usr/bin/issue-creds.sh s3
+	@docker exec aio /usr/bin/frostfs-s3-authmate issue-secret \
+		--wallet /config/user-wallet.json \
+		--peer localhost:8080 \
+		--gate-public-key $(S3_GATE_PUBLIC_KEY) \
+		--container-placement-policy "REP 1"
 
 # Generate S3 credentials based on imported wallets
 s3cred-custom:
-	@docker exec aio /usr/bin/issue-creds.sh s3 $(wallet)
-
-# Register user-wallet.json in FrostFS ID contract
-cred:
-	@docker exec aio /usr/bin/issue-creds.sh native
-
-# Register custom wallet in FrostFS ID contract
-cred-custom:
-	@docker exec aio /usr/bin/issue-creds.sh native $(wallet)
+	@docker exec aio /usr/bin/frostfs-s3-authmate issue-secret \
+		--wallet /wallets/$(wallet) \
+		--peer localhost:8080 \
+		--gate-public-key $(S3_GATE_PUBLIC_KEY) \
+		--container-placement-policy "REP 1"
 
 enable-oracle:
 	@docker exec aio neo-go contract invokefunction -r http://localhost:30333 \
@@ -92,37 +91,8 @@ enable-oracle:
 tick.epoch:
 	@docker exec aio /usr/bin/frostfs-adm --config /config/frostfs-adm.yml morph force-new-epoch
 
-# Create new wallet
-WALLET?=wallets/wallet1.json
-.PHONY: wallet
-wallet:
-	@! test -f "$(WALLET)" || { echo "File exists: $(WALLET)"; exit 1; }
-	@docker exec -it aio /usr/bin/neo-go wallet init -a -w "$(WALLET)"
-
-# Refill GAS in arbitrary WALLET (must be under wallets/ directory)
-GAS?=50.0
-.PHONY: refill
-refill:
-	@docker exec aio /usr/bin/frostfs-adm --config /config/frostfs-adm.yml \
-		morph refill-gas --storage-wallet=/$(WALLET) --gas=$(GAS)
-	@$(MAKE) --no-print-directory balance
-
-# Show wallet balance
-.PHONY: balance
-balance:
-	@echo "Balance for $(WALLET)"
-	@docker exec aio /usr/bin/neo-go wallet nep17 balance -r http://localhost:30333 -w "$(WALLET)"
-
 # Show current version
 version:
 	@echo $(VERSION)
 	@echo "frostfs-node: $(AIO_VERSION)"
 	@echo "neo-go: $(NEOGO_VERSION)"
-
-# Show all logs of the FrostFS service
-logs/%:
-	@docker exec aio cat ${LOG_DIR}/$*
-
-# Streaming of the frostFS service logs
-logs-stream/%:
-	@docker exec aio tail -f ${LOG_DIR}/$*

README.md

@@ -62,8 +62,7 @@ Data and the system configuration is stored in container's volume.
 Next time container is started, it will take less time to initialize.
 
 Before updating image version, reset `frostfs-aio` by clearing its local volume
-before starting container. Update may contain incompatible changes, e.g. neo-go
-database format update. This repository does not provide any migration tools.
+before starting container.
 
 ```
 $ make clean
@@ -288,55 +287,16 @@ If everything is rigth, the stdout will show a pretty-printed contents of the wa
 wallet successfully created, file location is /wallets/new-wallet.json
 ```
 
-You can use `make wallet WALLET=wallets/new-wallet.json`
-to execute the steps listed above a little faster.
-
-
-### Transfering GAS tokens to wallet
-
-To add GAS to a new wallet use `make refill`
-
-```console
-$ make refill GAS=12.3 WALLET=wallets/new-wallet.json
-Waiting for transactions to persist...
-Balance for wallets/new-wallet.json
-Account NgB6jnyVsMg6E3gQQLXoccZ3JBMLsgiut3
-GAS: GasToken (d2a4cff31913016155e38e474a2c06d08be276cf)
-        Amount : 12.3
-        Updated: 25069
-```
-
-### Checking wallet balance
-
-```console
-$ make balance WALLET=wallets/new-wallet.json
-Balance for wallets/new-wallet.json
-Account NgB6jnyVsMg6E3gQQLXoccZ3JBMLsgiut3
-GAS: GasToken (d2a4cff31913016155e38e474a2c06d08be276cf)
-        Amount : 12.3
-        Updated: 25069
-```
-
 
 ## frostfs-cli interface
 
-### Native credentials
-
-To use wallet file, it should be register in FrostFS ID contract and have  
-policies to perform FrostFS operations.
-
-Use `make cred` to register `s3-gw/user-wallet.json`. For custom wallets from
-`wallet` dir, use `make cred-custom wallet=wallet1.json`.
-
-These commands are similar to `make s3cred` and `make s3cred-custom`.
-
 ### Create container with frostfs-cli
 
 ``` sh
-$ frostfs-cli -r localhost:8080 -w s3-gw/user-wallet.json \
+$ frostfs-cli -r localhost:8080 -w /config/user-wallet.json \
             --address NWeByJPgNC97F83hTUnSbnZSBKaFvk5HNw \
             container create \
-            --policy "REP 1" --await
+            --policy "REP 1" --basic-acl public-read-write --await
 container ID: GfWw35kHds7gKWmSvW7Zi4U39K7NMLK8EfXBQ5FPJA46
 awaiting...
 container has been persisted on sidechain
@@ -345,7 +305,7 @@ container has been persisted on sidechain
 ### Put an object with frostfs-cli
 
 ``` sh
-$ frostfs-cli -r localhost:8080 -w s3-gw/user-wallet.json \
+$ frostfs-cli -r localhost:8080 -w /config/user-wallet.json \
             --address NWeByJPgNC97F83hTUnSbnZSBKaFvk5HNw \
             object put \
             --cid GfWw35kHds7gKWmSvW7Zi4U39K7NMLK8EfXBQ5FPJA46 \
@@ -358,25 +318,9 @@ $ frostfs-cli -r localhost:8080 -w s3-gw/user-wallet.json \
 ### Get and object with frostfs-cli
 
 ``` sh
-$ frostfs-cli -r localhost:8080 -w s3-gw/user-wallet.json \
+$ frostfs-cli -r localhost:8080 -w /config/user-wallet.json \
             --address NWeByJPgNC97F83hTUnSbnZSBKaFvk5HNw \
             object get \
             --cid GfWw35kHds7gKWmSvW7Zi4U39K7NMLK8EfXBQ5FPJA46 \
             --oid HByVC9A34i22BnzW3n83z9vEMxuYZoC7nNu11ZvGeCTe > new_cat.jpg
 ```
-
-## Viewing logs
-
-There are two targets for viewing logs.
-
-You can view all the logs by replacing the desired service name:
-
-``` sh
-make logs/frostfs-node
-```
-
-You can also make a continuous stream of logs by replacing the desired service name:
-
-``` sh
-make stream-logs/frostfs-node
-```

bin/init-aio.sh

@@ -14,7 +14,7 @@ die() {
 runBlockchain() {
   stage "Starting the blockchain"
 
-  /usr/bin/neo-go node --config-path /config --privnet |& tee -a ${LOG_DIR}/neo-go &
+  /usr/bin/neo-go node --config-path /config --privnet >dev/null 2>&1 &
 
   while [[ "$(curl -s -o /dev/null -w %{http_code} localhost:30333)" != "422" ]];
   do
@@ -31,15 +31,13 @@ configure() {
 
   /usr/bin/frostfs-adm morph refill-gas --config /config/frostfs-adm.yml --storage-wallet /config/wallet-sn.json --gas 10.0 || die "Failed to transfer GAS to alphabet wallets"
 
-  /usr/bin/frostfs-adm morph proxy-add-account --config /config/frostfs-adm.yml --account NejLbQpojKJWec4NQRMBhzsrmCyhXfGJJe || die "Failed to set storage wallet as proxy wallet"
-
   /usr/bin/frostfs-adm morph proxy-add-account --config /config/frostfs-adm.yml --account NN1RQ3qwnvDMVNsQw4WPkKi7BrjxdVTDZp || die "Failed to set s3 gateway wallet as proxy wallet"
 }
 
 runServices() {
   stage "Running services"
 
-  /usr/bin/frostfs-ir --config /config/config-ir.yaml |& tee -a ${LOG_DIR}/frostfs-ir &
+  /usr/bin/frostfs-ir --config /config/config-ir.yaml >/dev/null 2>&1 &
 
   while [[ -z "$(/usr/bin/frostfs-cli control ir healthcheck --endpoint localhost:16512 -c /config/cli-cfg-ir.yaml | grep 'Health status: READY')" ]];
   do
@@ -47,7 +45,7 @@ runServices() {
   done
 
   set -m
-  /usr/bin/frostfs-node --config /config/config-sn.yaml |& tee -a ${LOG_DIR}/frostfs-node &
+  /usr/bin/frostfs-node --config /config/config-sn.yaml >/dev/null 2>&1 &
 
   while [[ -z "$(/usr/bin/frostfs-cli control healthcheck --endpoint localhost:16513 -c /config/cli-cfg-sn.yaml | grep 'Health status: READY')" ]];
   do
@@ -65,8 +63,8 @@ runServices() {
     sleep 2
   done
 
-  /usr/bin/frostfs-s3-gw --config /config/s3-gw-config.yaml |& tee -a ${LOG_DIR}/frostfs-s3-gw &
-  /usr/bin/frostfs-http-gw --config /config/http-gw-config.yaml |& tee -a ${LOG_DIR}/frostfs-http-gw &
+  /usr/bin/frostfs-s3-gw --config /config/s3-gw-config.yaml &
+  /usr/bin/frostfs-http-gw --config /config/http-gw-config.yaml &
 }
 
 

bin/issue-creds.sh

@@ -1,35 +0,0 @@
-#!/bin/bash
-
-initUser() {
-  /usr/bin/frostfs-s3-authmate register-user \
-    --wallet $WALLET_PATH \
-    --rpc-endpoint http://localhost:30333 \
-    --username $USERNAME \
-    --contract-wallet /config/s3-gw-wallet.json >/dev/null 2>&1 && touch $WALLET_CACHE/$USERNAME
-}
-
-issueAWS() {
-  /usr/bin/frostfs-s3-authmate issue-secret \
-    --wallet $WALLET_PATH \
-    --peer localhost:8080 \
-    --gate-public-key $S3_GATE_PUBLIC_KEY \
-    --container-placement-policy "REP 1"
-}
-
-S3_GATE_PUBLIC_KEY=$(neo-go wallet dump-keys -w /config/s3-gw-wallet.json | tail -1)
-WALLET_PATH=/wallets/$2
-if [[ -z "$2" ]]; then
-  WALLET_PATH=/config/user-wallet.json
-fi
-
-WALLET_CACHE=/data/wallets
-mkdir -p $WALLET_CACHE
-
-USERNAME=$(echo $WALLET_PATH | md5sum | cut -d' ' -f1)
-if [ ! -e $WALLET_CACHE/$USERNAME ]; then
-  initUser
-fi
-
-if [ $1 == "s3" ]; then
-  issueAWS
-fi

docker-compose.yml

@@ -15,8 +15,6 @@ services:
       retries: 100
       start_period: 2s
       timeout: 2s
-    environment:
-      - LOG_DIR=/var/log
     volumes:
       - data:/data
       - ./wallets:/wallets

morph/protocol.privnet.yml

@@ -11,7 +11,6 @@ ProtocolConfiguration:
   VerifyTransactions: true
   StateRootInHeader: true
   P2PSigExtensions: true
-  Hardforks: {}
 
 ApplicationConfiguration:
   SkipBlockVerification: false

s3-gw/regions.json

@@ -1,4 +0,0 @@
-{
-  "us-east-1": "REP 1",
-  "us-west-1": "REP 1"
-}

s3-gw/s3-gw-config.yaml

@@ -32,7 +32,7 @@ stream_timeout: 10s
 healthcheck_timeout: 15s
 rebalance_interval: 60s
 pool_error_threshold: 100
-max_clients_count: 1000
+max_clients_count: 100
 max_clients_deadline: 30s
 
 cache:
@@ -60,7 +60,6 @@ cache:
 
 placement_policy:
   default: REP 1
-  region_mapping: /config/regions.json
 
 features:
   md5:
@@ -70,6 +69,15 @@ control:
   grpc:
     endpoint: localhost:16515
 
+frostfsid:
+  validation:
+    enabled: false
+
 kludge:
   use_default_xmlns: true
   bypass_content_encoding_check_in_chunks: true
+
+http_logging:
+  enabled: true
+  max_body: 100
+  log_response: true

sn/config.yaml

@@ -2,6 +2,7 @@ morph:
   rpc_endpoint:
     - address: ws://localhost:30333/ws
   dial_timeout: 20s
+  ape_chain_cache_size: 0
 
 control:
   grpc:
@@ -37,7 +38,6 @@ pprof:
 
 storage:
   shard_num: 1
-  shard_pool_size: 1000
   shard:
     0:
       writecache:
@@ -66,4 +66,6 @@ tree:
 
 object:
   put:
-    local_pool_size: 1000
+    pool_size_remote: 100
+    pool_size_local: 100
+