[#105] Add website container creation

Signed-off-by: Nikita Zinkevich <n.zinkevich@yadro.com>

.env

@@ -12,20 +12,20 @@ NEOGO_VERSION=0.106.3
 NEOGO_IMAGE=nspccdev/neo-go
 
 # FrostFS InnerRing nodes
-IR_VERSION=0.44.4
+IR_VERSION=0.45.0-rc.6
 IR_IMAGE=git.frostfs.info/truecloudlab/frostfs-ir
 
 # FrostFS Storage nodes
-NODE_VERSION=0.44.4
+NODE_VERSION=0.45.0-rc.6
 NODE_IMAGE=git.frostfs.info/truecloudlab/frostfs-storage
 
 # HTTP Gate
-HTTP_GW_VERSION=0.32.0
+HTTP_GW_VERSION=0.33.0-rc.3
-HTTP_GW_IMAGE=truecloudlab/frostfs-http-gw
+HTTP_GW_IMAGE=git.frostfs.info/truecloudlab/frostfs-http-gw
 
 # S3 Gate
-S3_GW_VERSION=0.32.0
+S3_GW_VERSION=0.33.0-rc.3
-S3_GW_IMAGE=truecloudlab/frostfs-s3-gw
+S3_GW_IMAGE=git.frostfs.info/truecloudlab/frostfs-s3-gw
 
 # Lifecycler
 S3_LIFECYCLER_VERSION=0.1.3
@@ -36,11 +36,13 @@ LOCODE_DB_URL=https://git.frostfs.info/attachments/a2e8def7-52b6-49f1-89cd-a0567
 #LOCODE_DB_PATH=/path/to/locode_db
 
 # FrostFS CLI binary
-FROSTFS_CLI_URL=https://git.frostfs.info/TrueCloudLab/frostfs-node/releases/download/v${NODE_VERSION}/frostfs-cli
+FROSTFS_CLI_VERSION=0.45.0-rc.6
+FROSTFS_CLI_IMAGE=git.frostfs.info/truecloudlab/frostfs-cli
 #FROSTFS_CLI_PATH=/path/to/frostfs-cli-binary
 
 # FrostFS ADM tool binary
-FROSTFS_ADM_URL=https://git.frostfs.info/TrueCloudLab/frostfs-node/releases/download/v${NODE_VERSION}/frostfs-adm
+FROSTFS_ADM_VERSION=0.45.0-rc.6
+FROSTFS_ADM_IMAGE=git.frostfs.info/truecloudlab/frostfs-adm
 #FROSTFS_ADM_PATH=/path/to/frostfs-adm-binary
 
 # Compiled FrostFS Smart Contracts

Makefile

@@ -58,9 +58,8 @@ get: $(foreach SVC, $(GET_SVCS), get.$(SVC))
 
 # Start environment
 .PHONY: up
-up: up/basic
+up: up/basic up/pre-services
 	@$(foreach SVC, $(START_SVCS), $(shell docker-compose -f services/$(SVC)/docker-compose.yml up -d))
-	./vendor/frostfs-adm morph proxy-add-account --config frostfs-adm.yml --account=`docker container exec morph_chain neo-go wallet dump-keys -w /wallets/s3-wallet.json | head -1 | awk '{print $1}'` || die "Couldn't set s3-gw wallet as proxy wallet"
 	@echo "Full FrostFS Developer Environment is ready"
 
 # Build up FrostFS
@@ -73,6 +72,8 @@ up/basic: up/bootstrap
 
 # Start bootstrap services
 .PHONY: up/bootstrap
+up/bootstrap: STORAGE_WALLETS = $(wildcard ./services/storage/wallet*.json)
+up/bootstrap: STORAGE_ACCOUNTS = $(foreach wallet,$(STORAGE_WALLETS),$(shell docker container exec morph_chain neo-go wallet dump-keys -w /wallets/storage/$(notdir $(wallet)) | head -1 | awk '{print $$1}' ))
 up/bootstrap: get vendor/hosts
 	@$(foreach SVC, $(START_BOOTSTRAP), $(shell docker-compose -f services/$(SVC)/docker-compose.yml up -d))
 	@source ./bin/helper.sh
@@ -81,13 +82,16 @@ up/bootstrap: get vendor/hosts
 	@./vendor/frostfs-adm --config frostfs-adm.yml morph \
 		ape add-rule-chain --target-type namespace --target-name "" \
 		--rule 'allow Container.* *' --chain-id "allow_container_ops"
-	@for f in ./services/storage/wallet*.json; do \
+
-		echo "Transfer GAS to wallet $${f}" \
+	echo -e "Transfer GAS to storage wallets: $(foreach wallet,$(STORAGE_WALLETS),\n\t$(wallet))"
-		&& ./vendor/frostfs-adm -c frostfs-adm.yml morph refill-gas --storage-wallet $${f} --gas 10.0 \
+	./vendor/frostfs-adm -c frostfs-adm.yml morph refill-gas \
-		|| die "Failed to transfer GAS to alphabet wallets"; \
+		$(foreach wallet,$(STORAGE_WALLETS),--storage-wallet $(wallet)) \
-		echo "Register storage wallet $${f} in proxy contract" \
+		--gas 10.0 \
-		&& ./vendor/frostfs-adm morph proxy-add-account --config frostfs-adm.yml --account=`docker container exec morph_chain neo-go wallet dump-keys -w /wallets/storage/$${f##*/} | head -1 | awk '{print $1}'` || die "Couldn't set storage allet as proxy wallet"
+		|| die "Failed to transfer GAS to alphabet wallets"
-		done
+	echo -e "Register storage accounts in proxy contract: $(foreach account,$(STORAGE_ACCOUNTS),\n\t$(account))"
+	./vendor/frostfs-adm morph proxy-add-account --config frostfs-adm.yml \
+		$(foreach account,$(STORAGE_ACCOUNTS),--account=$(account)) \
+		|| die "Couldn't set storage allet as proxy wallet"
 	@echo "Create frostfsid subject for ./wallets/wallet.json"; \
 	if [ -n "$$(./vendor/frostfs-adm -c frostfs-adm.yml morph frostfsid list-subjects --namespace '')" ]; then \
 		echo "Subject already exists"; \
@@ -99,6 +103,31 @@ up/bootstrap: get vendor/hosts
 	fi
 	echo "FrostFS sidechain environment is deployed"
 	
+# Prepare to start services
+.PHONY: up/pre-services
+up/pre-services:
+	@source ./bin/helper.sh
+	@echo "Prepare storage for services"; \
+	if [ -z "$$(./vendor/frostfs-cli -c cli-cfg.yml container list)" ]; then \
+		subj_key=`docker container exec -it morph_chain neo-go wallet dump-keys -w /wallets/system-wallet.json | tail -1 | tr -d ' \r\n'` \
+		&& echo "Subject key: $${subj_key}" \
+		&& ./vendor/frostfs-adm -c frostfs-adm.yml morph frostfsid create-subject --namespace "" --subject-key $${subj_key} --subject-name system \
+		|| die "Failed to create subject for system wallet"; \
+		proxy_acc=`docker container exec -it morph_chain neo-go wallet dump-keys -w /wallets/s3-wallet.json | head -1 | cut -d" " -f1` \
+		&& echo "Proxy acc: $${proxy_acc}" \
+		&& ./vendor/frostfs-adm morph proxy-add-account --config frostfs-adm.yml --account=$${proxy_acc} || die "Failed to register S3 gateway as proxy acc"; \
+		cid=`./vendor/frostfs-cli -c cli-cfg.yml container create -p "REP 4" --nns-name "cors" --nns-zone "container" --await | grep CID | cut -d" " -f2` \
+		&& echo "CORS Container: $${cid}" \
+		&& ./vendor/frostfs-cli -c cli-cfg.yml ape-manager add --target-type container --target-name $${cid} --rule "allow Object.* *" || die "Failed to create CORS container"; \
+		cid=`./vendor/frostfs-cli -c cli-cfg.yml container create -p "REP 4" --nns-name "mfa" --nns-zone "container" --await | grep CID | cut -d" " -f2` \
+		&& echo "MFA Container: $${cid}" \
+		&& ./vendor/frostfs-cli -c cli-cfg.yml ape-manager add --target-type container --target-name $${cid} --rule "allow Object.* *" || die "Failed to create MFA container"
+		cid=`./vendor/frostfs-cli -c cli-cfg.yml container create -p "REP 4" --nns-name "website" --nns-zone "container" --await | grep CID | cut -d" " -f2` \
+		&& echo "Website configuration Container: $${cid}" \
+		&& ./vendor/frostfs-cli -c cli-cfg.yml ape-manager add --target-type container --target-name $${cid} --rule "allow Object.* *" || die "Failed to create Website configuration container"
+	fi
+	@echo "Storage is prepared";
+
 # Build up certain service
 .PHONY: up/%
 up/%: get vendor/hosts

README.md

@@ -143,7 +143,7 @@ Registers user wallet and issues s3 credentials.
 
 Usage and default parameter values:
 ```sh
-make s3cred [password=""] [contract_password=s3] [wallet=/user_wallet.json] [gate_public_key=0313b1ac3a8076e155a7e797b24f0b650cccad5941ea59d7cfd51a024a8b2a06bf]
+make s3cred [password=""] [contract_password=s3] [wallet=""] [gate_public_key=0313b1ac3a8076e155a7e797b24f0b650cccad5941ea59d7cfd51a024a8b2a06bf]
 ```
 
 As soon as the storage node is in the network map (see above) you can generate S3
@@ -159,8 +159,9 @@ $ make s3cred
   "container_id": "EXArWh8x1zeHG3851s1RtoCo7dowxF6rhLGA15nbMffT"
 }                   
 ```
-Running without any parameters will result in defaults which are based on the private key from
+Running without any parameters results in defaults which are based on the private key from
-`/user-wallet.json` file and `/wallet.json` contract wallet.
+`/wallets/wallet.json` user wallet and `/wallet.json` contract wallet.
+If `wallet` parameter is set, gate searches custom user wallet file in `/wallets` directory.
 
 Now let's configure an S3 client (AWS CLI will be used as example):
 
@@ -172,7 +173,7 @@ Default region name []: us-east-1
 Default output format []: json
 ```
 
-If you need to create credentials for different users, put user wallets to `wallets` dir and specify them via `wallet` parameter.
+If you need to create credentials for different users, put user wallet to `wallets` dir and specify it via `wallet` parameter.
 Pass wallet password in `password` parameter if it's not default. The same is for `contract_wallet` and `gate_public_key` params.
 
 ```sh

cli-cfg.yml

@@ -0,0 +1,3 @@
+wallet: ./wallets/system-wallet.json
+password: ""
+rpc-endpoint: s01.frostfs.devenv:8080

services/http_gate/cfg/config.yml

@@ -22,3 +22,6 @@ server:
 wallet:
   path: /wallet.json  # Path to wallet 
   passphrase: one  # Passphrase to decrypt wallet
+  
+containers:
+  cors: cors.container

services/ir/artifacts.mk

@@ -30,11 +30,9 @@ get.cli:
 	@mkdir -p ./vendor
 
 ifeq (${FROSTFS_CLI_PATH},)
-	@echo "⇒ Download FrostFS CLI binary from ${FROSTFS_CLI_URL}"
+	echo "⇒ Download FrostFS CLI binary from ${FROSTFS_CLI_IMAGE}:${FROSTFS_CLI_VERSION}"
-	@curl \
+	$(shell docker cp `docker create --name tmp ${FROSTFS_CLI_IMAGE}:${FROSTFS_CLI_VERSION}`:/bin/frostfs-cli ${FROSTFS_CLI_FILE} && docker rm tmp >/dev/null)
-		-ksSL "${FROSTFS_CLI_URL}" \
+	chmod +x ${FROSTFS_CLI_FILE}
-		-o ${FROSTFS_CLI_FILE} 
-	@chmod +x ${FROSTFS_CLI_FILE}
 else
 	@echo "⇒ Copy local binary from ${FROSTFS_CLI_PATH}"
 	@cp ${FROSTFS_CLI_PATH} ${FROSTFS_CLI_FILE}

services/morph_chain/artifacts.mk

@@ -23,8 +23,8 @@ get.adm: FROSTFS_ADM_DEST=./vendor/frostfs-adm
 get.adm:
 
 ifeq (${FROSTFS_ADM_PATH},)
-	@echo "⇒ Download FrostFS ADM binary from ${FROSTFS_ADM_URL}"
+	@echo "⇒ Download FrostFS ADM binary from ${FROSTFS_ADM_IMAGE}:${FROSTFS_ADM_VERSION}"
-	@curl -skSL ${FROSTFS_ADM_URL} -o ${FROSTFS_ADM_DEST}
+	$(shell docker cp `docker create --name tmp ${FROSTFS_ADM_IMAGE}:${FROSTFS_ADM_VERSION}`:/bin/frostfs-adm ${FROSTFS_ADM_DEST} && docker rm tmp >/dev/null)
 	@chmod +x ${FROSTFS_ADM_DEST}
 else
 	@echo "⇒ Copy frostfs-adm binary from ${FROSTFS_ADM_PATH}"

services/morph_chain/docker-compose.yml

@@ -19,6 +19,7 @@ services:
     - ./config.yml:/wallets/config.yml
     - ./../../vendor/hosts:/etc/hosts
     - ./../../wallets/wallet.json:/wallets/wallet.json
+    - ./../../wallets/system-wallet.json:/wallets/system-wallet.json
     - ./../s3_gate/wallet.json:/wallets/s3-wallet.json
     - ./../storage/wallet01.json:/wallets/storage/wallet01.json
     - ./../storage/wallet02.json:/wallets/storage/wallet02.json

services/s3_gate/cfg/config.yml

@@ -47,3 +47,8 @@ frostfsid:
 
 policy:
   enabled: false
+ 
+containers:
+  cors: cors.container
+  mfa: mfa.container
+  website: website.container

services/s3_gate/docker-compose.yml

@@ -14,10 +14,8 @@ services:
     volumes:
       # Gate wallet
       - ./wallet.json:/wallet.json
-      # Custom user wallets
+      # Folder for custom user wallets
-      - ./wallets:/wallets
+      - ./../../wallets/:/wallets/
-      # Default user wallet
-      - ./../../wallets/wallet.json:/wallets/wallet.json
       - ./tls.key:/tls.key
       - ./tls.crt:/tls.crt
       - ./../../vendor/hosts:/etc/hosts

services/s3_lifecycler/docker-compose.yml

@@ -1,6 +1,5 @@
 ---
 
-version: "2.4"
 services:
   s3_lifecycler:
     image: ${S3_LIFECYCLER_IMAGE}:${S3_LIFECYCLER_VERSION}

wallets/system-wallet.json

@@ -0,0 +1 @@
+{"version":"1.0","accounts":[{"address":"NQijiVKHbL22PfF2AJQukv1CX75itxgzht","key":"6PYQKrpme57VqaucxuF7dDoSZRRA8d94oatHcScqhiFBauCXQvFDaYwEWa","label":"","contract":{"script":"DCEDRdLtpFIWeYyI7doTKRhIl4qYjaybGDveTyGpbqjsLZNBVuezJw==","parameters":[{"name":"parameter0","type":"Signature"}],"deployed":false},"lock":false,"isDefault":false}],"scrypt":{"n":16384,"r":8,"p":8},"extra":{"Tokens":null}}