[#85] Add s3 lifecycler

Signed-off-by: Pavel Pogodaev <p.pogodaev@yadro.com>

.basic_services

@@ -1,5 +1,4 @@
 # Services start/stop order
 # Will start from top to bottom and stop in reverse
-nats
 ir
 storage

.env

@@ -19,10 +19,6 @@ IR_IMAGE=git.frostfs.info/truecloudlab/frostfs-ir
 NODE_VERSION=0.42.9
 NODE_IMAGE=git.frostfs.info/truecloudlab/frostfs-storage
 
-# NATS Server
-NATS_VERSION=2.7.2
-NATS_IMAGE=nats
-
 # HTTP Gate
 HTTP_GW_VERSION=0.30.2
 HTTP_GW_IMAGE=truecloudlab/frostfs-http-gw
@@ -32,9 +28,13 @@ REST_GW_VERSION=c9c85e90
 REST_GW_IMAGE=truecloudlab/frostfs-rest-gw
 
 # S3 Gate
-S3_GW_VERSION=0.30.4
+S3_GW_VERSION=0.31.0-rc.4
 S3_GW_IMAGE=truecloudlab/frostfs-s3-gw
 
+# Lifecycler
+S3_LIFECYCLER_VERSION=0.1.3
+S3_LIFECYCLER_IMAGE=truecloudlab/frostfs-s3-lifecycler
+
 # FrostFS LOCODE database
 LOCODE_DB_URL=https://git.frostfs.info/attachments/a2e8def7-52b6-49f1-89cd-a056712e8e54
 #LOCODE_DB_PATH=/path/to/locode_db

.gitignore

@@ -15,4 +15,3 @@ sites/*
 # Runtime generation keys
 services/storage/*tls.crt
 services/storage/*tls.key
-services/nats/*.pem

.services

@@ -3,3 +3,4 @@
 http_gate
 s3_gate
 rest_gate
+s3_lifecycler

Makefile

@@ -150,7 +150,7 @@ hosts: vendor/hosts
 .PHONY: clean
 .ONESHELL:
 clean:
-	@rm -rf vendor/* services/storage/s04tls.* services/nats/*.pem
+	@rm -rf vendor/* services/storage/s04tls.*
 	@> .int_test.env
 	@for svc in $(PULL_SVCS)
 	do

services/nats/.hosts

@@ -1 +0,0 @@
-IPV4_PREFIX.101 nats.LOCAL_DOMAIN

services/nats/artifacts.mk

@@ -1,7 +0,0 @@
-# Create new TLS certs for NATS server and clients
-
-NATS_DIR=$(abspath services/nats)
-
-get.nats:
-	@echo "⇒ Creating certs for NATS server and clients"
-	${NATS_DIR}/generate_cert.sh ${LOCAL_DOMAIN} > /dev/null

services/nats/docker-compose.yml

@@ -1,30 +0,0 @@
----
-
-services:
-  nats:
-    image: ${NATS_IMAGE}:${NATS_VERSION}
-    domainname: ${LOCAL_DOMAIN}
-    hostname: nats
-    container_name: nats
-    restart: on-failure
-    dns: 
-      - ${IPV4_PREFIX}.101
-    networks:
-      nats_int:
-      internet:
-        ipv4_address: ${IPV4_PREFIX}.101
-    volumes:
-      - ./../../vendor/hosts:/etc/hosts
-      - ./nats.conf:/etc/nats/frostfs-nats-server.conf
-      - ./server-cert.pem:/certs/server-cert.pem
-      - ./server-key.pem:/certs/server-key.pem
-      - ./ca-cert.pem:/certs/ca-cert.pem
-    stop_signal: SIGKILL
-    env_file: [ ".env", ".int_test.env" ]
-    command: ["-c", "/etc/nats/frostfs-nats-server.conf"]
-
-networks:
-  nats_int:
-  internet:
-    external: true
-    name: basenet_internet

services/nats/generate_cert.sh

@@ -1,49 +0,0 @@
-#!/bin/bash
-
-source bin/helper.sh
-
-WORKDIR=$(dirname "$0")
-LOCAL_DOMAIN=$1
-
-CA_KEY=$WORKDIR/ca-key.pem
-CA_CRT=$WORKDIR/ca-cert.pem
-
-SRV_KEY=$WORKDIR/server-key.pem
-SRV_REQ=$WORKDIR/server-req.csr
-SRV_CRT=$WORKDIR/server-cert.pem
-
-CLI_KEY=$WORKDIR/client-key.pem
-CLI_REQ=$WORKDIR/client-req.csr
-CLI_CRT=$WORKDIR/client-cert.pem
-
-SUBJ="/O=TrueCloudLab"
-
-if [[ ! -f $CA_KEY || ! -f $CA_CRT ]]; then
-	openssl req -newkey rsa:4096 -x509 -days 365 -nodes -keyout $CA_KEY -out $CA_CRT -subj $SUBJ 2>&1 ||
-		die "CA certificate was not created"
-fi
-
-if [[ ! -f $SRV_KEY || ! -f $SRV_CRT ]]; then
-	openssl req -newkey rsa:4096 -nodes -keyout $SRV_KEY -out $SRV_REQ -subj $SUBJ 2>&1 ||
-		die "Server certificate was not created"
-
-	openssl x509 -req -days 365 -set_serial 01 -in $SRV_REQ -out $SRV_CRT -CA $CA_CRT -CAkey $CA_KEY \
- 		-extensions san -extfile <(printf "[san]\nsubjectAltName=DNS:nats.$LOCAL_DOMAIN") 2>&1 || {
-		rm $SRV_REQ
-		die "Server certificate was not signed by CA"
-	}
-
-	rm $SRV_REQ
-fi
-
-if [[ ! -f $CLI_KEY || ! -f $CLI_CRT ]]; then
-	openssl req -newkey rsa:4096 -nodes -keyout $CLI_KEY -out $CLI_REQ -subj $SUBJ 2>&1 ||
-		die "Client certificate was not created"
-
-	openssl x509 -req -days 365 -set_serial 01 -in $CLI_REQ -out $CLI_CRT -CA $CA_CRT -CAkey $CA_KEY 2>&1 || {
-		rm $CLI_REQ
-		die "Client certificate was not signed by CA"
-	}
-
-	rm $CLI_REQ
-fi

services/nats/nats.conf

@@ -1,15 +0,0 @@
-port: 4222
-monitor_port: 8222
-
-jetstream {
-   store_dir=nats
-   max_memory_store: 1GB
-   max_file_store: 2GB
-}
-
-tls {
-  cert_file: /certs/server-cert.pem
-  key_file:  /certs/server-key.pem
-  ca_file:   /certs/ca-cert.pem 
-  verify:    true
-}

services/s3_lifecycler/.hosts

@@ -0,0 +1 @@
+IPV4_PREFIX.84 lifecycler.LOCAL_DOMAIN

services/s3_lifecycler/cfg/config.yml

@@ -0,0 +1,42 @@
+logger:
+  level: debug
+
+prometheus:
+  enabled: true
+  address: :9090
+  
+lifecycle:
+  job_fetcher_buffer: 1000
+  executor_pool_size: 100  
+
+frostfs:
+  stream_timeout: 10s
+  connect_timeout: 10s
+  healthcheck_timeout: 15s
+  rebalance_interval: 60s
+  pool_error_threshold: 100
+  tree_pool_max_attempts: 4
+
+credential:
+  use: wallets
+  source:
+    wallets:
+      - path: /wallet.json
+        address: NTt1rxvmEDxEuuogLxs2xgxA71qhVaUcN7
+        passphrase: "cycle"
+      - path: /user-wallet.json
+        address: NbUgTSFvPmsRxmGeWpuuGeJUoRoi6PErcM
+        passphrase: ""
+        
+morph:
+  reconnect_clients_interval: 30s
+  dial_timeout: 5s
+  contract:
+    netmap: netmap.frostfs
+    frostfsid: frostfsid.frostfs
+    container: container.frostfs
+
+# Wallet configuration
+wallet:
+  path: /wallet.json # Path to wallet
+  passphrase: "cycle" # Passphrase to decrypt wallet

services/s3_lifecycler/docker-compose.yml

@@ -0,0 +1,38 @@
+---
+
+version: "2.4"
+services:
+  s3_lifecycler:
+    image: ${S3_LIFECYCLER_IMAGE}:${S3_LIFECYCLER_VERSION}
+    domainname: ${LOCAL_DOMAIN}
+    hostname: s3_lifecycler
+    container_name: s3_lifecycler
+    restart: on-failure
+    networks:
+      s3_lifecycler_int:
+      internet:
+        ipv4_address: ${IPV4_PREFIX}.84
+    volumes:
+      - ./wallet.json:/wallet.json
+      - ./../../vendor/hosts:/etc/hosts
+      - ./cfg:/etc/frostfs/s3-lifecycler
+      - ./../../wallets/wallet.json:/user-wallet.json
+    stop_signal: SIGKILL
+    env_file: [ ".env", ".int_test.env" ]
+    command: [ "frostfs-s3-lifecycler", "--config", "/etc/frostfs/s3-lifecycler/config.yml" ]
+    environment:
+      - S3_LIFECYCLER_MORPH_RPC_ENDPOINT_0_ADDRESS=ws://morph-chain:30333/ws
+      - S3_LIFECYCLER_FROSTFS_PEERS_0_ADDRESS=s01.${LOCAL_DOMAIN}:8080
+      - S3_LIFECYCLER_FROSTFS_PEERS_0_WEIGHT=0.2
+      - S3_LIFECYCLER_FROSTFS_PEERS_1_ADDRESS=s02.${LOCAL_DOMAIN}:8080
+      - S3_LIFECYCLER_FROSTFS_PEERS_1_WEIGHT=0.2
+      - S3_LIFECYCLER_FROSTFS_PEERS_2_ADDRESS=s03.${LOCAL_DOMAIN}:8080
+      - S3_LIFECYCLER_FROSTFS_PEERS_2_WEIGHT=0.2
+      - S3_LIFECYCLER_FROSTFS_PEERS_3_ADDRESS=s04.${LOCAL_DOMAIN}:8080
+      - S3_LIFECYCLER_FROSTFS_PEERS_3_WEIGHT=0.2
+
+networks:
+  s3_lifecycler_int:
+  internet:
+    external: true
+    name: basenet_internet

services/s3_lifecycler/wallet.json

@@ -0,0 +1,30 @@
+{
+  "version": "1.0",
+  "accounts": [
+    {
+      "address": "NTt1rxvmEDxEuuogLxs2xgxA71qhVaUcN7",
+      "key": "6PYR3XurAyTzVeDG5WV2Z8vnGdySw3mTLuKjr6Nwo7tae64SJ7XjZSMMPQ",
+      "label": "lifecycler",
+      "contract": {
+        "script": "DCED9z0M+WSGfXZGxYLj1yYwmgxJXE/kNA4+oWNi0q1uKCdBVuezJw==",
+        "parameters": [
+          {
+            "name": "parameter0",
+            "type": "Signature"
+          }
+        ],
+        "deployed": false
+      },
+      "lock": false,
+      "isDefault": false
+    }
+  ],
+  "scrypt": {
+    "n": 16384,
+    "r": 8,
+    "p": 8
+  },
+  "extra": {
+    "Tokens": null
+  }
+}

services/storage/cfg/config.yml

@@ -35,14 +35,6 @@ morph:
 # Common storage node settings
 node:
   attribute_0: "User-Agent:FrostFS/0.34"
-  notification:
-    enabled: true  # Turn on object notification service
-    endpoint: "tls://nats.frostfs.devenv:4222"  # Notification server endpoint
-    timeout: "6s"  # Timeout for object notification client connection
-    default_topic: "test"  # Default topic for object notifications if not found in object's meta
-    certificate: "/etc/frostfs-node/nats.tls.cert"  # Path to TLS certificate
-    key: "/etc/frostfs-node/nats.tls.key"  # Path to TLS key
-    ca: "/etc/frostfs-node/nats.ca.crt"  # Path to optional CA certificate
 
 # Tree section
 tree:

services/storage/docker-compose.yml

@@ -18,9 +18,6 @@ services:
       - ./../../vendor/frostfs-cli:/frostfs-cli
       - ./cli-cfg.yml:/cli-cfg.yml
       - ./s04tls.crt:/etc/ssl/certs/s04tls.crt
-      - ../nats/client-cert.pem:/etc/frostfs-node/nats.tls.cert
-      - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key
-      - ../nats/ca-cert.pem:/etc/frostfs-node/nats.ca.crt
       - ./cfg:/etc/frostfs/storage
     stop_signal: SIGTERM
     stop_grace_period: 15s
@@ -58,9 +55,6 @@ services:
       - ./../../vendor/frostfs-cli:/frostfs-cli
       - ./cli-cfg.yml:/cli-cfg.yml
       - ./s04tls.crt:/etc/ssl/certs/s04tls.crt
-      - ../nats/client-cert.pem:/etc/frostfs-node/nats.tls.cert
-      - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key
-      - ../nats/ca-cert.pem:/etc/frostfs-node/nats.ca.crt
       - ./cfg:/etc/frostfs/storage
     stop_signal: SIGTERM
     stop_grace_period: 15s
@@ -98,9 +92,6 @@ services:
       - ./../../vendor/frostfs-cli:/frostfs-cli
       - ./cli-cfg.yml:/cli-cfg.yml
       - ./s04tls.crt:/etc/ssl/certs/s04tls.crt
-      - ../nats/client-cert.pem:/etc/frostfs-node/nats.tls.cert
-      - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key
-      - ../nats/ca-cert.pem:/etc/frostfs-node/nats.ca.crt
       - ./cfg:/etc/frostfs/storage
     stop_signal: SIGTERM
     stop_grace_period: 15s
@@ -139,9 +130,6 @@ services:
       - ./cli-cfg.yml:/cli-cfg.yml
       - ./s04tls.crt:/tls.crt
       - ./s04tls.key:/tls.key
-      - ../nats/client-cert.pem:/etc/frostfs-node/nats.tls.cert
-      - ../nats/client-key.pem:/etc/frostfs-node/nats.tls.key
-      - ../nats/ca-cert.pem:/etc/frostfs-node/nats.ca.crt
       - ./cfg:/etc/frostfs/storage
     stop_signal: SIGTERM
     stop_grace_period: 15s