forked from TrueCloudLab/frostfs-aio
[#48] Add S3 Lifecycler
Signed-off-by: Pavel Pogodaev <p.pogodaev@yadro.com>
This commit is contained in:
parent
13bec4ff0d
commit
fe05390238
8 changed files with 78 additions and 0 deletions
1
.env
1
.env
|
|
@ -6,4 +6,5 @@ AIO_VERSION=1.7.0-nightly.2
|
|||
FROSTFS_CORE_TAG=0.44.0-rc.7
|
||||
FROSTFS_HTTP_GATE_TAG=0.31.0-rc.4
|
||||
FROSTFS_S3_GATE_TAG=0.31.0-rc.4
|
||||
FROSTFS_S3_LIFECYCLER_TAG=0.1.3
|
||||
NEOGO_TAG=0.106.0
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ ARG NEOGO_HUB_IMAGE=nspccdev/neo-go
|
|||
ARG FROSTFS_CORE_TAG=dev
|
||||
ARG FROSTFS_HTTP_GATE_TAG=dev
|
||||
ARG FROSTFS_S3_GATE_TAG=dev
|
||||
ARG FROSTFS_S3_LIFECYCLER_TAG=dev
|
||||
ARG NEOGO_TAG=dev
|
||||
|
||||
FROM ${NEOGO_HUB_IMAGE}:${NEOGO_TAG} AS neo-go
|
||||
|
|
@ -14,6 +15,7 @@ FROM ${FROSTFS_FORGEJO_IMAGE}-ir:${FROSTFS_CORE_TAG} AS frostfs-ir
|
|||
FROM ${FROSTFS_FORGEJO_IMAGE}-storage:${FROSTFS_CORE_TAG} AS frostfs-storage
|
||||
FROM ${FROSTFS_HUB_IMAGE}-s3-gw:${FROSTFS_S3_GATE_TAG} AS frostfs-s3-gw
|
||||
FROM ${FROSTFS_HUB_IMAGE}-http-gw:${FROSTFS_HTTP_GATE_TAG} AS frostfs-http-gw
|
||||
FROM ${FROSTFS_HUB_IMAGE}-s3-lifecycler:${FROSTFS_S3_LIFECYCLER_TAG} AS frostfs-s3-lifecycler
|
||||
|
||||
# Executable image
|
||||
FROM alpine AS frostfs-aio
|
||||
|
|
@ -35,6 +37,7 @@ COPY --from=frostfs-storage /bin/frostfs-node /usr/bin/frostfs-node
|
|||
COPY --from=frostfs-s3-gw /bin/frostfs-s3-gw /usr/bin/frostfs-s3-gw
|
||||
COPY --from=frostfs-s3-gw /bin/frostfs-s3-authmate /usr/bin/frostfs-s3-authmate
|
||||
COPY --from=frostfs-http-gw /bin/frostfs-http-gw /usr/bin/frostfs-http-gw
|
||||
COPY --from=frostfs-s3-lifecycler /bin/frostfs-s3-lifecycler /usr/bin/frostfs-s3-lifecycler
|
||||
|
||||
COPY ./adm/frostfs-adm.yml /config/frostfs-adm.yml
|
||||
COPY ./ir/cli-cfg.yaml /config/cli-cfg-ir.yaml
|
||||
|
|
@ -53,6 +56,7 @@ COPY ./s3-gw/s3-gw-wallet.json /config/s3-gw-wallet.json
|
|||
COPY ./s3-gw/s3-gw-tls.crt /config/s3-gw-tls.crt
|
||||
COPY ./s3-gw/s3-gw-tls.key /config/s3-gw-tls.key
|
||||
COPY ./s3-gw/user-wallet.json /config/user-wallet.json
|
||||
COPY ./s3-lifecycler/s3-lc-config.yaml /config/s3-lc-config.yaml
|
||||
COPY ./sn/cli-cfg.yaml /config/cli-cfg-sn.yaml
|
||||
COPY ./sn/wallet.json /config/wallet-sn.json
|
||||
COPY ./sn/config.yaml /config/config-sn.yaml
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ ARG NEOGO_HUB_IMAGE=nspccdev/neo-go
|
|||
ARG FROSTFS_CORE_TAG=dev
|
||||
ARG FROSTFS_HTTP_GATE_TAG=dev
|
||||
ARG FROSTFS_S3_GATE_TAG=dev
|
||||
ARG FROSTFS_S3_LIFECYCLER_TAG=dev
|
||||
ARG NEOGO_TAG=dev
|
||||
|
||||
FROM ${NEOGO_HUB_IMAGE}:${NEOGO_TAG} AS neo-go
|
||||
|
|
@ -13,6 +14,7 @@ FROM ${FROSTFS_FORGEJO_IMAGE}-cli:${FROSTFS_CORE_TAG} AS frostfs-cli
|
|||
FROM ${FROSTFS_FORGEJO_IMAGE}-ir:${FROSTFS_CORE_TAG} AS frostfs-ir
|
||||
FROM ${FROSTFS_FORGEJO_IMAGE}-storage:${FROSTFS_CORE_TAG} AS frostfs-storage
|
||||
FROM ${FROSTFS_HUB_IMAGE}-s3-gw:${FROSTFS_S3_GATE_TAG} AS frostfs-s3-gw
|
||||
FROM ${FROSTFS_HUB_IMAGE}-s3-lifecycler:${FROSTFS_S3_LIFECYCLER_TAG} AS frostfs-s3-lifecycler
|
||||
FROM ${FROSTFS_HUB_IMAGE}-http-gw:${FROSTFS_HTTP_GATE_TAG} AS frostfs-http-gw
|
||||
|
||||
# Executable image
|
||||
|
|
@ -35,6 +37,7 @@ COPY --from=frostfs-storage /bin/frostfs-node /usr/bin/frostfs-node
|
|||
COPY --from=frostfs-s3-gw /bin/frostfs-s3-gw /usr/bin/frostfs-s3-gw
|
||||
COPY --from=frostfs-s3-gw /bin/frostfs-s3-authmate /usr/bin/frostfs-s3-authmate
|
||||
COPY --from=frostfs-http-gw /bin/frostfs-http-gw /usr/bin/frostfs-http-gw
|
||||
COPY --from=frostfs-s3-lifecycler /bin/frostfs-s3-lifecycler /usr/bin/frostfs-s3-lifecycler
|
||||
|
||||
COPY ./adm/frostfs-adm.yml /config/frostfs-adm.yml
|
||||
COPY ./ir/cli-cfg.yaml /config/cli-cfg-ir.yaml
|
||||
|
|
@ -53,6 +56,7 @@ COPY ./s3-gw/s3-gw-wallet.json /config/s3-gw-wallet.json
|
|||
COPY ./s3-gw/s3-gw-tls.crt /config/s3-gw-tls.crt
|
||||
COPY ./s3-gw/s3-gw-tls.key /config/s3-gw-tls.key
|
||||
COPY ./s3-gw/user-wallet.json /config/user-wallet.json
|
||||
COPY ./s3-lifecycler/s3-lc-config.yaml /config/s3-lc-config.yaml
|
||||
COPY ./sn/cli-cfg.yaml /config/cli-cfg-sn.yaml
|
||||
COPY ./sn/wallet.json /config/wallet-sn.json
|
||||
COPY ./sn/config.yaml /config/config-sn.yaml
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ COPY ./bin/frostfs-s3-gw /usr/bin/frostfs-s3-gw
|
|||
COPY ./bin/frostfs-s3-authmate /usr/bin/frostfs-s3-authmate
|
||||
COPY ./bin/frostfs-http-gw /usr/bin/frostfs-http-gw
|
||||
COPY ./bin/issue-creds.sh /usr/bin/issue-creds.sh
|
||||
COPY ./bin/frostfs-s3-lifecycler /usr/bin/frostfs-s3-lifecycler
|
||||
|
||||
COPY ./adm/frostfs-adm.yml /config/frostfs-adm.yml
|
||||
COPY ./ir/cli-cfg.yaml /config/cli-cfg-ir.yaml
|
||||
|
|
@ -37,6 +38,7 @@ COPY ./s3-gw/s3-gw-wallet.json /config/s3-gw-wallet.json
|
|||
COPY ./s3-gw/s3-gw-tls.crt /config/s3-gw-tls.crt
|
||||
COPY ./s3-gw/s3-gw-tls.key /config/s3-gw-tls.key
|
||||
COPY ./s3-gw/user-wallet.json /config/user-wallet.json
|
||||
COPY ./s3-lifecycler/s3-lc-config.yaml /config/s3-lc-config.yaml
|
||||
COPY ./sn/cli-cfg.yaml /config/cli-cfg-sn.yaml
|
||||
COPY ./sn/wallet.json /config/wallet-sn.json
|
||||
COPY ./sn/config.yaml /config/config-sn.yaml
|
||||
|
|
|
|||
3
Makefile
3
Makefile
|
|
@ -34,6 +34,7 @@ image-aio:
|
|||
--build-arg FROSTFS_CORE_TAG=$(FROSTFS_CORE_TAG) \
|
||||
--build-arg FROSTFS_HTTP_GATE_TAG=$(FROSTFS_HTTP_GATE_TAG) \
|
||||
--build-arg FROSTFS_S3_GATE_TAG=$(FROSTFS_S3_GATE_TAG) \
|
||||
--build-arg FROSTFS_S3_LIFECYCLER_TAG=$(FROSTFS_S3_LIFECYCLER_TAG) \
|
||||
--build-arg NEOGO_TAG=$(NEOGO_TAG) \
|
||||
-t $(AIO_IMAGE):$(AIO_VERSION) .
|
||||
|
||||
|
|
@ -46,6 +47,7 @@ image-aio-local:
|
|||
--build-arg FROSTFS_CORE_TAG=$(FROSTFS_CORE_TAG) \
|
||||
--build-arg FROSTFS_HTTP_GATE_TAG=$(FROSTFS_HTTP_GATE_TAG) \
|
||||
--build-arg FROSTFS_S3_GATE_TAG=$(FROSTFS_S3_GATE_TAG) \
|
||||
--build-arg FROSTFS_S3_LIFECYCLER_TAG=$(FROSTFS_S3_LIFECYCLER_TAG) \
|
||||
--build-arg NEOGO_TAG=$(NEOGO_TAG) \
|
||||
-t $(AIO_IMAGE):$(AIO_VERSION) .
|
||||
|
||||
|
|
@ -58,6 +60,7 @@ image-aio-custom:
|
|||
--build-arg FROSTFS_CORE_TAG=$(FROSTFS_CORE_TAG) \
|
||||
--build-arg FROSTFS_HTTP_GATE_TAG=$(FROSTFS_HTTP_GATE_TAG) \
|
||||
--build-arg FROSTFS_S3_GATE_TAG=$(FROSTFS_S3_GATE_TAG) \
|
||||
--build-arg FROSTFS_S3_LIFECYCLER_TAG=$(FROSTFS_S3_LIFECYCLER_TAG) \
|
||||
--build-arg NEOGO_TAG=$(NEOGO_TAG) \
|
||||
-t $(AIO_IMAGE):$(AIO_VERSION) .
|
||||
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ files for:
|
|||
- frostfs-s3-gw
|
||||
- frostfs-s3-authmate
|
||||
- frostfs-http-gw
|
||||
- frostfs-s3-lifecycler
|
||||
|
||||
Entrypoint script starts blockchain, inner ring, storage, s3, and http gateway
|
||||
services and configures it in the initial start.
|
||||
|
|
@ -117,6 +118,7 @@ Put all the needed pre-built binaries to the `frostfs-aio/bin/`:
|
|||
- frostfs-s3-gw
|
||||
- frostfs-s3-authmate
|
||||
- frostfs-http-gw
|
||||
- frostfs-s3-lifecycler
|
||||
|
||||
Make sure they all have the `x` right for the execution enabled.
|
||||
Build frostfs-aio image using pre-built local binaries for all the services.
|
||||
|
|
|
|||
|
|
@ -66,6 +66,7 @@ runServices() {
|
|||
done
|
||||
|
||||
/usr/bin/frostfs-s3-gw --config /config/s3-gw-config.yaml |& tee -a ${LOG_DIR}/frostfs-s3-gw &
|
||||
/usr/bin/frostfs-s3-lifecycler --config /config/s3-lc-config.yaml |& tee -a ${LOG_DIR}/frostfs-s3-lifecycler &
|
||||
/usr/bin/frostfs-http-gw --config /config/http-gw-config.yaml |& tee -a ${LOG_DIR}/frostfs-http-gw &
|
||||
}
|
||||
|
||||
|
|
|
|||
61
s3-lifecycler/s3-lc-config.yaml
Normal file
61
s3-lifecycler/s3-lc-config.yaml
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
# Wallet address, path to the wallet must be set as cli parameter or environment variable
|
||||
wallet:
|
||||
path: /wallets/wallet1.json # Path to wallet
|
||||
address: NhCHDEtGgSph1v6PmjFC1gtzJWNKtNSadk # Account address. If omitted default one will be used.
|
||||
passphrase: "" # Passphrase to decrypt wallet. If you're using a wallet without a password, place '' here.
|
||||
|
||||
logger:
|
||||
level: debug # Log level.
|
||||
destination: stdout # Logging destination.
|
||||
sampling:
|
||||
enabled: false
|
||||
initial: 100
|
||||
thereafter: 100
|
||||
interval: 1s
|
||||
|
||||
morph:
|
||||
rpc_endpoint:
|
||||
- address: ws://localhost:30333/ws
|
||||
priority: 1
|
||||
reconnect_clients_interval: 30s
|
||||
dial_timeout: 5s
|
||||
contract:
|
||||
netmap: netmap.frostfs
|
||||
frostfsid: frostfsid.frostfs
|
||||
container: container.frostfs
|
||||
|
||||
credential:
|
||||
use: wallets
|
||||
source:
|
||||
wallets:
|
||||
- path: /wallets/wallet1.json
|
||||
address: NhCHDEtGgSph1v6PmjFC1gtzJWNKtNSadk
|
||||
passphrase: ""
|
||||
- path: /wallets/wallet2.json
|
||||
address: NivQRezLwGP8xFPr87DD5XaYGUKK3BMsTa
|
||||
passphrase: ""
|
||||
- path: /wallets/wallet3.json
|
||||
address: NR8qvrREk1eCeQJce9zBaDyPE3PHGZfT6t
|
||||
passphrase: ""
|
||||
- path: /config/user-wallet.json
|
||||
address: NWeByJPgNC97F83hTUnSbnZSBKaFvk5HNw
|
||||
passphrase: ""
|
||||
|
||||
|
||||
lifecycle:
|
||||
job_fetcher_buffer: 1000
|
||||
executor_pool_size: 100
|
||||
|
||||
frostfs:
|
||||
stream_timeout: 10s
|
||||
connect_timeout: 10s
|
||||
healthcheck_timeout: 15s
|
||||
rebalance_interval: 60s
|
||||
pool_error_threshold: 100
|
||||
tree_pool_max_attempts: 4
|
||||
|
||||
peers:
|
||||
0:
|
||||
address: localhost:8080
|
||||
priority: 1
|
||||
weight: 1
|
||||
Loading…
Reference in a new issue