Compare commits

..

1 commit

Author SHA1 Message Date
24777badf7 Update s3-gw
TrueCloudLab/frostfs-s3-gw#451

* Don't skip [Next]PartNumberMarker in ListParts response
* Support Location in CompleteMultipart response

Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
2024-08-08 10:46:59 +03:00
24 changed files with 53 additions and 187 deletions

2
.env
View file

@ -1,3 +1,3 @@
FROSTFS_HUB_IMAGE=truecloudlab/frostfs
AIO_IMAGE=truecloudlab/frostfs-aio
AIO_VERSION=1.6.4
AIO_VERSION=1.5.0-dev.1

View file

@ -1,18 +1,18 @@
ARG FROSTFS_HUB_IMAGE=truecloudlab/frostfs
ARG NEOGO_HUB_IMAGE=nspccdev/neo-go
ARG FROSTFS_CORE_TAG=0.42.15
ARG FROSTFS_HTTP_GATE_TAG=0.30.2
ARG FROSTFS_S3_GATE_TAG=0.30.7
ARG NEOGO_TAG=0.106.0
ARG FROSTFS_CORE_TAG=0.38.6
ARG FROSTFS_HTTP_GATE_TAG=0.29.0
ARG FROSTFS_S3_GATE_TAG=0.29.3-2-gff7f963f
ARG NEOGO_TAG=0.104.0
FROM ${NEOGO_HUB_IMAGE}:${NEOGO_TAG} AS neo-go
FROM ${FROSTFS_HUB_IMAGE}-adm:${FROSTFS_CORE_TAG} AS frostfs-adm
FROM ${FROSTFS_HUB_IMAGE}-cli:${FROSTFS_CORE_TAG} AS frostfs-cli
FROM ${FROSTFS_HUB_IMAGE}-ir:${FROSTFS_CORE_TAG} AS frostfs-ir
FROM ${FROSTFS_HUB_IMAGE}-storage:${FROSTFS_CORE_TAG} AS frostfs-storage
FROM ${FROSTFS_HUB_IMAGE}-s3-gw:${FROSTFS_S3_GATE_TAG} AS frostfs-s3-gw
FROM ${FROSTFS_HUB_IMAGE}-http-gw:${FROSTFS_HTTP_GATE_TAG} AS frostfs-http-gw
FROM ${NEOGO_HUB_IMAGE}:${NEOGO_TAG} as neo-go
FROM ${FROSTFS_HUB_IMAGE}-adm:${FROSTFS_CORE_TAG} as frostfs-adm
FROM ${FROSTFS_HUB_IMAGE}-cli:${FROSTFS_CORE_TAG} as frostfs-cli
FROM ${FROSTFS_HUB_IMAGE}-ir:${FROSTFS_CORE_TAG} as frostfs-ir
FROM ${FROSTFS_HUB_IMAGE}-storage:${FROSTFS_CORE_TAG} as frostfs-storage
FROM ${FROSTFS_HUB_IMAGE}-s3-gw:${FROSTFS_S3_GATE_TAG} as frostfs-s3-gw
FROM ${FROSTFS_HUB_IMAGE}-http-gw:${FROSTFS_HTTP_GATE_TAG} as frostfs-http-gw
# Executable image
FROM alpine AS frostfs-aio
@ -46,7 +46,6 @@ COPY ./morph/node-config.yaml /config/node-config.yaml
COPY ./http-gw/http-gw-config.yaml /config/http-gw-config.yaml
COPY ./http-gw/http-gw-wallet.json /config/http-gw-wallet.json
COPY ./s3-gw/rules.json /config/bearer-rules.json
COPY ./s3-gw/regions.json /config/regions.json
COPY ./s3-gw/s3-gw-config.yaml /config/s3-gw-config.yaml
COPY ./s3-gw/s3-gw-wallet.json /config/s3-gw-wallet.json
COPY ./s3-gw/user-wallet.json /config/user-wallet.json
@ -57,9 +56,7 @@ COPY ./vendor/locode_db /config/locode.db
COPY ./vendor/contracts/ /config/contracts
COPY ./bin/init-aio.sh /usr/bin/init-aio.sh
COPY ./bin/issue-creds.sh /usr/bin/issue-creds.sh
ENV AUTHMATE_WALLET_PASSPHRASE=""
ENV AUTHMATE_WALLET_CONTRACT_PASSPHRASE=""
ENTRYPOINT ["/usr/bin/init-aio.sh"]

View file

@ -1,18 +1,18 @@
ARG FROSTFS_HUB_IMAGE=truecloudlab/frostfs
ARG NEOGO_HUB_IMAGE=nspccdev/neo-go
ARG FROSTFS_CORE_TAG=0.42.15
ARG FROSTFS_HTTP_GATE_TAG=0.30.2
ARG FROSTFS_S3_GATE_TAG=0.30.7
ARG NEOGO_TAG=0.106.0
ARG FROSTFS_CORE_TAG=0.38.6
ARG FROSTFS_HTTP_GATE_TAG=0.29.0
ARG FROSTFS_S3_GATE_TAG=0.29.3-2-gff7f963f
ARG NEOGO_TAG=0.104.0
FROM ${NEOGO_HUB_IMAGE}:${NEOGO_TAG} AS neo-go
FROM ${FROSTFS_HUB_IMAGE}-adm:${FROSTFS_CORE_TAG} AS frostfs-adm
FROM ${FROSTFS_HUB_IMAGE}-cli:${FROSTFS_CORE_TAG} AS frostfs-cli
FROM ${FROSTFS_HUB_IMAGE}-ir:${FROSTFS_CORE_TAG} AS frostfs-ir
FROM ${FROSTFS_HUB_IMAGE}-storage:${FROSTFS_CORE_TAG} AS frostfs-storage
FROM ${FROSTFS_HUB_IMAGE}-s3-gw:${FROSTFS_S3_GATE_TAG} AS frostfs-s3-gw
FROM ${FROSTFS_HUB_IMAGE}-http-gw:${FROSTFS_HTTP_GATE_TAG} AS frostfs-http-gw
FROM ${NEOGO_HUB_IMAGE}:${NEOGO_TAG} as neo-go
FROM ${FROSTFS_HUB_IMAGE}-adm:${FROSTFS_CORE_TAG} as frostfs-adm
FROM ${FROSTFS_HUB_IMAGE}-cli:${FROSTFS_CORE_TAG} as frostfs-cli
FROM ${FROSTFS_HUB_IMAGE}-ir:${FROSTFS_CORE_TAG} as frostfs-ir
FROM ${FROSTFS_HUB_IMAGE}-storage:${FROSTFS_CORE_TAG} as frostfs-storage
FROM ${FROSTFS_HUB_IMAGE}-s3-gw:${FROSTFS_S3_GATE_TAG} as frostfs-s3-gw
FROM ${FROSTFS_HUB_IMAGE}-http-gw:${FROSTFS_HTTP_GATE_TAG} as frostfs-http-gw
# Executable image
FROM alpine AS frostfs-aio
@ -46,7 +46,6 @@ COPY ./morph/node-config.yaml /config/node-config.yaml
COPY ./http-gw/http-gw-config.yaml /config/http-gw-config.yaml
COPY ./http-gw/http-gw-wallet.json /config/http-gw-wallet.json
COPY ./s3-gw/rules.json /config/bearer-rules.json
COPY ./s3-gw/regions.json /config/regions.json
COPY ./s3-gw/s3-gw-config.yaml /config/s3-gw-config.yaml
COPY ./s3-gw/s3-gw-wallet.json /config/s3-gw-wallet.json
COPY ./s3-gw/user-wallet.json /config/user-wallet.json
@ -59,6 +58,5 @@ COPY ./vendor/contracts/ /config/contracts
COPY ./bin/* /usr/bin/
ENV AUTHMATE_WALLET_PASSPHRASE=""
ENV AUTHMATE_WALLET_CONTRACT_PASSPHRASE=""
ENTRYPOINT ["/usr/bin/init-aio.sh"]

View file

@ -18,7 +18,6 @@ COPY ./bin/frostfs-node /usr/bin/frostfs-node
COPY ./bin/frostfs-s3-gw /usr/bin/frostfs-s3-gw
COPY ./bin/frostfs-s3-authmate /usr/bin/frostfs-s3-authmate
COPY ./bin/frostfs-http-gw /usr/bin/frostfs-http-gw
COPY ./bin/issue-creds.sh /usr/bin/issue-creds.sh
COPY ./adm/frostfs-adm.yml /config/frostfs-adm.yml
COPY ./ir/cli-cfg.yaml /config/cli-cfg-ir.yaml
@ -31,7 +30,6 @@ COPY ./morph/node-config.yaml /config/node-config.yaml
COPY ./http-gw/http-gw-config.yaml /config/http-gw-config.yaml
COPY ./http-gw/http-gw-wallet.json /config/http-gw-wallet.json
COPY ./s3-gw/rules.json /config/bearer-rules.json
COPY ./s3-gw/regions.json /config/regions.json
COPY ./s3-gw/s3-gw-config.yaml /config/s3-gw-config.yaml
COPY ./s3-gw/s3-gw-wallet.json /config/s3-gw-wallet.json
COPY ./s3-gw/user-wallet.json /config/user-wallet.json
@ -44,6 +42,5 @@ COPY ./vendor/contracts/ /config/contracts
COPY ./bin/init-aio.sh /usr/bin/init-aio.sh
ENV AUTHMATE_WALLET_PASSPHRASE=""
ENV AUTHMATE_WALLET_CONTRACT_PASSPHRASE=""
ENTRYPOINT ["/usr/bin/init-aio.sh"]

View file

@ -9,7 +9,6 @@ VERSION ?= "$(shell git describe --tags --match "v*" 2>/dev/null || git rev-pars
# Variables for docker
AIO_IMAGE ?= "truecloudlab/frostfs-aio"
LOG_DIR ?= "/var/log"
# Variables for compose
COMPOSE_CMD_UP ?= docker-compose up -d
@ -63,19 +62,19 @@ clean: down
# Generate S3 credentials
s3cred:
@docker exec aio /usr/bin/issue-creds.sh s3
@docker exec aio /usr/bin/frostfs-s3-authmate issue-secret \
--wallet /config/user-wallet.json \
--peer localhost:8080 \
--gate-public-key $(S3_GATE_PUBLIC_KEY) \
--container-placement-policy "REP 1"
# Generate S3 credentials based on imported wallets
s3cred-custom:
@docker exec aio /usr/bin/issue-creds.sh s3 $(wallet)
# Register user-wallet.json in FrostFS ID contract
cred:
@docker exec aio /usr/bin/issue-creds.sh native
# Register custom wallet in FrostFS ID contract
cred-custom:
@docker exec aio /usr/bin/issue-creds.sh native $(wallet)
@docker exec aio /usr/bin/frostfs-s3-authmate issue-secret \
--wallet /wallets/$(wallet) \
--peer localhost:8080 \
--gate-public-key $(S3_GATE_PUBLIC_KEY) \
--container-placement-policy "REP 1"
enable-oracle:
@docker exec aio neo-go contract invokefunction -r http://localhost:30333 \
@ -92,37 +91,8 @@ enable-oracle:
tick.epoch:
@docker exec aio /usr/bin/frostfs-adm --config /config/frostfs-adm.yml morph force-new-epoch
# Create new wallet
WALLET?=wallets/wallet1.json
.PHONY: wallet
wallet:
@! test -f "$(WALLET)" || { echo "File exists: $(WALLET)"; exit 1; }
@docker exec -it aio /usr/bin/neo-go wallet init -a -w "$(WALLET)"
# Refill GAS in arbitrary WALLET (must be under wallets/ directory)
GAS?=50.0
.PHONY: refill
refill:
@docker exec aio /usr/bin/frostfs-adm --config /config/frostfs-adm.yml \
morph refill-gas --storage-wallet=/$(WALLET) --gas=$(GAS)
@$(MAKE) --no-print-directory balance
# Show wallet balance
.PHONY: balance
balance:
@echo "Balance for $(WALLET)"
@docker exec aio /usr/bin/neo-go wallet nep17 balance -r http://localhost:30333 -w "$(WALLET)"
# Show current version
version:
@echo $(VERSION)
@echo "frostfs-node: $(AIO_VERSION)"
@echo "neo-go: $(NEOGO_VERSION)"
# Show all logs of the FrostFS service
logs/%:
@docker exec aio cat ${LOG_DIR}/$*
# Streaming of the frostFS service logs
logs-stream/%:
@docker exec aio tail -f ${LOG_DIR}/$*

View file

@ -62,8 +62,7 @@ Data and the system configuration is stored in container's volume.
Next time container is started, it will take less time to initialize.
Before updating image version, reset `frostfs-aio` by clearing its local volume
before starting container. Update may contain incompatible changes, e.g. neo-go
database format update. This repository does not provide any migration tools.
before starting container.
```
$ make clean
@ -288,55 +287,16 @@ If everything is rigth, the stdout will show a pretty-printed contents of the wa
wallet successfully created, file location is /wallets/new-wallet.json
```
You can use `make wallet WALLET=wallets/new-wallet.json`
to execute the steps listed above a little faster.
### Transfering GAS tokens to wallet
To add GAS to a new wallet use `make refill`
```console
$ make refill GAS=12.3 WALLET=wallets/new-wallet.json
Waiting for transactions to persist...
Balance for wallets/new-wallet.json
Account NgB6jnyVsMg6E3gQQLXoccZ3JBMLsgiut3
GAS: GasToken (d2a4cff31913016155e38e474a2c06d08be276cf)
Amount : 12.3
Updated: 25069
```
### Checking wallet balance
```console
$ make balance WALLET=wallets/new-wallet.json
Balance for wallets/new-wallet.json
Account NgB6jnyVsMg6E3gQQLXoccZ3JBMLsgiut3
GAS: GasToken (d2a4cff31913016155e38e474a2c06d08be276cf)
Amount : 12.3
Updated: 25069
```
## frostfs-cli interface
### Native credentials
To use wallet file, it should be register in FrostFS ID contract and have
policies to perform FrostFS operations.
Use `make cred` to register `s3-gw/user-wallet.json`. For custom wallets from
`wallet` dir, use `make cred-custom wallet=wallet1.json`.
These commands are similar to `make s3cred` and `make s3cred-custom`.
### Create container with frostfs-cli
``` sh
$ frostfs-cli -r localhost:8080 -w s3-gw/user-wallet.json \
$ frostfs-cli -r localhost:8080 -w /config/user-wallet.json \
--address NWeByJPgNC97F83hTUnSbnZSBKaFvk5HNw \
container create \
--policy "REP 1" --await
--policy "REP 1" --basic-acl public-read-write --await
container ID: GfWw35kHds7gKWmSvW7Zi4U39K7NMLK8EfXBQ5FPJA46
awaiting...
container has been persisted on sidechain
@ -345,7 +305,7 @@ container has been persisted on sidechain
### Put an object with frostfs-cli
``` sh
$ frostfs-cli -r localhost:8080 -w s3-gw/user-wallet.json \
$ frostfs-cli -r localhost:8080 -w /config/user-wallet.json \
--address NWeByJPgNC97F83hTUnSbnZSBKaFvk5HNw \
object put \
--cid GfWw35kHds7gKWmSvW7Zi4U39K7NMLK8EfXBQ5FPJA46 \
@ -358,25 +318,9 @@ $ frostfs-cli -r localhost:8080 -w s3-gw/user-wallet.json \
### Get and object with frostfs-cli
``` sh
$ frostfs-cli -r localhost:8080 -w s3-gw/user-wallet.json \
$ frostfs-cli -r localhost:8080 -w /config/user-wallet.json \
--address NWeByJPgNC97F83hTUnSbnZSBKaFvk5HNw \
object get \
--cid GfWw35kHds7gKWmSvW7Zi4U39K7NMLK8EfXBQ5FPJA46 \
--oid HByVC9A34i22BnzW3n83z9vEMxuYZoC7nNu11ZvGeCTe > new_cat.jpg
```
## Viewing logs
There are two targets for viewing logs.
You can view all the logs by replacing the desired service name:
``` sh
make logs/frostfs-node
```
You can also make a continuous stream of logs by replacing the desired service name:
``` sh
make stream-logs/frostfs-node
```

View file

@ -14,7 +14,7 @@ die() {
runBlockchain() {
stage "Starting the blockchain"
/usr/bin/neo-go node --config-path /config --privnet |& tee -a ${LOG_DIR}/neo-go &
/usr/bin/neo-go node --config-path /config --privnet >dev/null 2>&1 &
while [[ "$(curl -s -o /dev/null -w %{http_code} localhost:30333)" != "422" ]];
do
@ -31,15 +31,13 @@ configure() {
/usr/bin/frostfs-adm morph refill-gas --config /config/frostfs-adm.yml --storage-wallet /config/wallet-sn.json --gas 10.0 || die "Failed to transfer GAS to alphabet wallets"
/usr/bin/frostfs-adm morph proxy-add-account --config /config/frostfs-adm.yml --account NejLbQpojKJWec4NQRMBhzsrmCyhXfGJJe || die "Failed to set storage wallet as proxy wallet"
/usr/bin/frostfs-adm morph proxy-add-account --config /config/frostfs-adm.yml --account NN1RQ3qwnvDMVNsQw4WPkKi7BrjxdVTDZp || die "Failed to set s3 gateway wallet as proxy wallet"
}
runServices() {
stage "Running services"
/usr/bin/frostfs-ir --config /config/config-ir.yaml |& tee -a ${LOG_DIR}/frostfs-ir &
/usr/bin/frostfs-ir --config /config/config-ir.yaml >/dev/null 2>&1 &
while [[ -z "$(/usr/bin/frostfs-cli control ir healthcheck --endpoint localhost:16512 -c /config/cli-cfg-ir.yaml | grep 'Health status: READY')" ]];
do
@ -47,7 +45,7 @@ runServices() {
done
set -m
/usr/bin/frostfs-node --config /config/config-sn.yaml |& tee -a ${LOG_DIR}/frostfs-node &
/usr/bin/frostfs-node --config /config/config-sn.yaml >/dev/null 2>&1 &
while [[ -z "$(/usr/bin/frostfs-cli control healthcheck --endpoint localhost:16513 -c /config/cli-cfg-sn.yaml | grep 'Health status: READY')" ]];
do
@ -65,8 +63,8 @@ runServices() {
sleep 2
done
/usr/bin/frostfs-s3-gw --config /config/s3-gw-config.yaml |& tee -a ${LOG_DIR}/frostfs-s3-gw &
/usr/bin/frostfs-http-gw --config /config/http-gw-config.yaml |& tee -a ${LOG_DIR}/frostfs-http-gw &
/usr/bin/frostfs-s3-gw --config /config/s3-gw-config.yaml &
/usr/bin/frostfs-http-gw --config /config/http-gw-config.yaml &
}

View file

@ -1,35 +0,0 @@
#!/bin/bash
initUser() {
/usr/bin/frostfs-s3-authmate register-user \
--wallet $WALLET_PATH \
--rpc-endpoint http://localhost:30333 \
--username $USERNAME \
--contract-wallet /config/s3-gw-wallet.json >/dev/null 2>&1 && touch $WALLET_CACHE/$USERNAME
}
issueAWS() {
/usr/bin/frostfs-s3-authmate issue-secret \
--wallet $WALLET_PATH \
--peer localhost:8080 \
--gate-public-key $S3_GATE_PUBLIC_KEY \
--container-placement-policy "REP 1"
}
S3_GATE_PUBLIC_KEY=$(neo-go wallet dump-keys -w /config/s3-gw-wallet.json | tail -1)
WALLET_PATH=/wallets/$2
if [[ -z "$2" ]]; then
WALLET_PATH=/config/user-wallet.json
fi
WALLET_CACHE=/data/wallets
mkdir -p $WALLET_CACHE
USERNAME=$(echo $WALLET_PATH | md5sum | cut -d' ' -f1)
if [ ! -e $WALLET_CACHE/$USERNAME ]; then
initUser
fi
if [ $1 == "s3" ]; then
issueAWS
fi

View file

@ -15,8 +15,6 @@ services:
retries: 100
start_period: 2s
timeout: 2s
environment:
- LOG_DIR=/var/log
volumes:
- data:/data
- ./wallets:/wallets

View file

@ -11,7 +11,6 @@ ProtocolConfiguration:
VerifyTransactions: true
StateRootInHeader: true
P2PSigExtensions: true
Hardforks: {}
ApplicationConfiguration:
SkipBlockVerification: false

View file

@ -1,4 +0,0 @@
{
"us-east-1": "REP 1",
"us-west-1": "REP 1"
}

View file

@ -32,7 +32,7 @@ stream_timeout: 10s
healthcheck_timeout: 15s
rebalance_interval: 60s
pool_error_threshold: 100
max_clients_count: 1000
max_clients_count: 100
max_clients_deadline: 30s
cache:
@ -60,7 +60,6 @@ cache:
placement_policy:
default: REP 1
region_mapping: /config/regions.json
features:
md5:
@ -70,6 +69,9 @@ control:
grpc:
endpoint: localhost:16515
frostfsid:
validation:
enabled: false
kludge:
use_default_xmlns: true
bypass_content_encoding_check_in_chunks: true

View file

@ -2,6 +2,7 @@ morph:
rpc_endpoint:
- address: ws://localhost:30333/ws
dial_timeout: 20s
ape_chain_cache_size: 0
control:
grpc:
@ -37,7 +38,6 @@ pprof:
storage:
shard_num: 1
shard_pool_size: 1000
shard:
0:
writecache:
@ -66,4 +66,6 @@ tree:
object:
put:
local_pool_size: 1000
pool_size_remote: 100
pool_size_local: 100

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

File diff suppressed because one or more lines are too long

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.