Compare commits

..

40 commits

Author SHA1 Message Date
85ea5de04c Update codeowners
Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
2023-01-25 18:32:18 +03:00
6e4558d792 Add waiting for epoch align for storage group lifetime test
Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
2023-01-16 16:21:24 +03:00
791c461da1 align all nodes to be in the same epoch
Signed-off-by: Vladislav Karakozov <v.karakozov@yadro.com>
2023-01-16 16:21:24 +03:00
anastasia prasolova
b869e8751b Remove aprasolova from CODEOWNERS file
Signed-off-by: anastasia prasolova <anastasia@nspcc.ru>
2023-01-16 16:21:24 +03:00
69199d301b Fix epoch duration in http test
Signed-off-by: Vladislav Karakozov <v.karakozov@yadro.com>
2023-01-16 16:21:24 +03:00
5083b5adad http test with bearer token
Signed-off-by: Vladislav Karakozov <v.karakozov@yadro.com>
2023-01-16 16:21:24 +03:00
a2d272eaee Add too many open files to logs analyzer
Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
2023-01-16 16:21:24 +03:00
6367e5e8ed #478 Update lock tests
Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
2023-01-16 16:21:24 +03:00
4b59c1f327 Add support data and internal ips
Signed-off-by: Yulia Kovshova <y.kovshova@yadro.com>
2023-01-16 16:21:24 +03:00
6bf672c203 add system http system header test
Signed-off-by: Vladislav Karakozov <v.karakozov@yadro.com>
2023-01-16 16:21:24 +03:00
359bb6be90 Unskip static session token tests
Signed-off-by: Vladimir Avdeev <v.avdeev@yadro.com>
2023-01-16 16:21:24 +03:00
Мария Малыгина
5d7fca14ee [fix] s3 load
Signed-off-by: Мария Малыгина <m.malygina@MacBook-Pro-Maria.local>
2023-01-16 16:21:24 +03:00
7a11560000 Add await to delete container
Signed-off-by: anikeev-yadro <a.anikeev@yadro.com>
2023-01-16 16:21:24 +03:00
Vlad K
37cc85028e Revert "Fix: IndexError: list index out of range"
This reverts commit 1dc4516258.
2023-01-16 16:21:24 +03:00
79cb3c3698 Fix: IndexError: list index out of range
Signed-off-by: Vladislav Karakozov <v.karakozov@yadro.com>
2023-01-16 16:21:24 +03:00
146aaa371f Updates for local dev env runs
Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
2023-01-16 16:21:24 +03:00
d357241a61 new http test
Signed-off-by: Vladislav Karakozov <v.karakozov@yadro.com>
2023-01-16 16:21:24 +03:00
dc6e521f59 Update range tests
Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
2023-01-16 16:21:24 +03:00
29a8c1e252 Fix wildcard flag value
Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
2023-01-16 16:21:24 +03:00
32b010df67 Fix allure attaches for failover test
Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
2023-01-16 16:21:24 +03:00
c33857fd80 fix lock_mode
Signed-off-by: Yulia Kovshova <y.kovshova@yadro.com>
2023-01-16 16:21:24 +03:00
523e7ebac0 Fix policy test s3
Signed-off-by: Yulia Kovshova <y.kovshova@yadro.com>
2023-01-16 16:21:24 +03:00
d438748b2a Fix s3 range tests
Signed-off-by: anikeev-yadro <a.anikeev@yadro.com>
2023-01-16 16:21:24 +03:00
de6434934d Add drop locked objects tests
Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
2023-01-16 16:21:24 +03:00
25a9443209 Fix delete all objects in bucket
Signed-off-by: anikeev-yadro <a.anikeev@yadro.com>
2023-01-16 16:21:24 +03:00
c670893805 Add load_param file, delete old tests, new universal, parametrized test, add stop unused nodes function.
Signed-off-by: a.lipay <a.lipay@yadro.com>
2023-01-16 16:21:24 +03:00
efc5f271bf Add load_param file, delete old tests, new universal, parametrized test, add stop unused nodes function.
Signed-off-by: a.lipay <a.lipay@yadro.com>
2023-01-16 16:21:24 +03:00
294e0fbae3 Add load_param file, delete old tests, new universal, parametrized test, add stop unused nodes function.
Signed-off-by: a.lipay <a.lipay@yadro.com>
2023-01-16 16:21:24 +03:00
ac987c49aa Fix s3 object tests after remove obj size hardcode
Signed-off-by: Vladimir Avdeev <v.avdeev@yadro.com>
2023-01-16 16:21:24 +03:00
bd78c9f4c0 fix generate file in http test
Signed-off-by: Vladislav Karakozov <v.karakozov@yadro.com>
2023-01-16 16:21:24 +03:00
8aab12d5a8 Fix after remove obj size hardcode
Signed-off-by: Vladimir Avdeev <v.avdeev@yadro.com>
2023-01-16 16:21:24 +03:00
33e20b0562 Bump neofs-testlib to 0.8.1 (Fix logging problem)
Signed-off-by: Vladimir Avdeev <v.avdeev@yadro.com>
2023-01-16 16:21:24 +03:00
cd98d362e3 Update shards test
Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
2023-01-16 16:21:24 +03:00
b9b8c4e1f8 create http folder, and adding a new test for http attributes
Signed-off-by: Vladislav Karakozov <v.karakozov@yadro.com>
2023-01-16 16:21:24 +03:00
e99cc7d05d Small fixes for tests
Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
2023-01-16 16:21:24 +03:00
071914b45c Remove SIMPLE_OBJ_SIZE and COMPLEX_OBJ_SIZE from env
Signed-off-by: Vladimir Avdeev <v.avdeev@yadro.com>
2023-01-16 16:21:24 +03:00
12d8b1b10b Return to session log analyzer
Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
2023-01-16 16:21:24 +03:00
089ac609c3 Add control shards test
Signed-off-by: Vladimir Avdeev <v.avdeev@yadro.com>
2023-01-16 16:21:24 +03:00
adc63b0221 Add bearer token tests for s3 wallet api calls
Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
2023-01-16 16:21:24 +03:00
7d3fc5d15c Delete node extra test
Signed-off-by: anikeev-yadro <a.anikeev@yadro.com>
2023-01-16 16:21:24 +03:00
94 changed files with 1637 additions and 1479 deletions

View file

@ -5,98 +5,98 @@ hosts:
- name: s01 - name: s01
attributes: attributes:
container_name: s01 container_name: s01
config_path: ../frostfs-dev-env/services/storage/.storage.env config_path: ../neofs-dev-env/services/storage/.storage.env
wallet_path: ../frostfs-dev-env/services/storage/wallet01.json wallet_path: ../neofs-dev-env/services/storage/wallet01.json
local_config_path: ./TemporaryDir/empty-password.yml local_config_path: ./TemporaryDir/empty-password.yml
local_wallet_path: ../frostfs-dev-env/services/storage/wallet01.json local_wallet_path: ../neofs-dev-env/services/storage/wallet01.json
wallet_password: "" wallet_password: ""
volume_name: storage_storage_s01 volume_name: storage_storage_s01
endpoint_data0: s01.frostfs.devenv:8080 endpoint_data0: s01.neofs.devenv:8080
control_endpoint: s01.frostfs.devenv:8081 control_endpoint: s01.neofs.devenv:8081
un_locode: "RU MOW" un_locode: "RU MOW"
- name: s02 - name: s02
attributes: attributes:
container_name: s02 container_name: s02
config_path: ../frostfs-dev-env/services/storage/.storage.env config_path: ../neofs-dev-env/services/storage/.storage.env
wallet_path: ../frostfs-dev-env/services/storage/wallet02.json wallet_path: ../neofs-dev-env/services/storage/wallet02.json
local_config_path: ./TemporaryDir/empty-password.yml local_config_path: ./TemporaryDir/empty-password.yml
local_wallet_path: ../frostfs-dev-env/services/storage/wallet02.json local_wallet_path: ../neofs-dev-env/services/storage/wallet02.json
wallet_password: "" wallet_password: ""
volume_name: storage_storage_s02 volume_name: storage_storage_s02
endpoint_data0: s02.frostfs.devenv:8080 endpoint_data0: s02.neofs.devenv:8080
control_endpoint: s02.frostfs.devenv:8081 control_endpoint: s02.neofs.devenv:8081
un_locode: "RU LED" un_locode: "RU LED"
- name: s03 - name: s03
attributes: attributes:
container_name: s03 container_name: s03
config_path: ../frostfs-dev-env/services/storage/.storage.env config_path: ../neofs-dev-env/services/storage/.storage.env
wallet_path: ../frostfs-dev-env/services/storage/wallet03.json wallet_path: ../neofs-dev-env/services/storage/wallet03.json
local_config_path: ./TemporaryDir/empty-password.yml local_config_path: ./TemporaryDir/empty-password.yml
local_wallet_path: ../frostfs-dev-env/services/storage/wallet03.json local_wallet_path: ../neofs-dev-env/services/storage/wallet03.json
wallet_password: "" wallet_password: ""
volume_name: storage_storage_s03 volume_name: storage_storage_s03
endpoint_data0: s03.frostfs.devenv:8080 endpoint_data0: s03.neofs.devenv:8080
control_endpoint: s03.frostfs.devenv:8081 control_endpoint: s03.neofs.devenv:8081
un_locode: "SE STO" un_locode: "SE STO"
- name: s04 - name: s04
attributes: attributes:
container_name: s04 container_name: s04
config_path: ../frostfs-dev-env/services/storage/.storage.env config_path: ../neofs-dev-env/services/storage/.storage.env
wallet_path: ../frostfs-dev-env/services/storage/wallet04.json wallet_path: ../neofs-dev-env/services/storage/wallet04.json
local_config_path: ./TemporaryDir/empty-password.yml local_config_path: ./TemporaryDir/empty-password.yml
local_wallet_path: ../frostfs-dev-env/services/storage/wallet04.json local_wallet_path: ../neofs-dev-env/services/storage/wallet04.json
wallet_password: "" wallet_password: ""
volume_name: storage_storage_s04 volume_name: storage_storage_s04
endpoint_data0: s04.frostfs.devenv:8080 endpoint_data0: s04.neofs.devenv:8080
control_endpoint: s04.frostfs.devenv:8081 control_endpoint: s04.neofs.devenv:8081
un_locode: "FI HEL" un_locode: "FI HEL"
- name: s3-gate01 - name: s3-gate01
attributes: attributes:
container_name: s3_gate container_name: s3_gate
config_path: ../frostfs-dev-env/services/s3_gate/.s3.env config_path: ../neofs-dev-env/services/s3_gate/.s3.env
wallet_path: ../frostfs-dev-env/services/s3_gate/wallet.json wallet_path: ../neofs-dev-env/services/s3_gate/wallet.json
local_config_path: ./TemporaryDir/password-s3.yml local_config_path: ./TemporaryDir/password-s3.yml
local_wallet_path: ../frostfs-dev-env/services/s3_gate/wallet.json local_wallet_path: ../neofs-dev-env/services/s3_gate/wallet.json
wallet_password: "s3" wallet_password: "s3"
endpoint_data0: https://s3.frostfs.devenv:8080 endpoint_data0: https://s3.neofs.devenv:8080
- name: http-gate01 - name: http-gate01
attributes: attributes:
container_name: http_gate container_name: http_gate
config_path: ../frostfs-dev-env/services/http_gate/.http.env config_path: ../neofs-dev-env/services/http_gate/.http.env
wallet_path: ../frostfs-dev-env/services/http_gate/wallet.json wallet_path: ../neofs-dev-env/services/http_gate/wallet.json
local_config_path: ./TemporaryDir/password-other.yml local_config_path: ./TemporaryDir/password-other.yml
local_wallet_path: ../frostfs-dev-env/services/http_gate/wallet.json local_wallet_path: ../neofs-dev-env/services/http_gate/wallet.json
wallet_password: "one" wallet_password: "one"
endpoint_data0: http://http.frostfs.devenv endpoint_data0: http://http.neofs.devenv
- name: ir01 - name: ir01
attributes: attributes:
container_name: ir01 container_name: ir01
config_path: ../frostfs-dev-env/services/ir/.ir.env config_path: ../neofs-dev-env/services/ir/.ir.env
wallet_path: ../frostfs-dev-env/services/ir/az.json wallet_path: ../neofs-dev-env/services/ir/az.json
local_config_path: ./TemporaryDir/password-other.yml local_config_path: ./TemporaryDir/password-other.yml
local_wallet_path: ../frostfs-dev-env/services/ir/az.json local_wallet_path: ../neofs-dev-env/services/ir/az.json
wallet_password: "one" wallet_password: "one"
- name: morph-chain01 - name: morph-chain01
attributes: attributes:
container_name: morph_chain container_name: morph_chain
config_path: ../frostfs-dev-env/services/morph_chain/protocol.privnet.yml config_path: ../neofs-dev-env/services/morph_chain/protocol.privnet.yml
wallet_path: ../frostfs-dev-env/services/morph_chain/node-wallet.json wallet_path: ../neofs-dev-env/services/morph_chain/node-wallet.json
local_config_path: ./TemporaryDir/password-other.yml local_config_path: ./TemporaryDir/password-other.yml
local_wallet_path: ../frostfs-dev-env/services/morph_chain/node-wallet.json local_wallet_path: ../neofs-dev-env/services/morph_chain/node-wallet.json
wallet_password: "one" wallet_password: "one"
endpoint_internal0: http://morph-chain.frostfs.devenv:30333 endpoint_internal0: http://morph-chain.neofs.devenv:30333
- name: main-chain01 - name: main-chain01
attributes: attributes:
container_name: main_chain container_name: main_chain
config_path: ../frostfs-dev-env/services/chain/protocol.privnet.yml config_path: ../neofs-dev-env/services/chain/protocol.privnet.yml
wallet_path: ../frostfs-dev-env/services/chain/node-wallet.json wallet_path: ../neofs-dev-env/services/chain/node-wallet.json
local_config_path: ./TemporaryDir/password-other.yml local_config_path: ./TemporaryDir/password-other.yml
local_wallet_path: ../frostfs-dev-env/services/chain/node-wallet.json local_wallet_path: ../neofs-dev-env/services/chain/node-wallet.json
wallet_password: "one" wallet_password: "one"
endpoint_internal0: http://main-chain.frostfs.devenv:30333 endpoint_internal0: http://main-chain.neofs.devenv:30333
- name: coredns01 - name: coredns01
attributes: attributes:
container_name: coredns container_name: coredns
clis: clis:
- name: frostfs-cli - name: neofs-cli
exec_path: frostfs-cli exec_path: neofs-cli

2
.github/CODEOWNERS vendored
View file

@ -1 +1 @@
* @vdomnich-yadro @dansingjulia @yadro-vavdeev @alexchetaev @abereziny * @vdomnich-yadro @dansingjulia @yadro-vavdeev @abereziny

8
.gitignore vendored
View file

@ -1,11 +1,5 @@
# ignore IDE files # ignore IDE files
.vscode .vscode
.idea
.DS_Store
venv_macos
# ignore test results # ignore test results
**/log.html **/log.html
@ -26,4 +20,4 @@ TemporaryDir/*
artifacts/* artifacts/*
docs/* docs/*
venv.*/* venv.*/*
wallet_config.yml /*wallet_config.yml

View file

@ -5,7 +5,7 @@ repos:
- id: black - id: black
language_version: python3.9 language_version: python3.9
- repo: https://github.com/pycqa/isort - repo: https://github.com/pycqa/isort
rev: 5.12.0 rev: 5.10.1
hooks: hooks:
- id: isort - id: isort
name: isort (python) name: isort (python)

View file

@ -3,8 +3,8 @@
First, thank you for contributing! We love and encourage pull requests from First, thank you for contributing! We love and encourage pull requests from
everyone. Please follow the guidelines: everyone. Please follow the guidelines:
- Check the open [issues](https://github.com/TrueCloudLab/frostfs-testcases/issues) and - Check the open [issues](https://github.com/nspcc-dev/neofs-testcases/issues) and
[pull requests](https://github.com/TrueCloudLab/frostfs-testcases/pulls) for existing [pull requests](https://github.com/nspcc-dev/neofs-testcases/pulls) for existing
discussions. discussions.
- Open an issue first, to discuss a new feature or enhancement. - Open an issue first, to discuss a new feature or enhancement.
@ -22,13 +22,13 @@ everyone. Please follow the guidelines:
## Development Workflow ## Development Workflow
Start by forking the `frostfs-testcases` repository, make changes in a branch and then Start by forking the `neofs-testcases` repository, make changes in a branch and then
send a pull request. We encourage pull requests to discuss code changes. Here send a pull request. We encourage pull requests to discuss code changes. Here
are the steps in details: are the steps in details:
### Set up your GitHub Repository ### Set up your GitHub Repository
Fork [FrosfFS testcases upstream](https://github.com/TrueCloudLab/frostfs-testcases/fork) source Fork [NeoFS testcases upstream](https://github.com/nspcc-dev/neofs-testcases/fork) source
repository to your own personal repository. Copy the URL of your fork and clone it: repository to your own personal repository. Copy the URL of your fork and clone it:
```shell ```shell
@ -36,20 +36,33 @@ $ git clone <url of your fork>
``` ```
### Set up git remote as ``upstream`` ### Set up git remote as ``upstream``
```sh
$ cd frostfs-testcases ```shell
$ git remote add upstream https://github.com/TrueCloudLab/frostfs-testcases $ cd neofs-testcases
$ git remote add upstream https://github.com/nspcc-dev/neofs-testcases
$ git fetch upstream $ git fetch upstream
``` ```
### Set up development environment ### Set up development environment
To setup development environment for `frosfs-testcases`, please, take the following steps: To setup development environment for `neofs-testcases`, please, take the following steps:
1. Prepare virtualenv 1. Prepare virtualenv
```shell ```shell
$ make venv $ virtualenv --python=python3.9 venv
$ source frostfs-testcases-3.10/bin/activate $ source venv/bin/activate
```
2. Install all dependencies:
```shell
$ pip install -r requirements.txt
```
3. Setup pre-commit hooks to run code formatters on staged files before you run a `git commit` command:
```shell
$ pre-commit install
``` ```
Optionally you might want to integrate code formatters with your code editor to apply formatters to code files as you go: Optionally you might want to integrate code formatters with your code editor to apply formatters to code files as you go:

View file

@ -1,29 +1,28 @@
SHELL := /bin/bash #!/usr/bin/make -f
PYTHON_VERSION := 3.10
VENV_NAME = frostfs-testcases-${PYTHON_VERSION}
VENV_DIR := venv.${VENV_NAME}
current_dir := $(shell pwd) .DEFAULT_GOAL := help
venv: create requirements paths precommit SHELL ?= bash
@echo Ready
precommit: VENVS = $(shell ls -1d venv/*/ | sort -u | xargs basename -a)
@echo Isntalling pre-commit hooks
. ${VENV_DIR}/bin/activate && pre-commit install
paths: .PHONY: all
@echo Append paths for project all: venvs
@echo Virtual environment: ${VENV_DIR}
@sudo rm -rf ${VENV_DIR}/lib/python${PYTHON_VERSION}/site-packages/_paths.pth
@sudo touch ${VENV_DIR}/lib/python${PYTHON_VERSION}/site-packages/_paths.pth
@echo ${current_dir} | sudo tee ${VENV_DIR}/lib/python${PYTHON_VERSION}/site-packages/_paths.pth
create: include venv_template.mk
@echo Create virtual environment for
virtualenv --python=python${PYTHON_VERSION} --prompt=${VENV_NAME} ${VENV_DIR}
requirements: .PHONY: venvs
@echo Isntalling pip requirements venvs:
. ${VENV_DIR}/bin/activate && pip install -e ../frostfs-testlib $(foreach venv,$(VENVS),venv.$(venv))
. ${VENV_DIR}/bin/activate && pip install -Ur pytest_tests/requirements.txt
$(foreach venv,$(VENVS),$(eval $(call VENV_template,$(venv))))
clean:
rm -rf venv.*
pytest-local:
@echo "⇒ Run Pytest"
python -m pytest pytest_tests/testsuites/
help:
@echo "⇒ run Run testcases ${R}"

View file

@ -2,58 +2,70 @@
Tests written with PyTest Framework are located under `pytest_tests/testsuites` directory. Tests written with PyTest Framework are located under `pytest_tests/testsuites` directory.
These tests rely on resources and utility modules that have been originally developed for Pytest Framework. These tests rely on resources and utility modules that have been originally developed for Robot Framework:
`robot/resources/files` - static files that are used in tests' commands.
`robot/resources/lib/` - common Python libraries that provide utility functions used as building blocks in tests.
`robot/variables/` - constants and configuration variables for tests.
## Testcases execution ## Testcases execution
### Initial preparation ### Initial preparation
1. Install frostfs-cli 1. Install neofs-cli
- `git clone git@github.com:TrueCloudLab/frostfs-node.git` - `git clone git@github.com:nspcc-dev/neofs-node.git`
- `cd frostfs-node` - `cd neofs-node`
- `make` - `make`
- `sudo cp bin/frostfs-cli /usr/local/bin/frostfs-cli` - `sudo cp bin/neofs-cli /usr/local/bin/neofs-cli`
2. Install frostfs-authmate 2. Install neofs-authmate
- `git clone git@github.com:TrueCloudLab/frostfs-s3-gw.git` - `git clone git@github.com:nspcc-dev/neofs-s3-gw.git`
- `cd frostfs-s3-gw` - `cd neofs-s3-gw`
- `make` - `make`
- `sudo cp bin/frostfs-s3-authmate /usr/local/bin/frostfs-authmate` - `sudo cp bin/neofs-authmate /usr/local/bin/neofs-authmate`
3. Install neo-go 3. Install neo-go
- `git clone git@github.com:nspcc-dev/neo-go.git` - `git clone git@github.com:nspcc-dev/neo-go.git`
- `cd neo-go` - `cd neo-go`
- `git checkout v0.101.0` (or the current version in the frostfs-dev-env) - `git checkout v0.92.0` (or the current version in the neofs-dev-env)
- `make` - `make`
- `sudo cp bin/neo-go /usr/local/bin/neo-go` - `sudo cp bin/neo-go /usr/local/bin/neo-go`
or download binary from releases: https://github.com/nspcc-dev/neo-go/releases or download binary from releases: https://github.com/nspcc-dev/neo-go/releases
4. Clone frostfs-dev-env 4. Clone neofs-dev-env
`git clone git@github.com:TrueCloudLab/frostfs-dev-env.git` `git clone git@github.com:nspcc-dev/neofs-dev-env.git`
Note that we expect frostfs-dev-env to be located under Note that we expect neofs-dev-env to be located under
the `<testcases_root_dir>/../frostfs-dev-env` directory. If you put this repo in any other place, the `<testcases_root_dir>/../neofs-dev-env` directory. If you put this repo in any other place,
manually set the full path to frostfs-dev-env in the environment variable `DEVENV_PATH` at this step. manually set the full path to neofs-dev-env in the environment variable `DEVENV_PATH` at this step.
5. Make sure you have installed all the following prerequisites on your machine 5. Make sure you have installed all of the following prerequisites on your machine
``` ```
make make
python3.10 python3.9
python3.10-dev python3.9-dev
libssl-dev libssl-dev
``` ```
As we use frostfs-dev-env, you'll also need to install As we use neofs-dev-env, you'll also need to install
[prerequisites](https://github.com/TrueCloudLab/frostfs-dev-env#prerequisites) of this repository. [prerequisites](https://github.com/nspcc-dev/neofs-dev-env#prerequisites) of this repository.
6. Prepare virtualenv 6. Prepare virtualenv
```shell ```shell
$ make venv $ make venv.local-pytest
$ source venv.frostfs-testcases-3.10/bin/activate $ . venv.local-pytest/bin/activate
``` ```
7. Optionally you might want to integrate code formatters with your code editor to apply formatters to code files as you go: 7. Setup pre-commit hooks to run code formatters on staged files before you run a `git commit` command:
```shell
$ pre-commit install
```
Optionally you might want to integrate code formatters with your code editor to apply formatters to code files as you go:
* isort is supported by [PyCharm](https://plugins.jetbrains.com/plugin/15434-isortconnect), [VS Code](https://cereblanco.medium.com/setup-black-and-isort-in-vscode-514804590bf9). Plugins exist for other IDEs/editors as well. * isort is supported by [PyCharm](https://plugins.jetbrains.com/plugin/15434-isortconnect), [VS Code](https://cereblanco.medium.com/setup-black-and-isort-in-vscode-514804590bf9). Plugins exist for other IDEs/editors as well.
* black can be integrated with multiple editors, please, instructions are available [here](https://black.readthedocs.io/en/stable/integrations/editors.html). * black can be integrated with multiple editors, please, instructions are available [here](https://black.readthedocs.io/en/stable/integrations/editors.html).
@ -110,7 +122,7 @@ $ docker run -p 5050:5050 -e CHECK_RESULTS_EVERY_SECONDS=30 -e KEEP_HISTORY=1 \
Then, you can check the allure report in your browser [by this link](http://localhost:5050/allure-docker-service/projects/default/reports/latest/index.html?redirect=false) Then, you can check the allure report in your browser [by this link](http://localhost:5050/allure-docker-service/projects/default/reports/latest/index.html?redirect=false)
NOTE: feel free to select a different location for `allure-reports` directory, there is no requirement to have it inside `frostfs-testcases`. For example, you can place it under `/tmp` path. NOTE: feel free to select a different location for `allure-reports` directory, there is no requirement to have it inside `neofs-testcases`. For example, you can place it under `/tmp` path.
# Contributing # Contributing

View file

@ -1,8 +1,8 @@
[tool.isort] [tool.isort]
profile = "black" profile = "black"
src_paths = ["pytest_tests"] src_paths = ["neofs-keywords", "pytest_tests", "robot"]
line_length = 100 line_length = 100
[tool.black] [tool.black]
line-length = 100 line-length = 100
target-version = ["py310"] target-version = ["py39"]

View file

@ -5,9 +5,8 @@ from datetime import datetime
from typing import Optional from typing import Optional
import allure import allure
from cli_helpers import _cmd_run
from pytest_tests.helpers.cli_helpers import _cmd_run from common import ASSETS_DIR
from pytest_tests.resources.common import ASSETS_DIR
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
REGULAR_TIMEOUT = 90 REGULAR_TIMEOUT = 90

View file

@ -1,11 +1,10 @@
import logging import logging
import re import re
from frostfs_testlib.cli import FrostfsAdm, FrostfsCli from common import NEOFS_ADM_EXEC, NEOFS_CLI_EXEC, WALLET_CONFIG
from frostfs_testlib.hosting import Hosting from neofs_testlib.cli import NeofsAdm, NeofsCli
from frostfs_testlib.shell import Shell from neofs_testlib.hosting import Hosting
from neofs_testlib.shell import Shell
from pytest_tests.resources.common import FROSTFS_ADM_EXEC, FROSTFS_CLI_EXEC, WALLET_CONFIG
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
@ -13,18 +12,18 @@ logger = logging.getLogger("NeoLogger")
def get_local_binaries_versions(shell: Shell) -> dict[str, str]: def get_local_binaries_versions(shell: Shell) -> dict[str, str]:
versions = {} versions = {}
for binary in ["neo-go", "frostfs-authmate"]: for binary in ["neo-go", "neofs-authmate"]:
out = shell.exec(f"{binary} --version").stdout out = shell.exec(f"{binary} --version").stdout
versions[binary] = _parse_version(out) versions[binary] = _parse_version(out)
frostfs_cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, WALLET_CONFIG) neofs_cli = NeofsCli(shell, NEOFS_CLI_EXEC, WALLET_CONFIG)
versions["frostfs-cli"] = _parse_version(frostfs_cli.version.get().stdout) versions["neofs-cli"] = _parse_version(neofs_cli.version.get().stdout)
try: try:
frostfs_adm = FrostfsAdm(shell, FROSTFS_ADM_EXEC) neofs_adm = NeofsAdm(shell, NEOFS_ADM_EXEC)
versions["frostfs-adm"] = _parse_version(frostfs_adm.version.get().stdout) versions["neofs-adm"] = _parse_version(neofs_adm.version.get().stdout)
except RuntimeError: except RuntimeError:
logger.info(f"frostfs-adm not installed") logger.info(f"neofs-adm not installed")
out = shell.exec("aws --version").stdout out = shell.exec("aws --version").stdout
out_lines = out.split("\n") out_lines = out.split("\n")

View file

@ -3,11 +3,12 @@ import re
from dataclasses import dataclass from dataclasses import dataclass
from typing import Any from typing import Any
import data_formatters
import yaml import yaml
from frostfs_testlib.blockchain import RPCClient from neofs_testlib.blockchain import RPCClient
from frostfs_testlib.hosting import Host, Hosting from neofs_testlib.hosting import Host, Hosting
from frostfs_testlib.hosting.config import ServiceConfig from neofs_testlib.hosting.config import ServiceConfig
from frostfs_testlib.utils import wallet_utils from test_control import wait_for_success
@dataclass @dataclass
@ -45,9 +46,11 @@ class NodeBase:
def label(self) -> str: def label(self) -> str:
return self.name return self.name
@wait_for_success(60, 1)
def start_service(self): def start_service(self):
self.host.start_service(self.name) self.host.start_service(self.name)
@wait_for_success(60, 1)
def stop_service(self): def stop_service(self):
self.host.stop_service(self.name) self.host.stop_service(self.name)
@ -85,7 +88,7 @@ class NodeBase:
def get_wallet_public_key(self): def get_wallet_public_key(self):
storage_wallet_path = self.get_wallet_path() storage_wallet_path = self.get_wallet_path()
storage_wallet_pass = self.get_wallet_password() storage_wallet_pass = self.get_wallet_password()
return wallet_utils.get_wallet_public_key(storage_wallet_path, storage_wallet_pass) return data_formatters.get_wallet_public_key(storage_wallet_path, storage_wallet_pass)
def _get_attribute(self, attribute_name: str, default_attribute_name: str = None) -> list[str]: def _get_attribute(self, attribute_name: str, default_attribute_name: str = None) -> list[str]:
config = self.host.get_service_config(self.name) config = self.host.get_service_config(self.name)
@ -107,7 +110,7 @@ class InnerRingNode(NodeBase):
Inner ring node is not always the same as physical host (or physical node, if you will): Inner ring node is not always the same as physical host (or physical node, if you will):
It can be service running in a container or on physical host It can be service running in a container or on physical host
For testing perspective, it's not relevant how it is actually running, For testing perspective, it's not relevant how it is actually running,
since frostfs network will still treat it as "node" since neofs network will still treat it as "node"
""" """
pass pass
@ -146,7 +149,7 @@ class MorphChain(NodeBase):
Consensus node is not always the same as physical host (or physical node, if you will): Consensus node is not always the same as physical host (or physical node, if you will):
It can be service running in a container or on physical host It can be service running in a container or on physical host
For testing perspective, it's not relevant how it is actually running, For testing perspective, it's not relevant how it is actually running,
since frostfs network will still treat it as "node" since neofs network will still treat it as "node"
""" """
rpc_client: RPCClient = None rpc_client: RPCClient = None
@ -169,7 +172,7 @@ class MainChain(NodeBase):
Consensus node is not always the same as physical host: Consensus node is not always the same as physical host:
It can be service running in a container or on physical host (or physical node, if you will): It can be service running in a container or on physical host (or physical node, if you will):
For testing perspective, it's not relevant how it is actually running, For testing perspective, it's not relevant how it is actually running,
since frostfs network will still treat it as "node" since neofs network will still treat it as "node"
""" """
rpc_client: RPCClient = None rpc_client: RPCClient = None
@ -192,7 +195,7 @@ class StorageNode(NodeBase):
Storage node is not always the same as physical host: Storage node is not always the same as physical host:
It can be service running in a container or on physical host (or physical node, if you will): It can be service running in a container or on physical host (or physical node, if you will):
For testing perspective, it's not relevant how it is actually running, For testing perspective, it's not relevant how it is actually running,
since frostfs network will still treat it as "node" since neofs network will still treat it as "node"
""" """
def get_rpc_endpoint(self) -> str: def get_rpc_endpoint(self) -> str:

View file

@ -1,22 +1,13 @@
import json
import logging
from dataclasses import dataclass from dataclasses import dataclass
from time import sleep from typing import Optional
from typing import Optional, Union
import allure import allure
from frostfs_testlib.cli import FrostfsCli from cluster import Cluster
from frostfs_testlib.shell import Shell from file_helper import generate_file, get_file_hash
from frostfs_testlib.utils import json_utils from neofs_testlib.shell import Shell
from neofs_verbs import put_object, put_object_to_random_node
from pytest_tests.helpers.cluster import Cluster from storage_object import StorageObjectInfo
from pytest_tests.helpers.file_helper import generate_file, get_file_hash from wallet import WalletFile
from pytest_tests.helpers.frostfs_verbs import put_object, put_object_to_random_node
from pytest_tests.helpers.storage_object_info import StorageObjectInfo
from pytest_tests.helpers.wallet import WalletFile
from pytest_tests.resources.common import CLI_DEFAULT_TIMEOUT, FROSTFS_CLI_EXEC, WALLET_CONFIG
logger = logging.getLogger("NeoLogger")
@dataclass @dataclass
@ -94,238 +85,3 @@ class StorageContainer:
) )
return storage_object return storage_object
DEFAULT_PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 4 FROM * AS X"
SINGLE_PLACEMENT_RULE = "REP 1 IN X CBF 1 SELECT 4 FROM * AS X"
REP_2_FOR_3_NODES_PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 3 FROM * AS X"
@allure.step("Create Container")
def create_container(
wallet: str,
shell: Shell,
endpoint: str,
rule: str = DEFAULT_PLACEMENT_RULE,
basic_acl: str = "",
attributes: Optional[dict] = None,
session_token: str = "",
session_wallet: str = "",
name: str = None,
options: dict = None,
await_mode: bool = True,
wait_for_creation: bool = True,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> str:
"""
A wrapper for `frostfs-cli container create` call.
Args:
wallet (str): a wallet on whose behalf a container is created
rule (optional, str): placement rule for container
basic_acl (optional, str): an ACL for container, will be
appended to `--basic-acl` key
attributes (optional, dict): container attributes , will be
appended to `--attributes` key
session_token (optional, str): a path to session token file
session_wallet(optional, str): a path to the wallet which signed
the session token; this parameter makes sense
when paired with `session_token`
shell: executor for cli command
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
options (optional, dict): any other options to pass to the call
name (optional, str): container name attribute
await_mode (bool): block execution until container is persisted
wait_for_creation (): Wait for container shows in container list
timeout: Timeout for the operation.
Returns:
(str): CID of the created container
"""
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, WALLET_CONFIG)
result = cli.container.create(
rpc_endpoint=endpoint,
wallet=session_wallet if session_wallet else wallet,
policy=rule,
basic_acl=basic_acl,
attributes=attributes,
name=name,
session=session_token,
await_mode=await_mode,
timeout=timeout,
**options or {},
)
cid = _parse_cid(result.stdout)
logger.info("Container created; waiting until it is persisted in the sidechain")
if wait_for_creation:
wait_for_container_creation(wallet, cid, shell, endpoint)
return cid
def wait_for_container_creation(
wallet: str, cid: str, shell: Shell, endpoint: str, attempts: int = 15, sleep_interval: int = 1
):
for _ in range(attempts):
containers = list_containers(wallet, shell, endpoint)
if cid in containers:
return
logger.info(f"There is no {cid} in {containers} yet; sleep {sleep_interval} and continue")
sleep(sleep_interval)
raise RuntimeError(
f"After {attempts * sleep_interval} seconds container {cid} hasn't been persisted; exiting"
)
def wait_for_container_deletion(
wallet: str, cid: str, shell: Shell, endpoint: str, attempts: int = 30, sleep_interval: int = 1
):
for _ in range(attempts):
try:
get_container(wallet, cid, shell=shell, endpoint=endpoint)
sleep(sleep_interval)
continue
except Exception as err:
if "container not found" not in str(err):
raise AssertionError(f'Expected "container not found" in error, got\n{err}')
return
raise AssertionError(f"Expected container deleted during {attempts * sleep_interval} sec.")
@allure.step("List Containers")
def list_containers(
wallet: str, shell: Shell, endpoint: str, timeout: Optional[str] = CLI_DEFAULT_TIMEOUT
) -> list[str]:
"""
A wrapper for `frostfs-cli container list` call. It returns all the
available containers for the given wallet.
Args:
wallet (str): a wallet on whose behalf we list the containers
shell: executor for cli command
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
timeout: Timeout for the operation.
Returns:
(list): list of containers
"""
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, WALLET_CONFIG)
result = cli.container.list(rpc_endpoint=endpoint, wallet=wallet, timeout=timeout)
logger.info(f"Containers: \n{result}")
return result.stdout.split()
@allure.step("Get Container")
def get_container(
wallet: str,
cid: str,
shell: Shell,
endpoint: str,
json_mode: bool = True,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> Union[dict, str]:
"""
A wrapper for `frostfs-cli container get` call. It extracts container's
attributes and rearranges them into a more compact view.
Args:
wallet (str): path to a wallet on whose behalf we get the container
cid (str): ID of the container to get
shell: executor for cli command
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
json_mode (bool): return container in JSON format
timeout: Timeout for the operation.
Returns:
(dict, str): dict of container attributes
"""
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, WALLET_CONFIG)
result = cli.container.get(
rpc_endpoint=endpoint, wallet=wallet, cid=cid, json_mode=json_mode, timeout=timeout
)
if not json_mode:
return result.stdout
container_info = json.loads(result.stdout)
attributes = dict()
for attr in container_info["attributes"]:
attributes[attr["key"]] = attr["value"]
container_info["attributes"] = attributes
container_info["ownerID"] = json_utils.json_reencode(container_info["ownerID"]["value"])
return container_info
@allure.step("Delete Container")
# TODO: make the error message about a non-found container more user-friendly
# https://github.com/nspcc-dev/frostfs-contract/issues/121
def delete_container(
wallet: str,
cid: str,
shell: Shell,
endpoint: str,
force: bool = False,
session_token: Optional[str] = None,
await_mode: bool = False,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> None:
"""
A wrapper for `frostfs-cli container delete` call.
Args:
wallet (str): path to a wallet on whose behalf we delete the container
cid (str): ID of the container to delete
shell: executor for cli command
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
force (bool): do not check whether container contains locks and remove immediately
session_token: a path to session token file
timeout: Timeout for the operation.
This function doesn't return anything.
"""
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, WALLET_CONFIG)
cli.container.delete(
wallet=wallet,
cid=cid,
rpc_endpoint=endpoint,
force=force,
session=session_token,
await_mode=await_mode,
timeout=timeout,
)
def _parse_cid(output: str) -> str:
"""
Parses container ID from a given CLI output. The input string we expect:
container ID: 2tz86kVTDpJxWHrhw3h6PbKMwkLtBEwoqhHQCKTre1FN
awaiting...
container has been persisted on sidechain
We want to take 'container ID' value from the string.
Args:
output (str): CLI output to parse
Returns:
(str): extracted CID
"""
try:
# taking first line from command's output
first_line = output.split("\n")[0]
except Exception:
first_line = ""
logger.error(f"Got empty output: {output}")
splitted = first_line.split(": ")
if len(splitted) != 2:
raise ValueError(f"no CID was parsed from command output: \t{first_line}")
return splitted[1]
@allure.step("Search container by name")
def search_container_by_name(wallet: str, name: str, shell: Shell, endpoint: str):
list_cids = list_containers(wallet, shell, endpoint)
for cid in list_cids:
cont_info = get_container(wallet, cid, shell, endpoint, True)
if cont_info.get("attributes").get("Name", None) == name:
return cid
return None

View file

@ -5,8 +5,7 @@ import uuid
from typing import Any, Optional from typing import Any, Optional
import allure import allure
from common import ASSETS_DIR
from pytest_tests.resources.common import ASSETS_DIR
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")

View file

@ -0,0 +1,36 @@
import re
# Regex patterns of status codes of Container service (https://github.com/nspcc-dev/neofs-spec/blob/98b154848116223e486ce8b43eaa35fec08b4a99/20-api-v2/container.md)
CONTAINER_NOT_FOUND = "code = 3072.*message = container not found"
# Regex patterns of status codes of Object service (https://github.com/nspcc-dev/neofs-spec/blob/98b154848116223e486ce8b43eaa35fec08b4a99/20-api-v2/object.md)
MALFORMED_REQUEST = "code = 1024.*message = malformed request"
OBJECT_ACCESS_DENIED = "code = 2048.*message = access to object operation denied"
OBJECT_NOT_FOUND = "code = 2049.*message = object not found"
OBJECT_ALREADY_REMOVED = "code = 2052.*message = object already removed"
SESSION_NOT_FOUND = "code = 4096.*message = session token not found"
OUT_OF_RANGE = "code = 2053.*message = out of range"
# TODO: Due to https://github.com/nspcc-dev/neofs-node/issues/2092 we have to check only codes until fixed
# OBJECT_IS_LOCKED = "code = 2050.*message = object is locked"
# LOCK_NON_REGULAR_OBJECT = "code = 2051.*message = ..." will be available once 2092 is fixed
OBJECT_IS_LOCKED = "code = 2050"
LOCK_NON_REGULAR_OBJECT = "code = 2051"
LIFETIME_REQUIRED = "either expiration epoch of a lifetime is required"
LOCK_OBJECT_REMOVAL = "lock object removal"
LOCK_OBJECT_EXPIRATION = "lock object expiration: {expiration_epoch}; current: {current_epoch}"
INVALID_RANGE_ZERO_LENGTH = "invalid '{range}' range: zero length"
INVALID_RANGE_OVERFLOW = "invalid '{range}' range: uint64 overflow"
INVALID_OFFSET_SPECIFIER = "invalid '{range}' range offset specifier"
INVALID_LENGTH_SPECIFIER = "invalid '{range}' range length specifier"
def error_matches_status(error: Exception, status_pattern: str) -> bool:
"""
Determines whether exception matches specified status pattern.
We use re.search to be consistent with pytest.raises.
"""
match = re.search(status_pattern, str(error))
return match is not None

View file

@ -1,4 +1,4 @@
from frostfs_testlib.shell import Shell from neofs_testlib.shell import Shell
class IpTablesHelper: class IpTablesHelper:

View file

@ -5,15 +5,14 @@ from time import sleep
from typing import Optional from typing import Optional
import allure import allure
from frostfs_testlib.shell import Shell from neofs_testlib.shell import Shell
from remote_process import RemoteProcess
from pytest_tests.helpers.remote_process import RemoteProcess
EXIT_RESULT_CODE = 0 EXIT_RESULT_CODE = 0
LOAD_RESULTS_PATTERNS = { LOAD_RESULTS_PATTERNS = {
"grpc": { "grpc": {
"write_ops": r"frostfs_obj_put_total\W*\d*\W*(?P<write_ops>\d*\.\d*)", "write_ops": r"neofs_obj_put_total\W*\d*\W*(?P<write_ops>\d*\.\d*)",
"read_ops": r"frostfs_obj_get_total\W*\d*\W*(?P<read_ops>\d*\.\d*)", "read_ops": r"neofs_obj_get_total\W*\d*\W*(?P<read_ops>\d*\.\d*)",
}, },
"s3": { "s3": {
"write_ops": r"aws_obj_put_total\W*\d*\W*(?P<write_ops>\d*\.\d*)", "write_ops": r"aws_obj_put_total\W*\d*\W*(?P<write_ops>\d*\.\d*)",

View file

@ -4,8 +4,8 @@ import uuid
from typing import Optional from typing import Optional
import allure import allure
from frostfs_testlib.shell import Shell from neofs_testlib.shell import Shell
from frostfs_testlib.shell.interfaces import CommandOptions from neofs_testlib.shell.interfaces import CommandOptions
from tenacity import retry, stop_after_attempt, wait_fixed from tenacity import retry, stop_after_attempt, wait_fixed

View file

@ -1,15 +1,13 @@
import logging import datetime
import os import os
from datetime import datetime, timedelta from datetime import datetime, timedelta
from typing import Optional from typing import Optional
import allure import allure
import s3_gate_bucket
import s3_gate_object
from dateutil.parser import parse from dateutil.parser import parse
from pytest_tests.steps import s3_gate_bucket, s3_gate_object
logger = logging.getLogger("NeoLogger")
@allure.step("Expected all objects are presented in the bucket") @allure.step("Expected all objects are presented in the bucket")
def check_objects_in_bucket( def check_objects_in_bucket(
@ -129,31 +127,3 @@ def assert_object_lock_mode(
assert ( assert (
retain_date - last_modify + timedelta(seconds=1) retain_date - last_modify + timedelta(seconds=1)
).days == retain_period, f"Expected retention period is {retain_period} days" ).days == retain_period, f"Expected retention period is {retain_period} days"
def assert_s3_acl(acl_grants: list, permitted_users: str):
if permitted_users == "AllUsers":
grantees = {"AllUsers": 0, "CanonicalUser": 0}
for acl_grant in acl_grants:
if acl_grant.get("Grantee", {}).get("Type") == "Group":
uri = acl_grant.get("Grantee", {}).get("URI")
permission = acl_grant.get("Permission")
assert (uri, permission) == (
"http://acs.amazonaws.com/groups/global/AllUsers",
"FULL_CONTROL",
), "All Groups should have FULL_CONTROL"
grantees["AllUsers"] += 1
if acl_grant.get("Grantee", {}).get("Type") == "CanonicalUser":
permission = acl_grant.get("Permission")
assert permission == "FULL_CONTROL", "Canonical User should have FULL_CONTROL"
grantees["CanonicalUser"] += 1
assert grantees["AllUsers"] >= 1, "All Users should have FULL_CONTROL"
assert grantees["CanonicalUser"] >= 1, "Canonical User should have FULL_CONTROL"
if permitted_users == "CanonicalUser":
for acl_grant in acl_grants:
if acl_grant.get("Grantee", {}).get("Type") == "CanonicalUser":
permission = acl_grant.get("Permission")
assert permission == "FULL_CONTROL", "Only CanonicalUser should have FULL_CONTROL"
else:
logger.error("FULL_CONTROL is given to All Users")

View file

@ -1,9 +1,33 @@
import time import time
import allure import allure
from frostfs_testlib.utils import datetime_utils from common import STORAGE_GC_TIME
from pytest_tests.resources.common import STORAGE_GC_TIME
def parse_time(value: str) -> int:
"""Converts time interval in text form into time interval as number of seconds.
Args:
value: time interval as text.
Returns:
Number of seconds in the parsed time interval.
"""
value = value.lower()
for suffix in ["s", "sec"]:
if value.endswith(suffix):
return int(value[: -len(suffix)])
for suffix in ["m", "min"]:
if value.endswith(suffix):
return int(value[: -len(suffix)]) * 60
for suffix in ["h", "hr", "hour"]:
if value.endswith(suffix):
return int(value[: -len(suffix)]) * 60 * 60
raise ValueError(f"Unknown units in time value '{value}'")
def placement_policy_from_container(container_info: str) -> str: def placement_policy_from_container(container_info: str) -> str:
@ -23,7 +47,7 @@ def placement_policy_from_container(container_info: str) -> str:
FILTER Country EQ Sweden AS LOC_SW FILTER Country EQ Sweden AS LOC_SW
Args: Args:
container_info: output from frostfs-cli container get command container_info: output from neofs-cli container get command
Returns: Returns:
placement policy as a string placement policy as a string
@ -33,6 +57,6 @@ def placement_policy_from_container(container_info: str) -> str:
def wait_for_gc_pass_on_storage_nodes() -> None: def wait_for_gc_pass_on_storage_nodes() -> None:
wait_time = datetime_utils.parse_time(STORAGE_GC_TIME) wait_time = parse_time(STORAGE_GC_TIME)
with allure.step(f"Wait {wait_time}s until GC completes on storage nodes"): with allure.step(f"Wait {wait_time}s until GC completes on storage nodes"):
time.sleep(wait_time) time.sleep(wait_time)

View file

@ -2,12 +2,11 @@ import os
import uuid import uuid
from dataclasses import dataclass from dataclasses import dataclass
from frostfs_testlib.shell import Shell from cluster import Cluster, NodeBase
from frostfs_testlib.utils import wallet_utils from common import FREE_STORAGE, WALLET_CONFIG, WALLET_PASS
from neofs_testlib.shell import Shell
from pytest_tests.helpers.cluster import Cluster, NodeBase from neofs_testlib.utils.wallet import get_last_address_from_wallet, init_wallet
from pytest_tests.helpers.payment_neogo import deposit_gas, transfer_gas from python_keywords.payment_neogo import deposit_gas, transfer_gas
from pytest_tests.resources.common import FREE_STORAGE, WALLET_CONFIG, WALLET_PASS
@dataclass @dataclass
@ -29,7 +28,7 @@ class WalletFile:
Returns: Returns:
The address of the wallet. The address of the wallet.
""" """
return wallet_utils.get_last_address_from_wallet(self.path, self.password) return get_last_address_from_wallet(self.path, self.password)
class WalletFactory: class WalletFactory:
@ -48,7 +47,7 @@ class WalletFactory:
WalletFile object of new wallet WalletFile object of new wallet
""" """
wallet_path = os.path.join(self.wallets_dir, f"{str(uuid.uuid4())}.json") wallet_path = os.path.join(self.wallets_dir, f"{str(uuid.uuid4())}.json")
wallet_utils.init_wallet(wallet_path, password) init_wallet(wallet_path, password)
if not FREE_STORAGE: if not FREE_STORAGE:
main_chain = self.cluster.main_chain_nodes[0] main_chain = self.cluster.main_chain_nodes[0]

View file

@ -13,7 +13,7 @@ markers =
# functional markers # functional markers
container: tests for container creation container: tests for container creation
grpc_api: standard gRPC API tests grpc_api: standard gRPC API tests
grpc_control: tests related to using frostfs-cli control commands grpc_control: tests related to using neofs-cli control commands
grpc_object_lock: gRPC lock tests grpc_object_lock: gRPC lock tests
http_gate: HTTP gate contract http_gate: HTTP gate contract
s3_gate: All S3 gate tests s3_gate: All S3 gate tests
@ -25,7 +25,7 @@ markers =
s3_gate_tagging: Tagging S3 gate tests s3_gate_tagging: Tagging S3 gate tests
s3_gate_versioning: Versioning S3 gate tests s3_gate_versioning: Versioning S3 gate tests
long: long tests (with long execution time) long: long tests (with long execution time)
node_mgmt: frostfs control commands node_mgmt: neofs control commands
session_token: tests for operations with session token session_token: tests for operations with session token
static_session: tests for operations with static session token static_session: tests for operations with static session token
bearer: tests for bearer tokens bearer: tests for bearer tokens
@ -40,6 +40,6 @@ markers =
failover_network: tests for network failure failover_network: tests for network failure
failover_reboot: tests for system recovery after reboot of a node failover_reboot: tests for system recovery after reboot of a node
add_nodes: add nodes to cluster add_nodes: add nodes to cluster
check_binaries: check frostfs installed binaries versions check_binaries: check neofs installed binaries versions
payments: tests for payment associated operations payments: tests for payment associated operations
load: performance tests load: performance tests

View file

@ -1,9 +1,8 @@
import allure import allure
import epoch
import pytest import pytest
from frostfs_testlib.shell import Shell from cluster import Cluster
from neofs_testlib.shell import Shell
from pytest_tests.helpers import epoch
from pytest_tests.helpers.cluster import Cluster
# To skip adding every mandatory singleton dependency to EACH test function # To skip adding every mandatory singleton dependency to EACH test function

View file

@ -3,16 +3,15 @@ import re
from dataclasses import asdict from dataclasses import asdict
import allure import allure
from frostfs_testlib.cli.frostfs_authmate import FrostfsAuthmate from common import STORAGE_NODE_SERVICE_NAME_REGEX
from frostfs_testlib.cli.neogo import NeoGo from k6 import K6, LoadParams, LoadResults
from frostfs_testlib.hosting import Hosting from neofs_testlib.cli.neofs_authmate import NeofsAuthmate
from frostfs_testlib.shell import CommandOptions, SSHShell from neofs_testlib.cli.neogo import NeoGo
from frostfs_testlib.shell.interfaces import InteractiveInput from neofs_testlib.hosting import Hosting
from neofs_testlib.shell import CommandOptions, SSHShell
from neofs_testlib.shell.interfaces import InteractiveInput
from pytest_tests.helpers.k6 import K6, LoadParams, LoadResults NEOFS_AUTHMATE_PATH = "neofs-s3-authmate"
from pytest_tests.resources.common import STORAGE_NODE_SERVICE_NAME_REGEX
FROSTFS_AUTHMATE_PATH = "frostfs-authmate"
STOPPED_HOSTS = [] STOPPED_HOSTS = []
@ -57,8 +56,8 @@ def init_s3_client(
path = ssh_client.exec(r"sudo find . -name 'k6' -exec dirname {} \; -quit").stdout.strip( path = ssh_client.exec(r"sudo find . -name 'k6' -exec dirname {} \; -quit").stdout.strip(
"\n" "\n"
) )
frostfs_authmate_exec = FrostfsAuthmate(ssh_client, FROSTFS_AUTHMATE_PATH) neofs_authmate_exec = NeofsAuthmate(ssh_client, NEOFS_AUTHMATE_PATH)
issue_secret_output = frostfs_authmate_exec.secret.issue( issue_secret_output = neofs_authmate_exec.secret.issue(
wallet=f"{path}/scenarios/files/wallet.json", wallet=f"{path}/scenarios/files/wallet.json",
peer=node_endpoint, peer=node_endpoint,
bearer_rules=f"{path}/scenarios/files/rules.json", bearer_rules=f"{path}/scenarios/files/rules.json",

View file

@ -8,20 +8,19 @@ from typing import Any, Optional
import allure import allure
import boto3 import boto3
import pytest import pytest
import s3_gate_bucket
import s3_gate_object
import urllib3 import urllib3
from aws_cli_client import AwsCliClient
from botocore.config import Config from botocore.config import Config
from botocore.exceptions import ClientError from botocore.exceptions import ClientError
from frostfs_testlib.shell import Shell from cli_helpers import _cmd_run, _configure_aws_cli, _run_with_passwd
from cluster import Cluster
from cluster_test_base import ClusterTestBase
from common import NEOFS_AUTHMATE_EXEC
from neofs_testlib.shell import Shell
from pytest import FixtureRequest from pytest import FixtureRequest
from python_keywords.container import list_containers
from pytest_tests.steps import s3_gate_bucket
from pytest_tests.steps import s3_gate_object
from pytest_tests.helpers.aws_cli_client import AwsCliClient
from pytest_tests.helpers.cli_helpers import _cmd_run, _configure_aws_cli, _run_with_passwd
from pytest_tests.helpers.cluster import Cluster
from pytest_tests.helpers.container import list_containers
from pytest_tests.resources.common import FROSTFS_AUTHMATE_EXEC
from pytest_tests.steps.cluster_test_base import ClusterTestBase
# Disable warnings on self-signed certificate which the # Disable warnings on self-signed certificate which the
# boto library produces on requests to S3-gate in dev-env # boto library produces on requests to S3-gate in dev-env
@ -45,7 +44,7 @@ class TestS3GateBase(ClusterTestBase):
self, default_wallet, client_shell: Shell, request: FixtureRequest, cluster: Cluster self, default_wallet, client_shell: Shell, request: FixtureRequest, cluster: Cluster
) -> Any: ) -> Any:
wallet = default_wallet wallet = default_wallet
s3_bearer_rules_file = f"{os.getcwd()}/pytest_tests/resources/files/s3_bearer_rules.json" s3_bearer_rules_file = f"{os.getcwd()}/robot/resources/files/s3_bearer_rules.json"
policy = None if isinstance(request.param, str) else request.param[1] policy = None if isinstance(request.param, str) else request.param[1]
(cid, bucket, access_key_id, secret_access_key, owner_private_key,) = init_s3_credentials( (cid, bucket, access_key_id, secret_access_key, owner_private_key,) = init_s3_credentials(
wallet, cluster, s3_bearer_rules_file=s3_bearer_rules_file, policy=policy wallet, cluster, s3_bearer_rules_file=s3_bearer_rules_file, policy=policy
@ -124,12 +123,12 @@ def init_s3_credentials(
policy: Optional[dict] = None, policy: Optional[dict] = None,
): ):
bucket = str(uuid.uuid4()) bucket = str(uuid.uuid4())
s3_bearer_rules = s3_bearer_rules_file or "pytest_tests/resources/files/s3_bearer_rules.json" s3_bearer_rules = s3_bearer_rules_file or "robot/resources/files/s3_bearer_rules.json"
s3gate_node = cluster.s3gates[0] s3gate_node = cluster.s3gates[0]
gate_public_key = s3gate_node.get_wallet_public_key() gate_public_key = s3gate_node.get_wallet_public_key()
cmd = ( cmd = (
f"{FROSTFS_AUTHMATE_EXEC} --debug --with-log --timeout {CREDENTIALS_CREATE_TIMEOUT} " f"{NEOFS_AUTHMATE_EXEC} --debug --with-log --timeout {CREDENTIALS_CREATE_TIMEOUT} "
f"issue-secret --wallet {wallet_path} --gate-public-key={gate_public_key} " f"issue-secret --wallet {wallet_path} --gate-public-key={gate_public_key} "
f"--peer {cluster.default_rpc_endpoint} --container-friendly-name {bucket} " f"--peer {cluster.default_rpc_endpoint} --container-friendly-name {bucket} "
f"--bearer-rules {s3_bearer_rules}" f"--bearer-rules {s3_bearer_rules}"

View file

@ -7,8 +7,7 @@ from typing import Optional
import allure import allure
from botocore.exceptions import ClientError from botocore.exceptions import ClientError
from cli_helpers import log_command_execution
from pytest_tests.helpers.cli_helpers import log_command_execution
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")

View file

@ -7,11 +7,10 @@ from typing import Optional
import allure import allure
import pytest import pytest
import urllib3 import urllib3
from aws_cli_client import AwsCliClient
from botocore.exceptions import ClientError from botocore.exceptions import ClientError
from cli_helpers import log_command_execution
from pytest_tests.helpers.aws_cli_client import AwsCliClient from s3_gate_bucket import S3_SYNC_WAIT_TIME
from pytest_tests.helpers.cli_helpers import log_command_execution
from pytest_tests.steps.s3_gate_bucket import S3_SYNC_WAIT_TIME
########################################################## ##########################################################
# Disabling warnings on self-signed certificate which the # Disabling warnings on self-signed certificate which the

View file

@ -8,13 +8,14 @@ from enum import Enum
from typing import Any, Optional from typing import Any, Optional
import allure import allure
from frostfs_testlib.cli import FrostfsCli import json_transformers
from frostfs_testlib.shell import Shell from common import ASSETS_DIR, NEOFS_CLI_EXEC, WALLET_CONFIG
from frostfs_testlib.utils import json_utils, wallet_utils from data_formatters import get_wallet_public_key
from json_transformers import encode_for_json
from pytest_tests.helpers.storage_object_info import StorageObjectInfo from neofs_testlib.cli import NeofsCli
from pytest_tests.helpers.wallet import WalletFile from neofs_testlib.shell import Shell
from pytest_tests.resources.common import ASSETS_DIR, FROSTFS_CLI_EXEC, WALLET_CONFIG from storage_object_info import StorageObjectInfo
from wallet import WalletFile
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
@ -70,16 +71,16 @@ def generate_session_token(
file_path = os.path.join(tokens_dir, str(uuid.uuid4())) file_path = os.path.join(tokens_dir, str(uuid.uuid4()))
pub_key_64 = wallet_utils.get_wallet_public_key( pub_key_64 = get_wallet_public_key(session_wallet.path, session_wallet.password, "base64")
session_wallet.path, session_wallet.password, "base64"
)
lifetime = lifetime or Lifetime() lifetime = lifetime or Lifetime()
session_token = { session_token = {
"body": { "body": {
"id": f"{base64.b64encode(uuid.uuid4().bytes).decode('utf-8')}", "id": f"{base64.b64encode(uuid.uuid4().bytes).decode('utf-8')}",
"ownerID": {"value": f"{json_utils.encode_for_json(owner_wallet.get_address())}"}, "ownerID": {
"value": f"{json_transformers.encode_for_json(owner_wallet.get_address())}"
},
"lifetime": { "lifetime": {
"exp": f"{lifetime.exp}", "exp": f"{lifetime.exp}",
"nbf": f"{lifetime.nbf}", "nbf": f"{lifetime.nbf}",
@ -124,11 +125,7 @@ def generate_container_session_token(
"container": { "container": {
"verb": verb.value, "verb": verb.value,
"wildcard": cid is None, "wildcard": cid is None,
**( **({"containerID": {"value": f"{encode_for_json(cid)}"}} if cid is not None else {}),
{"containerID": {"value": f"{json_utils.encode_for_json(cid)}"}}
if cid is not None
else {}
),
}, },
} }
@ -168,8 +165,8 @@ def generate_object_session_token(
"object": { "object": {
"verb": verb.value, "verb": verb.value,
"target": { "target": {
"container": {"value": json_utils.encode_for_json(cid)}, "container": {"value": encode_for_json(cid)},
"objects": [{"value": json_utils.encode_for_json(oid)} for oid in oids], "objects": [{"value": encode_for_json(oid)} for oid in oids],
}, },
}, },
} }
@ -252,8 +249,8 @@ def create_session_token(
The path to the generated session token file. The path to the generated session token file.
""" """
session_token = os.path.join(os.getcwd(), ASSETS_DIR, str(uuid.uuid4())) session_token = os.path.join(os.getcwd(), ASSETS_DIR, str(uuid.uuid4()))
frostfscli = FrostfsCli(shell=shell, frostfs_cli_exec_path=FROSTFS_CLI_EXEC) neofscli = NeofsCli(shell=shell, neofs_cli_exec_path=NEOFS_CLI_EXEC)
frostfscli.session.create( neofscli.session.create(
rpc_endpoint=rpc_endpoint, rpc_endpoint=rpc_endpoint,
address=owner, address=owner,
wallet=wallet_path, wallet=wallet_path,
@ -277,10 +274,8 @@ def sign_session_token(shell: Shell, session_token_file: str, wlt: WalletFile) -
The path to the signed token. The path to the signed token.
""" """
signed_token_file = os.path.join(os.getcwd(), ASSETS_DIR, str(uuid.uuid4())) signed_token_file = os.path.join(os.getcwd(), ASSETS_DIR, str(uuid.uuid4()))
frostfscli = FrostfsCli( neofscli = NeofsCli(shell=shell, neofs_cli_exec_path=NEOFS_CLI_EXEC, config_file=WALLET_CONFIG)
shell=shell, frostfs_cli_exec_path=FROSTFS_CLI_EXEC, config_file=WALLET_CONFIG neofscli.util.sign_session_token(
)
frostfscli.util.sign_session_token(
wallet=wlt.path, from_file=session_token_file, to_file=signed_token_file wallet=wlt.path, from_file=session_token_file, to_file=signed_token_file
) )
return signed_token_file return signed_token_file

View file

@ -3,14 +3,13 @@ from time import sleep
import allure import allure
import pytest import pytest
from frostfs_testlib.resources.common import OBJECT_ALREADY_REMOVED from cluster import Cluster
from frostfs_testlib.shell import Shell from epoch import tick_epoch
from grpc_responses import OBJECT_ALREADY_REMOVED
from pytest_tests.helpers.cluster import Cluster from neofs_testlib.shell import Shell
from pytest_tests.helpers.epoch import tick_epoch from python_keywords.neofs_verbs import delete_object, get_object
from pytest_tests.helpers.frostfs_verbs import delete_object, get_object from storage_object_info import StorageObjectInfo
from pytest_tests.helpers.storage_object_info import StorageObjectInfo from tombstone import verify_head_tombstone
from pytest_tests.helpers.tombstone import verify_head_tombstone
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")

View file

@ -5,16 +5,15 @@ from typing import Optional
import allure import allure
import pytest import pytest
from frostfs_testlib.resources.common import PUBLIC_ACL from cluster import Cluster
from frostfs_testlib.shell import Shell from common import WALLET_CONFIG, WALLET_PASS
from frostfs_testlib.utils import wallet_utils from file_helper import generate_file
from neofs_testlib.shell import Shell
from pytest_tests.helpers.acl import EACLRole from neofs_testlib.utils.wallet import init_wallet
from pytest_tests.helpers.cluster import Cluster from python_keywords.acl import EACLRole
from pytest_tests.helpers.container import create_container from python_keywords.container import create_container
from pytest_tests.helpers.file_helper import generate_file from python_keywords.neofs_verbs import put_object_to_random_node
from pytest_tests.helpers.frostfs_verbs import put_object_to_random_node from wellknown_acl import PUBLIC_ACL
from pytest_tests.resources.common import WALLET_CONFIG, WALLET_PASS
OBJECT_COUNT = 5 OBJECT_COUNT = 5
@ -42,7 +41,7 @@ def wallets(default_wallet, temp_directory, cluster: Cluster) -> Wallets:
os.path.join(temp_directory, f"{str(uuid.uuid4())}.json") for _ in range(2) os.path.join(temp_directory, f"{str(uuid.uuid4())}.json") for _ in range(2)
] ]
for other_wallet_path in other_wallets_paths: for other_wallet_path in other_wallets_paths:
wallet_utils.init_wallet(other_wallet_path, WALLET_PASS) init_wallet(other_wallet_path, WALLET_PASS)
ir_node = cluster.ir_nodes[0] ir_node = cluster.ir_nodes[0]
storage_node = cluster.storage_nodes[0] storage_node = cluster.storage_nodes[0]

View file

@ -5,10 +5,12 @@ from typing import Optional
import allure import allure
import pytest import pytest
from frostfs_testlib.resources.common import OBJECT_ACCESS_DENIED, OBJECT_NOT_FOUND from cluster_test_base import ClusterTestBase
from frostfs_testlib.utils import wallet_utils from common import ASSETS_DIR, FREE_STORAGE, WALLET_PASS
from file_helper import generate_file
from pytest_tests.helpers.acl import ( from grpc_responses import OBJECT_ACCESS_DENIED, OBJECT_NOT_FOUND
from neofs_testlib.utils.wallet import init_wallet
from python_keywords.acl import (
EACLAccess, EACLAccess,
EACLOperation, EACLOperation,
EACLRole, EACLRole,
@ -17,11 +19,10 @@ from pytest_tests.helpers.acl import (
form_bearertoken_file, form_bearertoken_file,
set_eacl, set_eacl,
) )
from pytest_tests.helpers.container import create_container from python_keywords.container import create_container
from pytest_tests.helpers.file_helper import generate_file from python_keywords.neofs_verbs import put_object_to_random_node
from pytest_tests.helpers.frostfs_verbs import put_object_to_random_node from python_keywords.payment_neogo import deposit_gas, transfer_gas
from pytest_tests.helpers.payment_neogo import deposit_gas, transfer_gas from python_keywords.storage_group import (
from pytest_tests.helpers.storage_group import (
delete_storagegroup, delete_storagegroup,
get_storagegroup, get_storagegroup,
list_storagegroup, list_storagegroup,
@ -29,8 +30,6 @@ from pytest_tests.helpers.storage_group import (
verify_get_storage_group, verify_get_storage_group,
verify_list_storage_group, verify_list_storage_group,
) )
from pytest_tests.resources.common import ASSETS_DIR, FREE_STORAGE, WALLET_PASS
from pytest_tests.steps.cluster_test_base import ClusterTestBase
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
deposit = 30 deposit = 30
@ -49,7 +48,7 @@ class TestStorageGroup(ClusterTestBase):
def prepare_two_wallets(self, default_wallet): def prepare_two_wallets(self, default_wallet):
self.main_wallet = default_wallet self.main_wallet = default_wallet
self.other_wallet = os.path.join(os.getcwd(), ASSETS_DIR, f"{str(uuid.uuid4())}.json") self.other_wallet = os.path.join(os.getcwd(), ASSETS_DIR, f"{str(uuid.uuid4())}.json")
wallet_utils.init_wallet(self.other_wallet, WALLET_PASS) init_wallet(self.other_wallet, WALLET_PASS)
if not FREE_STORAGE: if not FREE_STORAGE:
main_chain = self.cluster.main_chain_nodes[0] main_chain = self.cluster.main_chain_nodes[0]
deposit = 30 deposit = 30

View file

@ -1,16 +1,15 @@
import allure import allure
import pytest import pytest
from frostfs_testlib.resources.common import PRIVATE_ACL_F, PUBLIC_ACL_F, READONLY_ACL_F from cluster_test_base import ClusterTestBase
from python_keywords.acl import EACLRole
from pytest_tests.helpers.acl import EACLRole from python_keywords.container import create_container
from pytest_tests.helpers.container import create_container from python_keywords.container_access import (
from pytest_tests.helpers.container_access import (
check_full_access_to_container, check_full_access_to_container,
check_no_access_to_container, check_no_access_to_container,
check_read_only_container, check_read_only_container,
) )
from pytest_tests.helpers.frostfs_verbs import put_object_to_random_node from python_keywords.neofs_verbs import put_object_to_random_node
from pytest_tests.steps.cluster_test_base import ClusterTestBase from wellknown_acl import PRIVATE_ACL_F, PUBLIC_ACL_F, READONLY_ACL_F
@pytest.mark.sanity @pytest.mark.sanity

View file

@ -1,7 +1,7 @@
import allure import allure
import pytest import pytest
from cluster_test_base import ClusterTestBase
from pytest_tests.helpers.acl import ( from python_keywords.acl import (
EACLAccess, EACLAccess,
EACLOperation, EACLOperation,
EACLRole, EACLRole,
@ -11,12 +11,11 @@ from pytest_tests.helpers.acl import (
set_eacl, set_eacl,
wait_for_cache_expired, wait_for_cache_expired,
) )
from pytest_tests.helpers.container_access import ( from python_keywords.container_access import (
check_custom_access_to_container, check_custom_access_to_container,
check_full_access_to_container, check_full_access_to_container,
check_no_access_to_container, check_no_access_to_container,
) )
from pytest_tests.steps.cluster_test_base import ClusterTestBase
@pytest.mark.sanity @pytest.mark.sanity
@ -25,9 +24,7 @@ from pytest_tests.steps.cluster_test_base import ClusterTestBase
class TestACLBearer(ClusterTestBase): class TestACLBearer(ClusterTestBase):
@pytest.mark.parametrize("role", [EACLRole.USER, EACLRole.OTHERS]) @pytest.mark.parametrize("role", [EACLRole.USER, EACLRole.OTHERS])
def test_bearer_token_operations(self, wallets, eacl_container_with_objects, role): def test_bearer_token_operations(self, wallets, eacl_container_with_objects, role):
allure.dynamic.title( allure.dynamic.title(f"Testcase to validate NeoFS operations with {role.value} BearerToken")
f"Testcase to validate FrostFS operations with {role.value} BearerToken"
)
cid, objects_oids, file_path = eacl_container_with_objects cid, objects_oids, file_path = eacl_container_with_objects
user_wallet = wallets.get_wallet() user_wallet = wallets.get_wallet()
deny_wallet = wallets.get_wallet(role) deny_wallet = wallets.get_wallet(role)

View file

@ -1,8 +1,9 @@
import allure import allure
import pytest import pytest
from frostfs_testlib.resources.common import PUBLIC_ACL from cluster_test_base import ClusterTestBase
from failover_utils import wait_object_replication
from pytest_tests.helpers.acl import ( from neofs_testlib.shell import Shell
from python_keywords.acl import (
EACLAccess, EACLAccess,
EACLOperation, EACLOperation,
EACLRole, EACLRole,
@ -11,15 +12,14 @@ from pytest_tests.helpers.acl import (
set_eacl, set_eacl,
wait_for_cache_expired, wait_for_cache_expired,
) )
from pytest_tests.helpers.container import create_container from python_keywords.container import create_container
from pytest_tests.helpers.container_access import ( from python_keywords.container_access import (
check_full_access_to_container, check_full_access_to_container,
check_no_access_to_container, check_no_access_to_container,
) )
from pytest_tests.helpers.failover_utils import wait_object_replication from python_keywords.neofs_verbs import put_object_to_random_node
from pytest_tests.helpers.frostfs_verbs import put_object_to_random_node from python_keywords.node_management import drop_object
from pytest_tests.helpers.node_management import drop_object from python_keywords.object_access import (
from pytest_tests.helpers.object_access import (
can_delete_object, can_delete_object,
can_get_head_object, can_get_head_object,
can_get_object, can_get_object,
@ -28,7 +28,7 @@ from pytest_tests.helpers.object_access import (
can_put_object, can_put_object,
can_search_object, can_search_object,
) )
from pytest_tests.steps.cluster_test_base import ClusterTestBase from wellknown_acl import PUBLIC_ACL
@pytest.mark.sanity @pytest.mark.sanity
@ -74,7 +74,7 @@ class TestEACLContainer(ClusterTestBase):
not_deny_role_wallet = user_wallet if deny_role == EACLRole.OTHERS else other_wallet not_deny_role_wallet = user_wallet if deny_role == EACLRole.OTHERS else other_wallet
deny_role_str = "all others" if deny_role == EACLRole.OTHERS else "user" deny_role_str = "all others" if deny_role == EACLRole.OTHERS else "user"
not_deny_role_str = "user" if deny_role == EACLRole.OTHERS else "all others" not_deny_role_str = "user" if deny_role == EACLRole.OTHERS else "all others"
allure.dynamic.title(f"Testcase to deny FrostFS operations for {deny_role_str}.") allure.dynamic.title(f"Testcase to deny NeoFS operations for {deny_role_str}.")
cid, object_oids, file_path = eacl_container_with_objects cid, object_oids, file_path = eacl_container_with_objects
with allure.step(f"Deny all operations for {deny_role_str} via eACL"): with allure.step(f"Deny all operations for {deny_role_str} via eACL"):
@ -148,7 +148,7 @@ class TestEACLContainer(ClusterTestBase):
cluster=self.cluster, cluster=self.cluster,
) )
@allure.title("Testcase to allow FrostFS operations for only one other pubkey.") @allure.title("Testcase to allow NeoFS operations for only one other pubkey.")
def test_extended_acl_deny_all_operations_exclude_pubkey( def test_extended_acl_deny_all_operations_exclude_pubkey(
self, wallets, eacl_container_with_objects self, wallets, eacl_container_with_objects
): ):
@ -209,7 +209,7 @@ class TestEACLContainer(ClusterTestBase):
cluster=self.cluster, cluster=self.cluster,
) )
@allure.title("Testcase to validate FrostFS replication with eACL deny rules.") @allure.title("Testcase to validate NeoFS replication with eACL deny rules.")
def test_extended_acl_deny_replication( def test_extended_acl_deny_replication(
self, self,
wallets, wallets,
@ -251,7 +251,7 @@ class TestEACLContainer(ClusterTestBase):
storage_nodes, storage_nodes,
) )
@allure.title("Testcase to validate FrostFS system operations with extended ACL") @allure.title("Testcase to validate NeoFS system operations with extended ACL")
def test_extended_actions_system(self, wallets, eacl_container_with_objects): def test_extended_actions_system(self, wallets, eacl_container_with_objects):
user_wallet = wallets.get_wallet() user_wallet = wallets.get_wallet()
ir_wallet, storage_wallet = wallets.get_wallets_list(role=EACLRole.SYSTEM)[:2] ir_wallet, storage_wallet = wallets.get_wallets_list(role=EACLRole.SYSTEM)[:2]

View file

@ -1,8 +1,7 @@
import allure import allure
import pytest import pytest
from frostfs_testlib.resources.common import PUBLIC_ACL from cluster_test_base import ClusterTestBase
from python_keywords.acl import (
from pytest_tests.helpers.acl import (
EACLAccess, EACLAccess,
EACLFilter, EACLFilter,
EACLFilters, EACLFilters,
@ -16,14 +15,14 @@ from pytest_tests.helpers.acl import (
set_eacl, set_eacl,
wait_for_cache_expired, wait_for_cache_expired,
) )
from pytest_tests.helpers.container import create_container, delete_container from python_keywords.container import create_container, delete_container
from pytest_tests.helpers.container_access import ( from python_keywords.container_access import (
check_full_access_to_container, check_full_access_to_container,
check_no_access_to_container, check_no_access_to_container,
) )
from pytest_tests.helpers.frostfs_verbs import put_object_to_random_node from python_keywords.neofs_verbs import put_object_to_random_node
from pytest_tests.helpers.object_access import can_get_head_object, can_get_object, can_put_object from python_keywords.object_access import can_get_head_object, can_get_object, can_put_object
from pytest_tests.steps.cluster_test_base import ClusterTestBase from wellknown_acl import PUBLIC_ACL
@pytest.mark.sanity @pytest.mark.sanity
@ -129,7 +128,7 @@ class TestEACLFilters(ClusterTestBase):
"match_type", [EACLMatchType.STRING_EQUAL, EACLMatchType.STRING_NOT_EQUAL] "match_type", [EACLMatchType.STRING_EQUAL, EACLMatchType.STRING_NOT_EQUAL]
) )
def test_extended_acl_filters_request(self, wallets, eacl_container_with_objects, match_type): def test_extended_acl_filters_request(self, wallets, eacl_container_with_objects, match_type):
allure.dynamic.title(f"Validate FrostFS operations with request filter: {match_type.name}") allure.dynamic.title(f"Validate NeoFS operations with request filter: {match_type.name}")
user_wallet = wallets.get_wallet() user_wallet = wallets.get_wallet()
other_wallet = wallets.get_wallet(EACLRole.OTHERS) other_wallet = wallets.get_wallet(EACLRole.OTHERS)
( (
@ -244,7 +243,7 @@ class TestEACLFilters(ClusterTestBase):
self, wallets, eacl_container_with_objects, match_type self, wallets, eacl_container_with_objects, match_type
): ):
allure.dynamic.title( allure.dynamic.title(
f"Validate FrostFS operations with deny user headers filter: {match_type.name}" f"Validate NeoFS operations with deny user headers filter: {match_type.name}"
) )
user_wallet = wallets.get_wallet() user_wallet = wallets.get_wallet()
other_wallet = wallets.get_wallet(EACLRole.OTHERS) other_wallet = wallets.get_wallet(EACLRole.OTHERS)
@ -426,7 +425,7 @@ class TestEACLFilters(ClusterTestBase):
self, wallets, eacl_container_with_objects, match_type self, wallets, eacl_container_with_objects, match_type
): ):
allure.dynamic.title( allure.dynamic.title(
"Testcase to validate FrostFS operation with allow eACL user headers filters:" "Testcase to validate NeoFS operation with allow eACL user headers filters:"
f"{match_type.name}" f"{match_type.name}"
) )
user_wallet = wallets.get_wallet() user_wallet = wallets.get_wallet()

View file

@ -1,5 +1,6 @@
import logging import logging
import os import os
import re
import shutil import shutil
import uuid import uuid
from datetime import datetime from datetime import datetime
@ -7,19 +8,9 @@ from datetime import datetime
import allure import allure
import pytest import pytest
import yaml import yaml
from frostfs_testlib.hosting import Hosting from binary_version_helper import get_local_binaries_versions, get_remote_binaries_versions
from frostfs_testlib.reporter import AllureHandler, get_reporter from cluster import Cluster
from frostfs_testlib.shell import LocalShell, Shell from common import (
from frostfs_testlib.utils import wallet_utils
from pytest_tests.helpers import binary_version, env_properties
from pytest_tests.helpers.cluster import Cluster
from pytest_tests.helpers.frostfs_verbs import get_netmap_netinfo
from pytest_tests.helpers.k6 import LoadParams
from pytest_tests.helpers.node_management import storage_node_healthcheck
from pytest_tests.helpers.payment_neogo import deposit_gas, transfer_gas
from pytest_tests.helpers.wallet import WalletFactory
from pytest_tests.resources.common import (
ASSETS_DIR, ASSETS_DIR,
COMPLEX_OBJECT_CHUNKS_COUNT, COMPLEX_OBJECT_CHUNKS_COUNT,
COMPLEX_OBJECT_TAIL_SIZE, COMPLEX_OBJECT_TAIL_SIZE,
@ -29,7 +20,10 @@ from pytest_tests.resources.common import (
STORAGE_NODE_SERVICE_NAME_REGEX, STORAGE_NODE_SERVICE_NAME_REGEX,
WALLET_PASS, WALLET_PASS,
) )
from pytest_tests.resources.load_params import ( from env_properties import save_env_properties
from k6 import LoadParams
from load import get_services_endpoints, prepare_k6_instances
from load_params import (
BACKGROUND_LOAD_MAX_TIME, BACKGROUND_LOAD_MAX_TIME,
BACKGROUND_OBJ_SIZE, BACKGROUND_OBJ_SIZE,
BACKGROUND_READERS_COUNT, BACKGROUND_READERS_COUNT,
@ -38,25 +32,24 @@ from pytest_tests.resources.load_params import (
LOAD_NODE_SSH_USER, LOAD_NODE_SSH_USER,
LOAD_NODES, LOAD_NODES,
) )
from pytest_tests.steps.load import get_services_endpoints, prepare_k6_instances from neofs_testlib.hosting import Hosting
from neofs_testlib.reporter import AllureHandler, get_reporter
from neofs_testlib.shell import LocalShell, Shell
from neofs_testlib.utils.wallet import init_wallet
from payment_neogo import deposit_gas, transfer_gas
from python_keywords.neofs_verbs import get_netmap_netinfo
from python_keywords.node_management import storage_node_healthcheck
from helpers.wallet import WalletFactory
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
# Add logs check test even if it's not fit to mark selectors
def pytest_configure(config: pytest.Config):
markers = config.option.markexpr
if markers != "":
config.option.markexpr = f"logs_after_session or ({markers})"
# pytest hook. Do not rename
def pytest_collection_modifyitems(items): def pytest_collection_modifyitems(items):
# Make network tests last based on @pytest.mark.node_mgmt and logs_test to be latest # Make network tests last based on @pytest.mark.node_mgmt
def priority(item: pytest.Item) -> int: def priority(item: pytest.Item) -> int:
is_node_mgmt_test = 1 if item.get_closest_marker("node_mgmt") else 0 is_node_mgmt_test = item.get_closest_marker("node_mgmt")
is_logs_check_test = 100 if item.get_closest_marker("logs_after_session") else 0 return 0 if not is_node_mgmt_test else 1
return is_node_mgmt_test + is_logs_check_test
items.sort(key=lambda item: priority(item)) items.sort(key=lambda item: priority(item))
@ -132,11 +125,11 @@ def cluster(temp_directory: str, hosting: Hosting) -> Cluster:
@pytest.fixture(scope="session", autouse=True) @pytest.fixture(scope="session", autouse=True)
@allure.title("Check binary versions") @allure.title("Check binary versions")
def check_binary_versions(request, hosting: Hosting, client_shell: Shell): def check_binary_versions(request, hosting: Hosting, client_shell: Shell):
local_versions = binary_version.get_local_binaries_versions(client_shell) local_versions = get_local_binaries_versions(client_shell)
remote_versions = binary_version.get_remote_binaries_versions(hosting) remote_versions = get_remote_binaries_versions(hosting)
all_versions = {**local_versions, **remote_versions} all_versions = {**local_versions, **remote_versions}
env_properties.save_env_properties(request.config, all_versions) save_env_properties(request.config, all_versions)
@pytest.fixture(scope="session") @pytest.fixture(scope="session")
@ -153,16 +146,23 @@ def temp_directory():
shutil.rmtree(full_path) shutil.rmtree(full_path)
@allure.step("[Autouse/Session] Test session start time")
@pytest.fixture(scope="session", autouse=True) @pytest.fixture(scope="session", autouse=True)
def session_start_time(): @allure.title("Collect logs")
def collect_logs(temp_directory, hosting: Hosting):
start_time = datetime.utcnow() start_time = datetime.utcnow()
return start_time yield
end_time = datetime.utcnow()
# Dump logs to temp directory (because they might be too large to keep in RAM)
logs_dir = os.path.join(temp_directory, "logs")
dump_logs(hosting, logs_dir, start_time, end_time)
attach_logs(logs_dir)
check_logs(logs_dir)
@pytest.fixture(scope="session", autouse=True) @pytest.fixture(scope="session", autouse=True)
@allure.title("Run health check for all storage nodes") @allure.title("Run health check for all storage nodes")
def run_health_check(session_start_time, cluster: Cluster): def run_health_check(collect_logs, cluster: Cluster):
failed_nodes = [] failed_nodes = []
for node in cluster.storage_nodes: for node in cluster.storage_nodes:
health_check = storage_node_healthcheck(node) health_check = storage_node_healthcheck(node)
@ -242,7 +242,7 @@ def background_grpc_load(client_shell: Shell, hosting: Hosting):
@allure.title("Prepare wallet and deposit") @allure.title("Prepare wallet and deposit")
def default_wallet(client_shell: Shell, temp_directory: str, cluster: Cluster): def default_wallet(client_shell: Shell, temp_directory: str, cluster: Cluster):
wallet_path = os.path.join(os.getcwd(), ASSETS_DIR, f"{str(uuid.uuid4())}.json") wallet_path = os.path.join(os.getcwd(), ASSETS_DIR, f"{str(uuid.uuid4())}.json")
wallet_utils.init_wallet(wallet_path, WALLET_PASS) init_wallet(wallet_path, WALLET_PASS)
allure.attach.file(wallet_path, os.path.basename(wallet_path), allure.attachment_type.JSON) allure.attach.file(wallet_path, os.path.basename(wallet_path), allure.attachment_type.JSON)
if not FREE_STORAGE: if not FREE_STORAGE:
@ -264,3 +264,44 @@ def default_wallet(client_shell: Shell, temp_directory: str, cluster: Cluster):
) )
return wallet_path return wallet_path
@allure.title("Check logs for OOM and PANIC entries in {logs_dir}")
def check_logs(logs_dir: str):
problem_pattern = r"\Wpanic\W|\Woom\W|\Wtoo many open files\W"
log_file_paths = []
for directory_path, _, file_names in os.walk(logs_dir):
log_file_paths += [
os.path.join(directory_path, file_name)
for file_name in file_names
if re.match(r"\.(txt|log)", os.path.splitext(file_name)[-1], flags=re.IGNORECASE)
]
logs_with_problem = []
for file_path in log_file_paths:
with allure.step(f"Check log file {file_path}"):
with open(file_path, "r") as log_file:
if re.search(problem_pattern, log_file.read(), flags=re.IGNORECASE):
logs_with_problem.append(file_path)
if logs_with_problem:
raise pytest.fail(f"System logs {', '.join(logs_with_problem)} contain critical errors")
def dump_logs(hosting: Hosting, logs_dir: str, since: datetime, until: datetime) -> None:
# Dump logs to temp directory (because they might be too large to keep in RAM)
os.makedirs(logs_dir)
for host in hosting.hosts:
with allure.step(f"Dump logs from host {host.config.address}"):
try:
host.dump_logs(logs_dir, since=since, until=until)
except Exception as ex:
logger.warning(f"Exception during logs collection: {ex}")
def attach_logs(logs_dir: str) -> None:
# Zip all files and attach to Allure because it is more convenient to download a single
# zip with all logs rather than mess with individual logs files per service or node
logs_zip_file_path = shutil.make_archive(logs_dir, "zip", logs_dir)
allure.attach.file(logs_zip_file_path, name="logs.zip", extension="zip")

View file

@ -2,9 +2,8 @@ import json
import allure import allure
import pytest import pytest
from frostfs_testlib.resources.common import PRIVATE_ACL_F from epoch import tick_epoch
from python_keywords.container import (
from pytest_tests.helpers.container import (
create_container, create_container,
delete_container, delete_container,
get_container, get_container,
@ -12,8 +11,10 @@ from pytest_tests.helpers.container import (
wait_for_container_creation, wait_for_container_creation,
wait_for_container_deletion, wait_for_container_deletion,
) )
from pytest_tests.helpers.utility import placement_policy_from_container from utility import placement_policy_from_container
from pytest_tests.steps.cluster_test_base import ClusterTestBase from wellknown_acl import PRIVATE_ACL_F
from steps.cluster_test_base import ClusterTestBase
@pytest.mark.container @pytest.mark.container

View file

@ -4,18 +4,15 @@ from time import sleep
import allure import allure
import pytest import pytest
from frostfs_testlib.resources.common import PUBLIC_ACL from cluster import StorageNode
from failover_utils import wait_all_storage_nodes_returned, wait_object_replication
from file_helper import generate_file, get_file_hash
from iptables_helper import IpTablesHelper
from python_keywords.container import create_container
from python_keywords.neofs_verbs import get_object, put_object_to_random_node
from wellknown_acl import PUBLIC_ACL
from pytest_tests.helpers.cluster import StorageNode from steps.cluster_test_base import ClusterTestBase
from pytest_tests.helpers.container import create_container
from pytest_tests.helpers.failover_utils import (
wait_all_storage_nodes_returned,
wait_object_replication,
)
from pytest_tests.helpers.file_helper import generate_file, get_file_hash
from pytest_tests.helpers.frostfs_verbs import get_object, put_object_to_random_node
from pytest_tests.helpers.iptables_helper import IpTablesHelper
from pytest_tests.steps.cluster_test_base import ClusterTestBase
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
STORAGE_NODE_COMMUNICATION_PORT = "8080" STORAGE_NODE_COMMUNICATION_PORT = "8080"

View file

@ -2,26 +2,23 @@ import logging
import allure import allure
import pytest import pytest
from frostfs_testlib.hosting import Host from cluster import Cluster, StorageNode
from frostfs_testlib.resources.common import PUBLIC_ACL from failover_utils import wait_all_storage_nodes_returned, wait_object_replication
from frostfs_testlib.shell import CommandOptions from file_helper import generate_file, get_file_hash
from neofs_testlib.hosting import Host
from neofs_testlib.shell import CommandOptions
from python_keywords.container import create_container
from python_keywords.neofs_verbs import get_object, put_object_to_random_node
from wellknown_acl import PUBLIC_ACL
from pytest_tests.helpers.cluster import Cluster, StorageNode from steps.cluster_test_base import ClusterTestBase
from pytest_tests.helpers.container import create_container
from pytest_tests.helpers.failover_utils import (
wait_all_storage_nodes_returned,
wait_object_replication,
)
from pytest_tests.helpers.file_helper import generate_file, get_file_hash
from pytest_tests.helpers.frostfs_verbs import get_object, put_object_to_random_node
from pytest_tests.steps.cluster_test_base import ClusterTestBase
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
stopped_nodes: list[StorageNode] = [] stopped_nodes: list[StorageNode] = []
@allure.step("Return all stopped hosts")
@pytest.fixture(scope="function", autouse=True) @pytest.fixture(scope="function", autouse=True)
@allure.step("Return all stopped hosts")
def after_run_return_all_stopped_hosts(cluster: Cluster): def after_run_return_all_stopped_hosts(cluster: Cluster):
yield yield
return_stopped_hosts(cluster) return_stopped_hosts(cluster)

View file

@ -1,14 +1,22 @@
import allure import allure
import pytest import pytest
from frostfs_testlib.hosting import Hosting from cluster_test_base import ClusterTestBase
from common import (
from pytest_tests.helpers.k6 import LoadParams
from pytest_tests.resources.common import (
HTTP_GATE_SERVICE_NAME_REGEX, HTTP_GATE_SERVICE_NAME_REGEX,
S3_GATE_SERVICE_NAME_REGEX, S3_GATE_SERVICE_NAME_REGEX,
STORAGE_NODE_SERVICE_NAME_REGEX, STORAGE_NODE_SERVICE_NAME_REGEX,
) )
from pytest_tests.resources.load_params import ( from k6 import LoadParams
from load import (
clear_cache_and_data,
get_services_endpoints,
init_s3_client,
multi_node_k6_run,
prepare_k6_instances,
start_stopped_nodes,
stop_unused_nodes,
)
from load_params import (
CONTAINER_PLACEMENT_POLICY, CONTAINER_PLACEMENT_POLICY,
CONTAINERS_COUNT, CONTAINERS_COUNT,
DELETERS, DELETERS,
@ -25,16 +33,7 @@ from pytest_tests.resources.load_params import (
STORAGE_NODE_COUNT, STORAGE_NODE_COUNT,
WRITERS, WRITERS,
) )
from pytest_tests.steps.cluster_test_base import ClusterTestBase from neofs_testlib.hosting import Hosting
from pytest_tests.steps.load import (
clear_cache_and_data,
get_services_endpoints,
init_s3_client,
multi_node_k6_run,
prepare_k6_instances,
start_stopped_nodes,
stop_unused_nodes,
)
ENDPOINTS_ATTRIBUTES = { ENDPOINTS_ATTRIBUTES = {
"http": {"regex": HTTP_GATE_SERVICE_NAME_REGEX, "endpoint_attribute": "endpoint"}, "http": {"regex": HTTP_GATE_SERVICE_NAME_REGEX, "endpoint_attribute": "endpoint"},

View file

@ -5,15 +5,15 @@ from typing import Optional, Tuple
import allure import allure
import pytest import pytest
from frostfs_testlib.resources.common import OBJECT_NOT_FOUND, PUBLIC_ACL from cluster import StorageNode
from frostfs_testlib.utils import datetime_utils, string_utils from cluster_test_base import ClusterTestBase
from common import MORPH_BLOCK_TIME, NEOFS_CONTRACT_CACHE_TIMEOUT
from pytest_tests.helpers.cluster import StorageNode from epoch import tick_epoch
from pytest_tests.helpers.container import create_container, get_container from file_helper import generate_file
from pytest_tests.helpers.epoch import tick_epoch from grpc_responses import OBJECT_NOT_FOUND, error_matches_status
from pytest_tests.helpers.failover_utils import wait_object_replication from python_keywords.container import create_container, get_container
from pytest_tests.helpers.file_helper import generate_file from python_keywords.failover_utils import wait_object_replication
from pytest_tests.helpers.frostfs_verbs import ( from python_keywords.neofs_verbs import (
delete_object, delete_object,
get_object, get_object,
get_object_from_random_node, get_object_from_random_node,
@ -21,25 +21,23 @@ from pytest_tests.helpers.frostfs_verbs import (
put_object, put_object,
put_object_to_random_node, put_object_to_random_node,
) )
from pytest_tests.helpers.node_management import ( from python_keywords.node_management import (
check_node_in_map, check_node_in_map,
delete_node_data, delete_node_data,
drop_object, drop_object,
exclude_node_from_network_map, exclude_node_from_network_map,
get_locode_from_random_node, get_locode_from_random_node,
get_netmap_snapshot,
include_node_to_network_map, include_node_to_network_map,
node_shard_list, node_shard_list,
node_shard_set_mode, node_shard_set_mode,
start_storage_nodes,
storage_node_healthcheck, storage_node_healthcheck,
storage_node_set_status, storage_node_set_status,
) )
from pytest_tests.helpers.storage_policy import get_nodes_with_object, get_simple_object_copies from storage_policy import get_nodes_with_object, get_simple_object_copies
from pytest_tests.helpers.utility import ( from utility import parse_time, placement_policy_from_container, wait_for_gc_pass_on_storage_nodes
placement_policy_from_container, from wellknown_acl import PUBLIC_ACL
wait_for_gc_pass_on_storage_nodes,
)
from pytest_tests.resources.common import FROSTFS_CONTRACT_CACHE_TIMEOUT, MORPH_BLOCK_TIME
from pytest_tests.steps.cluster_test_base import ClusterTestBase
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
check_nodes: list[StorageNode] = [] check_nodes: list[StorageNode] = []
@ -113,13 +111,13 @@ class TestNodeManagement(ClusterTestBase):
# We need to wait for node to establish notifications from morph-chain # We need to wait for node to establish notifications from morph-chain
# Otherwise it will hang up when we will try to set status # Otherwise it will hang up when we will try to set status
sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME)) sleep(parse_time(MORPH_BLOCK_TIME))
with allure.step(f"Move node {node} to online state"): with allure.step(f"Move node {node} to online state"):
storage_node_set_status(node, status="online", retries=2) storage_node_set_status(node, status="online", retries=2)
check_nodes.remove(node) check_nodes.remove(node)
sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME)) sleep(parse_time(MORPH_BLOCK_TIME))
self.tick_epoch_with_retries(3) self.tick_epoch_with_retries(3)
check_node_in_map(node, shell=self.shell, alive_node=alive_node) check_node_in_map(node, shell=self.shell, alive_node=alive_node)
@ -132,7 +130,7 @@ class TestNodeManagement(ClusterTestBase):
simple_object_size, simple_object_size,
): ):
""" """
This test remove one node from pytest_tests.helpers.cluster then add it back. Test uses base control operations with storage nodes (healthcheck, netmap-snapshot, set-status). This test remove one node from cluster then add it back. Test uses base control operations with storage nodes (healthcheck, netmap-snapshot, set-status).
""" """
wallet = default_wallet wallet = default_wallet
placement_rule_3 = "REP 3 IN X CBF 1 SELECT 3 FROM * AS X" placement_rule_3 = "REP 3 IN X CBF 1 SELECT 3 FROM * AS X"
@ -326,10 +324,10 @@ class TestNodeManagement(ClusterTestBase):
self.validate_object_copies(wallet, placement_rule, file_path, expected_copies) self.validate_object_copies(wallet, placement_rule, file_path, expected_copies)
@pytest.mark.node_mgmt @pytest.mark.node_mgmt
@allure.title("FrostFS object could be dropped using control command") @allure.title("NeoFS object could be dropped using control command")
def test_drop_object(self, default_wallet, complex_object_size, simple_object_size): def test_drop_object(self, default_wallet, complex_object_size, simple_object_size):
""" """
Test checks object could be dropped using `frostfs-cli control drop-objects` command. Test checks object could be dropped using `neofs-cli control drop-objects` command.
""" """
wallet = default_wallet wallet = default_wallet
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
@ -476,7 +474,7 @@ class TestNodeManagement(ClusterTestBase):
if copies == expected_copies: if copies == expected_copies:
break break
tick_epoch(self.shell, self.cluster) tick_epoch(self.shell, self.cluster)
sleep(datetime_utils.parse_time(FROSTFS_CONTRACT_CACHE_TIMEOUT)) sleep(parse_time(NEOFS_CONTRACT_CACHE_TIMEOUT))
else: else:
raise AssertionError(f"There are no {expected_copies} copies during time") raise AssertionError(f"There are no {expected_copies} copies during time")
@ -487,7 +485,7 @@ class TestNodeManagement(ClusterTestBase):
checker(wallet, cid, oid, shell=self.shell, endpoint=endpoint) checker(wallet, cid, oid, shell=self.shell, endpoint=endpoint)
wait_for_gc_pass_on_storage_nodes() wait_for_gc_pass_on_storage_nodes()
except Exception as err: except Exception as err:
if string_utils.is_str_match_pattern(err, OBJECT_NOT_FOUND): if error_matches_status(err, OBJECT_NOT_FOUND):
return return
raise AssertionError(f'Expected "{OBJECT_NOT_FOUND}" error, got\n{err}') raise AssertionError(f'Expected "{OBJECT_NOT_FOUND}" error, got\n{err}')

View file

@ -4,21 +4,20 @@ import sys
import allure import allure
import pytest import pytest
from frostfs_testlib.resources.common import ( from cluster import Cluster
from complex_object_actions import get_complex_object_split_ranges
from file_helper import generate_file, get_file_content, get_file_hash
from grpc_responses import (
INVALID_LENGTH_SPECIFIER, INVALID_LENGTH_SPECIFIER,
INVALID_OFFSET_SPECIFIER, INVALID_OFFSET_SPECIFIER,
INVALID_RANGE_OVERFLOW, INVALID_RANGE_OVERFLOW,
INVALID_RANGE_ZERO_LENGTH, INVALID_RANGE_ZERO_LENGTH,
OUT_OF_RANGE, OUT_OF_RANGE,
) )
from frostfs_testlib.shell import Shell from neofs_testlib.shell import Shell
from pytest import FixtureRequest from pytest import FixtureRequest
from python_keywords.container import create_container
from pytest_tests.helpers.cluster import Cluster from python_keywords.neofs_verbs import (
from pytest_tests.helpers.complex_object_actions import get_complex_object_split_ranges
from pytest_tests.helpers.container import create_container
from pytest_tests.helpers.file_helper import generate_file, get_file_content, get_file_hash
from pytest_tests.helpers.frostfs_verbs import (
get_object_from_random_node, get_object_from_random_node,
get_range, get_range,
get_range_hash, get_range_hash,
@ -26,10 +25,11 @@ from pytest_tests.helpers.frostfs_verbs import (
put_object_to_random_node, put_object_to_random_node,
search_object, search_object,
) )
from pytest_tests.helpers.storage_object_info import StorageObjectInfo from python_keywords.storage_policy import get_complex_object_copies, get_simple_object_copies
from pytest_tests.helpers.storage_policy import get_complex_object_copies, get_simple_object_copies
from pytest_tests.steps.cluster_test_base import ClusterTestBase from helpers.storage_object_info import StorageObjectInfo
from pytest_tests.steps.storage_object import delete_objects from steps.cluster_test_base import ClusterTestBase
from steps.storage_object import delete_objects
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")

View file

@ -1,30 +1,19 @@
import allure import allure
import pytest import pytest
from frostfs_testlib.resources.common import EACL_PUBLIC_READ_WRITE from cluster import Cluster
from frostfs_testlib.shell import Shell from container import REP_2_FOR_3_NODES_PLACEMENT_RULE, SINGLE_PLACEMENT_RULE, create_container
from epoch import get_epoch
from neofs_testlib.shell import Shell
from neofs_verbs import delete_object, get_object
from pytest import FixtureRequest from pytest import FixtureRequest
from python_keywords.acl import EACLAccess, EACLOperation, EACLRole, EACLRule, form_bearertoken_file
from wellknown_acl import EACL_PUBLIC_READ_WRITE
from pytest_tests.helpers.acl import ( from helpers.container import StorageContainer, StorageContainerInfo
EACLAccess, from helpers.test_control import expect_not_raises
EACLOperation, from helpers.wallet import WalletFile
EACLRole, from steps.cluster_test_base import ClusterTestBase
EACLRule, from steps.storage_object import StorageObjectInfo
form_bearertoken_file,
)
from pytest_tests.helpers.cluster import Cluster
from pytest_tests.helpers.container import (
REP_2_FOR_3_NODES_PLACEMENT_RULE,
SINGLE_PLACEMENT_RULE,
StorageContainer,
StorageContainerInfo,
create_container,
)
from pytest_tests.helpers.epoch import get_epoch
from pytest_tests.helpers.frostfs_verbs import delete_object, get_object
from pytest_tests.helpers.test_control import expect_not_raises
from pytest_tests.helpers.wallet import WalletFile
from pytest_tests.steps.cluster_test_base import ClusterTestBase
from pytest_tests.steps.storage_object import StorageObjectInfo
@pytest.fixture(scope="module") @pytest.fixture(scope="module")

View file

@ -2,18 +2,15 @@ import logging
import allure import allure
import pytest import pytest
from frostfs_testlib.resources.common import OBJECT_NOT_FOUND from epoch import get_epoch, tick_epoch
from file_helper import generate_file, get_file_hash
from grpc_responses import OBJECT_NOT_FOUND
from pytest import FixtureRequest from pytest import FixtureRequest
from python_keywords.container import create_container
from python_keywords.neofs_verbs import get_object_from_random_node, put_object_to_random_node
from utility import wait_for_gc_pass_on_storage_nodes
from pytest_tests.helpers.container import create_container from steps.cluster_test_base import ClusterTestBase
from pytest_tests.helpers.epoch import get_epoch
from pytest_tests.helpers.file_helper import generate_file, get_file_hash
from pytest_tests.helpers.frostfs_verbs import (
get_object_from_random_node,
put_object_to_random_node,
)
from pytest_tests.helpers.utility import wait_for_gc_pass_on_storage_nodes
from pytest_tests.steps.cluster_test_base import ClusterTestBase
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")

View file

@ -3,7 +3,12 @@ import re
import allure import allure
import pytest import pytest
from frostfs_testlib.resources.common import ( from cluster import Cluster
from cluster_test_base import ClusterTestBase
from common import STORAGE_GC_TIME
from complex_object_actions import get_link_object, get_storage_object_chunks
from epoch import ensure_fresh_epoch, get_epoch, tick_epoch
from grpc_responses import (
LIFETIME_REQUIRED, LIFETIME_REQUIRED,
LOCK_NON_REGULAR_OBJECT, LOCK_NON_REGULAR_OBJECT,
LOCK_OBJECT_EXPIRATION, LOCK_OBJECT_EXPIRATION,
@ -12,24 +17,19 @@ from frostfs_testlib.resources.common import (
OBJECT_IS_LOCKED, OBJECT_IS_LOCKED,
OBJECT_NOT_FOUND, OBJECT_NOT_FOUND,
) )
from frostfs_testlib.shell import Shell from neofs_testlib.shell import Shell
from frostfs_testlib.utils import datetime_utils from node_management import drop_object
from pytest import FixtureRequest from pytest import FixtureRequest
from python_keywords.container import create_container
from python_keywords.neofs_verbs import delete_object, head_object, lock_object
from storage_policy import get_nodes_with_object
from test_control import expect_not_raises, wait_for_success
from utility import parse_time, wait_for_gc_pass_on_storage_nodes
from pytest_tests.helpers.cluster import Cluster from helpers.container import StorageContainer, StorageContainerInfo
from pytest_tests.helpers.complex_object_actions import get_link_object, get_storage_object_chunks from helpers.storage_object_info import LockObjectInfo, StorageObjectInfo
from pytest_tests.helpers.container import StorageContainer, StorageContainerInfo, create_container from helpers.wallet import WalletFactory, WalletFile
from pytest_tests.helpers.epoch import ensure_fresh_epoch, get_epoch, tick_epoch from steps.storage_object import delete_objects
from pytest_tests.helpers.frostfs_verbs import delete_object, head_object, lock_object
from pytest_tests.helpers.node_management import drop_object
from pytest_tests.helpers.storage_object_info import LockObjectInfo, StorageObjectInfo
from pytest_tests.helpers.storage_policy import get_nodes_with_object
from pytest_tests.helpers.test_control import expect_not_raises, wait_for_success
from pytest_tests.helpers.utility import wait_for_gc_pass_on_storage_nodes
from pytest_tests.helpers.wallet import WalletFactory, WalletFile
from pytest_tests.resources.common import STORAGE_GC_TIME
from pytest_tests.steps.cluster_test_base import ClusterTestBase
from pytest_tests.steps.storage_object import delete_objects
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
@ -321,7 +321,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
self.cluster.default_rpc_endpoint, self.cluster.default_rpc_endpoint,
) )
@wait_for_success(datetime_utils.parse_time(STORAGE_GC_TIME)) @wait_for_success(parse_time(STORAGE_GC_TIME))
def check_object_not_found(): def check_object_not_found():
with pytest.raises(Exception, match=OBJECT_NOT_FOUND): with pytest.raises(Exception, match=OBJECT_NOT_FOUND):
head_object( head_object(

View file

@ -4,12 +4,11 @@ import os
import allure import allure
import pytest import pytest
import yaml import yaml
from frostfs_testlib.cli import FrostfsCli from cluster_test_base import ClusterTestBase
from frostfs_testlib.shell import CommandResult, Shell from common import FREE_STORAGE, NEOFS_CLI_EXEC, WALLET_CONFIG
from neofs_testlib.cli import NeofsCli
from pytest_tests.helpers.wallet import WalletFactory, WalletFile from neofs_testlib.shell import CommandResult, Shell
from pytest_tests.resources.common import FREE_STORAGE, FROSTFS_CLI_EXEC, WALLET_CONFIG from wallet import WalletFactory, WalletFile
from pytest_tests.steps.cluster_test_base import ClusterTestBase
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
DEPOSIT_AMOUNT = 30 DEPOSIT_AMOUNT = 30
@ -28,8 +27,8 @@ class TestBalanceAccounting(ClusterTestBase):
return wallet_factory.create_wallet() return wallet_factory.create_wallet()
@pytest.fixture(scope="class") @pytest.fixture(scope="class")
def cli(self, client_shell: Shell) -> FrostfsCli: def cli(self, client_shell: Shell) -> NeofsCli:
return FrostfsCli(client_shell, FROSTFS_CLI_EXEC, WALLET_CONFIG) return NeofsCli(client_shell, NEOFS_CLI_EXEC, WALLET_CONFIG)
@allure.step("Check deposit amount") @allure.step("Check deposit amount")
def check_amount(self, result: CommandResult) -> None: def check_amount(self, result: CommandResult) -> None:
@ -54,13 +53,13 @@ class TestBalanceAccounting(ClusterTestBase):
"rpc-endpoint": endpoint, "rpc-endpoint": endpoint,
"wallet": wallet, "wallet": wallet,
} }
api_config_file = os.path.join(config_dir, "frostfs-cli-api-config.yaml") api_config_file = os.path.join(config_dir, "neofs-cli-api-config.yaml")
with open(api_config_file, "w") as file: with open(api_config_file, "w") as file:
yaml.dump(api_config, file) yaml.dump(api_config, file)
return api_config_file return api_config_file
@allure.title("Test balance request with wallet and address") @allure.title("Test balance request with wallet and address")
def test_balance_wallet_address(self, main_wallet: WalletFile, cli: FrostfsCli): def test_balance_wallet_address(self, main_wallet: WalletFile, cli: NeofsCli):
result = cli.accounting.balance( result = cli.accounting.balance(
wallet=main_wallet.path, wallet=main_wallet.path,
rpc_endpoint=self.cluster.default_rpc_endpoint, rpc_endpoint=self.cluster.default_rpc_endpoint,
@ -70,7 +69,7 @@ class TestBalanceAccounting(ClusterTestBase):
self.check_amount(result) self.check_amount(result)
@allure.title("Test balance request with wallet only") @allure.title("Test balance request with wallet only")
def test_balance_wallet(self, main_wallet: WalletFile, cli: FrostfsCli): def test_balance_wallet(self, main_wallet: WalletFile, cli: NeofsCli):
result = cli.accounting.balance( result = cli.accounting.balance(
wallet=main_wallet.path, rpc_endpoint=self.cluster.default_rpc_endpoint wallet=main_wallet.path, rpc_endpoint=self.cluster.default_rpc_endpoint
) )
@ -78,7 +77,7 @@ class TestBalanceAccounting(ClusterTestBase):
@allure.title("Test balance request with wallet and wrong address") @allure.title("Test balance request with wallet and wrong address")
def test_balance_wrong_address( def test_balance_wrong_address(
self, main_wallet: WalletFile, other_wallet: WalletFile, cli: FrostfsCli self, main_wallet: WalletFile, other_wallet: WalletFile, cli: NeofsCli
): ):
with pytest.raises(Exception, match="address option must be specified and valid"): with pytest.raises(Exception, match="address option must be specified and valid"):
cli.accounting.balance( cli.accounting.balance(
@ -96,7 +95,7 @@ class TestBalanceAccounting(ClusterTestBase):
) )
logger.info(f"Config with API endpoint: {config_file}") logger.info(f"Config with API endpoint: {config_file}")
cli = FrostfsCli(client_shell, FROSTFS_CLI_EXEC, config_file=config_file) cli = NeofsCli(client_shell, NEOFS_CLI_EXEC, config_file=config_file)
result = cli.accounting.balance() result = cli.accounting.balance()
self.check_amount(result) self.check_amount(result)

View file

@ -2,9 +2,10 @@ import logging
import allure import allure
import pytest import pytest
from frostfs_testlib.resources.common import PUBLIC_ACL from container import create_container
from file_helper import generate_file
from pytest_tests.helpers.acl import ( from http_gate import get_object_and_verify_hashes, upload_via_http_gate_curl
from python_keywords.acl import (
EACLAccess, EACLAccess,
EACLOperation, EACLOperation,
EACLRole, EACLRole,
@ -16,10 +17,9 @@ from pytest_tests.helpers.acl import (
sign_bearer, sign_bearer,
wait_for_cache_expired, wait_for_cache_expired,
) )
from pytest_tests.helpers.container import create_container from wellknown_acl import PUBLIC_ACL
from pytest_tests.helpers.file_helper import generate_file
from pytest_tests.helpers.http_gate import get_object_and_verify_hashes, upload_via_http_gate_curl from steps.cluster_test_base import ClusterTestBase
from pytest_tests.steps.cluster_test_base import ClusterTestBase
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")

View file

@ -3,13 +3,10 @@ import os
import allure import allure
import pytest import pytest
from frostfs_testlib.resources.common import PUBLIC_ACL from epoch import get_epoch, tick_epoch
from file_helper import generate_file, get_file_hash
from pytest_tests.helpers.container import create_container from python_keywords.container import create_container
from pytest_tests.helpers.epoch import get_epoch from python_keywords.http_gate import (
from pytest_tests.helpers.file_helper import generate_file, get_file_hash
from pytest_tests.helpers.frostfs_verbs import put_object_to_random_node
from pytest_tests.helpers.http_gate import (
attr_into_header, attr_into_header,
get_object_and_verify_hashes, get_object_and_verify_hashes,
get_object_by_attr_and_verify_hashes, get_object_by_attr_and_verify_hashes,
@ -20,19 +17,21 @@ from pytest_tests.helpers.http_gate import (
upload_via_http_gate, upload_via_http_gate,
upload_via_http_gate_curl, upload_via_http_gate_curl,
) )
from pytest_tests.helpers.utility import wait_for_gc_pass_on_storage_nodes from python_keywords.neofs_verbs import put_object_to_random_node
from pytest_tests.steps.cluster_test_base import ClusterTestBase from utility import wait_for_gc_pass_on_storage_nodes
from wellknown_acl import PUBLIC_ACL
from steps.cluster_test_base import ClusterTestBase
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
OBJECT_NOT_FOUND_ERROR = "not found" OBJECT_NOT_FOUND_ERROR = "not found"
@allure.link( @allure.link(
"https://github.com/TrueCloudLab/frostfs-http-gw#frostfs-http-gateway", "https://github.com/nspcc-dev/neofs-http-gw#neofs-http-gateway", name="neofs-http-gateway"
name="frostfs-http-gateway",
) )
@allure.link("https://github.com/TrueCloudLab/frostfs-http-gw#uploading", name="uploading") @allure.link("https://github.com/nspcc-dev/neofs-http-gw#uploading", name="uploading")
@allure.link("https://github.com/TrueCloudLab/frostfs-http-gw#downloading", name="downloading") @allure.link("https://github.com/nspcc-dev/neofs-http-gw#downloading", name="downloading")
@pytest.mark.sanity @pytest.mark.sanity
@pytest.mark.http_gate @pytest.mark.http_gate
class TestHttpGate(ClusterTestBase): class TestHttpGate(ClusterTestBase):
@ -51,9 +50,9 @@ class TestHttpGate(ClusterTestBase):
Steps: Steps:
1. Create simple and large objects. 1. Create simple and large objects.
2. Put objects using gRPC (frostfs-cli). 2. Put objects using gRPC (neofs-cli).
3. Download objects using HTTP gate (https://github.com/TrueCloudLab/frostfs-http-gw#downloading). 3. Download objects using HTTP gate (https://github.com/nspcc-dev/neofs-http-gw#downloading).
4. Get objects using gRPC (frostfs-cli). 4. Get objects using gRPC (neofs-cli).
5. Compare hashes for got objects. 5. Compare hashes for got objects.
6. Compare hashes for got and original objects. 6. Compare hashes for got and original objects.
@ -98,8 +97,8 @@ class TestHttpGate(ClusterTestBase):
endpoint=self.cluster.default_http_gate_endpoint, endpoint=self.cluster.default_http_gate_endpoint,
) )
@allure.link("https://github.com/TrueCloudLab/frostfs-http-gw#uploading", name="uploading") @allure.link("https://github.com/nspcc-dev/neofs-http-gw#uploading", name="uploading")
@allure.link("https://github.com/TrueCloudLab/frostfs-http-gw#downloading", name="downloading") @allure.link("https://github.com/nspcc-dev/neofs-http-gw#downloading", name="downloading")
@allure.title("Test Put over HTTP, Get over HTTP") @allure.title("Test Put over HTTP, Get over HTTP")
@pytest.mark.smoke @pytest.mark.smoke
def test_put_http_get_http(self, complex_object_size, simple_object_size): def test_put_http_get_http(self, complex_object_size, simple_object_size):
@ -108,8 +107,8 @@ class TestHttpGate(ClusterTestBase):
Steps: Steps:
1. Create simple and large objects. 1. Create simple and large objects.
2. Upload objects using HTTP (https://github.com/TrueCloudLab/frostfs-http-gw#uploading). 2. Upload objects using HTTP (https://github.com/nspcc-dev/neofs-http-gw#uploading).
3. Download objects using HTTP gate (https://github.com/TrueCloudLab/frostfs-http-gw#downloading). 3. Download objects using HTTP gate (https://github.com/nspcc-dev/neofs-http-gw#downloading).
4. Compare hashes for got and original objects. 4. Compare hashes for got and original objects.
Expected result: Expected result:
@ -146,8 +145,7 @@ class TestHttpGate(ClusterTestBase):
) )
@allure.link( @allure.link(
"https://github.com/TrueCloudLab/frostfs-http-gw#by-attributes", "https://github.com/nspcc-dev/neofs-http-gw#by-attributes", name="download by attributes"
name="download by attributes",
) )
@allure.title("Test Put over HTTP, Get over HTTP with headers") @allure.title("Test Put over HTTP, Get over HTTP with headers")
@pytest.mark.parametrize( @pytest.mark.parametrize(
@ -166,7 +164,7 @@ class TestHttpGate(ClusterTestBase):
Steps: Steps:
1. Create simple and large objects. 1. Create simple and large objects.
2. Upload objects using HTTP with particular attributes in the header. 2. Upload objects using HTTP with particular attributes in the header.
3. Download objects by attributes using HTTP gate (https://github.com/TrueCloudLab/frostfs-http-gw#by-attributes). 3. Download objects by attributes using HTTP gate (https://github.com/nspcc-dev/neofs-http-gw#by-attributes).
4. Compare hashes for got and original objects. 4. Compare hashes for got and original objects.
Expected result: Expected result:
@ -217,7 +215,7 @@ class TestHttpGate(ClusterTestBase):
epochs = (curr_epoch, curr_epoch + 1, curr_epoch + 2, curr_epoch + 100) epochs = (curr_epoch, curr_epoch + 1, curr_epoch + 2, curr_epoch + 100)
for epoch in epochs: for epoch in epochs:
headers = {"X-Attribute-Frostfs-Expiration-Epoch": str(epoch)} headers = {"X-Attribute-Neofs-Expiration-Epoch": str(epoch)}
with allure.step("Put objects using HTTP with attribute Expiration-Epoch"): with allure.step("Put objects using HTTP with attribute Expiration-Epoch"):
oids.append( oids.append(

View file

@ -3,26 +3,27 @@ import os
import allure import allure
import pytest import pytest
from frostfs_testlib.resources.common import PUBLIC_ACL from container import (
from pytest import FixtureRequest
from pytest_tests.helpers.container import (
create_container, create_container,
delete_container, delete_container,
list_containers, list_containers,
wait_for_container_deletion, wait_for_container_deletion,
) )
from pytest_tests.helpers.file_helper import generate_file from epoch import tick_epoch
from pytest_tests.helpers.frostfs_verbs import delete_object from file_helper import generate_file
from pytest_tests.helpers.http_gate import ( from http_gate import (
attr_into_str_header_curl, attr_into_str_header_curl,
get_object_by_attr_and_verify_hashes, get_object_by_attr_and_verify_hashes,
try_to_get_object_and_expect_error, try_to_get_object_and_expect_error,
try_to_get_object_via_passed_request_and_expect_error, try_to_get_object_via_passed_request_and_expect_error,
upload_via_http_gate_curl, upload_via_http_gate_curl,
) )
from pytest_tests.helpers.storage_object_info import StorageObjectInfo from pytest import FixtureRequest
from pytest_tests.steps.cluster_test_base import ClusterTestBase from python_keywords.neofs_verbs import delete_object
from wellknown_acl import PUBLIC_ACL
from helpers.storage_object_info import StorageObjectInfo
from steps.cluster_test_base import ClusterTestBase
OBJECT_ALREADY_REMOVED_ERROR = "object already removed" OBJECT_ALREADY_REMOVED_ERROR = "object already removed"
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")

View file

@ -1,18 +1,19 @@
import logging import logging
import os
import allure import allure
import pytest import pytest
from frostfs_testlib.resources.common import PUBLIC_ACL from container import create_container
from file_helper import generate_file
from pytest_tests.helpers.container import create_container from http_gate import (
from pytest_tests.helpers.file_helper import generate_file
from pytest_tests.helpers.frostfs_verbs import put_object_to_random_node
from pytest_tests.helpers.http_gate import (
get_object_and_verify_hashes, get_object_and_verify_hashes,
get_object_by_attr_and_verify_hashes, get_object_by_attr_and_verify_hashes,
try_to_get_object_via_passed_request_and_expect_error, try_to_get_object_via_passed_request_and_expect_error,
) )
from pytest_tests.steps.cluster_test_base import ClusterTestBase from python_keywords.neofs_verbs import put_object_to_random_node
from wellknown_acl import PUBLIC_ACL
from steps.cluster_test_base import ClusterTestBase
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
@ -39,8 +40,8 @@ class Test_http_object(ClusterTestBase):
Steps: Steps:
1. Create object; 1. Create object;
2. Put objects using gRPC (frostfs-cli) with attributes [--attributes chapter1=peace,chapter2=war]; 2. Put objects using gRPC (neofs-cli) with attributes [--attributes chapter1=peace,chapter2=war];
3. Download object using HTTP gate (https://github.com/TrueCloudLab/frostfs-http-gw#downloading); 3. Download object using HTTP gate (https://github.com/nspcc-dev/neofs-http-gw#downloading);
4. Compare hashes between original and downloaded object; 4. Compare hashes between original and downloaded object;
5. [Negative] Try to the get object with specified attributes and `get` request: [get/$CID/chapter1/peace]; 5. [Negative] Try to the get object with specified attributes and `get` request: [get/$CID/chapter1/peace];
6. Download the object with specified attributes and `get_by_attribute` request: [get_by_attribute/$CID/chapter1/peace]; 6. Download the object with specified attributes and `get_by_attribute` request: [get_by_attribute/$CID/chapter1/peace];
@ -51,7 +52,6 @@ class Test_http_object(ClusterTestBase):
Expected result: Expected result:
Hashes must be the same. Hashes must be the same.
""" """
with allure.step("Create public container"): with allure.step("Create public container"):
cid = create_container( cid = create_container(
self.wallet, self.wallet,

View file

@ -2,12 +2,12 @@ import logging
import allure import allure
import pytest import pytest
from frostfs_testlib.resources.common import PUBLIC_ACL from container import create_container
from file_helper import generate_file
from http_gate import get_object_and_verify_hashes, upload_via_http_gate_curl
from wellknown_acl import PUBLIC_ACL
from pytest_tests.helpers.container import create_container from steps.cluster_test_base import ClusterTestBase
from pytest_tests.helpers.file_helper import generate_file
from pytest_tests.helpers.http_gate import get_object_and_verify_hashes, upload_via_http_gate_curl
from pytest_tests.steps.cluster_test_base import ClusterTestBase
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
@ -35,7 +35,7 @@ class Test_http_streaming(ClusterTestBase):
Steps: Steps:
1. Create big object; 1. Create big object;
2. Put object using curl with pipe (streaming); 2. Put object using curl with pipe (streaming);
3. Download object using HTTP gate (https://github.com/TrueCloudLab/frostfs-http-gw#downloading); 3. Download object using HTTP gate (https://github.com/nspcc-dev/neofs-http-gw#downloading);
4. Compare hashes between original and downloaded object; 4. Compare hashes between original and downloaded object;
Expected result: Expected result:

View file

@ -5,33 +5,30 @@ from typing import Optional
import allure import allure
import pytest import pytest
from frostfs_testlib.resources.common import OBJECT_NOT_FOUND, PUBLIC_ACL from container import create_container
from epoch import get_epoch, wait_for_epochs_align
from pytest_tests.helpers.container import create_container from file_helper import generate_file
from pytest_tests.helpers.epoch import get_epoch, wait_for_epochs_align from grpc_responses import OBJECT_NOT_FOUND
from pytest_tests.helpers.file_helper import generate_file from http_gate import (
from pytest_tests.helpers.frostfs_verbs import (
get_netmap_netinfo,
get_object_from_random_node,
head_object,
)
from pytest_tests.helpers.http_gate import (
attr_into_str_header_curl, attr_into_str_header_curl,
get_object_and_verify_hashes, get_object_and_verify_hashes,
try_to_get_object_and_expect_error, try_to_get_object_and_expect_error,
upload_via_http_gate_curl, upload_via_http_gate_curl,
) )
from pytest_tests.steps.cluster_test_base import ClusterTestBase from python_keywords.neofs_verbs import get_netmap_netinfo, get_object_from_random_node, head_object
from wellknown_acl import PUBLIC_ACL
from steps.cluster_test_base import ClusterTestBase
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
EXPIRATION_TIMESTAMP_HEADER = "__FROSRFS__EXPIRATION_TIMESTAMP" EXPIRATION_TIMESTAMP_HEADER = "__NEOFS__EXPIRATION_TIMESTAMP"
EXPIRATION_EPOCH_HEADER = "__FROSRFS__EXPIRATION_EPOCH" EXPIRATION_EPOCH_HEADER = "__NEOFS__EXPIRATION_EPOCH"
EXPIRATION_DURATION_HEADER = "__FROSRFS__EXPIRATION_DURATION" EXPIRATION_DURATION_HEADER = "__NEOFS__EXPIRATION_DURATION"
EXPIRATION_EXPIRATION_RFC = "__FROSRFS__EXPIRATION_RFC3339" EXPIRATION_EXPIRATION_RFC = "__NEOFS__EXPIRATION_RFC3339"
FROSTFS_EXPIRATION_EPOCH = "Frostfs-Expiration-Epoch" NEOFS_EXPIRATION_EPOCH = "Neofs-Expiration-Epoch"
FROSTFS_EXPIRATION_DURATION = "Frostfs-Expiration-Duration" NEOFS_EXPIRATION_DURATION = "Neofs-Expiration-Duration"
FROSTFS_EXPIRATION_TIMESTAMP = "Frostfs-Expiration-Timestamp" NEOFS_EXPIRATION_TIMESTAMP = "Neofs-Expiration-Timestamp"
FROSTFS_EXPIRATION_RFC3339 = "Frostfs-Expiration-RFC3339" NEOFS_EXIPRATION_RFC3339 = "Neofs-Expiration-RFC3339"
@pytest.mark.sanity @pytest.mark.sanity
@ -97,7 +94,7 @@ class Test_http_system_header(ClusterTestBase):
f"Validate that only {EXPIRATION_EPOCH_HEADER} exists in header and other headers are abesent" f"Validate that only {EXPIRATION_EPOCH_HEADER} exists in header and other headers are abesent"
) )
def validation_for_http_header_attr(self, head_info: dict, expected_epoch: int) -> None: def validation_for_http_header_attr(self, head_info: dict, expected_epoch: int) -> None:
# check that __FROSTFS__EXPIRATION_EPOCH attribute has corresponding epoch # check that __NEOFS__EXPIRATION_EPOCH attribute has corresponding epoch
assert self.check_key_value_presented_header( assert self.check_key_value_presented_header(
head_info, {EXPIRATION_EPOCH_HEADER: str(expected_epoch)} head_info, {EXPIRATION_EPOCH_HEADER: str(expected_epoch)}
), f'Expected to find {EXPIRATION_EPOCH_HEADER}: {expected_epoch} in: {head_info["header"]["attributes"]}' ), f'Expected to find {EXPIRATION_EPOCH_HEADER}: {expected_epoch} in: {head_info["header"]["attributes"]}'
@ -143,7 +140,7 @@ class Test_http_system_header(ClusterTestBase):
@allure.title("[negative] attempt to put object with expired epoch") @allure.title("[negative] attempt to put object with expired epoch")
def test_unable_put_expired_epoch(self, user_container: str, simple_object_size: int): def test_unable_put_expired_epoch(self, user_container: str, simple_object_size: int):
headers = attr_into_str_header_curl( headers = attr_into_str_header_curl(
{"Frostfs-Expiration-Epoch": str(get_epoch(self.shell, self.cluster) - 1)} {"Neofs-Expiration-Epoch": str(get_epoch(self.shell, self.cluster) - 1)}
) )
file_path = generate_file(simple_object_size) file_path = generate_file(simple_object_size)
with allure.step( with allure.step(
@ -157,12 +154,12 @@ class Test_http_system_header(ClusterTestBase):
error_pattern="object has expired", error_pattern="object has expired",
) )
@allure.title("[negative] attempt to put object with negative Frostfs-Expiration-Duration") @allure.title("[negative] attempt to put object with negative Neofs-Expiration-Duration")
def test_unable_put_negative_duration(self, user_container: str, simple_object_size: int): def test_unable_put_negative_duration(self, user_container: str, simple_object_size: int):
headers = attr_into_str_header_curl({"Frostfs-Expiration-Duration": "-1h"}) headers = attr_into_str_header_curl({"Neofs-Expiration-Duration": "-1h"})
file_path = generate_file(simple_object_size) file_path = generate_file(simple_object_size)
with allure.step( with allure.step(
"Put object using HTTP with attribute Frostfs-Expiration-Duration where duration is negative" "Put object using HTTP with attribute Neofs-Expiration-Duration where duration is negative"
): ):
upload_via_http_gate_curl( upload_via_http_gate_curl(
cid=user_container, cid=user_container,
@ -173,13 +170,13 @@ class Test_http_system_header(ClusterTestBase):
) )
@allure.title( @allure.title(
"[negative] attempt to put object with Frostfs-Expiration-Timestamp value in the past" "[negative] attempt to put object with Neofs-Expiration-Timestamp value in the past"
) )
def test_unable_put_expired_timestamp(self, user_container: str, simple_object_size: int): def test_unable_put_expired_timestamp(self, user_container: str, simple_object_size: int):
headers = attr_into_str_header_curl({"Frostfs-Expiration-Timestamp": "1635075727"}) headers = attr_into_str_header_curl({"Neofs-Expiration-Timestamp": "1635075727"})
file_path = generate_file(simple_object_size) file_path = generate_file(simple_object_size)
with allure.step( with allure.step(
"Put object using HTTP with attribute Frostfs-Expiration-Timestamp where duration is in the past" "Put object using HTTP with attribute Neofs-Expiration-Timestamp where duration is in the past"
): ):
upload_via_http_gate_curl( upload_via_http_gate_curl(
cid=user_container, cid=user_container,
@ -190,10 +187,10 @@ class Test_http_system_header(ClusterTestBase):
) )
@allure.title( @allure.title(
"[negative] Put object using HTTP with attribute Frostfs-Expiration-RFC3339 where duration is in the past" "[negative] Put object using HTTP with attribute Neofs-Expiration-RFC3339 where duration is in the past"
) )
def test_unable_put_expired_rfc(self, user_container: str, simple_object_size: int): def test_unable_put_expired_rfc(self, user_container: str, simple_object_size: int):
headers = attr_into_str_header_curl({"Frostfs-Expiration-RFC3339": "2021-11-22T09:55:49Z"}) headers = attr_into_str_header_curl({"Neofs-Expiration-RFC3339": "2021-11-22T09:55:49Z"})
file_path = generate_file(simple_object_size) file_path = generate_file(simple_object_size)
upload_via_http_gate_curl( upload_via_http_gate_curl(
cid=user_container, cid=user_container,
@ -218,7 +215,7 @@ class Test_http_system_header(ClusterTestBase):
logger.info( logger.info(
f"epoch duration={epoch_duration}, current_epoch= {get_epoch(self.shell, self.cluster)} expected_epoch {expected_epoch}" f"epoch duration={epoch_duration}, current_epoch= {get_epoch(self.shell, self.cluster)} expected_epoch {expected_epoch}"
) )
attributes = {FROSTFS_EXPIRATION_EPOCH: expected_epoch, FROSTFS_EXPIRATION_DURATION: "1m"} attributes = {NEOFS_EXPIRATION_EPOCH: expected_epoch, NEOFS_EXPIRATION_DURATION: "1m"}
file_path = generate_file(object_size) file_path = generate_file(object_size)
with allure.step( with allure.step(
f"Put objects using HTTP with attributes and head command should display {EXPIRATION_EPOCH_HEADER}: {expected_epoch} attr" f"Put objects using HTTP with attributes and head command should display {EXPIRATION_EPOCH_HEADER}: {expected_epoch} attr"
@ -266,10 +263,10 @@ class Test_http_system_header(ClusterTestBase):
f"epoch duration={epoch_duration}, current_epoch= {get_epoch(self.shell, self.cluster)} expected_epoch {expected_epoch}" f"epoch duration={epoch_duration}, current_epoch= {get_epoch(self.shell, self.cluster)} expected_epoch {expected_epoch}"
) )
attributes = { attributes = {
FROSTFS_EXPIRATION_DURATION: self.epoch_count_into_mins( NEOFS_EXPIRATION_DURATION: self.epoch_count_into_mins(
epoch_duration=epoch_duration, epoch=2 epoch_duration=epoch_duration, epoch=2
), ),
FROSTFS_EXPIRATION_TIMESTAMP: self.epoch_count_into_timestamp( NEOFS_EXPIRATION_TIMESTAMP: self.epoch_count_into_timestamp(
epoch_duration=epoch_duration, epoch=1 epoch_duration=epoch_duration, epoch=1
), ),
} }
@ -320,10 +317,10 @@ class Test_http_system_header(ClusterTestBase):
f"epoch duration={epoch_duration}, current_epoch= {get_epoch(self.shell, self.cluster)} expected_epoch {expected_epoch}" f"epoch duration={epoch_duration}, current_epoch= {get_epoch(self.shell, self.cluster)} expected_epoch {expected_epoch}"
) )
attributes = { attributes = {
FROSTFS_EXPIRATION_TIMESTAMP: self.epoch_count_into_timestamp( NEOFS_EXPIRATION_TIMESTAMP: self.epoch_count_into_timestamp(
epoch_duration=epoch_duration, epoch=2 epoch_duration=epoch_duration, epoch=2
), ),
FROSTFS_EXPIRATION_RFC3339: self.epoch_count_into_timestamp( NEOFS_EXIPRATION_RFC3339: self.epoch_count_into_timestamp(
epoch_duration=epoch_duration, epoch=1, rfc3339=True epoch_duration=epoch_duration, epoch=1, rfc3339=True
), ),
} }
@ -372,7 +369,7 @@ class Test_http_system_header(ClusterTestBase):
f"epoch duration={epoch_duration}, current_epoch= {get_epoch(self.shell, self.cluster)} expected_epoch {expected_epoch}" f"epoch duration={epoch_duration}, current_epoch= {get_epoch(self.shell, self.cluster)} expected_epoch {expected_epoch}"
) )
attributes = { attributes = {
FROSTFS_EXPIRATION_RFC3339: self.epoch_count_into_timestamp( NEOFS_EXIPRATION_RFC3339: self.epoch_count_into_timestamp(
epoch_duration=epoch_duration, epoch=2, rfc3339=True epoch_duration=epoch_duration, epoch=2, rfc3339=True
) )
} }

View file

@ -1,10 +1,10 @@
import allure import allure
import pytest import pytest
from file_helper import generate_file
from s3_helper import object_key_from_file_path
from pytest_tests.helpers.file_helper import generate_file from steps import s3_gate_bucket, s3_gate_object
from pytest_tests.helpers.s3_helper import assert_s3_acl, object_key_from_file_path from steps.s3_gate_base import TestS3GateBase
from pytest_tests.steps import s3_gate_bucket, s3_gate_object
from pytest_tests.steps.s3_gate_base import TestS3GateBase
def pytest_generate_tests(metafunc): def pytest_generate_tests(metafunc):
@ -29,12 +29,19 @@ class TestS3GateACL(TestS3GateBase):
with allure.step("Put object ACL = public-read"): with allure.step("Put object ACL = public-read"):
s3_gate_object.put_object_acl_s3(self.s3_client, bucket, file_name, "public-read") s3_gate_object.put_object_acl_s3(self.s3_client, bucket, file_name, "public-read")
obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, file_name) obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, file_name)
assert_s3_acl(acl_grants=obj_acl, permitted_users="AllUsers") obj_permission = [permission.get("Permission") for permission in obj_acl]
assert obj_permission == [
"FULL_CONTROL",
"FULL_CONTROL",
], "Permission for all groups is FULL_CONTROL"
with allure.step("Put object ACL = private"): with allure.step("Put object ACL = private"):
s3_gate_object.put_object_acl_s3(self.s3_client, bucket, file_name, "private") s3_gate_object.put_object_acl_s3(self.s3_client, bucket, file_name, "private")
obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, file_name) obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, file_name)
assert_s3_acl(acl_grants=obj_acl, permitted_users="CanonicalUser") obj_permission = [permission.get("Permission") for permission in obj_acl]
assert obj_permission == [
"FULL_CONTROL",
], "Permission for Canonical User is FULL_CONTROL"
with allure.step( with allure.step(
"Put object with grant-read uri=http://acs.amazonaws.com/groups/global/AllUsers" "Put object with grant-read uri=http://acs.amazonaws.com/groups/global/AllUsers"
@ -46,19 +53,30 @@ class TestS3GateACL(TestS3GateBase):
grant_read="uri=http://acs.amazonaws.com/groups/global/AllUsers", grant_read="uri=http://acs.amazonaws.com/groups/global/AllUsers",
) )
obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, file_name) obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, file_name)
assert_s3_acl(acl_grants=obj_acl, permitted_users="AllUsers") obj_permission = [permission.get("Permission") for permission in obj_acl]
assert obj_permission == [
"FULL_CONTROL",
"FULL_CONTROL",
], "Permission for all groups is FULL_CONTROL"
@allure.title("Test S3: Bucket ACL") @allure.title("Test S3: Bucket ACL")
def test_s3_bucket_ACL(self): def test_s3_bucket_ACL(self):
with allure.step("Create bucket with ACL = public-read-write"): with allure.step("Create bucket with ACL = public-read-write"):
bucket = s3_gate_bucket.create_bucket_s3(self.s3_client, True, acl="public-read-write") bucket = s3_gate_bucket.create_bucket_s3(self.s3_client, True, acl="public-read-write")
bucket_acl = s3_gate_bucket.get_bucket_acl(self.s3_client, bucket) bucket_acl = s3_gate_bucket.get_bucket_acl(self.s3_client, bucket)
assert_s3_acl(acl_grants=bucket_acl, permitted_users="AllUsers") bucket_permission = [permission.get("Permission") for permission in bucket_acl]
assert bucket_permission == [
"FULL_CONTROL",
"FULL_CONTROL",
], "Permission for all groups is FULL_CONTROL"
with allure.step("Change bucket ACL to private"): with allure.step("Change bucket ACL to private"):
s3_gate_bucket.put_bucket_acl_s3(self.s3_client, bucket, acl="private") s3_gate_bucket.put_bucket_acl_s3(self.s3_client, bucket, acl="private")
bucket_acl = s3_gate_bucket.get_bucket_acl(self.s3_client, bucket) bucket_acl = s3_gate_bucket.get_bucket_acl(self.s3_client, bucket)
assert_s3_acl(acl_grants=bucket_acl, permitted_users="CanonicalUser") bucket_permission = [permission.get("Permission") for permission in bucket_acl]
assert bucket_permission == [
"FULL_CONTROL"
], "Permission for CanonicalUser is FULL_CONTROL"
with allure.step( with allure.step(
"Change bucket acl to --grant-write uri=http://acs.amazonaws.com/groups/global/AllUsers" "Change bucket acl to --grant-write uri=http://acs.amazonaws.com/groups/global/AllUsers"
@ -69,4 +87,8 @@ class TestS3GateACL(TestS3GateBase):
grant_write="uri=http://acs.amazonaws.com/groups/global/AllUsers", grant_write="uri=http://acs.amazonaws.com/groups/global/AllUsers",
) )
bucket_acl = s3_gate_bucket.get_bucket_acl(self.s3_client, bucket) bucket_acl = s3_gate_bucket.get_bucket_acl(self.s3_client, bucket)
assert_s3_acl(acl_grants=bucket_acl, permitted_users="AllUsers") bucket_permission = [permission.get("Permission") for permission in bucket_acl]
assert bucket_permission == [
"FULL_CONTROL",
"FULL_CONTROL",
], "Permission for all groups is FULL_CONTROL"

View file

@ -2,16 +2,11 @@ from datetime import datetime, timedelta
import allure import allure
import pytest import pytest
from file_helper import generate_file
from s3_helper import assert_object_lock_mode, check_objects_in_bucket, object_key_from_file_path
from pytest_tests.helpers.file_helper import generate_file from steps import s3_gate_bucket, s3_gate_object
from pytest_tests.helpers.s3_helper import ( from steps.s3_gate_base import TestS3GateBase
assert_object_lock_mode,
assert_s3_acl,
check_objects_in_bucket,
object_key_from_file_path,
)
from pytest_tests.steps import s3_gate_bucket, s3_gate_object
from pytest_tests.steps.s3_gate_base import TestS3GateBase
def pytest_generate_tests(metafunc): def pytest_generate_tests(metafunc):
@ -29,26 +24,41 @@ class TestS3GateBucket(TestS3GateBase):
with allure.step("Create bucket with ACL private"): with allure.step("Create bucket with ACL private"):
bucket = s3_gate_bucket.create_bucket_s3(self.s3_client, True, acl="private") bucket = s3_gate_bucket.create_bucket_s3(self.s3_client, True, acl="private")
bucket_acl = s3_gate_bucket.get_bucket_acl(self.s3_client, bucket) bucket_acl = s3_gate_bucket.get_bucket_acl(self.s3_client, bucket)
assert_s3_acl(acl_grants=bucket_acl, permitted_users="CanonicalUser") bucket_permission = [permission.get("Permission") for permission in bucket_acl]
assert bucket_permission == [
"FULL_CONTROL"
], "Permission for CanonicalUser is FULL_CONTROL"
with allure.step("Create bucket with ACL = public-read"): with allure.step("Create bucket with ACL = public-read"):
bucket_1 = s3_gate_bucket.create_bucket_s3(self.s3_client, True, acl="public-read") bucket_1 = s3_gate_bucket.create_bucket_s3(self.s3_client, True, acl="public-read")
bucket_acl_1 = s3_gate_bucket.get_bucket_acl(self.s3_client, bucket_1) bucket_acl_1 = s3_gate_bucket.get_bucket_acl(self.s3_client, bucket_1)
assert_s3_acl(acl_grants=bucket_acl_1, permitted_users="AllUsers") bucket_permission_1 = [permission.get("Permission") for permission in bucket_acl_1]
assert bucket_permission_1 == [
"FULL_CONTROL",
"FULL_CONTROL",
], "Permission for all groups is FULL_CONTROL"
with allure.step("Create bucket with ACL public-read-write"): with allure.step("Create bucket with ACL public-read-write"):
bucket_2 = s3_gate_bucket.create_bucket_s3( bucket_2 = s3_gate_bucket.create_bucket_s3(
self.s3_client, True, acl="public-read-write" self.s3_client, True, acl="public-read-write"
) )
bucket_acl_2 = s3_gate_bucket.get_bucket_acl(self.s3_client, bucket_2) bucket_acl_2 = s3_gate_bucket.get_bucket_acl(self.s3_client, bucket_2)
assert_s3_acl(acl_grants=bucket_acl_2, permitted_users="AllUsers") bucket_permission_2 = [permission.get("Permission") for permission in bucket_acl_2]
assert bucket_permission_2 == [
"FULL_CONTROL",
"FULL_CONTROL",
], "Permission for CanonicalUser is FULL_CONTROL"
with allure.step("Create bucket with ACL = authenticated-read"): with allure.step("Create bucket with ACL = authenticated-read"):
bucket_3 = s3_gate_bucket.create_bucket_s3( bucket_3 = s3_gate_bucket.create_bucket_s3(
self.s3_client, True, acl="authenticated-read" self.s3_client, True, acl="authenticated-read"
) )
bucket_acl_3 = s3_gate_bucket.get_bucket_acl(self.s3_client, bucket_3) bucket_acl_3 = s3_gate_bucket.get_bucket_acl(self.s3_client, bucket_3)
assert_s3_acl(acl_grants=bucket_acl_3, permitted_users="AllUsers") bucket_permission_3 = [permission.get("Permission") for permission in bucket_acl_3]
assert bucket_permission_3 == [
"FULL_CONTROL",
"FULL_CONTROL",
], "Permission for all groups is FULL_CONTROL"
@allure.title("Test S3: Create Bucket with different ACL by grand") @allure.title("Test S3: Create Bucket with different ACL by grand")
def test_s3_create_bucket_with_grands(self): def test_s3_create_bucket_with_grands(self):
@ -60,7 +70,11 @@ class TestS3GateBucket(TestS3GateBase):
grant_read="uri=http://acs.amazonaws.com/groups/global/AllUsers", grant_read="uri=http://acs.amazonaws.com/groups/global/AllUsers",
) )
bucket_acl = s3_gate_bucket.get_bucket_acl(self.s3_client, bucket) bucket_acl = s3_gate_bucket.get_bucket_acl(self.s3_client, bucket)
assert_s3_acl(acl_grants=bucket_acl, permitted_users="AllUsers") bucket_permission = [permission.get("Permission") for permission in bucket_acl]
assert bucket_permission == [
"FULL_CONTROL",
"FULL_CONTROL",
], "Permission for CanonicalUser is FULL_CONTROL"
with allure.step("Create bucket with --grant-wtite"): with allure.step("Create bucket with --grant-wtite"):
bucket_1 = s3_gate_bucket.create_bucket_s3( bucket_1 = s3_gate_bucket.create_bucket_s3(
@ -69,7 +83,11 @@ class TestS3GateBucket(TestS3GateBase):
grant_write="uri=http://acs.amazonaws.com/groups/global/AllUsers", grant_write="uri=http://acs.amazonaws.com/groups/global/AllUsers",
) )
bucket_acl_1 = s3_gate_bucket.get_bucket_acl(self.s3_client, bucket_1) bucket_acl_1 = s3_gate_bucket.get_bucket_acl(self.s3_client, bucket_1)
assert_s3_acl(acl_grants=bucket_acl_1, permitted_users="AllUsers") bucket_permission_1 = [permission.get("Permission") for permission in bucket_acl_1]
assert bucket_permission_1 == [
"FULL_CONTROL",
"FULL_CONTROL",
], "Permission for all groups is FULL_CONTROL"
with allure.step("Create bucket with --grant-full-control"): with allure.step("Create bucket with --grant-full-control"):
bucket_2 = s3_gate_bucket.create_bucket_s3( bucket_2 = s3_gate_bucket.create_bucket_s3(
@ -78,7 +96,11 @@ class TestS3GateBucket(TestS3GateBase):
grant_full_control="uri=http://acs.amazonaws.com/groups/global/AllUsers", grant_full_control="uri=http://acs.amazonaws.com/groups/global/AllUsers",
) )
bucket_acl_2 = s3_gate_bucket.get_bucket_acl(self.s3_client, bucket_2) bucket_acl_2 = s3_gate_bucket.get_bucket_acl(self.s3_client, bucket_2)
assert_s3_acl(acl_grants=bucket_acl_2, permitted_users="AllUsers") bucket_permission_2 = [permission.get("Permission") for permission in bucket_acl_2]
assert bucket_permission_2 == [
"FULL_CONTROL",
"FULL_CONTROL",
], "Permission for CanonicalUser is FULL_CONTROL"
@allure.title("Test S3: create bucket with object lock") @allure.title("Test S3: create bucket with object lock")
def test_s3_bucket_object_lock(self, simple_object_size): def test_s3_bucket_object_lock(self, simple_object_size):

View file

@ -4,26 +4,26 @@ from random import choice, choices
import allure import allure
import pytest import pytest
from aws_cli_client import AwsCliClient
from pytest_tests.helpers.aws_cli_client import AwsCliClient from common import ASSETS_DIR
from pytest_tests.helpers.epoch import tick_epoch from epoch import tick_epoch
from pytest_tests.helpers.file_helper import ( from file_helper import (
generate_file, generate_file,
generate_file_with_content, generate_file_with_content,
get_file_content, get_file_content,
get_file_hash, get_file_hash,
split_file, split_file,
) )
from pytest_tests.helpers.s3_helper import ( from s3_helper import (
check_objects_in_bucket, check_objects_in_bucket,
check_tags_by_bucket, check_tags_by_bucket,
check_tags_by_object, check_tags_by_object,
set_bucket_versioning, set_bucket_versioning,
try_to_get_objects_and_expect_error, try_to_get_objects_and_expect_error,
) )
from pytest_tests.resources.common import ASSETS_DIR
from pytest_tests.steps import s3_gate_bucket, s3_gate_object from steps import s3_gate_bucket, s3_gate_object
from pytest_tests.steps.s3_gate_base import TestS3GateBase from steps.s3_gate_base import TestS3GateBase
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
@ -33,9 +33,7 @@ def pytest_generate_tests(metafunc):
metafunc.parametrize("s3_client", ["aws cli", "boto3"], indirect=True) metafunc.parametrize("s3_client", ["aws cli", "boto3"], indirect=True)
@allure.link( @allure.link("https://github.com/nspcc-dev/neofs-s3-gw#neofs-s3-gateway", name="neofs-s3-gateway")
"https://github.com/TrueCloudLab/frostfs-s3-gw#frostfs-s3-gw", name="frostfs-s3-gateway"
)
@pytest.mark.sanity @pytest.mark.sanity
@pytest.mark.s3_gate @pytest.mark.s3_gate
@pytest.mark.s3_gate_base @pytest.mark.s3_gate_base

View file

@ -3,15 +3,11 @@ from datetime import datetime, timedelta
import allure import allure
import pytest import pytest
from file_helper import generate_file, generate_file_with_content
from s3_helper import assert_object_lock_mode, check_objects_in_bucket, object_key_from_file_path
from pytest_tests.helpers.file_helper import generate_file, generate_file_with_content from steps import s3_gate_bucket, s3_gate_object
from pytest_tests.helpers.s3_helper import ( from steps.s3_gate_base import TestS3GateBase
assert_object_lock_mode,
check_objects_in_bucket,
object_key_from_file_path,
)
from pytest_tests.steps import s3_gate_bucket, s3_gate_object
from pytest_tests.steps.s3_gate_base import TestS3GateBase
def pytest_generate_tests(metafunc): def pytest_generate_tests(metafunc):

View file

@ -1,14 +1,10 @@
import allure import allure
import pytest import pytest
from file_helper import generate_file, get_file_hash, split_file
from s3_helper import check_objects_in_bucket, object_key_from_file_path, set_bucket_versioning
from pytest_tests.helpers.file_helper import generate_file, get_file_hash, split_file from steps import s3_gate_bucket, s3_gate_object
from pytest_tests.helpers.s3_helper import ( from steps.s3_gate_base import TestS3GateBase
check_objects_in_bucket,
object_key_from_file_path,
set_bucket_versioning,
)
from pytest_tests.steps import s3_gate_bucket, s3_gate_object
from pytest_tests.steps.s3_gate_base import TestS3GateBase
PART_SIZE = 5 * 1024 * 1024 PART_SIZE = 5 * 1024 * 1024

View file

@ -6,25 +6,16 @@ from random import choices, sample
import allure import allure
import pytest import pytest
from frostfs_testlib.utils import wallet_utils from aws_cli_client import AwsCliClient
from common import ASSETS_DIR, FREE_STORAGE, WALLET_PASS
from data_formatters import get_wallet_public_key
from file_helper import concat_files, generate_file, generate_file_with_content, get_file_hash
from neofs_testlib.utils.wallet import init_wallet
from python_keywords.payment_neogo import deposit_gas, transfer_gas
from s3_helper import assert_object_lock_mode, check_objects_in_bucket, set_bucket_versioning
from pytest_tests.helpers.aws_cli_client import AwsCliClient from steps import s3_gate_bucket, s3_gate_object
from pytest_tests.helpers.file_helper import ( from steps.s3_gate_base import TestS3GateBase
concat_files,
generate_file,
generate_file_with_content,
get_file_hash,
)
from pytest_tests.helpers.payment_neogo import deposit_gas, transfer_gas
from pytest_tests.helpers.s3_helper import (
assert_object_lock_mode,
assert_s3_acl,
check_objects_in_bucket,
set_bucket_versioning,
)
from pytest_tests.resources.common import ASSETS_DIR, FREE_STORAGE, WALLET_PASS
from pytest_tests.steps import s3_gate_bucket, s3_gate_object
from pytest_tests.steps.s3_gate_base import TestS3GateBase
def pytest_generate_tests(metafunc): def pytest_generate_tests(metafunc):
@ -140,7 +131,10 @@ class TestS3GateObject(TestS3GateBase):
self.s3_client, bucket, obj_key, ACL="public-read-write" self.s3_client, bucket, obj_key, ACL="public-read-write"
) )
obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, copy_obj_path) obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, copy_obj_path)
assert_s3_acl(acl_grants=obj_acl, permitted_users="CanonicalUser") for control in obj_acl:
assert (
control.get("Permission") == "FULL_CONTROL"
), "Permission for all groups is FULL_CONTROL"
@allure.title("Test S3: Copy object with metadata") @allure.title("Test S3: Copy object with metadata")
def test_s3_copy_metadate(self, bucket, simple_object_size): def test_s3_copy_metadate(self, bucket, simple_object_size):
@ -665,10 +659,10 @@ class TestS3GateObject(TestS3GateBase):
@pytest.fixture @pytest.fixture
def prepare_two_wallets(self, default_wallet, client_shell): def prepare_two_wallets(self, default_wallet, client_shell):
self.main_wallet = default_wallet self.main_wallet = default_wallet
self.main_public_key = wallet_utils.get_wallet_public_key(self.main_wallet, WALLET_PASS) self.main_public_key = get_wallet_public_key(self.main_wallet, WALLET_PASS)
self.other_wallet = os.path.join(os.getcwd(), ASSETS_DIR, f"{str(uuid.uuid4())}.json") self.other_wallet = os.path.join(os.getcwd(), ASSETS_DIR, f"{str(uuid.uuid4())}.json")
wallet_utils.init_wallet(self.other_wallet, WALLET_PASS) init_wallet(self.other_wallet, WALLET_PASS)
self.other_public_key = wallet_utils.get_wallet_public_key(self.other_wallet, WALLET_PASS) self.other_public_key = get_wallet_public_key(self.other_wallet, WALLET_PASS)
if not FREE_STORAGE: if not FREE_STORAGE:
main_chain = self.cluster.main_chain_nodes[0] main_chain = self.cluster.main_chain_nodes[0]
@ -709,7 +703,8 @@ class TestS3GateObject(TestS3GateBase):
with allure.step("Put object with acl private"): with allure.step("Put object with acl private"):
s3_gate_object.put_object_s3(self.s3_client, bucket, file_path_1, ACL="private") s3_gate_object.put_object_s3(self.s3_client, bucket, file_path_1, ACL="private")
obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, file_name) obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, file_name)
assert_s3_acl(acl_grants=obj_acl, permitted_users="CanonicalUser") obj_permission = [permission.get("Permission") for permission in obj_acl]
assert obj_permission == ["FULL_CONTROL"], "Permission for all groups is FULL_CONTROL"
object_1 = s3_gate_object.get_object_s3(self.s3_client, bucket, file_name) object_1 = s3_gate_object.get_object_s3(self.s3_client, bucket, file_name)
assert get_file_hash(file_path_1) == get_file_hash(object_1), "Hashes must be the same" assert get_file_hash(file_path_1) == get_file_hash(object_1), "Hashes must be the same"
@ -717,7 +712,11 @@ class TestS3GateObject(TestS3GateBase):
file_path_2 = generate_file_with_content(simple_object_size, file_path=file_path_1) file_path_2 = generate_file_with_content(simple_object_size, file_path=file_path_1)
s3_gate_object.put_object_s3(self.s3_client, bucket, file_path_2, ACL="public-read") s3_gate_object.put_object_s3(self.s3_client, bucket, file_path_2, ACL="public-read")
obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, file_name) obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, file_name)
assert_s3_acl(acl_grants=obj_acl, permitted_users="AllUsers") obj_permission = [permission.get("Permission") for permission in obj_acl]
assert obj_permission == [
"FULL_CONTROL",
"FULL_CONTROL",
], "Permission for all groups is FULL_CONTROL"
object_2 = s3_gate_object.get_object_s3(self.s3_client, bucket, file_name) object_2 = s3_gate_object.get_object_s3(self.s3_client, bucket, file_name)
assert get_file_hash(file_path_2) == get_file_hash(object_2), "Hashes must be the same" assert get_file_hash(file_path_2) == get_file_hash(object_2), "Hashes must be the same"
@ -727,7 +726,11 @@ class TestS3GateObject(TestS3GateBase):
self.s3_client, bucket, file_path_3, ACL="public-read-write" self.s3_client, bucket, file_path_3, ACL="public-read-write"
) )
obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, file_name) obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, file_name)
assert_s3_acl(acl_grants=obj_acl, permitted_users="AllUsers") obj_permission = [permission.get("Permission") for permission in obj_acl]
assert obj_permission == [
"FULL_CONTROL",
"FULL_CONTROL",
], "Permission for all groups is FULL_CONTROL"
object_3 = s3_gate_object.get_object_s3(self.s3_client, bucket, file_name) object_3 = s3_gate_object.get_object_s3(self.s3_client, bucket, file_name)
assert get_file_hash(file_path_3) == get_file_hash(object_3), "Hashes must be the same" assert get_file_hash(file_path_3) == get_file_hash(object_3), "Hashes must be the same"
@ -737,7 +740,11 @@ class TestS3GateObject(TestS3GateBase):
self.s3_client, bucket, file_path_4, ACL="authenticated-read" self.s3_client, bucket, file_path_4, ACL="authenticated-read"
) )
obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, file_name) obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, file_name)
assert_s3_acl(acl_grants=obj_acl, permitted_users="AllUsers") obj_permission = [permission.get("Permission") for permission in obj_acl]
assert obj_permission == [
"FULL_CONTROL",
"FULL_CONTROL",
], "Permission for all groups is FULL_CONTROL"
object_4 = s3_gate_object.get_object_s3(self.s3_client, bucket, file_name) object_4 = s3_gate_object.get_object_s3(self.s3_client, bucket, file_name)
assert get_file_hash(file_path_4) == get_file_hash(object_4), "Hashes must be the same" assert get_file_hash(file_path_4) == get_file_hash(object_4), "Hashes must be the same"
@ -753,7 +760,11 @@ class TestS3GateObject(TestS3GateBase):
GrantFullControl=f"id={self.other_public_key}", GrantFullControl=f"id={self.other_public_key}",
) )
obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, file_name_5) obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, file_name_5)
assert_s3_acl(acl_grants=obj_acl, permitted_users="CanonicalUser") obj_permission = [permission.get("Permission") for permission in obj_acl]
assert obj_permission == [
"FULL_CONTROL",
"FULL_CONTROL",
], "Permission for all groups is FULL_CONTROL"
object_4 = s3_gate_object.get_object_s3(self.s3_client, bucket, file_name_5) object_4 = s3_gate_object.get_object_s3(self.s3_client, bucket, file_name_5)
assert get_file_hash(file_path_5) == get_file_hash(object_4), "Hashes must be the same" assert get_file_hash(file_path_5) == get_file_hash(object_4), "Hashes must be the same"
@ -768,7 +779,11 @@ class TestS3GateObject(TestS3GateBase):
GrantRead="uri=http://acs.amazonaws.com/groups/global/AllUsers", GrantRead="uri=http://acs.amazonaws.com/groups/global/AllUsers",
) )
obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, file_name_5) obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, file_name_5)
assert_s3_acl(acl_grants=obj_acl, permitted_users="AllUsers") obj_permission = [permission.get("Permission") for permission in obj_acl]
assert obj_permission == [
"FULL_CONTROL",
"FULL_CONTROL",
], "Permission for all groups is FULL_CONTROL"
object_7 = s3_gate_object.get_object_s3(self.s3_client, bucket, file_name_5) object_7 = s3_gate_object.get_object_s3(self.s3_client, bucket, file_name_5)
assert get_file_hash(file_path_7) == get_file_hash(object_7), "Hashes must be the same" assert get_file_hash(file_path_7) == get_file_hash(object_7), "Hashes must be the same"
@ -898,9 +913,12 @@ class TestS3GateObject(TestS3GateBase):
assert ( assert (
obj_head.get("Metadata") == object_metadata obj_head.get("Metadata") == object_metadata
), f"Metadata of object is {object_metadata}" ), f"Metadata of object is {object_metadata}"
# Uncomment after https://github.com/nspcc-dev/neofs-s3-gw/issues/685 is solved
# obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, obj_key) # obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, obj_key)
# assert_s3_acl(acl_grants = obj_acl, permitted_users = "AllUsers") # obj_permission = [permission.get("Permission") for permission in obj_acl]
# assert obj_permission == [
# "FULL_CONTROL",
# "FULL_CONTROL",
# ], "Permission for all groups is FULL_CONTROL"
@allure.title("Test S3 Put 10 nested level object") @allure.title("Test S3 Put 10 nested level object")
def test_s3_put_10_folder(self, bucket, temp_directory, simple_object_size): def test_s3_put_10_folder(self, bucket, temp_directory, simple_object_size):

View file

@ -2,21 +2,17 @@ import os
import allure import allure
import pytest import pytest
from file_helper import generate_file
from python_keywords.container import search_container_by_name
from python_keywords.storage_policy import get_simple_object_copies
from s3_helper import check_objects_in_bucket, object_key_from_file_path, set_bucket_versioning
from pytest_tests.helpers.container import search_container_by_name from steps import s3_gate_bucket, s3_gate_object
from pytest_tests.helpers.file_helper import generate_file from steps.s3_gate_base import TestS3GateBase
from pytest_tests.helpers.s3_helper import (
check_objects_in_bucket,
object_key_from_file_path,
set_bucket_versioning,
)
from pytest_tests.helpers.storage_policy import get_simple_object_copies
from pytest_tests.steps import s3_gate_bucket, s3_gate_object
from pytest_tests.steps.s3_gate_base import TestS3GateBase
def pytest_generate_tests(metafunc): def pytest_generate_tests(metafunc):
policy = f"{os.getcwd()}/pytest_tests/resources/files/policy.json" policy = f"{os.getcwd()}/robot/resources/files/policy.json"
if "s3_client" in metafunc.fixturenames: if "s3_client" in metafunc.fixturenames:
metafunc.parametrize( metafunc.parametrize(
"s3_client", "s3_client",
@ -99,7 +95,7 @@ class TestS3GatePolicy(TestS3GateBase):
s3_gate_bucket.get_bucket_policy(self.s3_client, bucket) s3_gate_bucket.get_bucket_policy(self.s3_client, bucket)
with allure.step("Put new policy"): with allure.step("Put new policy"):
custom_policy = f"file://{os.getcwd()}/pytest_tests/resources/files/bucket_policy.json" custom_policy = f"file://{os.getcwd()}/robot/resources/files/bucket_policy.json"
custom_policy = { custom_policy = {
"Version": "2008-10-17", "Version": "2008-10-17",
"Id": "aaaa-bbbb-cccc-dddd", "Id": "aaaa-bbbb-cccc-dddd",

View file

@ -1,18 +1,16 @@
import os
import uuid
from random import choice from random import choice
from string import ascii_letters from string import ascii_letters
from typing import Tuple from typing import Tuple
import allure import allure
import pytest import pytest
from file_helper import generate_file
from s3_helper import check_tags_by_bucket, check_tags_by_object, object_key_from_file_path
from pytest_tests.helpers.file_helper import generate_file from steps import s3_gate_bucket, s3_gate_object
from pytest_tests.helpers.s3_helper import ( from steps.s3_gate_base import TestS3GateBase
check_tags_by_bucket,
check_tags_by_object,
object_key_from_file_path,
)
from pytest_tests.steps import s3_gate_bucket, s3_gate_object
from pytest_tests.steps.s3_gate_base import TestS3GateBase
def pytest_generate_tests(metafunc): def pytest_generate_tests(metafunc):

View file

@ -2,11 +2,11 @@ import os
import allure import allure
import pytest import pytest
from file_helper import generate_file, generate_file_with_content
from s3_helper import set_bucket_versioning
from pytest_tests.helpers.file_helper import generate_file, generate_file_with_content from steps import s3_gate_bucket, s3_gate_object
from pytest_tests.helpers.s3_helper import set_bucket_versioning from steps.s3_gate_base import TestS3GateBase
from pytest_tests.steps import s3_gate_bucket, s3_gate_object
from pytest_tests.steps.s3_gate_base import TestS3GateBase
def pytest_generate_tests(metafunc): def pytest_generate_tests(metafunc):

View file

@ -5,11 +5,10 @@ from re import match
import allure import allure
import pytest import pytest
import requests import requests
from frostfs_testlib.hosting import Hosting from binary_version_helper import get_remote_binaries_versions
from common import BIN_VERSIONS_FILE
from pytest_tests.helpers.binary_version import get_remote_binaries_versions from env_properties import read_env_properties, save_env_properties
from pytest_tests.helpers.env_properties import read_env_properties, save_env_properties from neofs_testlib.hosting import Hosting
from pytest_tests.resources.common import BIN_VERSIONS_FILE
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")

View file

@ -1,6 +1,5 @@
import pytest import pytest
from wallet import WalletFactory, WalletFile
from pytest_tests.helpers.wallet import WalletFactory, WalletFile
@pytest.fixture(scope="module") @pytest.fixture(scope="module")

View file

@ -2,15 +2,15 @@ import random
import allure import allure
import pytest import pytest
from frostfs_testlib.resources.common import SESSION_NOT_FOUND from cluster_test_base import ClusterTestBase
from frostfs_testlib.utils import wallet_utils from common import WALLET_PASS
from file_helper import generate_file
from grpc_responses import SESSION_NOT_FOUND
from neofs_testlib.utils.wallet import get_last_address_from_wallet
from python_keywords.container import create_container
from python_keywords.neofs_verbs import delete_object, put_object, put_object_to_random_node
from pytest_tests.helpers.container import create_container from steps.session_token import create_session_token
from pytest_tests.helpers.file_helper import generate_file
from pytest_tests.helpers.frostfs_verbs import delete_object, put_object, put_object_to_random_node
from pytest_tests.resources.common import WALLET_PASS
from pytest_tests.steps.cluster_test_base import ClusterTestBase
from pytest_tests.steps.session_token import create_session_token
@pytest.mark.sanity @pytest.mark.sanity
@ -38,7 +38,7 @@ class TestDynamicObjectSession(ClusterTestBase):
with allure.step("Init wallet"): with allure.step("Init wallet"):
wallet = default_wallet wallet = default_wallet
address = wallet_utils.get_last_address_from_wallet(wallet, "") address = get_last_address_from_wallet(wallet, "")
with allure.step("Nodes Settlements"): with allure.step("Nodes Settlements"):
( (

View file

@ -2,20 +2,15 @@ import logging
import allure import allure
import pytest import pytest
from frostfs_testlib.resources.common import ( from cluster import Cluster
EXPIRED_SESSION_TOKEN, from cluster_test_base import ClusterTestBase
MALFORMED_REQUEST, from epoch import ensure_fresh_epoch
OBJECT_ACCESS_DENIED, from file_helper import generate_file
OBJECT_NOT_FOUND, from grpc_responses import MALFORMED_REQUEST, OBJECT_ACCESS_DENIED, OBJECT_NOT_FOUND
) from neofs_testlib.shell import Shell
from frostfs_testlib.shell import Shell
from pytest import FixtureRequest from pytest import FixtureRequest
from python_keywords.container import create_container
from pytest_tests.helpers.cluster import Cluster from python_keywords.neofs_verbs import (
from pytest_tests.helpers.container import create_container
from pytest_tests.helpers.epoch import ensure_fresh_epoch
from pytest_tests.helpers.file_helper import generate_file
from pytest_tests.helpers.frostfs_verbs import (
delete_object, delete_object,
get_object, get_object,
get_object_from_random_node, get_object_from_random_node,
@ -25,11 +20,11 @@ from pytest_tests.helpers.frostfs_verbs import (
put_object_to_random_node, put_object_to_random_node,
search_object, search_object,
) )
from pytest_tests.helpers.storage_object_info import StorageObjectInfo from test_control import expect_not_raises
from pytest_tests.helpers.test_control import expect_not_raises from wallet import WalletFile
from pytest_tests.helpers.wallet import WalletFile
from pytest_tests.steps.cluster_test_base import ClusterTestBase from helpers.storage_object_info import StorageObjectInfo
from pytest_tests.steps.session_token import ( from steps.session_token import (
INVALID_SIGNATURE, INVALID_SIGNATURE,
UNRELATED_CONTAINER, UNRELATED_CONTAINER,
UNRELATED_KEY, UNRELATED_KEY,
@ -41,7 +36,7 @@ from pytest_tests.steps.session_token import (
get_object_signed_token, get_object_signed_token,
sign_session_token, sign_session_token,
) )
from pytest_tests.steps.storage_object import delete_objects from steps.storage_object import delete_objects
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
@ -187,6 +182,7 @@ class TestObjectStaticSession(ClusterTestBase):
) )
@allure.title("Validate static session with range operations") @allure.title("Validate static session with range operations")
@pytest.mark.static_session
@pytest.mark.parametrize( @pytest.mark.parametrize(
"method_under_test,verb", "method_under_test,verb",
[(get_range, ObjectVerb.RANGE), (get_range_hash, ObjectVerb.RANGEHASH)], [(get_range, ObjectVerb.RANGE), (get_range_hash, ObjectVerb.RANGEHASH)],
@ -226,6 +222,7 @@ class TestObjectStaticSession(ClusterTestBase):
) )
@allure.title("Validate static session with search operation") @allure.title("Validate static session with search operation")
@pytest.mark.static_session
@pytest.mark.xfail @pytest.mark.xfail
# (see https://github.com/nspcc-dev/neofs-node/issues/2030) # (see https://github.com/nspcc-dev/neofs-node/issues/2030)
def test_static_session_search( def test_static_session_search(
@ -253,6 +250,7 @@ class TestObjectStaticSession(ClusterTestBase):
assert expected_object_ids == actual_object_ids assert expected_object_ids == actual_object_ids
@allure.title("Validate static session with object id not in session") @allure.title("Validate static session with object id not in session")
@pytest.mark.static_session
def test_static_session_unrelated_object( def test_static_session_unrelated_object(
self, self,
user_wallet: WalletFile, user_wallet: WalletFile,
@ -277,6 +275,7 @@ class TestObjectStaticSession(ClusterTestBase):
) )
@allure.title("Validate static session with user id not in session") @allure.title("Validate static session with user id not in session")
@pytest.mark.static_session
def test_static_session_head_unrelated_user( def test_static_session_head_unrelated_user(
self, self,
stranger_wallet: WalletFile, stranger_wallet: WalletFile,
@ -303,6 +302,7 @@ class TestObjectStaticSession(ClusterTestBase):
) )
@allure.title("Validate static session with wrong verb in session") @allure.title("Validate static session with wrong verb in session")
@pytest.mark.static_session
def test_static_session_head_wrong_verb( def test_static_session_head_wrong_verb(
self, self,
user_wallet: WalletFile, user_wallet: WalletFile,
@ -329,6 +329,7 @@ class TestObjectStaticSession(ClusterTestBase):
) )
@allure.title("Validate static session with container id not in session") @allure.title("Validate static session with container id not in session")
@pytest.mark.static_session
def test_static_session_unrelated_container( def test_static_session_unrelated_container(
self, self,
user_wallet: WalletFile, user_wallet: WalletFile,
@ -356,6 +357,7 @@ class TestObjectStaticSession(ClusterTestBase):
) )
@allure.title("Validate static session which signed by another wallet") @allure.title("Validate static session which signed by another wallet")
@pytest.mark.static_session
def test_static_session_signed_by_other( def test_static_session_signed_by_other(
self, self,
owner_wallet: WalletFile, owner_wallet: WalletFile,
@ -394,6 +396,7 @@ class TestObjectStaticSession(ClusterTestBase):
) )
@allure.title("Validate static session which signed for another container") @allure.title("Validate static session which signed for another container")
@pytest.mark.static_session
def test_static_session_signed_for_other_container( def test_static_session_signed_for_other_container(
self, self,
owner_wallet: WalletFile, owner_wallet: WalletFile,
@ -432,6 +435,7 @@ class TestObjectStaticSession(ClusterTestBase):
) )
@allure.title("Validate static session which wasn't signed") @allure.title("Validate static session which wasn't signed")
@pytest.mark.static_session
def test_static_session_without_sign( def test_static_session_without_sign(
self, self,
owner_wallet: WalletFile, owner_wallet: WalletFile,
@ -468,6 +472,7 @@ class TestObjectStaticSession(ClusterTestBase):
) )
@allure.title("Validate static session which expires at next epoch") @allure.title("Validate static session which expires at next epoch")
@pytest.mark.static_session
def test_static_session_expiration_at_next( def test_static_session_expiration_at_next(
self, self,
owner_wallet: WalletFile, owner_wallet: WalletFile,
@ -489,56 +494,40 @@ class TestObjectStaticSession(ClusterTestBase):
object_id = storage_objects[0].oid object_id = storage_objects[0].oid
expiration = Lifetime(epoch + 1, epoch, epoch) expiration = Lifetime(epoch + 1, epoch, epoch)
with allure.step("Create session token"): token_expire_at_next_epoch = get_object_signed_token(
token_expire_at_next_epoch = get_object_signed_token( owner_wallet,
owner_wallet, user_wallet,
user_wallet, container,
storage_objects,
ObjectVerb.HEAD,
self.shell,
temp_directory,
expiration,
)
head_object(
user_wallet.path,
container,
object_id,
self.shell,
self.cluster.default_rpc_endpoint,
session=token_expire_at_next_epoch,
)
self.tick_epoch()
with pytest.raises(Exception, match=MALFORMED_REQUEST):
head_object(
user_wallet.path,
container, container,
storage_objects, object_id,
ObjectVerb.HEAD,
self.shell, self.shell,
temp_directory, self.cluster.default_rpc_endpoint,
expiration, session=token_expire_at_next_epoch,
) )
with allure.step("Object should be available with session token after token creation"):
with expect_not_raises():
head_object(
user_wallet.path,
container,
object_id,
self.shell,
self.cluster.default_rpc_endpoint,
session=token_expire_at_next_epoch,
)
with allure.step(
"Object should be available at last epoch before session token expiration"
):
self.tick_epoch()
with expect_not_raises():
head_object(
user_wallet.path,
container,
object_id,
self.shell,
self.cluster.default_rpc_endpoint,
session=token_expire_at_next_epoch,
)
with allure.step("Object should NOT be available after session token expiration epoch"):
self.tick_epoch()
with pytest.raises(Exception, match=EXPIRED_SESSION_TOKEN):
head_object(
user_wallet.path,
container,
object_id,
self.shell,
self.cluster.default_rpc_endpoint,
session=token_expire_at_next_epoch,
)
@allure.title("Validate static session which is valid starting from next epoch") @allure.title("Validate static session which is valid starting from next epoch")
@pytest.mark.static_session
def test_static_session_start_at_next( def test_static_session_start_at_next(
self, self,
owner_wallet: WalletFile, owner_wallet: WalletFile,
@ -560,70 +549,50 @@ class TestObjectStaticSession(ClusterTestBase):
object_id = storage_objects[0].oid object_id = storage_objects[0].oid
expiration = Lifetime(epoch + 2, epoch + 1, epoch) expiration = Lifetime(epoch + 2, epoch + 1, epoch)
with allure.step("Create session token"): token_start_at_next_epoch = get_object_signed_token(
token_start_at_next_epoch = get_object_signed_token( owner_wallet,
owner_wallet, user_wallet,
user_wallet, container,
storage_objects,
ObjectVerb.HEAD,
self.shell,
temp_directory,
expiration,
)
with pytest.raises(Exception, match=MALFORMED_REQUEST):
head_object(
user_wallet.path,
container, container,
storage_objects, object_id,
ObjectVerb.HEAD,
self.shell, self.shell,
temp_directory, self.cluster.default_rpc_endpoint,
expiration, session=token_start_at_next_epoch,
) )
with allure.step("Object should NOT be available with session token after token creation"): self.tick_epoch()
with pytest.raises(Exception, match=MALFORMED_REQUEST): head_object(
head_object( user_wallet.path,
user_wallet.path, container,
container, object_id,
object_id, self.shell,
self.shell, self.cluster.default_rpc_endpoint,
self.cluster.default_rpc_endpoint, session=token_start_at_next_epoch,
session=token_start_at_next_epoch, )
)
with allure.step( self.tick_epoch()
"Object should be available with session token starting from token nbf epoch" with pytest.raises(Exception, match=MALFORMED_REQUEST):
): head_object(
self.tick_epoch() user_wallet.path,
with expect_not_raises(): container,
head_object( object_id,
user_wallet.path, self.shell,
container, self.cluster.default_rpc_endpoint,
object_id, session=token_start_at_next_epoch,
self.shell, )
self.cluster.default_rpc_endpoint,
session=token_start_at_next_epoch,
)
with allure.step(
"Object should be available at last epoch before session token expiration"
):
self.tick_epoch()
with expect_not_raises():
head_object(
user_wallet.path,
container,
object_id,
self.shell,
self.cluster.default_rpc_endpoint,
session=token_start_at_next_epoch,
)
with allure.step("Object should NOT be available after session token expiration epoch"):
self.tick_epoch()
with pytest.raises(Exception, match=EXPIRED_SESSION_TOKEN):
head_object(
user_wallet.path,
container,
object_id,
self.shell,
self.cluster.default_rpc_endpoint,
session=token_start_at_next_epoch,
)
@allure.title("Validate static session which is already expired") @allure.title("Validate static session which is already expired")
@pytest.mark.static_session
def test_static_session_already_expired( def test_static_session_already_expired(
self, self,
owner_wallet: WalletFile, owner_wallet: WalletFile,
@ -656,7 +625,7 @@ class TestObjectStaticSession(ClusterTestBase):
expiration, expiration,
) )
with pytest.raises(Exception, match=EXPIRED_SESSION_TOKEN): with pytest.raises(Exception, match=MALFORMED_REQUEST):
head_object( head_object(
user_wallet.path, user_wallet.path,
container, container,
@ -717,6 +686,7 @@ class TestObjectStaticSession(ClusterTestBase):
) )
@allure.title("Validate static session which is issued in future epoch") @allure.title("Validate static session which is issued in future epoch")
@pytest.mark.static_session
def test_static_session_invalid_issued_epoch( def test_static_session_invalid_issued_epoch(
self, self,
owner_wallet: WalletFile, owner_wallet: WalletFile,

View file

@ -1,9 +1,8 @@
import allure import allure
import pytest import pytest
from frostfs_testlib.resources.common import PUBLIC_ACL from file_helper import generate_file
from frostfs_testlib.shell import Shell from neofs_testlib.shell import Shell
from python_keywords.acl import (
from pytest_tests.helpers.acl import (
EACLAccess, EACLAccess,
EACLOperation, EACLOperation,
EACLRole, EACLRole,
@ -12,17 +11,18 @@ from pytest_tests.helpers.acl import (
set_eacl, set_eacl,
wait_for_cache_expired, wait_for_cache_expired,
) )
from pytest_tests.helpers.container import ( from python_keywords.container import (
create_container, create_container,
delete_container, delete_container,
get_container, get_container,
list_containers, list_containers,
) )
from pytest_tests.helpers.file_helper import generate_file from python_keywords.object_access import can_put_object
from pytest_tests.helpers.object_access import can_put_object from wallet import WalletFile
from pytest_tests.helpers.wallet import WalletFile from wellknown_acl import PUBLIC_ACL
from pytest_tests.steps.cluster_test_base import ClusterTestBase
from pytest_tests.steps.session_token import ContainerVerb, get_container_signed_token from steps.cluster_test_base import ClusterTestBase
from steps.session_token import ContainerVerb, get_container_signed_token
@pytest.mark.static_session_container @pytest.mark.static_session_container

View file

@ -7,13 +7,12 @@ from io import StringIO
import allure import allure
import pytest import pytest
import yaml import yaml
from cluster import Cluster, StorageNode
from common import WALLET_CONFIG
from configobj import ConfigObj from configobj import ConfigObj
from frostfs_testlib.cli import FrostfsCli from neofs_testlib.cli import NeofsCli
from pytest_tests.helpers.cluster import Cluster, StorageNode SHARD_PREFIX = "NEOFS_STORAGE_SHARD_"
from pytest_tests.resources.common import CLI_DEFAULT_TIMEOUT, WALLET_CONFIG
SHARD_PREFIX = "FROSTFS_STORAGE_SHARD_"
BLOBSTOR_PREFIX = "_BLOBSTOR_" BLOBSTOR_PREFIX = "_BLOBSTOR_"
@ -135,15 +134,14 @@ class TestControlShard:
wallet_password = node.get_wallet_password() wallet_password = node.get_wallet_password()
control_endpoint = node.get_control_endpoint() control_endpoint = node.get_control_endpoint()
cli_config = node.host.get_cli_config("frostfs-cli") cli_config = node.host.get_cli_config("neofs-cli")
cli = FrostfsCli(node.host.get_shell(), cli_config.exec_path, WALLET_CONFIG) cli = NeofsCli(node.host.get_shell(), cli_config.exec_path, WALLET_CONFIG)
result = cli.shards.list( result = cli.shards.list(
endpoint=control_endpoint, endpoint=control_endpoint,
wallet=wallet_path, wallet=wallet_path,
wallet_password=wallet_password, wallet_password=wallet_password,
json_mode=True, json_mode=True,
timeout=CLI_DEFAULT_TIMEOUT,
) )
return [Shard.from_object(shard) for shard in json.loads(result.stdout.split(">", 1)[1])] return [Shard.from_object(shard) for shard in json.loads(result.stdout.split(">", 1)[1])]

View file

@ -1,47 +0,0 @@
import os
import shutil
from datetime import datetime
import allure
import pytest
from pytest_tests.steps.cluster_test_base import ClusterTestBase
class TestLogs(ClusterTestBase):
@pytest.mark.logs_after_session
def test_logs_after_session(self, temp_directory: str, session_start_time: datetime):
"""
This test automatically added to any test run to check logs from cluster for critical errors.
"""
end_time = datetime.utcnow()
logs_dir = os.path.join(temp_directory, "logs")
os.makedirs(logs_dir)
issues_regex = r"\Wpanic\W|\Woom\W|\Wtoo many open files\W"
hosts_with_problems = []
for host in self.cluster.hosts:
with allure.step(f"Check logs on {host.config.address}"):
if host.is_message_in_logs(issues_regex, session_start_time, end_time):
hosts_with_problems.append(host.config.address)
host.dump_logs(
logs_dir,
since=session_start_time,
until=end_time,
filter_regex=issues_regex,
)
if hosts_with_problems:
self._attach_logs(logs_dir)
assert (
not hosts_with_problems
), f"The following hosts contains contain critical errors in system logs: {', '.join(hosts_with_problems)}"
def _attach_logs(self, logs_dir: str) -> None:
# Zip all files and attach to Allure because it is more convenient to download a single
# zip with all logs rather than mess with individual logs files per service or node
logs_zip_file_path = shutil.make_archive(logs_dir, "zip", logs_dir)
allure.attach.file(logs_zip_file_path, name="logs.zip", extension="zip")

View file

@ -0,0 +1 @@
password: ""

View file

@ -1,16 +1,71 @@
aiodns==3.0.0
aiohttp==3.7.4.post0
aioresponses==0.7.2
allure-pytest==2.9.45 allure-pytest==2.9.45
allure-python-commons==2.9.45 allure-python-commons==2.9.45
async-timeout==3.0.1
asynctest==0.13.0
attrs==21.4.0
base58==2.1.0 base58==2.1.0
bitarray==2.3.4
black==22.8.0
boto3==1.16.33 boto3==1.16.33
botocore==1.19.33 botocore==1.19.33
certifi==2022.5.18
cffi==1.15.0
chardet==4.0.0
charset-normalizer==2.0.12
coverage==6.3.3
configobj==5.0.6 configobj==5.0.6
frostfs-testlib==1.3.1 docker==4.4.0
neo-mamba==1.0.0 docutils==0.17.1
Events==0.4
flake8==4.0.1
idna==3.3
iniconfig==1.1.1
isort==5.10.1
jmespath==0.10.0
jsonschema==4.5.1
lz4==3.1.3
mccabe==0.6.1
mmh3==3.0.0
multidict==6.0.2
mypy==0.950
mypy-extensions==0.4.3
neo-mamba==0.10.0
neo3crypto==0.2.1
neo3vm==0.9.0
neo3vm-stubs==0.9.0
neofs-testlib==0.8.1
netaddr==0.8.0
orjson==3.6.8
packaging==21.3
paramiko==2.10.3
pexpect==4.8.0 pexpect==4.8.0
pyyaml==6.0 pluggy==1.0.0
pre-commit==2.20.0
ptyprocess==0.7.0
py==1.11.0
pybiginteger==1.2.6
pybiginteger-stubs==1.2.6
pycares==4.1.2
pycodestyle==2.8.0
pycparser==2.21
pycryptodome==3.11.0
pyflakes==2.4.0
pyparsing==3.0.9
pyrsistent==0.18.1
pytest==7.1.2 pytest==7.1.2
pytest-lazy-fixture==0.6.3 pytest-lazy-fixture==0.6.3
python-dateutil==2.8.2 python-dateutil==2.8.2
pyyaml==6.0
requests==2.28.0 requests==2.28.0
robotframework==4.1.2
s3transfer==0.3.7
six==1.16.0
tenacity==8.0.1 tenacity==8.0.1
urllib3==1.26.9 tomli==2.0.1
typing-extensions==4.2.0
urllib3==1.26.9
websocket-client==1.3.2
yarl==1.7.2

View file

@ -1,2 +0,0 @@
pre-commit==2.20.0
isort==5.12.0

View file

@ -10,15 +10,14 @@ from typing import Any, Dict, List, Optional, Union
import allure import allure
import base58 import base58
from frostfs_testlib.cli import FrostfsCli from common import ASSETS_DIR, NEOFS_CLI_EXEC, WALLET_CONFIG
from frostfs_testlib.shell import Shell from data_formatters import get_wallet_public_key
from frostfs_testlib.utils import wallet_utils from neofs_testlib.cli import NeofsCli
from neofs_testlib.shell import Shell
from pytest_tests.resources.common import ASSETS_DIR, FROSTFS_CLI_EXEC, WALLET_CONFIG
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
EACL_LIFETIME = 100500 EACL_LIFETIME = 100500
FROSTFS_CONTRACT_CACHE_TIMEOUT = 30 NEOFS_CONTRACT_CACHE_TIMEOUT = 30
class EACLOperation(Enum): class EACLOperation(Enum):
@ -45,7 +44,7 @@ class EACLRole(Enum):
class EACLHeaderType(Enum): class EACLHeaderType(Enum):
REQUEST = "req" # Filter request headers REQUEST = "req" # Filter request headers
OBJECT = "obj" # Filter object headers OBJECT = "obj" # Filter object headers
SERVICE = "SERVICE" # Filter service headers. These are not processed by FrostFS nodes and exist for service use only SERVICE = "SERVICE" # Filter service headers. These are not processed by NeoFS nodes and exist for service use only
class EACLMatchType(Enum): class EACLMatchType(Enum):
@ -111,14 +110,14 @@ class EACLRule:
role = ( role = (
self.role.value self.role.value
if isinstance(self.role, EACLRole) if isinstance(self.role, EACLRole)
else f'pubkey:{wallet_utils.get_wallet_public_key(self.role, "")}' else f'pubkey:{get_wallet_public_key(self.role, "")}'
) )
return f'{self.access.value} {self.operation.value} {self.filters or ""} {role}' return f'{self.access.value} {self.operation.value} {self.filters or ""} {role}'
@allure.title("Get extended ACL") @allure.title("Get extended ACL")
def get_eacl(wallet_path: str, cid: str, shell: Shell, endpoint: str) -> Optional[str]: def get_eacl(wallet_path: str, cid: str, shell: Shell, endpoint: str) -> Optional[str]:
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, WALLET_CONFIG) cli = NeofsCli(shell, NEOFS_CLI_EXEC, WALLET_CONFIG)
try: try:
result = cli.container.get_eacl(wallet=wallet_path, rpc_endpoint=endpoint, cid=cid) result = cli.container.get_eacl(wallet=wallet_path, rpc_endpoint=endpoint, cid=cid)
except RuntimeError as exc: except RuntimeError as exc:
@ -139,7 +138,7 @@ def set_eacl(
endpoint: str, endpoint: str,
session_token: Optional[str] = None, session_token: Optional[str] = None,
) -> None: ) -> None:
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, WALLET_CONFIG) cli = NeofsCli(shell, NEOFS_CLI_EXEC, WALLET_CONFIG)
cli.container.set_eacl( cli.container.set_eacl(
wallet=wallet_path, wallet=wallet_path,
rpc_endpoint=endpoint, rpc_endpoint=endpoint,
@ -157,7 +156,7 @@ def _encode_cid_for_eacl(cid: str) -> str:
def create_eacl(cid: str, rules_list: List[EACLRule], shell: Shell) -> str: def create_eacl(cid: str, rules_list: List[EACLRule], shell: Shell) -> str:
table_file_path = os.path.join(os.getcwd(), ASSETS_DIR, f"eacl_table_{str(uuid.uuid4())}.json") table_file_path = os.path.join(os.getcwd(), ASSETS_DIR, f"eacl_table_{str(uuid.uuid4())}.json")
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, WALLET_CONFIG) cli = NeofsCli(shell, NEOFS_CLI_EXEC, WALLET_CONFIG)
cli.acl.extended_create(cid=cid, out=table_file_path, rule=rules_list) cli.acl.extended_create(cid=cid, out=table_file_path, rule=rules_list)
with open(table_file_path, "r") as file: with open(table_file_path, "r") as file:
@ -245,7 +244,7 @@ def eacl_rules(access: str, verbs: list, user: str) -> list[str]:
(list): a list of eACL rules (list): a list of eACL rules
""" """
if user not in ("others", "user"): if user not in ("others", "user"):
pubkey = wallet_utils.get_wallet_public_key(user, wallet_password="") pubkey = get_wallet_public_key(user, wallet_password="")
user = f"pubkey:{pubkey}" user = f"pubkey:{pubkey}"
rules = [] rules = []
@ -258,17 +257,15 @@ def eacl_rules(access: str, verbs: list, user: str) -> list[str]:
def sign_bearer( def sign_bearer(
shell: Shell, wallet_path: str, eacl_rules_file_from: str, eacl_rules_file_to: str, json: bool shell: Shell, wallet_path: str, eacl_rules_file_from: str, eacl_rules_file_to: str, json: bool
) -> None: ) -> None:
frostfscli = FrostfsCli( neofscli = NeofsCli(shell=shell, neofs_cli_exec_path=NEOFS_CLI_EXEC, config_file=WALLET_CONFIG)
shell=shell, frostfs_cli_exec_path=FROSTFS_CLI_EXEC, config_file=WALLET_CONFIG neofscli.util.sign_bearer_token(
)
frostfscli.util.sign_bearer_token(
wallet=wallet_path, from_file=eacl_rules_file_from, to_file=eacl_rules_file_to, json=json wallet=wallet_path, from_file=eacl_rules_file_from, to_file=eacl_rules_file_to, json=json
) )
@allure.title("Wait for eACL cache expired") @allure.title("Wait for eACL cache expired")
def wait_for_cache_expired(): def wait_for_cache_expired():
sleep(FROSTFS_CONTRACT_CACHE_TIMEOUT) sleep(NEOFS_CONTRACT_CACHE_TIMEOUT)
return return

View file

@ -1,7 +1,7 @@
#!/usr/bin/python3.10 #!/usr/bin/python3.9
""" """
Helper functions to use with `frostfs-cli`, `neo-go` and other CLIs. Helper functions to use with `neofs-cli`, `neo-go` and other CLIs.
""" """
import json import json
import logging import logging

View file

@ -14,31 +14,26 @@ import logging
from typing import Optional, Tuple from typing import Optional, Tuple
import allure import allure
from frostfs_testlib.shell import Shell import neofs_verbs
from cluster import Cluster, StorageNode
from pytest_tests.helpers import frostfs_verbs from common import WALLET_CONFIG
from pytest_tests.helpers.cluster import Cluster, StorageNode from neofs_testlib.shell import Shell
from pytest_tests.helpers.frostfs_verbs import head_object from neofs_verbs import head_object
from pytest_tests.helpers.storage_object_info import StorageObjectInfo from storage_object import StorageObjectInfo
from pytest_tests.resources.common import CLI_DEFAULT_TIMEOUT, WALLET_CONFIG
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
def get_storage_object_chunks( def get_storage_object_chunks(
storage_object: StorageObjectInfo, storage_object: StorageObjectInfo, shell: Shell, cluster: Cluster
shell: Shell,
cluster: Cluster,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> list[str]: ) -> list[str]:
""" """
Get complex object split objects ids (no linker object) Get complex object split objects ids (no linker object)
Args: Args:
storage_object: storage_object to get it's chunks storage_object: storage_object to get it's chunks
shell: client shell to do cmd requests shell: client shell to do cmd requests
cluster: cluster object under test cluster: cluster object under test
timeout: Timeout for an operation.
Returns: Returns:
list of object ids of complex object chunks list of object ids of complex object chunks
@ -52,7 +47,6 @@ def get_storage_object_chunks(
shell, shell,
cluster.storage_nodes, cluster.storage_nodes,
is_direct=False, is_direct=False,
timeout=timeout,
) )
head = head_object( head = head_object(
storage_object.wallet_file_path, storage_object.wallet_file_path,
@ -60,7 +54,6 @@ def get_storage_object_chunks(
split_object_id, split_object_id,
shell, shell,
cluster.default_rpc_endpoint, cluster.default_rpc_endpoint,
timeout=timeout,
) )
chunks_object_ids = [] chunks_object_ids = []
@ -71,10 +64,7 @@ def get_storage_object_chunks(
def get_complex_object_split_ranges( def get_complex_object_split_ranges(
storage_object: StorageObjectInfo, storage_object: StorageObjectInfo, shell: Shell, cluster: Cluster
shell: Shell,
cluster: Cluster,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> list[Tuple[int, int]]: ) -> list[Tuple[int, int]]:
""" """
@ -84,10 +74,9 @@ def get_complex_object_split_ranges(
[(0, 30), (30, 30), (60, 30), (90, 10)] [(0, 30), (30, 30), (60, 30), (90, 10)]
Args: Args:
storage_object: storage_object to get it's chunks storage_object: storage_object to get it's chunks
shell: client shell to do cmd requests shell: client shell to do cmd requests
cluster: cluster object under test cluster: cluster object under test
timeout: Timeout for an operation.
Returns: Returns:
list of object ids of complex object chunks list of object ids of complex object chunks
@ -103,7 +92,6 @@ def get_complex_object_split_ranges(
chunk_id, chunk_id,
shell, shell,
cluster.default_rpc_endpoint, cluster.default_rpc_endpoint,
timeout=timeout,
) )
length = int(head["header"]["payloadLength"]) length = int(head["header"]["payloadLength"])
@ -124,7 +112,6 @@ def get_link_object(
bearer: str = "", bearer: str = "",
wallet_config: str = WALLET_CONFIG, wallet_config: str = WALLET_CONFIG,
is_direct: bool = True, is_direct: bool = True,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
): ):
""" """
Args: Args:
@ -135,10 +122,9 @@ def get_link_object(
shell: executor for cli command shell: executor for cli command
nodes: list of nodes to do search on nodes: list of nodes to do search on
bearer (optional, str): path to Bearer token file bearer (optional, str): path to Bearer token file
wallet_config (optional, str): path to the frostfs-cli config file wallet_config (optional, str): path to the neofs-cli config file
is_direct: send request directly to the node or not; this flag is_direct: send request directly to the node or not; this flag
turns into `--ttl 1` key turns into `--ttl 1` key
timeout: Timeout for an operation.
Returns: Returns:
(str): Link Object ID (str): Link Object ID
When no Link Object ID is found after all Storage Nodes polling, When no Link Object ID is found after all Storage Nodes polling,
@ -147,7 +133,7 @@ def get_link_object(
for node in nodes: for node in nodes:
endpoint = node.get_rpc_endpoint() endpoint = node.get_rpc_endpoint()
try: try:
resp = frostfs_verbs.head_object( resp = neofs_verbs.head_object(
wallet, wallet,
cid, cid,
oid, oid,
@ -157,7 +143,6 @@ def get_link_object(
is_direct=is_direct, is_direct=is_direct,
bearer=bearer, bearer=bearer,
wallet_config=wallet_config, wallet_config=wallet_config,
timeout=timeout,
) )
if resp["link"]: if resp["link"]:
return resp["link"] return resp["link"]
@ -169,12 +154,7 @@ def get_link_object(
@allure.step("Get Last Object") @allure.step("Get Last Object")
def get_last_object( def get_last_object(
wallet: str, wallet: str, cid: str, oid: str, shell: Shell, nodes: list[StorageNode]
cid: str,
oid: str,
shell: Shell,
nodes: list[StorageNode],
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> Optional[str]: ) -> Optional[str]:
""" """
Args: Args:
@ -184,7 +164,6 @@ def get_last_object(
oid (str): Large Object ID oid (str): Large Object ID
shell: executor for cli command shell: executor for cli command
nodes: list of nodes to do search on nodes: list of nodes to do search on
timeout: Timeout for an operation.
Returns: Returns:
(str): Last Object ID (str): Last Object ID
When no Last Object ID is found after all Storage Nodes polling, When no Last Object ID is found after all Storage Nodes polling,
@ -193,15 +172,8 @@ def get_last_object(
for node in nodes: for node in nodes:
endpoint = node.get_rpc_endpoint() endpoint = node.get_rpc_endpoint()
try: try:
resp = frostfs_verbs.head_object( resp = neofs_verbs.head_object(
wallet, wallet, cid, oid, shell=shell, endpoint=endpoint, is_raw=True, is_direct=True
cid,
oid,
shell=shell,
endpoint=endpoint,
is_raw=True,
is_direct=True,
timeout=timeout,
) )
if resp["lastPart"]: if resp["lastPart"]:
return resp["lastPart"] return resp["lastPart"]

View file

@ -0,0 +1,239 @@
#!/usr/bin/python3.9
"""
This module contains keywords that utilize `neofs-cli container` commands.
"""
import json
import logging
from time import sleep
from typing import Optional, Union
import allure
import json_transformers
from common import NEOFS_CLI_EXEC, WALLET_CONFIG
from neofs_testlib.cli import NeofsCli
from neofs_testlib.shell import Shell
logger = logging.getLogger("NeoLogger")
DEFAULT_PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 4 FROM * AS X"
SINGLE_PLACEMENT_RULE = "REP 1 IN X CBF 1 SELECT 4 FROM * AS X"
REP_2_FOR_3_NODES_PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 3 FROM * AS X"
@allure.step("Create Container")
def create_container(
wallet: str,
shell: Shell,
endpoint: str,
rule: str = DEFAULT_PLACEMENT_RULE,
basic_acl: str = "",
attributes: Optional[dict] = None,
session_token: str = "",
session_wallet: str = "",
name: str = None,
options: dict = None,
await_mode: bool = True,
wait_for_creation: bool = True,
) -> str:
"""
A wrapper for `neofs-cli container create` call.
Args:
wallet (str): a wallet on whose behalf a container is created
rule (optional, str): placement rule for container
basic_acl (optional, str): an ACL for container, will be
appended to `--basic-acl` key
attributes (optional, dict): container attributes , will be
appended to `--attributes` key
session_token (optional, str): a path to session token file
session_wallet(optional, str): a path to the wallet which signed
the session token; this parameter makes sense
when paired with `session_token`
shell: executor for cli command
endpoint: NeoFS endpoint to send request to, appends to `--rpc-endpoint` key
options (optional, dict): any other options to pass to the call
name (optional, str): container name attribute
await_mode (bool): block execution until container is persisted
wait_for_creation (): Wait for container shows in container list
Returns:
(str): CID of the created container
"""
cli = NeofsCli(shell, NEOFS_CLI_EXEC, WALLET_CONFIG)
result = cli.container.create(
rpc_endpoint=endpoint,
wallet=session_wallet if session_wallet else wallet,
policy=rule,
basic_acl=basic_acl,
attributes=attributes,
name=name,
session=session_token,
await_mode=await_mode,
**options or {},
)
cid = _parse_cid(result.stdout)
logger.info("Container created; waiting until it is persisted in the sidechain")
if wait_for_creation:
wait_for_container_creation(wallet, cid, shell, endpoint)
return cid
def wait_for_container_creation(
wallet: str, cid: str, shell: Shell, endpoint: str, attempts: int = 15, sleep_interval: int = 1
):
for _ in range(attempts):
containers = list_containers(wallet, shell, endpoint)
if cid in containers:
return
logger.info(f"There is no {cid} in {containers} yet; sleep {sleep_interval} and continue")
sleep(sleep_interval)
raise RuntimeError(
f"After {attempts * sleep_interval} seconds container {cid} hasn't been persisted; exiting"
)
def wait_for_container_deletion(
wallet: str, cid: str, shell: Shell, endpoint: str, attempts: int = 30, sleep_interval: int = 1
):
for _ in range(attempts):
try:
get_container(wallet, cid, shell=shell, endpoint=endpoint)
sleep(sleep_interval)
continue
except Exception as err:
if "container not found" not in str(err):
raise AssertionError(f'Expected "container not found" in error, got\n{err}')
return
raise AssertionError(f"Expected container deleted during {attempts * sleep_interval} sec.")
@allure.step("List Containers")
def list_containers(wallet: str, shell: Shell, endpoint: str) -> list[str]:
"""
A wrapper for `neofs-cli container list` call. It returns all the
available containers for the given wallet.
Args:
wallet (str): a wallet on whose behalf we list the containers
shell: executor for cli command
endpoint: NeoFS endpoint to send request to, appends to `--rpc-endpoint` key
Returns:
(list): list of containers
"""
cli = NeofsCli(shell, NEOFS_CLI_EXEC, WALLET_CONFIG)
result = cli.container.list(rpc_endpoint=endpoint, wallet=wallet)
logger.info(f"Containers: \n{result}")
return result.stdout.split()
@allure.step("Get Container")
def get_container(
wallet: str,
cid: str,
shell: Shell,
endpoint: str,
json_mode: bool = True,
) -> Union[dict, str]:
"""
A wrapper for `neofs-cli container get` call. It extracts container's
attributes and rearranges them into a more compact view.
Args:
wallet (str): path to a wallet on whose behalf we get the container
cid (str): ID of the container to get
shell: executor for cli command
endpoint: NeoFS endpoint to send request to, appends to `--rpc-endpoint` key
json_mode (bool): return container in JSON format
Returns:
(dict, str): dict of container attributes
"""
cli = NeofsCli(shell, NEOFS_CLI_EXEC, WALLET_CONFIG)
result = cli.container.get(rpc_endpoint=endpoint, wallet=wallet, cid=cid, json_mode=json_mode)
if not json_mode:
return result.stdout
container_info = json.loads(result.stdout)
attributes = dict()
for attr in container_info["attributes"]:
attributes[attr["key"]] = attr["value"]
container_info["attributes"] = attributes
container_info["ownerID"] = json_transformers.json_reencode(container_info["ownerID"]["value"])
return container_info
@allure.step("Delete Container")
# TODO: make the error message about a non-found container more user-friendly
# https://github.com/nspcc-dev/neofs-contract/issues/121
def delete_container(
wallet: str,
cid: str,
shell: Shell,
endpoint: str,
force: bool = False,
session_token: Optional[str] = None,
await_mode: bool = False,
) -> None:
"""
A wrapper for `neofs-cli container delete` call.
Args:
wallet (str): path to a wallet on whose behalf we delete the container
cid (str): ID of the container to delete
shell: executor for cli command
endpoint: NeoFS endpoint to send request to, appends to `--rpc-endpoint` key
force (bool): do not check whether container contains locks and remove immediately
session_token: a path to session token file
This function doesn't return anything.
"""
cli = NeofsCli(shell, NEOFS_CLI_EXEC, WALLET_CONFIG)
cli.container.delete(
wallet=wallet,
cid=cid,
rpc_endpoint=endpoint,
force=force,
session=session_token,
await_mode=await_mode,
)
def _parse_cid(output: str) -> str:
"""
Parses container ID from a given CLI output. The input string we expect:
container ID: 2tz86kVTDpJxWHrhw3h6PbKMwkLtBEwoqhHQCKTre1FN
awaiting...
container has been persisted on sidechain
We want to take 'container ID' value from the string.
Args:
output (str): CLI output to parse
Returns:
(str): extracted CID
"""
try:
# taking first line from command's output
first_line = output.split("\n")[0]
except Exception:
first_line = ""
logger.error(f"Got empty output: {output}")
splitted = first_line.split(": ")
if len(splitted) != 2:
raise ValueError(f"no CID was parsed from command output: \t{first_line}")
return splitted[1]
@allure.step("Search container by name")
def search_container_by_name(wallet: str, name: str, shell: Shell, endpoint: str):
list_cids = list_containers(wallet, shell, endpoint)
for cid in list_cids:
cont_info = get_container(wallet, cid, shell, endpoint, True)
if cont_info.get("attributes").get("Name", None) == name:
return cid
return None

View file

@ -1,10 +1,9 @@
from typing import List, Optional from typing import List, Optional
from frostfs_testlib.shell import Shell from acl import EACLOperation
from cluster import Cluster
from pytest_tests.helpers.acl import EACLOperation from neofs_testlib.shell import Shell
from pytest_tests.helpers.cluster import Cluster from python_keywords.object_access import (
from pytest_tests.helpers.object_access import (
can_delete_object, can_delete_object,
can_get_head_object, can_get_head_object,
can_get_object, can_get_object,

View file

@ -0,0 +1,50 @@
import base64
import json
import base58
from neo3 import wallet
def dict_to_attrs(attrs: dict) -> str:
"""
This function takes a dictionary of object's attributes and converts them
into string. The string is passed to `--attributes` key of neofs-cli.
Args:
attrs (dict): object attributes in {"a": "b", "c": "d"} format.
Returns:
(str): string in "a=b,c=d" format.
"""
return ",".join(f"{key}={value}" for key, value in attrs.items())
def __fix_wallet_schema(wallet: dict) -> None:
# Temporary function to fix wallets that do not conform to the schema
# TODO: get rid of it once issue is solved
if "name" not in wallet:
wallet["name"] = None
for account in wallet["accounts"]:
if "extra" not in account:
account["extra"] = None
def get_wallet_public_key(wallet_path: str, wallet_password: str, format: str = "hex") -> str:
# Get public key from wallet file
with open(wallet_path, "r") as file:
wallet_content = json.load(file)
__fix_wallet_schema(wallet_content)
wallet_from_json = wallet.Wallet.from_json(wallet_content, password=wallet_password)
public_key_hex = str(wallet_from_json.accounts[0].public_key)
# Convert public key to specified format
if format == "hex":
return public_key_hex
if format == "base58":
public_key_base58 = base58.b58encode(bytes.fromhex(public_key_hex))
return public_key_base58.decode("utf-8")
if format == "base64":
public_key_base64 = base64.b64encode(bytes.fromhex(public_key_hex))
return public_key_base64.decode("utf-8")
raise ValueError(f"Invalid public key format: {format}")

View file

@ -3,21 +3,20 @@ from time import sleep
from typing import Optional from typing import Optional
import allure import allure
from frostfs_testlib.cli import FrostfsAdm, FrostfsCli, NeoGo from cluster import Cluster, StorageNode
from frostfs_testlib.shell import Shell from common import (
from frostfs_testlib.utils import datetime_utils, wallet_utils
from pytest_tests.helpers.cluster import Cluster, StorageNode
from pytest_tests.helpers.payment_neogo import get_contract_hash
from pytest_tests.helpers.test_control import wait_for_success
from pytest_tests.resources.common import (
CLI_DEFAULT_TIMEOUT,
FROSTFS_ADM_CONFIG_PATH,
FROSTFS_ADM_EXEC,
FROSTFS_CLI_EXEC,
MAINNET_BLOCK_TIME, MAINNET_BLOCK_TIME,
NEOFS_ADM_CONFIG_PATH,
NEOFS_ADM_EXEC,
NEOFS_CLI_EXEC,
NEOGO_EXECUTABLE, NEOGO_EXECUTABLE,
) )
from neofs_testlib.cli import NeofsAdm, NeofsCli, NeoGo
from neofs_testlib.shell import Shell
from neofs_testlib.utils.wallet import get_last_address_from_wallet
from payment_neogo import get_contract_hash
from test_control import wait_for_success
from utility import parse_time
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
@ -54,16 +53,16 @@ def get_epoch(shell: Shell, cluster: Cluster, alive_node: Optional[StorageNode]
wallet_path = alive_node.get_wallet_path() wallet_path = alive_node.get_wallet_path()
wallet_config = alive_node.get_wallet_config_path() wallet_config = alive_node.get_wallet_config_path()
cli = FrostfsCli(shell=shell, frostfs_cli_exec_path=FROSTFS_CLI_EXEC, config_file=wallet_config) cli = NeofsCli(shell=shell, neofs_cli_exec_path=NEOFS_CLI_EXEC, config_file=wallet_config)
epoch = cli.netmap.epoch(endpoint, wallet_path, timeout=CLI_DEFAULT_TIMEOUT) epoch = cli.netmap.epoch(endpoint, wallet_path)
return int(epoch.stdout) return int(epoch.stdout)
@allure.step("Tick Epoch") @allure.step("Tick Epoch")
def tick_epoch(shell: Shell, cluster: Cluster, alive_node: Optional[StorageNode] = None): def tick_epoch(shell: Shell, cluster: Cluster, alive_node: Optional[StorageNode] = None):
""" """
Tick epoch using frostfs-adm or NeoGo if frostfs-adm is not available (DevEnv) Tick epoch using neofs-adm or NeoGo if neofs-adm is not available (DevEnv)
Args: Args:
shell: local shell to make queries about current epoch. Remote shell will be used to tick new one shell: local shell to make queries about current epoch. Remote shell will be used to tick new one
cluster: cluster instance under test cluster: cluster instance under test
@ -73,14 +72,14 @@ def tick_epoch(shell: Shell, cluster: Cluster, alive_node: Optional[StorageNode]
alive_node = alive_node if alive_node else cluster.storage_nodes[0] alive_node = alive_node if alive_node else cluster.storage_nodes[0]
remote_shell = alive_node.host.get_shell() remote_shell = alive_node.host.get_shell()
if FROSTFS_ADM_EXEC and FROSTFS_ADM_CONFIG_PATH: if NEOFS_ADM_EXEC and NEOFS_ADM_CONFIG_PATH:
# If frostfs-adm is available, then we tick epoch with it (to be consistent with UAT tests) # If neofs-adm is available, then we tick epoch with it (to be consistent with UAT tests)
frostfsadm = FrostfsAdm( neofsadm = NeofsAdm(
shell=remote_shell, shell=remote_shell,
frostfs_adm_exec_path=FROSTFS_ADM_EXEC, neofs_adm_exec_path=NEOFS_ADM_EXEC,
config_file=FROSTFS_ADM_CONFIG_PATH, config_file=NEOFS_ADM_CONFIG_PATH,
) )
frostfsadm.morph.force_new_epoch() neofsadm.morph.force_new_epoch()
return return
# Otherwise we tick epoch using transaction # Otherwise we tick epoch using transaction
@ -91,7 +90,7 @@ def tick_epoch(shell: Shell, cluster: Cluster, alive_node: Optional[StorageNode]
# In case if no local_wallet_path is provided, we use wallet_path # In case if no local_wallet_path is provided, we use wallet_path
ir_wallet_path = ir_node.get_wallet_path() ir_wallet_path = ir_node.get_wallet_path()
ir_wallet_pass = ir_node.get_wallet_password() ir_wallet_pass = ir_node.get_wallet_password()
ir_address = wallet_utils.get_last_address_from_wallet(ir_wallet_path, ir_wallet_pass) ir_address = get_last_address_from_wallet(ir_wallet_path, ir_wallet_pass)
morph_chain = cluster.morph_chain_nodes[0] morph_chain = cluster.morph_chain_nodes[0]
morph_endpoint = morph_chain.get_endpoint() morph_endpoint = morph_chain.get_endpoint()
@ -100,7 +99,7 @@ def tick_epoch(shell: Shell, cluster: Cluster, alive_node: Optional[StorageNode]
neogo.contract.invokefunction( neogo.contract.invokefunction(
wallet=ir_wallet_path, wallet=ir_wallet_path,
wallet_password=ir_wallet_pass, wallet_password=ir_wallet_pass,
scripthash=get_contract_hash(morph_chain, "netmap.frostfs", shell=shell), scripthash=get_contract_hash(morph_chain, "netmap.neofs", shell=shell),
method="newEpoch", method="newEpoch",
arguments=f"int:{cur_epoch + 1}", arguments=f"int:{cur_epoch + 1}",
multisig_hash=f"{ir_address}:Global", multisig_hash=f"{ir_address}:Global",
@ -109,4 +108,4 @@ def tick_epoch(shell: Shell, cluster: Cluster, alive_node: Optional[StorageNode]
force=True, force=True,
gas=1, gas=1,
) )
sleep(datetime_utils.parse_time(MAINNET_BLOCK_TIME)) sleep(parse_time(MAINNET_BLOCK_TIME))

View file

@ -2,11 +2,10 @@ import logging
from time import sleep from time import sleep
import allure import allure
from frostfs_testlib.shell import Shell from cluster import Cluster, StorageNode
from neofs_testlib.shell import Shell
from pytest_tests.helpers.cluster import Cluster, StorageNode from python_keywords.node_management import storage_node_healthcheck
from pytest_tests.helpers.node_management import storage_node_healthcheck from storage_policy import get_nodes_with_object
from pytest_tests.helpers.storage_policy import get_nodes_with_object
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")

View file

@ -1,3 +1,4 @@
import base64
import logging import logging
import os import os
import random import random
@ -10,15 +11,14 @@ from urllib.parse import quote_plus
import allure import allure
import requests import requests
from frostfs_testlib.shell import Shell from aws_cli_client import LONG_TIMEOUT
from cli_helpers import _cmd_run
from pytest_tests.helpers.aws_cli_client import LONG_TIMEOUT from cluster import StorageNode
from pytest_tests.helpers.cli_helpers import _cmd_run from common import SIMPLE_OBJECT_SIZE
from pytest_tests.helpers.cluster import StorageNode from file_helper import get_file_hash
from pytest_tests.helpers.file_helper import get_file_hash from neofs_testlib.shell import Shell
from pytest_tests.helpers.frostfs_verbs import get_object from python_keywords.neofs_verbs import get_object
from pytest_tests.helpers.storage_policy import get_nodes_without_object from python_keywords.storage_policy import get_nodes_without_object
from pytest_tests.resources.common import SIMPLE_OBJECT_SIZE
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")

View file

@ -0,0 +1,136 @@
"""
When doing requests to NeoFS, we get JSON output as an automatically decoded
structure from protobuf. Some fields are decoded with boilerplates and binary
values are Base64-encoded.
This module contains functions which rearrange the structure and reencode binary
data from Base64 to Base58.
"""
import base64
import base58
def decode_simple_header(data: dict) -> dict:
"""
This function reencodes Simple Object header and its attributes.
"""
try:
data = decode_common_fields(data)
# Normalize object attributes
data["header"]["attributes"] = {
attr["key"]: attr["value"] for attr in data["header"]["attributes"]
}
except Exception as exc:
raise ValueError(f"failed to decode JSON output: {exc}") from exc
return data
def decode_split_header(data: dict) -> dict:
"""
This function rearranges Complex Object header.
The header holds SplitID, a random unique
number, which is common among all splitted objects, and IDs of the Linking
Object and the last splitted Object.
"""
try:
data["splitId"] = json_reencode(data["splitId"])
data["lastPart"] = json_reencode(data["lastPart"]["value"]) if data["lastPart"] else None
data["link"] = json_reencode(data["link"]["value"]) if data["link"] else None
except Exception as exc:
raise ValueError(f"failed to decode JSON output: {exc}") from exc
return data
def decode_linking_object(data: dict) -> dict:
"""
This function reencodes Linking Object header.
It contains IDs of child Objects and Split Chain data.
"""
try:
data = decode_simple_header(data)
split = data["header"]["split"]
split["children"] = [json_reencode(item["value"]) for item in split["children"]]
split["splitID"] = json_reencode(split["splitID"])
split["previous"] = json_reencode(split["previous"]["value"]) if split["previous"] else None
split["parent"] = json_reencode(split["parent"]["value"]) if split["parent"] else None
except Exception as exc:
raise ValueError(f"failed to decode JSON output: {exc}") from exc
return data
def decode_storage_group(data: dict) -> dict:
"""
This function reencodes Storage Group header.
"""
try:
data = decode_common_fields(data)
except Exception as exc:
raise ValueError(f"failed to decode JSON output: {exc}") from exc
return data
def decode_tombstone(data: dict) -> dict:
"""
This function reencodes Tombstone header.
"""
try:
data = decode_simple_header(data)
data["header"]["sessionToken"] = decode_session_token(data["header"]["sessionToken"])
except Exception as exc:
raise ValueError(f"failed to decode JSON output: {exc}") from exc
return data
def decode_session_token(data: dict) -> dict:
"""
This function reencodes a fragment of header which contains
information about session token.
"""
target = data["body"]["object"]["target"]
target["container"] = json_reencode(target["container"]["value"])
target["objects"] = [json_reencode(obj["value"]) for obj in target["objects"]]
return data
def json_reencode(data: str) -> str:
"""
According to JSON protocol, binary data (Object/Container/Storage Group IDs, etc)
is converted to string via Base58 encoder. But we usually operate with Base64-encoded format.
This function reencodes given Base58 string into the Base64 one.
"""
return base58.b58encode(base64.b64decode(data)).decode("utf-8")
def encode_for_json(data: str) -> str:
"""
This function encodes binary data for sending them as protobuf
structures.
"""
return base64.b64encode(base58.b58decode(data)).decode("utf-8")
def decode_common_fields(data: dict) -> dict:
"""
Despite of type (simple/complex Object, Storage Group, etc) every Object
header contains several common fields.
This function rearranges these fields.
"""
data["objectID"] = json_reencode(data["objectID"]["value"])
header = data["header"]
header["containerID"] = json_reencode(header["containerID"]["value"])
header["ownerID"] = json_reencode(header["ownerID"]["value"])
header["payloadHash"] = json_reencode(header["payloadHash"]["sum"])
header["version"] = f"{header['version']['major']}{header['version']['minor']}"
# Homomorphic hash is optional and its calculation might be disabled in trusted network
if header.get("homomorphicHash"):
header["homomorphicHash"] = json_reencode(header["homomorphicHash"]["sum"])
return data

View file

@ -6,17 +6,11 @@ import uuid
from typing import Any, Optional from typing import Any, Optional
import allure import allure
from frostfs_testlib.cli import FrostfsCli import json_transformers
from frostfs_testlib.shell import Shell from cluster import Cluster
from frostfs_testlib.utils import json_utils from common import ASSETS_DIR, NEOFS_CLI_EXEC, WALLET_CONFIG
from neofs_testlib.cli import NeofsCli
from pytest_tests.helpers.cluster import Cluster from neofs_testlib.shell import Shell
from pytest_tests.resources.common import (
ASSETS_DIR,
CLI_DEFAULT_TIMEOUT,
FROSTFS_CLI_EXEC,
WALLET_CONFIG,
)
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
@ -34,10 +28,9 @@ def get_object_from_random_node(
wallet_config: Optional[str] = None, wallet_config: Optional[str] = None,
no_progress: bool = True, no_progress: bool = True,
session: Optional[str] = None, session: Optional[str] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> str: ) -> str:
""" """
GET from FrostFS random storage node GET from NeoFS random storage node
Args: Args:
wallet: wallet on whose behalf GET is done wallet: wallet on whose behalf GET is done
@ -46,12 +39,11 @@ def get_object_from_random_node(
shell: executor for cli command shell: executor for cli command
bearer (optional, str): path to Bearer Token file, appends to `--bearer` key bearer (optional, str): path to Bearer Token file, appends to `--bearer` key
write_object (optional, str): path to downloaded file, appends to `--file` key write_object (optional, str): path to downloaded file, appends to `--file` key
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key endpoint: NeoFS endpoint to send request to, appends to `--rpc-endpoint` key
wallet_config(optional, str): path to the wallet config wallet_config(optional, str): path to the wallet config
no_progress(optional, bool): do not show progress bar no_progress(optional, bool): do not show progress bar
xhdr (optional, dict): Request X-Headers in form of Key=Value xhdr (optional, dict): Request X-Headers in form of Key=Value
session (optional, dict): path to a JSON-encoded container session token session (optional, dict): path to a JSON-encoded container session token
timeout: Timeout for the operation.
Returns: Returns:
(str): path to downloaded file (str): path to downloaded file
""" """
@ -68,7 +60,6 @@ def get_object_from_random_node(
wallet_config, wallet_config,
no_progress, no_progress,
session, session,
timeout,
) )
@ -85,10 +76,9 @@ def get_object(
wallet_config: Optional[str] = None, wallet_config: Optional[str] = None,
no_progress: bool = True, no_progress: bool = True,
session: Optional[str] = None, session: Optional[str] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> str: ) -> str:
""" """
GET from FrostFS. GET from NeoFS.
Args: Args:
wallet (str): wallet on whose behalf GET is done wallet (str): wallet on whose behalf GET is done
@ -97,12 +87,11 @@ def get_object(
shell: executor for cli command shell: executor for cli command
bearer: path to Bearer Token file, appends to `--bearer` key bearer: path to Bearer Token file, appends to `--bearer` key
write_object: path to downloaded file, appends to `--file` key write_object: path to downloaded file, appends to `--file` key
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key endpoint: NeoFS endpoint to send request to, appends to `--rpc-endpoint` key
wallet_config(optional, str): path to the wallet config wallet_config(optional, str): path to the wallet config
no_progress(optional, bool): do not show progress bar no_progress(optional, bool): do not show progress bar
xhdr (optional, dict): Request X-Headers in form of Key=Value xhdr (optional, dict): Request X-Headers in form of Key=Value
session (optional, dict): path to a JSON-encoded container session token session (optional, dict): path to a JSON-encoded container session token
timeout: Timeout for the operation.
Returns: Returns:
(str): path to downloaded file (str): path to downloaded file
""" """
@ -111,7 +100,7 @@ def get_object(
write_object = str(uuid.uuid4()) write_object = str(uuid.uuid4())
file_path = os.path.join(ASSETS_DIR, write_object) file_path = os.path.join(ASSETS_DIR, write_object)
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or WALLET_CONFIG) cli = NeofsCli(shell, NEOFS_CLI_EXEC, wallet_config or WALLET_CONFIG)
cli.object.get( cli.object.get(
rpc_endpoint=endpoint, rpc_endpoint=endpoint,
wallet=wallet, wallet=wallet,
@ -122,7 +111,6 @@ def get_object(
no_progress=no_progress, no_progress=no_progress,
xhdr=xhdr, xhdr=xhdr,
session=session, session=session,
timeout=timeout,
) )
return file_path return file_path
@ -140,7 +128,6 @@ def get_range_hash(
wallet_config: Optional[str] = None, wallet_config: Optional[str] = None,
xhdr: Optional[dict] = None, xhdr: Optional[dict] = None,
session: Optional[str] = None, session: Optional[str] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
): ):
""" """
GETRANGEHASH of given Object. GETRANGEHASH of given Object.
@ -153,15 +140,14 @@ def get_range_hash(
bearer: path to Bearer Token file, appends to `--bearer` key bearer: path to Bearer Token file, appends to `--bearer` key
range_cut: Range to take hash from in the form offset1:length1,..., range_cut: Range to take hash from in the form offset1:length1,...,
value to pass to the `--range` parameter value to pass to the `--range` parameter
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key endpoint: NeoFS endpoint to send request to, appends to `--rpc-endpoint` key
wallet_config: path to the wallet config wallet_config: path to the wallet config
xhdr: Request X-Headers in form of Key=Values xhdr: Request X-Headers in form of Key=Values
session: Filepath to a JSON- or binary-encoded token of the object RANGEHASH session. session: Filepath to a JSON- or binary-encoded token of the object RANGEHASH session.
timeout: Timeout for the operation.
Returns: Returns:
None None
""" """
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or WALLET_CONFIG) cli = NeofsCli(shell, NEOFS_CLI_EXEC, wallet_config or WALLET_CONFIG)
result = cli.object.hash( result = cli.object.hash(
rpc_endpoint=endpoint, rpc_endpoint=endpoint,
wallet=wallet, wallet=wallet,
@ -171,7 +157,6 @@ def get_range_hash(
bearer=bearer, bearer=bearer,
xhdr=xhdr, xhdr=xhdr,
session=session, session=session,
timeout=timeout,
) )
# cutting off output about range offset and length # cutting off output about range offset and length
@ -192,7 +177,6 @@ def put_object_to_random_node(
expire_at: Optional[int] = None, expire_at: Optional[int] = None,
no_progress: bool = True, no_progress: bool = True,
session: Optional[str] = None, session: Optional[str] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
): ):
""" """
PUT of given file to a random storage node. PUT of given file to a random storage node.
@ -211,7 +195,6 @@ def put_object_to_random_node(
expire_at: Last epoch in the life of the object expire_at: Last epoch in the life of the object
xhdr: Request X-Headers in form of Key=Value xhdr: Request X-Headers in form of Key=Value
session: path to a JSON-encoded container session token session: path to a JSON-encoded container session token
timeout: Timeout for the operation.
Returns: Returns:
ID of uploaded Object ID of uploaded Object
""" """
@ -230,7 +213,6 @@ def put_object_to_random_node(
expire_at, expire_at,
no_progress, no_progress,
session, session,
timeout,
) )
@ -248,7 +230,6 @@ def put_object(
expire_at: Optional[int] = None, expire_at: Optional[int] = None,
no_progress: bool = True, no_progress: bool = True,
session: Optional[str] = None, session: Optional[str] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
): ):
""" """
PUT of given file. PUT of given file.
@ -260,18 +241,17 @@ def put_object(
shell: executor for cli command shell: executor for cli command
bearer: path to Bearer Token file, appends to `--bearer` key bearer: path to Bearer Token file, appends to `--bearer` key
attributes: User attributes in form of Key1=Value1,Key2=Value2 attributes: User attributes in form of Key1=Value1,Key2=Value2
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key endpoint: NeoFS endpoint to send request to, appends to `--rpc-endpoint` key
wallet_config: path to the wallet config wallet_config: path to the wallet config
no_progress: do not show progress bar no_progress: do not show progress bar
expire_at: Last epoch in the life of the object expire_at: Last epoch in the life of the object
xhdr: Request X-Headers in form of Key=Value xhdr: Request X-Headers in form of Key=Value
session: path to a JSON-encoded container session token session: path to a JSON-encoded container session token
timeout: Timeout for the operation.
Returns: Returns:
(str): ID of uploaded Object (str): ID of uploaded Object
""" """
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or WALLET_CONFIG) cli = NeofsCli(shell, NEOFS_CLI_EXEC, wallet_config or WALLET_CONFIG)
result = cli.object.put( result = cli.object.put(
rpc_endpoint=endpoint, rpc_endpoint=endpoint,
wallet=wallet, wallet=wallet,
@ -283,7 +263,6 @@ def put_object(
no_progress=no_progress, no_progress=no_progress,
xhdr=xhdr, xhdr=xhdr,
session=session, session=session,
timeout=timeout,
) )
# splitting CLI output to lines and taking the penultimate line # splitting CLI output to lines and taking the penultimate line
@ -303,7 +282,6 @@ def delete_object(
wallet_config: Optional[str] = None, wallet_config: Optional[str] = None,
xhdr: Optional[dict] = None, xhdr: Optional[dict] = None,
session: Optional[str] = None, session: Optional[str] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
): ):
""" """
DELETE an Object. DELETE an Object.
@ -314,16 +292,15 @@ def delete_object(
oid: ID of Object we are going to delete oid: ID of Object we are going to delete
shell: executor for cli command shell: executor for cli command
bearer: path to Bearer Token file, appends to `--bearer` key bearer: path to Bearer Token file, appends to `--bearer` key
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key endpoint: NeoFS endpoint to send request to, appends to `--rpc-endpoint` key
wallet_config: path to the wallet config wallet_config: path to the wallet config
xhdr: Request X-Headers in form of Key=Value xhdr: Request X-Headers in form of Key=Value
session: path to a JSON-encoded container session token session: path to a JSON-encoded container session token
timeout: Timeout for the operation.
Returns: Returns:
(str): Tombstone ID (str): Tombstone ID
""" """
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or WALLET_CONFIG) cli = NeofsCli(shell, NEOFS_CLI_EXEC, wallet_config or WALLET_CONFIG)
result = cli.object.delete( result = cli.object.delete(
rpc_endpoint=endpoint, rpc_endpoint=endpoint,
wallet=wallet, wallet=wallet,
@ -332,7 +309,6 @@ def delete_object(
bearer=bearer, bearer=bearer,
xhdr=xhdr, xhdr=xhdr,
session=session, session=session,
timeout=timeout,
) )
id_str = result.stdout.split("\n")[1] id_str = result.stdout.split("\n")[1]
@ -352,7 +328,6 @@ def get_range(
bearer: str = "", bearer: str = "",
xhdr: Optional[dict] = None, xhdr: Optional[dict] = None,
session: Optional[str] = None, session: Optional[str] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
): ):
""" """
GETRANGE an Object. GETRANGE an Object.
@ -363,18 +338,17 @@ def get_range(
oid: ID of Object we are going to request oid: ID of Object we are going to request
range_cut: range to take data from in the form offset:length range_cut: range to take data from in the form offset:length
shell: executor for cli command shell: executor for cli command
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key endpoint: NeoFS endpoint to send request to, appends to `--rpc-endpoint` key
bearer: path to Bearer Token file, appends to `--bearer` key bearer: path to Bearer Token file, appends to `--bearer` key
wallet_config: path to the wallet config wallet_config: path to the wallet config
xhdr: Request X-Headers in form of Key=Value xhdr: Request X-Headers in form of Key=Value
session: path to a JSON-encoded container session token session: path to a JSON-encoded container session token
timeout: Timeout for the operation.
Returns: Returns:
(str, bytes) - path to the file with range content and content of this file as bytes (str, bytes) - path to the file with range content and content of this file as bytes
""" """
range_file_path = os.path.join(ASSETS_DIR, str(uuid.uuid4())) range_file_path = os.path.join(ASSETS_DIR, str(uuid.uuid4()))
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or WALLET_CONFIG) cli = NeofsCli(shell, NEOFS_CLI_EXEC, wallet_config or WALLET_CONFIG)
cli.object.range( cli.object.range(
rpc_endpoint=endpoint, rpc_endpoint=endpoint,
wallet=wallet, wallet=wallet,
@ -385,7 +359,6 @@ def get_range(
bearer=bearer, bearer=bearer,
xhdr=xhdr, xhdr=xhdr,
session=session, session=session,
timeout=timeout,
) )
with open(range_file_path, "rb") as file: with open(range_file_path, "rb") as file:
@ -408,7 +381,6 @@ def lock_object(
wallet_config: Optional[str] = None, wallet_config: Optional[str] = None,
ttl: Optional[int] = None, ttl: Optional[int] = None,
xhdr: Optional[dict] = None, xhdr: Optional[dict] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> str: ) -> str:
""" """
Lock object in container. Lock object in container.
@ -421,18 +393,17 @@ def lock_object(
lifetime: Lock lifetime. lifetime: Lock lifetime.
expire_at: Lock expiration epoch. expire_at: Lock expiration epoch.
shell: executor for cli command shell: executor for cli command
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key endpoint: NeoFS endpoint to send request to, appends to `--rpc-endpoint` key
session: Path to a JSON-encoded container session token. session: Path to a JSON-encoded container session token.
ttl: TTL value in request meta header (default 2). ttl: TTL value in request meta header (default 2).
wallet: WIF (NEP-2) string or path to the wallet or binary key. wallet: WIF (NEP-2) string or path to the wallet or binary key.
xhdr: Dict with request X-Headers. xhdr: Dict with request X-Headers.
timeout: Timeout for the operation.
Returns: Returns:
Lock object ID Lock object ID
""" """
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or WALLET_CONFIG) cli = NeofsCli(shell, NEOFS_CLI_EXEC, wallet_config or WALLET_CONFIG)
result = cli.object.lock( result = cli.object.lock(
rpc_endpoint=endpoint, rpc_endpoint=endpoint,
lifetime=lifetime, lifetime=lifetime,
@ -445,7 +416,6 @@ def lock_object(
xhdr=xhdr, xhdr=xhdr,
session=session, session=session,
ttl=ttl, ttl=ttl,
timeout=timeout,
) )
# splitting CLI output to lines and taking the penultimate line # splitting CLI output to lines and taking the penultimate line
@ -468,7 +438,6 @@ def search_object(
session: Optional[str] = None, session: Optional[str] = None,
phy: bool = False, phy: bool = False,
root: bool = False, root: bool = False,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> list: ) -> list:
""" """
SEARCH an Object. SEARCH an Object.
@ -478,7 +447,7 @@ def search_object(
cid: ID of Container where we get the Object from cid: ID of Container where we get the Object from
shell: executor for cli command shell: executor for cli command
bearer: path to Bearer Token file, appends to `--bearer` key bearer: path to Bearer Token file, appends to `--bearer` key
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key endpoint: NeoFS endpoint to send request to, appends to `--rpc-endpoint` key
filters: key=value pairs to filter Objects filters: key=value pairs to filter Objects
expected_objects_list: a list of ObjectIDs to compare found Objects with expected_objects_list: a list of ObjectIDs to compare found Objects with
wallet_config: path to the wallet config wallet_config: path to the wallet config
@ -486,13 +455,12 @@ def search_object(
session: path to a JSON-encoded container session token session: path to a JSON-encoded container session token
phy: Search physically stored objects. phy: Search physically stored objects.
root: Search for user objects. root: Search for user objects.
timeout: Timeout for the operation.
Returns: Returns:
list of found ObjectIDs list of found ObjectIDs
""" """
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or WALLET_CONFIG) cli = NeofsCli(shell, NEOFS_CLI_EXEC, wallet_config or WALLET_CONFIG)
result = cli.object.search( result = cli.object.search(
rpc_endpoint=endpoint, rpc_endpoint=endpoint,
wallet=wallet, wallet=wallet,
@ -505,7 +473,6 @@ def search_object(
session=session, session=session,
phy=phy, phy=phy,
root=root, root=root,
timeout=timeout,
) )
found_objects = re.findall(r"(\w{43,44})", result.stdout) found_objects = re.findall(r"(\w{43,44})", result.stdout)
@ -534,7 +501,6 @@ def get_netmap_netinfo(
address: Optional[str] = None, address: Optional[str] = None,
ttl: Optional[int] = None, ttl: Optional[int] = None,
xhdr: Optional[dict] = None, xhdr: Optional[dict] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> dict[str, Any]: ) -> dict[str, Any]:
""" """
Get netmap netinfo output from node Get netmap netinfo output from node
@ -542,7 +508,7 @@ def get_netmap_netinfo(
Args: Args:
wallet (str): wallet on whose behalf request is done wallet (str): wallet on whose behalf request is done
shell: executor for cli command shell: executor for cli command
endpoint (optional, str): FrostFS endpoint to send request to, appends to `--rpc-endpoint` key endpoint (optional, str): NeoFS endpoint to send request to, appends to `--rpc-endpoint` key
address: Address of wallet account address: Address of wallet account
ttl: TTL value in request meta header (default 2) ttl: TTL value in request meta header (default 2)
wallet: Path to the wallet or binary key wallet: Path to the wallet or binary key
@ -552,14 +518,13 @@ def get_netmap_netinfo(
(dict): dict of parsed command output (dict): dict of parsed command output
""" """
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or WALLET_CONFIG) cli = NeofsCli(shell, NEOFS_CLI_EXEC, wallet_config or WALLET_CONFIG)
output = cli.netmap.netinfo( output = cli.netmap.netinfo(
wallet=wallet, wallet=wallet,
rpc_endpoint=endpoint, rpc_endpoint=endpoint,
address=address, address=address,
ttl=ttl, ttl=ttl,
xhdr=xhdr, xhdr=xhdr,
timeout=timeout,
) )
settings = dict() settings = dict()
@ -590,7 +555,6 @@ def head_object(
is_direct: bool = False, is_direct: bool = False,
wallet_config: Optional[str] = None, wallet_config: Optional[str] = None,
session: Optional[str] = None, session: Optional[str] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
): ):
""" """
HEAD an Object. HEAD an Object.
@ -601,7 +565,7 @@ def head_object(
oid (str): ObjectID to HEAD oid (str): ObjectID to HEAD
shell: executor for cli command shell: executor for cli command
bearer (optional, str): path to Bearer Token file, appends to `--bearer` key bearer (optional, str): path to Bearer Token file, appends to `--bearer` key
endpoint(optional, str): FrostFS endpoint to send request to endpoint(optional, str): NeoFS endpoint to send request to
json_output(optional, bool): return response in JSON format or not; this flag json_output(optional, bool): return response in JSON format or not; this flag
turns into `--json` key turns into `--json` key
is_raw(optional, bool): send "raw" request or not; this flag is_raw(optional, bool): send "raw" request or not; this flag
@ -611,7 +575,6 @@ def head_object(
wallet_config(optional, str): path to the wallet config wallet_config(optional, str): path to the wallet config
xhdr (optional, dict): Request X-Headers in form of Key=Value xhdr (optional, dict): Request X-Headers in form of Key=Value
session (optional, dict): path to a JSON-encoded container session token session (optional, dict): path to a JSON-encoded container session token
timeout: Timeout for the operation.
Returns: Returns:
depending on the `json_output` parameter value, the function returns depending on the `json_output` parameter value, the function returns
(dict): HEAD response in JSON format (dict): HEAD response in JSON format
@ -619,7 +582,7 @@ def head_object(
(str): HEAD response as a plain text (str): HEAD response as a plain text
""" """
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or WALLET_CONFIG) cli = NeofsCli(shell, NEOFS_CLI_EXEC, wallet_config or WALLET_CONFIG)
result = cli.object.head( result = cli.object.head(
rpc_endpoint=endpoint, rpc_endpoint=endpoint,
wallet=wallet, wallet=wallet,
@ -631,7 +594,6 @@ def head_object(
ttl=1 if is_direct else None, ttl=1 if is_direct else None,
xhdr=xhdr, xhdr=xhdr,
session=session, session=session,
timeout=timeout,
) )
if not json_output: if not json_output:
@ -651,22 +613,22 @@ def head_object(
# If response is Complex Object header, it has `splitId` key # If response is Complex Object header, it has `splitId` key
if "splitId" in decoded.keys(): if "splitId" in decoded.keys():
logger.info("decoding split header") logger.info("decoding split header")
return json_utils.decode_split_header(decoded) return json_transformers.decode_split_header(decoded)
# If response is Last or Linking Object header, # If response is Last or Linking Object header,
# it has `header` dictionary and non-null `split` dictionary # it has `header` dictionary and non-null `split` dictionary
if "split" in decoded["header"].keys(): if "split" in decoded["header"].keys():
if decoded["header"]["split"]: if decoded["header"]["split"]:
logger.info("decoding linking object") logger.info("decoding linking object")
return json_utils.decode_linking_object(decoded) return json_transformers.decode_linking_object(decoded)
if decoded["header"]["objectType"] == "STORAGE_GROUP": if decoded["header"]["objectType"] == "STORAGE_GROUP":
logger.info("decoding storage group") logger.info("decoding storage group")
return json_utils.decode_storage_group(decoded) return json_transformers.decode_storage_group(decoded)
if decoded["header"]["objectType"] == "TOMBSTONE": if decoded["header"]["objectType"] == "TOMBSTONE":
logger.info("decoding tombstone") logger.info("decoding tombstone")
return json_utils.decode_tombstone(decoded) return json_transformers.decode_tombstone(decoded)
logger.info("decoding simple header") logger.info("decoding simple header")
return json_utils.decode_simple_header(decoded) return json_transformers.decode_simple_header(decoded)

View file

@ -6,13 +6,12 @@ from dataclasses import dataclass
from typing import Optional from typing import Optional
import allure import allure
from frostfs_testlib.cli import FrostfsCli from cluster import Cluster, StorageNode
from frostfs_testlib.shell import Shell from common import MORPH_BLOCK_TIME, NEOFS_CLI_EXEC
from frostfs_testlib.utils import datetime_utils from epoch import tick_epoch
from neofs_testlib.cli import NeofsCli
from pytest_tests.helpers.cluster import Cluster, StorageNode from neofs_testlib.shell import Shell
from pytest_tests.helpers.epoch import tick_epoch from utility import parse_time
from pytest_tests.resources.common import FROSTFS_CLI_EXEC, MORPH_BLOCK_TIME
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
@ -108,7 +107,7 @@ def get_netmap_snapshot(node: StorageNode, shell: Shell) -> str:
storage_wallet_config = node.get_wallet_config_path() storage_wallet_config = node.get_wallet_config_path()
storage_wallet_path = node.get_wallet_path() storage_wallet_path = node.get_wallet_path()
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, config_file=storage_wallet_config) cli = NeofsCli(shell, NEOFS_CLI_EXEC, config_file=storage_wallet_config)
return cli.netmap.snapshot( return cli.netmap.snapshot(
rpc_endpoint=node.get_rpc_endpoint(), rpc_endpoint=node.get_rpc_endpoint(),
wallet=storage_wallet_path, wallet=storage_wallet_path,
@ -155,7 +154,7 @@ def drop_object(node: StorageNode, cid: str, oid: str) -> str:
def delete_node_data(node: StorageNode) -> None: def delete_node_data(node: StorageNode) -> None:
node.stop_service() node.stop_service()
node.host.delete_storage_node_data(node.name) node.host.delete_storage_node_data(node.name)
time.sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME)) time.sleep(parse_time(MORPH_BLOCK_TIME))
@allure.step("Exclude node {node_to_exclude} from network map") @allure.step("Exclude node {node_to_exclude} from network map")
@ -169,7 +168,7 @@ def exclude_node_from_network_map(
storage_node_set_status(node_to_exclude, status="offline") storage_node_set_status(node_to_exclude, status="offline")
time.sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME)) time.sleep(parse_time(MORPH_BLOCK_TIME))
tick_epoch(shell, cluster) tick_epoch(shell, cluster)
snapshot = get_netmap_snapshot(node=alive_node, shell=shell) snapshot = get_netmap_snapshot(node=alive_node, shell=shell)
@ -188,11 +187,11 @@ def include_node_to_network_map(
storage_node_set_status(node_to_include, status="online") storage_node_set_status(node_to_include, status="online")
# Per suggestion of @fyrchik we need to wait for 2 blocks after we set status and after tick epoch. # Per suggestion of @fyrchik we need to wait for 2 blocks after we set status and after tick epoch.
# First sleep can be omitted after https://github.com/nspcc-dev/frostfs-node/issues/1790 complete. # First sleep can be omitted after https://github.com/nspcc-dev/neofs-node/issues/1790 complete.
time.sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME) * 2) time.sleep(parse_time(MORPH_BLOCK_TIME) * 2)
tick_epoch(shell, cluster) tick_epoch(shell, cluster)
time.sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME) * 2) time.sleep(parse_time(MORPH_BLOCK_TIME) * 2)
check_node_in_map(node_to_include, shell, alive_node) check_node_in_map(node_to_include, shell, alive_node)
@ -236,10 +235,10 @@ def _run_control_command(node: StorageNode, command: str) -> None:
wallet_config = f'password: "{wallet_password}"' wallet_config = f'password: "{wallet_password}"'
shell.exec(f"echo '{wallet_config}' > {wallet_config_path}") shell.exec(f"echo '{wallet_config}' > {wallet_config_path}")
cli_config = host.get_cli_config("frostfs-cli") cli_config = host.get_cli_config("neofs-cli")
# TODO: implement cli.control # TODO: implement cli.control
# cli = frostfsCli(shell, cli_config.exec_path, wallet_config_path) # cli = NeofsCli(shell, cli_config.exec_path, wallet_config_path)
result = shell.exec( result = shell.exec(
f"{cli_config.exec_path} {command} --endpoint {control_endpoint} " f"{cli_config.exec_path} {command} --endpoint {control_endpoint} "
f"--wallet {wallet_path} --config {wallet_config_path}" f"--wallet {wallet_path} --config {wallet_config_path}"

View file

@ -1,13 +1,11 @@
from typing import Optional from typing import Optional
import allure import allure
from frostfs_testlib.resources.common import OBJECT_ACCESS_DENIED from cluster import Cluster
from frostfs_testlib.shell import Shell from file_helper import get_file_hash
from frostfs_testlib.utils import string_utils from grpc_responses import OBJECT_ACCESS_DENIED, error_matches_status
from neofs_testlib.shell import Shell
from pytest_tests.helpers.cluster import Cluster from python_keywords.neofs_verbs import (
from pytest_tests.helpers.file_helper import get_file_hash
from pytest_tests.helpers.frostfs_verbs import (
delete_object, delete_object,
get_object_from_random_node, get_object_from_random_node,
get_range, get_range,
@ -16,7 +14,6 @@ from pytest_tests.helpers.frostfs_verbs import (
put_object_to_random_node, put_object_to_random_node,
search_object, search_object,
) )
from pytest_tests.resources.common import CLI_DEFAULT_TIMEOUT
OPERATION_ERROR_TYPE = RuntimeError OPERATION_ERROR_TYPE = RuntimeError
@ -45,7 +42,7 @@ def can_get_object(
cluster=cluster, cluster=cluster,
) )
except OPERATION_ERROR_TYPE as err: except OPERATION_ERROR_TYPE as err:
assert string_utils.is_str_match_pattern( assert error_matches_status(
err, OBJECT_ACCESS_DENIED err, OBJECT_ACCESS_DENIED
), f"Expected {err} to match {OBJECT_ACCESS_DENIED}" ), f"Expected {err} to match {OBJECT_ACCESS_DENIED}"
return False return False
@ -78,7 +75,7 @@ def can_put_object(
cluster=cluster, cluster=cluster,
) )
except OPERATION_ERROR_TYPE as err: except OPERATION_ERROR_TYPE as err:
assert string_utils.is_str_match_pattern( assert error_matches_status(
err, OBJECT_ACCESS_DENIED err, OBJECT_ACCESS_DENIED
), f"Expected {err} to match {OBJECT_ACCESS_DENIED}" ), f"Expected {err} to match {OBJECT_ACCESS_DENIED}"
return False return False
@ -108,7 +105,7 @@ def can_delete_object(
endpoint=endpoint, endpoint=endpoint,
) )
except OPERATION_ERROR_TYPE as err: except OPERATION_ERROR_TYPE as err:
assert string_utils.is_str_match_pattern( assert error_matches_status(
err, OBJECT_ACCESS_DENIED err, OBJECT_ACCESS_DENIED
), f"Expected {err} to match {OBJECT_ACCESS_DENIED}" ), f"Expected {err} to match {OBJECT_ACCESS_DENIED}"
return False return False
@ -124,7 +121,6 @@ def can_get_head_object(
bearer: Optional[str] = None, bearer: Optional[str] = None,
wallet_config: Optional[str] = None, wallet_config: Optional[str] = None,
xhdr: Optional[dict] = None, xhdr: Optional[dict] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> bool: ) -> bool:
with allure.step("Try get head of object"): with allure.step("Try get head of object"):
try: try:
@ -137,10 +133,9 @@ def can_get_head_object(
xhdr=xhdr, xhdr=xhdr,
shell=shell, shell=shell,
endpoint=endpoint, endpoint=endpoint,
timeout=timeout,
) )
except OPERATION_ERROR_TYPE as err: except OPERATION_ERROR_TYPE as err:
assert string_utils.is_str_match_pattern( assert error_matches_status(
err, OBJECT_ACCESS_DENIED err, OBJECT_ACCESS_DENIED
), f"Expected {err} to match {OBJECT_ACCESS_DENIED}" ), f"Expected {err} to match {OBJECT_ACCESS_DENIED}"
return False return False
@ -156,7 +151,6 @@ def can_get_range_of_object(
bearer: Optional[str] = None, bearer: Optional[str] = None,
wallet_config: Optional[str] = None, wallet_config: Optional[str] = None,
xhdr: Optional[dict] = None, xhdr: Optional[dict] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> bool: ) -> bool:
with allure.step("Try get range of object"): with allure.step("Try get range of object"):
try: try:
@ -170,10 +164,9 @@ def can_get_range_of_object(
xhdr=xhdr, xhdr=xhdr,
shell=shell, shell=shell,
endpoint=endpoint, endpoint=endpoint,
timeout=timeout,
) )
except OPERATION_ERROR_TYPE as err: except OPERATION_ERROR_TYPE as err:
assert string_utils.is_str_match_pattern( assert error_matches_status(
err, OBJECT_ACCESS_DENIED err, OBJECT_ACCESS_DENIED
), f"Expected {err} to match {OBJECT_ACCESS_DENIED}" ), f"Expected {err} to match {OBJECT_ACCESS_DENIED}"
return False return False
@ -189,7 +182,6 @@ def can_get_range_hash_of_object(
bearer: Optional[str] = None, bearer: Optional[str] = None,
wallet_config: Optional[str] = None, wallet_config: Optional[str] = None,
xhdr: Optional[dict] = None, xhdr: Optional[dict] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> bool: ) -> bool:
with allure.step("Try get range hash of object"): with allure.step("Try get range hash of object"):
try: try:
@ -203,10 +195,9 @@ def can_get_range_hash_of_object(
xhdr=xhdr, xhdr=xhdr,
shell=shell, shell=shell,
endpoint=endpoint, endpoint=endpoint,
timeout=timeout,
) )
except OPERATION_ERROR_TYPE as err: except OPERATION_ERROR_TYPE as err:
assert string_utils.is_str_match_pattern( assert error_matches_status(
err, OBJECT_ACCESS_DENIED err, OBJECT_ACCESS_DENIED
), f"Expected {err} to match {OBJECT_ACCESS_DENIED}" ), f"Expected {err} to match {OBJECT_ACCESS_DENIED}"
return False return False
@ -222,7 +213,6 @@ def can_search_object(
bearer: Optional[str] = None, bearer: Optional[str] = None,
wallet_config: Optional[str] = None, wallet_config: Optional[str] = None,
xhdr: Optional[dict] = None, xhdr: Optional[dict] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> bool: ) -> bool:
with allure.step("Try search object in container"): with allure.step("Try search object in container"):
try: try:
@ -234,10 +224,9 @@ def can_search_object(
xhdr=xhdr, xhdr=xhdr,
shell=shell, shell=shell,
endpoint=endpoint, endpoint=endpoint,
timeout=timeout,
) )
except OPERATION_ERROR_TYPE as err: except OPERATION_ERROR_TYPE as err:
assert string_utils.is_str_match_pattern( assert error_matches_status(
err, OBJECT_ACCESS_DENIED err, OBJECT_ACCESS_DENIED
), f"Expected {err} to match {OBJECT_ACCESS_DENIED}" ), f"Expected {err} to match {OBJECT_ACCESS_DENIED}"
return False return False

View file

@ -6,19 +6,14 @@ import time
from typing import Optional from typing import Optional
import allure import allure
from frostfs_testlib.cli import NeoGo from cluster import MainChain, MorphChain
from frostfs_testlib.shell import Shell from common import GAS_HASH, MAINNET_BLOCK_TIME, NEOFS_CONTRACT, NEOGO_EXECUTABLE
from frostfs_testlib.utils import converting_utils, datetime_utils, wallet_utils from neo3 import wallet as neo3_wallet
from neo3.wallet import utils as neo3_utils from neofs_testlib.cli import NeoGo
from neo3.wallet import wallet as neo3_wallet from neofs_testlib.shell import Shell
from neofs_testlib.utils.converters import contract_hash_to_address
from pytest_tests.helpers.cluster import MainChain, MorphChain from neofs_testlib.utils.wallet import get_last_address_from_wallet
from pytest_tests.resources.common import ( from utility import parse_time
FROSTFS_CONTRACT,
GAS_HASH,
MAINNET_BLOCK_TIME,
NEOGO_EXECUTABLE,
)
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
@ -47,15 +42,15 @@ def get_contract_hash(morph_chain: MorphChain, resolve_name: str, shell: Shell)
@allure.step("Withdraw Mainnet Gas") @allure.step("Withdraw Mainnet Gas")
def withdraw_mainnet_gas(shell: Shell, main_chain: MainChain, wlt: str, amount: int): def withdraw_mainnet_gas(shell: Shell, main_chain: MainChain, wlt: str, amount: int):
address = wallet_utils.get_last_address_from_wallet(wlt, EMPTY_PASSWORD) address = get_last_address_from_wallet(wlt, EMPTY_PASSWORD)
scripthash = neo3_utils.address_to_script_hash(address) scripthash = neo3_wallet.Account.address_to_script_hash(address)
neogo = NeoGo(shell=shell, neo_go_exec_path=NEOGO_EXECUTABLE) neogo = NeoGo(shell=shell, neo_go_exec_path=NEOGO_EXECUTABLE)
out = neogo.contract.invokefunction( out = neogo.contract.invokefunction(
wallet=wlt, wallet=wlt,
address=address, address=address,
rpc_endpoint=main_chain.get_endpoint(), rpc_endpoint=main_chain.get_endpoint(),
scripthash=FROSTFS_CONTRACT, scripthash=NEOFS_CONTRACT,
method="withdraw", method="withdraw",
arguments=f"{scripthash} int:{amount}", arguments=f"{scripthash} int:{amount}",
multisig_hash=f"{scripthash}:Global", multisig_hash=f"{scripthash}:Global",
@ -92,10 +87,10 @@ def transaction_accepted(main_chain: MainChain, tx_id: str):
return False return False
@allure.step("Get FrostFS Balance") @allure.step("Get NeoFS Balance")
def get_balance(shell: Shell, morph_chain: MorphChain, wallet_path: str, wallet_password: str = ""): def get_balance(shell: Shell, morph_chain: MorphChain, wallet_path: str, wallet_password: str = ""):
""" """
This function returns FrostFS balance for given wallet. This function returns NeoFS balance for given wallet.
""" """
with open(wallet_path) as wallet_file: with open(wallet_path) as wallet_file:
wallet = neo3_wallet.Wallet.from_json(json.load(wallet_file), password=wallet_password) wallet = neo3_wallet.Wallet.from_json(json.load(wallet_file), password=wallet_password)
@ -103,7 +98,7 @@ def get_balance(shell: Shell, morph_chain: MorphChain, wallet_path: str, wallet_
payload = [{"type": "Hash160", "value": str(acc.script_hash)}] payload = [{"type": "Hash160", "value": str(acc.script_hash)}]
try: try:
resp = morph_chain.rpc_client.invoke_function( resp = morph_chain.rpc_client.invoke_function(
get_contract_hash(morph_chain, "balance.frostfs", shell=shell), "balanceOf", payload get_contract_hash(morph_chain, "balance.neofs", shell=shell), "balanceOf", payload
) )
logger.info(f"Got response \n{resp}") logger.info(f"Got response \n{resp}")
value = int(resp["stack"][0]["value"]) value = int(resp["stack"][0]["value"])
@ -146,12 +141,10 @@ def transfer_gas(
if wallet_from_password is not None if wallet_from_password is not None
else main_chain.get_wallet_password() else main_chain.get_wallet_password()
) )
address_from = address_from or wallet_utils.get_last_address_from_wallet( address_from = address_from or get_last_address_from_wallet(
wallet_from_path, wallet_from_password wallet_from_path, wallet_from_password
) )
address_to = address_to or wallet_utils.get_last_address_from_wallet( address_to = address_to or get_last_address_from_wallet(wallet_to_path, wallet_to_password)
wallet_to_path, wallet_to_password
)
neogo = NeoGo(shell, neo_go_exec_path=NEOGO_EXECUTABLE) neogo = NeoGo(shell, neo_go_exec_path=NEOGO_EXECUTABLE)
out = neogo.nep17.transfer( out = neogo.nep17.transfer(
@ -169,10 +162,10 @@ def transfer_gas(
raise Exception("Got no TXID after run the command") raise Exception("Got no TXID after run the command")
if not transaction_accepted(main_chain, txid): if not transaction_accepted(main_chain, txid):
raise AssertionError(f"TX {txid} hasn't been processed") raise AssertionError(f"TX {txid} hasn't been processed")
time.sleep(datetime_utils.parse_time(MAINNET_BLOCK_TIME)) time.sleep(parse_time(MAINNET_BLOCK_TIME))
@allure.step("FrostFS Deposit") @allure.step("NeoFS Deposit")
def deposit_gas( def deposit_gas(
shell: Shell, shell: Shell,
main_chain: MainChain, main_chain: MainChain,
@ -181,12 +174,12 @@ def deposit_gas(
wallet_from_password: str, wallet_from_password: str,
): ):
""" """
Transferring GAS from given wallet to FrostFS contract address. Transferring GAS from given wallet to NeoFS contract address.
""" """
# get FrostFS contract address # get NeoFS contract address
deposit_addr = converting_utils.contract_hash_to_address(FROSTFS_CONTRACT) deposit_addr = contract_hash_to_address(NEOFS_CONTRACT)
logger.info(f"FrostFS contract address: {deposit_addr}") logger.info(f"NeoFS contract address: {deposit_addr}")
address_from = wallet_utils.get_last_address_from_wallet( address_from = get_last_address_from_wallet(
wallet_path=wallet_from_path, wallet_password=wallet_from_password wallet_path=wallet_from_path, wallet_password=wallet_from_password
) )
transfer_gas( transfer_gas(

View file

@ -1,18 +1,17 @@
""" """
This module contains keywords for work with Storage Groups. This module contains keywords for work with Storage Groups.
It contains wrappers for `frostfs-cli storagegroup` verbs. It contains wrappers for `neofs-cli storagegroup` verbs.
""" """
import logging import logging
from typing import Optional from typing import Optional
import allure import allure
from frostfs_testlib.cli import FrostfsCli from cluster import Cluster
from frostfs_testlib.shell import Shell from common import NEOFS_CLI_EXEC, WALLET_CONFIG
from complex_object_actions import get_link_object
from pytest_tests.helpers.cluster import Cluster from neofs_testlib.cli import NeofsCli
from pytest_tests.helpers.complex_object_actions import get_link_object from neofs_testlib.shell import Shell
from pytest_tests.helpers.frostfs_verbs import head_object from neofs_verbs import head_object
from pytest_tests.resources.common import CLI_DEFAULT_TIMEOUT, FROSTFS_CLI_EXEC, WALLET_CONFIG
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
@ -27,11 +26,10 @@ def put_storagegroup(
bearer: Optional[str] = None, bearer: Optional[str] = None,
wallet_config: str = WALLET_CONFIG, wallet_config: str = WALLET_CONFIG,
lifetime: int = 10, lifetime: int = 10,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> str: ) -> str:
""" """
Wrapper for `frostfs-cli storagegroup put`. Before the SG is created, Wrapper for `neofs-cli storagegroup put`. Before the SG is created,
frostfs-cli performs HEAD on `objects`, so this verb must be allowed neofs-cli performs HEAD on `objects`, so this verb must be allowed
for `wallet` in `cid`. for `wallet` in `cid`.
Args: Args:
shell: Shell instance. shell: Shell instance.
@ -40,15 +38,12 @@ def put_storagegroup(
lifetime: Storage group lifetime in epochs. lifetime: Storage group lifetime in epochs.
objects: List of Object IDs to include into the SG. objects: List of Object IDs to include into the SG.
bearer: Path to Bearer token file. bearer: Path to Bearer token file.
wallet_config: Path to frostfs-cli config file. wallet_config: Path to neofs-cli config file.
timeout: Timeout for an operation.
Returns: Returns:
Object ID of created Storage Group. Object ID of created Storage Group.
""" """
frostfscli = FrostfsCli( neofscli = NeofsCli(shell=shell, neofs_cli_exec_path=NEOFS_CLI_EXEC, config_file=wallet_config)
shell=shell, frostfs_cli_exec_path=FROSTFS_CLI_EXEC, config_file=wallet_config result = neofscli.storagegroup.put(
)
result = frostfscli.storagegroup.put(
wallet=wallet, wallet=wallet,
cid=cid, cid=cid,
lifetime=lifetime, lifetime=lifetime,
@ -68,30 +63,25 @@ def list_storagegroup(
cid: str, cid: str,
bearer: Optional[str] = None, bearer: Optional[str] = None,
wallet_config: str = WALLET_CONFIG, wallet_config: str = WALLET_CONFIG,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> list: ) -> list:
""" """
Wrapper for `frostfs-cli storagegroup list`. This operation Wrapper for `neofs-cli storagegroup list`. This operation
requires SEARCH allowed for `wallet` in `cid`. requires SEARCH allowed for `wallet` in `cid`.
Args: Args:
shell: Shell instance. shell: Shell instance.
wallet: Path to wallet on whose behalf the SGs are listed in the container wallet: Path to wallet on whose behalf the SGs are listed in the container
cid: ID of Container to list. cid: ID of Container to list.
bearer: Path to Bearer token file. bearer: Path to Bearer token file.
wallet_config: Path to frostfs-cli config file. wallet_config: Path to neofs-cli config file.
timeout: Timeout for an operation.
Returns: Returns:
Object IDs of found Storage Groups. Object IDs of found Storage Groups.
""" """
frostfscli = FrostfsCli( neofscli = NeofsCli(shell=shell, neofs_cli_exec_path=NEOFS_CLI_EXEC, config_file=wallet_config)
shell=shell, frostfs_cli_exec_path=FROSTFS_CLI_EXEC, config_file=wallet_config result = neofscli.storagegroup.list(
)
result = frostfscli.storagegroup.list(
wallet=wallet, wallet=wallet,
cid=cid, cid=cid,
bearer=bearer, bearer=bearer,
rpc_endpoint=endpoint, rpc_endpoint=endpoint,
timeout=timeout,
) )
# throwing off the first string of output # throwing off the first string of output
found_objects = result.stdout.split("\n")[1:] found_objects = result.stdout.split("\n")[1:]
@ -107,35 +97,30 @@ def get_storagegroup(
gid: str, gid: str,
bearer: str = "", bearer: str = "",
wallet_config: str = WALLET_CONFIG, wallet_config: str = WALLET_CONFIG,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> dict: ) -> dict:
""" """
Wrapper for `frostfs-cli storagegroup get`. Wrapper for `neofs-cli storagegroup get`.
Args: Args:
shell: Shell instance. shell: Shell instance.
wallet: Path to wallet on whose behalf the SG is got. wallet: Path to wallet on whose behalf the SG is got.
cid: ID of Container where SG is stored. cid: ID of Container where SG is stored.
gid: ID of the Storage Group. gid: ID of the Storage Group.
bearer: Path to Bearer token file. bearer: Path to Bearer token file.
wallet_config: Path to frostfs-cli config file. wallet_config: Path to neofs-cli config file.
timeout: Timeout for an operation.
Returns: Returns:
Detailed information on the Storage Group. Detailed information on the Storage Group.
""" """
frostfscli = FrostfsCli( neofscli = NeofsCli(shell=shell, neofs_cli_exec_path=NEOFS_CLI_EXEC, config_file=wallet_config)
shell=shell, frostfs_cli_exec_path=FROSTFS_CLI_EXEC, config_file=wallet_config result = neofscli.storagegroup.get(
)
result = frostfscli.storagegroup.get(
wallet=wallet, wallet=wallet,
cid=cid, cid=cid,
bearer=bearer, bearer=bearer,
id=gid, id=gid,
rpc_endpoint=endpoint, rpc_endpoint=endpoint,
timeout=timeout,
) )
# TODO: temporary solution for parsing output. Needs to be replaced with # TODO: temporary solution for parsing output. Needs to be replaced with
# JSON parsing when https://github.com/nspcc-dev/frostfs-node/issues/1355 # JSON parsing when https://github.com/nspcc-dev/neofs-node/issues/1355
# is done. # is done.
strings = result.stdout.strip().split("\n") strings = result.stdout.strip().split("\n")
# first three strings go to `data`; # first three strings go to `data`;
@ -161,31 +146,26 @@ def delete_storagegroup(
gid: str, gid: str,
bearer: str = "", bearer: str = "",
wallet_config: str = WALLET_CONFIG, wallet_config: str = WALLET_CONFIG,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> str: ) -> str:
""" """
Wrapper for `frostfs-cli storagegroup delete`. Wrapper for `neofs-cli storagegroup delete`.
Args: Args:
shell: Shell instance. shell: Shell instance.
wallet: Path to wallet on whose behalf the SG is deleted. wallet: Path to wallet on whose behalf the SG is deleted.
cid: ID of Container where SG is stored. cid: ID of Container where SG is stored.
gid: ID of the Storage Group. gid: ID of the Storage Group.
bearer: Path to Bearer token file. bearer: Path to Bearer token file.
wallet_config: Path to frostfs-cli config file. wallet_config: Path to neofs-cli config file.
timeout: Timeout for an operation.
Returns: Returns:
Tombstone ID of the deleted Storage Group. Tombstone ID of the deleted Storage Group.
""" """
frostfscli = FrostfsCli( neofscli = NeofsCli(shell=shell, neofs_cli_exec_path=NEOFS_CLI_EXEC, config_file=wallet_config)
shell=shell, frostfs_cli_exec_path=FROSTFS_CLI_EXEC, config_file=wallet_config result = neofscli.storagegroup.delete(
)
result = frostfscli.storagegroup.delete(
wallet=wallet, wallet=wallet,
cid=cid, cid=cid,
bearer=bearer, bearer=bearer,
id=gid, id=gid,
rpc_endpoint=endpoint, rpc_endpoint=endpoint,
timeout=timeout,
) )
tombstone_id = result.stdout.strip().split("\n")[1].split(": ")[1] tombstone_id = result.stdout.strip().split("\n")[1].split(": ")[1]
return tombstone_id return tombstone_id
@ -200,7 +180,6 @@ def verify_list_storage_group(
gid: str, gid: str,
bearer: str = None, bearer: str = None,
wallet_config: str = WALLET_CONFIG, wallet_config: str = WALLET_CONFIG,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
): ):
storage_groups = list_storagegroup( storage_groups = list_storagegroup(
shell=shell, shell=shell,
@ -209,7 +188,6 @@ def verify_list_storage_group(
cid=cid, cid=cid,
bearer=bearer, bearer=bearer,
wallet_config=wallet_config, wallet_config=wallet_config,
timeout=timeout,
) )
assert gid in storage_groups assert gid in storage_groups
@ -226,7 +204,6 @@ def verify_get_storage_group(
max_object_size: int, max_object_size: int,
bearer: str = None, bearer: str = None,
wallet_config: str = WALLET_CONFIG, wallet_config: str = WALLET_CONFIG,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
): ):
obj_parts = [] obj_parts = []
endpoint = cluster.default_rpc_endpoint endpoint = cluster.default_rpc_endpoint
@ -240,7 +217,6 @@ def verify_get_storage_group(
nodes=cluster.storage_nodes, nodes=cluster.storage_nodes,
bearer=bearer, bearer=bearer,
wallet_config=wallet_config, wallet_config=wallet_config,
timeout=timeout,
) )
obj_head = head_object( obj_head = head_object(
wallet=wallet, wallet=wallet,
@ -251,7 +227,6 @@ def verify_get_storage_group(
is_raw=True, is_raw=True,
bearer=bearer, bearer=bearer,
wallet_config=wallet_config, wallet_config=wallet_config,
timeout=timeout,
) )
obj_parts = obj_head["header"]["split"]["children"] obj_parts = obj_head["header"]["split"]["children"]
@ -264,7 +239,6 @@ def verify_get_storage_group(
gid=gid, gid=gid,
bearer=bearer, bearer=bearer,
wallet_config=wallet_config, wallet_config=wallet_config,
timeout=timeout,
) )
exp_size = object_size * obj_num exp_size = object_size * obj_num
if object_size < max_object_size: if object_size < max_object_size:

View file

@ -6,14 +6,14 @@
""" """
import logging import logging
from typing import List
import allure import allure
from frostfs_testlib.resources.common import OBJECT_NOT_FOUND import complex_object_actions
from frostfs_testlib.shell import Shell import neofs_verbs
from frostfs_testlib.utils import string_utils from cluster import StorageNode
from grpc_responses import OBJECT_NOT_FOUND, error_matches_status
from pytest_tests.helpers import complex_object_actions, frostfs_verbs from neofs_testlib.shell import Shell
from pytest_tests.helpers.cluster import StorageNode
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
@ -66,7 +66,7 @@ def get_simple_object_copies(
copies = 0 copies = 0
for node in nodes: for node in nodes:
try: try:
response = frostfs_verbs.head_object( response = neofs_verbs.head_object(
wallet, cid, oid, shell=shell, endpoint=node.get_rpc_endpoint(), is_direct=True wallet, cid, oid, shell=shell, endpoint=node.get_rpc_endpoint(), is_direct=True
) )
if response: if response:
@ -123,7 +123,7 @@ def get_nodes_with_object(
wallet = node.get_wallet_path() wallet = node.get_wallet_path()
wallet_config = node.get_wallet_config_path() wallet_config = node.get_wallet_config_path()
try: try:
res = frostfs_verbs.head_object( res = neofs_verbs.head_object(
wallet, wallet,
cid, cid,
oid, oid,
@ -160,13 +160,13 @@ def get_nodes_without_object(
nodes_list = [] nodes_list = []
for node in nodes: for node in nodes:
try: try:
res = frostfs_verbs.head_object( res = neofs_verbs.head_object(
wallet, cid, oid, shell=shell, endpoint=node.get_rpc_endpoint(), is_direct=True wallet, cid, oid, shell=shell, endpoint=node.get_rpc_endpoint(), is_direct=True
) )
if res is None: if res is None:
nodes_list.append(node) nodes_list.append(node)
except Exception as err: except Exception as err:
if string_utils.is_str_match_pattern(err, OBJECT_NOT_FOUND): if error_matches_status(err, OBJECT_NOT_FOUND):
nodes_list.append(node) nodes_list.append(node)
else: else:
raise Exception(f"Got error {err} on head object command") from err raise Exception(f"Got error {err} on head object command") from err

View file

@ -2,10 +2,9 @@ import json
import logging import logging
import allure import allure
from frostfs_testlib.shell import Shell from neo3 import wallet
from neo3.wallet import wallet from neofs_testlib.shell import Shell
from neofs_verbs import head_object
from pytest_tests.helpers.frostfs_verbs import head_object
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")

View file

@ -11,7 +11,7 @@ COMPLEX_OBJECT_TAIL_SIZE = os.getenv("COMPLEX_OBJECT_TAIL_SIZE", "1000")
MAINNET_BLOCK_TIME = os.getenv("MAINNET_BLOCK_TIME", "1s") MAINNET_BLOCK_TIME = os.getenv("MAINNET_BLOCK_TIME", "1s")
MAINNET_TIMEOUT = os.getenv("MAINNET_TIMEOUT", "1min") MAINNET_TIMEOUT = os.getenv("MAINNET_TIMEOUT", "1min")
MORPH_BLOCK_TIME = os.getenv("MORPH_BLOCK_TIME", "1s") MORPH_BLOCK_TIME = os.getenv("MORPH_BLOCK_TIME", "1s")
FROSTFS_CONTRACT_CACHE_TIMEOUT = os.getenv("FROSTFS_CONTRACT_CACHE_TIMEOUT", "30s") NEOFS_CONTRACT_CACHE_TIMEOUT = os.getenv("NEOFS_CONTRACT_CACHE_TIMEOUT", "30s")
# Time interval that allows a GC pass on storage node (this includes GC sleep interval # Time interval that allows a GC pass on storage node (this includes GC sleep interval
# of 1min plus 15 seconds for GC pass itself) # of 1min plus 15 seconds for GC pass itself)
@ -19,10 +19,10 @@ STORAGE_GC_TIME = os.getenv("STORAGE_GC_TIME", "75s")
GAS_HASH = os.getenv("GAS_HASH", "0xd2a4cff31913016155e38e474a2c06d08be276cf") GAS_HASH = os.getenv("GAS_HASH", "0xd2a4cff31913016155e38e474a2c06d08be276cf")
FROSTFS_CONTRACT = os.getenv("FROSTFS_IR_CONTRACTS_FROSTFS") NEOFS_CONTRACT = os.getenv("NEOFS_IR_CONTRACTS_NEOFS")
ASSETS_DIR = os.getenv("ASSETS_DIR", "TemporaryDir") ASSETS_DIR = os.getenv("ASSETS_DIR", "TemporaryDir")
DEVENV_PATH = os.getenv("DEVENV_PATH", os.path.join("..", "frostfs-dev-env")) DEVENV_PATH = os.getenv("DEVENV_PATH", os.path.join("..", "neofs-dev-env"))
# Password of wallet owned by user on behalf of whom we are running tests # Password of wallet owned by user on behalf of whom we are running tests
WALLET_PASS = os.getenv("WALLET_PASS", "") WALLET_PASS = os.getenv("WALLET_PASS", "")
@ -30,12 +30,12 @@ WALLET_PASS = os.getenv("WALLET_PASS", "")
# Paths to CLI executables on machine that runs tests # Paths to CLI executables on machine that runs tests
NEOGO_EXECUTABLE = os.getenv("NEOGO_EXECUTABLE", "neo-go") NEOGO_EXECUTABLE = os.getenv("NEOGO_EXECUTABLE", "neo-go")
FROSTFS_CLI_EXEC = os.getenv("FROSTFS_CLI_EXEC", "frostfs-cli") NEOFS_CLI_EXEC = os.getenv("NEOFS_CLI_EXEC", "neofs-cli")
FROSTFS_AUTHMATE_EXEC = os.getenv("FROSTFS_AUTHMATE_EXEC", "frostfs-authmate") NEOFS_AUTHMATE_EXEC = os.getenv("NEOFS_AUTHMATE_EXEC", "neofs-authmate")
FROSTFS_ADM_EXEC = os.getenv("FROSTFS_ADM_EXEC", "frostfs-adm") NEOFS_ADM_EXEC = os.getenv("NEOFS_ADM_EXEC", "neofs-adm")
# Config for frostfs-adm utility. Optional if tests are running against devenv # Config for neofs-adm utility. Optional if tests are running against devenv
FROSTFS_ADM_CONFIG_PATH = os.getenv("FROSTFS_ADM_CONFIG_PATH") NEOFS_ADM_CONFIG_PATH = os.getenv("NEOFS_ADM_CONFIG_PATH")
FREE_STORAGE = os.getenv("FREE_STORAGE", "false").lower() == "true" FREE_STORAGE = os.getenv("FREE_STORAGE", "false").lower() == "true"
BIN_VERSIONS_FILE = os.getenv("BIN_VERSIONS_FILE") BIN_VERSIONS_FILE = os.getenv("BIN_VERSIONS_FILE")
@ -45,8 +45,6 @@ STORAGE_NODE_SERVICE_NAME_REGEX = r"s\d\d"
HTTP_GATE_SERVICE_NAME_REGEX = r"http-gate\d\d" HTTP_GATE_SERVICE_NAME_REGEX = r"http-gate\d\d"
S3_GATE_SERVICE_NAME_REGEX = r"s3-gate\d\d" S3_GATE_SERVICE_NAME_REGEX = r"s3-gate\d\d"
CLI_DEFAULT_TIMEOUT = os.getenv("CLI_DEFAULT_TIMEOUT", None)
# Generate wallet configs # Generate wallet configs
# TODO: we should move all info about wallet configs to fixtures # TODO: we should move all info about wallet configs to fixtures
WALLET_CONFIG = os.path.join(os.getcwd(), "wallet_config.yml") WALLET_CONFIG = os.path.join(os.getcwd(), "wallet_config.yml")

View file

@ -0,0 +1,23 @@
EACL_OBJ_FILTERS = {
"$Object:objectID": "objectID",
"$Object:containerID": "containerID",
"$Object:ownerID": "ownerID",
"$Object:creationEpoch": "creationEpoch",
"$Object:payloadLength": "payloadLength",
"$Object:payloadHash": "payloadHash",
"$Object:objectType": "objectType",
"$Object:homomorphicHash": "homomorphicHash",
"$Object:version": "version",
}
VERB_FILTER_DEP = {
"$Object:objectID": ["GET", "HEAD", "DELETE", "RANGE", "RANGEHASH"],
"$Object:containerID": ["GET", "PUT", "HEAD", "DELETE", "SEARCH", "RANGE", "RANGEHASH"],
"$Object:ownerID": ["GET", "HEAD"],
"$Object:creationEpoch": ["GET", "PUT", "HEAD"],
"$Object:payloadLength": ["GET", "PUT", "HEAD"],
"$Object:payloadHash": ["GET", "PUT", "HEAD"],
"$Object:objectType": ["GET", "PUT", "HEAD"],
"$Object:homomorphicHash": ["GET", "PUT", "HEAD"],
"$Object:version": ["GET", "PUT", "HEAD"],
}

View file

@ -0,0 +1,11 @@
# ACLs with final flag
PUBLIC_ACL_F = "1FBFBFFF"
PRIVATE_ACL_F = "1C8C8CCC"
READONLY_ACL_F = "1FBF8CFF"
# ACLs without final flag set
PUBLIC_ACL = "0FBFBFFF"
INACCESSIBLE_ACL = "40000000"
STICKYBIT_PUB_ACL = "3FFFFFFF"
EACL_PUBLIC_READ_WRITE = "eacl-public-read-write"

6
venv/local-pytest/environment.sh Executable file → Normal file
View file

@ -1,6 +1,8 @@
# DevEnv variables # DevEnv variables
export NEOFS_MORPH_DISABLE_CACHE=true export NEOFS_MORPH_DISABLE_CACHE=true
export DEVENV_PATH="${DEVENV_PATH:-${VIRTUAL_ENV}/../../frostfs-dev-env}" export DEVENV_PATH="${DEVENV_PATH:-${VIRTUAL_ENV}/../../neofs-dev-env}"
pushd $DEVENV_PATH > /dev/null pushd $DEVENV_PATH > /dev/null
export `make env` export `make env`
popd > /dev/null popd > /dev/null
export PYTHONPATH=${PYTHONPATH}:${VIRTUAL_ENV}/../robot/resources/lib/:${VIRTUAL_ENV}/../robot/resources/lib/python_keywords:${VIRTUAL_ENV}/../robot/resources/lib/robot:${VIRTUAL_ENV}/../robot/variables:${VIRTUAL_ENV}/../pytest_tests/helpers:${VIRTUAL_ENV}/../pytest_tests/steps:${VIRTUAL_ENV}/../pytest_tests/resources