Compare commits
40 commits
master
...
notsomaste
Author | SHA1 | Date | |
---|---|---|---|
85ea5de04c | |||
6e4558d792 | |||
791c461da1 | |||
|
b869e8751b | ||
69199d301b | |||
5083b5adad | |||
a2d272eaee | |||
6367e5e8ed | |||
4b59c1f327 | |||
6bf672c203 | |||
359bb6be90 | |||
|
5d7fca14ee | ||
7a11560000 | |||
|
37cc85028e | ||
79cb3c3698 | |||
146aaa371f | |||
d357241a61 | |||
dc6e521f59 | |||
29a8c1e252 | |||
32b010df67 | |||
c33857fd80 | |||
523e7ebac0 | |||
d438748b2a | |||
de6434934d | |||
25a9443209 | |||
c670893805 | |||
efc5f271bf | |||
294e0fbae3 | |||
ac987c49aa | |||
bd78c9f4c0 | |||
8aab12d5a8 | |||
33e20b0562 | |||
cd98d362e3 | |||
b9b8c4e1f8 | |||
e99cc7d05d | |||
071914b45c | |||
12d8b1b10b | |||
089ac609c3 | |||
adc63b0221 | |||
7d3fc5d15c |
94 changed files with 2211 additions and 1450 deletions
|
@ -5,98 +5,98 @@ hosts:
|
|||
- name: s01
|
||||
attributes:
|
||||
container_name: s01
|
||||
config_path: ../frostfs-dev-env/services/storage/.storage.env
|
||||
wallet_path: ../frostfs-dev-env/services/storage/wallet01.json
|
||||
config_path: ../neofs-dev-env/services/storage/.storage.env
|
||||
wallet_path: ../neofs-dev-env/services/storage/wallet01.json
|
||||
local_config_path: ./TemporaryDir/empty-password.yml
|
||||
local_wallet_path: ../frostfs-dev-env/services/storage/wallet01.json
|
||||
local_wallet_path: ../neofs-dev-env/services/storage/wallet01.json
|
||||
wallet_password: ""
|
||||
volume_name: storage_storage_s01
|
||||
endpoint_data0: s01.frostfs.devenv:8080
|
||||
control_endpoint: s01.frostfs.devenv:8081
|
||||
endpoint_data0: s01.neofs.devenv:8080
|
||||
control_endpoint: s01.neofs.devenv:8081
|
||||
un_locode: "RU MOW"
|
||||
- name: s02
|
||||
attributes:
|
||||
container_name: s02
|
||||
config_path: ../frostfs-dev-env/services/storage/.storage.env
|
||||
wallet_path: ../frostfs-dev-env/services/storage/wallet02.json
|
||||
config_path: ../neofs-dev-env/services/storage/.storage.env
|
||||
wallet_path: ../neofs-dev-env/services/storage/wallet02.json
|
||||
local_config_path: ./TemporaryDir/empty-password.yml
|
||||
local_wallet_path: ../frostfs-dev-env/services/storage/wallet02.json
|
||||
local_wallet_path: ../neofs-dev-env/services/storage/wallet02.json
|
||||
wallet_password: ""
|
||||
volume_name: storage_storage_s02
|
||||
endpoint_data0: s02.frostfs.devenv:8080
|
||||
control_endpoint: s02.frostfs.devenv:8081
|
||||
endpoint_data0: s02.neofs.devenv:8080
|
||||
control_endpoint: s02.neofs.devenv:8081
|
||||
un_locode: "RU LED"
|
||||
- name: s03
|
||||
attributes:
|
||||
container_name: s03
|
||||
config_path: ../frostfs-dev-env/services/storage/.storage.env
|
||||
wallet_path: ../frostfs-dev-env/services/storage/wallet03.json
|
||||
config_path: ../neofs-dev-env/services/storage/.storage.env
|
||||
wallet_path: ../neofs-dev-env/services/storage/wallet03.json
|
||||
local_config_path: ./TemporaryDir/empty-password.yml
|
||||
local_wallet_path: ../frostfs-dev-env/services/storage/wallet03.json
|
||||
local_wallet_path: ../neofs-dev-env/services/storage/wallet03.json
|
||||
wallet_password: ""
|
||||
volume_name: storage_storage_s03
|
||||
endpoint_data0: s03.frostfs.devenv:8080
|
||||
control_endpoint: s03.frostfs.devenv:8081
|
||||
endpoint_data0: s03.neofs.devenv:8080
|
||||
control_endpoint: s03.neofs.devenv:8081
|
||||
un_locode: "SE STO"
|
||||
- name: s04
|
||||
attributes:
|
||||
container_name: s04
|
||||
config_path: ../frostfs-dev-env/services/storage/.storage.env
|
||||
wallet_path: ../frostfs-dev-env/services/storage/wallet04.json
|
||||
config_path: ../neofs-dev-env/services/storage/.storage.env
|
||||
wallet_path: ../neofs-dev-env/services/storage/wallet04.json
|
||||
local_config_path: ./TemporaryDir/empty-password.yml
|
||||
local_wallet_path: ../frostfs-dev-env/services/storage/wallet04.json
|
||||
local_wallet_path: ../neofs-dev-env/services/storage/wallet04.json
|
||||
wallet_password: ""
|
||||
volume_name: storage_storage_s04
|
||||
endpoint_data0: s04.frostfs.devenv:8080
|
||||
control_endpoint: s04.frostfs.devenv:8081
|
||||
endpoint_data0: s04.neofs.devenv:8080
|
||||
control_endpoint: s04.neofs.devenv:8081
|
||||
un_locode: "FI HEL"
|
||||
- name: s3-gate01
|
||||
attributes:
|
||||
container_name: s3_gate
|
||||
config_path: ../frostfs-dev-env/services/s3_gate/.s3.env
|
||||
wallet_path: ../frostfs-dev-env/services/s3_gate/wallet.json
|
||||
config_path: ../neofs-dev-env/services/s3_gate/.s3.env
|
||||
wallet_path: ../neofs-dev-env/services/s3_gate/wallet.json
|
||||
local_config_path: ./TemporaryDir/password-s3.yml
|
||||
local_wallet_path: ../frostfs-dev-env/services/s3_gate/wallet.json
|
||||
local_wallet_path: ../neofs-dev-env/services/s3_gate/wallet.json
|
||||
wallet_password: "s3"
|
||||
endpoint_data0: https://s3.frostfs.devenv:8080
|
||||
endpoint_data0: https://s3.neofs.devenv:8080
|
||||
- name: http-gate01
|
||||
attributes:
|
||||
container_name: http_gate
|
||||
config_path: ../frostfs-dev-env/services/http_gate/.http.env
|
||||
wallet_path: ../frostfs-dev-env/services/http_gate/wallet.json
|
||||
config_path: ../neofs-dev-env/services/http_gate/.http.env
|
||||
wallet_path: ../neofs-dev-env/services/http_gate/wallet.json
|
||||
local_config_path: ./TemporaryDir/password-other.yml
|
||||
local_wallet_path: ../frostfs-dev-env/services/http_gate/wallet.json
|
||||
local_wallet_path: ../neofs-dev-env/services/http_gate/wallet.json
|
||||
wallet_password: "one"
|
||||
endpoint_data0: http://http.frostfs.devenv
|
||||
endpoint_data0: http://http.neofs.devenv
|
||||
- name: ir01
|
||||
attributes:
|
||||
container_name: ir01
|
||||
config_path: ../frostfs-dev-env/services/ir/.ir.env
|
||||
wallet_path: ../frostfs-dev-env/services/ir/az.json
|
||||
config_path: ../neofs-dev-env/services/ir/.ir.env
|
||||
wallet_path: ../neofs-dev-env/services/ir/az.json
|
||||
local_config_path: ./TemporaryDir/password-other.yml
|
||||
local_wallet_path: ../frostfs-dev-env/services/ir/az.json
|
||||
local_wallet_path: ../neofs-dev-env/services/ir/az.json
|
||||
wallet_password: "one"
|
||||
- name: morph-chain01
|
||||
attributes:
|
||||
container_name: morph_chain
|
||||
config_path: ../frostfs-dev-env/services/morph_chain/protocol.privnet.yml
|
||||
wallet_path: ../frostfs-dev-env/services/morph_chain/node-wallet.json
|
||||
config_path: ../neofs-dev-env/services/morph_chain/protocol.privnet.yml
|
||||
wallet_path: ../neofs-dev-env/services/morph_chain/node-wallet.json
|
||||
local_config_path: ./TemporaryDir/password-other.yml
|
||||
local_wallet_path: ../frostfs-dev-env/services/morph_chain/node-wallet.json
|
||||
local_wallet_path: ../neofs-dev-env/services/morph_chain/node-wallet.json
|
||||
wallet_password: "one"
|
||||
endpoint_internal0: http://morph-chain.frostfs.devenv:30333
|
||||
endpoint_internal0: http://morph-chain.neofs.devenv:30333
|
||||
- name: main-chain01
|
||||
attributes:
|
||||
container_name: main_chain
|
||||
config_path: ../frostfs-dev-env/services/chain/protocol.privnet.yml
|
||||
wallet_path: ../frostfs-dev-env/services/chain/node-wallet.json
|
||||
config_path: ../neofs-dev-env/services/chain/protocol.privnet.yml
|
||||
wallet_path: ../neofs-dev-env/services/chain/node-wallet.json
|
||||
local_config_path: ./TemporaryDir/password-other.yml
|
||||
local_wallet_path: ../frostfs-dev-env/services/chain/node-wallet.json
|
||||
local_wallet_path: ../neofs-dev-env/services/chain/node-wallet.json
|
||||
wallet_password: "one"
|
||||
endpoint_internal0: http://main-chain.frostfs.devenv:30333
|
||||
endpoint_internal0: http://main-chain.neofs.devenv:30333
|
||||
- name: coredns01
|
||||
attributes:
|
||||
container_name: coredns
|
||||
clis:
|
||||
- name: frostfs-cli
|
||||
exec_path: frostfs-cli
|
||||
- name: neofs-cli
|
||||
exec_path: neofs-cli
|
||||
|
|
2
.github/CODEOWNERS
vendored
2
.github/CODEOWNERS
vendored
|
@ -1 +1 @@
|
|||
* @vdomnich-yadro @dansingjulia @yadro-vavdeev @alexchetaev @abereziny
|
||||
* @vdomnich-yadro @dansingjulia @yadro-vavdeev @abereziny
|
||||
|
|
8
.gitignore
vendored
8
.gitignore
vendored
|
@ -1,11 +1,5 @@
|
|||
# ignore IDE files
|
||||
.vscode
|
||||
.idea
|
||||
|
||||
.DS_Store
|
||||
|
||||
venv_macos
|
||||
|
||||
|
||||
# ignore test results
|
||||
**/log.html
|
||||
|
@ -26,4 +20,4 @@ TemporaryDir/*
|
|||
artifacts/*
|
||||
docs/*
|
||||
venv.*/*
|
||||
wallet_config.yml
|
||||
/*wallet_config.yml
|
||||
|
|
|
@ -5,7 +5,7 @@ repos:
|
|||
- id: black
|
||||
language_version: python3.9
|
||||
- repo: https://github.com/pycqa/isort
|
||||
rev: 5.12.0
|
||||
rev: 5.10.1
|
||||
hooks:
|
||||
- id: isort
|
||||
name: isort (python)
|
||||
|
|
|
@ -3,8 +3,8 @@
|
|||
First, thank you for contributing! We love and encourage pull requests from
|
||||
everyone. Please follow the guidelines:
|
||||
|
||||
- Check the open [issues](https://github.com/TrueCloudLab/frostfs-testcases/issues) and
|
||||
[pull requests](https://github.com/TrueCloudLab/frostfs-testcases/pulls) for existing
|
||||
- Check the open [issues](https://github.com/nspcc-dev/neofs-testcases/issues) and
|
||||
[pull requests](https://github.com/nspcc-dev/neofs-testcases/pulls) for existing
|
||||
discussions.
|
||||
|
||||
- Open an issue first, to discuss a new feature or enhancement.
|
||||
|
@ -22,13 +22,13 @@ everyone. Please follow the guidelines:
|
|||
|
||||
## Development Workflow
|
||||
|
||||
Start by forking the `frostfs-testcases` repository, make changes in a branch and then
|
||||
Start by forking the `neofs-testcases` repository, make changes in a branch and then
|
||||
send a pull request. We encourage pull requests to discuss code changes. Here
|
||||
are the steps in details:
|
||||
|
||||
### Set up your GitHub Repository
|
||||
|
||||
Fork [FrosfFS testcases upstream](https://github.com/TrueCloudLab/frostfs-testcases/fork) source
|
||||
Fork [NeoFS testcases upstream](https://github.com/nspcc-dev/neofs-testcases/fork) source
|
||||
repository to your own personal repository. Copy the URL of your fork and clone it:
|
||||
|
||||
```shell
|
||||
|
@ -36,20 +36,33 @@ $ git clone <url of your fork>
|
|||
```
|
||||
|
||||
### Set up git remote as ``upstream``
|
||||
```sh
|
||||
$ cd frostfs-testcases
|
||||
$ git remote add upstream https://github.com/TrueCloudLab/frostfs-testcases
|
||||
|
||||
```shell
|
||||
$ cd neofs-testcases
|
||||
$ git remote add upstream https://github.com/nspcc-dev/neofs-testcases
|
||||
$ git fetch upstream
|
||||
```
|
||||
|
||||
### Set up development environment
|
||||
|
||||
To setup development environment for `frosfs-testcases`, please, take the following steps:
|
||||
To setup development environment for `neofs-testcases`, please, take the following steps:
|
||||
1. Prepare virtualenv
|
||||
|
||||
```shell
|
||||
$ make venv
|
||||
$ source frostfs-testcases-3.10/bin/activate
|
||||
$ virtualenv --python=python3.9 venv
|
||||
$ source venv/bin/activate
|
||||
```
|
||||
|
||||
2. Install all dependencies:
|
||||
|
||||
```shell
|
||||
$ pip install -r requirements.txt
|
||||
```
|
||||
|
||||
3. Setup pre-commit hooks to run code formatters on staged files before you run a `git commit` command:
|
||||
|
||||
```shell
|
||||
$ pre-commit install
|
||||
```
|
||||
|
||||
Optionally you might want to integrate code formatters with your code editor to apply formatters to code files as you go:
|
||||
|
|
45
Makefile
45
Makefile
|
@ -1,29 +1,28 @@
|
|||
SHELL := /bin/bash
|
||||
PYTHON_VERSION := 3.10
|
||||
VENV_NAME = frostfs-testcases-${PYTHON_VERSION}
|
||||
VENV_DIR := venv.${VENV_NAME}
|
||||
#!/usr/bin/make -f
|
||||
|
||||
current_dir := $(shell pwd)
|
||||
.DEFAULT_GOAL := help
|
||||
|
||||
venv: create requirements paths precommit
|
||||
@echo Ready
|
||||
SHELL ?= bash
|
||||
|
||||
precommit:
|
||||
@echo Isntalling pre-commit hooks
|
||||
. ${VENV_DIR}/bin/activate && pre-commit install
|
||||
VENVS = $(shell ls -1d venv/*/ | sort -u | xargs basename -a)
|
||||
|
||||
paths:
|
||||
@echo Append paths for project
|
||||
@echo Virtual environment: ${VENV_DIR}
|
||||
@sudo rm -rf ${VENV_DIR}/lib/python${PYTHON_VERSION}/site-packages/_paths.pth
|
||||
@sudo touch ${VENV_DIR}/lib/python${PYTHON_VERSION}/site-packages/_paths.pth
|
||||
@echo ${current_dir} | sudo tee ${VENV_DIR}/lib/python${PYTHON_VERSION}/site-packages/_paths.pth
|
||||
.PHONY: all
|
||||
all: venvs
|
||||
|
||||
create:
|
||||
@echo Create virtual environment for
|
||||
virtualenv --python=python${PYTHON_VERSION} --prompt=${VENV_NAME} ${VENV_DIR}
|
||||
include venv_template.mk
|
||||
|
||||
requirements:
|
||||
@echo Isntalling pip requirements
|
||||
. ${VENV_DIR}/bin/activate && pip install -e ../frostfs-testlib
|
||||
. ${VENV_DIR}/bin/activate && pip install -Ur pytest_tests/requirements.txt
|
||||
.PHONY: venvs
|
||||
venvs:
|
||||
$(foreach venv,$(VENVS),venv.$(venv))
|
||||
|
||||
$(foreach venv,$(VENVS),$(eval $(call VENV_template,$(venv))))
|
||||
|
||||
clean:
|
||||
rm -rf venv.*
|
||||
|
||||
pytest-local:
|
||||
@echo "⇒ Run Pytest"
|
||||
python -m pytest pytest_tests/testsuites/
|
||||
|
||||
help:
|
||||
@echo "⇒ run Run testcases ${R}"
|
||||
|
|
60
README.md
60
README.md
|
@ -2,58 +2,70 @@
|
|||
|
||||
Tests written with PyTest Framework are located under `pytest_tests/testsuites` directory.
|
||||
|
||||
These tests rely on resources and utility modules that have been originally developed for Pytest Framework.
|
||||
These tests rely on resources and utility modules that have been originally developed for Robot Framework:
|
||||
|
||||
`robot/resources/files` - static files that are used in tests' commands.
|
||||
|
||||
`robot/resources/lib/` - common Python libraries that provide utility functions used as building blocks in tests.
|
||||
|
||||
`robot/variables/` - constants and configuration variables for tests.
|
||||
|
||||
## Testcases execution
|
||||
|
||||
### Initial preparation
|
||||
|
||||
1. Install frostfs-cli
|
||||
- `git clone git@github.com:TrueCloudLab/frostfs-node.git`
|
||||
- `cd frostfs-node`
|
||||
1. Install neofs-cli
|
||||
- `git clone git@github.com:nspcc-dev/neofs-node.git`
|
||||
- `cd neofs-node`
|
||||
- `make`
|
||||
- `sudo cp bin/frostfs-cli /usr/local/bin/frostfs-cli`
|
||||
- `sudo cp bin/neofs-cli /usr/local/bin/neofs-cli`
|
||||
|
||||
2. Install frostfs-authmate
|
||||
- `git clone git@github.com:TrueCloudLab/frostfs-s3-gw.git`
|
||||
- `cd frostfs-s3-gw`
|
||||
2. Install neofs-authmate
|
||||
- `git clone git@github.com:nspcc-dev/neofs-s3-gw.git`
|
||||
- `cd neofs-s3-gw`
|
||||
- `make`
|
||||
- `sudo cp bin/frostfs-s3-authmate /usr/local/bin/frostfs-authmate`
|
||||
- `sudo cp bin/neofs-authmate /usr/local/bin/neofs-authmate`
|
||||
|
||||
3. Install neo-go
|
||||
- `git clone git@github.com:nspcc-dev/neo-go.git`
|
||||
- `cd neo-go`
|
||||
- `git checkout v0.101.0` (or the current version in the frostfs-dev-env)
|
||||
- `git checkout v0.92.0` (or the current version in the neofs-dev-env)
|
||||
- `make`
|
||||
- `sudo cp bin/neo-go /usr/local/bin/neo-go`
|
||||
or download binary from releases: https://github.com/nspcc-dev/neo-go/releases
|
||||
|
||||
4. Clone frostfs-dev-env
|
||||
`git clone git@github.com:TrueCloudLab/frostfs-dev-env.git`
|
||||
4. Clone neofs-dev-env
|
||||
`git clone git@github.com:nspcc-dev/neofs-dev-env.git`
|
||||
|
||||
Note that we expect frostfs-dev-env to be located under
|
||||
the `<testcases_root_dir>/../frostfs-dev-env` directory. If you put this repo in any other place,
|
||||
manually set the full path to frostfs-dev-env in the environment variable `DEVENV_PATH` at this step.
|
||||
Note that we expect neofs-dev-env to be located under
|
||||
the `<testcases_root_dir>/../neofs-dev-env` directory. If you put this repo in any other place,
|
||||
manually set the full path to neofs-dev-env in the environment variable `DEVENV_PATH` at this step.
|
||||
|
||||
5. Make sure you have installed all the following prerequisites on your machine
|
||||
5. Make sure you have installed all of the following prerequisites on your machine
|
||||
|
||||
```
|
||||
make
|
||||
python3.10
|
||||
python3.10-dev
|
||||
python3.9
|
||||
python3.9-dev
|
||||
libssl-dev
|
||||
```
|
||||
As we use frostfs-dev-env, you'll also need to install
|
||||
[prerequisites](https://github.com/TrueCloudLab/frostfs-dev-env#prerequisites) of this repository.
|
||||
As we use neofs-dev-env, you'll also need to install
|
||||
[prerequisites](https://github.com/nspcc-dev/neofs-dev-env#prerequisites) of this repository.
|
||||
|
||||
6. Prepare virtualenv
|
||||
|
||||
```shell
|
||||
$ make venv
|
||||
$ source venv.frostfs-testcases-3.10/bin/activate
|
||||
$ make venv.local-pytest
|
||||
$ . venv.local-pytest/bin/activate
|
||||
```
|
||||
|
||||
7. Optionally you might want to integrate code formatters with your code editor to apply formatters to code files as you go:
|
||||
7. Setup pre-commit hooks to run code formatters on staged files before you run a `git commit` command:
|
||||
|
||||
```shell
|
||||
$ pre-commit install
|
||||
```
|
||||
|
||||
Optionally you might want to integrate code formatters with your code editor to apply formatters to code files as you go:
|
||||
* isort is supported by [PyCharm](https://plugins.jetbrains.com/plugin/15434-isortconnect), [VS Code](https://cereblanco.medium.com/setup-black-and-isort-in-vscode-514804590bf9). Plugins exist for other IDEs/editors as well.
|
||||
* black can be integrated with multiple editors, please, instructions are available [here](https://black.readthedocs.io/en/stable/integrations/editors.html).
|
||||
|
||||
|
@ -110,7 +122,7 @@ $ docker run -p 5050:5050 -e CHECK_RESULTS_EVERY_SECONDS=30 -e KEEP_HISTORY=1 \
|
|||
|
||||
Then, you can check the allure report in your browser [by this link](http://localhost:5050/allure-docker-service/projects/default/reports/latest/index.html?redirect=false)
|
||||
|
||||
NOTE: feel free to select a different location for `allure-reports` directory, there is no requirement to have it inside `frostfs-testcases`. For example, you can place it under `/tmp` path.
|
||||
NOTE: feel free to select a different location for `allure-reports` directory, there is no requirement to have it inside `neofs-testcases`. For example, you can place it under `/tmp` path.
|
||||
|
||||
# Contributing
|
||||
|
||||
|
|
|
@ -1,8 +1,8 @@
|
|||
[tool.isort]
|
||||
profile = "black"
|
||||
src_paths = ["pytest_tests"]
|
||||
src_paths = ["neofs-keywords", "pytest_tests", "robot"]
|
||||
line_length = 100
|
||||
|
||||
[tool.black]
|
||||
line-length = 100
|
||||
target-version = ["py310"]
|
||||
target-version = ["py39"]
|
||||
|
|
|
@ -5,9 +5,8 @@ from datetime import datetime
|
|||
from typing import Optional
|
||||
|
||||
import allure
|
||||
|
||||
from pytest_tests.helpers.cli_helpers import _cmd_run
|
||||
from pytest_tests.resources.common import ASSETS_DIR
|
||||
from cli_helpers import _cmd_run
|
||||
from common import ASSETS_DIR
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
REGULAR_TIMEOUT = 90
|
||||
|
|
|
@ -1,11 +1,10 @@
|
|||
import logging
|
||||
import re
|
||||
|
||||
from frostfs_testlib.cli import FrostfsAdm, FrostfsCli
|
||||
from frostfs_testlib.hosting import Hosting
|
||||
from frostfs_testlib.shell import Shell
|
||||
|
||||
from pytest_tests.resources.common import FROSTFS_ADM_EXEC, FROSTFS_CLI_EXEC, WALLET_CONFIG
|
||||
from common import NEOFS_ADM_EXEC, NEOFS_CLI_EXEC, WALLET_CONFIG
|
||||
from neofs_testlib.cli import NeofsAdm, NeofsCli
|
||||
from neofs_testlib.hosting import Hosting
|
||||
from neofs_testlib.shell import Shell
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
|
||||
|
@ -13,18 +12,18 @@ logger = logging.getLogger("NeoLogger")
|
|||
def get_local_binaries_versions(shell: Shell) -> dict[str, str]:
|
||||
versions = {}
|
||||
|
||||
for binary in ["neo-go", "frostfs-authmate"]:
|
||||
for binary in ["neo-go", "neofs-authmate"]:
|
||||
out = shell.exec(f"{binary} --version").stdout
|
||||
versions[binary] = _parse_version(out)
|
||||
|
||||
frostfs_cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, WALLET_CONFIG)
|
||||
versions["frostfs-cli"] = _parse_version(frostfs_cli.version.get().stdout)
|
||||
neofs_cli = NeofsCli(shell, NEOFS_CLI_EXEC, WALLET_CONFIG)
|
||||
versions["neofs-cli"] = _parse_version(neofs_cli.version.get().stdout)
|
||||
|
||||
try:
|
||||
frostfs_adm = FrostfsAdm(shell, FROSTFS_ADM_EXEC)
|
||||
versions["frostfs-adm"] = _parse_version(frostfs_adm.version.get().stdout)
|
||||
neofs_adm = NeofsAdm(shell, NEOFS_ADM_EXEC)
|
||||
versions["neofs-adm"] = _parse_version(neofs_adm.version.get().stdout)
|
||||
except RuntimeError:
|
||||
logger.info(f"frostfs-adm not installed")
|
||||
logger.info(f"neofs-adm not installed")
|
||||
|
||||
out = shell.exec("aws --version").stdout
|
||||
out_lines = out.split("\n")
|
|
@ -3,11 +3,12 @@ import re
|
|||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import data_formatters
|
||||
import yaml
|
||||
from frostfs_testlib.blockchain import RPCClient
|
||||
from frostfs_testlib.hosting import Host, Hosting
|
||||
from frostfs_testlib.hosting.config import ServiceConfig
|
||||
from frostfs_testlib.utils import wallet_utils
|
||||
from neofs_testlib.blockchain import RPCClient
|
||||
from neofs_testlib.hosting import Host, Hosting
|
||||
from neofs_testlib.hosting.config import ServiceConfig
|
||||
from test_control import wait_for_success
|
||||
|
||||
|
||||
@dataclass
|
||||
|
@ -45,9 +46,11 @@ class NodeBase:
|
|||
def label(self) -> str:
|
||||
return self.name
|
||||
|
||||
@wait_for_success(60, 1)
|
||||
def start_service(self):
|
||||
self.host.start_service(self.name)
|
||||
|
||||
@wait_for_success(60, 1)
|
||||
def stop_service(self):
|
||||
self.host.stop_service(self.name)
|
||||
|
||||
|
@ -85,7 +88,7 @@ class NodeBase:
|
|||
def get_wallet_public_key(self):
|
||||
storage_wallet_path = self.get_wallet_path()
|
||||
storage_wallet_pass = self.get_wallet_password()
|
||||
return wallet_utils.get_wallet_public_key(storage_wallet_path, storage_wallet_pass)
|
||||
return data_formatters.get_wallet_public_key(storage_wallet_path, storage_wallet_pass)
|
||||
|
||||
def _get_attribute(self, attribute_name: str, default_attribute_name: str = None) -> list[str]:
|
||||
config = self.host.get_service_config(self.name)
|
||||
|
@ -107,7 +110,7 @@ class InnerRingNode(NodeBase):
|
|||
Inner ring node is not always the same as physical host (or physical node, if you will):
|
||||
It can be service running in a container or on physical host
|
||||
For testing perspective, it's not relevant how it is actually running,
|
||||
since frostfs network will still treat it as "node"
|
||||
since neofs network will still treat it as "node"
|
||||
"""
|
||||
|
||||
pass
|
||||
|
@ -146,7 +149,7 @@ class MorphChain(NodeBase):
|
|||
Consensus node is not always the same as physical host (or physical node, if you will):
|
||||
It can be service running in a container or on physical host
|
||||
For testing perspective, it's not relevant how it is actually running,
|
||||
since frostfs network will still treat it as "node"
|
||||
since neofs network will still treat it as "node"
|
||||
"""
|
||||
|
||||
rpc_client: RPCClient = None
|
||||
|
@ -169,7 +172,7 @@ class MainChain(NodeBase):
|
|||
Consensus node is not always the same as physical host:
|
||||
It can be service running in a container or on physical host (or physical node, if you will):
|
||||
For testing perspective, it's not relevant how it is actually running,
|
||||
since frostfs network will still treat it as "node"
|
||||
since neofs network will still treat it as "node"
|
||||
"""
|
||||
|
||||
rpc_client: RPCClient = None
|
||||
|
@ -192,7 +195,7 @@ class StorageNode(NodeBase):
|
|||
Storage node is not always the same as physical host:
|
||||
It can be service running in a container or on physical host (or physical node, if you will):
|
||||
For testing perspective, it's not relevant how it is actually running,
|
||||
since frostfs network will still treat it as "node"
|
||||
since neofs network will still treat it as "node"
|
||||
"""
|
||||
|
||||
def get_rpc_endpoint(self) -> str:
|
||||
|
|
|
@ -1,22 +1,13 @@
|
|||
import json
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from time import sleep
|
||||
from typing import Optional, Union
|
||||
from typing import Optional
|
||||
|
||||
import allure
|
||||
from frostfs_testlib.cli import FrostfsCli
|
||||
from frostfs_testlib.shell import Shell
|
||||
from frostfs_testlib.utils import json_utils
|
||||
|
||||
from pytest_tests.helpers.cluster import Cluster
|
||||
from pytest_tests.helpers.file_helper import generate_file, get_file_hash
|
||||
from pytest_tests.helpers.frostfs_verbs import put_object, put_object_to_random_node
|
||||
from pytest_tests.helpers.storage_object_info import StorageObjectInfo
|
||||
from pytest_tests.helpers.wallet import WalletFile
|
||||
from pytest_tests.resources.common import CLI_DEFAULT_TIMEOUT, FROSTFS_CLI_EXEC, WALLET_CONFIG
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
from cluster import Cluster
|
||||
from file_helper import generate_file, get_file_hash
|
||||
from neofs_testlib.shell import Shell
|
||||
from neofs_verbs import put_object, put_object_to_random_node
|
||||
from storage_object import StorageObjectInfo
|
||||
from wallet import WalletFile
|
||||
|
||||
|
||||
@dataclass
|
||||
|
@ -94,238 +85,3 @@ class StorageContainer:
|
|||
)
|
||||
|
||||
return storage_object
|
||||
|
||||
|
||||
DEFAULT_PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 4 FROM * AS X"
|
||||
SINGLE_PLACEMENT_RULE = "REP 1 IN X CBF 1 SELECT 4 FROM * AS X"
|
||||
REP_2_FOR_3_NODES_PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 3 FROM * AS X"
|
||||
|
||||
|
||||
@allure.step("Create Container")
|
||||
def create_container(
|
||||
wallet: str,
|
||||
shell: Shell,
|
||||
endpoint: str,
|
||||
rule: str = DEFAULT_PLACEMENT_RULE,
|
||||
basic_acl: str = "",
|
||||
attributes: Optional[dict] = None,
|
||||
session_token: str = "",
|
||||
session_wallet: str = "",
|
||||
name: str = None,
|
||||
options: dict = None,
|
||||
await_mode: bool = True,
|
||||
wait_for_creation: bool = True,
|
||||
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||
) -> str:
|
||||
"""
|
||||
A wrapper for `frostfs-cli container create` call.
|
||||
|
||||
Args:
|
||||
wallet (str): a wallet on whose behalf a container is created
|
||||
rule (optional, str): placement rule for container
|
||||
basic_acl (optional, str): an ACL for container, will be
|
||||
appended to `--basic-acl` key
|
||||
attributes (optional, dict): container attributes , will be
|
||||
appended to `--attributes` key
|
||||
session_token (optional, str): a path to session token file
|
||||
session_wallet(optional, str): a path to the wallet which signed
|
||||
the session token; this parameter makes sense
|
||||
when paired with `session_token`
|
||||
shell: executor for cli command
|
||||
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
|
||||
options (optional, dict): any other options to pass to the call
|
||||
name (optional, str): container name attribute
|
||||
await_mode (bool): block execution until container is persisted
|
||||
wait_for_creation (): Wait for container shows in container list
|
||||
timeout: Timeout for the operation.
|
||||
|
||||
Returns:
|
||||
(str): CID of the created container
|
||||
"""
|
||||
|
||||
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, WALLET_CONFIG)
|
||||
result = cli.container.create(
|
||||
rpc_endpoint=endpoint,
|
||||
wallet=session_wallet if session_wallet else wallet,
|
||||
policy=rule,
|
||||
basic_acl=basic_acl,
|
||||
attributes=attributes,
|
||||
name=name,
|
||||
session=session_token,
|
||||
await_mode=await_mode,
|
||||
timeout=timeout,
|
||||
**options or {},
|
||||
)
|
||||
|
||||
cid = _parse_cid(result.stdout)
|
||||
|
||||
logger.info("Container created; waiting until it is persisted in the sidechain")
|
||||
|
||||
if wait_for_creation:
|
||||
wait_for_container_creation(wallet, cid, shell, endpoint)
|
||||
|
||||
return cid
|
||||
|
||||
|
||||
def wait_for_container_creation(
|
||||
wallet: str, cid: str, shell: Shell, endpoint: str, attempts: int = 15, sleep_interval: int = 1
|
||||
):
|
||||
for _ in range(attempts):
|
||||
containers = list_containers(wallet, shell, endpoint)
|
||||
if cid in containers:
|
||||
return
|
||||
logger.info(f"There is no {cid} in {containers} yet; sleep {sleep_interval} and continue")
|
||||
sleep(sleep_interval)
|
||||
raise RuntimeError(
|
||||
f"After {attempts * sleep_interval} seconds container {cid} hasn't been persisted; exiting"
|
||||
)
|
||||
|
||||
|
||||
def wait_for_container_deletion(
|
||||
wallet: str, cid: str, shell: Shell, endpoint: str, attempts: int = 30, sleep_interval: int = 1
|
||||
):
|
||||
for _ in range(attempts):
|
||||
try:
|
||||
get_container(wallet, cid, shell=shell, endpoint=endpoint)
|
||||
sleep(sleep_interval)
|
||||
continue
|
||||
except Exception as err:
|
||||
if "container not found" not in str(err):
|
||||
raise AssertionError(f'Expected "container not found" in error, got\n{err}')
|
||||
return
|
||||
raise AssertionError(f"Expected container deleted during {attempts * sleep_interval} sec.")
|
||||
|
||||
|
||||
@allure.step("List Containers")
|
||||
def list_containers(
|
||||
wallet: str, shell: Shell, endpoint: str, timeout: Optional[str] = CLI_DEFAULT_TIMEOUT
|
||||
) -> list[str]:
|
||||
"""
|
||||
A wrapper for `frostfs-cli container list` call. It returns all the
|
||||
available containers for the given wallet.
|
||||
Args:
|
||||
wallet (str): a wallet on whose behalf we list the containers
|
||||
shell: executor for cli command
|
||||
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
|
||||
timeout: Timeout for the operation.
|
||||
Returns:
|
||||
(list): list of containers
|
||||
"""
|
||||
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, WALLET_CONFIG)
|
||||
result = cli.container.list(rpc_endpoint=endpoint, wallet=wallet, timeout=timeout)
|
||||
logger.info(f"Containers: \n{result}")
|
||||
return result.stdout.split()
|
||||
|
||||
|
||||
@allure.step("Get Container")
|
||||
def get_container(
|
||||
wallet: str,
|
||||
cid: str,
|
||||
shell: Shell,
|
||||
endpoint: str,
|
||||
json_mode: bool = True,
|
||||
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||
) -> Union[dict, str]:
|
||||
"""
|
||||
A wrapper for `frostfs-cli container get` call. It extracts container's
|
||||
attributes and rearranges them into a more compact view.
|
||||
Args:
|
||||
wallet (str): path to a wallet on whose behalf we get the container
|
||||
cid (str): ID of the container to get
|
||||
shell: executor for cli command
|
||||
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
|
||||
json_mode (bool): return container in JSON format
|
||||
timeout: Timeout for the operation.
|
||||
Returns:
|
||||
(dict, str): dict of container attributes
|
||||
"""
|
||||
|
||||
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, WALLET_CONFIG)
|
||||
result = cli.container.get(
|
||||
rpc_endpoint=endpoint, wallet=wallet, cid=cid, json_mode=json_mode, timeout=timeout
|
||||
)
|
||||
|
||||
if not json_mode:
|
||||
return result.stdout
|
||||
|
||||
container_info = json.loads(result.stdout)
|
||||
attributes = dict()
|
||||
for attr in container_info["attributes"]:
|
||||
attributes[attr["key"]] = attr["value"]
|
||||
container_info["attributes"] = attributes
|
||||
container_info["ownerID"] = json_utils.json_reencode(container_info["ownerID"]["value"])
|
||||
return container_info
|
||||
|
||||
|
||||
@allure.step("Delete Container")
|
||||
# TODO: make the error message about a non-found container more user-friendly
|
||||
# https://github.com/nspcc-dev/frostfs-contract/issues/121
|
||||
def delete_container(
|
||||
wallet: str,
|
||||
cid: str,
|
||||
shell: Shell,
|
||||
endpoint: str,
|
||||
force: bool = False,
|
||||
session_token: Optional[str] = None,
|
||||
await_mode: bool = False,
|
||||
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||
) -> None:
|
||||
"""
|
||||
A wrapper for `frostfs-cli container delete` call.
|
||||
Args:
|
||||
wallet (str): path to a wallet on whose behalf we delete the container
|
||||
cid (str): ID of the container to delete
|
||||
shell: executor for cli command
|
||||
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
|
||||
force (bool): do not check whether container contains locks and remove immediately
|
||||
session_token: a path to session token file
|
||||
timeout: Timeout for the operation.
|
||||
This function doesn't return anything.
|
||||
"""
|
||||
|
||||
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, WALLET_CONFIG)
|
||||
cli.container.delete(
|
||||
wallet=wallet,
|
||||
cid=cid,
|
||||
rpc_endpoint=endpoint,
|
||||
force=force,
|
||||
session=session_token,
|
||||
await_mode=await_mode,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
|
||||
def _parse_cid(output: str) -> str:
|
||||
"""
|
||||
Parses container ID from a given CLI output. The input string we expect:
|
||||
container ID: 2tz86kVTDpJxWHrhw3h6PbKMwkLtBEwoqhHQCKTre1FN
|
||||
awaiting...
|
||||
container has been persisted on sidechain
|
||||
We want to take 'container ID' value from the string.
|
||||
|
||||
Args:
|
||||
output (str): CLI output to parse
|
||||
|
||||
Returns:
|
||||
(str): extracted CID
|
||||
"""
|
||||
try:
|
||||
# taking first line from command's output
|
||||
first_line = output.split("\n")[0]
|
||||
except Exception:
|
||||
first_line = ""
|
||||
logger.error(f"Got empty output: {output}")
|
||||
splitted = first_line.split(": ")
|
||||
if len(splitted) != 2:
|
||||
raise ValueError(f"no CID was parsed from command output: \t{first_line}")
|
||||
return splitted[1]
|
||||
|
||||
|
||||
@allure.step("Search container by name")
|
||||
def search_container_by_name(wallet: str, name: str, shell: Shell, endpoint: str):
|
||||
list_cids = list_containers(wallet, shell, endpoint)
|
||||
for cid in list_cids:
|
||||
cont_info = get_container(wallet, cid, shell, endpoint, True)
|
||||
if cont_info.get("attributes").get("Name", None) == name:
|
||||
return cid
|
||||
return None
|
||||
|
|
|
@ -5,8 +5,7 @@ import uuid
|
|||
from typing import Any, Optional
|
||||
|
||||
import allure
|
||||
|
||||
from pytest_tests.resources.common import ASSETS_DIR
|
||||
from common import ASSETS_DIR
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
|
||||
|
|
36
pytest_tests/helpers/grpc_responses.py
Normal file
36
pytest_tests/helpers/grpc_responses.py
Normal file
|
@ -0,0 +1,36 @@
|
|||
import re
|
||||
|
||||
# Regex patterns of status codes of Container service (https://github.com/nspcc-dev/neofs-spec/blob/98b154848116223e486ce8b43eaa35fec08b4a99/20-api-v2/container.md)
|
||||
CONTAINER_NOT_FOUND = "code = 3072.*message = container not found"
|
||||
|
||||
|
||||
# Regex patterns of status codes of Object service (https://github.com/nspcc-dev/neofs-spec/blob/98b154848116223e486ce8b43eaa35fec08b4a99/20-api-v2/object.md)
|
||||
MALFORMED_REQUEST = "code = 1024.*message = malformed request"
|
||||
OBJECT_ACCESS_DENIED = "code = 2048.*message = access to object operation denied"
|
||||
OBJECT_NOT_FOUND = "code = 2049.*message = object not found"
|
||||
OBJECT_ALREADY_REMOVED = "code = 2052.*message = object already removed"
|
||||
SESSION_NOT_FOUND = "code = 4096.*message = session token not found"
|
||||
OUT_OF_RANGE = "code = 2053.*message = out of range"
|
||||
# TODO: Due to https://github.com/nspcc-dev/neofs-node/issues/2092 we have to check only codes until fixed
|
||||
# OBJECT_IS_LOCKED = "code = 2050.*message = object is locked"
|
||||
# LOCK_NON_REGULAR_OBJECT = "code = 2051.*message = ..." will be available once 2092 is fixed
|
||||
OBJECT_IS_LOCKED = "code = 2050"
|
||||
LOCK_NON_REGULAR_OBJECT = "code = 2051"
|
||||
|
||||
LIFETIME_REQUIRED = "either expiration epoch of a lifetime is required"
|
||||
LOCK_OBJECT_REMOVAL = "lock object removal"
|
||||
LOCK_OBJECT_EXPIRATION = "lock object expiration: {expiration_epoch}; current: {current_epoch}"
|
||||
INVALID_RANGE_ZERO_LENGTH = "invalid '{range}' range: zero length"
|
||||
INVALID_RANGE_OVERFLOW = "invalid '{range}' range: uint64 overflow"
|
||||
INVALID_OFFSET_SPECIFIER = "invalid '{range}' range offset specifier"
|
||||
INVALID_LENGTH_SPECIFIER = "invalid '{range}' range length specifier"
|
||||
|
||||
|
||||
def error_matches_status(error: Exception, status_pattern: str) -> bool:
|
||||
"""
|
||||
Determines whether exception matches specified status pattern.
|
||||
|
||||
We use re.search to be consistent with pytest.raises.
|
||||
"""
|
||||
match = re.search(status_pattern, str(error))
|
||||
return match is not None
|
|
@ -1,4 +1,4 @@
|
|||
from frostfs_testlib.shell import Shell
|
||||
from neofs_testlib.shell import Shell
|
||||
|
||||
|
||||
class IpTablesHelper:
|
||||
|
|
|
@ -5,15 +5,14 @@ from time import sleep
|
|||
from typing import Optional
|
||||
|
||||
import allure
|
||||
from frostfs_testlib.shell import Shell
|
||||
|
||||
from pytest_tests.helpers.remote_process import RemoteProcess
|
||||
from neofs_testlib.shell import Shell
|
||||
from remote_process import RemoteProcess
|
||||
|
||||
EXIT_RESULT_CODE = 0
|
||||
LOAD_RESULTS_PATTERNS = {
|
||||
"grpc": {
|
||||
"write_ops": r"frostfs_obj_put_total\W*\d*\W*(?P<write_ops>\d*\.\d*)",
|
||||
"read_ops": r"frostfs_obj_get_total\W*\d*\W*(?P<read_ops>\d*\.\d*)",
|
||||
"write_ops": r"neofs_obj_put_total\W*\d*\W*(?P<write_ops>\d*\.\d*)",
|
||||
"read_ops": r"neofs_obj_get_total\W*\d*\W*(?P<read_ops>\d*\.\d*)",
|
||||
},
|
||||
"s3": {
|
||||
"write_ops": r"aws_obj_put_total\W*\d*\W*(?P<write_ops>\d*\.\d*)",
|
||||
|
|
|
@ -4,8 +4,8 @@ import uuid
|
|||
from typing import Optional
|
||||
|
||||
import allure
|
||||
from frostfs_testlib.shell import Shell
|
||||
from frostfs_testlib.shell.interfaces import CommandOptions
|
||||
from neofs_testlib.shell import Shell
|
||||
from neofs_testlib.shell.interfaces import CommandOptions
|
||||
from tenacity import retry, stop_after_attempt, wait_fixed
|
||||
|
||||
|
||||
|
|
|
@ -1,15 +1,13 @@
|
|||
import logging
|
||||
import datetime
|
||||
import os
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Optional
|
||||
|
||||
import allure
|
||||
import s3_gate_bucket
|
||||
import s3_gate_object
|
||||
from dateutil.parser import parse
|
||||
|
||||
from pytest_tests.steps import s3_gate_bucket, s3_gate_object
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
|
||||
|
||||
@allure.step("Expected all objects are presented in the bucket")
|
||||
def check_objects_in_bucket(
|
||||
|
@ -129,31 +127,3 @@ def assert_object_lock_mode(
|
|||
assert (
|
||||
retain_date - last_modify + timedelta(seconds=1)
|
||||
).days == retain_period, f"Expected retention period is {retain_period} days"
|
||||
|
||||
|
||||
def assert_s3_acl(acl_grants: list, permitted_users: str):
|
||||
if permitted_users == "AllUsers":
|
||||
grantees = {"AllUsers": 0, "CanonicalUser": 0}
|
||||
for acl_grant in acl_grants:
|
||||
if acl_grant.get("Grantee", {}).get("Type") == "Group":
|
||||
uri = acl_grant.get("Grantee", {}).get("URI")
|
||||
permission = acl_grant.get("Permission")
|
||||
assert (uri, permission) == (
|
||||
"http://acs.amazonaws.com/groups/global/AllUsers",
|
||||
"FULL_CONTROL",
|
||||
), "All Groups should have FULL_CONTROL"
|
||||
grantees["AllUsers"] += 1
|
||||
if acl_grant.get("Grantee", {}).get("Type") == "CanonicalUser":
|
||||
permission = acl_grant.get("Permission")
|
||||
assert permission == "FULL_CONTROL", "Canonical User should have FULL_CONTROL"
|
||||
grantees["CanonicalUser"] += 1
|
||||
assert grantees["AllUsers"] >= 1, "All Users should have FULL_CONTROL"
|
||||
assert grantees["CanonicalUser"] >= 1, "Canonical User should have FULL_CONTROL"
|
||||
|
||||
if permitted_users == "CanonicalUser":
|
||||
for acl_grant in acl_grants:
|
||||
if acl_grant.get("Grantee", {}).get("Type") == "CanonicalUser":
|
||||
permission = acl_grant.get("Permission")
|
||||
assert permission == "FULL_CONTROL", "Only CanonicalUser should have FULL_CONTROL"
|
||||
else:
|
||||
logger.error("FULL_CONTROL is given to All Users")
|
||||
|
|
|
@ -1,9 +1,33 @@
|
|||
import time
|
||||
|
||||
import allure
|
||||
from frostfs_testlib.utils import datetime_utils
|
||||
from common import STORAGE_GC_TIME
|
||||
|
||||
from pytest_tests.resources.common import STORAGE_GC_TIME
|
||||
|
||||
def parse_time(value: str) -> int:
|
||||
"""Converts time interval in text form into time interval as number of seconds.
|
||||
|
||||
Args:
|
||||
value: time interval as text.
|
||||
|
||||
Returns:
|
||||
Number of seconds in the parsed time interval.
|
||||
"""
|
||||
value = value.lower()
|
||||
|
||||
for suffix in ["s", "sec"]:
|
||||
if value.endswith(suffix):
|
||||
return int(value[: -len(suffix)])
|
||||
|
||||
for suffix in ["m", "min"]:
|
||||
if value.endswith(suffix):
|
||||
return int(value[: -len(suffix)]) * 60
|
||||
|
||||
for suffix in ["h", "hr", "hour"]:
|
||||
if value.endswith(suffix):
|
||||
return int(value[: -len(suffix)]) * 60 * 60
|
||||
|
||||
raise ValueError(f"Unknown units in time value '{value}'")
|
||||
|
||||
|
||||
def placement_policy_from_container(container_info: str) -> str:
|
||||
|
@ -23,7 +47,7 @@ def placement_policy_from_container(container_info: str) -> str:
|
|||
FILTER Country EQ Sweden AS LOC_SW
|
||||
|
||||
Args:
|
||||
container_info: output from frostfs-cli container get command
|
||||
container_info: output from neofs-cli container get command
|
||||
|
||||
Returns:
|
||||
placement policy as a string
|
||||
|
@ -33,6 +57,6 @@ def placement_policy_from_container(container_info: str) -> str:
|
|||
|
||||
|
||||
def wait_for_gc_pass_on_storage_nodes() -> None:
|
||||
wait_time = datetime_utils.parse_time(STORAGE_GC_TIME)
|
||||
wait_time = parse_time(STORAGE_GC_TIME)
|
||||
with allure.step(f"Wait {wait_time}s until GC completes on storage nodes"):
|
||||
time.sleep(wait_time)
|
||||
|
|
|
@ -2,12 +2,11 @@ import os
|
|||
import uuid
|
||||
from dataclasses import dataclass
|
||||
|
||||
from frostfs_testlib.shell import Shell
|
||||
from frostfs_testlib.utils import wallet_utils
|
||||
|
||||
from pytest_tests.helpers.cluster import Cluster, NodeBase
|
||||
from pytest_tests.helpers.payment_neogo import deposit_gas, transfer_gas
|
||||
from pytest_tests.resources.common import FREE_STORAGE, WALLET_CONFIG, WALLET_PASS
|
||||
from cluster import Cluster, NodeBase
|
||||
from common import FREE_STORAGE, WALLET_CONFIG, WALLET_PASS
|
||||
from neofs_testlib.shell import Shell
|
||||
from neofs_testlib.utils.wallet import get_last_address_from_wallet, init_wallet
|
||||
from python_keywords.payment_neogo import deposit_gas, transfer_gas
|
||||
|
||||
|
||||
@dataclass
|
||||
|
@ -29,7 +28,7 @@ class WalletFile:
|
|||
Returns:
|
||||
The address of the wallet.
|
||||
"""
|
||||
return wallet_utils.get_last_address_from_wallet(self.path, self.password)
|
||||
return get_last_address_from_wallet(self.path, self.password)
|
||||
|
||||
|
||||
class WalletFactory:
|
||||
|
@ -48,7 +47,7 @@ class WalletFactory:
|
|||
WalletFile object of new wallet
|
||||
"""
|
||||
wallet_path = os.path.join(self.wallets_dir, f"{str(uuid.uuid4())}.json")
|
||||
wallet_utils.init_wallet(wallet_path, password)
|
||||
init_wallet(wallet_path, password)
|
||||
|
||||
if not FREE_STORAGE:
|
||||
main_chain = self.cluster.main_chain_nodes[0]
|
||||
|
|
|
@ -13,7 +13,7 @@ markers =
|
|||
# functional markers
|
||||
container: tests for container creation
|
||||
grpc_api: standard gRPC API tests
|
||||
grpc_control: tests related to using frostfs-cli control commands
|
||||
grpc_control: tests related to using neofs-cli control commands
|
||||
grpc_object_lock: gRPC lock tests
|
||||
http_gate: HTTP gate contract
|
||||
s3_gate: All S3 gate tests
|
||||
|
@ -25,7 +25,7 @@ markers =
|
|||
s3_gate_tagging: Tagging S3 gate tests
|
||||
s3_gate_versioning: Versioning S3 gate tests
|
||||
long: long tests (with long execution time)
|
||||
node_mgmt: frostfs control commands
|
||||
node_mgmt: neofs control commands
|
||||
session_token: tests for operations with session token
|
||||
static_session: tests for operations with static session token
|
||||
bearer: tests for bearer tokens
|
||||
|
@ -40,6 +40,6 @@ markers =
|
|||
failover_network: tests for network failure
|
||||
failover_reboot: tests for system recovery after reboot of a node
|
||||
add_nodes: add nodes to cluster
|
||||
check_binaries: check frostfs installed binaries versions
|
||||
check_binaries: check neofs installed binaries versions
|
||||
payments: tests for payment associated operations
|
||||
load: performance tests
|
||||
|
|
|
@ -1,9 +1,8 @@
|
|||
import allure
|
||||
import epoch
|
||||
import pytest
|
||||
from frostfs_testlib.shell import Shell
|
||||
|
||||
from pytest_tests.helpers import epoch
|
||||
from pytest_tests.helpers.cluster import Cluster
|
||||
from cluster import Cluster
|
||||
from neofs_testlib.shell import Shell
|
||||
|
||||
|
||||
# To skip adding every mandatory singleton dependency to EACH test function
|
||||
|
|
|
@ -3,16 +3,15 @@ import re
|
|||
from dataclasses import asdict
|
||||
|
||||
import allure
|
||||
from frostfs_testlib.cli.frostfs_authmate import FrostfsAuthmate
|
||||
from frostfs_testlib.cli.neogo import NeoGo
|
||||
from frostfs_testlib.hosting import Hosting
|
||||
from frostfs_testlib.shell import CommandOptions, SSHShell
|
||||
from frostfs_testlib.shell.interfaces import InteractiveInput
|
||||
from common import STORAGE_NODE_SERVICE_NAME_REGEX
|
||||
from k6 import K6, LoadParams, LoadResults
|
||||
from neofs_testlib.cli.neofs_authmate import NeofsAuthmate
|
||||
from neofs_testlib.cli.neogo import NeoGo
|
||||
from neofs_testlib.hosting import Hosting
|
||||
from neofs_testlib.shell import CommandOptions, SSHShell
|
||||
from neofs_testlib.shell.interfaces import InteractiveInput
|
||||
|
||||
from pytest_tests.helpers.k6 import K6, LoadParams, LoadResults
|
||||
from pytest_tests.resources.common import STORAGE_NODE_SERVICE_NAME_REGEX
|
||||
|
||||
FROSTFS_AUTHMATE_PATH = "frostfs-authmate"
|
||||
NEOFS_AUTHMATE_PATH = "neofs-s3-authmate"
|
||||
STOPPED_HOSTS = []
|
||||
|
||||
|
||||
|
@ -57,8 +56,8 @@ def init_s3_client(
|
|||
path = ssh_client.exec(r"sudo find . -name 'k6' -exec dirname {} \; -quit").stdout.strip(
|
||||
"\n"
|
||||
)
|
||||
frostfs_authmate_exec = FrostfsAuthmate(ssh_client, FROSTFS_AUTHMATE_PATH)
|
||||
issue_secret_output = frostfs_authmate_exec.secret.issue(
|
||||
neofs_authmate_exec = NeofsAuthmate(ssh_client, NEOFS_AUTHMATE_PATH)
|
||||
issue_secret_output = neofs_authmate_exec.secret.issue(
|
||||
wallet=f"{path}/scenarios/files/wallet.json",
|
||||
peer=node_endpoint,
|
||||
bearer_rules=f"{path}/scenarios/files/rules.json",
|
||||
|
|
|
@ -8,20 +8,19 @@ from typing import Any, Optional
|
|||
import allure
|
||||
import boto3
|
||||
import pytest
|
||||
import s3_gate_bucket
|
||||
import s3_gate_object
|
||||
import urllib3
|
||||
from aws_cli_client import AwsCliClient
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import ClientError
|
||||
from frostfs_testlib.shell import Shell
|
||||
from cli_helpers import _cmd_run, _configure_aws_cli, _run_with_passwd
|
||||
from cluster import Cluster
|
||||
from cluster_test_base import ClusterTestBase
|
||||
from common import NEOFS_AUTHMATE_EXEC
|
||||
from neofs_testlib.shell import Shell
|
||||
from pytest import FixtureRequest
|
||||
|
||||
from pytest_tests.steps import s3_gate_bucket
|
||||
from pytest_tests.steps import s3_gate_object
|
||||
from pytest_tests.helpers.aws_cli_client import AwsCliClient
|
||||
from pytest_tests.helpers.cli_helpers import _cmd_run, _configure_aws_cli, _run_with_passwd
|
||||
from pytest_tests.helpers.cluster import Cluster
|
||||
from pytest_tests.helpers.container import list_containers
|
||||
from pytest_tests.resources.common import FROSTFS_AUTHMATE_EXEC
|
||||
from pytest_tests.steps.cluster_test_base import ClusterTestBase
|
||||
from python_keywords.container import list_containers
|
||||
|
||||
# Disable warnings on self-signed certificate which the
|
||||
# boto library produces on requests to S3-gate in dev-env
|
||||
|
@ -45,7 +44,7 @@ class TestS3GateBase(ClusterTestBase):
|
|||
self, default_wallet, client_shell: Shell, request: FixtureRequest, cluster: Cluster
|
||||
) -> Any:
|
||||
wallet = default_wallet
|
||||
s3_bearer_rules_file = f"{os.getcwd()}/pytest_tests/resources/files/s3_bearer_rules.json"
|
||||
s3_bearer_rules_file = f"{os.getcwd()}/robot/resources/files/s3_bearer_rules.json"
|
||||
policy = None if isinstance(request.param, str) else request.param[1]
|
||||
(cid, bucket, access_key_id, secret_access_key, owner_private_key,) = init_s3_credentials(
|
||||
wallet, cluster, s3_bearer_rules_file=s3_bearer_rules_file, policy=policy
|
||||
|
@ -124,12 +123,12 @@ def init_s3_credentials(
|
|||
policy: Optional[dict] = None,
|
||||
):
|
||||
bucket = str(uuid.uuid4())
|
||||
s3_bearer_rules = s3_bearer_rules_file or "pytest_tests/resources/files/s3_bearer_rules.json"
|
||||
s3_bearer_rules = s3_bearer_rules_file or "robot/resources/files/s3_bearer_rules.json"
|
||||
|
||||
s3gate_node = cluster.s3gates[0]
|
||||
gate_public_key = s3gate_node.get_wallet_public_key()
|
||||
cmd = (
|
||||
f"{FROSTFS_AUTHMATE_EXEC} --debug --with-log --timeout {CREDENTIALS_CREATE_TIMEOUT} "
|
||||
f"{NEOFS_AUTHMATE_EXEC} --debug --with-log --timeout {CREDENTIALS_CREATE_TIMEOUT} "
|
||||
f"issue-secret --wallet {wallet_path} --gate-public-key={gate_public_key} "
|
||||
f"--peer {cluster.default_rpc_endpoint} --container-friendly-name {bucket} "
|
||||
f"--bearer-rules {s3_bearer_rules}"
|
||||
|
|
|
@ -7,8 +7,7 @@ from typing import Optional
|
|||
|
||||
import allure
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
from pytest_tests.helpers.cli_helpers import log_command_execution
|
||||
from cli_helpers import log_command_execution
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
|
||||
|
|
|
@ -7,11 +7,10 @@ from typing import Optional
|
|||
import allure
|
||||
import pytest
|
||||
import urllib3
|
||||
from aws_cli_client import AwsCliClient
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
from pytest_tests.helpers.aws_cli_client import AwsCliClient
|
||||
from pytest_tests.helpers.cli_helpers import log_command_execution
|
||||
from pytest_tests.steps.s3_gate_bucket import S3_SYNC_WAIT_TIME
|
||||
from cli_helpers import log_command_execution
|
||||
from s3_gate_bucket import S3_SYNC_WAIT_TIME
|
||||
|
||||
##########################################################
|
||||
# Disabling warnings on self-signed certificate which the
|
||||
|
|
|
@ -8,13 +8,14 @@ from enum import Enum
|
|||
from typing import Any, Optional
|
||||
|
||||
import allure
|
||||
from frostfs_testlib.cli import FrostfsCli
|
||||
from frostfs_testlib.shell import Shell
|
||||
from frostfs_testlib.utils import json_utils, wallet_utils
|
||||
|
||||
from pytest_tests.helpers.storage_object_info import StorageObjectInfo
|
||||
from pytest_tests.helpers.wallet import WalletFile
|
||||
from pytest_tests.resources.common import ASSETS_DIR, FROSTFS_CLI_EXEC, WALLET_CONFIG
|
||||
import json_transformers
|
||||
from common import ASSETS_DIR, NEOFS_CLI_EXEC, WALLET_CONFIG
|
||||
from data_formatters import get_wallet_public_key
|
||||
from json_transformers import encode_for_json
|
||||
from neofs_testlib.cli import NeofsCli
|
||||
from neofs_testlib.shell import Shell
|
||||
from storage_object_info import StorageObjectInfo
|
||||
from wallet import WalletFile
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
|
||||
|
@ -70,16 +71,16 @@ def generate_session_token(
|
|||
|
||||
file_path = os.path.join(tokens_dir, str(uuid.uuid4()))
|
||||
|
||||
pub_key_64 = wallet_utils.get_wallet_public_key(
|
||||
session_wallet.path, session_wallet.password, "base64"
|
||||
)
|
||||
pub_key_64 = get_wallet_public_key(session_wallet.path, session_wallet.password, "base64")
|
||||
|
||||
lifetime = lifetime or Lifetime()
|
||||
|
||||
session_token = {
|
||||
"body": {
|
||||
"id": f"{base64.b64encode(uuid.uuid4().bytes).decode('utf-8')}",
|
||||
"ownerID": {"value": f"{json_utils.encode_for_json(owner_wallet.get_address())}"},
|
||||
"ownerID": {
|
||||
"value": f"{json_transformers.encode_for_json(owner_wallet.get_address())}"
|
||||
},
|
||||
"lifetime": {
|
||||
"exp": f"{lifetime.exp}",
|
||||
"nbf": f"{lifetime.nbf}",
|
||||
|
@ -124,11 +125,7 @@ def generate_container_session_token(
|
|||
"container": {
|
||||
"verb": verb.value,
|
||||
"wildcard": cid is None,
|
||||
**(
|
||||
{"containerID": {"value": f"{json_utils.encode_for_json(cid)}"}}
|
||||
if cid is not None
|
||||
else {}
|
||||
),
|
||||
**({"containerID": {"value": f"{encode_for_json(cid)}"}} if cid is not None else {}),
|
||||
},
|
||||
}
|
||||
|
||||
|
@ -168,8 +165,8 @@ def generate_object_session_token(
|
|||
"object": {
|
||||
"verb": verb.value,
|
||||
"target": {
|
||||
"container": {"value": json_utils.encode_for_json(cid)},
|
||||
"objects": [{"value": json_utils.encode_for_json(oid)} for oid in oids],
|
||||
"container": {"value": encode_for_json(cid)},
|
||||
"objects": [{"value": encode_for_json(oid)} for oid in oids],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
@ -252,8 +249,8 @@ def create_session_token(
|
|||
The path to the generated session token file.
|
||||
"""
|
||||
session_token = os.path.join(os.getcwd(), ASSETS_DIR, str(uuid.uuid4()))
|
||||
frostfscli = FrostfsCli(shell=shell, frostfs_cli_exec_path=FROSTFS_CLI_EXEC)
|
||||
frostfscli.session.create(
|
||||
neofscli = NeofsCli(shell=shell, neofs_cli_exec_path=NEOFS_CLI_EXEC)
|
||||
neofscli.session.create(
|
||||
rpc_endpoint=rpc_endpoint,
|
||||
address=owner,
|
||||
wallet=wallet_path,
|
||||
|
@ -277,10 +274,8 @@ def sign_session_token(shell: Shell, session_token_file: str, wlt: WalletFile) -
|
|||
The path to the signed token.
|
||||
"""
|
||||
signed_token_file = os.path.join(os.getcwd(), ASSETS_DIR, str(uuid.uuid4()))
|
||||
frostfscli = FrostfsCli(
|
||||
shell=shell, frostfs_cli_exec_path=FROSTFS_CLI_EXEC, config_file=WALLET_CONFIG
|
||||
)
|
||||
frostfscli.util.sign_session_token(
|
||||
neofscli = NeofsCli(shell=shell, neofs_cli_exec_path=NEOFS_CLI_EXEC, config_file=WALLET_CONFIG)
|
||||
neofscli.util.sign_session_token(
|
||||
wallet=wlt.path, from_file=session_token_file, to_file=signed_token_file
|
||||
)
|
||||
return signed_token_file
|
||||
|
|
|
@ -3,14 +3,13 @@ from time import sleep
|
|||
|
||||
import allure
|
||||
import pytest
|
||||
from frostfs_testlib.resources.common import OBJECT_ALREADY_REMOVED
|
||||
from frostfs_testlib.shell import Shell
|
||||
|
||||
from pytest_tests.helpers.cluster import Cluster
|
||||
from pytest_tests.helpers.epoch import tick_epoch
|
||||
from pytest_tests.helpers.frostfs_verbs import delete_object, get_object
|
||||
from pytest_tests.helpers.storage_object_info import StorageObjectInfo
|
||||
from pytest_tests.helpers.tombstone import verify_head_tombstone
|
||||
from cluster import Cluster
|
||||
from epoch import tick_epoch
|
||||
from grpc_responses import OBJECT_ALREADY_REMOVED
|
||||
from neofs_testlib.shell import Shell
|
||||
from python_keywords.neofs_verbs import delete_object, get_object
|
||||
from storage_object_info import StorageObjectInfo
|
||||
from tombstone import verify_head_tombstone
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
|
||||
|
|
|
@ -5,16 +5,15 @@ from typing import Optional
|
|||
|
||||
import allure
|
||||
import pytest
|
||||
from frostfs_testlib.resources.common import PUBLIC_ACL
|
||||
from frostfs_testlib.shell import Shell
|
||||
from frostfs_testlib.utils import wallet_utils
|
||||
|
||||
from pytest_tests.helpers.acl import EACLRole
|
||||
from pytest_tests.helpers.cluster import Cluster
|
||||
from pytest_tests.helpers.container import create_container
|
||||
from pytest_tests.helpers.file_helper import generate_file
|
||||
from pytest_tests.helpers.frostfs_verbs import put_object_to_random_node
|
||||
from pytest_tests.resources.common import WALLET_CONFIG, WALLET_PASS
|
||||
from cluster import Cluster
|
||||
from common import WALLET_CONFIG, WALLET_PASS
|
||||
from file_helper import generate_file
|
||||
from neofs_testlib.shell import Shell
|
||||
from neofs_testlib.utils.wallet import init_wallet
|
||||
from python_keywords.acl import EACLRole
|
||||
from python_keywords.container import create_container
|
||||
from python_keywords.neofs_verbs import put_object_to_random_node
|
||||
from wellknown_acl import PUBLIC_ACL
|
||||
|
||||
OBJECT_COUNT = 5
|
||||
|
||||
|
@ -42,7 +41,7 @@ def wallets(default_wallet, temp_directory, cluster: Cluster) -> Wallets:
|
|||
os.path.join(temp_directory, f"{str(uuid.uuid4())}.json") for _ in range(2)
|
||||
]
|
||||
for other_wallet_path in other_wallets_paths:
|
||||
wallet_utils.init_wallet(other_wallet_path, WALLET_PASS)
|
||||
init_wallet(other_wallet_path, WALLET_PASS)
|
||||
|
||||
ir_node = cluster.ir_nodes[0]
|
||||
storage_node = cluster.storage_nodes[0]
|
||||
|
|
472
pytest_tests/testsuites/acl/storage_group/test_storagegroup.py
Normal file
472
pytest_tests/testsuites/acl/storage_group/test_storagegroup.py
Normal file
|
@ -0,0 +1,472 @@
|
|||
import logging
|
||||
import os
|
||||
import uuid
|
||||
from typing import Optional
|
||||
|
||||
import allure
|
||||
import pytest
|
||||
from cluster_test_base import ClusterTestBase
|
||||
from common import ASSETS_DIR, FREE_STORAGE, WALLET_PASS
|
||||
from file_helper import generate_file
|
||||
from grpc_responses import OBJECT_ACCESS_DENIED, OBJECT_NOT_FOUND
|
||||
from neofs_testlib.utils.wallet import init_wallet
|
||||
from python_keywords.acl import (
|
||||
EACLAccess,
|
||||
EACLOperation,
|
||||
EACLRole,
|
||||
EACLRule,
|
||||
create_eacl,
|
||||
form_bearertoken_file,
|
||||
set_eacl,
|
||||
)
|
||||
from python_keywords.container import create_container
|
||||
from python_keywords.neofs_verbs import put_object_to_random_node
|
||||
from python_keywords.payment_neogo import deposit_gas, transfer_gas
|
||||
from python_keywords.storage_group import (
|
||||
delete_storagegroup,
|
||||
get_storagegroup,
|
||||
list_storagegroup,
|
||||
put_storagegroup,
|
||||
verify_get_storage_group,
|
||||
verify_list_storage_group,
|
||||
)
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
deposit = 30
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"object_size",
|
||||
[pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")],
|
||||
ids=["simple object", "complex object"],
|
||||
)
|
||||
@pytest.mark.sanity
|
||||
@pytest.mark.acl
|
||||
@pytest.mark.storage_group
|
||||
class TestStorageGroup(ClusterTestBase):
|
||||
@pytest.fixture(autouse=True)
|
||||
def prepare_two_wallets(self, default_wallet):
|
||||
self.main_wallet = default_wallet
|
||||
self.other_wallet = os.path.join(os.getcwd(), ASSETS_DIR, f"{str(uuid.uuid4())}.json")
|
||||
init_wallet(self.other_wallet, WALLET_PASS)
|
||||
if not FREE_STORAGE:
|
||||
main_chain = self.cluster.main_chain_nodes[0]
|
||||
deposit = 30
|
||||
transfer_gas(
|
||||
shell=self.shell,
|
||||
amount=deposit + 1,
|
||||
main_chain=main_chain,
|
||||
wallet_to_path=self.other_wallet,
|
||||
wallet_to_password=WALLET_PASS,
|
||||
)
|
||||
deposit_gas(
|
||||
shell=self.shell,
|
||||
amount=deposit,
|
||||
main_chain=main_chain,
|
||||
wallet_from_path=self.other_wallet,
|
||||
wallet_from_password=WALLET_PASS,
|
||||
)
|
||||
|
||||
@allure.title("Test Storage Group in Private Container")
|
||||
def test_storagegroup_basic_private_container(self, object_size, max_object_size):
|
||||
cid = create_container(
|
||||
self.main_wallet, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint
|
||||
)
|
||||
file_path = generate_file(object_size)
|
||||
oid = put_object_to_random_node(self.main_wallet, file_path, cid, self.shell, self.cluster)
|
||||
objects = [oid]
|
||||
storage_group = put_storagegroup(
|
||||
shell=self.shell,
|
||||
endpoint=self.cluster.default_rpc_endpoint,
|
||||
wallet=self.main_wallet,
|
||||
cid=cid,
|
||||
objects=objects,
|
||||
)
|
||||
|
||||
self.expect_success_for_storagegroup_operations(
|
||||
wallet=self.main_wallet,
|
||||
cid=cid,
|
||||
obj_list=objects,
|
||||
object_size=object_size,
|
||||
max_object_size=max_object_size,
|
||||
)
|
||||
self.expect_failure_for_storagegroup_operations(
|
||||
wallet=self.other_wallet,
|
||||
cid=cid,
|
||||
obj_list=objects,
|
||||
gid=storage_group,
|
||||
)
|
||||
self.storagegroup_operations_by_system_ro_container(
|
||||
wallet=self.main_wallet,
|
||||
cid=cid,
|
||||
obj_list=objects,
|
||||
object_size=object_size,
|
||||
max_object_size=max_object_size,
|
||||
)
|
||||
|
||||
@allure.title("Test Storage Group in Public Container")
|
||||
def test_storagegroup_basic_public_container(self, object_size, max_object_size):
|
||||
cid = create_container(
|
||||
self.main_wallet,
|
||||
basic_acl="public-read-write",
|
||||
shell=self.shell,
|
||||
endpoint=self.cluster.default_rpc_endpoint,
|
||||
)
|
||||
file_path = generate_file(object_size)
|
||||
oid = put_object_to_random_node(
|
||||
self.main_wallet, file_path, cid, shell=self.shell, cluster=self.cluster
|
||||
)
|
||||
objects = [oid]
|
||||
self.expect_success_for_storagegroup_operations(
|
||||
wallet=self.main_wallet,
|
||||
cid=cid,
|
||||
obj_list=objects,
|
||||
object_size=object_size,
|
||||
max_object_size=max_object_size,
|
||||
)
|
||||
self.expect_success_for_storagegroup_operations(
|
||||
wallet=self.other_wallet,
|
||||
cid=cid,
|
||||
obj_list=objects,
|
||||
object_size=object_size,
|
||||
max_object_size=max_object_size,
|
||||
)
|
||||
self.storagegroup_operations_by_system_ro_container(
|
||||
wallet=self.main_wallet,
|
||||
cid=cid,
|
||||
obj_list=objects,
|
||||
object_size=object_size,
|
||||
max_object_size=max_object_size,
|
||||
)
|
||||
|
||||
@allure.title("Test Storage Group in Read-Only Container")
|
||||
def test_storagegroup_basic_ro_container(self, object_size, max_object_size):
|
||||
cid = create_container(
|
||||
self.main_wallet,
|
||||
basic_acl="public-read",
|
||||
shell=self.shell,
|
||||
endpoint=self.cluster.default_rpc_endpoint,
|
||||
)
|
||||
file_path = generate_file(object_size)
|
||||
oid = put_object_to_random_node(
|
||||
self.main_wallet, file_path, cid, shell=self.shell, cluster=self.cluster
|
||||
)
|
||||
objects = [oid]
|
||||
self.expect_success_for_storagegroup_operations(
|
||||
wallet=self.main_wallet,
|
||||
cid=cid,
|
||||
obj_list=objects,
|
||||
object_size=object_size,
|
||||
max_object_size=max_object_size,
|
||||
)
|
||||
self.storagegroup_operations_by_other_ro_container(
|
||||
owner_wallet=self.main_wallet,
|
||||
other_wallet=self.other_wallet,
|
||||
cid=cid,
|
||||
obj_list=objects,
|
||||
object_size=object_size,
|
||||
max_object_size=max_object_size,
|
||||
)
|
||||
self.storagegroup_operations_by_system_ro_container(
|
||||
wallet=self.main_wallet,
|
||||
cid=cid,
|
||||
obj_list=objects,
|
||||
object_size=object_size,
|
||||
max_object_size=max_object_size,
|
||||
)
|
||||
|
||||
@allure.title("Test Storage Group with Bearer Allow")
|
||||
def test_storagegroup_bearer_allow(self, object_size, max_object_size):
|
||||
cid = create_container(
|
||||
self.main_wallet,
|
||||
basic_acl="eacl-public-read-write",
|
||||
shell=self.shell,
|
||||
endpoint=self.cluster.default_rpc_endpoint,
|
||||
)
|
||||
file_path = generate_file(object_size)
|
||||
oid = put_object_to_random_node(
|
||||
self.main_wallet, file_path, cid, shell=self.shell, cluster=self.cluster
|
||||
)
|
||||
objects = [oid]
|
||||
self.expect_success_for_storagegroup_operations(
|
||||
wallet=self.main_wallet,
|
||||
cid=cid,
|
||||
obj_list=objects,
|
||||
object_size=object_size,
|
||||
max_object_size=max_object_size,
|
||||
)
|
||||
storage_group = put_storagegroup(
|
||||
self.shell, self.cluster.default_rpc_endpoint, self.main_wallet, cid, objects
|
||||
)
|
||||
eacl_deny = [
|
||||
EACLRule(access=EACLAccess.DENY, role=role, operation=op)
|
||||
for op in EACLOperation
|
||||
for role in EACLRole
|
||||
]
|
||||
set_eacl(
|
||||
self.main_wallet,
|
||||
cid,
|
||||
create_eacl(cid, eacl_deny, shell=self.shell),
|
||||
shell=self.shell,
|
||||
endpoint=self.cluster.default_rpc_endpoint,
|
||||
)
|
||||
self.expect_failure_for_storagegroup_operations(
|
||||
self.main_wallet, cid, objects, storage_group
|
||||
)
|
||||
bearer_file = form_bearertoken_file(
|
||||
self.main_wallet,
|
||||
cid,
|
||||
[
|
||||
EACLRule(operation=op, access=EACLAccess.ALLOW, role=EACLRole.USER)
|
||||
for op in EACLOperation
|
||||
],
|
||||
shell=self.shell,
|
||||
endpoint=self.cluster.default_rpc_endpoint,
|
||||
)
|
||||
self.expect_success_for_storagegroup_operations(
|
||||
wallet=self.main_wallet,
|
||||
cid=cid,
|
||||
obj_list=objects,
|
||||
object_size=object_size,
|
||||
max_object_size=max_object_size,
|
||||
bearer=bearer_file,
|
||||
)
|
||||
|
||||
@allure.title("Test to check Storage Group lifetime")
|
||||
def test_storagegroup_lifetime(self, object_size):
|
||||
cid = create_container(
|
||||
self.main_wallet, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint
|
||||
)
|
||||
file_path = generate_file(object_size)
|
||||
oid = put_object_to_random_node(
|
||||
self.main_wallet, file_path, cid, shell=self.shell, cluster=self.cluster
|
||||
)
|
||||
objects = [oid]
|
||||
storage_group = put_storagegroup(
|
||||
self.shell,
|
||||
self.cluster.default_rpc_endpoint,
|
||||
self.main_wallet,
|
||||
cid,
|
||||
objects,
|
||||
lifetime=1,
|
||||
)
|
||||
with allure.step("Tick two epochs"):
|
||||
for _ in range(2):
|
||||
self.tick_epoch()
|
||||
self.wait_for_epochs_align()
|
||||
with pytest.raises(Exception, match=OBJECT_NOT_FOUND):
|
||||
get_storagegroup(
|
||||
shell=self.shell,
|
||||
endpoint=self.cluster.default_rpc_endpoint,
|
||||
wallet=self.main_wallet,
|
||||
cid=cid,
|
||||
gid=storage_group,
|
||||
)
|
||||
|
||||
@allure.step("Run Storage Group Operations And Expect Success")
|
||||
def expect_success_for_storagegroup_operations(
|
||||
self,
|
||||
wallet: str,
|
||||
cid: str,
|
||||
obj_list: list,
|
||||
object_size: int,
|
||||
max_object_size: int,
|
||||
bearer: Optional[str] = None,
|
||||
):
|
||||
"""
|
||||
This func verifies if the Object's owner is allowed to
|
||||
Put, List, Get and Delete the Storage Group which contains
|
||||
the Object.
|
||||
"""
|
||||
storage_group = put_storagegroup(
|
||||
self.shell, self.cluster.default_rpc_endpoint, wallet, cid, obj_list, bearer
|
||||
)
|
||||
verify_list_storage_group(
|
||||
shell=self.shell,
|
||||
endpoint=self.cluster.default_rpc_endpoint,
|
||||
wallet=wallet,
|
||||
cid=cid,
|
||||
gid=storage_group,
|
||||
bearer=bearer,
|
||||
)
|
||||
verify_get_storage_group(
|
||||
shell=self.shell,
|
||||
cluster=self.cluster,
|
||||
wallet=wallet,
|
||||
cid=cid,
|
||||
gid=storage_group,
|
||||
obj_list=obj_list,
|
||||
object_size=object_size,
|
||||
max_object_size=max_object_size,
|
||||
bearer=bearer,
|
||||
)
|
||||
delete_storagegroup(
|
||||
shell=self.shell,
|
||||
endpoint=self.cluster.default_rpc_endpoint,
|
||||
wallet=wallet,
|
||||
cid=cid,
|
||||
gid=storage_group,
|
||||
bearer=bearer,
|
||||
)
|
||||
|
||||
@allure.step("Run Storage Group Operations And Expect Failure")
|
||||
def expect_failure_for_storagegroup_operations(
|
||||
self, wallet: str, cid: str, obj_list: list, gid: str
|
||||
):
|
||||
"""
|
||||
This func verifies if the Object's owner isn't allowed to
|
||||
Put, List, Get and Delete the Storage Group which contains
|
||||
the Object.
|
||||
"""
|
||||
with pytest.raises(Exception, match=OBJECT_ACCESS_DENIED):
|
||||
put_storagegroup(
|
||||
shell=self.shell,
|
||||
endpoint=self.cluster.default_rpc_endpoint,
|
||||
wallet=wallet,
|
||||
cid=cid,
|
||||
objects=obj_list,
|
||||
)
|
||||
with pytest.raises(Exception, match=OBJECT_ACCESS_DENIED):
|
||||
list_storagegroup(
|
||||
shell=self.shell, endpoint=self.cluster.default_rpc_endpoint, wallet=wallet, cid=cid
|
||||
)
|
||||
with pytest.raises(Exception, match=OBJECT_ACCESS_DENIED):
|
||||
get_storagegroup(
|
||||
shell=self.shell,
|
||||
endpoint=self.cluster.default_rpc_endpoint,
|
||||
wallet=wallet,
|
||||
cid=cid,
|
||||
gid=gid,
|
||||
)
|
||||
with pytest.raises(Exception, match=OBJECT_ACCESS_DENIED):
|
||||
delete_storagegroup(
|
||||
shell=self.shell,
|
||||
endpoint=self.cluster.default_rpc_endpoint,
|
||||
wallet=wallet,
|
||||
cid=cid,
|
||||
gid=gid,
|
||||
)
|
||||
|
||||
@allure.step("Run Storage Group Operations On Other's Behalf In RO Container")
|
||||
def storagegroup_operations_by_other_ro_container(
|
||||
self,
|
||||
owner_wallet: str,
|
||||
other_wallet: str,
|
||||
cid: str,
|
||||
obj_list: list,
|
||||
object_size: int,
|
||||
max_object_size: int,
|
||||
):
|
||||
storage_group = put_storagegroup(
|
||||
self.shell, self.cluster.default_rpc_endpoint, owner_wallet, cid, obj_list
|
||||
)
|
||||
with pytest.raises(Exception, match=OBJECT_ACCESS_DENIED):
|
||||
put_storagegroup(
|
||||
shell=self.shell,
|
||||
endpoint=self.cluster.default_rpc_endpoint,
|
||||
wallet=other_wallet,
|
||||
cid=cid,
|
||||
objects=obj_list,
|
||||
)
|
||||
verify_list_storage_group(
|
||||
shell=self.shell,
|
||||
endpoint=self.cluster.default_rpc_endpoint,
|
||||
wallet=other_wallet,
|
||||
cid=cid,
|
||||
gid=storage_group,
|
||||
)
|
||||
verify_get_storage_group(
|
||||
shell=self.shell,
|
||||
cluster=self.cluster,
|
||||
wallet=other_wallet,
|
||||
cid=cid,
|
||||
gid=storage_group,
|
||||
obj_list=obj_list,
|
||||
object_size=object_size,
|
||||
max_object_size=max_object_size,
|
||||
)
|
||||
with pytest.raises(Exception, match=OBJECT_ACCESS_DENIED):
|
||||
delete_storagegroup(
|
||||
shell=self.shell,
|
||||
endpoint=self.cluster.default_rpc_endpoint,
|
||||
wallet=other_wallet,
|
||||
cid=cid,
|
||||
gid=storage_group,
|
||||
)
|
||||
|
||||
@allure.step("Run Storage Group Operations On Systems's Behalf In RO Container")
|
||||
def storagegroup_operations_by_system_ro_container(
|
||||
self,
|
||||
wallet: str,
|
||||
cid: str,
|
||||
obj_list: list,
|
||||
object_size: int,
|
||||
max_object_size: int,
|
||||
):
|
||||
"""
|
||||
In this func we create a Storage Group on Inner Ring's key behalf
|
||||
and include an Object created on behalf of some user. We expect
|
||||
that System key is granted to make all operations except PUT and DELETE.
|
||||
"""
|
||||
ir_node = self.cluster.ir_nodes[0]
|
||||
ir_wallet_path = ir_node.get_wallet_path()
|
||||
ir_wallet_password = ir_node.get_wallet_password()
|
||||
ir_wallet_config = ir_node.get_wallet_config_path()
|
||||
|
||||
if not FREE_STORAGE:
|
||||
main_chain = self.cluster.main_chain_nodes[0]
|
||||
deposit = 30
|
||||
transfer_gas(
|
||||
shell=self.shell,
|
||||
amount=deposit + 1,
|
||||
main_chain=main_chain,
|
||||
wallet_to_path=ir_wallet_path,
|
||||
wallet_to_password=ir_wallet_password,
|
||||
)
|
||||
deposit_gas(
|
||||
shell=self.shell,
|
||||
amount=deposit,
|
||||
main_chain=main_chain,
|
||||
wallet_from_path=ir_wallet_path,
|
||||
wallet_from_password=ir_wallet_password,
|
||||
)
|
||||
storage_group = put_storagegroup(
|
||||
self.shell, self.cluster.default_rpc_endpoint, wallet, cid, obj_list
|
||||
)
|
||||
with pytest.raises(Exception, match=OBJECT_ACCESS_DENIED):
|
||||
put_storagegroup(
|
||||
self.shell,
|
||||
self.cluster.default_rpc_endpoint,
|
||||
ir_wallet_path,
|
||||
cid,
|
||||
obj_list,
|
||||
wallet_config=ir_wallet_config,
|
||||
)
|
||||
verify_list_storage_group(
|
||||
shell=self.shell,
|
||||
endpoint=self.cluster.default_rpc_endpoint,
|
||||
wallet=ir_wallet_path,
|
||||
cid=cid,
|
||||
gid=storage_group,
|
||||
wallet_config=ir_wallet_config,
|
||||
)
|
||||
verify_get_storage_group(
|
||||
shell=self.shell,
|
||||
cluster=self.cluster,
|
||||
wallet=ir_wallet_path,
|
||||
cid=cid,
|
||||
gid=storage_group,
|
||||
obj_list=obj_list,
|
||||
object_size=object_size,
|
||||
max_object_size=max_object_size,
|
||||
wallet_config=ir_wallet_config,
|
||||
)
|
||||
with pytest.raises(Exception, match=OBJECT_ACCESS_DENIED):
|
||||
delete_storagegroup(
|
||||
shell=self.shell,
|
||||
endpoint=self.cluster.default_rpc_endpoint,
|
||||
wallet=ir_wallet_path,
|
||||
cid=cid,
|
||||
gid=storage_group,
|
||||
wallet_config=ir_wallet_config,
|
||||
)
|
|
@ -1,16 +1,15 @@
|
|||
import allure
|
||||
import pytest
|
||||
from frostfs_testlib.resources.common import PRIVATE_ACL_F, PUBLIC_ACL_F, READONLY_ACL_F
|
||||
|
||||
from pytest_tests.helpers.acl import EACLRole
|
||||
from pytest_tests.helpers.container import create_container
|
||||
from pytest_tests.helpers.container_access import (
|
||||
from cluster_test_base import ClusterTestBase
|
||||
from python_keywords.acl import EACLRole
|
||||
from python_keywords.container import create_container
|
||||
from python_keywords.container_access import (
|
||||
check_full_access_to_container,
|
||||
check_no_access_to_container,
|
||||
check_read_only_container,
|
||||
)
|
||||
from pytest_tests.helpers.frostfs_verbs import put_object_to_random_node
|
||||
from pytest_tests.steps.cluster_test_base import ClusterTestBase
|
||||
from python_keywords.neofs_verbs import put_object_to_random_node
|
||||
from wellknown_acl import PRIVATE_ACL_F, PUBLIC_ACL_F, READONLY_ACL_F
|
||||
|
||||
|
||||
@pytest.mark.sanity
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
import allure
|
||||
import pytest
|
||||
|
||||
from pytest_tests.helpers.acl import (
|
||||
from cluster_test_base import ClusterTestBase
|
||||
from python_keywords.acl import (
|
||||
EACLAccess,
|
||||
EACLOperation,
|
||||
EACLRole,
|
||||
|
@ -11,12 +11,11 @@ from pytest_tests.helpers.acl import (
|
|||
set_eacl,
|
||||
wait_for_cache_expired,
|
||||
)
|
||||
from pytest_tests.helpers.container_access import (
|
||||
from python_keywords.container_access import (
|
||||
check_custom_access_to_container,
|
||||
check_full_access_to_container,
|
||||
check_no_access_to_container,
|
||||
)
|
||||
from pytest_tests.steps.cluster_test_base import ClusterTestBase
|
||||
|
||||
|
||||
@pytest.mark.sanity
|
||||
|
@ -25,9 +24,7 @@ from pytest_tests.steps.cluster_test_base import ClusterTestBase
|
|||
class TestACLBearer(ClusterTestBase):
|
||||
@pytest.mark.parametrize("role", [EACLRole.USER, EACLRole.OTHERS])
|
||||
def test_bearer_token_operations(self, wallets, eacl_container_with_objects, role):
|
||||
allure.dynamic.title(
|
||||
f"Testcase to validate FrostFS operations with {role.value} BearerToken"
|
||||
)
|
||||
allure.dynamic.title(f"Testcase to validate NeoFS operations with {role.value} BearerToken")
|
||||
cid, objects_oids, file_path = eacl_container_with_objects
|
||||
user_wallet = wallets.get_wallet()
|
||||
deny_wallet = wallets.get_wallet(role)
|
||||
|
|
|
@ -1,8 +1,9 @@
|
|||
import allure
|
||||
import pytest
|
||||
from frostfs_testlib.resources.common import PUBLIC_ACL
|
||||
|
||||
from pytest_tests.helpers.acl import (
|
||||
from cluster_test_base import ClusterTestBase
|
||||
from failover_utils import wait_object_replication
|
||||
from neofs_testlib.shell import Shell
|
||||
from python_keywords.acl import (
|
||||
EACLAccess,
|
||||
EACLOperation,
|
||||
EACLRole,
|
||||
|
@ -11,15 +12,14 @@ from pytest_tests.helpers.acl import (
|
|||
set_eacl,
|
||||
wait_for_cache_expired,
|
||||
)
|
||||
from pytest_tests.helpers.container import create_container
|
||||
from pytest_tests.helpers.container_access import (
|
||||
from python_keywords.container import create_container
|
||||
from python_keywords.container_access import (
|
||||
check_full_access_to_container,
|
||||
check_no_access_to_container,
|
||||
)
|
||||
from pytest_tests.helpers.failover_utils import wait_object_replication
|
||||
from pytest_tests.helpers.frostfs_verbs import put_object_to_random_node
|
||||
from pytest_tests.helpers.node_management import drop_object
|
||||
from pytest_tests.helpers.object_access import (
|
||||
from python_keywords.neofs_verbs import put_object_to_random_node
|
||||
from python_keywords.node_management import drop_object
|
||||
from python_keywords.object_access import (
|
||||
can_delete_object,
|
||||
can_get_head_object,
|
||||
can_get_object,
|
||||
|
@ -28,7 +28,7 @@ from pytest_tests.helpers.object_access import (
|
|||
can_put_object,
|
||||
can_search_object,
|
||||
)
|
||||
from pytest_tests.steps.cluster_test_base import ClusterTestBase
|
||||
from wellknown_acl import PUBLIC_ACL
|
||||
|
||||
|
||||
@pytest.mark.sanity
|
||||
|
@ -74,7 +74,7 @@ class TestEACLContainer(ClusterTestBase):
|
|||
not_deny_role_wallet = user_wallet if deny_role == EACLRole.OTHERS else other_wallet
|
||||
deny_role_str = "all others" if deny_role == EACLRole.OTHERS else "user"
|
||||
not_deny_role_str = "user" if deny_role == EACLRole.OTHERS else "all others"
|
||||
allure.dynamic.title(f"Testcase to deny FrostFS operations for {deny_role_str}.")
|
||||
allure.dynamic.title(f"Testcase to deny NeoFS operations for {deny_role_str}.")
|
||||
cid, object_oids, file_path = eacl_container_with_objects
|
||||
|
||||
with allure.step(f"Deny all operations for {deny_role_str} via eACL"):
|
||||
|
@ -148,7 +148,7 @@ class TestEACLContainer(ClusterTestBase):
|
|||
cluster=self.cluster,
|
||||
)
|
||||
|
||||
@allure.title("Testcase to allow FrostFS operations for only one other pubkey.")
|
||||
@allure.title("Testcase to allow NeoFS operations for only one other pubkey.")
|
||||
def test_extended_acl_deny_all_operations_exclude_pubkey(
|
||||
self, wallets, eacl_container_with_objects
|
||||
):
|
||||
|
@ -209,7 +209,7 @@ class TestEACLContainer(ClusterTestBase):
|
|||
cluster=self.cluster,
|
||||
)
|
||||
|
||||
@allure.title("Testcase to validate FrostFS replication with eACL deny rules.")
|
||||
@allure.title("Testcase to validate NeoFS replication with eACL deny rules.")
|
||||
def test_extended_acl_deny_replication(
|
||||
self,
|
||||
wallets,
|
||||
|
@ -251,7 +251,7 @@ class TestEACLContainer(ClusterTestBase):
|
|||
storage_nodes,
|
||||
)
|
||||
|
||||
@allure.title("Testcase to validate FrostFS system operations with extended ACL")
|
||||
@allure.title("Testcase to validate NeoFS system operations with extended ACL")
|
||||
def test_extended_actions_system(self, wallets, eacl_container_with_objects):
|
||||
user_wallet = wallets.get_wallet()
|
||||
ir_wallet, storage_wallet = wallets.get_wallets_list(role=EACLRole.SYSTEM)[:2]
|
||||
|
|
|
@ -1,8 +1,7 @@
|
|||
import allure
|
||||
import pytest
|
||||
from frostfs_testlib.resources.common import PUBLIC_ACL
|
||||
|
||||
from pytest_tests.helpers.acl import (
|
||||
from cluster_test_base import ClusterTestBase
|
||||
from python_keywords.acl import (
|
||||
EACLAccess,
|
||||
EACLFilter,
|
||||
EACLFilters,
|
||||
|
@ -16,14 +15,14 @@ from pytest_tests.helpers.acl import (
|
|||
set_eacl,
|
||||
wait_for_cache_expired,
|
||||
)
|
||||
from pytest_tests.helpers.container import create_container, delete_container
|
||||
from pytest_tests.helpers.container_access import (
|
||||
from python_keywords.container import create_container, delete_container
|
||||
from python_keywords.container_access import (
|
||||
check_full_access_to_container,
|
||||
check_no_access_to_container,
|
||||
)
|
||||
from pytest_tests.helpers.frostfs_verbs import put_object_to_random_node
|
||||
from pytest_tests.helpers.object_access import can_get_head_object, can_get_object, can_put_object
|
||||
from pytest_tests.steps.cluster_test_base import ClusterTestBase
|
||||
from python_keywords.neofs_verbs import put_object_to_random_node
|
||||
from python_keywords.object_access import can_get_head_object, can_get_object, can_put_object
|
||||
from wellknown_acl import PUBLIC_ACL
|
||||
|
||||
|
||||
@pytest.mark.sanity
|
||||
|
@ -129,7 +128,7 @@ class TestEACLFilters(ClusterTestBase):
|
|||
"match_type", [EACLMatchType.STRING_EQUAL, EACLMatchType.STRING_NOT_EQUAL]
|
||||
)
|
||||
def test_extended_acl_filters_request(self, wallets, eacl_container_with_objects, match_type):
|
||||
allure.dynamic.title(f"Validate FrostFS operations with request filter: {match_type.name}")
|
||||
allure.dynamic.title(f"Validate NeoFS operations with request filter: {match_type.name}")
|
||||
user_wallet = wallets.get_wallet()
|
||||
other_wallet = wallets.get_wallet(EACLRole.OTHERS)
|
||||
(
|
||||
|
@ -244,7 +243,7 @@ class TestEACLFilters(ClusterTestBase):
|
|||
self, wallets, eacl_container_with_objects, match_type
|
||||
):
|
||||
allure.dynamic.title(
|
||||
f"Validate FrostFS operations with deny user headers filter: {match_type.name}"
|
||||
f"Validate NeoFS operations with deny user headers filter: {match_type.name}"
|
||||
)
|
||||
user_wallet = wallets.get_wallet()
|
||||
other_wallet = wallets.get_wallet(EACLRole.OTHERS)
|
||||
|
@ -426,7 +425,7 @@ class TestEACLFilters(ClusterTestBase):
|
|||
self, wallets, eacl_container_with_objects, match_type
|
||||
):
|
||||
allure.dynamic.title(
|
||||
"Testcase to validate FrostFS operation with allow eACL user headers filters:"
|
||||
"Testcase to validate NeoFS operation with allow eACL user headers filters:"
|
||||
f"{match_type.name}"
|
||||
)
|
||||
user_wallet = wallets.get_wallet()
|
||||
|
|
|
@ -1,5 +1,6 @@
|
|||
import logging
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
@ -7,19 +8,9 @@ from datetime import datetime
|
|||
import allure
|
||||
import pytest
|
||||
import yaml
|
||||
from frostfs_testlib.hosting import Hosting
|
||||
from frostfs_testlib.reporter import AllureHandler, get_reporter
|
||||
from frostfs_testlib.shell import LocalShell, Shell
|
||||
from frostfs_testlib.utils import wallet_utils
|
||||
|
||||
from pytest_tests.helpers import binary_version, env_properties
|
||||
from pytest_tests.helpers.cluster import Cluster
|
||||
from pytest_tests.helpers.frostfs_verbs import get_netmap_netinfo
|
||||
from pytest_tests.helpers.k6 import LoadParams
|
||||
from pytest_tests.helpers.node_management import storage_node_healthcheck
|
||||
from pytest_tests.helpers.payment_neogo import deposit_gas, transfer_gas
|
||||
from pytest_tests.helpers.wallet import WalletFactory
|
||||
from pytest_tests.resources.common import (
|
||||
from binary_version_helper import get_local_binaries_versions, get_remote_binaries_versions
|
||||
from cluster import Cluster
|
||||
from common import (
|
||||
ASSETS_DIR,
|
||||
COMPLEX_OBJECT_CHUNKS_COUNT,
|
||||
COMPLEX_OBJECT_TAIL_SIZE,
|
||||
|
@ -29,7 +20,10 @@ from pytest_tests.resources.common import (
|
|||
STORAGE_NODE_SERVICE_NAME_REGEX,
|
||||
WALLET_PASS,
|
||||
)
|
||||
from pytest_tests.resources.load_params import (
|
||||
from env_properties import save_env_properties
|
||||
from k6 import LoadParams
|
||||
from load import get_services_endpoints, prepare_k6_instances
|
||||
from load_params import (
|
||||
BACKGROUND_LOAD_MAX_TIME,
|
||||
BACKGROUND_OBJ_SIZE,
|
||||
BACKGROUND_READERS_COUNT,
|
||||
|
@ -38,25 +32,24 @@ from pytest_tests.resources.load_params import (
|
|||
LOAD_NODE_SSH_USER,
|
||||
LOAD_NODES,
|
||||
)
|
||||
from pytest_tests.steps.load import get_services_endpoints, prepare_k6_instances
|
||||
from neofs_testlib.hosting import Hosting
|
||||
from neofs_testlib.reporter import AllureHandler, get_reporter
|
||||
from neofs_testlib.shell import LocalShell, Shell
|
||||
from neofs_testlib.utils.wallet import init_wallet
|
||||
from payment_neogo import deposit_gas, transfer_gas
|
||||
from python_keywords.neofs_verbs import get_netmap_netinfo
|
||||
from python_keywords.node_management import storage_node_healthcheck
|
||||
|
||||
from helpers.wallet import WalletFactory
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
|
||||
|
||||
# Add logs check test even if it's not fit to mark selectors
|
||||
def pytest_configure(config: pytest.Config):
|
||||
markers = config.option.markexpr
|
||||
if markers != "":
|
||||
config.option.markexpr = f"logs_after_session or ({markers})"
|
||||
|
||||
|
||||
# pytest hook. Do not rename
|
||||
def pytest_collection_modifyitems(items):
|
||||
# Make network tests last based on @pytest.mark.node_mgmt and logs_test to be latest
|
||||
# Make network tests last based on @pytest.mark.node_mgmt
|
||||
def priority(item: pytest.Item) -> int:
|
||||
is_node_mgmt_test = 1 if item.get_closest_marker("node_mgmt") else 0
|
||||
is_logs_check_test = 100 if item.get_closest_marker("logs_after_session") else 0
|
||||
return is_node_mgmt_test + is_logs_check_test
|
||||
is_node_mgmt_test = item.get_closest_marker("node_mgmt")
|
||||
return 0 if not is_node_mgmt_test else 1
|
||||
|
||||
items.sort(key=lambda item: priority(item))
|
||||
|
||||
|
@ -132,11 +125,11 @@ def cluster(temp_directory: str, hosting: Hosting) -> Cluster:
|
|||
@pytest.fixture(scope="session", autouse=True)
|
||||
@allure.title("Check binary versions")
|
||||
def check_binary_versions(request, hosting: Hosting, client_shell: Shell):
|
||||
local_versions = binary_version.get_local_binaries_versions(client_shell)
|
||||
remote_versions = binary_version.get_remote_binaries_versions(hosting)
|
||||
local_versions = get_local_binaries_versions(client_shell)
|
||||
remote_versions = get_remote_binaries_versions(hosting)
|
||||
|
||||
all_versions = {**local_versions, **remote_versions}
|
||||
env_properties.save_env_properties(request.config, all_versions)
|
||||
save_env_properties(request.config, all_versions)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
|
@ -153,16 +146,23 @@ def temp_directory():
|
|||
shutil.rmtree(full_path)
|
||||
|
||||
|
||||
@allure.step("[Autouse/Session] Test session start time")
|
||||
@pytest.fixture(scope="session", autouse=True)
|
||||
def session_start_time():
|
||||
@allure.title("Collect logs")
|
||||
def collect_logs(temp_directory, hosting: Hosting):
|
||||
start_time = datetime.utcnow()
|
||||
return start_time
|
||||
yield
|
||||
end_time = datetime.utcnow()
|
||||
|
||||
# Dump logs to temp directory (because they might be too large to keep in RAM)
|
||||
logs_dir = os.path.join(temp_directory, "logs")
|
||||
dump_logs(hosting, logs_dir, start_time, end_time)
|
||||
attach_logs(logs_dir)
|
||||
check_logs(logs_dir)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session", autouse=True)
|
||||
@allure.title("Run health check for all storage nodes")
|
||||
def run_health_check(session_start_time, cluster: Cluster):
|
||||
def run_health_check(collect_logs, cluster: Cluster):
|
||||
failed_nodes = []
|
||||
for node in cluster.storage_nodes:
|
||||
health_check = storage_node_healthcheck(node)
|
||||
|
@ -242,7 +242,7 @@ def background_grpc_load(client_shell: Shell, hosting: Hosting):
|
|||
@allure.title("Prepare wallet and deposit")
|
||||
def default_wallet(client_shell: Shell, temp_directory: str, cluster: Cluster):
|
||||
wallet_path = os.path.join(os.getcwd(), ASSETS_DIR, f"{str(uuid.uuid4())}.json")
|
||||
wallet_utils.init_wallet(wallet_path, WALLET_PASS)
|
||||
init_wallet(wallet_path, WALLET_PASS)
|
||||
allure.attach.file(wallet_path, os.path.basename(wallet_path), allure.attachment_type.JSON)
|
||||
|
||||
if not FREE_STORAGE:
|
||||
|
@ -264,3 +264,44 @@ def default_wallet(client_shell: Shell, temp_directory: str, cluster: Cluster):
|
|||
)
|
||||
|
||||
return wallet_path
|
||||
|
||||
|
||||
@allure.title("Check logs for OOM and PANIC entries in {logs_dir}")
|
||||
def check_logs(logs_dir: str):
|
||||
problem_pattern = r"\Wpanic\W|\Woom\W|\Wtoo many open files\W"
|
||||
|
||||
log_file_paths = []
|
||||
for directory_path, _, file_names in os.walk(logs_dir):
|
||||
log_file_paths += [
|
||||
os.path.join(directory_path, file_name)
|
||||
for file_name in file_names
|
||||
if re.match(r"\.(txt|log)", os.path.splitext(file_name)[-1], flags=re.IGNORECASE)
|
||||
]
|
||||
|
||||
logs_with_problem = []
|
||||
for file_path in log_file_paths:
|
||||
with allure.step(f"Check log file {file_path}"):
|
||||
with open(file_path, "r") as log_file:
|
||||
if re.search(problem_pattern, log_file.read(), flags=re.IGNORECASE):
|
||||
logs_with_problem.append(file_path)
|
||||
if logs_with_problem:
|
||||
raise pytest.fail(f"System logs {', '.join(logs_with_problem)} contain critical errors")
|
||||
|
||||
|
||||
def dump_logs(hosting: Hosting, logs_dir: str, since: datetime, until: datetime) -> None:
|
||||
# Dump logs to temp directory (because they might be too large to keep in RAM)
|
||||
os.makedirs(logs_dir)
|
||||
|
||||
for host in hosting.hosts:
|
||||
with allure.step(f"Dump logs from host {host.config.address}"):
|
||||
try:
|
||||
host.dump_logs(logs_dir, since=since, until=until)
|
||||
except Exception as ex:
|
||||
logger.warning(f"Exception during logs collection: {ex}")
|
||||
|
||||
|
||||
def attach_logs(logs_dir: str) -> None:
|
||||
# Zip all files and attach to Allure because it is more convenient to download a single
|
||||
# zip with all logs rather than mess with individual logs files per service or node
|
||||
logs_zip_file_path = shutil.make_archive(logs_dir, "zip", logs_dir)
|
||||
allure.attach.file(logs_zip_file_path, name="logs.zip", extension="zip")
|
||||
|
|
|
@ -2,9 +2,8 @@ import json
|
|||
|
||||
import allure
|
||||
import pytest
|
||||
from frostfs_testlib.resources.common import PRIVATE_ACL_F
|
||||
|
||||
from pytest_tests.helpers.container import (
|
||||
from epoch import tick_epoch
|
||||
from python_keywords.container import (
|
||||
create_container,
|
||||
delete_container,
|
||||
get_container,
|
||||
|
@ -12,8 +11,10 @@ from pytest_tests.helpers.container import (
|
|||
wait_for_container_creation,
|
||||
wait_for_container_deletion,
|
||||
)
|
||||
from pytest_tests.helpers.utility import placement_policy_from_container
|
||||
from pytest_tests.steps.cluster_test_base import ClusterTestBase
|
||||
from utility import placement_policy_from_container
|
||||
from wellknown_acl import PRIVATE_ACL_F
|
||||
|
||||
from steps.cluster_test_base import ClusterTestBase
|
||||
|
||||
|
||||
@pytest.mark.container
|
||||
|
|
|
@ -4,18 +4,15 @@ from time import sleep
|
|||
|
||||
import allure
|
||||
import pytest
|
||||
from frostfs_testlib.resources.common import PUBLIC_ACL
|
||||
from cluster import StorageNode
|
||||
from failover_utils import wait_all_storage_nodes_returned, wait_object_replication
|
||||
from file_helper import generate_file, get_file_hash
|
||||
from iptables_helper import IpTablesHelper
|
||||
from python_keywords.container import create_container
|
||||
from python_keywords.neofs_verbs import get_object, put_object_to_random_node
|
||||
from wellknown_acl import PUBLIC_ACL
|
||||
|
||||
from pytest_tests.helpers.cluster import StorageNode
|
||||
from pytest_tests.helpers.container import create_container
|
||||
from pytest_tests.helpers.failover_utils import (
|
||||
wait_all_storage_nodes_returned,
|
||||
wait_object_replication,
|
||||
)
|
||||
from pytest_tests.helpers.file_helper import generate_file, get_file_hash
|
||||
from pytest_tests.helpers.frostfs_verbs import get_object, put_object_to_random_node
|
||||
from pytest_tests.helpers.iptables_helper import IpTablesHelper
|
||||
from pytest_tests.steps.cluster_test_base import ClusterTestBase
|
||||
from steps.cluster_test_base import ClusterTestBase
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
STORAGE_NODE_COMMUNICATION_PORT = "8080"
|
||||
|
|
|
@ -2,26 +2,23 @@ import logging
|
|||
|
||||
import allure
|
||||
import pytest
|
||||
from frostfs_testlib.hosting import Host
|
||||
from frostfs_testlib.resources.common import PUBLIC_ACL
|
||||
from frostfs_testlib.shell import CommandOptions
|
||||
from cluster import Cluster, StorageNode
|
||||
from failover_utils import wait_all_storage_nodes_returned, wait_object_replication
|
||||
from file_helper import generate_file, get_file_hash
|
||||
from neofs_testlib.hosting import Host
|
||||
from neofs_testlib.shell import CommandOptions
|
||||
from python_keywords.container import create_container
|
||||
from python_keywords.neofs_verbs import get_object, put_object_to_random_node
|
||||
from wellknown_acl import PUBLIC_ACL
|
||||
|
||||
from pytest_tests.helpers.cluster import Cluster, StorageNode
|
||||
from pytest_tests.helpers.container import create_container
|
||||
from pytest_tests.helpers.failover_utils import (
|
||||
wait_all_storage_nodes_returned,
|
||||
wait_object_replication,
|
||||
)
|
||||
from pytest_tests.helpers.file_helper import generate_file, get_file_hash
|
||||
from pytest_tests.helpers.frostfs_verbs import get_object, put_object_to_random_node
|
||||
from pytest_tests.steps.cluster_test_base import ClusterTestBase
|
||||
from steps.cluster_test_base import ClusterTestBase
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
stopped_nodes: list[StorageNode] = []
|
||||
|
||||
|
||||
@allure.step("Return all stopped hosts")
|
||||
@pytest.fixture(scope="function", autouse=True)
|
||||
@allure.step("Return all stopped hosts")
|
||||
def after_run_return_all_stopped_hosts(cluster: Cluster):
|
||||
yield
|
||||
return_stopped_hosts(cluster)
|
||||
|
|
|
@ -1,14 +1,22 @@
|
|||
import allure
|
||||
import pytest
|
||||
from frostfs_testlib.hosting import Hosting
|
||||
|
||||
from pytest_tests.helpers.k6 import LoadParams
|
||||
from pytest_tests.resources.common import (
|
||||
from cluster_test_base import ClusterTestBase
|
||||
from common import (
|
||||
HTTP_GATE_SERVICE_NAME_REGEX,
|
||||
S3_GATE_SERVICE_NAME_REGEX,
|
||||
STORAGE_NODE_SERVICE_NAME_REGEX,
|
||||
)
|
||||
from pytest_tests.resources.load_params import (
|
||||
from k6 import LoadParams
|
||||
from load import (
|
||||
clear_cache_and_data,
|
||||
get_services_endpoints,
|
||||
init_s3_client,
|
||||
multi_node_k6_run,
|
||||
prepare_k6_instances,
|
||||
start_stopped_nodes,
|
||||
stop_unused_nodes,
|
||||
)
|
||||
from load_params import (
|
||||
CONTAINER_PLACEMENT_POLICY,
|
||||
CONTAINERS_COUNT,
|
||||
DELETERS,
|
||||
|
@ -25,16 +33,7 @@ from pytest_tests.resources.load_params import (
|
|||
STORAGE_NODE_COUNT,
|
||||
WRITERS,
|
||||
)
|
||||
from pytest_tests.steps.cluster_test_base import ClusterTestBase
|
||||
from pytest_tests.steps.load import (
|
||||
clear_cache_and_data,
|
||||
get_services_endpoints,
|
||||
init_s3_client,
|
||||
multi_node_k6_run,
|
||||
prepare_k6_instances,
|
||||
start_stopped_nodes,
|
||||
stop_unused_nodes,
|
||||
)
|
||||
from neofs_testlib.hosting import Hosting
|
||||
|
||||
ENDPOINTS_ATTRIBUTES = {
|
||||
"http": {"regex": HTTP_GATE_SERVICE_NAME_REGEX, "endpoint_attribute": "endpoint"},
|
||||
|
|
|
@ -5,15 +5,15 @@ from typing import Optional, Tuple
|
|||
|
||||
import allure
|
||||
import pytest
|
||||
from frostfs_testlib.resources.common import OBJECT_NOT_FOUND, PUBLIC_ACL
|
||||
from frostfs_testlib.utils import datetime_utils, string_utils
|
||||
|
||||
from pytest_tests.helpers.cluster import StorageNode
|
||||
from pytest_tests.helpers.container import create_container, get_container
|
||||
from pytest_tests.helpers.epoch import tick_epoch
|
||||
from pytest_tests.helpers.failover_utils import wait_object_replication
|
||||
from pytest_tests.helpers.file_helper import generate_file
|
||||
from pytest_tests.helpers.frostfs_verbs import (
|
||||
from cluster import StorageNode
|
||||
from cluster_test_base import ClusterTestBase
|
||||
from common import MORPH_BLOCK_TIME, NEOFS_CONTRACT_CACHE_TIMEOUT
|
||||
from epoch import tick_epoch
|
||||
from file_helper import generate_file
|
||||
from grpc_responses import OBJECT_NOT_FOUND, error_matches_status
|
||||
from python_keywords.container import create_container, get_container
|
||||
from python_keywords.failover_utils import wait_object_replication
|
||||
from python_keywords.neofs_verbs import (
|
||||
delete_object,
|
||||
get_object,
|
||||
get_object_from_random_node,
|
||||
|
@ -21,25 +21,23 @@ from pytest_tests.helpers.frostfs_verbs import (
|
|||
put_object,
|
||||
put_object_to_random_node,
|
||||
)
|
||||
from pytest_tests.helpers.node_management import (
|
||||
from python_keywords.node_management import (
|
||||
check_node_in_map,
|
||||
delete_node_data,
|
||||
drop_object,
|
||||
exclude_node_from_network_map,
|
||||
get_locode_from_random_node,
|
||||
get_netmap_snapshot,
|
||||
include_node_to_network_map,
|
||||
node_shard_list,
|
||||
node_shard_set_mode,
|
||||
start_storage_nodes,
|
||||
storage_node_healthcheck,
|
||||
storage_node_set_status,
|
||||
)
|
||||
from pytest_tests.helpers.storage_policy import get_nodes_with_object, get_simple_object_copies
|
||||
from pytest_tests.helpers.utility import (
|
||||
placement_policy_from_container,
|
||||
wait_for_gc_pass_on_storage_nodes,
|
||||
)
|
||||
from pytest_tests.resources.common import FROSTFS_CONTRACT_CACHE_TIMEOUT, MORPH_BLOCK_TIME
|
||||
from pytest_tests.steps.cluster_test_base import ClusterTestBase
|
||||
from storage_policy import get_nodes_with_object, get_simple_object_copies
|
||||
from utility import parse_time, placement_policy_from_container, wait_for_gc_pass_on_storage_nodes
|
||||
from wellknown_acl import PUBLIC_ACL
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
check_nodes: list[StorageNode] = []
|
||||
|
@ -113,13 +111,13 @@ class TestNodeManagement(ClusterTestBase):
|
|||
|
||||
# We need to wait for node to establish notifications from morph-chain
|
||||
# Otherwise it will hang up when we will try to set status
|
||||
sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME))
|
||||
sleep(parse_time(MORPH_BLOCK_TIME))
|
||||
|
||||
with allure.step(f"Move node {node} to online state"):
|
||||
storage_node_set_status(node, status="online", retries=2)
|
||||
|
||||
check_nodes.remove(node)
|
||||
sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME))
|
||||
sleep(parse_time(MORPH_BLOCK_TIME))
|
||||
self.tick_epoch_with_retries(3)
|
||||
check_node_in_map(node, shell=self.shell, alive_node=alive_node)
|
||||
|
||||
|
@ -132,7 +130,7 @@ class TestNodeManagement(ClusterTestBase):
|
|||
simple_object_size,
|
||||
):
|
||||
"""
|
||||
This test remove one node from pytest_tests.helpers.cluster then add it back. Test uses base control operations with storage nodes (healthcheck, netmap-snapshot, set-status).
|
||||
This test remove one node from cluster then add it back. Test uses base control operations with storage nodes (healthcheck, netmap-snapshot, set-status).
|
||||
"""
|
||||
wallet = default_wallet
|
||||
placement_rule_3 = "REP 3 IN X CBF 1 SELECT 3 FROM * AS X"
|
||||
|
@ -326,10 +324,10 @@ class TestNodeManagement(ClusterTestBase):
|
|||
self.validate_object_copies(wallet, placement_rule, file_path, expected_copies)
|
||||
|
||||
@pytest.mark.node_mgmt
|
||||
@allure.title("FrostFS object could be dropped using control command")
|
||||
@allure.title("NeoFS object could be dropped using control command")
|
||||
def test_drop_object(self, default_wallet, complex_object_size, simple_object_size):
|
||||
"""
|
||||
Test checks object could be dropped using `frostfs-cli control drop-objects` command.
|
||||
Test checks object could be dropped using `neofs-cli control drop-objects` command.
|
||||
"""
|
||||
wallet = default_wallet
|
||||
endpoint = self.cluster.default_rpc_endpoint
|
||||
|
@ -476,7 +474,7 @@ class TestNodeManagement(ClusterTestBase):
|
|||
if copies == expected_copies:
|
||||
break
|
||||
tick_epoch(self.shell, self.cluster)
|
||||
sleep(datetime_utils.parse_time(FROSTFS_CONTRACT_CACHE_TIMEOUT))
|
||||
sleep(parse_time(NEOFS_CONTRACT_CACHE_TIMEOUT))
|
||||
else:
|
||||
raise AssertionError(f"There are no {expected_copies} copies during time")
|
||||
|
||||
|
@ -487,7 +485,7 @@ class TestNodeManagement(ClusterTestBase):
|
|||
checker(wallet, cid, oid, shell=self.shell, endpoint=endpoint)
|
||||
wait_for_gc_pass_on_storage_nodes()
|
||||
except Exception as err:
|
||||
if string_utils.is_str_match_pattern(err, OBJECT_NOT_FOUND):
|
||||
if error_matches_status(err, OBJECT_NOT_FOUND):
|
||||
return
|
||||
raise AssertionError(f'Expected "{OBJECT_NOT_FOUND}" error, got\n{err}')
|
||||
|
||||
|
|
|
@ -4,21 +4,20 @@ import sys
|
|||
|
||||
import allure
|
||||
import pytest
|
||||
from frostfs_testlib.resources.common import (
|
||||
from cluster import Cluster
|
||||
from complex_object_actions import get_complex_object_split_ranges
|
||||
from file_helper import generate_file, get_file_content, get_file_hash
|
||||
from grpc_responses import (
|
||||
INVALID_LENGTH_SPECIFIER,
|
||||
INVALID_OFFSET_SPECIFIER,
|
||||
INVALID_RANGE_OVERFLOW,
|
||||
INVALID_RANGE_ZERO_LENGTH,
|
||||
OUT_OF_RANGE,
|
||||
)
|
||||
from frostfs_testlib.shell import Shell
|
||||
from neofs_testlib.shell import Shell
|
||||
from pytest import FixtureRequest
|
||||
|
||||
from pytest_tests.helpers.cluster import Cluster
|
||||
from pytest_tests.helpers.complex_object_actions import get_complex_object_split_ranges
|
||||
from pytest_tests.helpers.container import create_container
|
||||
from pytest_tests.helpers.file_helper import generate_file, get_file_content, get_file_hash
|
||||
from pytest_tests.helpers.frostfs_verbs import (
|
||||
from python_keywords.container import create_container
|
||||
from python_keywords.neofs_verbs import (
|
||||
get_object_from_random_node,
|
||||
get_range,
|
||||
get_range_hash,
|
||||
|
@ -26,10 +25,11 @@ from pytest_tests.helpers.frostfs_verbs import (
|
|||
put_object_to_random_node,
|
||||
search_object,
|
||||
)
|
||||
from pytest_tests.helpers.storage_object_info import StorageObjectInfo
|
||||
from pytest_tests.helpers.storage_policy import get_complex_object_copies, get_simple_object_copies
|
||||
from pytest_tests.steps.cluster_test_base import ClusterTestBase
|
||||
from pytest_tests.steps.storage_object import delete_objects
|
||||
from python_keywords.storage_policy import get_complex_object_copies, get_simple_object_copies
|
||||
|
||||
from helpers.storage_object_info import StorageObjectInfo
|
||||
from steps.cluster_test_base import ClusterTestBase
|
||||
from steps.storage_object import delete_objects
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
|
||||
|
|
|
@ -1,30 +1,19 @@
|
|||
import allure
|
||||
import pytest
|
||||
from frostfs_testlib.resources.common import EACL_PUBLIC_READ_WRITE
|
||||
from frostfs_testlib.shell import Shell
|
||||
from cluster import Cluster
|
||||
from container import REP_2_FOR_3_NODES_PLACEMENT_RULE, SINGLE_PLACEMENT_RULE, create_container
|
||||
from epoch import get_epoch
|
||||
from neofs_testlib.shell import Shell
|
||||
from neofs_verbs import delete_object, get_object
|
||||
from pytest import FixtureRequest
|
||||
from python_keywords.acl import EACLAccess, EACLOperation, EACLRole, EACLRule, form_bearertoken_file
|
||||
from wellknown_acl import EACL_PUBLIC_READ_WRITE
|
||||
|
||||
from pytest_tests.helpers.acl import (
|
||||
EACLAccess,
|
||||
EACLOperation,
|
||||
EACLRole,
|
||||
EACLRule,
|
||||
form_bearertoken_file,
|
||||
)
|
||||
from pytest_tests.helpers.cluster import Cluster
|
||||
from pytest_tests.helpers.container import (
|
||||
REP_2_FOR_3_NODES_PLACEMENT_RULE,
|
||||
SINGLE_PLACEMENT_RULE,
|
||||
StorageContainer,
|
||||
StorageContainerInfo,
|
||||
create_container,
|
||||
)
|
||||
from pytest_tests.helpers.epoch import get_epoch
|
||||
from pytest_tests.helpers.frostfs_verbs import delete_object, get_object
|
||||
from pytest_tests.helpers.test_control import expect_not_raises
|
||||
from pytest_tests.helpers.wallet import WalletFile
|
||||
from pytest_tests.steps.cluster_test_base import ClusterTestBase
|
||||
from pytest_tests.steps.storage_object import StorageObjectInfo
|
||||
from helpers.container import StorageContainer, StorageContainerInfo
|
||||
from helpers.test_control import expect_not_raises
|
||||
from helpers.wallet import WalletFile
|
||||
from steps.cluster_test_base import ClusterTestBase
|
||||
from steps.storage_object import StorageObjectInfo
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
|
|
|
@ -2,18 +2,15 @@ import logging
|
|||
|
||||
import allure
|
||||
import pytest
|
||||
from frostfs_testlib.resources.common import OBJECT_NOT_FOUND
|
||||
from epoch import get_epoch, tick_epoch
|
||||
from file_helper import generate_file, get_file_hash
|
||||
from grpc_responses import OBJECT_NOT_FOUND
|
||||
from pytest import FixtureRequest
|
||||
from python_keywords.container import create_container
|
||||
from python_keywords.neofs_verbs import get_object_from_random_node, put_object_to_random_node
|
||||
from utility import wait_for_gc_pass_on_storage_nodes
|
||||
|
||||
from pytest_tests.helpers.container import create_container
|
||||
from pytest_tests.helpers.epoch import get_epoch
|
||||
from pytest_tests.helpers.file_helper import generate_file, get_file_hash
|
||||
from pytest_tests.helpers.frostfs_verbs import (
|
||||
get_object_from_random_node,
|
||||
put_object_to_random_node,
|
||||
)
|
||||
from pytest_tests.helpers.utility import wait_for_gc_pass_on_storage_nodes
|
||||
from pytest_tests.steps.cluster_test_base import ClusterTestBase
|
||||
from steps.cluster_test_base import ClusterTestBase
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
|
||||
|
|
|
@ -3,7 +3,12 @@ import re
|
|||
|
||||
import allure
|
||||
import pytest
|
||||
from frostfs_testlib.resources.common import (
|
||||
from cluster import Cluster
|
||||
from cluster_test_base import ClusterTestBase
|
||||
from common import STORAGE_GC_TIME
|
||||
from complex_object_actions import get_link_object, get_storage_object_chunks
|
||||
from epoch import ensure_fresh_epoch, get_epoch, tick_epoch
|
||||
from grpc_responses import (
|
||||
LIFETIME_REQUIRED,
|
||||
LOCK_NON_REGULAR_OBJECT,
|
||||
LOCK_OBJECT_EXPIRATION,
|
||||
|
@ -12,24 +17,19 @@ from frostfs_testlib.resources.common import (
|
|||
OBJECT_IS_LOCKED,
|
||||
OBJECT_NOT_FOUND,
|
||||
)
|
||||
from frostfs_testlib.shell import Shell
|
||||
from frostfs_testlib.utils import datetime_utils
|
||||
from neofs_testlib.shell import Shell
|
||||
from node_management import drop_object
|
||||
from pytest import FixtureRequest
|
||||
from python_keywords.container import create_container
|
||||
from python_keywords.neofs_verbs import delete_object, head_object, lock_object
|
||||
from storage_policy import get_nodes_with_object
|
||||
from test_control import expect_not_raises, wait_for_success
|
||||
from utility import parse_time, wait_for_gc_pass_on_storage_nodes
|
||||
|
||||
from pytest_tests.helpers.cluster import Cluster
|
||||
from pytest_tests.helpers.complex_object_actions import get_link_object, get_storage_object_chunks
|
||||
from pytest_tests.helpers.container import StorageContainer, StorageContainerInfo, create_container
|
||||
from pytest_tests.helpers.epoch import ensure_fresh_epoch, get_epoch, tick_epoch
|
||||
from pytest_tests.helpers.frostfs_verbs import delete_object, head_object, lock_object
|
||||
from pytest_tests.helpers.node_management import drop_object
|
||||
from pytest_tests.helpers.storage_object_info import LockObjectInfo, StorageObjectInfo
|
||||
from pytest_tests.helpers.storage_policy import get_nodes_with_object
|
||||
from pytest_tests.helpers.test_control import expect_not_raises, wait_for_success
|
||||
from pytest_tests.helpers.utility import wait_for_gc_pass_on_storage_nodes
|
||||
from pytest_tests.helpers.wallet import WalletFactory, WalletFile
|
||||
from pytest_tests.resources.common import STORAGE_GC_TIME
|
||||
from pytest_tests.steps.cluster_test_base import ClusterTestBase
|
||||
from pytest_tests.steps.storage_object import delete_objects
|
||||
from helpers.container import StorageContainer, StorageContainerInfo
|
||||
from helpers.storage_object_info import LockObjectInfo, StorageObjectInfo
|
||||
from helpers.wallet import WalletFactory, WalletFile
|
||||
from steps.storage_object import delete_objects
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
|
||||
|
@ -321,7 +321,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
|
|||
self.cluster.default_rpc_endpoint,
|
||||
)
|
||||
|
||||
@wait_for_success(datetime_utils.parse_time(STORAGE_GC_TIME))
|
||||
@wait_for_success(parse_time(STORAGE_GC_TIME))
|
||||
def check_object_not_found():
|
||||
with pytest.raises(Exception, match=OBJECT_NOT_FOUND):
|
||||
head_object(
|
||||
|
|
101
pytest_tests/testsuites/payment/test_balance.py
Normal file
101
pytest_tests/testsuites/payment/test_balance.py
Normal file
|
@ -0,0 +1,101 @@
|
|||
import logging
|
||||
import os
|
||||
|
||||
import allure
|
||||
import pytest
|
||||
import yaml
|
||||
from cluster_test_base import ClusterTestBase
|
||||
from common import FREE_STORAGE, NEOFS_CLI_EXEC, WALLET_CONFIG
|
||||
from neofs_testlib.cli import NeofsCli
|
||||
from neofs_testlib.shell import CommandResult, Shell
|
||||
from wallet import WalletFactory, WalletFile
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
DEPOSIT_AMOUNT = 30
|
||||
|
||||
|
||||
@pytest.mark.sanity
|
||||
@pytest.mark.payments
|
||||
@pytest.mark.skipif(FREE_STORAGE, reason="Test only works on public network with paid storage")
|
||||
class TestBalanceAccounting(ClusterTestBase):
|
||||
@pytest.fixture(scope="class")
|
||||
def main_wallet(self, wallet_factory: WalletFactory) -> WalletFile:
|
||||
return wallet_factory.create_wallet()
|
||||
|
||||
@pytest.fixture(scope="class")
|
||||
def other_wallet(self, wallet_factory: WalletFactory) -> WalletFile:
|
||||
return wallet_factory.create_wallet()
|
||||
|
||||
@pytest.fixture(scope="class")
|
||||
def cli(self, client_shell: Shell) -> NeofsCli:
|
||||
return NeofsCli(client_shell, NEOFS_CLI_EXEC, WALLET_CONFIG)
|
||||
|
||||
@allure.step("Check deposit amount")
|
||||
def check_amount(self, result: CommandResult) -> None:
|
||||
amount_str = result.stdout.rstrip()
|
||||
|
||||
try:
|
||||
amount = int(amount_str)
|
||||
except Exception as ex:
|
||||
pytest.fail(
|
||||
f"Amount parse error, should be parsable as int({DEPOSIT_AMOUNT}), but given {amount_str}: {ex}"
|
||||
)
|
||||
|
||||
assert amount == DEPOSIT_AMOUNT
|
||||
|
||||
@staticmethod
|
||||
@allure.step("Write config with API endpoint")
|
||||
def write_api_config(config_dir: str, endpoint: str, wallet: str) -> str:
|
||||
with open(WALLET_CONFIG, "r") as file:
|
||||
wallet_config = yaml.full_load(file)
|
||||
api_config = {
|
||||
**wallet_config,
|
||||
"rpc-endpoint": endpoint,
|
||||
"wallet": wallet,
|
||||
}
|
||||
api_config_file = os.path.join(config_dir, "neofs-cli-api-config.yaml")
|
||||
with open(api_config_file, "w") as file:
|
||||
yaml.dump(api_config, file)
|
||||
return api_config_file
|
||||
|
||||
@allure.title("Test balance request with wallet and address")
|
||||
def test_balance_wallet_address(self, main_wallet: WalletFile, cli: NeofsCli):
|
||||
result = cli.accounting.balance(
|
||||
wallet=main_wallet.path,
|
||||
rpc_endpoint=self.cluster.default_rpc_endpoint,
|
||||
address=main_wallet.get_address(),
|
||||
)
|
||||
|
||||
self.check_amount(result)
|
||||
|
||||
@allure.title("Test balance request with wallet only")
|
||||
def test_balance_wallet(self, main_wallet: WalletFile, cli: NeofsCli):
|
||||
result = cli.accounting.balance(
|
||||
wallet=main_wallet.path, rpc_endpoint=self.cluster.default_rpc_endpoint
|
||||
)
|
||||
self.check_amount(result)
|
||||
|
||||
@allure.title("Test balance request with wallet and wrong address")
|
||||
def test_balance_wrong_address(
|
||||
self, main_wallet: WalletFile, other_wallet: WalletFile, cli: NeofsCli
|
||||
):
|
||||
with pytest.raises(Exception, match="address option must be specified and valid"):
|
||||
cli.accounting.balance(
|
||||
wallet=main_wallet.path,
|
||||
rpc_endpoint=self.cluster.default_rpc_endpoint,
|
||||
address=other_wallet.get_address(),
|
||||
)
|
||||
|
||||
@allure.title("Test balance request with config file")
|
||||
def test_balance_api(self, temp_directory: str, main_wallet: WalletFile, client_shell: Shell):
|
||||
config_file = self.write_api_config(
|
||||
config_dir=temp_directory,
|
||||
endpoint=self.cluster.default_rpc_endpoint,
|
||||
wallet=main_wallet.path,
|
||||
)
|
||||
logger.info(f"Config with API endpoint: {config_file}")
|
||||
|
||||
cli = NeofsCli(client_shell, NEOFS_CLI_EXEC, config_file=config_file)
|
||||
result = cli.accounting.balance()
|
||||
|
||||
self.check_amount(result)
|
|
@ -2,9 +2,10 @@ import logging
|
|||
|
||||
import allure
|
||||
import pytest
|
||||
from frostfs_testlib.resources.common import PUBLIC_ACL
|
||||
|
||||
from pytest_tests.helpers.acl import (
|
||||
from container import create_container
|
||||
from file_helper import generate_file
|
||||
from http_gate import get_object_and_verify_hashes, upload_via_http_gate_curl
|
||||
from python_keywords.acl import (
|
||||
EACLAccess,
|
||||
EACLOperation,
|
||||
EACLRole,
|
||||
|
@ -16,10 +17,9 @@ from pytest_tests.helpers.acl import (
|
|||
sign_bearer,
|
||||
wait_for_cache_expired,
|
||||
)
|
||||
from pytest_tests.helpers.container import create_container
|
||||
from pytest_tests.helpers.file_helper import generate_file
|
||||
from pytest_tests.helpers.http_gate import get_object_and_verify_hashes, upload_via_http_gate_curl
|
||||
from pytest_tests.steps.cluster_test_base import ClusterTestBase
|
||||
from wellknown_acl import PUBLIC_ACL
|
||||
|
||||
from steps.cluster_test_base import ClusterTestBase
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
|
||||
|
|
|
@ -3,13 +3,10 @@ import os
|
|||
|
||||
import allure
|
||||
import pytest
|
||||
from frostfs_testlib.resources.common import PUBLIC_ACL
|
||||
|
||||
from pytest_tests.helpers.container import create_container
|
||||
from pytest_tests.helpers.epoch import get_epoch
|
||||
from pytest_tests.helpers.file_helper import generate_file, get_file_hash
|
||||
from pytest_tests.helpers.frostfs_verbs import put_object_to_random_node
|
||||
from pytest_tests.helpers.http_gate import (
|
||||
from epoch import get_epoch, tick_epoch
|
||||
from file_helper import generate_file, get_file_hash
|
||||
from python_keywords.container import create_container
|
||||
from python_keywords.http_gate import (
|
||||
attr_into_header,
|
||||
get_object_and_verify_hashes,
|
||||
get_object_by_attr_and_verify_hashes,
|
||||
|
@ -20,19 +17,21 @@ from pytest_tests.helpers.http_gate import (
|
|||
upload_via_http_gate,
|
||||
upload_via_http_gate_curl,
|
||||
)
|
||||
from pytest_tests.helpers.utility import wait_for_gc_pass_on_storage_nodes
|
||||
from pytest_tests.steps.cluster_test_base import ClusterTestBase
|
||||
from python_keywords.neofs_verbs import put_object_to_random_node
|
||||
from utility import wait_for_gc_pass_on_storage_nodes
|
||||
from wellknown_acl import PUBLIC_ACL
|
||||
|
||||
from steps.cluster_test_base import ClusterTestBase
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
OBJECT_NOT_FOUND_ERROR = "not found"
|
||||
|
||||
|
||||
@allure.link(
|
||||
"https://github.com/TrueCloudLab/frostfs-http-gw#frostfs-http-gateway",
|
||||
name="frostfs-http-gateway",
|
||||
"https://github.com/nspcc-dev/neofs-http-gw#neofs-http-gateway", name="neofs-http-gateway"
|
||||
)
|
||||
@allure.link("https://github.com/TrueCloudLab/frostfs-http-gw#uploading", name="uploading")
|
||||
@allure.link("https://github.com/TrueCloudLab/frostfs-http-gw#downloading", name="downloading")
|
||||
@allure.link("https://github.com/nspcc-dev/neofs-http-gw#uploading", name="uploading")
|
||||
@allure.link("https://github.com/nspcc-dev/neofs-http-gw#downloading", name="downloading")
|
||||
@pytest.mark.sanity
|
||||
@pytest.mark.http_gate
|
||||
class TestHttpGate(ClusterTestBase):
|
||||
|
@ -51,9 +50,9 @@ class TestHttpGate(ClusterTestBase):
|
|||
|
||||
Steps:
|
||||
1. Create simple and large objects.
|
||||
2. Put objects using gRPC (frostfs-cli).
|
||||
3. Download objects using HTTP gate (https://github.com/TrueCloudLab/frostfs-http-gw#downloading).
|
||||
4. Get objects using gRPC (frostfs-cli).
|
||||
2. Put objects using gRPC (neofs-cli).
|
||||
3. Download objects using HTTP gate (https://github.com/nspcc-dev/neofs-http-gw#downloading).
|
||||
4. Get objects using gRPC (neofs-cli).
|
||||
5. Compare hashes for got objects.
|
||||
6. Compare hashes for got and original objects.
|
||||
|
||||
|
@ -98,8 +97,8 @@ class TestHttpGate(ClusterTestBase):
|
|||
endpoint=self.cluster.default_http_gate_endpoint,
|
||||
)
|
||||
|
||||
@allure.link("https://github.com/TrueCloudLab/frostfs-http-gw#uploading", name="uploading")
|
||||
@allure.link("https://github.com/TrueCloudLab/frostfs-http-gw#downloading", name="downloading")
|
||||
@allure.link("https://github.com/nspcc-dev/neofs-http-gw#uploading", name="uploading")
|
||||
@allure.link("https://github.com/nspcc-dev/neofs-http-gw#downloading", name="downloading")
|
||||
@allure.title("Test Put over HTTP, Get over HTTP")
|
||||
@pytest.mark.smoke
|
||||
def test_put_http_get_http(self, complex_object_size, simple_object_size):
|
||||
|
@ -108,8 +107,8 @@ class TestHttpGate(ClusterTestBase):
|
|||
|
||||
Steps:
|
||||
1. Create simple and large objects.
|
||||
2. Upload objects using HTTP (https://github.com/TrueCloudLab/frostfs-http-gw#uploading).
|
||||
3. Download objects using HTTP gate (https://github.com/TrueCloudLab/frostfs-http-gw#downloading).
|
||||
2. Upload objects using HTTP (https://github.com/nspcc-dev/neofs-http-gw#uploading).
|
||||
3. Download objects using HTTP gate (https://github.com/nspcc-dev/neofs-http-gw#downloading).
|
||||
4. Compare hashes for got and original objects.
|
||||
|
||||
Expected result:
|
||||
|
@ -146,8 +145,7 @@ class TestHttpGate(ClusterTestBase):
|
|||
)
|
||||
|
||||
@allure.link(
|
||||
"https://github.com/TrueCloudLab/frostfs-http-gw#by-attributes",
|
||||
name="download by attributes",
|
||||
"https://github.com/nspcc-dev/neofs-http-gw#by-attributes", name="download by attributes"
|
||||
)
|
||||
@allure.title("Test Put over HTTP, Get over HTTP with headers")
|
||||
@pytest.mark.parametrize(
|
||||
|
@ -166,7 +164,7 @@ class TestHttpGate(ClusterTestBase):
|
|||
Steps:
|
||||
1. Create simple and large objects.
|
||||
2. Upload objects using HTTP with particular attributes in the header.
|
||||
3. Download objects by attributes using HTTP gate (https://github.com/TrueCloudLab/frostfs-http-gw#by-attributes).
|
||||
3. Download objects by attributes using HTTP gate (https://github.com/nspcc-dev/neofs-http-gw#by-attributes).
|
||||
4. Compare hashes for got and original objects.
|
||||
|
||||
Expected result:
|
||||
|
@ -217,7 +215,7 @@ class TestHttpGate(ClusterTestBase):
|
|||
epochs = (curr_epoch, curr_epoch + 1, curr_epoch + 2, curr_epoch + 100)
|
||||
|
||||
for epoch in epochs:
|
||||
headers = {"X-Attribute-Frostfs-Expiration-Epoch": str(epoch)}
|
||||
headers = {"X-Attribute-Neofs-Expiration-Epoch": str(epoch)}
|
||||
|
||||
with allure.step("Put objects using HTTP with attribute Expiration-Epoch"):
|
||||
oids.append(
|
||||
|
|
|
@ -3,26 +3,27 @@ import os
|
|||
|
||||
import allure
|
||||
import pytest
|
||||
from frostfs_testlib.resources.common import PUBLIC_ACL
|
||||
from pytest import FixtureRequest
|
||||
|
||||
from pytest_tests.helpers.container import (
|
||||
from container import (
|
||||
create_container,
|
||||
delete_container,
|
||||
list_containers,
|
||||
wait_for_container_deletion,
|
||||
)
|
||||
from pytest_tests.helpers.file_helper import generate_file
|
||||
from pytest_tests.helpers.frostfs_verbs import delete_object
|
||||
from pytest_tests.helpers.http_gate import (
|
||||
from epoch import tick_epoch
|
||||
from file_helper import generate_file
|
||||
from http_gate import (
|
||||
attr_into_str_header_curl,
|
||||
get_object_by_attr_and_verify_hashes,
|
||||
try_to_get_object_and_expect_error,
|
||||
try_to_get_object_via_passed_request_and_expect_error,
|
||||
upload_via_http_gate_curl,
|
||||
)
|
||||
from pytest_tests.helpers.storage_object_info import StorageObjectInfo
|
||||
from pytest_tests.steps.cluster_test_base import ClusterTestBase
|
||||
from pytest import FixtureRequest
|
||||
from python_keywords.neofs_verbs import delete_object
|
||||
from wellknown_acl import PUBLIC_ACL
|
||||
|
||||
from helpers.storage_object_info import StorageObjectInfo
|
||||
from steps.cluster_test_base import ClusterTestBase
|
||||
|
||||
OBJECT_ALREADY_REMOVED_ERROR = "object already removed"
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
|
|
|
@ -1,18 +1,19 @@
|
|||
import logging
|
||||
import os
|
||||
|
||||
import allure
|
||||
import pytest
|
||||
from frostfs_testlib.resources.common import PUBLIC_ACL
|
||||
|
||||
from pytest_tests.helpers.container import create_container
|
||||
from pytest_tests.helpers.file_helper import generate_file
|
||||
from pytest_tests.helpers.frostfs_verbs import put_object_to_random_node
|
||||
from pytest_tests.helpers.http_gate import (
|
||||
from container import create_container
|
||||
from file_helper import generate_file
|
||||
from http_gate import (
|
||||
get_object_and_verify_hashes,
|
||||
get_object_by_attr_and_verify_hashes,
|
||||
try_to_get_object_via_passed_request_and_expect_error,
|
||||
)
|
||||
from pytest_tests.steps.cluster_test_base import ClusterTestBase
|
||||
from python_keywords.neofs_verbs import put_object_to_random_node
|
||||
from wellknown_acl import PUBLIC_ACL
|
||||
|
||||
from steps.cluster_test_base import ClusterTestBase
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
|
||||
|
@ -39,8 +40,8 @@ class Test_http_object(ClusterTestBase):
|
|||
|
||||
Steps:
|
||||
1. Create object;
|
||||
2. Put objects using gRPC (frostfs-cli) with attributes [--attributes chapter1=peace,chapter2=war];
|
||||
3. Download object using HTTP gate (https://github.com/TrueCloudLab/frostfs-http-gw#downloading);
|
||||
2. Put objects using gRPC (neofs-cli) with attributes [--attributes chapter1=peace,chapter2=war];
|
||||
3. Download object using HTTP gate (https://github.com/nspcc-dev/neofs-http-gw#downloading);
|
||||
4. Compare hashes between original and downloaded object;
|
||||
5. [Negative] Try to the get object with specified attributes and `get` request: [get/$CID/chapter1/peace];
|
||||
6. Download the object with specified attributes and `get_by_attribute` request: [get_by_attribute/$CID/chapter1/peace];
|
||||
|
@ -51,7 +52,6 @@ class Test_http_object(ClusterTestBase):
|
|||
Expected result:
|
||||
Hashes must be the same.
|
||||
"""
|
||||
|
||||
with allure.step("Create public container"):
|
||||
cid = create_container(
|
||||
self.wallet,
|
||||
|
|
|
@ -2,12 +2,12 @@ import logging
|
|||
|
||||
import allure
|
||||
import pytest
|
||||
from frostfs_testlib.resources.common import PUBLIC_ACL
|
||||
from container import create_container
|
||||
from file_helper import generate_file
|
||||
from http_gate import get_object_and_verify_hashes, upload_via_http_gate_curl
|
||||
from wellknown_acl import PUBLIC_ACL
|
||||
|
||||
from pytest_tests.helpers.container import create_container
|
||||
from pytest_tests.helpers.file_helper import generate_file
|
||||
from pytest_tests.helpers.http_gate import get_object_and_verify_hashes, upload_via_http_gate_curl
|
||||
from pytest_tests.steps.cluster_test_base import ClusterTestBase
|
||||
from steps.cluster_test_base import ClusterTestBase
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
|
||||
|
@ -35,7 +35,7 @@ class Test_http_streaming(ClusterTestBase):
|
|||
Steps:
|
||||
1. Create big object;
|
||||
2. Put object using curl with pipe (streaming);
|
||||
3. Download object using HTTP gate (https://github.com/TrueCloudLab/frostfs-http-gw#downloading);
|
||||
3. Download object using HTTP gate (https://github.com/nspcc-dev/neofs-http-gw#downloading);
|
||||
4. Compare hashes between original and downloaded object;
|
||||
|
||||
Expected result:
|
||||
|
|
|
@ -5,33 +5,30 @@ from typing import Optional
|
|||
|
||||
import allure
|
||||
import pytest
|
||||
from frostfs_testlib.resources.common import OBJECT_NOT_FOUND, PUBLIC_ACL
|
||||
|
||||
from pytest_tests.helpers.container import create_container
|
||||
from pytest_tests.helpers.epoch import get_epoch, wait_for_epochs_align
|
||||
from pytest_tests.helpers.file_helper import generate_file
|
||||
from pytest_tests.helpers.frostfs_verbs import (
|
||||
get_netmap_netinfo,
|
||||
get_object_from_random_node,
|
||||
head_object,
|
||||
)
|
||||
from pytest_tests.helpers.http_gate import (
|
||||
from container import create_container
|
||||
from epoch import get_epoch, wait_for_epochs_align
|
||||
from file_helper import generate_file
|
||||
from grpc_responses import OBJECT_NOT_FOUND
|
||||
from http_gate import (
|
||||
attr_into_str_header_curl,
|
||||
get_object_and_verify_hashes,
|
||||
try_to_get_object_and_expect_error,
|
||||
upload_via_http_gate_curl,
|
||||
)
|
||||
from pytest_tests.steps.cluster_test_base import ClusterTestBase
|
||||
from python_keywords.neofs_verbs import get_netmap_netinfo, get_object_from_random_node, head_object
|
||||
from wellknown_acl import PUBLIC_ACL
|
||||
|
||||
from steps.cluster_test_base import ClusterTestBase
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
EXPIRATION_TIMESTAMP_HEADER = "__FROSTFS__EXPIRATION_TIMESTAMP"
|
||||
EXPIRATION_EPOCH_HEADER = "__FROSTFS__EXPIRATION_EPOCH"
|
||||
EXPIRATION_DURATION_HEADER = "__FROSTFS__EXPIRATION_DURATION"
|
||||
EXPIRATION_EXPIRATION_RFC = "__FROSTFS__EXPIRATION_RFC3339"
|
||||
FROSTFS_EXPIRATION_EPOCH = "Frostfs-Expiration-Epoch"
|
||||
FROSTFS_EXPIRATION_DURATION = "Frostfs-Expiration-Duration"
|
||||
FROSTFS_EXPIRATION_TIMESTAMP = "Frostfs-Expiration-Timestamp"
|
||||
FROSTFS_EXPIRATION_RFC3339 = "Frostfs-Expiration-RFC3339"
|
||||
EXPIRATION_TIMESTAMP_HEADER = "__NEOFS__EXPIRATION_TIMESTAMP"
|
||||
EXPIRATION_EPOCH_HEADER = "__NEOFS__EXPIRATION_EPOCH"
|
||||
EXPIRATION_DURATION_HEADER = "__NEOFS__EXPIRATION_DURATION"
|
||||
EXPIRATION_EXPIRATION_RFC = "__NEOFS__EXPIRATION_RFC3339"
|
||||
NEOFS_EXPIRATION_EPOCH = "Neofs-Expiration-Epoch"
|
||||
NEOFS_EXPIRATION_DURATION = "Neofs-Expiration-Duration"
|
||||
NEOFS_EXPIRATION_TIMESTAMP = "Neofs-Expiration-Timestamp"
|
||||
NEOFS_EXIPRATION_RFC3339 = "Neofs-Expiration-RFC3339"
|
||||
|
||||
|
||||
@pytest.mark.sanity
|
||||
|
@ -97,7 +94,7 @@ class Test_http_system_header(ClusterTestBase):
|
|||
f"Validate that only {EXPIRATION_EPOCH_HEADER} exists in header and other headers are abesent"
|
||||
)
|
||||
def validation_for_http_header_attr(self, head_info: dict, expected_epoch: int) -> None:
|
||||
# check that __FROSTFS__EXPIRATION_EPOCH attribute has corresponding epoch
|
||||
# check that __NEOFS__EXPIRATION_EPOCH attribute has corresponding epoch
|
||||
assert self.check_key_value_presented_header(
|
||||
head_info, {EXPIRATION_EPOCH_HEADER: str(expected_epoch)}
|
||||
), f'Expected to find {EXPIRATION_EPOCH_HEADER}: {expected_epoch} in: {head_info["header"]["attributes"]}'
|
||||
|
@ -143,7 +140,7 @@ class Test_http_system_header(ClusterTestBase):
|
|||
@allure.title("[negative] attempt to put object with expired epoch")
|
||||
def test_unable_put_expired_epoch(self, user_container: str, simple_object_size: int):
|
||||
headers = attr_into_str_header_curl(
|
||||
{"Frostfs-Expiration-Epoch": str(get_epoch(self.shell, self.cluster) - 1)}
|
||||
{"Neofs-Expiration-Epoch": str(get_epoch(self.shell, self.cluster) - 1)}
|
||||
)
|
||||
file_path = generate_file(simple_object_size)
|
||||
with allure.step(
|
||||
|
@ -157,12 +154,12 @@ class Test_http_system_header(ClusterTestBase):
|
|||
error_pattern="object has expired",
|
||||
)
|
||||
|
||||
@allure.title("[negative] attempt to put object with negative Frostfs-Expiration-Duration")
|
||||
@allure.title("[negative] attempt to put object with negative Neofs-Expiration-Duration")
|
||||
def test_unable_put_negative_duration(self, user_container: str, simple_object_size: int):
|
||||
headers = attr_into_str_header_curl({"Frostfs-Expiration-Duration": "-1h"})
|
||||
headers = attr_into_str_header_curl({"Neofs-Expiration-Duration": "-1h"})
|
||||
file_path = generate_file(simple_object_size)
|
||||
with allure.step(
|
||||
"Put object using HTTP with attribute Frostfs-Expiration-Duration where duration is negative"
|
||||
"Put object using HTTP with attribute Neofs-Expiration-Duration where duration is negative"
|
||||
):
|
||||
upload_via_http_gate_curl(
|
||||
cid=user_container,
|
||||
|
@ -173,13 +170,13 @@ class Test_http_system_header(ClusterTestBase):
|
|||
)
|
||||
|
||||
@allure.title(
|
||||
"[negative] attempt to put object with Frostfs-Expiration-Timestamp value in the past"
|
||||
"[negative] attempt to put object with Neofs-Expiration-Timestamp value in the past"
|
||||
)
|
||||
def test_unable_put_expired_timestamp(self, user_container: str, simple_object_size: int):
|
||||
headers = attr_into_str_header_curl({"Frostfs-Expiration-Timestamp": "1635075727"})
|
||||
headers = attr_into_str_header_curl({"Neofs-Expiration-Timestamp": "1635075727"})
|
||||
file_path = generate_file(simple_object_size)
|
||||
with allure.step(
|
||||
"Put object using HTTP with attribute Frostfs-Expiration-Timestamp where duration is in the past"
|
||||
"Put object using HTTP with attribute Neofs-Expiration-Timestamp where duration is in the past"
|
||||
):
|
||||
upload_via_http_gate_curl(
|
||||
cid=user_container,
|
||||
|
@ -190,10 +187,10 @@ class Test_http_system_header(ClusterTestBase):
|
|||
)
|
||||
|
||||
@allure.title(
|
||||
"[negative] Put object using HTTP with attribute Frostfs-Expiration-RFC3339 where duration is in the past"
|
||||
"[negative] Put object using HTTP with attribute Neofs-Expiration-RFC3339 where duration is in the past"
|
||||
)
|
||||
def test_unable_put_expired_rfc(self, user_container: str, simple_object_size: int):
|
||||
headers = attr_into_str_header_curl({"Frostfs-Expiration-RFC3339": "2021-11-22T09:55:49Z"})
|
||||
headers = attr_into_str_header_curl({"Neofs-Expiration-RFC3339": "2021-11-22T09:55:49Z"})
|
||||
file_path = generate_file(simple_object_size)
|
||||
upload_via_http_gate_curl(
|
||||
cid=user_container,
|
||||
|
@ -218,7 +215,7 @@ class Test_http_system_header(ClusterTestBase):
|
|||
logger.info(
|
||||
f"epoch duration={epoch_duration}, current_epoch= {get_epoch(self.shell, self.cluster)} expected_epoch {expected_epoch}"
|
||||
)
|
||||
attributes = {FROSTFS_EXPIRATION_EPOCH: expected_epoch, FROSTFS_EXPIRATION_DURATION: "1m"}
|
||||
attributes = {NEOFS_EXPIRATION_EPOCH: expected_epoch, NEOFS_EXPIRATION_DURATION: "1m"}
|
||||
file_path = generate_file(object_size)
|
||||
with allure.step(
|
||||
f"Put objects using HTTP with attributes and head command should display {EXPIRATION_EPOCH_HEADER}: {expected_epoch} attr"
|
||||
|
@ -266,10 +263,10 @@ class Test_http_system_header(ClusterTestBase):
|
|||
f"epoch duration={epoch_duration}, current_epoch= {get_epoch(self.shell, self.cluster)} expected_epoch {expected_epoch}"
|
||||
)
|
||||
attributes = {
|
||||
FROSTFS_EXPIRATION_DURATION: self.epoch_count_into_mins(
|
||||
NEOFS_EXPIRATION_DURATION: self.epoch_count_into_mins(
|
||||
epoch_duration=epoch_duration, epoch=2
|
||||
),
|
||||
FROSTFS_EXPIRATION_TIMESTAMP: self.epoch_count_into_timestamp(
|
||||
NEOFS_EXPIRATION_TIMESTAMP: self.epoch_count_into_timestamp(
|
||||
epoch_duration=epoch_duration, epoch=1
|
||||
),
|
||||
}
|
||||
|
@ -320,10 +317,10 @@ class Test_http_system_header(ClusterTestBase):
|
|||
f"epoch duration={epoch_duration}, current_epoch= {get_epoch(self.shell, self.cluster)} expected_epoch {expected_epoch}"
|
||||
)
|
||||
attributes = {
|
||||
FROSTFS_EXPIRATION_TIMESTAMP: self.epoch_count_into_timestamp(
|
||||
NEOFS_EXPIRATION_TIMESTAMP: self.epoch_count_into_timestamp(
|
||||
epoch_duration=epoch_duration, epoch=2
|
||||
),
|
||||
FROSTFS_EXPIRATION_RFC3339: self.epoch_count_into_timestamp(
|
||||
NEOFS_EXIPRATION_RFC3339: self.epoch_count_into_timestamp(
|
||||
epoch_duration=epoch_duration, epoch=1, rfc3339=True
|
||||
),
|
||||
}
|
||||
|
@ -372,7 +369,7 @@ class Test_http_system_header(ClusterTestBase):
|
|||
f"epoch duration={epoch_duration}, current_epoch= {get_epoch(self.shell, self.cluster)} expected_epoch {expected_epoch}"
|
||||
)
|
||||
attributes = {
|
||||
FROSTFS_EXPIRATION_RFC3339: self.epoch_count_into_timestamp(
|
||||
NEOFS_EXIPRATION_RFC3339: self.epoch_count_into_timestamp(
|
||||
epoch_duration=epoch_duration, epoch=2, rfc3339=True
|
||||
)
|
||||
}
|
||||
|
|
|
@ -1,10 +1,10 @@
|
|||
import allure
|
||||
import pytest
|
||||
from file_helper import generate_file
|
||||
from s3_helper import object_key_from_file_path
|
||||
|
||||
from pytest_tests.helpers.file_helper import generate_file
|
||||
from pytest_tests.helpers.s3_helper import assert_s3_acl, object_key_from_file_path
|
||||
from pytest_tests.steps import s3_gate_bucket, s3_gate_object
|
||||
from pytest_tests.steps.s3_gate_base import TestS3GateBase
|
||||
from steps import s3_gate_bucket, s3_gate_object
|
||||
from steps.s3_gate_base import TestS3GateBase
|
||||
|
||||
|
||||
def pytest_generate_tests(metafunc):
|
||||
|
@ -29,12 +29,19 @@ class TestS3GateACL(TestS3GateBase):
|
|||
with allure.step("Put object ACL = public-read"):
|
||||
s3_gate_object.put_object_acl_s3(self.s3_client, bucket, file_name, "public-read")
|
||||
obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, file_name)
|
||||
assert_s3_acl(acl_grants=obj_acl, permitted_users="AllUsers")
|
||||
obj_permission = [permission.get("Permission") for permission in obj_acl]
|
||||
assert obj_permission == [
|
||||
"FULL_CONTROL",
|
||||
"FULL_CONTROL",
|
||||
], "Permission for all groups is FULL_CONTROL"
|
||||
|
||||
with allure.step("Put object ACL = private"):
|
||||
s3_gate_object.put_object_acl_s3(self.s3_client, bucket, file_name, "private")
|
||||
obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, file_name)
|
||||
assert_s3_acl(acl_grants=obj_acl, permitted_users="CanonicalUser")
|
||||
obj_permission = [permission.get("Permission") for permission in obj_acl]
|
||||
assert obj_permission == [
|
||||
"FULL_CONTROL",
|
||||
], "Permission for Canonical User is FULL_CONTROL"
|
||||
|
||||
with allure.step(
|
||||
"Put object with grant-read uri=http://acs.amazonaws.com/groups/global/AllUsers"
|
||||
|
@ -46,19 +53,30 @@ class TestS3GateACL(TestS3GateBase):
|
|||
grant_read="uri=http://acs.amazonaws.com/groups/global/AllUsers",
|
||||
)
|
||||
obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, file_name)
|
||||
assert_s3_acl(acl_grants=obj_acl, permitted_users="AllUsers")
|
||||
obj_permission = [permission.get("Permission") for permission in obj_acl]
|
||||
assert obj_permission == [
|
||||
"FULL_CONTROL",
|
||||
"FULL_CONTROL",
|
||||
], "Permission for all groups is FULL_CONTROL"
|
||||
|
||||
@allure.title("Test S3: Bucket ACL")
|
||||
def test_s3_bucket_ACL(self):
|
||||
with allure.step("Create bucket with ACL = public-read-write"):
|
||||
bucket = s3_gate_bucket.create_bucket_s3(self.s3_client, True, acl="public-read-write")
|
||||
bucket_acl = s3_gate_bucket.get_bucket_acl(self.s3_client, bucket)
|
||||
assert_s3_acl(acl_grants=bucket_acl, permitted_users="AllUsers")
|
||||
bucket_permission = [permission.get("Permission") for permission in bucket_acl]
|
||||
assert bucket_permission == [
|
||||
"FULL_CONTROL",
|
||||
"FULL_CONTROL",
|
||||
], "Permission for all groups is FULL_CONTROL"
|
||||
|
||||
with allure.step("Change bucket ACL to private"):
|
||||
s3_gate_bucket.put_bucket_acl_s3(self.s3_client, bucket, acl="private")
|
||||
bucket_acl = s3_gate_bucket.get_bucket_acl(self.s3_client, bucket)
|
||||
assert_s3_acl(acl_grants=bucket_acl, permitted_users="CanonicalUser")
|
||||
bucket_permission = [permission.get("Permission") for permission in bucket_acl]
|
||||
assert bucket_permission == [
|
||||
"FULL_CONTROL"
|
||||
], "Permission for CanonicalUser is FULL_CONTROL"
|
||||
|
||||
with allure.step(
|
||||
"Change bucket acl to --grant-write uri=http://acs.amazonaws.com/groups/global/AllUsers"
|
||||
|
@ -69,4 +87,8 @@ class TestS3GateACL(TestS3GateBase):
|
|||
grant_write="uri=http://acs.amazonaws.com/groups/global/AllUsers",
|
||||
)
|
||||
bucket_acl = s3_gate_bucket.get_bucket_acl(self.s3_client, bucket)
|
||||
assert_s3_acl(acl_grants=bucket_acl, permitted_users="AllUsers")
|
||||
bucket_permission = [permission.get("Permission") for permission in bucket_acl]
|
||||
assert bucket_permission == [
|
||||
"FULL_CONTROL",
|
||||
"FULL_CONTROL",
|
||||
], "Permission for all groups is FULL_CONTROL"
|
||||
|
|
|
@ -2,16 +2,11 @@ from datetime import datetime, timedelta
|
|||
|
||||
import allure
|
||||
import pytest
|
||||
from file_helper import generate_file
|
||||
from s3_helper import assert_object_lock_mode, check_objects_in_bucket, object_key_from_file_path
|
||||
|
||||
from pytest_tests.helpers.file_helper import generate_file
|
||||
from pytest_tests.helpers.s3_helper import (
|
||||
assert_object_lock_mode,
|
||||
assert_s3_acl,
|
||||
check_objects_in_bucket,
|
||||
object_key_from_file_path,
|
||||
)
|
||||
from pytest_tests.steps import s3_gate_bucket, s3_gate_object
|
||||
from pytest_tests.steps.s3_gate_base import TestS3GateBase
|
||||
from steps import s3_gate_bucket, s3_gate_object
|
||||
from steps.s3_gate_base import TestS3GateBase
|
||||
|
||||
|
||||
def pytest_generate_tests(metafunc):
|
||||
|
@ -29,26 +24,41 @@ class TestS3GateBucket(TestS3GateBase):
|
|||
with allure.step("Create bucket with ACL private"):
|
||||
bucket = s3_gate_bucket.create_bucket_s3(self.s3_client, True, acl="private")
|
||||
bucket_acl = s3_gate_bucket.get_bucket_acl(self.s3_client, bucket)
|
||||
assert_s3_acl(acl_grants=bucket_acl, permitted_users="CanonicalUser")
|
||||
bucket_permission = [permission.get("Permission") for permission in bucket_acl]
|
||||
assert bucket_permission == [
|
||||
"FULL_CONTROL"
|
||||
], "Permission for CanonicalUser is FULL_CONTROL"
|
||||
|
||||
with allure.step("Create bucket with ACL = public-read"):
|
||||
bucket_1 = s3_gate_bucket.create_bucket_s3(self.s3_client, True, acl="public-read")
|
||||
bucket_acl_1 = s3_gate_bucket.get_bucket_acl(self.s3_client, bucket_1)
|
||||
assert_s3_acl(acl_grants=bucket_acl_1, permitted_users="AllUsers")
|
||||
bucket_permission_1 = [permission.get("Permission") for permission in bucket_acl_1]
|
||||
assert bucket_permission_1 == [
|
||||
"FULL_CONTROL",
|
||||
"FULL_CONTROL",
|
||||
], "Permission for all groups is FULL_CONTROL"
|
||||
|
||||
with allure.step("Create bucket with ACL public-read-write"):
|
||||
bucket_2 = s3_gate_bucket.create_bucket_s3(
|
||||
self.s3_client, True, acl="public-read-write"
|
||||
)
|
||||
bucket_acl_2 = s3_gate_bucket.get_bucket_acl(self.s3_client, bucket_2)
|
||||
assert_s3_acl(acl_grants=bucket_acl_2, permitted_users="AllUsers")
|
||||
bucket_permission_2 = [permission.get("Permission") for permission in bucket_acl_2]
|
||||
assert bucket_permission_2 == [
|
||||
"FULL_CONTROL",
|
||||
"FULL_CONTROL",
|
||||
], "Permission for CanonicalUser is FULL_CONTROL"
|
||||
|
||||
with allure.step("Create bucket with ACL = authenticated-read"):
|
||||
bucket_3 = s3_gate_bucket.create_bucket_s3(
|
||||
self.s3_client, True, acl="authenticated-read"
|
||||
)
|
||||
bucket_acl_3 = s3_gate_bucket.get_bucket_acl(self.s3_client, bucket_3)
|
||||
assert_s3_acl(acl_grants=bucket_acl_3, permitted_users="AllUsers")
|
||||
bucket_permission_3 = [permission.get("Permission") for permission in bucket_acl_3]
|
||||
assert bucket_permission_3 == [
|
||||
"FULL_CONTROL",
|
||||
"FULL_CONTROL",
|
||||
], "Permission for all groups is FULL_CONTROL"
|
||||
|
||||
@allure.title("Test S3: Create Bucket with different ACL by grand")
|
||||
def test_s3_create_bucket_with_grands(self):
|
||||
|
@ -60,7 +70,11 @@ class TestS3GateBucket(TestS3GateBase):
|
|||
grant_read="uri=http://acs.amazonaws.com/groups/global/AllUsers",
|
||||
)
|
||||
bucket_acl = s3_gate_bucket.get_bucket_acl(self.s3_client, bucket)
|
||||
assert_s3_acl(acl_grants=bucket_acl, permitted_users="AllUsers")
|
||||
bucket_permission = [permission.get("Permission") for permission in bucket_acl]
|
||||
assert bucket_permission == [
|
||||
"FULL_CONTROL",
|
||||
"FULL_CONTROL",
|
||||
], "Permission for CanonicalUser is FULL_CONTROL"
|
||||
|
||||
with allure.step("Create bucket with --grant-wtite"):
|
||||
bucket_1 = s3_gate_bucket.create_bucket_s3(
|
||||
|
@ -69,7 +83,11 @@ class TestS3GateBucket(TestS3GateBase):
|
|||
grant_write="uri=http://acs.amazonaws.com/groups/global/AllUsers",
|
||||
)
|
||||
bucket_acl_1 = s3_gate_bucket.get_bucket_acl(self.s3_client, bucket_1)
|
||||
assert_s3_acl(acl_grants=bucket_acl_1, permitted_users="AllUsers")
|
||||
bucket_permission_1 = [permission.get("Permission") for permission in bucket_acl_1]
|
||||
assert bucket_permission_1 == [
|
||||
"FULL_CONTROL",
|
||||
"FULL_CONTROL",
|
||||
], "Permission for all groups is FULL_CONTROL"
|
||||
|
||||
with allure.step("Create bucket with --grant-full-control"):
|
||||
bucket_2 = s3_gate_bucket.create_bucket_s3(
|
||||
|
@ -78,7 +96,11 @@ class TestS3GateBucket(TestS3GateBase):
|
|||
grant_full_control="uri=http://acs.amazonaws.com/groups/global/AllUsers",
|
||||
)
|
||||
bucket_acl_2 = s3_gate_bucket.get_bucket_acl(self.s3_client, bucket_2)
|
||||
assert_s3_acl(acl_grants=bucket_acl_2, permitted_users="AllUsers")
|
||||
bucket_permission_2 = [permission.get("Permission") for permission in bucket_acl_2]
|
||||
assert bucket_permission_2 == [
|
||||
"FULL_CONTROL",
|
||||
"FULL_CONTROL",
|
||||
], "Permission for CanonicalUser is FULL_CONTROL"
|
||||
|
||||
@allure.title("Test S3: create bucket with object lock")
|
||||
def test_s3_bucket_object_lock(self, simple_object_size):
|
||||
|
|
|
@ -4,26 +4,26 @@ from random import choice, choices
|
|||
|
||||
import allure
|
||||
import pytest
|
||||
|
||||
from pytest_tests.helpers.aws_cli_client import AwsCliClient
|
||||
from pytest_tests.helpers.epoch import tick_epoch
|
||||
from pytest_tests.helpers.file_helper import (
|
||||
from aws_cli_client import AwsCliClient
|
||||
from common import ASSETS_DIR
|
||||
from epoch import tick_epoch
|
||||
from file_helper import (
|
||||
generate_file,
|
||||
generate_file_with_content,
|
||||
get_file_content,
|
||||
get_file_hash,
|
||||
split_file,
|
||||
)
|
||||
from pytest_tests.helpers.s3_helper import (
|
||||
from s3_helper import (
|
||||
check_objects_in_bucket,
|
||||
check_tags_by_bucket,
|
||||
check_tags_by_object,
|
||||
set_bucket_versioning,
|
||||
try_to_get_objects_and_expect_error,
|
||||
)
|
||||
from pytest_tests.resources.common import ASSETS_DIR
|
||||
from pytest_tests.steps import s3_gate_bucket, s3_gate_object
|
||||
from pytest_tests.steps.s3_gate_base import TestS3GateBase
|
||||
|
||||
from steps import s3_gate_bucket, s3_gate_object
|
||||
from steps.s3_gate_base import TestS3GateBase
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
|
||||
|
@ -33,9 +33,7 @@ def pytest_generate_tests(metafunc):
|
|||
metafunc.parametrize("s3_client", ["aws cli", "boto3"], indirect=True)
|
||||
|
||||
|
||||
@allure.link(
|
||||
"https://github.com/TrueCloudLab/frostfs-s3-gw#frostfs-s3-gw", name="frostfs-s3-gateway"
|
||||
)
|
||||
@allure.link("https://github.com/nspcc-dev/neofs-s3-gw#neofs-s3-gateway", name="neofs-s3-gateway")
|
||||
@pytest.mark.sanity
|
||||
@pytest.mark.s3_gate
|
||||
@pytest.mark.s3_gate_base
|
||||
|
|
|
@ -3,15 +3,11 @@ from datetime import datetime, timedelta
|
|||
|
||||
import allure
|
||||
import pytest
|
||||
from file_helper import generate_file, generate_file_with_content
|
||||
from s3_helper import assert_object_lock_mode, check_objects_in_bucket, object_key_from_file_path
|
||||
|
||||
from pytest_tests.helpers.file_helper import generate_file, generate_file_with_content
|
||||
from pytest_tests.helpers.s3_helper import (
|
||||
assert_object_lock_mode,
|
||||
check_objects_in_bucket,
|
||||
object_key_from_file_path,
|
||||
)
|
||||
from pytest_tests.steps import s3_gate_bucket, s3_gate_object
|
||||
from pytest_tests.steps.s3_gate_base import TestS3GateBase
|
||||
from steps import s3_gate_bucket, s3_gate_object
|
||||
from steps.s3_gate_base import TestS3GateBase
|
||||
|
||||
|
||||
def pytest_generate_tests(metafunc):
|
||||
|
|
|
@ -1,14 +1,10 @@
|
|||
import allure
|
||||
import pytest
|
||||
from file_helper import generate_file, get_file_hash, split_file
|
||||
from s3_helper import check_objects_in_bucket, object_key_from_file_path, set_bucket_versioning
|
||||
|
||||
from pytest_tests.helpers.file_helper import generate_file, get_file_hash, split_file
|
||||
from pytest_tests.helpers.s3_helper import (
|
||||
check_objects_in_bucket,
|
||||
object_key_from_file_path,
|
||||
set_bucket_versioning,
|
||||
)
|
||||
from pytest_tests.steps import s3_gate_bucket, s3_gate_object
|
||||
from pytest_tests.steps.s3_gate_base import TestS3GateBase
|
||||
from steps import s3_gate_bucket, s3_gate_object
|
||||
from steps.s3_gate_base import TestS3GateBase
|
||||
|
||||
PART_SIZE = 5 * 1024 * 1024
|
||||
|
||||
|
|
|
@ -6,24 +6,16 @@ from random import choices, sample
|
|||
|
||||
import allure
|
||||
import pytest
|
||||
from aws_cli_client import AwsCliClient
|
||||
from common import ASSETS_DIR, FREE_STORAGE, WALLET_PASS
|
||||
from data_formatters import get_wallet_public_key
|
||||
from file_helper import concat_files, generate_file, generate_file_with_content, get_file_hash
|
||||
from neofs_testlib.utils.wallet import init_wallet
|
||||
from python_keywords.payment_neogo import deposit_gas, transfer_gas
|
||||
from s3_helper import assert_object_lock_mode, check_objects_in_bucket, set_bucket_versioning
|
||||
|
||||
from pytest_tests.helpers.aws_cli_client import AwsCliClient
|
||||
from pytest_tests.helpers.file_helper import (
|
||||
concat_files,
|
||||
generate_file,
|
||||
generate_file_with_content,
|
||||
get_file_hash,
|
||||
)
|
||||
from pytest_tests.helpers.payment_neogo import deposit_gas, transfer_gas
|
||||
from pytest_tests.helpers.s3_helper import (
|
||||
assert_object_lock_mode,
|
||||
assert_s3_acl,
|
||||
check_objects_in_bucket,
|
||||
set_bucket_versioning,
|
||||
)
|
||||
from pytest_tests.resources.common import ASSETS_DIR, FREE_STORAGE, WALLET_PASS
|
||||
from pytest_tests.steps import s3_gate_bucket, s3_gate_object
|
||||
from pytest_tests.steps.s3_gate_base import TestS3GateBase
|
||||
from steps import s3_gate_bucket, s3_gate_object
|
||||
from steps.s3_gate_base import TestS3GateBase
|
||||
|
||||
|
||||
def pytest_generate_tests(metafunc):
|
||||
|
@ -139,7 +131,10 @@ class TestS3GateObject(TestS3GateBase):
|
|||
self.s3_client, bucket, obj_key, ACL="public-read-write"
|
||||
)
|
||||
obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, copy_obj_path)
|
||||
assert_s3_acl(acl_grants=obj_acl, permitted_users="CanonicalUser")
|
||||
for control in obj_acl:
|
||||
assert (
|
||||
control.get("Permission") == "FULL_CONTROL"
|
||||
), "Permission for all groups is FULL_CONTROL"
|
||||
|
||||
@allure.title("Test S3: Copy object with metadata")
|
||||
def test_s3_copy_metadate(self, bucket, simple_object_size):
|
||||
|
@ -661,10 +656,37 @@ class TestS3GateObject(TestS3GateBase):
|
|||
{"Key": tag_key_3, "Value": str(tag_value_3)}
|
||||
], "Tags must be the same"
|
||||
|
||||
@pytest.fixture
|
||||
def prepare_two_wallets(self, default_wallet, client_shell):
|
||||
self.main_wallet = default_wallet
|
||||
self.main_public_key = get_wallet_public_key(self.main_wallet, WALLET_PASS)
|
||||
self.other_wallet = os.path.join(os.getcwd(), ASSETS_DIR, f"{str(uuid.uuid4())}.json")
|
||||
init_wallet(self.other_wallet, WALLET_PASS)
|
||||
self.other_public_key = get_wallet_public_key(self.other_wallet, WALLET_PASS)
|
||||
|
||||
if not FREE_STORAGE:
|
||||
main_chain = self.cluster.main_chain_nodes[0]
|
||||
deposit = 30
|
||||
transfer_gas(
|
||||
shell=client_shell,
|
||||
amount=deposit + 1,
|
||||
main_chain=main_chain,
|
||||
wallet_to_path=self.other_wallet,
|
||||
wallet_to_password=WALLET_PASS,
|
||||
)
|
||||
deposit_gas(
|
||||
shell=client_shell,
|
||||
main_chain=main_chain,
|
||||
amount=deposit,
|
||||
wallet_from_path=self.other_wallet,
|
||||
wallet_from_password=WALLET_PASS,
|
||||
)
|
||||
|
||||
@allure.title("Test S3: put object with ACL")
|
||||
@pytest.mark.parametrize("bucket_versioning", ["ENABLED", "SUSPENDED"])
|
||||
def test_s3_put_object_acl(
|
||||
self,
|
||||
prepare_two_wallets,
|
||||
bucket_versioning,
|
||||
bucket,
|
||||
complex_object_size,
|
||||
|
@ -681,7 +703,8 @@ class TestS3GateObject(TestS3GateBase):
|
|||
with allure.step("Put object with acl private"):
|
||||
s3_gate_object.put_object_s3(self.s3_client, bucket, file_path_1, ACL="private")
|
||||
obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, file_name)
|
||||
assert_s3_acl(acl_grants=obj_acl, permitted_users="CanonicalUser")
|
||||
obj_permission = [permission.get("Permission") for permission in obj_acl]
|
||||
assert obj_permission == ["FULL_CONTROL"], "Permission for all groups is FULL_CONTROL"
|
||||
object_1 = s3_gate_object.get_object_s3(self.s3_client, bucket, file_name)
|
||||
assert get_file_hash(file_path_1) == get_file_hash(object_1), "Hashes must be the same"
|
||||
|
||||
|
@ -689,7 +712,11 @@ class TestS3GateObject(TestS3GateBase):
|
|||
file_path_2 = generate_file_with_content(simple_object_size, file_path=file_path_1)
|
||||
s3_gate_object.put_object_s3(self.s3_client, bucket, file_path_2, ACL="public-read")
|
||||
obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, file_name)
|
||||
assert_s3_acl(acl_grants=obj_acl, permitted_users="AllUsers")
|
||||
obj_permission = [permission.get("Permission") for permission in obj_acl]
|
||||
assert obj_permission == [
|
||||
"FULL_CONTROL",
|
||||
"FULL_CONTROL",
|
||||
], "Permission for all groups is FULL_CONTROL"
|
||||
object_2 = s3_gate_object.get_object_s3(self.s3_client, bucket, file_name)
|
||||
assert get_file_hash(file_path_2) == get_file_hash(object_2), "Hashes must be the same"
|
||||
|
||||
|
@ -699,7 +726,11 @@ class TestS3GateObject(TestS3GateBase):
|
|||
self.s3_client, bucket, file_path_3, ACL="public-read-write"
|
||||
)
|
||||
obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, file_name)
|
||||
assert_s3_acl(acl_grants=obj_acl, permitted_users="AllUsers")
|
||||
obj_permission = [permission.get("Permission") for permission in obj_acl]
|
||||
assert obj_permission == [
|
||||
"FULL_CONTROL",
|
||||
"FULL_CONTROL",
|
||||
], "Permission for all groups is FULL_CONTROL"
|
||||
object_3 = s3_gate_object.get_object_s3(self.s3_client, bucket, file_name)
|
||||
assert get_file_hash(file_path_3) == get_file_hash(object_3), "Hashes must be the same"
|
||||
|
||||
|
@ -709,7 +740,11 @@ class TestS3GateObject(TestS3GateBase):
|
|||
self.s3_client, bucket, file_path_4, ACL="authenticated-read"
|
||||
)
|
||||
obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, file_name)
|
||||
assert_s3_acl(acl_grants=obj_acl, permitted_users="AllUsers")
|
||||
obj_permission = [permission.get("Permission") for permission in obj_acl]
|
||||
assert obj_permission == [
|
||||
"FULL_CONTROL",
|
||||
"FULL_CONTROL",
|
||||
], "Permission for all groups is FULL_CONTROL"
|
||||
object_4 = s3_gate_object.get_object_s3(self.s3_client, bucket, file_name)
|
||||
assert get_file_hash(file_path_4) == get_file_hash(object_4), "Hashes must be the same"
|
||||
|
||||
|
@ -725,7 +760,11 @@ class TestS3GateObject(TestS3GateBase):
|
|||
GrantFullControl=f"id={self.other_public_key}",
|
||||
)
|
||||
obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, file_name_5)
|
||||
assert_s3_acl(acl_grants=obj_acl, permitted_users="CanonicalUser")
|
||||
obj_permission = [permission.get("Permission") for permission in obj_acl]
|
||||
assert obj_permission == [
|
||||
"FULL_CONTROL",
|
||||
"FULL_CONTROL",
|
||||
], "Permission for all groups is FULL_CONTROL"
|
||||
object_4 = s3_gate_object.get_object_s3(self.s3_client, bucket, file_name_5)
|
||||
assert get_file_hash(file_path_5) == get_file_hash(object_4), "Hashes must be the same"
|
||||
|
||||
|
@ -740,7 +779,11 @@ class TestS3GateObject(TestS3GateBase):
|
|||
GrantRead="uri=http://acs.amazonaws.com/groups/global/AllUsers",
|
||||
)
|
||||
obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, file_name_5)
|
||||
assert_s3_acl(acl_grants=obj_acl, permitted_users="AllUsers")
|
||||
obj_permission = [permission.get("Permission") for permission in obj_acl]
|
||||
assert obj_permission == [
|
||||
"FULL_CONTROL",
|
||||
"FULL_CONTROL",
|
||||
], "Permission for all groups is FULL_CONTROL"
|
||||
object_7 = s3_gate_object.get_object_s3(self.s3_client, bucket, file_name_5)
|
||||
assert get_file_hash(file_path_7) == get_file_hash(object_7), "Hashes must be the same"
|
||||
|
||||
|
@ -870,9 +913,12 @@ class TestS3GateObject(TestS3GateBase):
|
|||
assert (
|
||||
obj_head.get("Metadata") == object_metadata
|
||||
), f"Metadata of object is {object_metadata}"
|
||||
# Uncomment after https://github.com/nspcc-dev/neofs-s3-gw/issues/685 is solved
|
||||
# obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, obj_key)
|
||||
# assert_s3_acl(acl_grants = obj_acl, permitted_users = "AllUsers")
|
||||
# obj_permission = [permission.get("Permission") for permission in obj_acl]
|
||||
# assert obj_permission == [
|
||||
# "FULL_CONTROL",
|
||||
# "FULL_CONTROL",
|
||||
# ], "Permission for all groups is FULL_CONTROL"
|
||||
|
||||
@allure.title("Test S3 Put 10 nested level object")
|
||||
def test_s3_put_10_folder(self, bucket, temp_directory, simple_object_size):
|
||||
|
|
|
@ -2,21 +2,17 @@ import os
|
|||
|
||||
import allure
|
||||
import pytest
|
||||
from file_helper import generate_file
|
||||
from python_keywords.container import search_container_by_name
|
||||
from python_keywords.storage_policy import get_simple_object_copies
|
||||
from s3_helper import check_objects_in_bucket, object_key_from_file_path, set_bucket_versioning
|
||||
|
||||
from pytest_tests.helpers.container import search_container_by_name
|
||||
from pytest_tests.helpers.file_helper import generate_file
|
||||
from pytest_tests.helpers.s3_helper import (
|
||||
check_objects_in_bucket,
|
||||
object_key_from_file_path,
|
||||
set_bucket_versioning,
|
||||
)
|
||||
from pytest_tests.helpers.storage_policy import get_simple_object_copies
|
||||
from pytest_tests.steps import s3_gate_bucket, s3_gate_object
|
||||
from pytest_tests.steps.s3_gate_base import TestS3GateBase
|
||||
from steps import s3_gate_bucket, s3_gate_object
|
||||
from steps.s3_gate_base import TestS3GateBase
|
||||
|
||||
|
||||
def pytest_generate_tests(metafunc):
|
||||
policy = f"{os.getcwd()}/pytest_tests/resources/files/policy.json"
|
||||
policy = f"{os.getcwd()}/robot/resources/files/policy.json"
|
||||
if "s3_client" in metafunc.fixturenames:
|
||||
metafunc.parametrize(
|
||||
"s3_client",
|
||||
|
@ -99,7 +95,7 @@ class TestS3GatePolicy(TestS3GateBase):
|
|||
s3_gate_bucket.get_bucket_policy(self.s3_client, bucket)
|
||||
|
||||
with allure.step("Put new policy"):
|
||||
custom_policy = f"file://{os.getcwd()}/pytest_tests/resources/files/bucket_policy.json"
|
||||
custom_policy = f"file://{os.getcwd()}/robot/resources/files/bucket_policy.json"
|
||||
custom_policy = {
|
||||
"Version": "2008-10-17",
|
||||
"Id": "aaaa-bbbb-cccc-dddd",
|
||||
|
|
|
@ -1,18 +1,16 @@
|
|||
import os
|
||||
import uuid
|
||||
from random import choice
|
||||
from string import ascii_letters
|
||||
from typing import Tuple
|
||||
|
||||
import allure
|
||||
import pytest
|
||||
from file_helper import generate_file
|
||||
from s3_helper import check_tags_by_bucket, check_tags_by_object, object_key_from_file_path
|
||||
|
||||
from pytest_tests.helpers.file_helper import generate_file
|
||||
from pytest_tests.helpers.s3_helper import (
|
||||
check_tags_by_bucket,
|
||||
check_tags_by_object,
|
||||
object_key_from_file_path,
|
||||
)
|
||||
from pytest_tests.steps import s3_gate_bucket, s3_gate_object
|
||||
from pytest_tests.steps.s3_gate_base import TestS3GateBase
|
||||
from steps import s3_gate_bucket, s3_gate_object
|
||||
from steps.s3_gate_base import TestS3GateBase
|
||||
|
||||
|
||||
def pytest_generate_tests(metafunc):
|
||||
|
|
|
@ -2,11 +2,11 @@ import os
|
|||
|
||||
import allure
|
||||
import pytest
|
||||
from file_helper import generate_file, generate_file_with_content
|
||||
from s3_helper import set_bucket_versioning
|
||||
|
||||
from pytest_tests.helpers.file_helper import generate_file, generate_file_with_content
|
||||
from pytest_tests.helpers.s3_helper import set_bucket_versioning
|
||||
from pytest_tests.steps import s3_gate_bucket, s3_gate_object
|
||||
from pytest_tests.steps.s3_gate_base import TestS3GateBase
|
||||
from steps import s3_gate_bucket, s3_gate_object
|
||||
from steps.s3_gate_base import TestS3GateBase
|
||||
|
||||
|
||||
def pytest_generate_tests(metafunc):
|
||||
|
|
|
@ -5,11 +5,10 @@ from re import match
|
|||
import allure
|
||||
import pytest
|
||||
import requests
|
||||
from frostfs_testlib.hosting import Hosting
|
||||
|
||||
from pytest_tests.helpers.binary_version import get_remote_binaries_versions
|
||||
from pytest_tests.helpers.env_properties import read_env_properties, save_env_properties
|
||||
from pytest_tests.resources.common import BIN_VERSIONS_FILE
|
||||
from binary_version_helper import get_remote_binaries_versions
|
||||
from common import BIN_VERSIONS_FILE
|
||||
from env_properties import read_env_properties, save_env_properties
|
||||
from neofs_testlib.hosting import Hosting
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
|
||||
|
|
|
@ -1,6 +1,5 @@
|
|||
import pytest
|
||||
|
||||
from pytest_tests.helpers.wallet import WalletFactory, WalletFile
|
||||
from wallet import WalletFactory, WalletFile
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
|
|
|
@ -2,15 +2,15 @@ import random
|
|||
|
||||
import allure
|
||||
import pytest
|
||||
from frostfs_testlib.resources.common import SESSION_NOT_FOUND
|
||||
from frostfs_testlib.utils import wallet_utils
|
||||
from cluster_test_base import ClusterTestBase
|
||||
from common import WALLET_PASS
|
||||
from file_helper import generate_file
|
||||
from grpc_responses import SESSION_NOT_FOUND
|
||||
from neofs_testlib.utils.wallet import get_last_address_from_wallet
|
||||
from python_keywords.container import create_container
|
||||
from python_keywords.neofs_verbs import delete_object, put_object, put_object_to_random_node
|
||||
|
||||
from pytest_tests.helpers.container import create_container
|
||||
from pytest_tests.helpers.file_helper import generate_file
|
||||
from pytest_tests.helpers.frostfs_verbs import delete_object, put_object, put_object_to_random_node
|
||||
from pytest_tests.resources.common import WALLET_PASS
|
||||
from pytest_tests.steps.cluster_test_base import ClusterTestBase
|
||||
from pytest_tests.steps.session_token import create_session_token
|
||||
from steps.session_token import create_session_token
|
||||
|
||||
|
||||
@pytest.mark.sanity
|
||||
|
@ -38,7 +38,7 @@ class TestDynamicObjectSession(ClusterTestBase):
|
|||
|
||||
with allure.step("Init wallet"):
|
||||
wallet = default_wallet
|
||||
address = wallet_utils.get_last_address_from_wallet(wallet, "")
|
||||
address = get_last_address_from_wallet(wallet, "")
|
||||
|
||||
with allure.step("Nodes Settlements"):
|
||||
(
|
||||
|
|
|
@ -2,20 +2,15 @@ import logging
|
|||
|
||||
import allure
|
||||
import pytest
|
||||
from frostfs_testlib.resources.common import (
|
||||
EXPIRED_SESSION_TOKEN,
|
||||
MALFORMED_REQUEST,
|
||||
OBJECT_ACCESS_DENIED,
|
||||
OBJECT_NOT_FOUND,
|
||||
)
|
||||
from frostfs_testlib.shell import Shell
|
||||
from cluster import Cluster
|
||||
from cluster_test_base import ClusterTestBase
|
||||
from epoch import ensure_fresh_epoch
|
||||
from file_helper import generate_file
|
||||
from grpc_responses import MALFORMED_REQUEST, OBJECT_ACCESS_DENIED, OBJECT_NOT_FOUND
|
||||
from neofs_testlib.shell import Shell
|
||||
from pytest import FixtureRequest
|
||||
|
||||
from pytest_tests.helpers.cluster import Cluster
|
||||
from pytest_tests.helpers.container import create_container
|
||||
from pytest_tests.helpers.epoch import ensure_fresh_epoch
|
||||
from pytest_tests.helpers.file_helper import generate_file
|
||||
from pytest_tests.helpers.frostfs_verbs import (
|
||||
from python_keywords.container import create_container
|
||||
from python_keywords.neofs_verbs import (
|
||||
delete_object,
|
||||
get_object,
|
||||
get_object_from_random_node,
|
||||
|
@ -25,11 +20,11 @@ from pytest_tests.helpers.frostfs_verbs import (
|
|||
put_object_to_random_node,
|
||||
search_object,
|
||||
)
|
||||
from pytest_tests.helpers.storage_object_info import StorageObjectInfo
|
||||
from pytest_tests.helpers.test_control import expect_not_raises
|
||||
from pytest_tests.helpers.wallet import WalletFile
|
||||
from pytest_tests.steps.cluster_test_base import ClusterTestBase
|
||||
from pytest_tests.steps.session_token import (
|
||||
from test_control import expect_not_raises
|
||||
from wallet import WalletFile
|
||||
|
||||
from helpers.storage_object_info import StorageObjectInfo
|
||||
from steps.session_token import (
|
||||
INVALID_SIGNATURE,
|
||||
UNRELATED_CONTAINER,
|
||||
UNRELATED_KEY,
|
||||
|
@ -41,7 +36,7 @@ from pytest_tests.steps.session_token import (
|
|||
get_object_signed_token,
|
||||
sign_session_token,
|
||||
)
|
||||
from pytest_tests.steps.storage_object import delete_objects
|
||||
from steps.storage_object import delete_objects
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
|
||||
|
@ -187,6 +182,7 @@ class TestObjectStaticSession(ClusterTestBase):
|
|||
)
|
||||
|
||||
@allure.title("Validate static session with range operations")
|
||||
@pytest.mark.static_session
|
||||
@pytest.mark.parametrize(
|
||||
"method_under_test,verb",
|
||||
[(get_range, ObjectVerb.RANGE), (get_range_hash, ObjectVerb.RANGEHASH)],
|
||||
|
@ -226,6 +222,7 @@ class TestObjectStaticSession(ClusterTestBase):
|
|||
)
|
||||
|
||||
@allure.title("Validate static session with search operation")
|
||||
@pytest.mark.static_session
|
||||
@pytest.mark.xfail
|
||||
# (see https://github.com/nspcc-dev/neofs-node/issues/2030)
|
||||
def test_static_session_search(
|
||||
|
@ -253,6 +250,7 @@ class TestObjectStaticSession(ClusterTestBase):
|
|||
assert expected_object_ids == actual_object_ids
|
||||
|
||||
@allure.title("Validate static session with object id not in session")
|
||||
@pytest.mark.static_session
|
||||
def test_static_session_unrelated_object(
|
||||
self,
|
||||
user_wallet: WalletFile,
|
||||
|
@ -277,6 +275,7 @@ class TestObjectStaticSession(ClusterTestBase):
|
|||
)
|
||||
|
||||
@allure.title("Validate static session with user id not in session")
|
||||
@pytest.mark.static_session
|
||||
def test_static_session_head_unrelated_user(
|
||||
self,
|
||||
stranger_wallet: WalletFile,
|
||||
|
@ -303,6 +302,7 @@ class TestObjectStaticSession(ClusterTestBase):
|
|||
)
|
||||
|
||||
@allure.title("Validate static session with wrong verb in session")
|
||||
@pytest.mark.static_session
|
||||
def test_static_session_head_wrong_verb(
|
||||
self,
|
||||
user_wallet: WalletFile,
|
||||
|
@ -329,6 +329,7 @@ class TestObjectStaticSession(ClusterTestBase):
|
|||
)
|
||||
|
||||
@allure.title("Validate static session with container id not in session")
|
||||
@pytest.mark.static_session
|
||||
def test_static_session_unrelated_container(
|
||||
self,
|
||||
user_wallet: WalletFile,
|
||||
|
@ -356,6 +357,7 @@ class TestObjectStaticSession(ClusterTestBase):
|
|||
)
|
||||
|
||||
@allure.title("Validate static session which signed by another wallet")
|
||||
@pytest.mark.static_session
|
||||
def test_static_session_signed_by_other(
|
||||
self,
|
||||
owner_wallet: WalletFile,
|
||||
|
@ -394,6 +396,7 @@ class TestObjectStaticSession(ClusterTestBase):
|
|||
)
|
||||
|
||||
@allure.title("Validate static session which signed for another container")
|
||||
@pytest.mark.static_session
|
||||
def test_static_session_signed_for_other_container(
|
||||
self,
|
||||
owner_wallet: WalletFile,
|
||||
|
@ -432,6 +435,7 @@ class TestObjectStaticSession(ClusterTestBase):
|
|||
)
|
||||
|
||||
@allure.title("Validate static session which wasn't signed")
|
||||
@pytest.mark.static_session
|
||||
def test_static_session_without_sign(
|
||||
self,
|
||||
owner_wallet: WalletFile,
|
||||
|
@ -468,6 +472,7 @@ class TestObjectStaticSession(ClusterTestBase):
|
|||
)
|
||||
|
||||
@allure.title("Validate static session which expires at next epoch")
|
||||
@pytest.mark.static_session
|
||||
def test_static_session_expiration_at_next(
|
||||
self,
|
||||
owner_wallet: WalletFile,
|
||||
|
@ -489,56 +494,40 @@ class TestObjectStaticSession(ClusterTestBase):
|
|||
object_id = storage_objects[0].oid
|
||||
expiration = Lifetime(epoch + 1, epoch, epoch)
|
||||
|
||||
with allure.step("Create session token"):
|
||||
token_expire_at_next_epoch = get_object_signed_token(
|
||||
owner_wallet,
|
||||
user_wallet,
|
||||
token_expire_at_next_epoch = get_object_signed_token(
|
||||
owner_wallet,
|
||||
user_wallet,
|
||||
container,
|
||||
storage_objects,
|
||||
ObjectVerb.HEAD,
|
||||
self.shell,
|
||||
temp_directory,
|
||||
expiration,
|
||||
)
|
||||
|
||||
head_object(
|
||||
user_wallet.path,
|
||||
container,
|
||||
object_id,
|
||||
self.shell,
|
||||
self.cluster.default_rpc_endpoint,
|
||||
session=token_expire_at_next_epoch,
|
||||
)
|
||||
|
||||
self.tick_epoch()
|
||||
|
||||
with pytest.raises(Exception, match=MALFORMED_REQUEST):
|
||||
head_object(
|
||||
user_wallet.path,
|
||||
container,
|
||||
storage_objects,
|
||||
ObjectVerb.HEAD,
|
||||
object_id,
|
||||
self.shell,
|
||||
temp_directory,
|
||||
expiration,
|
||||
self.cluster.default_rpc_endpoint,
|
||||
session=token_expire_at_next_epoch,
|
||||
)
|
||||
|
||||
with allure.step("Object should be available with session token after token creation"):
|
||||
with expect_not_raises():
|
||||
head_object(
|
||||
user_wallet.path,
|
||||
container,
|
||||
object_id,
|
||||
self.shell,
|
||||
self.cluster.default_rpc_endpoint,
|
||||
session=token_expire_at_next_epoch,
|
||||
)
|
||||
|
||||
with allure.step(
|
||||
"Object should be available at last epoch before session token expiration"
|
||||
):
|
||||
self.tick_epoch()
|
||||
with expect_not_raises():
|
||||
head_object(
|
||||
user_wallet.path,
|
||||
container,
|
||||
object_id,
|
||||
self.shell,
|
||||
self.cluster.default_rpc_endpoint,
|
||||
session=token_expire_at_next_epoch,
|
||||
)
|
||||
|
||||
with allure.step("Object should NOT be available after session token expiration epoch"):
|
||||
self.tick_epoch()
|
||||
with pytest.raises(Exception, match=EXPIRED_SESSION_TOKEN):
|
||||
head_object(
|
||||
user_wallet.path,
|
||||
container,
|
||||
object_id,
|
||||
self.shell,
|
||||
self.cluster.default_rpc_endpoint,
|
||||
session=token_expire_at_next_epoch,
|
||||
)
|
||||
|
||||
@allure.title("Validate static session which is valid starting from next epoch")
|
||||
@pytest.mark.static_session
|
||||
def test_static_session_start_at_next(
|
||||
self,
|
||||
owner_wallet: WalletFile,
|
||||
|
@ -560,70 +549,50 @@ class TestObjectStaticSession(ClusterTestBase):
|
|||
object_id = storage_objects[0].oid
|
||||
expiration = Lifetime(epoch + 2, epoch + 1, epoch)
|
||||
|
||||
with allure.step("Create session token"):
|
||||
token_start_at_next_epoch = get_object_signed_token(
|
||||
owner_wallet,
|
||||
user_wallet,
|
||||
token_start_at_next_epoch = get_object_signed_token(
|
||||
owner_wallet,
|
||||
user_wallet,
|
||||
container,
|
||||
storage_objects,
|
||||
ObjectVerb.HEAD,
|
||||
self.shell,
|
||||
temp_directory,
|
||||
expiration,
|
||||
)
|
||||
|
||||
with pytest.raises(Exception, match=MALFORMED_REQUEST):
|
||||
head_object(
|
||||
user_wallet.path,
|
||||
container,
|
||||
storage_objects,
|
||||
ObjectVerb.HEAD,
|
||||
object_id,
|
||||
self.shell,
|
||||
temp_directory,
|
||||
expiration,
|
||||
self.cluster.default_rpc_endpoint,
|
||||
session=token_start_at_next_epoch,
|
||||
)
|
||||
|
||||
with allure.step("Object should NOT be available with session token after token creation"):
|
||||
with pytest.raises(Exception, match=MALFORMED_REQUEST):
|
||||
head_object(
|
||||
user_wallet.path,
|
||||
container,
|
||||
object_id,
|
||||
self.shell,
|
||||
self.cluster.default_rpc_endpoint,
|
||||
session=token_start_at_next_epoch,
|
||||
)
|
||||
self.tick_epoch()
|
||||
head_object(
|
||||
user_wallet.path,
|
||||
container,
|
||||
object_id,
|
||||
self.shell,
|
||||
self.cluster.default_rpc_endpoint,
|
||||
session=token_start_at_next_epoch,
|
||||
)
|
||||
|
||||
with allure.step(
|
||||
"Object should be available with session token starting from token nbf epoch"
|
||||
):
|
||||
self.tick_epoch()
|
||||
with expect_not_raises():
|
||||
head_object(
|
||||
user_wallet.path,
|
||||
container,
|
||||
object_id,
|
||||
self.shell,
|
||||
self.cluster.default_rpc_endpoint,
|
||||
session=token_start_at_next_epoch,
|
||||
)
|
||||
|
||||
with allure.step(
|
||||
"Object should be available at last epoch before session token expiration"
|
||||
):
|
||||
self.tick_epoch()
|
||||
with expect_not_raises():
|
||||
head_object(
|
||||
user_wallet.path,
|
||||
container,
|
||||
object_id,
|
||||
self.shell,
|
||||
self.cluster.default_rpc_endpoint,
|
||||
session=token_start_at_next_epoch,
|
||||
)
|
||||
|
||||
with allure.step("Object should NOT be available after session token expiration epoch"):
|
||||
self.tick_epoch()
|
||||
with pytest.raises(Exception, match=EXPIRED_SESSION_TOKEN):
|
||||
head_object(
|
||||
user_wallet.path,
|
||||
container,
|
||||
object_id,
|
||||
self.shell,
|
||||
self.cluster.default_rpc_endpoint,
|
||||
session=token_start_at_next_epoch,
|
||||
)
|
||||
self.tick_epoch()
|
||||
with pytest.raises(Exception, match=MALFORMED_REQUEST):
|
||||
head_object(
|
||||
user_wallet.path,
|
||||
container,
|
||||
object_id,
|
||||
self.shell,
|
||||
self.cluster.default_rpc_endpoint,
|
||||
session=token_start_at_next_epoch,
|
||||
)
|
||||
|
||||
@allure.title("Validate static session which is already expired")
|
||||
@pytest.mark.static_session
|
||||
def test_static_session_already_expired(
|
||||
self,
|
||||
owner_wallet: WalletFile,
|
||||
|
@ -656,7 +625,7 @@ class TestObjectStaticSession(ClusterTestBase):
|
|||
expiration,
|
||||
)
|
||||
|
||||
with pytest.raises(Exception, match=EXPIRED_SESSION_TOKEN):
|
||||
with pytest.raises(Exception, match=MALFORMED_REQUEST):
|
||||
head_object(
|
||||
user_wallet.path,
|
||||
container,
|
||||
|
@ -717,6 +686,7 @@ class TestObjectStaticSession(ClusterTestBase):
|
|||
)
|
||||
|
||||
@allure.title("Validate static session which is issued in future epoch")
|
||||
@pytest.mark.static_session
|
||||
def test_static_session_invalid_issued_epoch(
|
||||
self,
|
||||
owner_wallet: WalletFile,
|
||||
|
|
|
@ -1,9 +1,8 @@
|
|||
import allure
|
||||
import pytest
|
||||
from frostfs_testlib.resources.common import PUBLIC_ACL
|
||||
from frostfs_testlib.shell import Shell
|
||||
|
||||
from pytest_tests.helpers.acl import (
|
||||
from file_helper import generate_file
|
||||
from neofs_testlib.shell import Shell
|
||||
from python_keywords.acl import (
|
||||
EACLAccess,
|
||||
EACLOperation,
|
||||
EACLRole,
|
||||
|
@ -12,17 +11,18 @@ from pytest_tests.helpers.acl import (
|
|||
set_eacl,
|
||||
wait_for_cache_expired,
|
||||
)
|
||||
from pytest_tests.helpers.container import (
|
||||
from python_keywords.container import (
|
||||
create_container,
|
||||
delete_container,
|
||||
get_container,
|
||||
list_containers,
|
||||
)
|
||||
from pytest_tests.helpers.file_helper import generate_file
|
||||
from pytest_tests.helpers.object_access import can_put_object
|
||||
from pytest_tests.helpers.wallet import WalletFile
|
||||
from pytest_tests.steps.cluster_test_base import ClusterTestBase
|
||||
from pytest_tests.steps.session_token import ContainerVerb, get_container_signed_token
|
||||
from python_keywords.object_access import can_put_object
|
||||
from wallet import WalletFile
|
||||
from wellknown_acl import PUBLIC_ACL
|
||||
|
||||
from steps.cluster_test_base import ClusterTestBase
|
||||
from steps.session_token import ContainerVerb, get_container_signed_token
|
||||
|
||||
|
||||
@pytest.mark.static_session_container
|
||||
|
|
|
@ -7,13 +7,12 @@ from io import StringIO
|
|||
import allure
|
||||
import pytest
|
||||
import yaml
|
||||
from cluster import Cluster, StorageNode
|
||||
from common import WALLET_CONFIG
|
||||
from configobj import ConfigObj
|
||||
from frostfs_testlib.cli import FrostfsCli
|
||||
from neofs_testlib.cli import NeofsCli
|
||||
|
||||
from pytest_tests.helpers.cluster import Cluster, StorageNode
|
||||
from pytest_tests.resources.common import CLI_DEFAULT_TIMEOUT, WALLET_CONFIG
|
||||
|
||||
SHARD_PREFIX = "FROSTFS_STORAGE_SHARD_"
|
||||
SHARD_PREFIX = "NEOFS_STORAGE_SHARD_"
|
||||
BLOBSTOR_PREFIX = "_BLOBSTOR_"
|
||||
|
||||
|
||||
|
@ -135,15 +134,14 @@ class TestControlShard:
|
|||
wallet_password = node.get_wallet_password()
|
||||
control_endpoint = node.get_control_endpoint()
|
||||
|
||||
cli_config = node.host.get_cli_config("frostfs-cli")
|
||||
cli_config = node.host.get_cli_config("neofs-cli")
|
||||
|
||||
cli = FrostfsCli(node.host.get_shell(), cli_config.exec_path, WALLET_CONFIG)
|
||||
cli = NeofsCli(node.host.get_shell(), cli_config.exec_path, WALLET_CONFIG)
|
||||
result = cli.shards.list(
|
||||
endpoint=control_endpoint,
|
||||
wallet=wallet_path,
|
||||
wallet_password=wallet_password,
|
||||
json_mode=True,
|
||||
timeout=CLI_DEFAULT_TIMEOUT,
|
||||
)
|
||||
return [Shard.from_object(shard) for shard in json.loads(result.stdout.split(">", 1)[1])]
|
||||
|
||||
|
|
|
@ -1,47 +0,0 @@
|
|||
import os
|
||||
import shutil
|
||||
from datetime import datetime
|
||||
|
||||
import allure
|
||||
import pytest
|
||||
|
||||
from pytest_tests.steps.cluster_test_base import ClusterTestBase
|
||||
|
||||
|
||||
class TestLogs(ClusterTestBase):
|
||||
@pytest.mark.logs_after_session
|
||||
def test_logs_after_session(self, temp_directory: str, session_start_time: datetime):
|
||||
"""
|
||||
This test automatically added to any test run to check logs from cluster for critical errors.
|
||||
|
||||
"""
|
||||
|
||||
end_time = datetime.utcnow()
|
||||
logs_dir = os.path.join(temp_directory, "logs")
|
||||
os.makedirs(logs_dir)
|
||||
issues_regex = r"\Wpanic\W|\Woom\W|\Wtoo many open files\W"
|
||||
|
||||
hosts_with_problems = []
|
||||
for host in self.cluster.hosts:
|
||||
with allure.step(f"Check logs on {host.config.address}"):
|
||||
if host.is_message_in_logs(issues_regex, session_start_time, end_time):
|
||||
hosts_with_problems.append(host.config.address)
|
||||
host.dump_logs(
|
||||
logs_dir,
|
||||
since=session_start_time,
|
||||
until=end_time,
|
||||
filter_regex=issues_regex,
|
||||
)
|
||||
|
||||
if hosts_with_problems:
|
||||
self._attach_logs(logs_dir)
|
||||
|
||||
assert (
|
||||
not hosts_with_problems
|
||||
), f"The following hosts contains contain critical errors in system logs: {', '.join(hosts_with_problems)}"
|
||||
|
||||
def _attach_logs(self, logs_dir: str) -> None:
|
||||
# Zip all files and attach to Allure because it is more convenient to download a single
|
||||
# zip with all logs rather than mess with individual logs files per service or node
|
||||
logs_zip_file_path = shutil.make_archive(logs_dir, "zip", logs_dir)
|
||||
allure.attach.file(logs_zip_file_path, name="logs.zip", extension="zip")
|
1
pytest_tests/wallet_pass.yml
Normal file
1
pytest_tests/wallet_pass.yml
Normal file
|
@ -0,0 +1 @@
|
|||
password: ""
|
|
@ -1,16 +1,71 @@
|
|||
aiodns==3.0.0
|
||||
aiohttp==3.7.4.post0
|
||||
aioresponses==0.7.2
|
||||
allure-pytest==2.9.45
|
||||
allure-python-commons==2.9.45
|
||||
async-timeout==3.0.1
|
||||
asynctest==0.13.0
|
||||
attrs==21.4.0
|
||||
base58==2.1.0
|
||||
bitarray==2.3.4
|
||||
black==22.8.0
|
||||
boto3==1.16.33
|
||||
botocore==1.19.33
|
||||
certifi==2022.5.18
|
||||
cffi==1.15.0
|
||||
chardet==4.0.0
|
||||
charset-normalizer==2.0.12
|
||||
coverage==6.3.3
|
||||
configobj==5.0.6
|
||||
frostfs-testlib==1.3.1
|
||||
neo-mamba==1.0.0
|
||||
docker==4.4.0
|
||||
docutils==0.17.1
|
||||
Events==0.4
|
||||
flake8==4.0.1
|
||||
idna==3.3
|
||||
iniconfig==1.1.1
|
||||
isort==5.10.1
|
||||
jmespath==0.10.0
|
||||
jsonschema==4.5.1
|
||||
lz4==3.1.3
|
||||
mccabe==0.6.1
|
||||
mmh3==3.0.0
|
||||
multidict==6.0.2
|
||||
mypy==0.950
|
||||
mypy-extensions==0.4.3
|
||||
neo-mamba==0.10.0
|
||||
neo3crypto==0.2.1
|
||||
neo3vm==0.9.0
|
||||
neo3vm-stubs==0.9.0
|
||||
neofs-testlib==0.8.1
|
||||
netaddr==0.8.0
|
||||
orjson==3.6.8
|
||||
packaging==21.3
|
||||
paramiko==2.10.3
|
||||
pexpect==4.8.0
|
||||
pyyaml==6.0
|
||||
pluggy==1.0.0
|
||||
pre-commit==2.20.0
|
||||
ptyprocess==0.7.0
|
||||
py==1.11.0
|
||||
pybiginteger==1.2.6
|
||||
pybiginteger-stubs==1.2.6
|
||||
pycares==4.1.2
|
||||
pycodestyle==2.8.0
|
||||
pycparser==2.21
|
||||
pycryptodome==3.11.0
|
||||
pyflakes==2.4.0
|
||||
pyparsing==3.0.9
|
||||
pyrsistent==0.18.1
|
||||
pytest==7.1.2
|
||||
pytest-lazy-fixture==0.6.3
|
||||
python-dateutil==2.8.2
|
||||
pyyaml==6.0
|
||||
requests==2.28.0
|
||||
robotframework==4.1.2
|
||||
s3transfer==0.3.7
|
||||
six==1.16.0
|
||||
tenacity==8.0.1
|
||||
urllib3==1.26.9
|
||||
tomli==2.0.1
|
||||
typing-extensions==4.2.0
|
||||
urllib3==1.26.9
|
||||
websocket-client==1.3.2
|
||||
yarl==1.7.2
|
||||
|
|
|
@ -1,2 +0,0 @@
|
|||
pre-commit==2.20.0
|
||||
isort==5.12.0
|
|
@ -10,15 +10,14 @@ from typing import Any, Dict, List, Optional, Union
|
|||
|
||||
import allure
|
||||
import base58
|
||||
from frostfs_testlib.cli import FrostfsCli
|
||||
from frostfs_testlib.shell import Shell
|
||||
from frostfs_testlib.utils import wallet_utils
|
||||
|
||||
from pytest_tests.resources.common import ASSETS_DIR, FROSTFS_CLI_EXEC, WALLET_CONFIG
|
||||
from common import ASSETS_DIR, NEOFS_CLI_EXEC, WALLET_CONFIG
|
||||
from data_formatters import get_wallet_public_key
|
||||
from neofs_testlib.cli import NeofsCli
|
||||
from neofs_testlib.shell import Shell
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
EACL_LIFETIME = 100500
|
||||
FROSTFS_CONTRACT_CACHE_TIMEOUT = 30
|
||||
NEOFS_CONTRACT_CACHE_TIMEOUT = 30
|
||||
|
||||
|
||||
class EACLOperation(Enum):
|
||||
|
@ -45,7 +44,7 @@ class EACLRole(Enum):
|
|||
class EACLHeaderType(Enum):
|
||||
REQUEST = "req" # Filter request headers
|
||||
OBJECT = "obj" # Filter object headers
|
||||
SERVICE = "SERVICE" # Filter service headers. These are not processed by FrostFS nodes and exist for service use only
|
||||
SERVICE = "SERVICE" # Filter service headers. These are not processed by NeoFS nodes and exist for service use only
|
||||
|
||||
|
||||
class EACLMatchType(Enum):
|
||||
|
@ -111,14 +110,14 @@ class EACLRule:
|
|||
role = (
|
||||
self.role.value
|
||||
if isinstance(self.role, EACLRole)
|
||||
else f'pubkey:{wallet_utils.get_wallet_public_key(self.role, "")}'
|
||||
else f'pubkey:{get_wallet_public_key(self.role, "")}'
|
||||
)
|
||||
return f'{self.access.value} {self.operation.value} {self.filters or ""} {role}'
|
||||
|
||||
|
||||
@allure.title("Get extended ACL")
|
||||
def get_eacl(wallet_path: str, cid: str, shell: Shell, endpoint: str) -> Optional[str]:
|
||||
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, WALLET_CONFIG)
|
||||
cli = NeofsCli(shell, NEOFS_CLI_EXEC, WALLET_CONFIG)
|
||||
try:
|
||||
result = cli.container.get_eacl(wallet=wallet_path, rpc_endpoint=endpoint, cid=cid)
|
||||
except RuntimeError as exc:
|
||||
|
@ -139,7 +138,7 @@ def set_eacl(
|
|||
endpoint: str,
|
||||
session_token: Optional[str] = None,
|
||||
) -> None:
|
||||
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, WALLET_CONFIG)
|
||||
cli = NeofsCli(shell, NEOFS_CLI_EXEC, WALLET_CONFIG)
|
||||
cli.container.set_eacl(
|
||||
wallet=wallet_path,
|
||||
rpc_endpoint=endpoint,
|
||||
|
@ -157,7 +156,7 @@ def _encode_cid_for_eacl(cid: str) -> str:
|
|||
|
||||
def create_eacl(cid: str, rules_list: List[EACLRule], shell: Shell) -> str:
|
||||
table_file_path = os.path.join(os.getcwd(), ASSETS_DIR, f"eacl_table_{str(uuid.uuid4())}.json")
|
||||
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, WALLET_CONFIG)
|
||||
cli = NeofsCli(shell, NEOFS_CLI_EXEC, WALLET_CONFIG)
|
||||
cli.acl.extended_create(cid=cid, out=table_file_path, rule=rules_list)
|
||||
|
||||
with open(table_file_path, "r") as file:
|
||||
|
@ -245,7 +244,7 @@ def eacl_rules(access: str, verbs: list, user: str) -> list[str]:
|
|||
(list): a list of eACL rules
|
||||
"""
|
||||
if user not in ("others", "user"):
|
||||
pubkey = wallet_utils.get_wallet_public_key(user, wallet_password="")
|
||||
pubkey = get_wallet_public_key(user, wallet_password="")
|
||||
user = f"pubkey:{pubkey}"
|
||||
|
||||
rules = []
|
||||
|
@ -258,17 +257,15 @@ def eacl_rules(access: str, verbs: list, user: str) -> list[str]:
|
|||
def sign_bearer(
|
||||
shell: Shell, wallet_path: str, eacl_rules_file_from: str, eacl_rules_file_to: str, json: bool
|
||||
) -> None:
|
||||
frostfscli = FrostfsCli(
|
||||
shell=shell, frostfs_cli_exec_path=FROSTFS_CLI_EXEC, config_file=WALLET_CONFIG
|
||||
)
|
||||
frostfscli.util.sign_bearer_token(
|
||||
neofscli = NeofsCli(shell=shell, neofs_cli_exec_path=NEOFS_CLI_EXEC, config_file=WALLET_CONFIG)
|
||||
neofscli.util.sign_bearer_token(
|
||||
wallet=wallet_path, from_file=eacl_rules_file_from, to_file=eacl_rules_file_to, json=json
|
||||
)
|
||||
|
||||
|
||||
@allure.title("Wait for eACL cache expired")
|
||||
def wait_for_cache_expired():
|
||||
sleep(FROSTFS_CONTRACT_CACHE_TIMEOUT)
|
||||
sleep(NEOFS_CONTRACT_CACHE_TIMEOUT)
|
||||
return
|
||||
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
#!/usr/bin/python3.10
|
||||
#!/usr/bin/python3.9
|
||||
|
||||
"""
|
||||
Helper functions to use with `frostfs-cli`, `neo-go` and other CLIs.
|
||||
Helper functions to use with `neofs-cli`, `neo-go` and other CLIs.
|
||||
"""
|
||||
import json
|
||||
import logging
|
|
@ -14,31 +14,26 @@ import logging
|
|||
from typing import Optional, Tuple
|
||||
|
||||
import allure
|
||||
from frostfs_testlib.shell import Shell
|
||||
|
||||
from pytest_tests.helpers import frostfs_verbs
|
||||
from pytest_tests.helpers.cluster import Cluster, StorageNode
|
||||
from pytest_tests.helpers.frostfs_verbs import head_object
|
||||
from pytest_tests.helpers.storage_object_info import StorageObjectInfo
|
||||
from pytest_tests.resources.common import CLI_DEFAULT_TIMEOUT, WALLET_CONFIG
|
||||
import neofs_verbs
|
||||
from cluster import Cluster, StorageNode
|
||||
from common import WALLET_CONFIG
|
||||
from neofs_testlib.shell import Shell
|
||||
from neofs_verbs import head_object
|
||||
from storage_object import StorageObjectInfo
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
|
||||
|
||||
def get_storage_object_chunks(
|
||||
storage_object: StorageObjectInfo,
|
||||
shell: Shell,
|
||||
cluster: Cluster,
|
||||
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||
storage_object: StorageObjectInfo, shell: Shell, cluster: Cluster
|
||||
) -> list[str]:
|
||||
"""
|
||||
Get complex object split objects ids (no linker object)
|
||||
|
||||
Args:
|
||||
storage_object: storage_object to get it's chunks
|
||||
shell: client shell to do cmd requests
|
||||
cluster: cluster object under test
|
||||
timeout: Timeout for an operation.
|
||||
storage_object: storage_object to get it's chunks
|
||||
shell: client shell to do cmd requests
|
||||
cluster: cluster object under test
|
||||
|
||||
Returns:
|
||||
list of object ids of complex object chunks
|
||||
|
@ -52,7 +47,6 @@ def get_storage_object_chunks(
|
|||
shell,
|
||||
cluster.storage_nodes,
|
||||
is_direct=False,
|
||||
timeout=timeout,
|
||||
)
|
||||
head = head_object(
|
||||
storage_object.wallet_file_path,
|
||||
|
@ -60,7 +54,6 @@ def get_storage_object_chunks(
|
|||
split_object_id,
|
||||
shell,
|
||||
cluster.default_rpc_endpoint,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
chunks_object_ids = []
|
||||
|
@ -71,10 +64,7 @@ def get_storage_object_chunks(
|
|||
|
||||
|
||||
def get_complex_object_split_ranges(
|
||||
storage_object: StorageObjectInfo,
|
||||
shell: Shell,
|
||||
cluster: Cluster,
|
||||
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||
storage_object: StorageObjectInfo, shell: Shell, cluster: Cluster
|
||||
) -> list[Tuple[int, int]]:
|
||||
|
||||
"""
|
||||
|
@ -84,10 +74,9 @@ def get_complex_object_split_ranges(
|
|||
[(0, 30), (30, 30), (60, 30), (90, 10)]
|
||||
|
||||
Args:
|
||||
storage_object: storage_object to get it's chunks
|
||||
shell: client shell to do cmd requests
|
||||
cluster: cluster object under test
|
||||
timeout: Timeout for an operation.
|
||||
storage_object: storage_object to get it's chunks
|
||||
shell: client shell to do cmd requests
|
||||
cluster: cluster object under test
|
||||
|
||||
Returns:
|
||||
list of object ids of complex object chunks
|
||||
|
@ -103,7 +92,6 @@ def get_complex_object_split_ranges(
|
|||
chunk_id,
|
||||
shell,
|
||||
cluster.default_rpc_endpoint,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
length = int(head["header"]["payloadLength"])
|
||||
|
@ -124,7 +112,6 @@ def get_link_object(
|
|||
bearer: str = "",
|
||||
wallet_config: str = WALLET_CONFIG,
|
||||
is_direct: bool = True,
|
||||
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||
):
|
||||
"""
|
||||
Args:
|
||||
|
@ -135,10 +122,9 @@ def get_link_object(
|
|||
shell: executor for cli command
|
||||
nodes: list of nodes to do search on
|
||||
bearer (optional, str): path to Bearer token file
|
||||
wallet_config (optional, str): path to the frostfs-cli config file
|
||||
wallet_config (optional, str): path to the neofs-cli config file
|
||||
is_direct: send request directly to the node or not; this flag
|
||||
turns into `--ttl 1` key
|
||||
timeout: Timeout for an operation.
|
||||
Returns:
|
||||
(str): Link Object ID
|
||||
When no Link Object ID is found after all Storage Nodes polling,
|
||||
|
@ -147,7 +133,7 @@ def get_link_object(
|
|||
for node in nodes:
|
||||
endpoint = node.get_rpc_endpoint()
|
||||
try:
|
||||
resp = frostfs_verbs.head_object(
|
||||
resp = neofs_verbs.head_object(
|
||||
wallet,
|
||||
cid,
|
||||
oid,
|
||||
|
@ -157,7 +143,6 @@ def get_link_object(
|
|||
is_direct=is_direct,
|
||||
bearer=bearer,
|
||||
wallet_config=wallet_config,
|
||||
timeout=timeout,
|
||||
)
|
||||
if resp["link"]:
|
||||
return resp["link"]
|
||||
|
@ -169,12 +154,7 @@ def get_link_object(
|
|||
|
||||
@allure.step("Get Last Object")
|
||||
def get_last_object(
|
||||
wallet: str,
|
||||
cid: str,
|
||||
oid: str,
|
||||
shell: Shell,
|
||||
nodes: list[StorageNode],
|
||||
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||
wallet: str, cid: str, oid: str, shell: Shell, nodes: list[StorageNode]
|
||||
) -> Optional[str]:
|
||||
"""
|
||||
Args:
|
||||
|
@ -184,7 +164,6 @@ def get_last_object(
|
|||
oid (str): Large Object ID
|
||||
shell: executor for cli command
|
||||
nodes: list of nodes to do search on
|
||||
timeout: Timeout for an operation.
|
||||
Returns:
|
||||
(str): Last Object ID
|
||||
When no Last Object ID is found after all Storage Nodes polling,
|
||||
|
@ -193,15 +172,8 @@ def get_last_object(
|
|||
for node in nodes:
|
||||
endpoint = node.get_rpc_endpoint()
|
||||
try:
|
||||
resp = frostfs_verbs.head_object(
|
||||
wallet,
|
||||
cid,
|
||||
oid,
|
||||
shell=shell,
|
||||
endpoint=endpoint,
|
||||
is_raw=True,
|
||||
is_direct=True,
|
||||
timeout=timeout,
|
||||
resp = neofs_verbs.head_object(
|
||||
wallet, cid, oid, shell=shell, endpoint=endpoint, is_raw=True, is_direct=True
|
||||
)
|
||||
if resp["lastPart"]:
|
||||
return resp["lastPart"]
|
239
robot/resources/lib/python_keywords/container.py
Normal file
239
robot/resources/lib/python_keywords/container.py
Normal file
|
@ -0,0 +1,239 @@
|
|||
#!/usr/bin/python3.9
|
||||
|
||||
"""
|
||||
This module contains keywords that utilize `neofs-cli container` commands.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from time import sleep
|
||||
from typing import Optional, Union
|
||||
|
||||
import allure
|
||||
import json_transformers
|
||||
from common import NEOFS_CLI_EXEC, WALLET_CONFIG
|
||||
from neofs_testlib.cli import NeofsCli
|
||||
from neofs_testlib.shell import Shell
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
|
||||
DEFAULT_PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 4 FROM * AS X"
|
||||
SINGLE_PLACEMENT_RULE = "REP 1 IN X CBF 1 SELECT 4 FROM * AS X"
|
||||
REP_2_FOR_3_NODES_PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 3 FROM * AS X"
|
||||
|
||||
|
||||
@allure.step("Create Container")
|
||||
def create_container(
|
||||
wallet: str,
|
||||
shell: Shell,
|
||||
endpoint: str,
|
||||
rule: str = DEFAULT_PLACEMENT_RULE,
|
||||
basic_acl: str = "",
|
||||
attributes: Optional[dict] = None,
|
||||
session_token: str = "",
|
||||
session_wallet: str = "",
|
||||
name: str = None,
|
||||
options: dict = None,
|
||||
await_mode: bool = True,
|
||||
wait_for_creation: bool = True,
|
||||
) -> str:
|
||||
"""
|
||||
A wrapper for `neofs-cli container create` call.
|
||||
|
||||
Args:
|
||||
wallet (str): a wallet on whose behalf a container is created
|
||||
rule (optional, str): placement rule for container
|
||||
basic_acl (optional, str): an ACL for container, will be
|
||||
appended to `--basic-acl` key
|
||||
attributes (optional, dict): container attributes , will be
|
||||
appended to `--attributes` key
|
||||
session_token (optional, str): a path to session token file
|
||||
session_wallet(optional, str): a path to the wallet which signed
|
||||
the session token; this parameter makes sense
|
||||
when paired with `session_token`
|
||||
shell: executor for cli command
|
||||
endpoint: NeoFS endpoint to send request to, appends to `--rpc-endpoint` key
|
||||
options (optional, dict): any other options to pass to the call
|
||||
name (optional, str): container name attribute
|
||||
await_mode (bool): block execution until container is persisted
|
||||
wait_for_creation (): Wait for container shows in container list
|
||||
|
||||
Returns:
|
||||
(str): CID of the created container
|
||||
"""
|
||||
|
||||
cli = NeofsCli(shell, NEOFS_CLI_EXEC, WALLET_CONFIG)
|
||||
result = cli.container.create(
|
||||
rpc_endpoint=endpoint,
|
||||
wallet=session_wallet if session_wallet else wallet,
|
||||
policy=rule,
|
||||
basic_acl=basic_acl,
|
||||
attributes=attributes,
|
||||
name=name,
|
||||
session=session_token,
|
||||
await_mode=await_mode,
|
||||
**options or {},
|
||||
)
|
||||
|
||||
cid = _parse_cid(result.stdout)
|
||||
|
||||
logger.info("Container created; waiting until it is persisted in the sidechain")
|
||||
|
||||
if wait_for_creation:
|
||||
wait_for_container_creation(wallet, cid, shell, endpoint)
|
||||
|
||||
return cid
|
||||
|
||||
|
||||
def wait_for_container_creation(
|
||||
wallet: str, cid: str, shell: Shell, endpoint: str, attempts: int = 15, sleep_interval: int = 1
|
||||
):
|
||||
for _ in range(attempts):
|
||||
containers = list_containers(wallet, shell, endpoint)
|
||||
if cid in containers:
|
||||
return
|
||||
logger.info(f"There is no {cid} in {containers} yet; sleep {sleep_interval} and continue")
|
||||
sleep(sleep_interval)
|
||||
raise RuntimeError(
|
||||
f"After {attempts * sleep_interval} seconds container {cid} hasn't been persisted; exiting"
|
||||
)
|
||||
|
||||
|
||||
def wait_for_container_deletion(
|
||||
wallet: str, cid: str, shell: Shell, endpoint: str, attempts: int = 30, sleep_interval: int = 1
|
||||
):
|
||||
for _ in range(attempts):
|
||||
try:
|
||||
get_container(wallet, cid, shell=shell, endpoint=endpoint)
|
||||
sleep(sleep_interval)
|
||||
continue
|
||||
except Exception as err:
|
||||
if "container not found" not in str(err):
|
||||
raise AssertionError(f'Expected "container not found" in error, got\n{err}')
|
||||
return
|
||||
raise AssertionError(f"Expected container deleted during {attempts * sleep_interval} sec.")
|
||||
|
||||
|
||||
@allure.step("List Containers")
|
||||
def list_containers(wallet: str, shell: Shell, endpoint: str) -> list[str]:
|
||||
"""
|
||||
A wrapper for `neofs-cli container list` call. It returns all the
|
||||
available containers for the given wallet.
|
||||
Args:
|
||||
wallet (str): a wallet on whose behalf we list the containers
|
||||
shell: executor for cli command
|
||||
endpoint: NeoFS endpoint to send request to, appends to `--rpc-endpoint` key
|
||||
Returns:
|
||||
(list): list of containers
|
||||
"""
|
||||
cli = NeofsCli(shell, NEOFS_CLI_EXEC, WALLET_CONFIG)
|
||||
result = cli.container.list(rpc_endpoint=endpoint, wallet=wallet)
|
||||
logger.info(f"Containers: \n{result}")
|
||||
return result.stdout.split()
|
||||
|
||||
|
||||
@allure.step("Get Container")
|
||||
def get_container(
|
||||
wallet: str,
|
||||
cid: str,
|
||||
shell: Shell,
|
||||
endpoint: str,
|
||||
json_mode: bool = True,
|
||||
) -> Union[dict, str]:
|
||||
"""
|
||||
A wrapper for `neofs-cli container get` call. It extracts container's
|
||||
attributes and rearranges them into a more compact view.
|
||||
Args:
|
||||
wallet (str): path to a wallet on whose behalf we get the container
|
||||
cid (str): ID of the container to get
|
||||
shell: executor for cli command
|
||||
endpoint: NeoFS endpoint to send request to, appends to `--rpc-endpoint` key
|
||||
json_mode (bool): return container in JSON format
|
||||
Returns:
|
||||
(dict, str): dict of container attributes
|
||||
"""
|
||||
|
||||
cli = NeofsCli(shell, NEOFS_CLI_EXEC, WALLET_CONFIG)
|
||||
result = cli.container.get(rpc_endpoint=endpoint, wallet=wallet, cid=cid, json_mode=json_mode)
|
||||
|
||||
if not json_mode:
|
||||
return result.stdout
|
||||
|
||||
container_info = json.loads(result.stdout)
|
||||
attributes = dict()
|
||||
for attr in container_info["attributes"]:
|
||||
attributes[attr["key"]] = attr["value"]
|
||||
container_info["attributes"] = attributes
|
||||
container_info["ownerID"] = json_transformers.json_reencode(container_info["ownerID"]["value"])
|
||||
return container_info
|
||||
|
||||
|
||||
@allure.step("Delete Container")
|
||||
# TODO: make the error message about a non-found container more user-friendly
|
||||
# https://github.com/nspcc-dev/neofs-contract/issues/121
|
||||
def delete_container(
|
||||
wallet: str,
|
||||
cid: str,
|
||||
shell: Shell,
|
||||
endpoint: str,
|
||||
force: bool = False,
|
||||
session_token: Optional[str] = None,
|
||||
await_mode: bool = False,
|
||||
) -> None:
|
||||
"""
|
||||
A wrapper for `neofs-cli container delete` call.
|
||||
Args:
|
||||
wallet (str): path to a wallet on whose behalf we delete the container
|
||||
cid (str): ID of the container to delete
|
||||
shell: executor for cli command
|
||||
endpoint: NeoFS endpoint to send request to, appends to `--rpc-endpoint` key
|
||||
force (bool): do not check whether container contains locks and remove immediately
|
||||
session_token: a path to session token file
|
||||
This function doesn't return anything.
|
||||
"""
|
||||
|
||||
cli = NeofsCli(shell, NEOFS_CLI_EXEC, WALLET_CONFIG)
|
||||
cli.container.delete(
|
||||
wallet=wallet,
|
||||
cid=cid,
|
||||
rpc_endpoint=endpoint,
|
||||
force=force,
|
||||
session=session_token,
|
||||
await_mode=await_mode,
|
||||
)
|
||||
|
||||
|
||||
def _parse_cid(output: str) -> str:
|
||||
"""
|
||||
Parses container ID from a given CLI output. The input string we expect:
|
||||
container ID: 2tz86kVTDpJxWHrhw3h6PbKMwkLtBEwoqhHQCKTre1FN
|
||||
awaiting...
|
||||
container has been persisted on sidechain
|
||||
We want to take 'container ID' value from the string.
|
||||
|
||||
Args:
|
||||
output (str): CLI output to parse
|
||||
|
||||
Returns:
|
||||
(str): extracted CID
|
||||
"""
|
||||
try:
|
||||
# taking first line from command's output
|
||||
first_line = output.split("\n")[0]
|
||||
except Exception:
|
||||
first_line = ""
|
||||
logger.error(f"Got empty output: {output}")
|
||||
splitted = first_line.split(": ")
|
||||
if len(splitted) != 2:
|
||||
raise ValueError(f"no CID was parsed from command output: \t{first_line}")
|
||||
return splitted[1]
|
||||
|
||||
|
||||
@allure.step("Search container by name")
|
||||
def search_container_by_name(wallet: str, name: str, shell: Shell, endpoint: str):
|
||||
list_cids = list_containers(wallet, shell, endpoint)
|
||||
for cid in list_cids:
|
||||
cont_info = get_container(wallet, cid, shell, endpoint, True)
|
||||
if cont_info.get("attributes").get("Name", None) == name:
|
||||
return cid
|
||||
return None
|
|
@ -1,10 +1,9 @@
|
|||
from typing import List, Optional
|
||||
|
||||
from frostfs_testlib.shell import Shell
|
||||
|
||||
from pytest_tests.helpers.acl import EACLOperation
|
||||
from pytest_tests.helpers.cluster import Cluster
|
||||
from pytest_tests.helpers.object_access import (
|
||||
from acl import EACLOperation
|
||||
from cluster import Cluster
|
||||
from neofs_testlib.shell import Shell
|
||||
from python_keywords.object_access import (
|
||||
can_delete_object,
|
||||
can_get_head_object,
|
||||
can_get_object,
|
50
robot/resources/lib/python_keywords/data_formatters.py
Normal file
50
robot/resources/lib/python_keywords/data_formatters.py
Normal file
|
@ -0,0 +1,50 @@
|
|||
import base64
|
||||
import json
|
||||
|
||||
import base58
|
||||
from neo3 import wallet
|
||||
|
||||
|
||||
def dict_to_attrs(attrs: dict) -> str:
|
||||
"""
|
||||
This function takes a dictionary of object's attributes and converts them
|
||||
into string. The string is passed to `--attributes` key of neofs-cli.
|
||||
|
||||
Args:
|
||||
attrs (dict): object attributes in {"a": "b", "c": "d"} format.
|
||||
|
||||
Returns:
|
||||
(str): string in "a=b,c=d" format.
|
||||
"""
|
||||
return ",".join(f"{key}={value}" for key, value in attrs.items())
|
||||
|
||||
|
||||
def __fix_wallet_schema(wallet: dict) -> None:
|
||||
# Temporary function to fix wallets that do not conform to the schema
|
||||
# TODO: get rid of it once issue is solved
|
||||
if "name" not in wallet:
|
||||
wallet["name"] = None
|
||||
for account in wallet["accounts"]:
|
||||
if "extra" not in account:
|
||||
account["extra"] = None
|
||||
|
||||
|
||||
def get_wallet_public_key(wallet_path: str, wallet_password: str, format: str = "hex") -> str:
|
||||
# Get public key from wallet file
|
||||
with open(wallet_path, "r") as file:
|
||||
wallet_content = json.load(file)
|
||||
__fix_wallet_schema(wallet_content)
|
||||
|
||||
wallet_from_json = wallet.Wallet.from_json(wallet_content, password=wallet_password)
|
||||
public_key_hex = str(wallet_from_json.accounts[0].public_key)
|
||||
|
||||
# Convert public key to specified format
|
||||
if format == "hex":
|
||||
return public_key_hex
|
||||
if format == "base58":
|
||||
public_key_base58 = base58.b58encode(bytes.fromhex(public_key_hex))
|
||||
return public_key_base58.decode("utf-8")
|
||||
if format == "base64":
|
||||
public_key_base64 = base64.b64encode(bytes.fromhex(public_key_hex))
|
||||
return public_key_base64.decode("utf-8")
|
||||
raise ValueError(f"Invalid public key format: {format}")
|
|
@ -3,21 +3,20 @@ from time import sleep
|
|||
from typing import Optional
|
||||
|
||||
import allure
|
||||
from frostfs_testlib.cli import FrostfsAdm, FrostfsCli, NeoGo
|
||||
from frostfs_testlib.shell import Shell
|
||||
from frostfs_testlib.utils import datetime_utils, wallet_utils
|
||||
|
||||
from pytest_tests.helpers.cluster import Cluster, StorageNode
|
||||
from pytest_tests.helpers.payment_neogo import get_contract_hash
|
||||
from pytest_tests.helpers.test_control import wait_for_success
|
||||
from pytest_tests.resources.common import (
|
||||
CLI_DEFAULT_TIMEOUT,
|
||||
FROSTFS_ADM_CONFIG_PATH,
|
||||
FROSTFS_ADM_EXEC,
|
||||
FROSTFS_CLI_EXEC,
|
||||
from cluster import Cluster, StorageNode
|
||||
from common import (
|
||||
MAINNET_BLOCK_TIME,
|
||||
NEOFS_ADM_CONFIG_PATH,
|
||||
NEOFS_ADM_EXEC,
|
||||
NEOFS_CLI_EXEC,
|
||||
NEOGO_EXECUTABLE,
|
||||
)
|
||||
from neofs_testlib.cli import NeofsAdm, NeofsCli, NeoGo
|
||||
from neofs_testlib.shell import Shell
|
||||
from neofs_testlib.utils.wallet import get_last_address_from_wallet
|
||||
from payment_neogo import get_contract_hash
|
||||
from test_control import wait_for_success
|
||||
from utility import parse_time
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
|
||||
|
@ -54,16 +53,16 @@ def get_epoch(shell: Shell, cluster: Cluster, alive_node: Optional[StorageNode]
|
|||
wallet_path = alive_node.get_wallet_path()
|
||||
wallet_config = alive_node.get_wallet_config_path()
|
||||
|
||||
cli = FrostfsCli(shell=shell, frostfs_cli_exec_path=FROSTFS_CLI_EXEC, config_file=wallet_config)
|
||||
cli = NeofsCli(shell=shell, neofs_cli_exec_path=NEOFS_CLI_EXEC, config_file=wallet_config)
|
||||
|
||||
epoch = cli.netmap.epoch(endpoint, wallet_path, timeout=CLI_DEFAULT_TIMEOUT)
|
||||
epoch = cli.netmap.epoch(endpoint, wallet_path)
|
||||
return int(epoch.stdout)
|
||||
|
||||
|
||||
@allure.step("Tick Epoch")
|
||||
def tick_epoch(shell: Shell, cluster: Cluster, alive_node: Optional[StorageNode] = None):
|
||||
"""
|
||||
Tick epoch using frostfs-adm or NeoGo if frostfs-adm is not available (DevEnv)
|
||||
Tick epoch using neofs-adm or NeoGo if neofs-adm is not available (DevEnv)
|
||||
Args:
|
||||
shell: local shell to make queries about current epoch. Remote shell will be used to tick new one
|
||||
cluster: cluster instance under test
|
||||
|
@ -73,14 +72,14 @@ def tick_epoch(shell: Shell, cluster: Cluster, alive_node: Optional[StorageNode]
|
|||
alive_node = alive_node if alive_node else cluster.storage_nodes[0]
|
||||
remote_shell = alive_node.host.get_shell()
|
||||
|
||||
if FROSTFS_ADM_EXEC and FROSTFS_ADM_CONFIG_PATH:
|
||||
# If frostfs-adm is available, then we tick epoch with it (to be consistent with UAT tests)
|
||||
frostfsadm = FrostfsAdm(
|
||||
if NEOFS_ADM_EXEC and NEOFS_ADM_CONFIG_PATH:
|
||||
# If neofs-adm is available, then we tick epoch with it (to be consistent with UAT tests)
|
||||
neofsadm = NeofsAdm(
|
||||
shell=remote_shell,
|
||||
frostfs_adm_exec_path=FROSTFS_ADM_EXEC,
|
||||
config_file=FROSTFS_ADM_CONFIG_PATH,
|
||||
neofs_adm_exec_path=NEOFS_ADM_EXEC,
|
||||
config_file=NEOFS_ADM_CONFIG_PATH,
|
||||
)
|
||||
frostfsadm.morph.force_new_epoch()
|
||||
neofsadm.morph.force_new_epoch()
|
||||
return
|
||||
|
||||
# Otherwise we tick epoch using transaction
|
||||
|
@ -91,7 +90,7 @@ def tick_epoch(shell: Shell, cluster: Cluster, alive_node: Optional[StorageNode]
|
|||
# In case if no local_wallet_path is provided, we use wallet_path
|
||||
ir_wallet_path = ir_node.get_wallet_path()
|
||||
ir_wallet_pass = ir_node.get_wallet_password()
|
||||
ir_address = wallet_utils.get_last_address_from_wallet(ir_wallet_path, ir_wallet_pass)
|
||||
ir_address = get_last_address_from_wallet(ir_wallet_path, ir_wallet_pass)
|
||||
|
||||
morph_chain = cluster.morph_chain_nodes[0]
|
||||
morph_endpoint = morph_chain.get_endpoint()
|
||||
|
@ -100,7 +99,7 @@ def tick_epoch(shell: Shell, cluster: Cluster, alive_node: Optional[StorageNode]
|
|||
neogo.contract.invokefunction(
|
||||
wallet=ir_wallet_path,
|
||||
wallet_password=ir_wallet_pass,
|
||||
scripthash=get_contract_hash(morph_chain, "netmap.frostfs", shell=shell),
|
||||
scripthash=get_contract_hash(morph_chain, "netmap.neofs", shell=shell),
|
||||
method="newEpoch",
|
||||
arguments=f"int:{cur_epoch + 1}",
|
||||
multisig_hash=f"{ir_address}:Global",
|
||||
|
@ -109,4 +108,4 @@ def tick_epoch(shell: Shell, cluster: Cluster, alive_node: Optional[StorageNode]
|
|||
force=True,
|
||||
gas=1,
|
||||
)
|
||||
sleep(datetime_utils.parse_time(MAINNET_BLOCK_TIME))
|
||||
sleep(parse_time(MAINNET_BLOCK_TIME))
|
|
@ -2,11 +2,10 @@ import logging
|
|||
from time import sleep
|
||||
|
||||
import allure
|
||||
from frostfs_testlib.shell import Shell
|
||||
|
||||
from pytest_tests.helpers.cluster import Cluster, StorageNode
|
||||
from pytest_tests.helpers.node_management import storage_node_healthcheck
|
||||
from pytest_tests.helpers.storage_policy import get_nodes_with_object
|
||||
from cluster import Cluster, StorageNode
|
||||
from neofs_testlib.shell import Shell
|
||||
from python_keywords.node_management import storage_node_healthcheck
|
||||
from storage_policy import get_nodes_with_object
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
|
|
@ -1,3 +1,4 @@
|
|||
import base64
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
|
@ -10,15 +11,14 @@ from urllib.parse import quote_plus
|
|||
|
||||
import allure
|
||||
import requests
|
||||
from frostfs_testlib.shell import Shell
|
||||
|
||||
from pytest_tests.helpers.aws_cli_client import LONG_TIMEOUT
|
||||
from pytest_tests.helpers.cli_helpers import _cmd_run
|
||||
from pytest_tests.helpers.cluster import StorageNode
|
||||
from pytest_tests.helpers.file_helper import get_file_hash
|
||||
from pytest_tests.helpers.frostfs_verbs import get_object
|
||||
from pytest_tests.helpers.storage_policy import get_nodes_without_object
|
||||
from pytest_tests.resources.common import SIMPLE_OBJECT_SIZE
|
||||
from aws_cli_client import LONG_TIMEOUT
|
||||
from cli_helpers import _cmd_run
|
||||
from cluster import StorageNode
|
||||
from common import SIMPLE_OBJECT_SIZE
|
||||
from file_helper import get_file_hash
|
||||
from neofs_testlib.shell import Shell
|
||||
from python_keywords.neofs_verbs import get_object
|
||||
from python_keywords.storage_policy import get_nodes_without_object
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
|
136
robot/resources/lib/python_keywords/json_transformers.py
Normal file
136
robot/resources/lib/python_keywords/json_transformers.py
Normal file
|
@ -0,0 +1,136 @@
|
|||
"""
|
||||
When doing requests to NeoFS, we get JSON output as an automatically decoded
|
||||
structure from protobuf. Some fields are decoded with boilerplates and binary
|
||||
values are Base64-encoded.
|
||||
|
||||
This module contains functions which rearrange the structure and reencode binary
|
||||
data from Base64 to Base58.
|
||||
"""
|
||||
|
||||
import base64
|
||||
|
||||
import base58
|
||||
|
||||
|
||||
def decode_simple_header(data: dict) -> dict:
|
||||
"""
|
||||
This function reencodes Simple Object header and its attributes.
|
||||
"""
|
||||
try:
|
||||
data = decode_common_fields(data)
|
||||
|
||||
# Normalize object attributes
|
||||
data["header"]["attributes"] = {
|
||||
attr["key"]: attr["value"] for attr in data["header"]["attributes"]
|
||||
}
|
||||
except Exception as exc:
|
||||
raise ValueError(f"failed to decode JSON output: {exc}") from exc
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def decode_split_header(data: dict) -> dict:
|
||||
"""
|
||||
This function rearranges Complex Object header.
|
||||
The header holds SplitID, a random unique
|
||||
number, which is common among all splitted objects, and IDs of the Linking
|
||||
Object and the last splitted Object.
|
||||
"""
|
||||
try:
|
||||
data["splitId"] = json_reencode(data["splitId"])
|
||||
data["lastPart"] = json_reencode(data["lastPart"]["value"]) if data["lastPart"] else None
|
||||
data["link"] = json_reencode(data["link"]["value"]) if data["link"] else None
|
||||
except Exception as exc:
|
||||
raise ValueError(f"failed to decode JSON output: {exc}") from exc
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def decode_linking_object(data: dict) -> dict:
|
||||
"""
|
||||
This function reencodes Linking Object header.
|
||||
It contains IDs of child Objects and Split Chain data.
|
||||
"""
|
||||
try:
|
||||
data = decode_simple_header(data)
|
||||
split = data["header"]["split"]
|
||||
split["children"] = [json_reencode(item["value"]) for item in split["children"]]
|
||||
split["splitID"] = json_reencode(split["splitID"])
|
||||
split["previous"] = json_reencode(split["previous"]["value"]) if split["previous"] else None
|
||||
split["parent"] = json_reencode(split["parent"]["value"]) if split["parent"] else None
|
||||
except Exception as exc:
|
||||
raise ValueError(f"failed to decode JSON output: {exc}") from exc
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def decode_storage_group(data: dict) -> dict:
|
||||
"""
|
||||
This function reencodes Storage Group header.
|
||||
"""
|
||||
try:
|
||||
data = decode_common_fields(data)
|
||||
except Exception as exc:
|
||||
raise ValueError(f"failed to decode JSON output: {exc}") from exc
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def decode_tombstone(data: dict) -> dict:
|
||||
"""
|
||||
This function reencodes Tombstone header.
|
||||
"""
|
||||
try:
|
||||
data = decode_simple_header(data)
|
||||
data["header"]["sessionToken"] = decode_session_token(data["header"]["sessionToken"])
|
||||
except Exception as exc:
|
||||
raise ValueError(f"failed to decode JSON output: {exc}") from exc
|
||||
return data
|
||||
|
||||
|
||||
def decode_session_token(data: dict) -> dict:
|
||||
"""
|
||||
This function reencodes a fragment of header which contains
|
||||
information about session token.
|
||||
"""
|
||||
target = data["body"]["object"]["target"]
|
||||
target["container"] = json_reencode(target["container"]["value"])
|
||||
target["objects"] = [json_reencode(obj["value"]) for obj in target["objects"]]
|
||||
return data
|
||||
|
||||
|
||||
def json_reencode(data: str) -> str:
|
||||
"""
|
||||
According to JSON protocol, binary data (Object/Container/Storage Group IDs, etc)
|
||||
is converted to string via Base58 encoder. But we usually operate with Base64-encoded format.
|
||||
This function reencodes given Base58 string into the Base64 one.
|
||||
"""
|
||||
return base58.b58encode(base64.b64decode(data)).decode("utf-8")
|
||||
|
||||
|
||||
def encode_for_json(data: str) -> str:
|
||||
"""
|
||||
This function encodes binary data for sending them as protobuf
|
||||
structures.
|
||||
"""
|
||||
return base64.b64encode(base58.b58decode(data)).decode("utf-8")
|
||||
|
||||
|
||||
def decode_common_fields(data: dict) -> dict:
|
||||
"""
|
||||
Despite of type (simple/complex Object, Storage Group, etc) every Object
|
||||
header contains several common fields.
|
||||
This function rearranges these fields.
|
||||
"""
|
||||
data["objectID"] = json_reencode(data["objectID"]["value"])
|
||||
|
||||
header = data["header"]
|
||||
header["containerID"] = json_reencode(header["containerID"]["value"])
|
||||
header["ownerID"] = json_reencode(header["ownerID"]["value"])
|
||||
header["payloadHash"] = json_reencode(header["payloadHash"]["sum"])
|
||||
header["version"] = f"{header['version']['major']}{header['version']['minor']}"
|
||||
# Homomorphic hash is optional and its calculation might be disabled in trusted network
|
||||
if header.get("homomorphicHash"):
|
||||
header["homomorphicHash"] = json_reencode(header["homomorphicHash"]["sum"])
|
||||
|
||||
return data
|
|
@ -6,17 +6,11 @@ import uuid
|
|||
from typing import Any, Optional
|
||||
|
||||
import allure
|
||||
from frostfs_testlib.cli import FrostfsCli
|
||||
from frostfs_testlib.shell import Shell
|
||||
from frostfs_testlib.utils import json_utils
|
||||
|
||||
from pytest_tests.helpers.cluster import Cluster
|
||||
from pytest_tests.resources.common import (
|
||||
ASSETS_DIR,
|
||||
CLI_DEFAULT_TIMEOUT,
|
||||
FROSTFS_CLI_EXEC,
|
||||
WALLET_CONFIG,
|
||||
)
|
||||
import json_transformers
|
||||
from cluster import Cluster
|
||||
from common import ASSETS_DIR, NEOFS_CLI_EXEC, WALLET_CONFIG
|
||||
from neofs_testlib.cli import NeofsCli
|
||||
from neofs_testlib.shell import Shell
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
|
||||
|
@ -34,10 +28,9 @@ def get_object_from_random_node(
|
|||
wallet_config: Optional[str] = None,
|
||||
no_progress: bool = True,
|
||||
session: Optional[str] = None,
|
||||
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||
) -> str:
|
||||
"""
|
||||
GET from FrostFS random storage node
|
||||
GET from NeoFS random storage node
|
||||
|
||||
Args:
|
||||
wallet: wallet on whose behalf GET is done
|
||||
|
@ -46,12 +39,11 @@ def get_object_from_random_node(
|
|||
shell: executor for cli command
|
||||
bearer (optional, str): path to Bearer Token file, appends to `--bearer` key
|
||||
write_object (optional, str): path to downloaded file, appends to `--file` key
|
||||
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
|
||||
endpoint: NeoFS endpoint to send request to, appends to `--rpc-endpoint` key
|
||||
wallet_config(optional, str): path to the wallet config
|
||||
no_progress(optional, bool): do not show progress bar
|
||||
xhdr (optional, dict): Request X-Headers in form of Key=Value
|
||||
session (optional, dict): path to a JSON-encoded container session token
|
||||
timeout: Timeout for the operation.
|
||||
Returns:
|
||||
(str): path to downloaded file
|
||||
"""
|
||||
|
@ -68,7 +60,6 @@ def get_object_from_random_node(
|
|||
wallet_config,
|
||||
no_progress,
|
||||
session,
|
||||
timeout,
|
||||
)
|
||||
|
||||
|
||||
|
@ -85,10 +76,9 @@ def get_object(
|
|||
wallet_config: Optional[str] = None,
|
||||
no_progress: bool = True,
|
||||
session: Optional[str] = None,
|
||||
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||
) -> str:
|
||||
"""
|
||||
GET from FrostFS.
|
||||
GET from NeoFS.
|
||||
|
||||
Args:
|
||||
wallet (str): wallet on whose behalf GET is done
|
||||
|
@ -97,12 +87,11 @@ def get_object(
|
|||
shell: executor for cli command
|
||||
bearer: path to Bearer Token file, appends to `--bearer` key
|
||||
write_object: path to downloaded file, appends to `--file` key
|
||||
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
|
||||
endpoint: NeoFS endpoint to send request to, appends to `--rpc-endpoint` key
|
||||
wallet_config(optional, str): path to the wallet config
|
||||
no_progress(optional, bool): do not show progress bar
|
||||
xhdr (optional, dict): Request X-Headers in form of Key=Value
|
||||
session (optional, dict): path to a JSON-encoded container session token
|
||||
timeout: Timeout for the operation.
|
||||
Returns:
|
||||
(str): path to downloaded file
|
||||
"""
|
||||
|
@ -111,7 +100,7 @@ def get_object(
|
|||
write_object = str(uuid.uuid4())
|
||||
file_path = os.path.join(ASSETS_DIR, write_object)
|
||||
|
||||
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or WALLET_CONFIG)
|
||||
cli = NeofsCli(shell, NEOFS_CLI_EXEC, wallet_config or WALLET_CONFIG)
|
||||
cli.object.get(
|
||||
rpc_endpoint=endpoint,
|
||||
wallet=wallet,
|
||||
|
@ -122,7 +111,6 @@ def get_object(
|
|||
no_progress=no_progress,
|
||||
xhdr=xhdr,
|
||||
session=session,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
return file_path
|
||||
|
@ -140,7 +128,6 @@ def get_range_hash(
|
|||
wallet_config: Optional[str] = None,
|
||||
xhdr: Optional[dict] = None,
|
||||
session: Optional[str] = None,
|
||||
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||
):
|
||||
"""
|
||||
GETRANGEHASH of given Object.
|
||||
|
@ -153,15 +140,14 @@ def get_range_hash(
|
|||
bearer: path to Bearer Token file, appends to `--bearer` key
|
||||
range_cut: Range to take hash from in the form offset1:length1,...,
|
||||
value to pass to the `--range` parameter
|
||||
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
|
||||
endpoint: NeoFS endpoint to send request to, appends to `--rpc-endpoint` key
|
||||
wallet_config: path to the wallet config
|
||||
xhdr: Request X-Headers in form of Key=Values
|
||||
session: Filepath to a JSON- or binary-encoded token of the object RANGEHASH session.
|
||||
timeout: Timeout for the operation.
|
||||
Returns:
|
||||
None
|
||||
"""
|
||||
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or WALLET_CONFIG)
|
||||
cli = NeofsCli(shell, NEOFS_CLI_EXEC, wallet_config or WALLET_CONFIG)
|
||||
result = cli.object.hash(
|
||||
rpc_endpoint=endpoint,
|
||||
wallet=wallet,
|
||||
|
@ -171,7 +157,6 @@ def get_range_hash(
|
|||
bearer=bearer,
|
||||
xhdr=xhdr,
|
||||
session=session,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
# cutting off output about range offset and length
|
||||
|
@ -192,7 +177,6 @@ def put_object_to_random_node(
|
|||
expire_at: Optional[int] = None,
|
||||
no_progress: bool = True,
|
||||
session: Optional[str] = None,
|
||||
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||
):
|
||||
"""
|
||||
PUT of given file to a random storage node.
|
||||
|
@ -211,7 +195,6 @@ def put_object_to_random_node(
|
|||
expire_at: Last epoch in the life of the object
|
||||
xhdr: Request X-Headers in form of Key=Value
|
||||
session: path to a JSON-encoded container session token
|
||||
timeout: Timeout for the operation.
|
||||
Returns:
|
||||
ID of uploaded Object
|
||||
"""
|
||||
|
@ -230,7 +213,6 @@ def put_object_to_random_node(
|
|||
expire_at,
|
||||
no_progress,
|
||||
session,
|
||||
timeout,
|
||||
)
|
||||
|
||||
|
||||
|
@ -248,7 +230,6 @@ def put_object(
|
|||
expire_at: Optional[int] = None,
|
||||
no_progress: bool = True,
|
||||
session: Optional[str] = None,
|
||||
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||
):
|
||||
"""
|
||||
PUT of given file.
|
||||
|
@ -260,18 +241,17 @@ def put_object(
|
|||
shell: executor for cli command
|
||||
bearer: path to Bearer Token file, appends to `--bearer` key
|
||||
attributes: User attributes in form of Key1=Value1,Key2=Value2
|
||||
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
|
||||
endpoint: NeoFS endpoint to send request to, appends to `--rpc-endpoint` key
|
||||
wallet_config: path to the wallet config
|
||||
no_progress: do not show progress bar
|
||||
expire_at: Last epoch in the life of the object
|
||||
xhdr: Request X-Headers in form of Key=Value
|
||||
session: path to a JSON-encoded container session token
|
||||
timeout: Timeout for the operation.
|
||||
Returns:
|
||||
(str): ID of uploaded Object
|
||||
"""
|
||||
|
||||
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or WALLET_CONFIG)
|
||||
cli = NeofsCli(shell, NEOFS_CLI_EXEC, wallet_config or WALLET_CONFIG)
|
||||
result = cli.object.put(
|
||||
rpc_endpoint=endpoint,
|
||||
wallet=wallet,
|
||||
|
@ -283,7 +263,6 @@ def put_object(
|
|||
no_progress=no_progress,
|
||||
xhdr=xhdr,
|
||||
session=session,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
# splitting CLI output to lines and taking the penultimate line
|
||||
|
@ -303,7 +282,6 @@ def delete_object(
|
|||
wallet_config: Optional[str] = None,
|
||||
xhdr: Optional[dict] = None,
|
||||
session: Optional[str] = None,
|
||||
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||
):
|
||||
"""
|
||||
DELETE an Object.
|
||||
|
@ -314,16 +292,15 @@ def delete_object(
|
|||
oid: ID of Object we are going to delete
|
||||
shell: executor for cli command
|
||||
bearer: path to Bearer Token file, appends to `--bearer` key
|
||||
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
|
||||
endpoint: NeoFS endpoint to send request to, appends to `--rpc-endpoint` key
|
||||
wallet_config: path to the wallet config
|
||||
xhdr: Request X-Headers in form of Key=Value
|
||||
session: path to a JSON-encoded container session token
|
||||
timeout: Timeout for the operation.
|
||||
Returns:
|
||||
(str): Tombstone ID
|
||||
"""
|
||||
|
||||
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or WALLET_CONFIG)
|
||||
cli = NeofsCli(shell, NEOFS_CLI_EXEC, wallet_config or WALLET_CONFIG)
|
||||
result = cli.object.delete(
|
||||
rpc_endpoint=endpoint,
|
||||
wallet=wallet,
|
||||
|
@ -332,7 +309,6 @@ def delete_object(
|
|||
bearer=bearer,
|
||||
xhdr=xhdr,
|
||||
session=session,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
id_str = result.stdout.split("\n")[1]
|
||||
|
@ -352,7 +328,6 @@ def get_range(
|
|||
bearer: str = "",
|
||||
xhdr: Optional[dict] = None,
|
||||
session: Optional[str] = None,
|
||||
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||
):
|
||||
"""
|
||||
GETRANGE an Object.
|
||||
|
@ -363,18 +338,17 @@ def get_range(
|
|||
oid: ID of Object we are going to request
|
||||
range_cut: range to take data from in the form offset:length
|
||||
shell: executor for cli command
|
||||
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
|
||||
endpoint: NeoFS endpoint to send request to, appends to `--rpc-endpoint` key
|
||||
bearer: path to Bearer Token file, appends to `--bearer` key
|
||||
wallet_config: path to the wallet config
|
||||
xhdr: Request X-Headers in form of Key=Value
|
||||
session: path to a JSON-encoded container session token
|
||||
timeout: Timeout for the operation.
|
||||
Returns:
|
||||
(str, bytes) - path to the file with range content and content of this file as bytes
|
||||
"""
|
||||
range_file_path = os.path.join(ASSETS_DIR, str(uuid.uuid4()))
|
||||
|
||||
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or WALLET_CONFIG)
|
||||
cli = NeofsCli(shell, NEOFS_CLI_EXEC, wallet_config or WALLET_CONFIG)
|
||||
cli.object.range(
|
||||
rpc_endpoint=endpoint,
|
||||
wallet=wallet,
|
||||
|
@ -385,7 +359,6 @@ def get_range(
|
|||
bearer=bearer,
|
||||
xhdr=xhdr,
|
||||
session=session,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
with open(range_file_path, "rb") as file:
|
||||
|
@ -408,7 +381,6 @@ def lock_object(
|
|||
wallet_config: Optional[str] = None,
|
||||
ttl: Optional[int] = None,
|
||||
xhdr: Optional[dict] = None,
|
||||
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||
) -> str:
|
||||
"""
|
||||
Lock object in container.
|
||||
|
@ -421,18 +393,17 @@ def lock_object(
|
|||
lifetime: Lock lifetime.
|
||||
expire_at: Lock expiration epoch.
|
||||
shell: executor for cli command
|
||||
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
|
||||
endpoint: NeoFS endpoint to send request to, appends to `--rpc-endpoint` key
|
||||
session: Path to a JSON-encoded container session token.
|
||||
ttl: TTL value in request meta header (default 2).
|
||||
wallet: WIF (NEP-2) string or path to the wallet or binary key.
|
||||
xhdr: Dict with request X-Headers.
|
||||
timeout: Timeout for the operation.
|
||||
|
||||
Returns:
|
||||
Lock object ID
|
||||
"""
|
||||
|
||||
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or WALLET_CONFIG)
|
||||
cli = NeofsCli(shell, NEOFS_CLI_EXEC, wallet_config or WALLET_CONFIG)
|
||||
result = cli.object.lock(
|
||||
rpc_endpoint=endpoint,
|
||||
lifetime=lifetime,
|
||||
|
@ -445,7 +416,6 @@ def lock_object(
|
|||
xhdr=xhdr,
|
||||
session=session,
|
||||
ttl=ttl,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
# splitting CLI output to lines and taking the penultimate line
|
||||
|
@ -468,7 +438,6 @@ def search_object(
|
|||
session: Optional[str] = None,
|
||||
phy: bool = False,
|
||||
root: bool = False,
|
||||
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||
) -> list:
|
||||
"""
|
||||
SEARCH an Object.
|
||||
|
@ -478,7 +447,7 @@ def search_object(
|
|||
cid: ID of Container where we get the Object from
|
||||
shell: executor for cli command
|
||||
bearer: path to Bearer Token file, appends to `--bearer` key
|
||||
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
|
||||
endpoint: NeoFS endpoint to send request to, appends to `--rpc-endpoint` key
|
||||
filters: key=value pairs to filter Objects
|
||||
expected_objects_list: a list of ObjectIDs to compare found Objects with
|
||||
wallet_config: path to the wallet config
|
||||
|
@ -486,13 +455,12 @@ def search_object(
|
|||
session: path to a JSON-encoded container session token
|
||||
phy: Search physically stored objects.
|
||||
root: Search for user objects.
|
||||
timeout: Timeout for the operation.
|
||||
|
||||
Returns:
|
||||
list of found ObjectIDs
|
||||
"""
|
||||
|
||||
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or WALLET_CONFIG)
|
||||
cli = NeofsCli(shell, NEOFS_CLI_EXEC, wallet_config or WALLET_CONFIG)
|
||||
result = cli.object.search(
|
||||
rpc_endpoint=endpoint,
|
||||
wallet=wallet,
|
||||
|
@ -505,7 +473,6 @@ def search_object(
|
|||
session=session,
|
||||
phy=phy,
|
||||
root=root,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
found_objects = re.findall(r"(\w{43,44})", result.stdout)
|
||||
|
@ -534,7 +501,6 @@ def get_netmap_netinfo(
|
|||
address: Optional[str] = None,
|
||||
ttl: Optional[int] = None,
|
||||
xhdr: Optional[dict] = None,
|
||||
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Get netmap netinfo output from node
|
||||
|
@ -542,7 +508,7 @@ def get_netmap_netinfo(
|
|||
Args:
|
||||
wallet (str): wallet on whose behalf request is done
|
||||
shell: executor for cli command
|
||||
endpoint (optional, str): FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
|
||||
endpoint (optional, str): NeoFS endpoint to send request to, appends to `--rpc-endpoint` key
|
||||
address: Address of wallet account
|
||||
ttl: TTL value in request meta header (default 2)
|
||||
wallet: Path to the wallet or binary key
|
||||
|
@ -552,14 +518,13 @@ def get_netmap_netinfo(
|
|||
(dict): dict of parsed command output
|
||||
"""
|
||||
|
||||
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or WALLET_CONFIG)
|
||||
cli = NeofsCli(shell, NEOFS_CLI_EXEC, wallet_config or WALLET_CONFIG)
|
||||
output = cli.netmap.netinfo(
|
||||
wallet=wallet,
|
||||
rpc_endpoint=endpoint,
|
||||
address=address,
|
||||
ttl=ttl,
|
||||
xhdr=xhdr,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
settings = dict()
|
||||
|
@ -590,7 +555,6 @@ def head_object(
|
|||
is_direct: bool = False,
|
||||
wallet_config: Optional[str] = None,
|
||||
session: Optional[str] = None,
|
||||
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||
):
|
||||
"""
|
||||
HEAD an Object.
|
||||
|
@ -601,7 +565,7 @@ def head_object(
|
|||
oid (str): ObjectID to HEAD
|
||||
shell: executor for cli command
|
||||
bearer (optional, str): path to Bearer Token file, appends to `--bearer` key
|
||||
endpoint(optional, str): FrostFS endpoint to send request to
|
||||
endpoint(optional, str): NeoFS endpoint to send request to
|
||||
json_output(optional, bool): return response in JSON format or not; this flag
|
||||
turns into `--json` key
|
||||
is_raw(optional, bool): send "raw" request or not; this flag
|
||||
|
@ -611,7 +575,6 @@ def head_object(
|
|||
wallet_config(optional, str): path to the wallet config
|
||||
xhdr (optional, dict): Request X-Headers in form of Key=Value
|
||||
session (optional, dict): path to a JSON-encoded container session token
|
||||
timeout: Timeout for the operation.
|
||||
Returns:
|
||||
depending on the `json_output` parameter value, the function returns
|
||||
(dict): HEAD response in JSON format
|
||||
|
@ -619,7 +582,7 @@ def head_object(
|
|||
(str): HEAD response as a plain text
|
||||
"""
|
||||
|
||||
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or WALLET_CONFIG)
|
||||
cli = NeofsCli(shell, NEOFS_CLI_EXEC, wallet_config or WALLET_CONFIG)
|
||||
result = cli.object.head(
|
||||
rpc_endpoint=endpoint,
|
||||
wallet=wallet,
|
||||
|
@ -631,7 +594,6 @@ def head_object(
|
|||
ttl=1 if is_direct else None,
|
||||
xhdr=xhdr,
|
||||
session=session,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
if not json_output:
|
||||
|
@ -651,22 +613,22 @@ def head_object(
|
|||
# If response is Complex Object header, it has `splitId` key
|
||||
if "splitId" in decoded.keys():
|
||||
logger.info("decoding split header")
|
||||
return json_utils.decode_split_header(decoded)
|
||||
return json_transformers.decode_split_header(decoded)
|
||||
|
||||
# If response is Last or Linking Object header,
|
||||
# it has `header` dictionary and non-null `split` dictionary
|
||||
if "split" in decoded["header"].keys():
|
||||
if decoded["header"]["split"]:
|
||||
logger.info("decoding linking object")
|
||||
return json_utils.decode_linking_object(decoded)
|
||||
return json_transformers.decode_linking_object(decoded)
|
||||
|
||||
if decoded["header"]["objectType"] == "STORAGE_GROUP":
|
||||
logger.info("decoding storage group")
|
||||
return json_utils.decode_storage_group(decoded)
|
||||
return json_transformers.decode_storage_group(decoded)
|
||||
|
||||
if decoded["header"]["objectType"] == "TOMBSTONE":
|
||||
logger.info("decoding tombstone")
|
||||
return json_utils.decode_tombstone(decoded)
|
||||
return json_transformers.decode_tombstone(decoded)
|
||||
|
||||
logger.info("decoding simple header")
|
||||
return json_utils.decode_simple_header(decoded)
|
||||
return json_transformers.decode_simple_header(decoded)
|
|
@ -6,13 +6,12 @@ from dataclasses import dataclass
|
|||
from typing import Optional
|
||||
|
||||
import allure
|
||||
from frostfs_testlib.cli import FrostfsCli
|
||||
from frostfs_testlib.shell import Shell
|
||||
from frostfs_testlib.utils import datetime_utils
|
||||
|
||||
from pytest_tests.helpers.cluster import Cluster, StorageNode
|
||||
from pytest_tests.helpers.epoch import tick_epoch
|
||||
from pytest_tests.resources.common import FROSTFS_CLI_EXEC, MORPH_BLOCK_TIME
|
||||
from cluster import Cluster, StorageNode
|
||||
from common import MORPH_BLOCK_TIME, NEOFS_CLI_EXEC
|
||||
from epoch import tick_epoch
|
||||
from neofs_testlib.cli import NeofsCli
|
||||
from neofs_testlib.shell import Shell
|
||||
from utility import parse_time
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
|
||||
|
@ -108,7 +107,7 @@ def get_netmap_snapshot(node: StorageNode, shell: Shell) -> str:
|
|||
storage_wallet_config = node.get_wallet_config_path()
|
||||
storage_wallet_path = node.get_wallet_path()
|
||||
|
||||
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, config_file=storage_wallet_config)
|
||||
cli = NeofsCli(shell, NEOFS_CLI_EXEC, config_file=storage_wallet_config)
|
||||
return cli.netmap.snapshot(
|
||||
rpc_endpoint=node.get_rpc_endpoint(),
|
||||
wallet=storage_wallet_path,
|
||||
|
@ -155,7 +154,7 @@ def drop_object(node: StorageNode, cid: str, oid: str) -> str:
|
|||
def delete_node_data(node: StorageNode) -> None:
|
||||
node.stop_service()
|
||||
node.host.delete_storage_node_data(node.name)
|
||||
time.sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME))
|
||||
time.sleep(parse_time(MORPH_BLOCK_TIME))
|
||||
|
||||
|
||||
@allure.step("Exclude node {node_to_exclude} from network map")
|
||||
|
@ -169,7 +168,7 @@ def exclude_node_from_network_map(
|
|||
|
||||
storage_node_set_status(node_to_exclude, status="offline")
|
||||
|
||||
time.sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME))
|
||||
time.sleep(parse_time(MORPH_BLOCK_TIME))
|
||||
tick_epoch(shell, cluster)
|
||||
|
||||
snapshot = get_netmap_snapshot(node=alive_node, shell=shell)
|
||||
|
@ -188,11 +187,11 @@ def include_node_to_network_map(
|
|||
storage_node_set_status(node_to_include, status="online")
|
||||
|
||||
# Per suggestion of @fyrchik we need to wait for 2 blocks after we set status and after tick epoch.
|
||||
# First sleep can be omitted after https://github.com/nspcc-dev/frostfs-node/issues/1790 complete.
|
||||
# First sleep can be omitted after https://github.com/nspcc-dev/neofs-node/issues/1790 complete.
|
||||
|
||||
time.sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME) * 2)
|
||||
time.sleep(parse_time(MORPH_BLOCK_TIME) * 2)
|
||||
tick_epoch(shell, cluster)
|
||||
time.sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME) * 2)
|
||||
time.sleep(parse_time(MORPH_BLOCK_TIME) * 2)
|
||||
|
||||
check_node_in_map(node_to_include, shell, alive_node)
|
||||
|
||||
|
@ -236,10 +235,10 @@ def _run_control_command(node: StorageNode, command: str) -> None:
|
|||
wallet_config = f'password: "{wallet_password}"'
|
||||
shell.exec(f"echo '{wallet_config}' > {wallet_config_path}")
|
||||
|
||||
cli_config = host.get_cli_config("frostfs-cli")
|
||||
cli_config = host.get_cli_config("neofs-cli")
|
||||
|
||||
# TODO: implement cli.control
|
||||
# cli = frostfsCli(shell, cli_config.exec_path, wallet_config_path)
|
||||
# cli = NeofsCli(shell, cli_config.exec_path, wallet_config_path)
|
||||
result = shell.exec(
|
||||
f"{cli_config.exec_path} {command} --endpoint {control_endpoint} "
|
||||
f"--wallet {wallet_path} --config {wallet_config_path}"
|
|
@ -1,13 +1,11 @@
|
|||
from typing import Optional
|
||||
|
||||
import allure
|
||||
from frostfs_testlib.resources.common import OBJECT_ACCESS_DENIED
|
||||
from frostfs_testlib.shell import Shell
|
||||
from frostfs_testlib.utils import string_utils
|
||||
|
||||
from pytest_tests.helpers.cluster import Cluster
|
||||
from pytest_tests.helpers.file_helper import get_file_hash
|
||||
from pytest_tests.helpers.frostfs_verbs import (
|
||||
from cluster import Cluster
|
||||
from file_helper import get_file_hash
|
||||
from grpc_responses import OBJECT_ACCESS_DENIED, error_matches_status
|
||||
from neofs_testlib.shell import Shell
|
||||
from python_keywords.neofs_verbs import (
|
||||
delete_object,
|
||||
get_object_from_random_node,
|
||||
get_range,
|
||||
|
@ -16,7 +14,6 @@ from pytest_tests.helpers.frostfs_verbs import (
|
|||
put_object_to_random_node,
|
||||
search_object,
|
||||
)
|
||||
from pytest_tests.resources.common import CLI_DEFAULT_TIMEOUT
|
||||
|
||||
OPERATION_ERROR_TYPE = RuntimeError
|
||||
|
||||
|
@ -45,7 +42,7 @@ def can_get_object(
|
|||
cluster=cluster,
|
||||
)
|
||||
except OPERATION_ERROR_TYPE as err:
|
||||
assert string_utils.is_str_match_pattern(
|
||||
assert error_matches_status(
|
||||
err, OBJECT_ACCESS_DENIED
|
||||
), f"Expected {err} to match {OBJECT_ACCESS_DENIED}"
|
||||
return False
|
||||
|
@ -78,7 +75,7 @@ def can_put_object(
|
|||
cluster=cluster,
|
||||
)
|
||||
except OPERATION_ERROR_TYPE as err:
|
||||
assert string_utils.is_str_match_pattern(
|
||||
assert error_matches_status(
|
||||
err, OBJECT_ACCESS_DENIED
|
||||
), f"Expected {err} to match {OBJECT_ACCESS_DENIED}"
|
||||
return False
|
||||
|
@ -108,7 +105,7 @@ def can_delete_object(
|
|||
endpoint=endpoint,
|
||||
)
|
||||
except OPERATION_ERROR_TYPE as err:
|
||||
assert string_utils.is_str_match_pattern(
|
||||
assert error_matches_status(
|
||||
err, OBJECT_ACCESS_DENIED
|
||||
), f"Expected {err} to match {OBJECT_ACCESS_DENIED}"
|
||||
return False
|
||||
|
@ -124,7 +121,6 @@ def can_get_head_object(
|
|||
bearer: Optional[str] = None,
|
||||
wallet_config: Optional[str] = None,
|
||||
xhdr: Optional[dict] = None,
|
||||
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||
) -> bool:
|
||||
with allure.step("Try get head of object"):
|
||||
try:
|
||||
|
@ -137,10 +133,9 @@ def can_get_head_object(
|
|||
xhdr=xhdr,
|
||||
shell=shell,
|
||||
endpoint=endpoint,
|
||||
timeout=timeout,
|
||||
)
|
||||
except OPERATION_ERROR_TYPE as err:
|
||||
assert string_utils.is_str_match_pattern(
|
||||
assert error_matches_status(
|
||||
err, OBJECT_ACCESS_DENIED
|
||||
), f"Expected {err} to match {OBJECT_ACCESS_DENIED}"
|
||||
return False
|
||||
|
@ -156,7 +151,6 @@ def can_get_range_of_object(
|
|||
bearer: Optional[str] = None,
|
||||
wallet_config: Optional[str] = None,
|
||||
xhdr: Optional[dict] = None,
|
||||
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||
) -> bool:
|
||||
with allure.step("Try get range of object"):
|
||||
try:
|
||||
|
@ -170,10 +164,9 @@ def can_get_range_of_object(
|
|||
xhdr=xhdr,
|
||||
shell=shell,
|
||||
endpoint=endpoint,
|
||||
timeout=timeout,
|
||||
)
|
||||
except OPERATION_ERROR_TYPE as err:
|
||||
assert string_utils.is_str_match_pattern(
|
||||
assert error_matches_status(
|
||||
err, OBJECT_ACCESS_DENIED
|
||||
), f"Expected {err} to match {OBJECT_ACCESS_DENIED}"
|
||||
return False
|
||||
|
@ -189,7 +182,6 @@ def can_get_range_hash_of_object(
|
|||
bearer: Optional[str] = None,
|
||||
wallet_config: Optional[str] = None,
|
||||
xhdr: Optional[dict] = None,
|
||||
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||
) -> bool:
|
||||
with allure.step("Try get range hash of object"):
|
||||
try:
|
||||
|
@ -203,10 +195,9 @@ def can_get_range_hash_of_object(
|
|||
xhdr=xhdr,
|
||||
shell=shell,
|
||||
endpoint=endpoint,
|
||||
timeout=timeout,
|
||||
)
|
||||
except OPERATION_ERROR_TYPE as err:
|
||||
assert string_utils.is_str_match_pattern(
|
||||
assert error_matches_status(
|
||||
err, OBJECT_ACCESS_DENIED
|
||||
), f"Expected {err} to match {OBJECT_ACCESS_DENIED}"
|
||||
return False
|
||||
|
@ -222,7 +213,6 @@ def can_search_object(
|
|||
bearer: Optional[str] = None,
|
||||
wallet_config: Optional[str] = None,
|
||||
xhdr: Optional[dict] = None,
|
||||
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||
) -> bool:
|
||||
with allure.step("Try search object in container"):
|
||||
try:
|
||||
|
@ -234,10 +224,9 @@ def can_search_object(
|
|||
xhdr=xhdr,
|
||||
shell=shell,
|
||||
endpoint=endpoint,
|
||||
timeout=timeout,
|
||||
)
|
||||
except OPERATION_ERROR_TYPE as err:
|
||||
assert string_utils.is_str_match_pattern(
|
||||
assert error_matches_status(
|
||||
err, OBJECT_ACCESS_DENIED
|
||||
), f"Expected {err} to match {OBJECT_ACCESS_DENIED}"
|
||||
return False
|
|
@ -6,19 +6,14 @@ import time
|
|||
from typing import Optional
|
||||
|
||||
import allure
|
||||
from frostfs_testlib.cli import NeoGo
|
||||
from frostfs_testlib.shell import Shell
|
||||
from frostfs_testlib.utils import converting_utils, datetime_utils, wallet_utils
|
||||
from neo3.wallet import utils as neo3_utils
|
||||
from neo3.wallet import wallet as neo3_wallet
|
||||
|
||||
from pytest_tests.helpers.cluster import MainChain, MorphChain
|
||||
from pytest_tests.resources.common import (
|
||||
FROSTFS_CONTRACT,
|
||||
GAS_HASH,
|
||||
MAINNET_BLOCK_TIME,
|
||||
NEOGO_EXECUTABLE,
|
||||
)
|
||||
from cluster import MainChain, MorphChain
|
||||
from common import GAS_HASH, MAINNET_BLOCK_TIME, NEOFS_CONTRACT, NEOGO_EXECUTABLE
|
||||
from neo3 import wallet as neo3_wallet
|
||||
from neofs_testlib.cli import NeoGo
|
||||
from neofs_testlib.shell import Shell
|
||||
from neofs_testlib.utils.converters import contract_hash_to_address
|
||||
from neofs_testlib.utils.wallet import get_last_address_from_wallet
|
||||
from utility import parse_time
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
|
||||
|
@ -47,15 +42,15 @@ def get_contract_hash(morph_chain: MorphChain, resolve_name: str, shell: Shell)
|
|||
|
||||
@allure.step("Withdraw Mainnet Gas")
|
||||
def withdraw_mainnet_gas(shell: Shell, main_chain: MainChain, wlt: str, amount: int):
|
||||
address = wallet_utils.get_last_address_from_wallet(wlt, EMPTY_PASSWORD)
|
||||
scripthash = neo3_utils.address_to_script_hash(address)
|
||||
address = get_last_address_from_wallet(wlt, EMPTY_PASSWORD)
|
||||
scripthash = neo3_wallet.Account.address_to_script_hash(address)
|
||||
|
||||
neogo = NeoGo(shell=shell, neo_go_exec_path=NEOGO_EXECUTABLE)
|
||||
out = neogo.contract.invokefunction(
|
||||
wallet=wlt,
|
||||
address=address,
|
||||
rpc_endpoint=main_chain.get_endpoint(),
|
||||
scripthash=FROSTFS_CONTRACT,
|
||||
scripthash=NEOFS_CONTRACT,
|
||||
method="withdraw",
|
||||
arguments=f"{scripthash} int:{amount}",
|
||||
multisig_hash=f"{scripthash}:Global",
|
||||
|
@ -92,10 +87,10 @@ def transaction_accepted(main_chain: MainChain, tx_id: str):
|
|||
return False
|
||||
|
||||
|
||||
@allure.step("Get FrostFS Balance")
|
||||
@allure.step("Get NeoFS Balance")
|
||||
def get_balance(shell: Shell, morph_chain: MorphChain, wallet_path: str, wallet_password: str = ""):
|
||||
"""
|
||||
This function returns FrostFS balance for given wallet.
|
||||
This function returns NeoFS balance for given wallet.
|
||||
"""
|
||||
with open(wallet_path) as wallet_file:
|
||||
wallet = neo3_wallet.Wallet.from_json(json.load(wallet_file), password=wallet_password)
|
||||
|
@ -103,7 +98,7 @@ def get_balance(shell: Shell, morph_chain: MorphChain, wallet_path: str, wallet_
|
|||
payload = [{"type": "Hash160", "value": str(acc.script_hash)}]
|
||||
try:
|
||||
resp = morph_chain.rpc_client.invoke_function(
|
||||
get_contract_hash(morph_chain, "balance.frostfs", shell=shell), "balanceOf", payload
|
||||
get_contract_hash(morph_chain, "balance.neofs", shell=shell), "balanceOf", payload
|
||||
)
|
||||
logger.info(f"Got response \n{resp}")
|
||||
value = int(resp["stack"][0]["value"])
|
||||
|
@ -146,12 +141,10 @@ def transfer_gas(
|
|||
if wallet_from_password is not None
|
||||
else main_chain.get_wallet_password()
|
||||
)
|
||||
address_from = address_from or wallet_utils.get_last_address_from_wallet(
|
||||
address_from = address_from or get_last_address_from_wallet(
|
||||
wallet_from_path, wallet_from_password
|
||||
)
|
||||
address_to = address_to or wallet_utils.get_last_address_from_wallet(
|
||||
wallet_to_path, wallet_to_password
|
||||
)
|
||||
address_to = address_to or get_last_address_from_wallet(wallet_to_path, wallet_to_password)
|
||||
|
||||
neogo = NeoGo(shell, neo_go_exec_path=NEOGO_EXECUTABLE)
|
||||
out = neogo.nep17.transfer(
|
||||
|
@ -169,10 +162,10 @@ def transfer_gas(
|
|||
raise Exception("Got no TXID after run the command")
|
||||
if not transaction_accepted(main_chain, txid):
|
||||
raise AssertionError(f"TX {txid} hasn't been processed")
|
||||
time.sleep(datetime_utils.parse_time(MAINNET_BLOCK_TIME))
|
||||
time.sleep(parse_time(MAINNET_BLOCK_TIME))
|
||||
|
||||
|
||||
@allure.step("FrostFS Deposit")
|
||||
@allure.step("NeoFS Deposit")
|
||||
def deposit_gas(
|
||||
shell: Shell,
|
||||
main_chain: MainChain,
|
||||
|
@ -181,12 +174,12 @@ def deposit_gas(
|
|||
wallet_from_password: str,
|
||||
):
|
||||
"""
|
||||
Transferring GAS from given wallet to FrostFS contract address.
|
||||
Transferring GAS from given wallet to NeoFS contract address.
|
||||
"""
|
||||
# get FrostFS contract address
|
||||
deposit_addr = converting_utils.contract_hash_to_address(FROSTFS_CONTRACT)
|
||||
logger.info(f"FrostFS contract address: {deposit_addr}")
|
||||
address_from = wallet_utils.get_last_address_from_wallet(
|
||||
# get NeoFS contract address
|
||||
deposit_addr = contract_hash_to_address(NEOFS_CONTRACT)
|
||||
logger.info(f"NeoFS contract address: {deposit_addr}")
|
||||
address_from = get_last_address_from_wallet(
|
||||
wallet_path=wallet_from_path, wallet_password=wallet_from_password
|
||||
)
|
||||
transfer_gas(
|
|
@ -1,18 +1,17 @@
|
|||
"""
|
||||
This module contains keywords for work with Storage Groups.
|
||||
It contains wrappers for `frostfs-cli storagegroup` verbs.
|
||||
It contains wrappers for `neofs-cli storagegroup` verbs.
|
||||
"""
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
import allure
|
||||
from frostfs_testlib.cli import FrostfsCli
|
||||
from frostfs_testlib.shell import Shell
|
||||
|
||||
from pytest_tests.helpers.cluster import Cluster
|
||||
from pytest_tests.helpers.complex_object_actions import get_link_object
|
||||
from pytest_tests.helpers.frostfs_verbs import head_object
|
||||
from pytest_tests.resources.common import CLI_DEFAULT_TIMEOUT, FROSTFS_CLI_EXEC, WALLET_CONFIG
|
||||
from cluster import Cluster
|
||||
from common import NEOFS_CLI_EXEC, WALLET_CONFIG
|
||||
from complex_object_actions import get_link_object
|
||||
from neofs_testlib.cli import NeofsCli
|
||||
from neofs_testlib.shell import Shell
|
||||
from neofs_verbs import head_object
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
|
||||
|
@ -27,11 +26,10 @@ def put_storagegroup(
|
|||
bearer: Optional[str] = None,
|
||||
wallet_config: str = WALLET_CONFIG,
|
||||
lifetime: int = 10,
|
||||
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||
) -> str:
|
||||
"""
|
||||
Wrapper for `frostfs-cli storagegroup put`. Before the SG is created,
|
||||
frostfs-cli performs HEAD on `objects`, so this verb must be allowed
|
||||
Wrapper for `neofs-cli storagegroup put`. Before the SG is created,
|
||||
neofs-cli performs HEAD on `objects`, so this verb must be allowed
|
||||
for `wallet` in `cid`.
|
||||
Args:
|
||||
shell: Shell instance.
|
||||
|
@ -40,15 +38,12 @@ def put_storagegroup(
|
|||
lifetime: Storage group lifetime in epochs.
|
||||
objects: List of Object IDs to include into the SG.
|
||||
bearer: Path to Bearer token file.
|
||||
wallet_config: Path to frostfs-cli config file.
|
||||
timeout: Timeout for an operation.
|
||||
wallet_config: Path to neofs-cli config file.
|
||||
Returns:
|
||||
Object ID of created Storage Group.
|
||||
"""
|
||||
frostfscli = FrostfsCli(
|
||||
shell=shell, frostfs_cli_exec_path=FROSTFS_CLI_EXEC, config_file=wallet_config
|
||||
)
|
||||
result = frostfscli.storagegroup.put(
|
||||
neofscli = NeofsCli(shell=shell, neofs_cli_exec_path=NEOFS_CLI_EXEC, config_file=wallet_config)
|
||||
result = neofscli.storagegroup.put(
|
||||
wallet=wallet,
|
||||
cid=cid,
|
||||
lifetime=lifetime,
|
||||
|
@ -68,30 +63,25 @@ def list_storagegroup(
|
|||
cid: str,
|
||||
bearer: Optional[str] = None,
|
||||
wallet_config: str = WALLET_CONFIG,
|
||||
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||
) -> list:
|
||||
"""
|
||||
Wrapper for `frostfs-cli storagegroup list`. This operation
|
||||
Wrapper for `neofs-cli storagegroup list`. This operation
|
||||
requires SEARCH allowed for `wallet` in `cid`.
|
||||
Args:
|
||||
shell: Shell instance.
|
||||
wallet: Path to wallet on whose behalf the SGs are listed in the container
|
||||
cid: ID of Container to list.
|
||||
bearer: Path to Bearer token file.
|
||||
wallet_config: Path to frostfs-cli config file.
|
||||
timeout: Timeout for an operation.
|
||||
wallet_config: Path to neofs-cli config file.
|
||||
Returns:
|
||||
Object IDs of found Storage Groups.
|
||||
"""
|
||||
frostfscli = FrostfsCli(
|
||||
shell=shell, frostfs_cli_exec_path=FROSTFS_CLI_EXEC, config_file=wallet_config
|
||||
)
|
||||
result = frostfscli.storagegroup.list(
|
||||
neofscli = NeofsCli(shell=shell, neofs_cli_exec_path=NEOFS_CLI_EXEC, config_file=wallet_config)
|
||||
result = neofscli.storagegroup.list(
|
||||
wallet=wallet,
|
||||
cid=cid,
|
||||
bearer=bearer,
|
||||
rpc_endpoint=endpoint,
|
||||
timeout=timeout,
|
||||
)
|
||||
# throwing off the first string of output
|
||||
found_objects = result.stdout.split("\n")[1:]
|
||||
|
@ -107,35 +97,30 @@ def get_storagegroup(
|
|||
gid: str,
|
||||
bearer: str = "",
|
||||
wallet_config: str = WALLET_CONFIG,
|
||||
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||
) -> dict:
|
||||
"""
|
||||
Wrapper for `frostfs-cli storagegroup get`.
|
||||
Wrapper for `neofs-cli storagegroup get`.
|
||||
Args:
|
||||
shell: Shell instance.
|
||||
wallet: Path to wallet on whose behalf the SG is got.
|
||||
cid: ID of Container where SG is stored.
|
||||
gid: ID of the Storage Group.
|
||||
bearer: Path to Bearer token file.
|
||||
wallet_config: Path to frostfs-cli config file.
|
||||
timeout: Timeout for an operation.
|
||||
wallet_config: Path to neofs-cli config file.
|
||||
Returns:
|
||||
Detailed information on the Storage Group.
|
||||
"""
|
||||
frostfscli = FrostfsCli(
|
||||
shell=shell, frostfs_cli_exec_path=FROSTFS_CLI_EXEC, config_file=wallet_config
|
||||
)
|
||||
result = frostfscli.storagegroup.get(
|
||||
neofscli = NeofsCli(shell=shell, neofs_cli_exec_path=NEOFS_CLI_EXEC, config_file=wallet_config)
|
||||
result = neofscli.storagegroup.get(
|
||||
wallet=wallet,
|
||||
cid=cid,
|
||||
bearer=bearer,
|
||||
id=gid,
|
||||
rpc_endpoint=endpoint,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
# TODO: temporary solution for parsing output. Needs to be replaced with
|
||||
# JSON parsing when https://github.com/nspcc-dev/frostfs-node/issues/1355
|
||||
# JSON parsing when https://github.com/nspcc-dev/neofs-node/issues/1355
|
||||
# is done.
|
||||
strings = result.stdout.strip().split("\n")
|
||||
# first three strings go to `data`;
|
||||
|
@ -161,31 +146,26 @@ def delete_storagegroup(
|
|||
gid: str,
|
||||
bearer: str = "",
|
||||
wallet_config: str = WALLET_CONFIG,
|
||||
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||
) -> str:
|
||||
"""
|
||||
Wrapper for `frostfs-cli storagegroup delete`.
|
||||
Wrapper for `neofs-cli storagegroup delete`.
|
||||
Args:
|
||||
shell: Shell instance.
|
||||
wallet: Path to wallet on whose behalf the SG is deleted.
|
||||
cid: ID of Container where SG is stored.
|
||||
gid: ID of the Storage Group.
|
||||
bearer: Path to Bearer token file.
|
||||
wallet_config: Path to frostfs-cli config file.
|
||||
timeout: Timeout for an operation.
|
||||
wallet_config: Path to neofs-cli config file.
|
||||
Returns:
|
||||
Tombstone ID of the deleted Storage Group.
|
||||
"""
|
||||
frostfscli = FrostfsCli(
|
||||
shell=shell, frostfs_cli_exec_path=FROSTFS_CLI_EXEC, config_file=wallet_config
|
||||
)
|
||||
result = frostfscli.storagegroup.delete(
|
||||
neofscli = NeofsCli(shell=shell, neofs_cli_exec_path=NEOFS_CLI_EXEC, config_file=wallet_config)
|
||||
result = neofscli.storagegroup.delete(
|
||||
wallet=wallet,
|
||||
cid=cid,
|
||||
bearer=bearer,
|
||||
id=gid,
|
||||
rpc_endpoint=endpoint,
|
||||
timeout=timeout,
|
||||
)
|
||||
tombstone_id = result.stdout.strip().split("\n")[1].split(": ")[1]
|
||||
return tombstone_id
|
||||
|
@ -200,7 +180,6 @@ def verify_list_storage_group(
|
|||
gid: str,
|
||||
bearer: str = None,
|
||||
wallet_config: str = WALLET_CONFIG,
|
||||
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||
):
|
||||
storage_groups = list_storagegroup(
|
||||
shell=shell,
|
||||
|
@ -209,7 +188,6 @@ def verify_list_storage_group(
|
|||
cid=cid,
|
||||
bearer=bearer,
|
||||
wallet_config=wallet_config,
|
||||
timeout=timeout,
|
||||
)
|
||||
assert gid in storage_groups
|
||||
|
||||
|
@ -226,7 +204,6 @@ def verify_get_storage_group(
|
|||
max_object_size: int,
|
||||
bearer: str = None,
|
||||
wallet_config: str = WALLET_CONFIG,
|
||||
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||
):
|
||||
obj_parts = []
|
||||
endpoint = cluster.default_rpc_endpoint
|
||||
|
@ -240,7 +217,6 @@ def verify_get_storage_group(
|
|||
nodes=cluster.storage_nodes,
|
||||
bearer=bearer,
|
||||
wallet_config=wallet_config,
|
||||
timeout=timeout,
|
||||
)
|
||||
obj_head = head_object(
|
||||
wallet=wallet,
|
||||
|
@ -251,7 +227,6 @@ def verify_get_storage_group(
|
|||
is_raw=True,
|
||||
bearer=bearer,
|
||||
wallet_config=wallet_config,
|
||||
timeout=timeout,
|
||||
)
|
||||
obj_parts = obj_head["header"]["split"]["children"]
|
||||
|
||||
|
@ -264,7 +239,6 @@ def verify_get_storage_group(
|
|||
gid=gid,
|
||||
bearer=bearer,
|
||||
wallet_config=wallet_config,
|
||||
timeout=timeout,
|
||||
)
|
||||
exp_size = object_size * obj_num
|
||||
if object_size < max_object_size:
|
|
@ -6,14 +6,14 @@
|
|||
"""
|
||||
|
||||
import logging
|
||||
from typing import List
|
||||
|
||||
import allure
|
||||
from frostfs_testlib.resources.common import OBJECT_NOT_FOUND
|
||||
from frostfs_testlib.shell import Shell
|
||||
from frostfs_testlib.utils import string_utils
|
||||
|
||||
from pytest_tests.helpers import complex_object_actions, frostfs_verbs
|
||||
from pytest_tests.helpers.cluster import StorageNode
|
||||
import complex_object_actions
|
||||
import neofs_verbs
|
||||
from cluster import StorageNode
|
||||
from grpc_responses import OBJECT_NOT_FOUND, error_matches_status
|
||||
from neofs_testlib.shell import Shell
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
|
||||
|
@ -66,7 +66,7 @@ def get_simple_object_copies(
|
|||
copies = 0
|
||||
for node in nodes:
|
||||
try:
|
||||
response = frostfs_verbs.head_object(
|
||||
response = neofs_verbs.head_object(
|
||||
wallet, cid, oid, shell=shell, endpoint=node.get_rpc_endpoint(), is_direct=True
|
||||
)
|
||||
if response:
|
||||
|
@ -123,7 +123,7 @@ def get_nodes_with_object(
|
|||
wallet = node.get_wallet_path()
|
||||
wallet_config = node.get_wallet_config_path()
|
||||
try:
|
||||
res = frostfs_verbs.head_object(
|
||||
res = neofs_verbs.head_object(
|
||||
wallet,
|
||||
cid,
|
||||
oid,
|
||||
|
@ -160,13 +160,13 @@ def get_nodes_without_object(
|
|||
nodes_list = []
|
||||
for node in nodes:
|
||||
try:
|
||||
res = frostfs_verbs.head_object(
|
||||
res = neofs_verbs.head_object(
|
||||
wallet, cid, oid, shell=shell, endpoint=node.get_rpc_endpoint(), is_direct=True
|
||||
)
|
||||
if res is None:
|
||||
nodes_list.append(node)
|
||||
except Exception as err:
|
||||
if string_utils.is_str_match_pattern(err, OBJECT_NOT_FOUND):
|
||||
if error_matches_status(err, OBJECT_NOT_FOUND):
|
||||
nodes_list.append(node)
|
||||
else:
|
||||
raise Exception(f"Got error {err} on head object command") from err
|
|
@ -2,10 +2,9 @@ import json
|
|||
import logging
|
||||
|
||||
import allure
|
||||
from frostfs_testlib.shell import Shell
|
||||
from neo3.wallet import wallet
|
||||
|
||||
from pytest_tests.helpers.frostfs_verbs import head_object
|
||||
from neo3 import wallet
|
||||
from neofs_testlib.shell import Shell
|
||||
from neofs_verbs import head_object
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
|
|
@ -11,7 +11,7 @@ COMPLEX_OBJECT_TAIL_SIZE = os.getenv("COMPLEX_OBJECT_TAIL_SIZE", "1000")
|
|||
MAINNET_BLOCK_TIME = os.getenv("MAINNET_BLOCK_TIME", "1s")
|
||||
MAINNET_TIMEOUT = os.getenv("MAINNET_TIMEOUT", "1min")
|
||||
MORPH_BLOCK_TIME = os.getenv("MORPH_BLOCK_TIME", "1s")
|
||||
FROSTFS_CONTRACT_CACHE_TIMEOUT = os.getenv("FROSTFS_CONTRACT_CACHE_TIMEOUT", "30s")
|
||||
NEOFS_CONTRACT_CACHE_TIMEOUT = os.getenv("NEOFS_CONTRACT_CACHE_TIMEOUT", "30s")
|
||||
|
||||
# Time interval that allows a GC pass on storage node (this includes GC sleep interval
|
||||
# of 1min plus 15 seconds for GC pass itself)
|
||||
|
@ -19,10 +19,10 @@ STORAGE_GC_TIME = os.getenv("STORAGE_GC_TIME", "75s")
|
|||
|
||||
GAS_HASH = os.getenv("GAS_HASH", "0xd2a4cff31913016155e38e474a2c06d08be276cf")
|
||||
|
||||
FROSTFS_CONTRACT = os.getenv("FROSTFS_IR_CONTRACTS_FROSTFS")
|
||||
NEOFS_CONTRACT = os.getenv("NEOFS_IR_CONTRACTS_NEOFS")
|
||||
|
||||
ASSETS_DIR = os.getenv("ASSETS_DIR", "TemporaryDir")
|
||||
DEVENV_PATH = os.getenv("DEVENV_PATH", os.path.join("..", "frostfs-dev-env"))
|
||||
DEVENV_PATH = os.getenv("DEVENV_PATH", os.path.join("..", "neofs-dev-env"))
|
||||
|
||||
# Password of wallet owned by user on behalf of whom we are running tests
|
||||
WALLET_PASS = os.getenv("WALLET_PASS", "")
|
||||
|
@ -30,12 +30,12 @@ WALLET_PASS = os.getenv("WALLET_PASS", "")
|
|||
|
||||
# Paths to CLI executables on machine that runs tests
|
||||
NEOGO_EXECUTABLE = os.getenv("NEOGO_EXECUTABLE", "neo-go")
|
||||
FROSTFS_CLI_EXEC = os.getenv("FROSTFS_CLI_EXEC", "frostfs-cli")
|
||||
FROSTFS_AUTHMATE_EXEC = os.getenv("FROSTFS_AUTHMATE_EXEC", "frostfs-authmate")
|
||||
FROSTFS_ADM_EXEC = os.getenv("FROSTFS_ADM_EXEC", "frostfs-adm")
|
||||
NEOFS_CLI_EXEC = os.getenv("NEOFS_CLI_EXEC", "neofs-cli")
|
||||
NEOFS_AUTHMATE_EXEC = os.getenv("NEOFS_AUTHMATE_EXEC", "neofs-authmate")
|
||||
NEOFS_ADM_EXEC = os.getenv("NEOFS_ADM_EXEC", "neofs-adm")
|
||||
|
||||
# Config for frostfs-adm utility. Optional if tests are running against devenv
|
||||
FROSTFS_ADM_CONFIG_PATH = os.getenv("FROSTFS_ADM_CONFIG_PATH")
|
||||
# Config for neofs-adm utility. Optional if tests are running against devenv
|
||||
NEOFS_ADM_CONFIG_PATH = os.getenv("NEOFS_ADM_CONFIG_PATH")
|
||||
|
||||
FREE_STORAGE = os.getenv("FREE_STORAGE", "false").lower() == "true"
|
||||
BIN_VERSIONS_FILE = os.getenv("BIN_VERSIONS_FILE")
|
||||
|
@ -45,8 +45,6 @@ STORAGE_NODE_SERVICE_NAME_REGEX = r"s\d\d"
|
|||
HTTP_GATE_SERVICE_NAME_REGEX = r"http-gate\d\d"
|
||||
S3_GATE_SERVICE_NAME_REGEX = r"s3-gate\d\d"
|
||||
|
||||
CLI_DEFAULT_TIMEOUT = os.getenv("CLI_DEFAULT_TIMEOUT", None)
|
||||
|
||||
# Generate wallet configs
|
||||
# TODO: we should move all info about wallet configs to fixtures
|
||||
WALLET_CONFIG = os.path.join(os.getcwd(), "wallet_config.yml")
|
23
robot/variables/eacl_object_filters.py
Normal file
23
robot/variables/eacl_object_filters.py
Normal file
|
@ -0,0 +1,23 @@
|
|||
EACL_OBJ_FILTERS = {
|
||||
"$Object:objectID": "objectID",
|
||||
"$Object:containerID": "containerID",
|
||||
"$Object:ownerID": "ownerID",
|
||||
"$Object:creationEpoch": "creationEpoch",
|
||||
"$Object:payloadLength": "payloadLength",
|
||||
"$Object:payloadHash": "payloadHash",
|
||||
"$Object:objectType": "objectType",
|
||||
"$Object:homomorphicHash": "homomorphicHash",
|
||||
"$Object:version": "version",
|
||||
}
|
||||
|
||||
VERB_FILTER_DEP = {
|
||||
"$Object:objectID": ["GET", "HEAD", "DELETE", "RANGE", "RANGEHASH"],
|
||||
"$Object:containerID": ["GET", "PUT", "HEAD", "DELETE", "SEARCH", "RANGE", "RANGEHASH"],
|
||||
"$Object:ownerID": ["GET", "HEAD"],
|
||||
"$Object:creationEpoch": ["GET", "PUT", "HEAD"],
|
||||
"$Object:payloadLength": ["GET", "PUT", "HEAD"],
|
||||
"$Object:payloadHash": ["GET", "PUT", "HEAD"],
|
||||
"$Object:objectType": ["GET", "PUT", "HEAD"],
|
||||
"$Object:homomorphicHash": ["GET", "PUT", "HEAD"],
|
||||
"$Object:version": ["GET", "PUT", "HEAD"],
|
||||
}
|
11
robot/variables/wellknown_acl.py
Normal file
11
robot/variables/wellknown_acl.py
Normal file
|
@ -0,0 +1,11 @@
|
|||
# ACLs with final flag
|
||||
PUBLIC_ACL_F = "1FBFBFFF"
|
||||
PRIVATE_ACL_F = "1C8C8CCC"
|
||||
READONLY_ACL_F = "1FBF8CFF"
|
||||
|
||||
# ACLs without final flag set
|
||||
PUBLIC_ACL = "0FBFBFFF"
|
||||
INACCESSIBLE_ACL = "40000000"
|
||||
STICKYBIT_PUB_ACL = "3FFFFFFF"
|
||||
|
||||
EACL_PUBLIC_READ_WRITE = "eacl-public-read-write"
|
6
venv/local-pytest/environment.sh
Executable file → Normal file
6
venv/local-pytest/environment.sh
Executable file → Normal file
|
@ -1,6 +1,8 @@
|
|||
# DevEnv variables
|
||||
export NEOFS_MORPH_DISABLE_CACHE=true
|
||||
export DEVENV_PATH="${DEVENV_PATH:-${VIRTUAL_ENV}/../../frostfs-dev-env}"
|
||||
export DEVENV_PATH="${DEVENV_PATH:-${VIRTUAL_ENV}/../../neofs-dev-env}"
|
||||
pushd $DEVENV_PATH > /dev/null
|
||||
export `make env`
|
||||
popd > /dev/null
|
||||
popd > /dev/null
|
||||
|
||||
export PYTHONPATH=${PYTHONPATH}:${VIRTUAL_ENV}/../robot/resources/lib/:${VIRTUAL_ENV}/../robot/resources/lib/python_keywords:${VIRTUAL_ENV}/../robot/resources/lib/robot:${VIRTUAL_ENV}/../robot/variables:${VIRTUAL_ENV}/../pytest_tests/helpers:${VIRTUAL_ENV}/../pytest_tests/steps:${VIRTUAL_ENV}/../pytest_tests/resources
|
||||
|
|
Loading…
Reference in a new issue