Compare commits
9 commits
EliChin/fe
...
master
Author | SHA1 | Date | |
---|---|---|---|
c3947b0716 | |||
c97855dcee | |||
c997e23194 | |||
cff0e0f23e | |||
ef5e142015 | |||
06dc226ef8 | |||
ac7dae0d2d | |||
3802df25fe | |||
4755a2e167 |
22 changed files with 343 additions and 213 deletions
|
@ -48,20 +48,8 @@ To setup development environment for `frosfs-testcases`, please, take the follow
|
||||||
1. Prepare virtualenv
|
1. Prepare virtualenv
|
||||||
|
|
||||||
```shell
|
```shell
|
||||||
$ virtualenv --python=python3.9 venv
|
$ make venv
|
||||||
$ source venv/bin/activate
|
$ source frostfs-testcases-3.10/bin/activate
|
||||||
```
|
|
||||||
|
|
||||||
2. Install all dependencies:
|
|
||||||
|
|
||||||
```shell
|
|
||||||
$ pip install -r requirements.txt
|
|
||||||
```
|
|
||||||
|
|
||||||
3. Setup pre-commit hooks to run code formatters on staged files before you run a `git commit` command:
|
|
||||||
|
|
||||||
```shell
|
|
||||||
$ pre-commit install
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Optionally you might want to integrate code formatters with your code editor to apply formatters to code files as you go:
|
Optionally you might want to integrate code formatters with your code editor to apply formatters to code files as you go:
|
||||||
|
|
45
Makefile
45
Makefile
|
@ -1,28 +1,29 @@
|
||||||
#!/usr/bin/make -f
|
SHELL := /bin/bash
|
||||||
|
PYTHON_VERSION := 3.10
|
||||||
|
VENV_NAME = frostfs-testcases-${PYTHON_VERSION}
|
||||||
|
VENV_DIR := venv.${VENV_NAME}
|
||||||
|
|
||||||
.DEFAULT_GOAL := help
|
current_dir := $(shell pwd)
|
||||||
|
|
||||||
SHELL ?= bash
|
venv: create requirements paths precommit
|
||||||
|
@echo Ready
|
||||||
|
|
||||||
VENVS = $(shell ls -1d venv/*/ | sort -u | xargs basename -a)
|
precommit:
|
||||||
|
@echo Isntalling pre-commit hooks
|
||||||
|
. ${VENV_DIR}/bin/activate && pre-commit install
|
||||||
|
|
||||||
.PHONY: all
|
paths:
|
||||||
all: venvs
|
@echo Append paths for project
|
||||||
|
@echo Virtual environment: ${VENV_DIR}
|
||||||
|
@sudo rm -rf ${VENV_DIR}/lib/python${PYTHON_VERSION}/site-packages/_paths.pth
|
||||||
|
@sudo touch ${VENV_DIR}/lib/python${PYTHON_VERSION}/site-packages/_paths.pth
|
||||||
|
@echo ${current_dir} | sudo tee ${VENV_DIR}/lib/python${PYTHON_VERSION}/site-packages/_paths.pth
|
||||||
|
|
||||||
include venv_template.mk
|
create:
|
||||||
|
@echo Create virtual environment for
|
||||||
|
virtualenv --python=python${PYTHON_VERSION} --prompt=${VENV_NAME} ${VENV_DIR}
|
||||||
|
|
||||||
.PHONY: venvs
|
requirements:
|
||||||
venvs:
|
@echo Isntalling pip requirements
|
||||||
$(foreach venv,$(VENVS),venv.$(venv))
|
. ${VENV_DIR}/bin/activate && pip install -e ../frostfs-testlib
|
||||||
|
. ${VENV_DIR}/bin/activate && pip install -Ur pytest_tests/requirements.txt
|
||||||
$(foreach venv,$(VENVS),$(eval $(call VENV_template,$(venv))))
|
|
||||||
|
|
||||||
clean:
|
|
||||||
rm -rf venv.*
|
|
||||||
|
|
||||||
pytest-local:
|
|
||||||
@echo "⇒ Run Pytest"
|
|
||||||
python -m pytest pytest_tests/testsuites/
|
|
||||||
|
|
||||||
help:
|
|
||||||
@echo "⇒ run Run testcases ${R}"
|
|
12
README.md
12
README.md
|
@ -49,17 +49,11 @@ As we use frostfs-dev-env, you'll also need to install
|
||||||
6. Prepare virtualenv
|
6. Prepare virtualenv
|
||||||
|
|
||||||
```shell
|
```shell
|
||||||
$ make venv.local-pytest
|
$ make venv
|
||||||
$ . venv.local-pytest/bin/activate
|
$ source venv.frostfs-testcases-3.10/bin/activate
|
||||||
```
|
```
|
||||||
|
|
||||||
7. Setup pre-commit hooks to run code formatters on staged files before you run a `git commit` command:
|
7. Optionally you might want to integrate code formatters with your code editor to apply formatters to code files as you go:
|
||||||
|
|
||||||
```shell
|
|
||||||
$ pre-commit install
|
|
||||||
```
|
|
||||||
|
|
||||||
Optionally you might want to integrate code formatters with your code editor to apply formatters to code files as you go:
|
|
||||||
* isort is supported by [PyCharm](https://plugins.jetbrains.com/plugin/15434-isortconnect), [VS Code](https://cereblanco.medium.com/setup-black-and-isort-in-vscode-514804590bf9). Plugins exist for other IDEs/editors as well.
|
* isort is supported by [PyCharm](https://plugins.jetbrains.com/plugin/15434-isortconnect), [VS Code](https://cereblanco.medium.com/setup-black-and-isort-in-vscode-514804590bf9). Plugins exist for other IDEs/editors as well.
|
||||||
* black can be integrated with multiple editors, please, instructions are available [here](https://black.readthedocs.io/en/stable/integrations/editors.html).
|
* black can be integrated with multiple editors, please, instructions are available [here](https://black.readthedocs.io/en/stable/integrations/editors.html).
|
||||||
|
|
||||||
|
|
|
@ -20,21 +20,25 @@ from pytest_tests.helpers import frostfs_verbs
|
||||||
from pytest_tests.helpers.cluster import Cluster, StorageNode
|
from pytest_tests.helpers.cluster import Cluster, StorageNode
|
||||||
from pytest_tests.helpers.frostfs_verbs import head_object
|
from pytest_tests.helpers.frostfs_verbs import head_object
|
||||||
from pytest_tests.helpers.storage_object_info import StorageObjectInfo
|
from pytest_tests.helpers.storage_object_info import StorageObjectInfo
|
||||||
from pytest_tests.resources.common import WALLET_CONFIG
|
from pytest_tests.resources.common import CLI_DEFAULT_TIMEOUT, WALLET_CONFIG
|
||||||
|
|
||||||
logger = logging.getLogger("NeoLogger")
|
logger = logging.getLogger("NeoLogger")
|
||||||
|
|
||||||
|
|
||||||
def get_storage_object_chunks(
|
def get_storage_object_chunks(
|
||||||
storage_object: StorageObjectInfo, shell: Shell, cluster: Cluster
|
storage_object: StorageObjectInfo,
|
||||||
|
shell: Shell,
|
||||||
|
cluster: Cluster,
|
||||||
|
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||||
) -> list[str]:
|
) -> list[str]:
|
||||||
"""
|
"""
|
||||||
Get complex object split objects ids (no linker object)
|
Get complex object split objects ids (no linker object)
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
storage_object: storage_object to get it's chunks
|
storage_object: storage_object to get it's chunks
|
||||||
shell: client shell to do cmd requests
|
shell: client shell to do cmd requests
|
||||||
cluster: cluster object under test
|
cluster: cluster object under test
|
||||||
|
timeout: Timeout for an operation.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
list of object ids of complex object chunks
|
list of object ids of complex object chunks
|
||||||
|
@ -48,6 +52,7 @@ def get_storage_object_chunks(
|
||||||
shell,
|
shell,
|
||||||
cluster.storage_nodes,
|
cluster.storage_nodes,
|
||||||
is_direct=False,
|
is_direct=False,
|
||||||
|
timeout=timeout,
|
||||||
)
|
)
|
||||||
head = head_object(
|
head = head_object(
|
||||||
storage_object.wallet_file_path,
|
storage_object.wallet_file_path,
|
||||||
|
@ -55,6 +60,7 @@ def get_storage_object_chunks(
|
||||||
split_object_id,
|
split_object_id,
|
||||||
shell,
|
shell,
|
||||||
cluster.default_rpc_endpoint,
|
cluster.default_rpc_endpoint,
|
||||||
|
timeout=timeout,
|
||||||
)
|
)
|
||||||
|
|
||||||
chunks_object_ids = []
|
chunks_object_ids = []
|
||||||
|
@ -65,7 +71,10 @@ def get_storage_object_chunks(
|
||||||
|
|
||||||
|
|
||||||
def get_complex_object_split_ranges(
|
def get_complex_object_split_ranges(
|
||||||
storage_object: StorageObjectInfo, shell: Shell, cluster: Cluster
|
storage_object: StorageObjectInfo,
|
||||||
|
shell: Shell,
|
||||||
|
cluster: Cluster,
|
||||||
|
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||||
) -> list[Tuple[int, int]]:
|
) -> list[Tuple[int, int]]:
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
@ -75,9 +84,10 @@ def get_complex_object_split_ranges(
|
||||||
[(0, 30), (30, 30), (60, 30), (90, 10)]
|
[(0, 30), (30, 30), (60, 30), (90, 10)]
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
storage_object: storage_object to get it's chunks
|
storage_object: storage_object to get it's chunks
|
||||||
shell: client shell to do cmd requests
|
shell: client shell to do cmd requests
|
||||||
cluster: cluster object under test
|
cluster: cluster object under test
|
||||||
|
timeout: Timeout for an operation.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
list of object ids of complex object chunks
|
list of object ids of complex object chunks
|
||||||
|
@ -93,6 +103,7 @@ def get_complex_object_split_ranges(
|
||||||
chunk_id,
|
chunk_id,
|
||||||
shell,
|
shell,
|
||||||
cluster.default_rpc_endpoint,
|
cluster.default_rpc_endpoint,
|
||||||
|
timeout=timeout,
|
||||||
)
|
)
|
||||||
|
|
||||||
length = int(head["header"]["payloadLength"])
|
length = int(head["header"]["payloadLength"])
|
||||||
|
@ -113,6 +124,7 @@ def get_link_object(
|
||||||
bearer: str = "",
|
bearer: str = "",
|
||||||
wallet_config: str = WALLET_CONFIG,
|
wallet_config: str = WALLET_CONFIG,
|
||||||
is_direct: bool = True,
|
is_direct: bool = True,
|
||||||
|
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||||
):
|
):
|
||||||
"""
|
"""
|
||||||
Args:
|
Args:
|
||||||
|
@ -126,6 +138,7 @@ def get_link_object(
|
||||||
wallet_config (optional, str): path to the frostfs-cli config file
|
wallet_config (optional, str): path to the frostfs-cli config file
|
||||||
is_direct: send request directly to the node or not; this flag
|
is_direct: send request directly to the node or not; this flag
|
||||||
turns into `--ttl 1` key
|
turns into `--ttl 1` key
|
||||||
|
timeout: Timeout for an operation.
|
||||||
Returns:
|
Returns:
|
||||||
(str): Link Object ID
|
(str): Link Object ID
|
||||||
When no Link Object ID is found after all Storage Nodes polling,
|
When no Link Object ID is found after all Storage Nodes polling,
|
||||||
|
@ -144,6 +157,7 @@ def get_link_object(
|
||||||
is_direct=is_direct,
|
is_direct=is_direct,
|
||||||
bearer=bearer,
|
bearer=bearer,
|
||||||
wallet_config=wallet_config,
|
wallet_config=wallet_config,
|
||||||
|
timeout=timeout,
|
||||||
)
|
)
|
||||||
if resp["link"]:
|
if resp["link"]:
|
||||||
return resp["link"]
|
return resp["link"]
|
||||||
|
@ -155,7 +169,12 @@ def get_link_object(
|
||||||
|
|
||||||
@allure.step("Get Last Object")
|
@allure.step("Get Last Object")
|
||||||
def get_last_object(
|
def get_last_object(
|
||||||
wallet: str, cid: str, oid: str, shell: Shell, nodes: list[StorageNode]
|
wallet: str,
|
||||||
|
cid: str,
|
||||||
|
oid: str,
|
||||||
|
shell: Shell,
|
||||||
|
nodes: list[StorageNode],
|
||||||
|
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||||
) -> Optional[str]:
|
) -> Optional[str]:
|
||||||
"""
|
"""
|
||||||
Args:
|
Args:
|
||||||
|
@ -165,6 +184,7 @@ def get_last_object(
|
||||||
oid (str): Large Object ID
|
oid (str): Large Object ID
|
||||||
shell: executor for cli command
|
shell: executor for cli command
|
||||||
nodes: list of nodes to do search on
|
nodes: list of nodes to do search on
|
||||||
|
timeout: Timeout for an operation.
|
||||||
Returns:
|
Returns:
|
||||||
(str): Last Object ID
|
(str): Last Object ID
|
||||||
When no Last Object ID is found after all Storage Nodes polling,
|
When no Last Object ID is found after all Storage Nodes polling,
|
||||||
|
@ -174,7 +194,14 @@ def get_last_object(
|
||||||
endpoint = node.get_rpc_endpoint()
|
endpoint = node.get_rpc_endpoint()
|
||||||
try:
|
try:
|
||||||
resp = frostfs_verbs.head_object(
|
resp = frostfs_verbs.head_object(
|
||||||
wallet, cid, oid, shell=shell, endpoint=endpoint, is_raw=True, is_direct=True
|
wallet,
|
||||||
|
cid,
|
||||||
|
oid,
|
||||||
|
shell=shell,
|
||||||
|
endpoint=endpoint,
|
||||||
|
is_raw=True,
|
||||||
|
is_direct=True,
|
||||||
|
timeout=timeout,
|
||||||
)
|
)
|
||||||
if resp["lastPart"]:
|
if resp["lastPart"]:
|
||||||
return resp["lastPart"]
|
return resp["lastPart"]
|
||||||
|
|
|
@ -8,13 +8,13 @@ import allure
|
||||||
from frostfs_testlib.cli import FrostfsCli
|
from frostfs_testlib.cli import FrostfsCli
|
||||||
from frostfs_testlib.shell import Shell
|
from frostfs_testlib.shell import Shell
|
||||||
from frostfs_testlib.utils import json_utils
|
from frostfs_testlib.utils import json_utils
|
||||||
from wallet import WalletFile
|
|
||||||
|
|
||||||
from pytest_tests.helpers.cluster import Cluster
|
from pytest_tests.helpers.cluster import Cluster
|
||||||
from pytest_tests.helpers.file_helper import generate_file, get_file_hash
|
from pytest_tests.helpers.file_helper import generate_file, get_file_hash
|
||||||
from pytest_tests.helpers.frostfs_verbs import put_object, put_object_to_random_node
|
from pytest_tests.helpers.frostfs_verbs import put_object, put_object_to_random_node
|
||||||
from pytest_tests.helpers.storage_object_info import StorageObjectInfo
|
from pytest_tests.helpers.storage_object_info import StorageObjectInfo
|
||||||
from pytest_tests.resources.common import FROSTFS_CLI_EXEC, WALLET_CONFIG
|
from pytest_tests.helpers.wallet import WalletFile
|
||||||
|
from pytest_tests.resources.common import CLI_DEFAULT_TIMEOUT, FROSTFS_CLI_EXEC, WALLET_CONFIG
|
||||||
|
|
||||||
logger = logging.getLogger("NeoLogger")
|
logger = logging.getLogger("NeoLogger")
|
||||||
|
|
||||||
|
@ -115,6 +115,7 @@ def create_container(
|
||||||
options: dict = None,
|
options: dict = None,
|
||||||
await_mode: bool = True,
|
await_mode: bool = True,
|
||||||
wait_for_creation: bool = True,
|
wait_for_creation: bool = True,
|
||||||
|
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||||
) -> str:
|
) -> str:
|
||||||
"""
|
"""
|
||||||
A wrapper for `frostfs-cli container create` call.
|
A wrapper for `frostfs-cli container create` call.
|
||||||
|
@ -136,6 +137,7 @@ def create_container(
|
||||||
name (optional, str): container name attribute
|
name (optional, str): container name attribute
|
||||||
await_mode (bool): block execution until container is persisted
|
await_mode (bool): block execution until container is persisted
|
||||||
wait_for_creation (): Wait for container shows in container list
|
wait_for_creation (): Wait for container shows in container list
|
||||||
|
timeout: Timeout for the operation.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
(str): CID of the created container
|
(str): CID of the created container
|
||||||
|
@ -151,6 +153,7 @@ def create_container(
|
||||||
name=name,
|
name=name,
|
||||||
session=session_token,
|
session=session_token,
|
||||||
await_mode=await_mode,
|
await_mode=await_mode,
|
||||||
|
timeout=timeout,
|
||||||
**options or {},
|
**options or {},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -194,7 +197,9 @@ def wait_for_container_deletion(
|
||||||
|
|
||||||
|
|
||||||
@allure.step("List Containers")
|
@allure.step("List Containers")
|
||||||
def list_containers(wallet: str, shell: Shell, endpoint: str) -> list[str]:
|
def list_containers(
|
||||||
|
wallet: str, shell: Shell, endpoint: str, timeout: Optional[str] = CLI_DEFAULT_TIMEOUT
|
||||||
|
) -> list[str]:
|
||||||
"""
|
"""
|
||||||
A wrapper for `frostfs-cli container list` call. It returns all the
|
A wrapper for `frostfs-cli container list` call. It returns all the
|
||||||
available containers for the given wallet.
|
available containers for the given wallet.
|
||||||
|
@ -202,11 +207,12 @@ def list_containers(wallet: str, shell: Shell, endpoint: str) -> list[str]:
|
||||||
wallet (str): a wallet on whose behalf we list the containers
|
wallet (str): a wallet on whose behalf we list the containers
|
||||||
shell: executor for cli command
|
shell: executor for cli command
|
||||||
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
|
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
|
||||||
|
timeout: Timeout for the operation.
|
||||||
Returns:
|
Returns:
|
||||||
(list): list of containers
|
(list): list of containers
|
||||||
"""
|
"""
|
||||||
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, WALLET_CONFIG)
|
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, WALLET_CONFIG)
|
||||||
result = cli.container.list(rpc_endpoint=endpoint, wallet=wallet)
|
result = cli.container.list(rpc_endpoint=endpoint, wallet=wallet, timeout=timeout)
|
||||||
logger.info(f"Containers: \n{result}")
|
logger.info(f"Containers: \n{result}")
|
||||||
return result.stdout.split()
|
return result.stdout.split()
|
||||||
|
|
||||||
|
@ -218,6 +224,7 @@ def get_container(
|
||||||
shell: Shell,
|
shell: Shell,
|
||||||
endpoint: str,
|
endpoint: str,
|
||||||
json_mode: bool = True,
|
json_mode: bool = True,
|
||||||
|
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||||
) -> Union[dict, str]:
|
) -> Union[dict, str]:
|
||||||
"""
|
"""
|
||||||
A wrapper for `frostfs-cli container get` call. It extracts container's
|
A wrapper for `frostfs-cli container get` call. It extracts container's
|
||||||
|
@ -228,12 +235,15 @@ def get_container(
|
||||||
shell: executor for cli command
|
shell: executor for cli command
|
||||||
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
|
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
|
||||||
json_mode (bool): return container in JSON format
|
json_mode (bool): return container in JSON format
|
||||||
|
timeout: Timeout for the operation.
|
||||||
Returns:
|
Returns:
|
||||||
(dict, str): dict of container attributes
|
(dict, str): dict of container attributes
|
||||||
"""
|
"""
|
||||||
|
|
||||||
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, WALLET_CONFIG)
|
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, WALLET_CONFIG)
|
||||||
result = cli.container.get(rpc_endpoint=endpoint, wallet=wallet, cid=cid, json_mode=json_mode)
|
result = cli.container.get(
|
||||||
|
rpc_endpoint=endpoint, wallet=wallet, cid=cid, json_mode=json_mode, timeout=timeout
|
||||||
|
)
|
||||||
|
|
||||||
if not json_mode:
|
if not json_mode:
|
||||||
return result.stdout
|
return result.stdout
|
||||||
|
@ -258,6 +268,7 @@ def delete_container(
|
||||||
force: bool = False,
|
force: bool = False,
|
||||||
session_token: Optional[str] = None,
|
session_token: Optional[str] = None,
|
||||||
await_mode: bool = False,
|
await_mode: bool = False,
|
||||||
|
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""
|
"""
|
||||||
A wrapper for `frostfs-cli container delete` call.
|
A wrapper for `frostfs-cli container delete` call.
|
||||||
|
@ -268,6 +279,7 @@ def delete_container(
|
||||||
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
|
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
|
||||||
force (bool): do not check whether container contains locks and remove immediately
|
force (bool): do not check whether container contains locks and remove immediately
|
||||||
session_token: a path to session token file
|
session_token: a path to session token file
|
||||||
|
timeout: Timeout for the operation.
|
||||||
This function doesn't return anything.
|
This function doesn't return anything.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
@ -279,6 +291,7 @@ def delete_container(
|
||||||
force=force,
|
force=force,
|
||||||
session=session_token,
|
session=session_token,
|
||||||
await_mode=await_mode,
|
await_mode=await_mode,
|
||||||
|
timeout=timeout,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -6,11 +6,12 @@ import allure
|
||||||
from frostfs_testlib.cli import FrostfsAdm, FrostfsCli, NeoGo
|
from frostfs_testlib.cli import FrostfsAdm, FrostfsCli, NeoGo
|
||||||
from frostfs_testlib.shell import Shell
|
from frostfs_testlib.shell import Shell
|
||||||
from frostfs_testlib.utils import datetime_utils, wallet_utils
|
from frostfs_testlib.utils import datetime_utils, wallet_utils
|
||||||
from payment_neogo import get_contract_hash
|
|
||||||
|
|
||||||
from pytest_tests.helpers.cluster import Cluster, StorageNode
|
from pytest_tests.helpers.cluster import Cluster, StorageNode
|
||||||
|
from pytest_tests.helpers.payment_neogo import get_contract_hash
|
||||||
from pytest_tests.helpers.test_control import wait_for_success
|
from pytest_tests.helpers.test_control import wait_for_success
|
||||||
from pytest_tests.resources.common import (
|
from pytest_tests.resources.common import (
|
||||||
|
CLI_DEFAULT_TIMEOUT,
|
||||||
FROSTFS_ADM_CONFIG_PATH,
|
FROSTFS_ADM_CONFIG_PATH,
|
||||||
FROSTFS_ADM_EXEC,
|
FROSTFS_ADM_EXEC,
|
||||||
FROSTFS_CLI_EXEC,
|
FROSTFS_CLI_EXEC,
|
||||||
|
@ -55,7 +56,7 @@ def get_epoch(shell: Shell, cluster: Cluster, alive_node: Optional[StorageNode]
|
||||||
|
|
||||||
cli = FrostfsCli(shell=shell, frostfs_cli_exec_path=FROSTFS_CLI_EXEC, config_file=wallet_config)
|
cli = FrostfsCli(shell=shell, frostfs_cli_exec_path=FROSTFS_CLI_EXEC, config_file=wallet_config)
|
||||||
|
|
||||||
epoch = cli.netmap.epoch(endpoint, wallet_path)
|
epoch = cli.netmap.epoch(endpoint, wallet_path, timeout=CLI_DEFAULT_TIMEOUT)
|
||||||
return int(epoch.stdout)
|
return int(epoch.stdout)
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -11,7 +11,12 @@ from frostfs_testlib.shell import Shell
|
||||||
from frostfs_testlib.utils import json_utils
|
from frostfs_testlib.utils import json_utils
|
||||||
|
|
||||||
from pytest_tests.helpers.cluster import Cluster
|
from pytest_tests.helpers.cluster import Cluster
|
||||||
from pytest_tests.resources.common import ASSETS_DIR, FROSTFS_CLI_EXEC, WALLET_CONFIG
|
from pytest_tests.resources.common import (
|
||||||
|
ASSETS_DIR,
|
||||||
|
CLI_DEFAULT_TIMEOUT,
|
||||||
|
FROSTFS_CLI_EXEC,
|
||||||
|
WALLET_CONFIG,
|
||||||
|
)
|
||||||
|
|
||||||
logger = logging.getLogger("NeoLogger")
|
logger = logging.getLogger("NeoLogger")
|
||||||
|
|
||||||
|
@ -29,6 +34,7 @@ def get_object_from_random_node(
|
||||||
wallet_config: Optional[str] = None,
|
wallet_config: Optional[str] = None,
|
||||||
no_progress: bool = True,
|
no_progress: bool = True,
|
||||||
session: Optional[str] = None,
|
session: Optional[str] = None,
|
||||||
|
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||||
) -> str:
|
) -> str:
|
||||||
"""
|
"""
|
||||||
GET from FrostFS random storage node
|
GET from FrostFS random storage node
|
||||||
|
@ -45,6 +51,7 @@ def get_object_from_random_node(
|
||||||
no_progress(optional, bool): do not show progress bar
|
no_progress(optional, bool): do not show progress bar
|
||||||
xhdr (optional, dict): Request X-Headers in form of Key=Value
|
xhdr (optional, dict): Request X-Headers in form of Key=Value
|
||||||
session (optional, dict): path to a JSON-encoded container session token
|
session (optional, dict): path to a JSON-encoded container session token
|
||||||
|
timeout: Timeout for the operation.
|
||||||
Returns:
|
Returns:
|
||||||
(str): path to downloaded file
|
(str): path to downloaded file
|
||||||
"""
|
"""
|
||||||
|
@ -61,6 +68,7 @@ def get_object_from_random_node(
|
||||||
wallet_config,
|
wallet_config,
|
||||||
no_progress,
|
no_progress,
|
||||||
session,
|
session,
|
||||||
|
timeout,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -77,6 +85,7 @@ def get_object(
|
||||||
wallet_config: Optional[str] = None,
|
wallet_config: Optional[str] = None,
|
||||||
no_progress: bool = True,
|
no_progress: bool = True,
|
||||||
session: Optional[str] = None,
|
session: Optional[str] = None,
|
||||||
|
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||||
) -> str:
|
) -> str:
|
||||||
"""
|
"""
|
||||||
GET from FrostFS.
|
GET from FrostFS.
|
||||||
|
@ -93,6 +102,7 @@ def get_object(
|
||||||
no_progress(optional, bool): do not show progress bar
|
no_progress(optional, bool): do not show progress bar
|
||||||
xhdr (optional, dict): Request X-Headers in form of Key=Value
|
xhdr (optional, dict): Request X-Headers in form of Key=Value
|
||||||
session (optional, dict): path to a JSON-encoded container session token
|
session (optional, dict): path to a JSON-encoded container session token
|
||||||
|
timeout: Timeout for the operation.
|
||||||
Returns:
|
Returns:
|
||||||
(str): path to downloaded file
|
(str): path to downloaded file
|
||||||
"""
|
"""
|
||||||
|
@ -112,6 +122,7 @@ def get_object(
|
||||||
no_progress=no_progress,
|
no_progress=no_progress,
|
||||||
xhdr=xhdr,
|
xhdr=xhdr,
|
||||||
session=session,
|
session=session,
|
||||||
|
timeout=timeout,
|
||||||
)
|
)
|
||||||
|
|
||||||
return file_path
|
return file_path
|
||||||
|
@ -129,6 +140,7 @@ def get_range_hash(
|
||||||
wallet_config: Optional[str] = None,
|
wallet_config: Optional[str] = None,
|
||||||
xhdr: Optional[dict] = None,
|
xhdr: Optional[dict] = None,
|
||||||
session: Optional[str] = None,
|
session: Optional[str] = None,
|
||||||
|
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||||
):
|
):
|
||||||
"""
|
"""
|
||||||
GETRANGEHASH of given Object.
|
GETRANGEHASH of given Object.
|
||||||
|
@ -145,6 +157,7 @@ def get_range_hash(
|
||||||
wallet_config: path to the wallet config
|
wallet_config: path to the wallet config
|
||||||
xhdr: Request X-Headers in form of Key=Values
|
xhdr: Request X-Headers in form of Key=Values
|
||||||
session: Filepath to a JSON- or binary-encoded token of the object RANGEHASH session.
|
session: Filepath to a JSON- or binary-encoded token of the object RANGEHASH session.
|
||||||
|
timeout: Timeout for the operation.
|
||||||
Returns:
|
Returns:
|
||||||
None
|
None
|
||||||
"""
|
"""
|
||||||
|
@ -158,6 +171,7 @@ def get_range_hash(
|
||||||
bearer=bearer,
|
bearer=bearer,
|
||||||
xhdr=xhdr,
|
xhdr=xhdr,
|
||||||
session=session,
|
session=session,
|
||||||
|
timeout=timeout,
|
||||||
)
|
)
|
||||||
|
|
||||||
# cutting off output about range offset and length
|
# cutting off output about range offset and length
|
||||||
|
@ -178,6 +192,7 @@ def put_object_to_random_node(
|
||||||
expire_at: Optional[int] = None,
|
expire_at: Optional[int] = None,
|
||||||
no_progress: bool = True,
|
no_progress: bool = True,
|
||||||
session: Optional[str] = None,
|
session: Optional[str] = None,
|
||||||
|
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||||
):
|
):
|
||||||
"""
|
"""
|
||||||
PUT of given file to a random storage node.
|
PUT of given file to a random storage node.
|
||||||
|
@ -196,6 +211,7 @@ def put_object_to_random_node(
|
||||||
expire_at: Last epoch in the life of the object
|
expire_at: Last epoch in the life of the object
|
||||||
xhdr: Request X-Headers in form of Key=Value
|
xhdr: Request X-Headers in form of Key=Value
|
||||||
session: path to a JSON-encoded container session token
|
session: path to a JSON-encoded container session token
|
||||||
|
timeout: Timeout for the operation.
|
||||||
Returns:
|
Returns:
|
||||||
ID of uploaded Object
|
ID of uploaded Object
|
||||||
"""
|
"""
|
||||||
|
@ -214,6 +230,7 @@ def put_object_to_random_node(
|
||||||
expire_at,
|
expire_at,
|
||||||
no_progress,
|
no_progress,
|
||||||
session,
|
session,
|
||||||
|
timeout,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -231,6 +248,7 @@ def put_object(
|
||||||
expire_at: Optional[int] = None,
|
expire_at: Optional[int] = None,
|
||||||
no_progress: bool = True,
|
no_progress: bool = True,
|
||||||
session: Optional[str] = None,
|
session: Optional[str] = None,
|
||||||
|
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||||
):
|
):
|
||||||
"""
|
"""
|
||||||
PUT of given file.
|
PUT of given file.
|
||||||
|
@ -248,6 +266,7 @@ def put_object(
|
||||||
expire_at: Last epoch in the life of the object
|
expire_at: Last epoch in the life of the object
|
||||||
xhdr: Request X-Headers in form of Key=Value
|
xhdr: Request X-Headers in form of Key=Value
|
||||||
session: path to a JSON-encoded container session token
|
session: path to a JSON-encoded container session token
|
||||||
|
timeout: Timeout for the operation.
|
||||||
Returns:
|
Returns:
|
||||||
(str): ID of uploaded Object
|
(str): ID of uploaded Object
|
||||||
"""
|
"""
|
||||||
|
@ -264,6 +283,7 @@ def put_object(
|
||||||
no_progress=no_progress,
|
no_progress=no_progress,
|
||||||
xhdr=xhdr,
|
xhdr=xhdr,
|
||||||
session=session,
|
session=session,
|
||||||
|
timeout=timeout,
|
||||||
)
|
)
|
||||||
|
|
||||||
# splitting CLI output to lines and taking the penultimate line
|
# splitting CLI output to lines and taking the penultimate line
|
||||||
|
@ -283,6 +303,7 @@ def delete_object(
|
||||||
wallet_config: Optional[str] = None,
|
wallet_config: Optional[str] = None,
|
||||||
xhdr: Optional[dict] = None,
|
xhdr: Optional[dict] = None,
|
||||||
session: Optional[str] = None,
|
session: Optional[str] = None,
|
||||||
|
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||||
):
|
):
|
||||||
"""
|
"""
|
||||||
DELETE an Object.
|
DELETE an Object.
|
||||||
|
@ -297,6 +318,7 @@ def delete_object(
|
||||||
wallet_config: path to the wallet config
|
wallet_config: path to the wallet config
|
||||||
xhdr: Request X-Headers in form of Key=Value
|
xhdr: Request X-Headers in form of Key=Value
|
||||||
session: path to a JSON-encoded container session token
|
session: path to a JSON-encoded container session token
|
||||||
|
timeout: Timeout for the operation.
|
||||||
Returns:
|
Returns:
|
||||||
(str): Tombstone ID
|
(str): Tombstone ID
|
||||||
"""
|
"""
|
||||||
|
@ -310,6 +332,7 @@ def delete_object(
|
||||||
bearer=bearer,
|
bearer=bearer,
|
||||||
xhdr=xhdr,
|
xhdr=xhdr,
|
||||||
session=session,
|
session=session,
|
||||||
|
timeout=timeout,
|
||||||
)
|
)
|
||||||
|
|
||||||
id_str = result.stdout.split("\n")[1]
|
id_str = result.stdout.split("\n")[1]
|
||||||
|
@ -329,6 +352,7 @@ def get_range(
|
||||||
bearer: str = "",
|
bearer: str = "",
|
||||||
xhdr: Optional[dict] = None,
|
xhdr: Optional[dict] = None,
|
||||||
session: Optional[str] = None,
|
session: Optional[str] = None,
|
||||||
|
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||||
):
|
):
|
||||||
"""
|
"""
|
||||||
GETRANGE an Object.
|
GETRANGE an Object.
|
||||||
|
@ -344,6 +368,7 @@ def get_range(
|
||||||
wallet_config: path to the wallet config
|
wallet_config: path to the wallet config
|
||||||
xhdr: Request X-Headers in form of Key=Value
|
xhdr: Request X-Headers in form of Key=Value
|
||||||
session: path to a JSON-encoded container session token
|
session: path to a JSON-encoded container session token
|
||||||
|
timeout: Timeout for the operation.
|
||||||
Returns:
|
Returns:
|
||||||
(str, bytes) - path to the file with range content and content of this file as bytes
|
(str, bytes) - path to the file with range content and content of this file as bytes
|
||||||
"""
|
"""
|
||||||
|
@ -360,6 +385,7 @@ def get_range(
|
||||||
bearer=bearer,
|
bearer=bearer,
|
||||||
xhdr=xhdr,
|
xhdr=xhdr,
|
||||||
session=session,
|
session=session,
|
||||||
|
timeout=timeout,
|
||||||
)
|
)
|
||||||
|
|
||||||
with open(range_file_path, "rb") as file:
|
with open(range_file_path, "rb") as file:
|
||||||
|
@ -382,6 +408,7 @@ def lock_object(
|
||||||
wallet_config: Optional[str] = None,
|
wallet_config: Optional[str] = None,
|
||||||
ttl: Optional[int] = None,
|
ttl: Optional[int] = None,
|
||||||
xhdr: Optional[dict] = None,
|
xhdr: Optional[dict] = None,
|
||||||
|
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||||
) -> str:
|
) -> str:
|
||||||
"""
|
"""
|
||||||
Lock object in container.
|
Lock object in container.
|
||||||
|
@ -399,6 +426,7 @@ def lock_object(
|
||||||
ttl: TTL value in request meta header (default 2).
|
ttl: TTL value in request meta header (default 2).
|
||||||
wallet: WIF (NEP-2) string or path to the wallet or binary key.
|
wallet: WIF (NEP-2) string or path to the wallet or binary key.
|
||||||
xhdr: Dict with request X-Headers.
|
xhdr: Dict with request X-Headers.
|
||||||
|
timeout: Timeout for the operation.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Lock object ID
|
Lock object ID
|
||||||
|
@ -417,6 +445,7 @@ def lock_object(
|
||||||
xhdr=xhdr,
|
xhdr=xhdr,
|
||||||
session=session,
|
session=session,
|
||||||
ttl=ttl,
|
ttl=ttl,
|
||||||
|
timeout=timeout,
|
||||||
)
|
)
|
||||||
|
|
||||||
# splitting CLI output to lines and taking the penultimate line
|
# splitting CLI output to lines and taking the penultimate line
|
||||||
|
@ -439,6 +468,7 @@ def search_object(
|
||||||
session: Optional[str] = None,
|
session: Optional[str] = None,
|
||||||
phy: bool = False,
|
phy: bool = False,
|
||||||
root: bool = False,
|
root: bool = False,
|
||||||
|
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||||
) -> list:
|
) -> list:
|
||||||
"""
|
"""
|
||||||
SEARCH an Object.
|
SEARCH an Object.
|
||||||
|
@ -456,6 +486,7 @@ def search_object(
|
||||||
session: path to a JSON-encoded container session token
|
session: path to a JSON-encoded container session token
|
||||||
phy: Search physically stored objects.
|
phy: Search physically stored objects.
|
||||||
root: Search for user objects.
|
root: Search for user objects.
|
||||||
|
timeout: Timeout for the operation.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
list of found ObjectIDs
|
list of found ObjectIDs
|
||||||
|
@ -474,6 +505,7 @@ def search_object(
|
||||||
session=session,
|
session=session,
|
||||||
phy=phy,
|
phy=phy,
|
||||||
root=root,
|
root=root,
|
||||||
|
timeout=timeout,
|
||||||
)
|
)
|
||||||
|
|
||||||
found_objects = re.findall(r"(\w{43,44})", result.stdout)
|
found_objects = re.findall(r"(\w{43,44})", result.stdout)
|
||||||
|
@ -502,6 +534,7 @@ def get_netmap_netinfo(
|
||||||
address: Optional[str] = None,
|
address: Optional[str] = None,
|
||||||
ttl: Optional[int] = None,
|
ttl: Optional[int] = None,
|
||||||
xhdr: Optional[dict] = None,
|
xhdr: Optional[dict] = None,
|
||||||
|
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
"""
|
"""
|
||||||
Get netmap netinfo output from node
|
Get netmap netinfo output from node
|
||||||
|
@ -526,6 +559,7 @@ def get_netmap_netinfo(
|
||||||
address=address,
|
address=address,
|
||||||
ttl=ttl,
|
ttl=ttl,
|
||||||
xhdr=xhdr,
|
xhdr=xhdr,
|
||||||
|
timeout=timeout,
|
||||||
)
|
)
|
||||||
|
|
||||||
settings = dict()
|
settings = dict()
|
||||||
|
@ -556,6 +590,7 @@ def head_object(
|
||||||
is_direct: bool = False,
|
is_direct: bool = False,
|
||||||
wallet_config: Optional[str] = None,
|
wallet_config: Optional[str] = None,
|
||||||
session: Optional[str] = None,
|
session: Optional[str] = None,
|
||||||
|
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||||
):
|
):
|
||||||
"""
|
"""
|
||||||
HEAD an Object.
|
HEAD an Object.
|
||||||
|
@ -576,6 +611,7 @@ def head_object(
|
||||||
wallet_config(optional, str): path to the wallet config
|
wallet_config(optional, str): path to the wallet config
|
||||||
xhdr (optional, dict): Request X-Headers in form of Key=Value
|
xhdr (optional, dict): Request X-Headers in form of Key=Value
|
||||||
session (optional, dict): path to a JSON-encoded container session token
|
session (optional, dict): path to a JSON-encoded container session token
|
||||||
|
timeout: Timeout for the operation.
|
||||||
Returns:
|
Returns:
|
||||||
depending on the `json_output` parameter value, the function returns
|
depending on the `json_output` parameter value, the function returns
|
||||||
(dict): HEAD response in JSON format
|
(dict): HEAD response in JSON format
|
||||||
|
@ -595,6 +631,7 @@ def head_object(
|
||||||
ttl=1 if is_direct else None,
|
ttl=1 if is_direct else None,
|
||||||
xhdr=xhdr,
|
xhdr=xhdr,
|
||||||
session=session,
|
session=session,
|
||||||
|
timeout=timeout,
|
||||||
)
|
)
|
||||||
|
|
||||||
if not json_output:
|
if not json_output:
|
||||||
|
|
|
@ -10,9 +10,9 @@ from urllib.parse import quote_plus
|
||||||
|
|
||||||
import allure
|
import allure
|
||||||
import requests
|
import requests
|
||||||
from aws_cli_client import LONG_TIMEOUT
|
|
||||||
from frostfs_testlib.shell import Shell
|
from frostfs_testlib.shell import Shell
|
||||||
|
|
||||||
|
from pytest_tests.helpers.aws_cli_client import LONG_TIMEOUT
|
||||||
from pytest_tests.helpers.cli_helpers import _cmd_run
|
from pytest_tests.helpers.cli_helpers import _cmd_run
|
||||||
from pytest_tests.helpers.cluster import StorageNode
|
from pytest_tests.helpers.cluster import StorageNode
|
||||||
from pytest_tests.helpers.file_helper import get_file_hash
|
from pytest_tests.helpers.file_helper import get_file_hash
|
||||||
|
|
|
@ -6,7 +6,8 @@ from typing import Optional
|
||||||
|
|
||||||
import allure
|
import allure
|
||||||
from frostfs_testlib.shell import Shell
|
from frostfs_testlib.shell import Shell
|
||||||
from remote_process import RemoteProcess
|
|
||||||
|
from pytest_tests.helpers.remote_process import RemoteProcess
|
||||||
|
|
||||||
EXIT_RESULT_CODE = 0
|
EXIT_RESULT_CODE = 0
|
||||||
LOAD_RESULTS_PATTERNS = {
|
LOAD_RESULTS_PATTERNS = {
|
||||||
|
|
|
@ -16,6 +16,7 @@ from pytest_tests.helpers.frostfs_verbs import (
|
||||||
put_object_to_random_node,
|
put_object_to_random_node,
|
||||||
search_object,
|
search_object,
|
||||||
)
|
)
|
||||||
|
from pytest_tests.resources.common import CLI_DEFAULT_TIMEOUT
|
||||||
|
|
||||||
OPERATION_ERROR_TYPE = RuntimeError
|
OPERATION_ERROR_TYPE = RuntimeError
|
||||||
|
|
||||||
|
@ -123,6 +124,7 @@ def can_get_head_object(
|
||||||
bearer: Optional[str] = None,
|
bearer: Optional[str] = None,
|
||||||
wallet_config: Optional[str] = None,
|
wallet_config: Optional[str] = None,
|
||||||
xhdr: Optional[dict] = None,
|
xhdr: Optional[dict] = None,
|
||||||
|
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||||
) -> bool:
|
) -> bool:
|
||||||
with allure.step("Try get head of object"):
|
with allure.step("Try get head of object"):
|
||||||
try:
|
try:
|
||||||
|
@ -135,6 +137,7 @@ def can_get_head_object(
|
||||||
xhdr=xhdr,
|
xhdr=xhdr,
|
||||||
shell=shell,
|
shell=shell,
|
||||||
endpoint=endpoint,
|
endpoint=endpoint,
|
||||||
|
timeout=timeout,
|
||||||
)
|
)
|
||||||
except OPERATION_ERROR_TYPE as err:
|
except OPERATION_ERROR_TYPE as err:
|
||||||
assert string_utils.is_str_match_pattern(
|
assert string_utils.is_str_match_pattern(
|
||||||
|
@ -153,6 +156,7 @@ def can_get_range_of_object(
|
||||||
bearer: Optional[str] = None,
|
bearer: Optional[str] = None,
|
||||||
wallet_config: Optional[str] = None,
|
wallet_config: Optional[str] = None,
|
||||||
xhdr: Optional[dict] = None,
|
xhdr: Optional[dict] = None,
|
||||||
|
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||||
) -> bool:
|
) -> bool:
|
||||||
with allure.step("Try get range of object"):
|
with allure.step("Try get range of object"):
|
||||||
try:
|
try:
|
||||||
|
@ -166,6 +170,7 @@ def can_get_range_of_object(
|
||||||
xhdr=xhdr,
|
xhdr=xhdr,
|
||||||
shell=shell,
|
shell=shell,
|
||||||
endpoint=endpoint,
|
endpoint=endpoint,
|
||||||
|
timeout=timeout,
|
||||||
)
|
)
|
||||||
except OPERATION_ERROR_TYPE as err:
|
except OPERATION_ERROR_TYPE as err:
|
||||||
assert string_utils.is_str_match_pattern(
|
assert string_utils.is_str_match_pattern(
|
||||||
|
@ -184,6 +189,7 @@ def can_get_range_hash_of_object(
|
||||||
bearer: Optional[str] = None,
|
bearer: Optional[str] = None,
|
||||||
wallet_config: Optional[str] = None,
|
wallet_config: Optional[str] = None,
|
||||||
xhdr: Optional[dict] = None,
|
xhdr: Optional[dict] = None,
|
||||||
|
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||||
) -> bool:
|
) -> bool:
|
||||||
with allure.step("Try get range hash of object"):
|
with allure.step("Try get range hash of object"):
|
||||||
try:
|
try:
|
||||||
|
@ -197,6 +203,7 @@ def can_get_range_hash_of_object(
|
||||||
xhdr=xhdr,
|
xhdr=xhdr,
|
||||||
shell=shell,
|
shell=shell,
|
||||||
endpoint=endpoint,
|
endpoint=endpoint,
|
||||||
|
timeout=timeout,
|
||||||
)
|
)
|
||||||
except OPERATION_ERROR_TYPE as err:
|
except OPERATION_ERROR_TYPE as err:
|
||||||
assert string_utils.is_str_match_pattern(
|
assert string_utils.is_str_match_pattern(
|
||||||
|
@ -215,6 +222,7 @@ def can_search_object(
|
||||||
bearer: Optional[str] = None,
|
bearer: Optional[str] = None,
|
||||||
wallet_config: Optional[str] = None,
|
wallet_config: Optional[str] = None,
|
||||||
xhdr: Optional[dict] = None,
|
xhdr: Optional[dict] = None,
|
||||||
|
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||||
) -> bool:
|
) -> bool:
|
||||||
with allure.step("Try search object in container"):
|
with allure.step("Try search object in container"):
|
||||||
try:
|
try:
|
||||||
|
@ -226,6 +234,7 @@ def can_search_object(
|
||||||
xhdr=xhdr,
|
xhdr=xhdr,
|
||||||
shell=shell,
|
shell=shell,
|
||||||
endpoint=endpoint,
|
endpoint=endpoint,
|
||||||
|
timeout=timeout,
|
||||||
)
|
)
|
||||||
except OPERATION_ERROR_TYPE as err:
|
except OPERATION_ERROR_TYPE as err:
|
||||||
assert string_utils.is_str_match_pattern(
|
assert string_utils.is_str_match_pattern(
|
||||||
|
|
|
@ -12,7 +12,7 @@ from frostfs_testlib.shell import Shell
|
||||||
from pytest_tests.helpers.cluster import Cluster
|
from pytest_tests.helpers.cluster import Cluster
|
||||||
from pytest_tests.helpers.complex_object_actions import get_link_object
|
from pytest_tests.helpers.complex_object_actions import get_link_object
|
||||||
from pytest_tests.helpers.frostfs_verbs import head_object
|
from pytest_tests.helpers.frostfs_verbs import head_object
|
||||||
from pytest_tests.resources.common import FROSTFS_CLI_EXEC, WALLET_CONFIG
|
from pytest_tests.resources.common import CLI_DEFAULT_TIMEOUT, FROSTFS_CLI_EXEC, WALLET_CONFIG
|
||||||
|
|
||||||
logger = logging.getLogger("NeoLogger")
|
logger = logging.getLogger("NeoLogger")
|
||||||
|
|
||||||
|
@ -27,6 +27,7 @@ def put_storagegroup(
|
||||||
bearer: Optional[str] = None,
|
bearer: Optional[str] = None,
|
||||||
wallet_config: str = WALLET_CONFIG,
|
wallet_config: str = WALLET_CONFIG,
|
||||||
lifetime: int = 10,
|
lifetime: int = 10,
|
||||||
|
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||||
) -> str:
|
) -> str:
|
||||||
"""
|
"""
|
||||||
Wrapper for `frostfs-cli storagegroup put`. Before the SG is created,
|
Wrapper for `frostfs-cli storagegroup put`. Before the SG is created,
|
||||||
|
@ -40,6 +41,7 @@ def put_storagegroup(
|
||||||
objects: List of Object IDs to include into the SG.
|
objects: List of Object IDs to include into the SG.
|
||||||
bearer: Path to Bearer token file.
|
bearer: Path to Bearer token file.
|
||||||
wallet_config: Path to frostfs-cli config file.
|
wallet_config: Path to frostfs-cli config file.
|
||||||
|
timeout: Timeout for an operation.
|
||||||
Returns:
|
Returns:
|
||||||
Object ID of created Storage Group.
|
Object ID of created Storage Group.
|
||||||
"""
|
"""
|
||||||
|
@ -66,6 +68,7 @@ def list_storagegroup(
|
||||||
cid: str,
|
cid: str,
|
||||||
bearer: Optional[str] = None,
|
bearer: Optional[str] = None,
|
||||||
wallet_config: str = WALLET_CONFIG,
|
wallet_config: str = WALLET_CONFIG,
|
||||||
|
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||||
) -> list:
|
) -> list:
|
||||||
"""
|
"""
|
||||||
Wrapper for `frostfs-cli storagegroup list`. This operation
|
Wrapper for `frostfs-cli storagegroup list`. This operation
|
||||||
|
@ -76,6 +79,7 @@ def list_storagegroup(
|
||||||
cid: ID of Container to list.
|
cid: ID of Container to list.
|
||||||
bearer: Path to Bearer token file.
|
bearer: Path to Bearer token file.
|
||||||
wallet_config: Path to frostfs-cli config file.
|
wallet_config: Path to frostfs-cli config file.
|
||||||
|
timeout: Timeout for an operation.
|
||||||
Returns:
|
Returns:
|
||||||
Object IDs of found Storage Groups.
|
Object IDs of found Storage Groups.
|
||||||
"""
|
"""
|
||||||
|
@ -87,6 +91,7 @@ def list_storagegroup(
|
||||||
cid=cid,
|
cid=cid,
|
||||||
bearer=bearer,
|
bearer=bearer,
|
||||||
rpc_endpoint=endpoint,
|
rpc_endpoint=endpoint,
|
||||||
|
timeout=timeout,
|
||||||
)
|
)
|
||||||
# throwing off the first string of output
|
# throwing off the first string of output
|
||||||
found_objects = result.stdout.split("\n")[1:]
|
found_objects = result.stdout.split("\n")[1:]
|
||||||
|
@ -102,6 +107,7 @@ def get_storagegroup(
|
||||||
gid: str,
|
gid: str,
|
||||||
bearer: str = "",
|
bearer: str = "",
|
||||||
wallet_config: str = WALLET_CONFIG,
|
wallet_config: str = WALLET_CONFIG,
|
||||||
|
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||||
) -> dict:
|
) -> dict:
|
||||||
"""
|
"""
|
||||||
Wrapper for `frostfs-cli storagegroup get`.
|
Wrapper for `frostfs-cli storagegroup get`.
|
||||||
|
@ -112,6 +118,7 @@ def get_storagegroup(
|
||||||
gid: ID of the Storage Group.
|
gid: ID of the Storage Group.
|
||||||
bearer: Path to Bearer token file.
|
bearer: Path to Bearer token file.
|
||||||
wallet_config: Path to frostfs-cli config file.
|
wallet_config: Path to frostfs-cli config file.
|
||||||
|
timeout: Timeout for an operation.
|
||||||
Returns:
|
Returns:
|
||||||
Detailed information on the Storage Group.
|
Detailed information on the Storage Group.
|
||||||
"""
|
"""
|
||||||
|
@ -124,6 +131,7 @@ def get_storagegroup(
|
||||||
bearer=bearer,
|
bearer=bearer,
|
||||||
id=gid,
|
id=gid,
|
||||||
rpc_endpoint=endpoint,
|
rpc_endpoint=endpoint,
|
||||||
|
timeout=timeout,
|
||||||
)
|
)
|
||||||
|
|
||||||
# TODO: temporary solution for parsing output. Needs to be replaced with
|
# TODO: temporary solution for parsing output. Needs to be replaced with
|
||||||
|
@ -153,6 +161,7 @@ def delete_storagegroup(
|
||||||
gid: str,
|
gid: str,
|
||||||
bearer: str = "",
|
bearer: str = "",
|
||||||
wallet_config: str = WALLET_CONFIG,
|
wallet_config: str = WALLET_CONFIG,
|
||||||
|
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||||
) -> str:
|
) -> str:
|
||||||
"""
|
"""
|
||||||
Wrapper for `frostfs-cli storagegroup delete`.
|
Wrapper for `frostfs-cli storagegroup delete`.
|
||||||
|
@ -163,6 +172,7 @@ def delete_storagegroup(
|
||||||
gid: ID of the Storage Group.
|
gid: ID of the Storage Group.
|
||||||
bearer: Path to Bearer token file.
|
bearer: Path to Bearer token file.
|
||||||
wallet_config: Path to frostfs-cli config file.
|
wallet_config: Path to frostfs-cli config file.
|
||||||
|
timeout: Timeout for an operation.
|
||||||
Returns:
|
Returns:
|
||||||
Tombstone ID of the deleted Storage Group.
|
Tombstone ID of the deleted Storage Group.
|
||||||
"""
|
"""
|
||||||
|
@ -175,6 +185,7 @@ def delete_storagegroup(
|
||||||
bearer=bearer,
|
bearer=bearer,
|
||||||
id=gid,
|
id=gid,
|
||||||
rpc_endpoint=endpoint,
|
rpc_endpoint=endpoint,
|
||||||
|
timeout=timeout,
|
||||||
)
|
)
|
||||||
tombstone_id = result.stdout.strip().split("\n")[1].split(": ")[1]
|
tombstone_id = result.stdout.strip().split("\n")[1].split(": ")[1]
|
||||||
return tombstone_id
|
return tombstone_id
|
||||||
|
@ -189,6 +200,7 @@ def verify_list_storage_group(
|
||||||
gid: str,
|
gid: str,
|
||||||
bearer: str = None,
|
bearer: str = None,
|
||||||
wallet_config: str = WALLET_CONFIG,
|
wallet_config: str = WALLET_CONFIG,
|
||||||
|
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||||
):
|
):
|
||||||
storage_groups = list_storagegroup(
|
storage_groups = list_storagegroup(
|
||||||
shell=shell,
|
shell=shell,
|
||||||
|
@ -197,6 +209,7 @@ def verify_list_storage_group(
|
||||||
cid=cid,
|
cid=cid,
|
||||||
bearer=bearer,
|
bearer=bearer,
|
||||||
wallet_config=wallet_config,
|
wallet_config=wallet_config,
|
||||||
|
timeout=timeout,
|
||||||
)
|
)
|
||||||
assert gid in storage_groups
|
assert gid in storage_groups
|
||||||
|
|
||||||
|
@ -213,6 +226,7 @@ def verify_get_storage_group(
|
||||||
max_object_size: int,
|
max_object_size: int,
|
||||||
bearer: str = None,
|
bearer: str = None,
|
||||||
wallet_config: str = WALLET_CONFIG,
|
wallet_config: str = WALLET_CONFIG,
|
||||||
|
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||||
):
|
):
|
||||||
obj_parts = []
|
obj_parts = []
|
||||||
endpoint = cluster.default_rpc_endpoint
|
endpoint = cluster.default_rpc_endpoint
|
||||||
|
@ -226,6 +240,7 @@ def verify_get_storage_group(
|
||||||
nodes=cluster.storage_nodes,
|
nodes=cluster.storage_nodes,
|
||||||
bearer=bearer,
|
bearer=bearer,
|
||||||
wallet_config=wallet_config,
|
wallet_config=wallet_config,
|
||||||
|
timeout=timeout,
|
||||||
)
|
)
|
||||||
obj_head = head_object(
|
obj_head = head_object(
|
||||||
wallet=wallet,
|
wallet=wallet,
|
||||||
|
@ -236,6 +251,7 @@ def verify_get_storage_group(
|
||||||
is_raw=True,
|
is_raw=True,
|
||||||
bearer=bearer,
|
bearer=bearer,
|
||||||
wallet_config=wallet_config,
|
wallet_config=wallet_config,
|
||||||
|
timeout=timeout,
|
||||||
)
|
)
|
||||||
obj_parts = obj_head["header"]["split"]["children"]
|
obj_parts = obj_head["header"]["split"]["children"]
|
||||||
|
|
||||||
|
@ -248,6 +264,7 @@ def verify_get_storage_group(
|
||||||
gid=gid,
|
gid=gid,
|
||||||
bearer=bearer,
|
bearer=bearer,
|
||||||
wallet_config=wallet_config,
|
wallet_config=wallet_config,
|
||||||
|
timeout=timeout,
|
||||||
)
|
)
|
||||||
exp_size = object_size * obj_num
|
exp_size = object_size * obj_num
|
||||||
if object_size < max_object_size:
|
if object_size < max_object_size:
|
||||||
|
|
|
@ -45,6 +45,8 @@ STORAGE_NODE_SERVICE_NAME_REGEX = r"s\d\d"
|
||||||
HTTP_GATE_SERVICE_NAME_REGEX = r"http-gate\d\d"
|
HTTP_GATE_SERVICE_NAME_REGEX = r"http-gate\d\d"
|
||||||
S3_GATE_SERVICE_NAME_REGEX = r"s3-gate\d\d"
|
S3_GATE_SERVICE_NAME_REGEX = r"s3-gate\d\d"
|
||||||
|
|
||||||
|
CLI_DEFAULT_TIMEOUT = os.getenv("CLI_DEFAULT_TIMEOUT", None)
|
||||||
|
|
||||||
# Generate wallet configs
|
# Generate wallet configs
|
||||||
# TODO: we should move all info about wallet configs to fixtures
|
# TODO: we should move all info about wallet configs to fixtures
|
||||||
WALLET_CONFIG = os.path.join(os.getcwd(), "wallet_config.yml")
|
WALLET_CONFIG = os.path.join(os.getcwd(), "wallet_config.yml")
|
||||||
|
|
|
@ -8,14 +8,14 @@ from typing import Any, Optional
|
||||||
import allure
|
import allure
|
||||||
import boto3
|
import boto3
|
||||||
import pytest
|
import pytest
|
||||||
import s3_gate_bucket
|
|
||||||
import s3_gate_object
|
|
||||||
import urllib3
|
import urllib3
|
||||||
from botocore.config import Config
|
from botocore.config import Config
|
||||||
from botocore.exceptions import ClientError
|
from botocore.exceptions import ClientError
|
||||||
from frostfs_testlib.shell import Shell
|
from frostfs_testlib.shell import Shell
|
||||||
from pytest import FixtureRequest
|
from pytest import FixtureRequest
|
||||||
|
|
||||||
|
from pytest_tests.steps import s3_gate_bucket
|
||||||
|
from pytest_tests.steps import s3_gate_object
|
||||||
from pytest_tests.helpers.aws_cli_client import AwsCliClient
|
from pytest_tests.helpers.aws_cli_client import AwsCliClient
|
||||||
from pytest_tests.helpers.cli_helpers import _cmd_run, _configure_aws_cli, _run_with_passwd
|
from pytest_tests.helpers.cli_helpers import _cmd_run, _configure_aws_cli, _run_with_passwd
|
||||||
from pytest_tests.helpers.cluster import Cluster
|
from pytest_tests.helpers.cluster import Cluster
|
||||||
|
|
|
@ -8,10 +8,10 @@ import allure
|
||||||
import pytest
|
import pytest
|
||||||
import urllib3
|
import urllib3
|
||||||
from botocore.exceptions import ClientError
|
from botocore.exceptions import ClientError
|
||||||
from s3_gate_bucket import S3_SYNC_WAIT_TIME
|
|
||||||
|
|
||||||
from pytest_tests.helpers.aws_cli_client import AwsCliClient
|
from pytest_tests.helpers.aws_cli_client import AwsCliClient
|
||||||
from pytest_tests.helpers.cli_helpers import log_command_execution
|
from pytest_tests.helpers.cli_helpers import log_command_execution
|
||||||
|
from pytest_tests.steps.s3_gate_bucket import S3_SYNC_WAIT_TIME
|
||||||
|
|
||||||
##########################################################
|
##########################################################
|
||||||
# Disabling warnings on self-signed certificate which the
|
# Disabling warnings on self-signed certificate which the
|
||||||
|
|
|
@ -1,6 +1,5 @@
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import re
|
|
||||||
import shutil
|
import shutil
|
||||||
import uuid
|
import uuid
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
@ -44,11 +43,20 @@ from pytest_tests.steps.load import get_services_endpoints, prepare_k6_instances
|
||||||
logger = logging.getLogger("NeoLogger")
|
logger = logging.getLogger("NeoLogger")
|
||||||
|
|
||||||
|
|
||||||
|
# Add logs check test even if it's not fit to mark selectors
|
||||||
|
def pytest_configure(config: pytest.Config):
|
||||||
|
markers = config.option.markexpr
|
||||||
|
if markers != "":
|
||||||
|
config.option.markexpr = f"logs_after_session or ({markers})"
|
||||||
|
|
||||||
|
|
||||||
|
# pytest hook. Do not rename
|
||||||
def pytest_collection_modifyitems(items):
|
def pytest_collection_modifyitems(items):
|
||||||
# Make network tests last based on @pytest.mark.node_mgmt
|
# Make network tests last based on @pytest.mark.node_mgmt and logs_test to be latest
|
||||||
def priority(item: pytest.Item) -> int:
|
def priority(item: pytest.Item) -> int:
|
||||||
is_node_mgmt_test = item.get_closest_marker("node_mgmt")
|
is_node_mgmt_test = 1 if item.get_closest_marker("node_mgmt") else 0
|
||||||
return 0 if not is_node_mgmt_test else 1
|
is_logs_check_test = 100 if item.get_closest_marker("logs_after_session") else 0
|
||||||
|
return is_node_mgmt_test + is_logs_check_test
|
||||||
|
|
||||||
items.sort(key=lambda item: priority(item))
|
items.sort(key=lambda item: priority(item))
|
||||||
|
|
||||||
|
@ -145,23 +153,16 @@ def temp_directory():
|
||||||
shutil.rmtree(full_path)
|
shutil.rmtree(full_path)
|
||||||
|
|
||||||
|
|
||||||
|
@allure.step("[Autouse/Session] Test session start time")
|
||||||
@pytest.fixture(scope="session", autouse=True)
|
@pytest.fixture(scope="session", autouse=True)
|
||||||
@allure.title("Collect logs")
|
def session_start_time():
|
||||||
def collect_logs(temp_directory, hosting: Hosting):
|
|
||||||
start_time = datetime.utcnow()
|
start_time = datetime.utcnow()
|
||||||
yield
|
return start_time
|
||||||
end_time = datetime.utcnow()
|
|
||||||
|
|
||||||
# Dump logs to temp directory (because they might be too large to keep in RAM)
|
|
||||||
logs_dir = os.path.join(temp_directory, "logs")
|
|
||||||
dump_logs(hosting, logs_dir, start_time, end_time)
|
|
||||||
attach_logs(logs_dir)
|
|
||||||
check_logs(logs_dir)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="session", autouse=True)
|
@pytest.fixture(scope="session", autouse=True)
|
||||||
@allure.title("Run health check for all storage nodes")
|
@allure.title("Run health check for all storage nodes")
|
||||||
def run_health_check(collect_logs, cluster: Cluster):
|
def run_health_check(session_start_time, cluster: Cluster):
|
||||||
failed_nodes = []
|
failed_nodes = []
|
||||||
for node in cluster.storage_nodes:
|
for node in cluster.storage_nodes:
|
||||||
health_check = storage_node_healthcheck(node)
|
health_check = storage_node_healthcheck(node)
|
||||||
|
@ -263,44 +264,3 @@ def default_wallet(client_shell: Shell, temp_directory: str, cluster: Cluster):
|
||||||
)
|
)
|
||||||
|
|
||||||
return wallet_path
|
return wallet_path
|
||||||
|
|
||||||
|
|
||||||
@allure.title("Check logs for OOM and PANIC entries in {logs_dir}")
|
|
||||||
def check_logs(logs_dir: str):
|
|
||||||
problem_pattern = r"\Wpanic\W|\Woom\W|\Wtoo many open files\W"
|
|
||||||
|
|
||||||
log_file_paths = []
|
|
||||||
for directory_path, _, file_names in os.walk(logs_dir):
|
|
||||||
log_file_paths += [
|
|
||||||
os.path.join(directory_path, file_name)
|
|
||||||
for file_name in file_names
|
|
||||||
if re.match(r"\.(txt|log)", os.path.splitext(file_name)[-1], flags=re.IGNORECASE)
|
|
||||||
]
|
|
||||||
|
|
||||||
logs_with_problem = []
|
|
||||||
for file_path in log_file_paths:
|
|
||||||
with allure.step(f"Check log file {file_path}"):
|
|
||||||
with open(file_path, "r") as log_file:
|
|
||||||
if re.search(problem_pattern, log_file.read(), flags=re.IGNORECASE):
|
|
||||||
logs_with_problem.append(file_path)
|
|
||||||
if logs_with_problem:
|
|
||||||
raise pytest.fail(f"System logs {', '.join(logs_with_problem)} contain critical errors")
|
|
||||||
|
|
||||||
|
|
||||||
def dump_logs(hosting: Hosting, logs_dir: str, since: datetime, until: datetime) -> None:
|
|
||||||
# Dump logs to temp directory (because they might be too large to keep in RAM)
|
|
||||||
os.makedirs(logs_dir)
|
|
||||||
|
|
||||||
for host in hosting.hosts:
|
|
||||||
with allure.step(f"Dump logs from host {host.config.address}"):
|
|
||||||
try:
|
|
||||||
host.dump_logs(logs_dir, since=since, until=until)
|
|
||||||
except Exception as ex:
|
|
||||||
logger.warning(f"Exception during logs collection: {ex}")
|
|
||||||
|
|
||||||
|
|
||||||
def attach_logs(logs_dir: str) -> None:
|
|
||||||
# Zip all files and attach to Allure because it is more convenient to download a single
|
|
||||||
# zip with all logs rather than mess with individual logs files per service or node
|
|
||||||
logs_zip_file_path = shutil.make_archive(logs_dir, "zip", logs_dir)
|
|
||||||
allure.attach.file(logs_zip_file_path, name="logs.zip", extension="zip")
|
|
||||||
|
|
|
@ -20,8 +20,8 @@ logger = logging.getLogger("NeoLogger")
|
||||||
stopped_nodes: list[StorageNode] = []
|
stopped_nodes: list[StorageNode] = []
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="function", autouse=True)
|
|
||||||
@allure.step("Return all stopped hosts")
|
@allure.step("Return all stopped hosts")
|
||||||
|
@pytest.fixture(scope="function", autouse=True)
|
||||||
def after_run_return_all_stopped_hosts(cluster: Cluster):
|
def after_run_return_all_stopped_hosts(cluster: Cluster):
|
||||||
yield
|
yield
|
||||||
return_stopped_hosts(cluster)
|
return_stopped_hosts(cluster)
|
||||||
|
|
|
@ -51,6 +51,7 @@ class Test_http_object(ClusterTestBase):
|
||||||
Expected result:
|
Expected result:
|
||||||
Hashes must be the same.
|
Hashes must be the same.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
with allure.step("Create public container"):
|
with allure.step("Create public container"):
|
||||||
cid = create_container(
|
cid = create_container(
|
||||||
self.wallet,
|
self.wallet,
|
||||||
|
|
|
@ -24,10 +24,10 @@ from pytest_tests.helpers.http_gate import (
|
||||||
from pytest_tests.steps.cluster_test_base import ClusterTestBase
|
from pytest_tests.steps.cluster_test_base import ClusterTestBase
|
||||||
|
|
||||||
logger = logging.getLogger("NeoLogger")
|
logger = logging.getLogger("NeoLogger")
|
||||||
EXPIRATION_TIMESTAMP_HEADER = "__FROSRFS__EXPIRATION_TIMESTAMP"
|
EXPIRATION_TIMESTAMP_HEADER = "__FROSTFS__EXPIRATION_TIMESTAMP"
|
||||||
EXPIRATION_EPOCH_HEADER = "__FROSRFS__EXPIRATION_EPOCH"
|
EXPIRATION_EPOCH_HEADER = "__FROSTFS__EXPIRATION_EPOCH"
|
||||||
EXPIRATION_DURATION_HEADER = "__FROSRFS__EXPIRATION_DURATION"
|
EXPIRATION_DURATION_HEADER = "__FROSTFS__EXPIRATION_DURATION"
|
||||||
EXPIRATION_EXPIRATION_RFC = "__FROSRFS__EXPIRATION_RFC3339"
|
EXPIRATION_EXPIRATION_RFC = "__FROSTFS__EXPIRATION_RFC3339"
|
||||||
FROSTFS_EXPIRATION_EPOCH = "Frostfs-Expiration-Epoch"
|
FROSTFS_EXPIRATION_EPOCH = "Frostfs-Expiration-Epoch"
|
||||||
FROSTFS_EXPIRATION_DURATION = "Frostfs-Expiration-Duration"
|
FROSTFS_EXPIRATION_DURATION = "Frostfs-Expiration-Duration"
|
||||||
FROSTFS_EXPIRATION_TIMESTAMP = "Frostfs-Expiration-Timestamp"
|
FROSTFS_EXPIRATION_TIMESTAMP = "Frostfs-Expiration-Timestamp"
|
||||||
|
|
|
@ -6,10 +6,16 @@ from random import choices, sample
|
||||||
|
|
||||||
import allure
|
import allure
|
||||||
import pytest
|
import pytest
|
||||||
from aws_cli_client import AwsCliClient
|
|
||||||
from common import ASSETS_DIR
|
from pytest_tests.helpers.aws_cli_client import AwsCliClient
|
||||||
from file_helper import concat_files, generate_file, generate_file_with_content, get_file_hash
|
from pytest_tests.helpers.file_helper import (
|
||||||
from s3_helper import (
|
concat_files,
|
||||||
|
generate_file,
|
||||||
|
generate_file_with_content,
|
||||||
|
get_file_hash,
|
||||||
|
)
|
||||||
|
from pytest_tests.helpers.payment_neogo import deposit_gas, transfer_gas
|
||||||
|
from pytest_tests.helpers.s3_helper import (
|
||||||
assert_object_lock_mode,
|
assert_object_lock_mode,
|
||||||
assert_s3_acl,
|
assert_s3_acl,
|
||||||
check_objects_in_bucket,
|
check_objects_in_bucket,
|
||||||
|
|
|
@ -187,7 +187,6 @@ class TestObjectStaticSession(ClusterTestBase):
|
||||||
)
|
)
|
||||||
|
|
||||||
@allure.title("Validate static session with range operations")
|
@allure.title("Validate static session with range operations")
|
||||||
@pytest.mark.static_session
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
"method_under_test,verb",
|
"method_under_test,verb",
|
||||||
[(get_range, ObjectVerb.RANGE), (get_range_hash, ObjectVerb.RANGEHASH)],
|
[(get_range, ObjectVerb.RANGE), (get_range_hash, ObjectVerb.RANGEHASH)],
|
||||||
|
@ -227,7 +226,6 @@ class TestObjectStaticSession(ClusterTestBase):
|
||||||
)
|
)
|
||||||
|
|
||||||
@allure.title("Validate static session with search operation")
|
@allure.title("Validate static session with search operation")
|
||||||
@pytest.mark.static_session
|
|
||||||
@pytest.mark.xfail
|
@pytest.mark.xfail
|
||||||
# (see https://github.com/nspcc-dev/neofs-node/issues/2030)
|
# (see https://github.com/nspcc-dev/neofs-node/issues/2030)
|
||||||
def test_static_session_search(
|
def test_static_session_search(
|
||||||
|
@ -255,7 +253,6 @@ class TestObjectStaticSession(ClusterTestBase):
|
||||||
assert expected_object_ids == actual_object_ids
|
assert expected_object_ids == actual_object_ids
|
||||||
|
|
||||||
@allure.title("Validate static session with object id not in session")
|
@allure.title("Validate static session with object id not in session")
|
||||||
@pytest.mark.static_session
|
|
||||||
def test_static_session_unrelated_object(
|
def test_static_session_unrelated_object(
|
||||||
self,
|
self,
|
||||||
user_wallet: WalletFile,
|
user_wallet: WalletFile,
|
||||||
|
@ -280,7 +277,6 @@ class TestObjectStaticSession(ClusterTestBase):
|
||||||
)
|
)
|
||||||
|
|
||||||
@allure.title("Validate static session with user id not in session")
|
@allure.title("Validate static session with user id not in session")
|
||||||
@pytest.mark.static_session
|
|
||||||
def test_static_session_head_unrelated_user(
|
def test_static_session_head_unrelated_user(
|
||||||
self,
|
self,
|
||||||
stranger_wallet: WalletFile,
|
stranger_wallet: WalletFile,
|
||||||
|
@ -307,7 +303,6 @@ class TestObjectStaticSession(ClusterTestBase):
|
||||||
)
|
)
|
||||||
|
|
||||||
@allure.title("Validate static session with wrong verb in session")
|
@allure.title("Validate static session with wrong verb in session")
|
||||||
@pytest.mark.static_session
|
|
||||||
def test_static_session_head_wrong_verb(
|
def test_static_session_head_wrong_verb(
|
||||||
self,
|
self,
|
||||||
user_wallet: WalletFile,
|
user_wallet: WalletFile,
|
||||||
|
@ -334,7 +329,6 @@ class TestObjectStaticSession(ClusterTestBase):
|
||||||
)
|
)
|
||||||
|
|
||||||
@allure.title("Validate static session with container id not in session")
|
@allure.title("Validate static session with container id not in session")
|
||||||
@pytest.mark.static_session
|
|
||||||
def test_static_session_unrelated_container(
|
def test_static_session_unrelated_container(
|
||||||
self,
|
self,
|
||||||
user_wallet: WalletFile,
|
user_wallet: WalletFile,
|
||||||
|
@ -362,7 +356,6 @@ class TestObjectStaticSession(ClusterTestBase):
|
||||||
)
|
)
|
||||||
|
|
||||||
@allure.title("Validate static session which signed by another wallet")
|
@allure.title("Validate static session which signed by another wallet")
|
||||||
@pytest.mark.static_session
|
|
||||||
def test_static_session_signed_by_other(
|
def test_static_session_signed_by_other(
|
||||||
self,
|
self,
|
||||||
owner_wallet: WalletFile,
|
owner_wallet: WalletFile,
|
||||||
|
@ -401,7 +394,6 @@ class TestObjectStaticSession(ClusterTestBase):
|
||||||
)
|
)
|
||||||
|
|
||||||
@allure.title("Validate static session which signed for another container")
|
@allure.title("Validate static session which signed for another container")
|
||||||
@pytest.mark.static_session
|
|
||||||
def test_static_session_signed_for_other_container(
|
def test_static_session_signed_for_other_container(
|
||||||
self,
|
self,
|
||||||
owner_wallet: WalletFile,
|
owner_wallet: WalletFile,
|
||||||
|
@ -440,7 +432,6 @@ class TestObjectStaticSession(ClusterTestBase):
|
||||||
)
|
)
|
||||||
|
|
||||||
@allure.title("Validate static session which wasn't signed")
|
@allure.title("Validate static session which wasn't signed")
|
||||||
@pytest.mark.static_session
|
|
||||||
def test_static_session_without_sign(
|
def test_static_session_without_sign(
|
||||||
self,
|
self,
|
||||||
owner_wallet: WalletFile,
|
owner_wallet: WalletFile,
|
||||||
|
@ -477,7 +468,6 @@ class TestObjectStaticSession(ClusterTestBase):
|
||||||
)
|
)
|
||||||
|
|
||||||
@allure.title("Validate static session which expires at next epoch")
|
@allure.title("Validate static session which expires at next epoch")
|
||||||
@pytest.mark.static_session
|
|
||||||
def test_static_session_expiration_at_next(
|
def test_static_session_expiration_at_next(
|
||||||
self,
|
self,
|
||||||
owner_wallet: WalletFile,
|
owner_wallet: WalletFile,
|
||||||
|
@ -499,40 +489,56 @@ class TestObjectStaticSession(ClusterTestBase):
|
||||||
object_id = storage_objects[0].oid
|
object_id = storage_objects[0].oid
|
||||||
expiration = Lifetime(epoch + 1, epoch, epoch)
|
expiration = Lifetime(epoch + 1, epoch, epoch)
|
||||||
|
|
||||||
token_expire_at_next_epoch = get_object_signed_token(
|
with allure.step("Create session token"):
|
||||||
owner_wallet,
|
token_expire_at_next_epoch = get_object_signed_token(
|
||||||
user_wallet,
|
owner_wallet,
|
||||||
container,
|
user_wallet,
|
||||||
storage_objects,
|
|
||||||
ObjectVerb.HEAD,
|
|
||||||
self.shell,
|
|
||||||
temp_directory,
|
|
||||||
expiration,
|
|
||||||
)
|
|
||||||
|
|
||||||
head_object(
|
|
||||||
user_wallet.path,
|
|
||||||
container,
|
|
||||||
object_id,
|
|
||||||
self.shell,
|
|
||||||
self.cluster.default_rpc_endpoint,
|
|
||||||
session=token_expire_at_next_epoch,
|
|
||||||
)
|
|
||||||
|
|
||||||
self.tick_epoch()
|
|
||||||
|
|
||||||
with pytest.raises(Exception, match=EXPIRED_SESSION_TOKEN):
|
|
||||||
head_object(
|
|
||||||
user_wallet.path,
|
|
||||||
container,
|
container,
|
||||||
object_id,
|
storage_objects,
|
||||||
|
ObjectVerb.HEAD,
|
||||||
self.shell,
|
self.shell,
|
||||||
self.cluster.default_rpc_endpoint,
|
temp_directory,
|
||||||
session=token_expire_at_next_epoch,
|
expiration,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
with allure.step("Object should be available with session token after token creation"):
|
||||||
|
with expect_not_raises():
|
||||||
|
head_object(
|
||||||
|
user_wallet.path,
|
||||||
|
container,
|
||||||
|
object_id,
|
||||||
|
self.shell,
|
||||||
|
self.cluster.default_rpc_endpoint,
|
||||||
|
session=token_expire_at_next_epoch,
|
||||||
|
)
|
||||||
|
|
||||||
|
with allure.step(
|
||||||
|
"Object should be available at last epoch before session token expiration"
|
||||||
|
):
|
||||||
|
self.tick_epoch()
|
||||||
|
with expect_not_raises():
|
||||||
|
head_object(
|
||||||
|
user_wallet.path,
|
||||||
|
container,
|
||||||
|
object_id,
|
||||||
|
self.shell,
|
||||||
|
self.cluster.default_rpc_endpoint,
|
||||||
|
session=token_expire_at_next_epoch,
|
||||||
|
)
|
||||||
|
|
||||||
|
with allure.step("Object should NOT be available after session token expiration epoch"):
|
||||||
|
self.tick_epoch()
|
||||||
|
with pytest.raises(Exception, match=EXPIRED_SESSION_TOKEN):
|
||||||
|
head_object(
|
||||||
|
user_wallet.path,
|
||||||
|
container,
|
||||||
|
object_id,
|
||||||
|
self.shell,
|
||||||
|
self.cluster.default_rpc_endpoint,
|
||||||
|
session=token_expire_at_next_epoch,
|
||||||
|
)
|
||||||
|
|
||||||
@allure.title("Validate static session which is valid starting from next epoch")
|
@allure.title("Validate static session which is valid starting from next epoch")
|
||||||
@pytest.mark.static_session
|
|
||||||
def test_static_session_start_at_next(
|
def test_static_session_start_at_next(
|
||||||
self,
|
self,
|
||||||
owner_wallet: WalletFile,
|
owner_wallet: WalletFile,
|
||||||
|
@ -554,50 +560,70 @@ class TestObjectStaticSession(ClusterTestBase):
|
||||||
object_id = storage_objects[0].oid
|
object_id = storage_objects[0].oid
|
||||||
expiration = Lifetime(epoch + 2, epoch + 1, epoch)
|
expiration = Lifetime(epoch + 2, epoch + 1, epoch)
|
||||||
|
|
||||||
token_start_at_next_epoch = get_object_signed_token(
|
with allure.step("Create session token"):
|
||||||
owner_wallet,
|
token_start_at_next_epoch = get_object_signed_token(
|
||||||
user_wallet,
|
owner_wallet,
|
||||||
container,
|
user_wallet,
|
||||||
storage_objects,
|
|
||||||
ObjectVerb.HEAD,
|
|
||||||
self.shell,
|
|
||||||
temp_directory,
|
|
||||||
expiration,
|
|
||||||
)
|
|
||||||
|
|
||||||
with pytest.raises(Exception, match=MALFORMED_REQUEST):
|
|
||||||
head_object(
|
|
||||||
user_wallet.path,
|
|
||||||
container,
|
container,
|
||||||
object_id,
|
storage_objects,
|
||||||
|
ObjectVerb.HEAD,
|
||||||
self.shell,
|
self.shell,
|
||||||
self.cluster.default_rpc_endpoint,
|
temp_directory,
|
||||||
session=token_start_at_next_epoch,
|
expiration,
|
||||||
)
|
)
|
||||||
|
|
||||||
self.tick_epoch()
|
with allure.step("Object should NOT be available with session token after token creation"):
|
||||||
head_object(
|
with pytest.raises(Exception, match=MALFORMED_REQUEST):
|
||||||
user_wallet.path,
|
head_object(
|
||||||
container,
|
user_wallet.path,
|
||||||
object_id,
|
container,
|
||||||
self.shell,
|
object_id,
|
||||||
self.cluster.default_rpc_endpoint,
|
self.shell,
|
||||||
session=token_start_at_next_epoch,
|
self.cluster.default_rpc_endpoint,
|
||||||
)
|
session=token_start_at_next_epoch,
|
||||||
|
)
|
||||||
|
|
||||||
self.tick_epoch()
|
with allure.step(
|
||||||
with pytest.raises(Exception, match=EXPIRED_SESSION_TOKEN):
|
"Object should be available with session token starting from token nbf epoch"
|
||||||
head_object(
|
):
|
||||||
user_wallet.path,
|
self.tick_epoch()
|
||||||
container,
|
with expect_not_raises():
|
||||||
object_id,
|
head_object(
|
||||||
self.shell,
|
user_wallet.path,
|
||||||
self.cluster.default_rpc_endpoint,
|
container,
|
||||||
session=token_start_at_next_epoch,
|
object_id,
|
||||||
)
|
self.shell,
|
||||||
|
self.cluster.default_rpc_endpoint,
|
||||||
|
session=token_start_at_next_epoch,
|
||||||
|
)
|
||||||
|
|
||||||
|
with allure.step(
|
||||||
|
"Object should be available at last epoch before session token expiration"
|
||||||
|
):
|
||||||
|
self.tick_epoch()
|
||||||
|
with expect_not_raises():
|
||||||
|
head_object(
|
||||||
|
user_wallet.path,
|
||||||
|
container,
|
||||||
|
object_id,
|
||||||
|
self.shell,
|
||||||
|
self.cluster.default_rpc_endpoint,
|
||||||
|
session=token_start_at_next_epoch,
|
||||||
|
)
|
||||||
|
|
||||||
|
with allure.step("Object should NOT be available after session token expiration epoch"):
|
||||||
|
self.tick_epoch()
|
||||||
|
with pytest.raises(Exception, match=EXPIRED_SESSION_TOKEN):
|
||||||
|
head_object(
|
||||||
|
user_wallet.path,
|
||||||
|
container,
|
||||||
|
object_id,
|
||||||
|
self.shell,
|
||||||
|
self.cluster.default_rpc_endpoint,
|
||||||
|
session=token_start_at_next_epoch,
|
||||||
|
)
|
||||||
|
|
||||||
@allure.title("Validate static session which is already expired")
|
@allure.title("Validate static session which is already expired")
|
||||||
@pytest.mark.static_session
|
|
||||||
def test_static_session_already_expired(
|
def test_static_session_already_expired(
|
||||||
self,
|
self,
|
||||||
owner_wallet: WalletFile,
|
owner_wallet: WalletFile,
|
||||||
|
@ -691,7 +717,6 @@ class TestObjectStaticSession(ClusterTestBase):
|
||||||
)
|
)
|
||||||
|
|
||||||
@allure.title("Validate static session which is issued in future epoch")
|
@allure.title("Validate static session which is issued in future epoch")
|
||||||
@pytest.mark.static_session
|
|
||||||
def test_static_session_invalid_issued_epoch(
|
def test_static_session_invalid_issued_epoch(
|
||||||
self,
|
self,
|
||||||
owner_wallet: WalletFile,
|
owner_wallet: WalletFile,
|
||||||
|
|
|
@ -11,7 +11,7 @@ from configobj import ConfigObj
|
||||||
from frostfs_testlib.cli import FrostfsCli
|
from frostfs_testlib.cli import FrostfsCli
|
||||||
|
|
||||||
from pytest_tests.helpers.cluster import Cluster, StorageNode
|
from pytest_tests.helpers.cluster import Cluster, StorageNode
|
||||||
from pytest_tests.resources.common import WALLET_CONFIG
|
from pytest_tests.resources.common import CLI_DEFAULT_TIMEOUT, WALLET_CONFIG
|
||||||
|
|
||||||
SHARD_PREFIX = "FROSTFS_STORAGE_SHARD_"
|
SHARD_PREFIX = "FROSTFS_STORAGE_SHARD_"
|
||||||
BLOBSTOR_PREFIX = "_BLOBSTOR_"
|
BLOBSTOR_PREFIX = "_BLOBSTOR_"
|
||||||
|
@ -143,6 +143,7 @@ class TestControlShard:
|
||||||
wallet=wallet_path,
|
wallet=wallet_path,
|
||||||
wallet_password=wallet_password,
|
wallet_password=wallet_password,
|
||||||
json_mode=True,
|
json_mode=True,
|
||||||
|
timeout=CLI_DEFAULT_TIMEOUT,
|
||||||
)
|
)
|
||||||
return [Shard.from_object(shard) for shard in json.loads(result.stdout.split(">", 1)[1])]
|
return [Shard.from_object(shard) for shard in json.loads(result.stdout.split(">", 1)[1])]
|
||||||
|
|
||||||
|
|
47
pytest_tests/testsuites/special/test_logs.py
Normal file
47
pytest_tests/testsuites/special/test_logs.py
Normal file
|
@ -0,0 +1,47 @@
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
import allure
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from pytest_tests.steps.cluster_test_base import ClusterTestBase
|
||||||
|
|
||||||
|
|
||||||
|
class TestLogs(ClusterTestBase):
|
||||||
|
@pytest.mark.logs_after_session
|
||||||
|
def test_logs_after_session(self, temp_directory: str, session_start_time: datetime):
|
||||||
|
"""
|
||||||
|
This test automatically added to any test run to check logs from cluster for critical errors.
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
end_time = datetime.utcnow()
|
||||||
|
logs_dir = os.path.join(temp_directory, "logs")
|
||||||
|
os.makedirs(logs_dir)
|
||||||
|
issues_regex = r"\Wpanic\W|\Woom\W|\Wtoo many open files\W"
|
||||||
|
|
||||||
|
hosts_with_problems = []
|
||||||
|
for host in self.cluster.hosts:
|
||||||
|
with allure.step(f"Check logs on {host.config.address}"):
|
||||||
|
if host.is_message_in_logs(issues_regex, session_start_time, end_time):
|
||||||
|
hosts_with_problems.append(host.config.address)
|
||||||
|
host.dump_logs(
|
||||||
|
logs_dir,
|
||||||
|
since=session_start_time,
|
||||||
|
until=end_time,
|
||||||
|
filter_regex=issues_regex,
|
||||||
|
)
|
||||||
|
|
||||||
|
if hosts_with_problems:
|
||||||
|
self._attach_logs(logs_dir)
|
||||||
|
|
||||||
|
assert (
|
||||||
|
not hosts_with_problems
|
||||||
|
), f"The following hosts contains contain critical errors in system logs: {', '.join(hosts_with_problems)}"
|
||||||
|
|
||||||
|
def _attach_logs(self, logs_dir: str) -> None:
|
||||||
|
# Zip all files and attach to Allure because it is more convenient to download a single
|
||||||
|
# zip with all logs rather than mess with individual logs files per service or node
|
||||||
|
logs_zip_file_path = shutil.make_archive(logs_dir, "zip", logs_dir)
|
||||||
|
allure.attach.file(logs_zip_file_path, name="logs.zip", extension="zip")
|
Loading…
Reference in a new issue