From 52001dc23a9b5c5eb0c6bc5097db744965641962 Mon Sep 17 00:00:00 2001 From: Aleksei Chetaev Date: Mon, 27 Feb 2023 17:54:27 +0100 Subject: [PATCH] Change all imports to imports from root and remove robot Signed-off-by: Aleksei Chetaev --- .gitignore | 2 +- README.md | 14 +- pyproject.toml | 4 +- .../__init__.py | 0 .../helpers}/acl.py | 3 +- pytest_tests/helpers/aws_cli_client.py | 5 +- ...ry_version_helper.py => binary_version.py} | 3 +- .../helpers}/cli_helpers.py | 0 .../helpers}/complex_object_actions.py | 11 +- pytest_tests/helpers/container.py | 241 +++++++++++++++++- .../helpers}/container_access.py | 7 +- .../helpers}/epoch.py | 15 +- .../helpers}/failover_utils.py | 7 +- pytest_tests/helpers/file_helper.py | 3 +- .../helpers}/frostfs_verbs.py | 5 +- .../helpers}/http_gate.py | 14 +- .../helpers}/node_management.py | 7 +- .../helpers}/object_access.py | 11 +- .../helpers}/payment_neogo.py | 10 +- pytest_tests/helpers/s3_helper.py | 5 +- .../helpers}/storage_group.py | 9 +- .../helpers}/storage_policy.py | 6 +- .../helpers}/tombstone.py | 3 +- pytest_tests/helpers/utility.py | 5 +- pytest_tests/helpers/wallet.py | 7 +- .../resources}/common.py | 2 +- .../resources/files/policy.json | 0 .../resources/files/s3_bearer_rules.json | 0 pytest_tests/steps/cluster_test_base.py | 5 +- pytest_tests/steps/load.py | 5 +- pytest_tests/steps/s3_gate_base.py | 17 +- pytest_tests/steps/s3_gate_bucket.py | 3 +- pytest_tests/steps/s3_gate_object.py | 5 +- pytest_tests/steps/session_token.py | 7 +- pytest_tests/steps/storage_object.py | 11 +- pytest_tests/testsuites/acl/conftest.py | 13 +- .../acl/storage_group/test_storagegroup.py | 17 +- pytest_tests/testsuites/acl/test_acl.py | 11 +- pytest_tests/testsuites/acl/test_bearer.py | 7 +- pytest_tests/testsuites/acl/test_eacl.py | 17 +- .../testsuites/acl/test_eacl_filters.py | 13 +- pytest_tests/testsuites/conftest.py | 37 ++- .../testsuites/container/test_container.py | 8 +- .../failovers/test_failover_network.py | 17 +- .../failovers/test_failover_storage.py | 15 +- pytest_tests/testsuites/load/test_load.py | 29 ++- .../network/test_node_management.py | 28 +- .../testsuites/object/test_object_api.py | 20 +- .../object/test_object_api_bearer.py | 31 ++- .../testsuites/object/test_object_lifetime.py | 15 +- .../testsuites/object/test_object_lock.py | 29 +-- .../testsuites/payment/test_balance.py | 7 +- .../services/http_gate/test_http_bearer.py | 12 +- .../services/http_gate/test_http_gate.py | 16 +- .../services/http_gate/test_http_headers.py | 18 +- .../services/http_gate/test_http_object.py | 12 +- .../services/http_gate/test_http_streaming.py | 8 +- .../http_gate/test_http_system_header.py | 22 +- .../services/s3_gate/test_s3_ACL.py | 8 +- .../services/s3_gate/test_s3_bucket.py | 10 +- .../services/s3_gate/test_s3_gate.py | 16 +- .../services/s3_gate/test_s3_locking.py | 12 +- .../services/s3_gate/test_s3_multipart.py | 12 +- .../services/s3_gate/test_s3_object.py | 21 +- .../services/s3_gate/test_s3_policy.py | 20 +- .../services/s3_gate/test_s3_tagging.py | 14 +- .../services/s3_gate/test_s3_versioning.py | 8 +- .../testsuites/services/test_binaries.py | 7 +- .../testsuites/session_token/conftest.py | 3 +- .../test_object_session_token.py | 12 +- .../test_static_object_session_token.py | 24 +- .../test_static_session_token_container.py | 16 +- .../testsuites/shard/test_control_shard.py | 5 +- pytest_tests/wallet_pass.yml | 1 - requirements.txt | 1 - .../lib/python_keywords/container.py | 239 ----------------- robot/variables/eacl_object_filters.py | 23 -- venv/local-pytest/environment.sh | 4 +- 78 files changed, 674 insertions(+), 636 deletions(-) rename {robot/resources/lib/python_keywords => pytest_tests}/__init__.py (100%) rename {robot/resources/lib/python_keywords => pytest_tests/helpers}/acl.py (98%) rename pytest_tests/helpers/{binary_version_helper.py => binary_version.py} (96%) rename {robot/resources/lib/python_keywords => pytest_tests/helpers}/cli_helpers.py (100%) rename {robot/resources/lib/python_keywords => pytest_tests/helpers}/complex_object_actions.py (95%) rename {robot/resources/lib/python_keywords => pytest_tests/helpers}/container_access.py (97%) rename {robot/resources/lib/python_keywords => pytest_tests/helpers}/epoch.py (95%) rename {robot/resources/lib/python_keywords => pytest_tests/helpers}/failover_utils.py (89%) rename {robot/resources/lib/python_keywords => pytest_tests/helpers}/frostfs_verbs.py (99%) rename {robot/resources/lib/python_keywords => pytest_tests/helpers}/http_gate.py (96%) rename {robot/resources/lib/python_keywords => pytest_tests/helpers}/node_management.py (97%) rename {robot/resources/lib/python_keywords => pytest_tests/helpers}/object_access.py (95%) rename {robot/resources/lib/python_keywords => pytest_tests/helpers}/payment_neogo.py (97%) rename {robot/resources/lib/python_keywords => pytest_tests/helpers}/storage_group.py (96%) rename {robot/resources/lib/python_keywords => pytest_tests/helpers}/storage_policy.py (97%) rename {robot/resources/lib/python_keywords => pytest_tests/helpers}/tombstone.py (95%) rename {robot/variables => pytest_tests/resources}/common.py (95%) rename {robot => pytest_tests}/resources/files/policy.json (100%) rename {robot => pytest_tests}/resources/files/s3_bearer_rules.json (100%) delete mode 100644 pytest_tests/wallet_pass.yml delete mode 100644 robot/resources/lib/python_keywords/container.py delete mode 100644 robot/variables/eacl_object_filters.py diff --git a/.gitignore b/.gitignore index 6efdff25..f955749a 100644 --- a/.gitignore +++ b/.gitignore @@ -26,4 +26,4 @@ TemporaryDir/* artifacts/* docs/* venv.*/* -/*wallet_config.yml +wallet_config.yml diff --git a/README.md b/README.md index a565d43e..9f9fe8a3 100644 --- a/README.md +++ b/README.md @@ -2,13 +2,7 @@ Tests written with PyTest Framework are located under `pytest_tests/testsuites` directory. -These tests rely on resources and utility modules that have been originally developed for Robot Framework: - -`robot/resources/files` - static files that are used in tests' commands. - -`robot/resources/lib/` - common Python libraries that provide utility functions used as building blocks in tests. - -`robot/variables/` - constants and configuration variables for tests. +These tests rely on resources and utility modules that have been originally developed for Pytest Framework. ## Testcases execution @@ -41,12 +35,12 @@ Note that we expect frostfs-dev-env to be located under the `/../frostfs-dev-env` directory. If you put this repo in any other place, manually set the full path to frostfs-dev-env in the environment variable `DEVENV_PATH` at this step. -5. Make sure you have installed all of the following prerequisites on your machine +5. Make sure you have installed all the following prerequisites on your machine ``` make -python3.9 -python3.9-dev +python3.10 +python3.10-dev libssl-dev ``` As we use frostfs-dev-env, you'll also need to install diff --git a/pyproject.toml b/pyproject.toml index c8f7731d..9897c3fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,8 +1,8 @@ [tool.isort] profile = "black" -src_paths = ["pytest_tests", "robot"] +src_paths = ["pytest_tests"] line_length = 100 [tool.black] line-length = 100 -target-version = ["py39"] +target-version = ["py310"] diff --git a/robot/resources/lib/python_keywords/__init__.py b/pytest_tests/__init__.py similarity index 100% rename from robot/resources/lib/python_keywords/__init__.py rename to pytest_tests/__init__.py diff --git a/robot/resources/lib/python_keywords/acl.py b/pytest_tests/helpers/acl.py similarity index 98% rename from robot/resources/lib/python_keywords/acl.py rename to pytest_tests/helpers/acl.py index 74ae6517..980e4765 100644 --- a/robot/resources/lib/python_keywords/acl.py +++ b/pytest_tests/helpers/acl.py @@ -10,11 +10,12 @@ from typing import Any, Dict, List, Optional, Union import allure import base58 -from common import ASSETS_DIR, FROSTFS_CLI_EXEC, WALLET_CONFIG from frostfs_testlib.cli import FrostfsCli from frostfs_testlib.shell import Shell from frostfs_testlib.utils import wallet_utils +from pytest_tests.resources.common import ASSETS_DIR, FROSTFS_CLI_EXEC, WALLET_CONFIG + logger = logging.getLogger("NeoLogger") EACL_LIFETIME = 100500 FROSTFS_CONTRACT_CACHE_TIMEOUT = 30 diff --git a/pytest_tests/helpers/aws_cli_client.py b/pytest_tests/helpers/aws_cli_client.py index 213a383c..36d067e6 100644 --- a/pytest_tests/helpers/aws_cli_client.py +++ b/pytest_tests/helpers/aws_cli_client.py @@ -5,8 +5,9 @@ from datetime import datetime from typing import Optional import allure -from cli_helpers import _cmd_run -from common import ASSETS_DIR + +from pytest_tests.helpers.cli_helpers import _cmd_run +from pytest_tests.resources.common import ASSETS_DIR logger = logging.getLogger("NeoLogger") REGULAR_TIMEOUT = 90 diff --git a/pytest_tests/helpers/binary_version_helper.py b/pytest_tests/helpers/binary_version.py similarity index 96% rename from pytest_tests/helpers/binary_version_helper.py rename to pytest_tests/helpers/binary_version.py index d3f15a3d..485f7946 100644 --- a/pytest_tests/helpers/binary_version_helper.py +++ b/pytest_tests/helpers/binary_version.py @@ -1,11 +1,12 @@ import logging import re -from common import FROSTFS_ADM_EXEC, FROSTFS_CLI_EXEC, WALLET_CONFIG from frostfs_testlib.cli import FrostfsAdm, FrostfsCli from frostfs_testlib.hosting import Hosting from frostfs_testlib.shell import Shell +from pytest_tests.resources.common import FROSTFS_ADM_EXEC, FROSTFS_CLI_EXEC, WALLET_CONFIG + logger = logging.getLogger("NeoLogger") diff --git a/robot/resources/lib/python_keywords/cli_helpers.py b/pytest_tests/helpers/cli_helpers.py similarity index 100% rename from robot/resources/lib/python_keywords/cli_helpers.py rename to pytest_tests/helpers/cli_helpers.py diff --git a/robot/resources/lib/python_keywords/complex_object_actions.py b/pytest_tests/helpers/complex_object_actions.py similarity index 95% rename from robot/resources/lib/python_keywords/complex_object_actions.py rename to pytest_tests/helpers/complex_object_actions.py index bf1fe6b2..5dc58405 100644 --- a/robot/resources/lib/python_keywords/complex_object_actions.py +++ b/pytest_tests/helpers/complex_object_actions.py @@ -14,12 +14,13 @@ import logging from typing import Optional, Tuple import allure -import frostfs_verbs -from cluster import Cluster, StorageNode -from common import WALLET_CONFIG from frostfs_testlib.shell import Shell -from frostfs_verbs import head_object -from storage_object import StorageObjectInfo + +from pytest_tests.helpers import frostfs_verbs +from pytest_tests.helpers.cluster import Cluster, StorageNode +from pytest_tests.helpers.frostfs_verbs import head_object +from pytest_tests.helpers.storage_object_info import StorageObjectInfo +from pytest_tests.resources.common import WALLET_CONFIG logger = logging.getLogger("NeoLogger") diff --git a/pytest_tests/helpers/container.py b/pytest_tests/helpers/container.py index 3b1b49ab..264eea8f 100644 --- a/pytest_tests/helpers/container.py +++ b/pytest_tests/helpers/container.py @@ -1,14 +1,23 @@ +import json +import logging from dataclasses import dataclass -from typing import Optional +from time import sleep +from typing import Optional, Union import allure -from cluster import Cluster -from file_helper import generate_file, get_file_hash +from frostfs_testlib.cli import FrostfsCli from frostfs_testlib.shell import Shell -from frostfs_verbs import put_object, put_object_to_random_node -from storage_object import StorageObjectInfo +from frostfs_testlib.utils import json_utils from wallet import WalletFile +from pytest_tests.helpers.cluster import Cluster +from pytest_tests.helpers.file_helper import generate_file, get_file_hash +from pytest_tests.helpers.frostfs_verbs import put_object, put_object_to_random_node +from pytest_tests.helpers.storage_object_info import StorageObjectInfo +from pytest_tests.resources.common import FROSTFS_CLI_EXEC, WALLET_CONFIG + +logger = logging.getLogger("NeoLogger") + @dataclass class StorageContainerInfo: @@ -85,3 +94,225 @@ class StorageContainer: ) return storage_object + + +DEFAULT_PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 4 FROM * AS X" +SINGLE_PLACEMENT_RULE = "REP 1 IN X CBF 1 SELECT 4 FROM * AS X" +REP_2_FOR_3_NODES_PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 3 FROM * AS X" + + +@allure.step("Create Container") +def create_container( + wallet: str, + shell: Shell, + endpoint: str, + rule: str = DEFAULT_PLACEMENT_RULE, + basic_acl: str = "", + attributes: Optional[dict] = None, + session_token: str = "", + session_wallet: str = "", + name: str = None, + options: dict = None, + await_mode: bool = True, + wait_for_creation: bool = True, +) -> str: + """ + A wrapper for `frostfs-cli container create` call. + + Args: + wallet (str): a wallet on whose behalf a container is created + rule (optional, str): placement rule for container + basic_acl (optional, str): an ACL for container, will be + appended to `--basic-acl` key + attributes (optional, dict): container attributes , will be + appended to `--attributes` key + session_token (optional, str): a path to session token file + session_wallet(optional, str): a path to the wallet which signed + the session token; this parameter makes sense + when paired with `session_token` + shell: executor for cli command + endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key + options (optional, dict): any other options to pass to the call + name (optional, str): container name attribute + await_mode (bool): block execution until container is persisted + wait_for_creation (): Wait for container shows in container list + + Returns: + (str): CID of the created container + """ + + cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, WALLET_CONFIG) + result = cli.container.create( + rpc_endpoint=endpoint, + wallet=session_wallet if session_wallet else wallet, + policy=rule, + basic_acl=basic_acl, + attributes=attributes, + name=name, + session=session_token, + await_mode=await_mode, + **options or {}, + ) + + cid = _parse_cid(result.stdout) + + logger.info("Container created; waiting until it is persisted in the sidechain") + + if wait_for_creation: + wait_for_container_creation(wallet, cid, shell, endpoint) + + return cid + + +def wait_for_container_creation( + wallet: str, cid: str, shell: Shell, endpoint: str, attempts: int = 15, sleep_interval: int = 1 +): + for _ in range(attempts): + containers = list_containers(wallet, shell, endpoint) + if cid in containers: + return + logger.info(f"There is no {cid} in {containers} yet; sleep {sleep_interval} and continue") + sleep(sleep_interval) + raise RuntimeError( + f"After {attempts * sleep_interval} seconds container {cid} hasn't been persisted; exiting" + ) + + +def wait_for_container_deletion( + wallet: str, cid: str, shell: Shell, endpoint: str, attempts: int = 30, sleep_interval: int = 1 +): + for _ in range(attempts): + try: + get_container(wallet, cid, shell=shell, endpoint=endpoint) + sleep(sleep_interval) + continue + except Exception as err: + if "container not found" not in str(err): + raise AssertionError(f'Expected "container not found" in error, got\n{err}') + return + raise AssertionError(f"Expected container deleted during {attempts * sleep_interval} sec.") + + +@allure.step("List Containers") +def list_containers(wallet: str, shell: Shell, endpoint: str) -> list[str]: + """ + A wrapper for `frostfs-cli container list` call. It returns all the + available containers for the given wallet. + Args: + wallet (str): a wallet on whose behalf we list the containers + shell: executor for cli command + endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key + Returns: + (list): list of containers + """ + cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, WALLET_CONFIG) + result = cli.container.list(rpc_endpoint=endpoint, wallet=wallet) + logger.info(f"Containers: \n{result}") + return result.stdout.split() + + +@allure.step("Get Container") +def get_container( + wallet: str, + cid: str, + shell: Shell, + endpoint: str, + json_mode: bool = True, +) -> Union[dict, str]: + """ + A wrapper for `frostfs-cli container get` call. It extracts container's + attributes and rearranges them into a more compact view. + Args: + wallet (str): path to a wallet on whose behalf we get the container + cid (str): ID of the container to get + shell: executor for cli command + endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key + json_mode (bool): return container in JSON format + Returns: + (dict, str): dict of container attributes + """ + + cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, WALLET_CONFIG) + result = cli.container.get(rpc_endpoint=endpoint, wallet=wallet, cid=cid, json_mode=json_mode) + + if not json_mode: + return result.stdout + + container_info = json.loads(result.stdout) + attributes = dict() + for attr in container_info["attributes"]: + attributes[attr["key"]] = attr["value"] + container_info["attributes"] = attributes + container_info["ownerID"] = json_utils.json_reencode(container_info["ownerID"]["value"]) + return container_info + + +@allure.step("Delete Container") +# TODO: make the error message about a non-found container more user-friendly +# https://github.com/nspcc-dev/frostfs-contract/issues/121 +def delete_container( + wallet: str, + cid: str, + shell: Shell, + endpoint: str, + force: bool = False, + session_token: Optional[str] = None, + await_mode: bool = False, +) -> None: + """ + A wrapper for `frostfs-cli container delete` call. + Args: + wallet (str): path to a wallet on whose behalf we delete the container + cid (str): ID of the container to delete + shell: executor for cli command + endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key + force (bool): do not check whether container contains locks and remove immediately + session_token: a path to session token file + This function doesn't return anything. + """ + + cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, WALLET_CONFIG) + cli.container.delete( + wallet=wallet, + cid=cid, + rpc_endpoint=endpoint, + force=force, + session=session_token, + await_mode=await_mode, + ) + + +def _parse_cid(output: str) -> str: + """ + Parses container ID from a given CLI output. The input string we expect: + container ID: 2tz86kVTDpJxWHrhw3h6PbKMwkLtBEwoqhHQCKTre1FN + awaiting... + container has been persisted on sidechain + We want to take 'container ID' value from the string. + + Args: + output (str): CLI output to parse + + Returns: + (str): extracted CID + """ + try: + # taking first line from command's output + first_line = output.split("\n")[0] + except Exception: + first_line = "" + logger.error(f"Got empty output: {output}") + splitted = first_line.split(": ") + if len(splitted) != 2: + raise ValueError(f"no CID was parsed from command output: \t{first_line}") + return splitted[1] + + +@allure.step("Search container by name") +def search_container_by_name(wallet: str, name: str, shell: Shell, endpoint: str): + list_cids = list_containers(wallet, shell, endpoint) + for cid in list_cids: + cont_info = get_container(wallet, cid, shell, endpoint, True) + if cont_info.get("attributes").get("Name", None) == name: + return cid + return None diff --git a/robot/resources/lib/python_keywords/container_access.py b/pytest_tests/helpers/container_access.py similarity index 97% rename from robot/resources/lib/python_keywords/container_access.py rename to pytest_tests/helpers/container_access.py index 67eb4c5b..9114a899 100644 --- a/robot/resources/lib/python_keywords/container_access.py +++ b/pytest_tests/helpers/container_access.py @@ -1,9 +1,10 @@ from typing import List, Optional -from acl import EACLOperation -from cluster import Cluster from frostfs_testlib.shell import Shell -from python_keywords.object_access import ( + +from pytest_tests.helpers.acl import EACLOperation +from pytest_tests.helpers.cluster import Cluster +from pytest_tests.helpers.object_access import ( can_delete_object, can_get_head_object, can_get_object, diff --git a/robot/resources/lib/python_keywords/epoch.py b/pytest_tests/helpers/epoch.py similarity index 95% rename from robot/resources/lib/python_keywords/epoch.py rename to pytest_tests/helpers/epoch.py index 6932661c..3c04c6fa 100644 --- a/robot/resources/lib/python_keywords/epoch.py +++ b/pytest_tests/helpers/epoch.py @@ -3,19 +3,20 @@ from time import sleep from typing import Optional import allure -from cluster import Cluster, StorageNode -from common import ( +from frostfs_testlib.cli import FrostfsAdm, FrostfsCli, NeoGo +from frostfs_testlib.shell import Shell +from frostfs_testlib.utils import datetime_utils, wallet_utils +from payment_neogo import get_contract_hash + +from pytest_tests.helpers.cluster import Cluster, StorageNode +from pytest_tests.helpers.test_control import wait_for_success +from pytest_tests.resources.common import ( FROSTFS_ADM_CONFIG_PATH, FROSTFS_ADM_EXEC, FROSTFS_CLI_EXEC, MAINNET_BLOCK_TIME, NEOGO_EXECUTABLE, ) -from frostfs_testlib.cli import FrostfsAdm, FrostfsCli, NeoGo -from frostfs_testlib.shell import Shell -from frostfs_testlib.utils import datetime_utils, wallet_utils -from payment_neogo import get_contract_hash -from test_control import wait_for_success logger = logging.getLogger("NeoLogger") diff --git a/robot/resources/lib/python_keywords/failover_utils.py b/pytest_tests/helpers/failover_utils.py similarity index 89% rename from robot/resources/lib/python_keywords/failover_utils.py rename to pytest_tests/helpers/failover_utils.py index 0f862280..a9b257b7 100644 --- a/robot/resources/lib/python_keywords/failover_utils.py +++ b/pytest_tests/helpers/failover_utils.py @@ -2,10 +2,11 @@ import logging from time import sleep import allure -from cluster import Cluster, StorageNode from frostfs_testlib.shell import Shell -from python_keywords.node_management import storage_node_healthcheck -from storage_policy import get_nodes_with_object + +from pytest_tests.helpers.cluster import Cluster, StorageNode +from pytest_tests.helpers.node_management import storage_node_healthcheck +from pytest_tests.helpers.storage_policy import get_nodes_with_object logger = logging.getLogger("NeoLogger") diff --git a/pytest_tests/helpers/file_helper.py b/pytest_tests/helpers/file_helper.py index 0a503895..c556a1dd 100644 --- a/pytest_tests/helpers/file_helper.py +++ b/pytest_tests/helpers/file_helper.py @@ -5,7 +5,8 @@ import uuid from typing import Any, Optional import allure -from common import ASSETS_DIR + +from pytest_tests.resources.common import ASSETS_DIR logger = logging.getLogger("NeoLogger") diff --git a/robot/resources/lib/python_keywords/frostfs_verbs.py b/pytest_tests/helpers/frostfs_verbs.py similarity index 99% rename from robot/resources/lib/python_keywords/frostfs_verbs.py rename to pytest_tests/helpers/frostfs_verbs.py index 24aaee14..21b52294 100644 --- a/robot/resources/lib/python_keywords/frostfs_verbs.py +++ b/pytest_tests/helpers/frostfs_verbs.py @@ -6,12 +6,13 @@ import uuid from typing import Any, Optional import allure -from cluster import Cluster -from common import ASSETS_DIR, FROSTFS_CLI_EXEC, WALLET_CONFIG from frostfs_testlib.cli import FrostfsCli from frostfs_testlib.shell import Shell from frostfs_testlib.utils import json_utils +from pytest_tests.helpers.cluster import Cluster +from pytest_tests.resources.common import ASSETS_DIR, FROSTFS_CLI_EXEC, WALLET_CONFIG + logger = logging.getLogger("NeoLogger") diff --git a/robot/resources/lib/python_keywords/http_gate.py b/pytest_tests/helpers/http_gate.py similarity index 96% rename from robot/resources/lib/python_keywords/http_gate.py rename to pytest_tests/helpers/http_gate.py index 68d9ca8c..226fde23 100644 --- a/robot/resources/lib/python_keywords/http_gate.py +++ b/pytest_tests/helpers/http_gate.py @@ -1,4 +1,3 @@ -import base64 import logging import os import random @@ -12,13 +11,14 @@ from urllib.parse import quote_plus import allure import requests from aws_cli_client import LONG_TIMEOUT -from cli_helpers import _cmd_run -from cluster import StorageNode -from common import SIMPLE_OBJECT_SIZE -from file_helper import get_file_hash from frostfs_testlib.shell import Shell -from python_keywords.frostfs_verbs import get_object -from python_keywords.storage_policy import get_nodes_without_object + +from pytest_tests.helpers.cli_helpers import _cmd_run +from pytest_tests.helpers.cluster import StorageNode +from pytest_tests.helpers.file_helper import get_file_hash +from pytest_tests.helpers.frostfs_verbs import get_object +from pytest_tests.helpers.storage_policy import get_nodes_without_object +from pytest_tests.resources.common import SIMPLE_OBJECT_SIZE logger = logging.getLogger("NeoLogger") diff --git a/robot/resources/lib/python_keywords/node_management.py b/pytest_tests/helpers/node_management.py similarity index 97% rename from robot/resources/lib/python_keywords/node_management.py rename to pytest_tests/helpers/node_management.py index efd10b7f..a4e99843 100644 --- a/robot/resources/lib/python_keywords/node_management.py +++ b/pytest_tests/helpers/node_management.py @@ -6,13 +6,14 @@ from dataclasses import dataclass from typing import Optional import allure -from cluster import Cluster, StorageNode -from common import FROSTFS_CLI_EXEC, MORPH_BLOCK_TIME -from epoch import tick_epoch from frostfs_testlib.cli import FrostfsCli from frostfs_testlib.shell import Shell from frostfs_testlib.utils import datetime_utils +from pytest_tests.helpers.cluster import Cluster, StorageNode +from pytest_tests.helpers.epoch import tick_epoch +from pytest_tests.resources.common import FROSTFS_CLI_EXEC, MORPH_BLOCK_TIME + logger = logging.getLogger("NeoLogger") diff --git a/robot/resources/lib/python_keywords/object_access.py b/pytest_tests/helpers/object_access.py similarity index 95% rename from robot/resources/lib/python_keywords/object_access.py rename to pytest_tests/helpers/object_access.py index ba599716..bc6d1938 100644 --- a/robot/resources/lib/python_keywords/object_access.py +++ b/pytest_tests/helpers/object_access.py @@ -1,12 +1,13 @@ from typing import Optional import allure -from cluster import Cluster -from file_helper import get_file_hash from frostfs_testlib.resources.common import OBJECT_ACCESS_DENIED from frostfs_testlib.shell import Shell from frostfs_testlib.utils import string_utils -from python_keywords.frostfs_verbs import ( + +from pytest_tests.helpers.cluster import Cluster +from pytest_tests.helpers.file_helper import get_file_hash +from pytest_tests.helpers.frostfs_verbs import ( delete_object, get_object_from_random_node, get_range, @@ -30,7 +31,7 @@ def can_get_object( wallet_config: Optional[str] = None, xhdr: Optional[dict] = None, ) -> bool: - with allure.step("Try get object from container"): + with allure.step("Try get object from pytest_tests.helpers.container"): try: got_file_path = get_object_from_random_node( wallet, @@ -93,7 +94,7 @@ def can_delete_object( wallet_config: Optional[str] = None, xhdr: Optional[dict] = None, ) -> bool: - with allure.step("Try delete object from container"): + with allure.step("Try delete object from pytest_tests.helpers.container"): try: delete_object( wallet, diff --git a/robot/resources/lib/python_keywords/payment_neogo.py b/pytest_tests/helpers/payment_neogo.py similarity index 97% rename from robot/resources/lib/python_keywords/payment_neogo.py rename to pytest_tests/helpers/payment_neogo.py index b65474a4..833ca83c 100644 --- a/robot/resources/lib/python_keywords/payment_neogo.py +++ b/pytest_tests/helpers/payment_neogo.py @@ -6,14 +6,20 @@ import time from typing import Optional import allure -from cluster import MainChain, MorphChain -from common import FROSTFS_CONTRACT, GAS_HASH, MAINNET_BLOCK_TIME, NEOGO_EXECUTABLE from frostfs_testlib.cli import NeoGo from frostfs_testlib.shell import Shell from frostfs_testlib.utils import converting_utils, datetime_utils, wallet_utils from neo3.wallet import utils as neo3_utils from neo3.wallet import wallet as neo3_wallet +from pytest_tests.helpers.cluster import MainChain, MorphChain +from pytest_tests.resources.common import ( + FROSTFS_CONTRACT, + GAS_HASH, + MAINNET_BLOCK_TIME, + NEOGO_EXECUTABLE, +) + logger = logging.getLogger("NeoLogger") EMPTY_PASSWORD = "" diff --git a/pytest_tests/helpers/s3_helper.py b/pytest_tests/helpers/s3_helper.py index 013ab396..3c2db215 100644 --- a/pytest_tests/helpers/s3_helper.py +++ b/pytest_tests/helpers/s3_helper.py @@ -1,14 +1,13 @@ -import datetime import logging import os from datetime import datetime, timedelta from typing import Optional import allure -import s3_gate_bucket -import s3_gate_object from dateutil.parser import parse +from pytest_tests.steps import s3_gate_bucket, s3_gate_object + logger = logging.getLogger("NeoLogger") diff --git a/robot/resources/lib/python_keywords/storage_group.py b/pytest_tests/helpers/storage_group.py similarity index 96% rename from robot/resources/lib/python_keywords/storage_group.py rename to pytest_tests/helpers/storage_group.py index 1941fe7a..cce22717 100644 --- a/robot/resources/lib/python_keywords/storage_group.py +++ b/pytest_tests/helpers/storage_group.py @@ -6,12 +6,13 @@ import logging from typing import Optional import allure -from cluster import Cluster -from common import FROSTFS_CLI_EXEC, WALLET_CONFIG -from complex_object_actions import get_link_object from frostfs_testlib.cli import FrostfsCli from frostfs_testlib.shell import Shell -from frostfs_verbs import head_object + +from pytest_tests.helpers.cluster import Cluster +from pytest_tests.helpers.complex_object_actions import get_link_object +from pytest_tests.helpers.frostfs_verbs import head_object +from pytest_tests.resources.common import FROSTFS_CLI_EXEC, WALLET_CONFIG logger = logging.getLogger("NeoLogger") diff --git a/robot/resources/lib/python_keywords/storage_policy.py b/pytest_tests/helpers/storage_policy.py similarity index 97% rename from robot/resources/lib/python_keywords/storage_policy.py rename to pytest_tests/helpers/storage_policy.py index a2d2a09d..13f45cf6 100644 --- a/robot/resources/lib/python_keywords/storage_policy.py +++ b/pytest_tests/helpers/storage_policy.py @@ -8,13 +8,13 @@ import logging import allure -import complex_object_actions -import frostfs_verbs -from cluster import StorageNode from frostfs_testlib.resources.common import OBJECT_NOT_FOUND from frostfs_testlib.shell import Shell from frostfs_testlib.utils import string_utils +from pytest_tests.helpers import complex_object_actions, frostfs_verbs +from pytest_tests.helpers.cluster import StorageNode + logger = logging.getLogger("NeoLogger") diff --git a/robot/resources/lib/python_keywords/tombstone.py b/pytest_tests/helpers/tombstone.py similarity index 95% rename from robot/resources/lib/python_keywords/tombstone.py rename to pytest_tests/helpers/tombstone.py index a3630400..939d447a 100644 --- a/robot/resources/lib/python_keywords/tombstone.py +++ b/pytest_tests/helpers/tombstone.py @@ -3,9 +3,10 @@ import logging import allure from frostfs_testlib.shell import Shell -from frostfs_verbs import head_object from neo3.wallet import wallet +from pytest_tests.helpers.frostfs_verbs import head_object + logger = logging.getLogger("NeoLogger") diff --git a/pytest_tests/helpers/utility.py b/pytest_tests/helpers/utility.py index fe03d499..9fd37b8c 100644 --- a/pytest_tests/helpers/utility.py +++ b/pytest_tests/helpers/utility.py @@ -1,13 +1,14 @@ import time import allure -from common import STORAGE_GC_TIME from frostfs_testlib.utils import datetime_utils +from pytest_tests.resources.common import STORAGE_GC_TIME + def placement_policy_from_container(container_info: str) -> str: """ - Get placement policy from container info: + Get placement policy from pytest_tests.helpers.container info: container ID: j7k4auNHRmiPMSmnH2qENLECD2au2y675fvTX6csDwd version: 2.12 diff --git a/pytest_tests/helpers/wallet.py b/pytest_tests/helpers/wallet.py index 597490dc..935d2e49 100644 --- a/pytest_tests/helpers/wallet.py +++ b/pytest_tests/helpers/wallet.py @@ -2,11 +2,12 @@ import os import uuid from dataclasses import dataclass -from cluster import Cluster, NodeBase -from common import FREE_STORAGE, WALLET_CONFIG, WALLET_PASS from frostfs_testlib.shell import Shell from frostfs_testlib.utils import wallet_utils -from python_keywords.payment_neogo import deposit_gas, transfer_gas + +from pytest_tests.helpers.cluster import Cluster, NodeBase +from pytest_tests.helpers.payment_neogo import deposit_gas, transfer_gas +from pytest_tests.resources.common import FREE_STORAGE, WALLET_CONFIG, WALLET_PASS @dataclass diff --git a/robot/variables/common.py b/pytest_tests/resources/common.py similarity index 95% rename from robot/variables/common.py rename to pytest_tests/resources/common.py index 1f81786c..596d31fc 100644 --- a/robot/variables/common.py +++ b/pytest_tests/resources/common.py @@ -22,7 +22,7 @@ GAS_HASH = os.getenv("GAS_HASH", "0xd2a4cff31913016155e38e474a2c06d08be276cf") FROSTFS_CONTRACT = os.getenv("FROSTFS_IR_CONTRACTS_FROSTFS") ASSETS_DIR = os.getenv("ASSETS_DIR", "TemporaryDir") -DEVENV_PATH = os.getenv("DEVENV_PATH", os.path.join("..", "frostfs-dev-env")) +DEVENV_PATH = os.getenv("DEVENV_PATH", os.path.join("../../pytest_tests", "frostfs-dev-env")) # Password of wallet owned by user on behalf of whom we are running tests WALLET_PASS = os.getenv("WALLET_PASS", "") diff --git a/robot/resources/files/policy.json b/pytest_tests/resources/files/policy.json similarity index 100% rename from robot/resources/files/policy.json rename to pytest_tests/resources/files/policy.json diff --git a/robot/resources/files/s3_bearer_rules.json b/pytest_tests/resources/files/s3_bearer_rules.json similarity index 100% rename from robot/resources/files/s3_bearer_rules.json rename to pytest_tests/resources/files/s3_bearer_rules.json diff --git a/pytest_tests/steps/cluster_test_base.py b/pytest_tests/steps/cluster_test_base.py index 918581bf..a0e6c965 100644 --- a/pytest_tests/steps/cluster_test_base.py +++ b/pytest_tests/steps/cluster_test_base.py @@ -1,9 +1,10 @@ import allure -import epoch import pytest -from cluster import Cluster from frostfs_testlib.shell import Shell +from pytest_tests.helpers import epoch +from pytest_tests.helpers.cluster import Cluster + # To skip adding every mandatory singleton dependency to EACH test function class ClusterTestBase: diff --git a/pytest_tests/steps/load.py b/pytest_tests/steps/load.py index 60f1b1cb..3951e6a1 100644 --- a/pytest_tests/steps/load.py +++ b/pytest_tests/steps/load.py @@ -3,13 +3,14 @@ import re from dataclasses import asdict import allure -from common import STORAGE_NODE_SERVICE_NAME_REGEX from frostfs_testlib.cli.frostfs_authmate import FrostfsAuthmate from frostfs_testlib.cli.neogo import NeoGo from frostfs_testlib.hosting import Hosting from frostfs_testlib.shell import CommandOptions, SSHShell from frostfs_testlib.shell.interfaces import InteractiveInput -from k6 import K6, LoadParams, LoadResults + +from pytest_tests.helpers.k6 import K6, LoadParams, LoadResults +from pytest_tests.resources.common import STORAGE_NODE_SERVICE_NAME_REGEX FROSTFS_AUTHMATE_PATH = "frostfs-s3-authmate" STOPPED_HOSTS = [] diff --git a/pytest_tests/steps/s3_gate_base.py b/pytest_tests/steps/s3_gate_base.py index 96d1ab81..72fa8239 100644 --- a/pytest_tests/steps/s3_gate_base.py +++ b/pytest_tests/steps/s3_gate_base.py @@ -11,16 +11,17 @@ import pytest import s3_gate_bucket import s3_gate_object import urllib3 -from aws_cli_client import AwsCliClient from botocore.config import Config from botocore.exceptions import ClientError -from cli_helpers import _cmd_run, _configure_aws_cli, _run_with_passwd -from cluster import Cluster -from cluster_test_base import ClusterTestBase -from common import FROSTFS_AUTHMATE_EXEC from frostfs_testlib.shell import Shell from pytest import FixtureRequest -from python_keywords.container import list_containers + +from pytest_tests.helpers.aws_cli_client import AwsCliClient +from pytest_tests.helpers.cli_helpers import _cmd_run, _configure_aws_cli, _run_with_passwd +from pytest_tests.helpers.cluster import Cluster +from pytest_tests.helpers.container import list_containers +from pytest_tests.resources.common import FROSTFS_AUTHMATE_EXEC +from pytest_tests.steps.cluster_test_base import ClusterTestBase # Disable warnings on self-signed certificate which the # boto library produces on requests to S3-gate in dev-env @@ -44,7 +45,7 @@ class TestS3GateBase(ClusterTestBase): self, default_wallet, client_shell: Shell, request: FixtureRequest, cluster: Cluster ) -> Any: wallet = default_wallet - s3_bearer_rules_file = f"{os.getcwd()}/robot/resources/files/s3_bearer_rules.json" + s3_bearer_rules_file = f"{os.getcwd()}/pytest_tests/resources/files/s3_bearer_rules.json" policy = None if isinstance(request.param, str) else request.param[1] (cid, bucket, access_key_id, secret_access_key, owner_private_key,) = init_s3_credentials( wallet, cluster, s3_bearer_rules_file=s3_bearer_rules_file, policy=policy @@ -123,7 +124,7 @@ def init_s3_credentials( policy: Optional[dict] = None, ): bucket = str(uuid.uuid4()) - s3_bearer_rules = s3_bearer_rules_file or "robot/resources/files/s3_bearer_rules.json" + s3_bearer_rules = s3_bearer_rules_file or "pytest_tests/resources/files/s3_bearer_rules.json" s3gate_node = cluster.s3gates[0] gate_public_key = s3gate_node.get_wallet_public_key() diff --git a/pytest_tests/steps/s3_gate_bucket.py b/pytest_tests/steps/s3_gate_bucket.py index cd0c92bd..db9e4126 100644 --- a/pytest_tests/steps/s3_gate_bucket.py +++ b/pytest_tests/steps/s3_gate_bucket.py @@ -7,7 +7,8 @@ from typing import Optional import allure from botocore.exceptions import ClientError -from cli_helpers import log_command_execution + +from pytest_tests.helpers.cli_helpers import log_command_execution logger = logging.getLogger("NeoLogger") diff --git a/pytest_tests/steps/s3_gate_object.py b/pytest_tests/steps/s3_gate_object.py index 1b30baef..e8b0c4be 100644 --- a/pytest_tests/steps/s3_gate_object.py +++ b/pytest_tests/steps/s3_gate_object.py @@ -7,11 +7,12 @@ from typing import Optional import allure import pytest import urllib3 -from aws_cli_client import AwsCliClient from botocore.exceptions import ClientError -from cli_helpers import log_command_execution from s3_gate_bucket import S3_SYNC_WAIT_TIME +from pytest_tests.helpers.aws_cli_client import AwsCliClient +from pytest_tests.helpers.cli_helpers import log_command_execution + ########################################################## # Disabling warnings on self-signed certificate which the # boto library produces on requests to S3-gate in dev-env. diff --git a/pytest_tests/steps/session_token.py b/pytest_tests/steps/session_token.py index 0498def3..5334b84a 100644 --- a/pytest_tests/steps/session_token.py +++ b/pytest_tests/steps/session_token.py @@ -8,12 +8,13 @@ from enum import Enum from typing import Any, Optional import allure -from common import ASSETS_DIR, FROSTFS_CLI_EXEC, WALLET_CONFIG from frostfs_testlib.cli import FrostfsCli from frostfs_testlib.shell import Shell from frostfs_testlib.utils import json_utils, wallet_utils -from storage_object_info import StorageObjectInfo -from wallet import WalletFile + +from pytest_tests.helpers.storage_object_info import StorageObjectInfo +from pytest_tests.helpers.wallet import WalletFile +from pytest_tests.resources.common import ASSETS_DIR, FROSTFS_CLI_EXEC, WALLET_CONFIG logger = logging.getLogger("NeoLogger") diff --git a/pytest_tests/steps/storage_object.py b/pytest_tests/steps/storage_object.py index 40d53ac6..2a5addac 100644 --- a/pytest_tests/steps/storage_object.py +++ b/pytest_tests/steps/storage_object.py @@ -3,13 +3,14 @@ from time import sleep import allure import pytest -from cluster import Cluster -from epoch import tick_epoch from frostfs_testlib.resources.common import OBJECT_ALREADY_REMOVED from frostfs_testlib.shell import Shell -from python_keywords.frostfs_verbs import delete_object, get_object -from storage_object_info import StorageObjectInfo -from tombstone import verify_head_tombstone + +from pytest_tests.helpers.cluster import Cluster +from pytest_tests.helpers.epoch import tick_epoch +from pytest_tests.helpers.frostfs_verbs import delete_object, get_object +from pytest_tests.helpers.storage_object_info import StorageObjectInfo +from pytest_tests.helpers.tombstone import verify_head_tombstone logger = logging.getLogger("NeoLogger") diff --git a/pytest_tests/testsuites/acl/conftest.py b/pytest_tests/testsuites/acl/conftest.py index 737082d5..c0ca9563 100644 --- a/pytest_tests/testsuites/acl/conftest.py +++ b/pytest_tests/testsuites/acl/conftest.py @@ -5,15 +5,16 @@ from typing import Optional import allure import pytest -from cluster import Cluster -from common import WALLET_CONFIG, WALLET_PASS -from file_helper import generate_file from frostfs_testlib.resources.common import PUBLIC_ACL from frostfs_testlib.shell import Shell from frostfs_testlib.utils import wallet_utils -from python_keywords.acl import EACLRole -from python_keywords.container import create_container -from python_keywords.frostfs_verbs import put_object_to_random_node + +from pytest_tests.helpers.acl import EACLRole +from pytest_tests.helpers.cluster import Cluster +from pytest_tests.helpers.container import create_container +from pytest_tests.helpers.file_helper import generate_file +from pytest_tests.helpers.frostfs_verbs import put_object_to_random_node +from pytest_tests.resources.common import WALLET_CONFIG, WALLET_PASS OBJECT_COUNT = 5 diff --git a/pytest_tests/testsuites/acl/storage_group/test_storagegroup.py b/pytest_tests/testsuites/acl/storage_group/test_storagegroup.py index 44538cb8..ac9e2947 100644 --- a/pytest_tests/testsuites/acl/storage_group/test_storagegroup.py +++ b/pytest_tests/testsuites/acl/storage_group/test_storagegroup.py @@ -5,12 +5,10 @@ from typing import Optional import allure import pytest -from cluster_test_base import ClusterTestBase -from common import ASSETS_DIR, FREE_STORAGE, WALLET_PASS -from file_helper import generate_file from frostfs_testlib.resources.common import OBJECT_ACCESS_DENIED, OBJECT_NOT_FOUND from frostfs_testlib.utils import wallet_utils -from python_keywords.acl import ( + +from pytest_tests.helpers.acl import ( EACLAccess, EACLOperation, EACLRole, @@ -19,10 +17,11 @@ from python_keywords.acl import ( form_bearertoken_file, set_eacl, ) -from python_keywords.container import create_container -from python_keywords.frostfs_verbs import put_object_to_random_node -from python_keywords.payment_neogo import deposit_gas, transfer_gas -from python_keywords.storage_group import ( +from pytest_tests.helpers.container import create_container +from pytest_tests.helpers.file_helper import generate_file +from pytest_tests.helpers.frostfs_verbs import put_object_to_random_node +from pytest_tests.helpers.payment_neogo import deposit_gas, transfer_gas +from pytest_tests.helpers.storage_group import ( delete_storagegroup, get_storagegroup, list_storagegroup, @@ -30,6 +29,8 @@ from python_keywords.storage_group import ( verify_get_storage_group, verify_list_storage_group, ) +from pytest_tests.resources.common import ASSETS_DIR, FREE_STORAGE, WALLET_PASS +from pytest_tests.steps.cluster_test_base import ClusterTestBase logger = logging.getLogger("NeoLogger") deposit = 30 diff --git a/pytest_tests/testsuites/acl/test_acl.py b/pytest_tests/testsuites/acl/test_acl.py index 4a0cc24b..dee01a56 100644 --- a/pytest_tests/testsuites/acl/test_acl.py +++ b/pytest_tests/testsuites/acl/test_acl.py @@ -1,15 +1,16 @@ import allure import pytest -from cluster_test_base import ClusterTestBase from frostfs_testlib.resources.common import PRIVATE_ACL_F, PUBLIC_ACL_F, READONLY_ACL_F -from python_keywords.acl import EACLRole -from python_keywords.container import create_container -from python_keywords.container_access import ( + +from pytest_tests.helpers.acl import EACLRole +from pytest_tests.helpers.container import create_container +from pytest_tests.helpers.container_access import ( check_full_access_to_container, check_no_access_to_container, check_read_only_container, ) -from python_keywords.frostfs_verbs import put_object_to_random_node +from pytest_tests.helpers.frostfs_verbs import put_object_to_random_node +from pytest_tests.steps.cluster_test_base import ClusterTestBase @pytest.mark.sanity diff --git a/pytest_tests/testsuites/acl/test_bearer.py b/pytest_tests/testsuites/acl/test_bearer.py index 3f7bdf55..99298951 100644 --- a/pytest_tests/testsuites/acl/test_bearer.py +++ b/pytest_tests/testsuites/acl/test_bearer.py @@ -1,7 +1,7 @@ import allure import pytest -from cluster_test_base import ClusterTestBase -from python_keywords.acl import ( + +from pytest_tests.helpers.acl import ( EACLAccess, EACLOperation, EACLRole, @@ -11,11 +11,12 @@ from python_keywords.acl import ( set_eacl, wait_for_cache_expired, ) -from python_keywords.container_access import ( +from pytest_tests.helpers.container_access import ( check_custom_access_to_container, check_full_access_to_container, check_no_access_to_container, ) +from pytest_tests.steps.cluster_test_base import ClusterTestBase @pytest.mark.sanity diff --git a/pytest_tests/testsuites/acl/test_eacl.py b/pytest_tests/testsuites/acl/test_eacl.py index d80cc06b..3d751f1d 100644 --- a/pytest_tests/testsuites/acl/test_eacl.py +++ b/pytest_tests/testsuites/acl/test_eacl.py @@ -1,9 +1,8 @@ import allure import pytest -from cluster_test_base import ClusterTestBase -from failover_utils import wait_object_replication from frostfs_testlib.resources.common import PUBLIC_ACL -from python_keywords.acl import ( + +from pytest_tests.helpers.acl import ( EACLAccess, EACLOperation, EACLRole, @@ -12,14 +11,15 @@ from python_keywords.acl import ( set_eacl, wait_for_cache_expired, ) -from python_keywords.container import create_container -from python_keywords.container_access import ( +from pytest_tests.helpers.container import create_container +from pytest_tests.helpers.container_access import ( check_full_access_to_container, check_no_access_to_container, ) -from python_keywords.frostfs_verbs import put_object_to_random_node -from python_keywords.node_management import drop_object -from python_keywords.object_access import ( +from pytest_tests.helpers.failover_utils import wait_object_replication +from pytest_tests.helpers.frostfs_verbs import put_object_to_random_node +from pytest_tests.helpers.node_management import drop_object +from pytest_tests.helpers.object_access import ( can_delete_object, can_get_head_object, can_get_object, @@ -28,6 +28,7 @@ from python_keywords.object_access import ( can_put_object, can_search_object, ) +from pytest_tests.steps.cluster_test_base import ClusterTestBase @pytest.mark.sanity diff --git a/pytest_tests/testsuites/acl/test_eacl_filters.py b/pytest_tests/testsuites/acl/test_eacl_filters.py index 1bfe6c09..d6e6a69b 100644 --- a/pytest_tests/testsuites/acl/test_eacl_filters.py +++ b/pytest_tests/testsuites/acl/test_eacl_filters.py @@ -1,8 +1,8 @@ import allure import pytest -from cluster_test_base import ClusterTestBase from frostfs_testlib.resources.common import PUBLIC_ACL -from python_keywords.acl import ( + +from pytest_tests.helpers.acl import ( EACLAccess, EACLFilter, EACLFilters, @@ -16,13 +16,14 @@ from python_keywords.acl import ( set_eacl, wait_for_cache_expired, ) -from python_keywords.container import create_container, delete_container -from python_keywords.container_access import ( +from pytest_tests.helpers.container import create_container, delete_container +from pytest_tests.helpers.container_access import ( check_full_access_to_container, check_no_access_to_container, ) -from python_keywords.frostfs_verbs import put_object_to_random_node -from python_keywords.object_access import can_get_head_object, can_get_object, can_put_object +from pytest_tests.helpers.frostfs_verbs import put_object_to_random_node +from pytest_tests.helpers.object_access import can_get_head_object, can_get_object, can_put_object +from pytest_tests.steps.cluster_test_base import ClusterTestBase @pytest.mark.sanity diff --git a/pytest_tests/testsuites/conftest.py b/pytest_tests/testsuites/conftest.py index 963f9acd..cf372f02 100644 --- a/pytest_tests/testsuites/conftest.py +++ b/pytest_tests/testsuites/conftest.py @@ -8,9 +8,19 @@ from datetime import datetime import allure import pytest import yaml -from binary_version_helper import get_local_binaries_versions, get_remote_binaries_versions -from cluster import Cluster -from common import ( +from frostfs_testlib.hosting import Hosting +from frostfs_testlib.reporter import AllureHandler, get_reporter +from frostfs_testlib.shell import LocalShell, Shell +from frostfs_testlib.utils import wallet_utils + +from pytest_tests.helpers import binary_version, env_properties +from pytest_tests.helpers.cluster import Cluster +from pytest_tests.helpers.frostfs_verbs import get_netmap_netinfo +from pytest_tests.helpers.k6 import LoadParams +from pytest_tests.helpers.node_management import storage_node_healthcheck +from pytest_tests.helpers.payment_neogo import deposit_gas, transfer_gas +from pytest_tests.helpers.wallet import WalletFactory +from pytest_tests.resources.common import ( ASSETS_DIR, COMPLEX_OBJECT_CHUNKS_COUNT, COMPLEX_OBJECT_TAIL_SIZE, @@ -20,14 +30,7 @@ from common import ( STORAGE_NODE_SERVICE_NAME_REGEX, WALLET_PASS, ) -from env_properties import save_env_properties -from frostfs_testlib.hosting import Hosting -from frostfs_testlib.reporter import AllureHandler, get_reporter -from frostfs_testlib.shell import LocalShell, Shell -from frostfs_testlib.utils import wallet_utils -from k6 import LoadParams -from load import get_services_endpoints, prepare_k6_instances -from load_params import ( +from pytest_tests.resources.load_params import ( BACKGROUND_LOAD_MAX_TIME, BACKGROUND_OBJ_SIZE, BACKGROUND_READERS_COUNT, @@ -36,11 +39,7 @@ from load_params import ( LOAD_NODE_SSH_USER, LOAD_NODES, ) -from payment_neogo import deposit_gas, transfer_gas -from python_keywords.frostfs_verbs import get_netmap_netinfo -from python_keywords.node_management import storage_node_healthcheck - -from helpers.wallet import WalletFactory +from pytest_tests.steps.load import get_services_endpoints, prepare_k6_instances logger = logging.getLogger("NeoLogger") @@ -125,11 +124,11 @@ def cluster(temp_directory: str, hosting: Hosting) -> Cluster: @pytest.fixture(scope="session", autouse=True) @allure.title("Check binary versions") def check_binary_versions(request, hosting: Hosting, client_shell: Shell): - local_versions = get_local_binaries_versions(client_shell) - remote_versions = get_remote_binaries_versions(hosting) + local_versions = binary_version.get_local_binaries_versions(client_shell) + remote_versions = binary_version.get_remote_binaries_versions(hosting) all_versions = {**local_versions, **remote_versions} - save_env_properties(request.config, all_versions) + env_properties.save_env_properties(request.config, all_versions) @pytest.fixture(scope="session") diff --git a/pytest_tests/testsuites/container/test_container.py b/pytest_tests/testsuites/container/test_container.py index 75079b3d..3c40267a 100644 --- a/pytest_tests/testsuites/container/test_container.py +++ b/pytest_tests/testsuites/container/test_container.py @@ -3,7 +3,8 @@ import json import allure import pytest from frostfs_testlib.resources.common import PRIVATE_ACL_F -from python_keywords.container import ( + +from pytest_tests.helpers.container import ( create_container, delete_container, get_container, @@ -11,9 +12,8 @@ from python_keywords.container import ( wait_for_container_creation, wait_for_container_deletion, ) -from utility import placement_policy_from_container - -from steps.cluster_test_base import ClusterTestBase +from pytest_tests.helpers.utility import placement_policy_from_container +from pytest_tests.steps.cluster_test_base import ClusterTestBase @pytest.mark.container diff --git a/pytest_tests/testsuites/failovers/test_failover_network.py b/pytest_tests/testsuites/failovers/test_failover_network.py index a0157e06..e84c3784 100644 --- a/pytest_tests/testsuites/failovers/test_failover_network.py +++ b/pytest_tests/testsuites/failovers/test_failover_network.py @@ -4,15 +4,18 @@ from time import sleep import allure import pytest -from cluster import StorageNode -from failover_utils import wait_all_storage_nodes_returned, wait_object_replication -from file_helper import generate_file, get_file_hash from frostfs_testlib.resources.common import PUBLIC_ACL -from iptables_helper import IpTablesHelper -from python_keywords.container import create_container -from python_keywords.frostfs_verbs import get_object, put_object_to_random_node -from steps.cluster_test_base import ClusterTestBase +from pytest_tests.helpers.cluster import StorageNode +from pytest_tests.helpers.container import create_container +from pytest_tests.helpers.failover_utils import ( + wait_all_storage_nodes_returned, + wait_object_replication, +) +from pytest_tests.helpers.file_helper import generate_file, get_file_hash +from pytest_tests.helpers.frostfs_verbs import get_object, put_object_to_random_node +from pytest_tests.helpers.iptables_helper import IpTablesHelper +from pytest_tests.steps.cluster_test_base import ClusterTestBase logger = logging.getLogger("NeoLogger") STORAGE_NODE_COMMUNICATION_PORT = "8080" diff --git a/pytest_tests/testsuites/failovers/test_failover_storage.py b/pytest_tests/testsuites/failovers/test_failover_storage.py index 9bb1356c..d906a6d7 100644 --- a/pytest_tests/testsuites/failovers/test_failover_storage.py +++ b/pytest_tests/testsuites/failovers/test_failover_storage.py @@ -2,16 +2,19 @@ import logging import allure import pytest -from cluster import Cluster, StorageNode -from failover_utils import wait_all_storage_nodes_returned, wait_object_replication -from file_helper import generate_file, get_file_hash from frostfs_testlib.hosting import Host from frostfs_testlib.resources.common import PUBLIC_ACL from frostfs_testlib.shell import CommandOptions -from python_keywords.container import create_container -from python_keywords.frostfs_verbs import get_object, put_object_to_random_node -from steps.cluster_test_base import ClusterTestBase +from pytest_tests.helpers.cluster import Cluster, StorageNode +from pytest_tests.helpers.container import create_container +from pytest_tests.helpers.failover_utils import ( + wait_all_storage_nodes_returned, + wait_object_replication, +) +from pytest_tests.helpers.file_helper import generate_file, get_file_hash +from pytest_tests.helpers.frostfs_verbs import get_object, put_object_to_random_node +from pytest_tests.steps.cluster_test_base import ClusterTestBase logger = logging.getLogger("NeoLogger") stopped_nodes: list[StorageNode] = [] diff --git a/pytest_tests/testsuites/load/test_load.py b/pytest_tests/testsuites/load/test_load.py index 7845af0e..df712557 100644 --- a/pytest_tests/testsuites/load/test_load.py +++ b/pytest_tests/testsuites/load/test_load.py @@ -1,23 +1,14 @@ import allure import pytest -from cluster_test_base import ClusterTestBase -from common import ( +from frostfs_testlib.hosting import Hosting + +from pytest_tests.helpers.k6 import LoadParams +from pytest_tests.resources.common import ( HTTP_GATE_SERVICE_NAME_REGEX, S3_GATE_SERVICE_NAME_REGEX, STORAGE_NODE_SERVICE_NAME_REGEX, ) -from frostfs_testlib.hosting import Hosting -from k6 import LoadParams -from load import ( - clear_cache_and_data, - get_services_endpoints, - init_s3_client, - multi_node_k6_run, - prepare_k6_instances, - start_stopped_nodes, - stop_unused_nodes, -) -from load_params import ( +from pytest_tests.resources.load_params import ( CONTAINER_PLACEMENT_POLICY, CONTAINERS_COUNT, DELETERS, @@ -34,6 +25,16 @@ from load_params import ( STORAGE_NODE_COUNT, WRITERS, ) +from pytest_tests.steps.cluster_test_base import ClusterTestBase +from pytest_tests.steps.load import ( + clear_cache_and_data, + get_services_endpoints, + init_s3_client, + multi_node_k6_run, + prepare_k6_instances, + start_stopped_nodes, + stop_unused_nodes, +) ENDPOINTS_ATTRIBUTES = { "http": {"regex": HTTP_GATE_SERVICE_NAME_REGEX, "endpoint_attribute": "endpoint"}, diff --git a/pytest_tests/testsuites/network/test_node_management.py b/pytest_tests/testsuites/network/test_node_management.py index ffe7cf88..888a2b0c 100644 --- a/pytest_tests/testsuites/network/test_node_management.py +++ b/pytest_tests/testsuites/network/test_node_management.py @@ -5,16 +5,15 @@ from typing import Optional, Tuple import allure import pytest -from cluster import StorageNode -from cluster_test_base import ClusterTestBase -from common import FROSTFS_CONTRACT_CACHE_TIMEOUT, MORPH_BLOCK_TIME -from epoch import tick_epoch -from file_helper import generate_file from frostfs_testlib.resources.common import OBJECT_NOT_FOUND, PUBLIC_ACL from frostfs_testlib.utils import datetime_utils, string_utils -from python_keywords.container import create_container, get_container -from python_keywords.failover_utils import wait_object_replication -from python_keywords.frostfs_verbs import ( + +from pytest_tests.helpers.cluster import StorageNode +from pytest_tests.helpers.container import create_container, get_container +from pytest_tests.helpers.epoch import tick_epoch +from pytest_tests.helpers.failover_utils import wait_object_replication +from pytest_tests.helpers.file_helper import generate_file +from pytest_tests.helpers.frostfs_verbs import ( delete_object, get_object, get_object_from_random_node, @@ -22,7 +21,7 @@ from python_keywords.frostfs_verbs import ( put_object, put_object_to_random_node, ) -from python_keywords.node_management import ( +from pytest_tests.helpers.node_management import ( check_node_in_map, delete_node_data, drop_object, @@ -34,8 +33,13 @@ from python_keywords.node_management import ( storage_node_healthcheck, storage_node_set_status, ) -from storage_policy import get_nodes_with_object, get_simple_object_copies -from utility import placement_policy_from_container, wait_for_gc_pass_on_storage_nodes +from pytest_tests.helpers.storage_policy import get_nodes_with_object, get_simple_object_copies +from pytest_tests.helpers.utility import ( + placement_policy_from_container, + wait_for_gc_pass_on_storage_nodes, +) +from pytest_tests.resources.common import FROSTFS_CONTRACT_CACHE_TIMEOUT, MORPH_BLOCK_TIME +from pytest_tests.steps.cluster_test_base import ClusterTestBase logger = logging.getLogger("NeoLogger") check_nodes: list[StorageNode] = [] @@ -128,7 +132,7 @@ class TestNodeManagement(ClusterTestBase): simple_object_size, ): """ - This test remove one node from cluster then add it back. Test uses base control operations with storage nodes (healthcheck, netmap-snapshot, set-status). + This test remove one node from pytest_tests.helpers.cluster then add it back. Test uses base control operations with storage nodes (healthcheck, netmap-snapshot, set-status). """ wallet = default_wallet placement_rule_3 = "REP 3 IN X CBF 1 SELECT 3 FROM * AS X" diff --git a/pytest_tests/testsuites/object/test_object_api.py b/pytest_tests/testsuites/object/test_object_api.py index 7dde156e..0b1fa8d3 100755 --- a/pytest_tests/testsuites/object/test_object_api.py +++ b/pytest_tests/testsuites/object/test_object_api.py @@ -4,9 +4,6 @@ import sys import allure import pytest -from cluster import Cluster -from complex_object_actions import get_complex_object_split_ranges -from file_helper import generate_file, get_file_content, get_file_hash from frostfs_testlib.resources.common import ( INVALID_LENGTH_SPECIFIER, INVALID_OFFSET_SPECIFIER, @@ -16,8 +13,12 @@ from frostfs_testlib.resources.common import ( ) from frostfs_testlib.shell import Shell from pytest import FixtureRequest -from python_keywords.container import create_container -from python_keywords.frostfs_verbs import ( + +from pytest_tests.helpers.cluster import Cluster +from pytest_tests.helpers.complex_object_actions import get_complex_object_split_ranges +from pytest_tests.helpers.container import create_container +from pytest_tests.helpers.file_helper import generate_file, get_file_content, get_file_hash +from pytest_tests.helpers.frostfs_verbs import ( get_object_from_random_node, get_range, get_range_hash, @@ -25,11 +26,10 @@ from python_keywords.frostfs_verbs import ( put_object_to_random_node, search_object, ) -from python_keywords.storage_policy import get_complex_object_copies, get_simple_object_copies - -from helpers.storage_object_info import StorageObjectInfo -from steps.cluster_test_base import ClusterTestBase -from steps.storage_object import delete_objects +from pytest_tests.helpers.storage_object_info import StorageObjectInfo +from pytest_tests.helpers.storage_policy import get_complex_object_copies, get_simple_object_copies +from pytest_tests.steps.cluster_test_base import ClusterTestBase +from pytest_tests.steps.storage_object import delete_objects logger = logging.getLogger("NeoLogger") diff --git a/pytest_tests/testsuites/object/test_object_api_bearer.py b/pytest_tests/testsuites/object/test_object_api_bearer.py index 883fcf0c..26df4852 100644 --- a/pytest_tests/testsuites/object/test_object_api_bearer.py +++ b/pytest_tests/testsuites/object/test_object_api_bearer.py @@ -1,19 +1,30 @@ import allure import pytest -from cluster import Cluster -from container import REP_2_FOR_3_NODES_PLACEMENT_RULE, SINGLE_PLACEMENT_RULE, create_container -from epoch import get_epoch from frostfs_testlib.resources.common import EACL_PUBLIC_READ_WRITE from frostfs_testlib.shell import Shell -from frostfs_verbs import delete_object, get_object from pytest import FixtureRequest -from python_keywords.acl import EACLAccess, EACLOperation, EACLRole, EACLRule, form_bearertoken_file -from helpers.container import StorageContainer, StorageContainerInfo -from helpers.test_control import expect_not_raises -from helpers.wallet import WalletFile -from steps.cluster_test_base import ClusterTestBase -from steps.storage_object import StorageObjectInfo +from pytest_tests.helpers.acl import ( + EACLAccess, + EACLOperation, + EACLRole, + EACLRule, + form_bearertoken_file, +) +from pytest_tests.helpers.cluster import Cluster +from pytest_tests.helpers.container import ( + REP_2_FOR_3_NODES_PLACEMENT_RULE, + SINGLE_PLACEMENT_RULE, + StorageContainer, + StorageContainerInfo, + create_container, +) +from pytest_tests.helpers.epoch import get_epoch +from pytest_tests.helpers.frostfs_verbs import delete_object, get_object +from pytest_tests.helpers.test_control import expect_not_raises +from pytest_tests.helpers.wallet import WalletFile +from pytest_tests.steps.cluster_test_base import ClusterTestBase +from pytest_tests.steps.storage_object import StorageObjectInfo @pytest.fixture(scope="module") diff --git a/pytest_tests/testsuites/object/test_object_lifetime.py b/pytest_tests/testsuites/object/test_object_lifetime.py index 2bf15269..d36de345 100644 --- a/pytest_tests/testsuites/object/test_object_lifetime.py +++ b/pytest_tests/testsuites/object/test_object_lifetime.py @@ -2,15 +2,18 @@ import logging import allure import pytest -from epoch import get_epoch -from file_helper import generate_file, get_file_hash from frostfs_testlib.resources.common import OBJECT_NOT_FOUND from pytest import FixtureRequest -from python_keywords.container import create_container -from python_keywords.frostfs_verbs import get_object_from_random_node, put_object_to_random_node -from utility import wait_for_gc_pass_on_storage_nodes -from steps.cluster_test_base import ClusterTestBase +from pytest_tests.helpers.container import create_container +from pytest_tests.helpers.epoch import get_epoch +from pytest_tests.helpers.file_helper import generate_file, get_file_hash +from pytest_tests.helpers.frostfs_verbs import ( + get_object_from_random_node, + put_object_to_random_node, +) +from pytest_tests.helpers.utility import wait_for_gc_pass_on_storage_nodes +from pytest_tests.steps.cluster_test_base import ClusterTestBase logger = logging.getLogger("NeoLogger") diff --git a/pytest_tests/testsuites/object/test_object_lock.py b/pytest_tests/testsuites/object/test_object_lock.py index 7794eb60..640c092e 100755 --- a/pytest_tests/testsuites/object/test_object_lock.py +++ b/pytest_tests/testsuites/object/test_object_lock.py @@ -3,11 +3,6 @@ import re import allure import pytest -from cluster import Cluster -from cluster_test_base import ClusterTestBase -from common import STORAGE_GC_TIME -from complex_object_actions import get_link_object, get_storage_object_chunks -from epoch import ensure_fresh_epoch, get_epoch, tick_epoch from frostfs_testlib.resources.common import ( LIFETIME_REQUIRED, LOCK_NON_REGULAR_OBJECT, @@ -19,18 +14,22 @@ from frostfs_testlib.resources.common import ( ) from frostfs_testlib.shell import Shell from frostfs_testlib.utils import datetime_utils -from node_management import drop_object from pytest import FixtureRequest -from python_keywords.container import create_container -from python_keywords.frostfs_verbs import delete_object, head_object, lock_object -from storage_policy import get_nodes_with_object -from test_control import expect_not_raises, wait_for_success -from utility import wait_for_gc_pass_on_storage_nodes -from helpers.container import StorageContainer, StorageContainerInfo -from helpers.storage_object_info import LockObjectInfo, StorageObjectInfo -from helpers.wallet import WalletFactory, WalletFile -from steps.storage_object import delete_objects +from pytest_tests.helpers.cluster import Cluster +from pytest_tests.helpers.complex_object_actions import get_link_object, get_storage_object_chunks +from pytest_tests.helpers.container import StorageContainer, StorageContainerInfo, create_container +from pytest_tests.helpers.epoch import ensure_fresh_epoch, get_epoch, tick_epoch +from pytest_tests.helpers.frostfs_verbs import delete_object, head_object, lock_object +from pytest_tests.helpers.node_management import drop_object +from pytest_tests.helpers.storage_object_info import LockObjectInfo, StorageObjectInfo +from pytest_tests.helpers.storage_policy import get_nodes_with_object +from pytest_tests.helpers.test_control import expect_not_raises, wait_for_success +from pytest_tests.helpers.utility import wait_for_gc_pass_on_storage_nodes +from pytest_tests.helpers.wallet import WalletFactory, WalletFile +from pytest_tests.resources.common import STORAGE_GC_TIME +from pytest_tests.steps.cluster_test_base import ClusterTestBase +from pytest_tests.steps.storage_object import delete_objects logger = logging.getLogger("NeoLogger") diff --git a/pytest_tests/testsuites/payment/test_balance.py b/pytest_tests/testsuites/payment/test_balance.py index f8b06a89..f34ad1f0 100644 --- a/pytest_tests/testsuites/payment/test_balance.py +++ b/pytest_tests/testsuites/payment/test_balance.py @@ -4,11 +4,12 @@ import os import allure import pytest import yaml -from cluster_test_base import ClusterTestBase -from common import FREE_STORAGE, FROSTFS_CLI_EXEC, WALLET_CONFIG from frostfs_testlib.cli import FrostfsCli from frostfs_testlib.shell import CommandResult, Shell -from wallet import WalletFactory, WalletFile + +from pytest_tests.helpers.wallet import WalletFactory, WalletFile +from pytest_tests.resources.common import FREE_STORAGE, FROSTFS_CLI_EXEC, WALLET_CONFIG +from pytest_tests.steps.cluster_test_base import ClusterTestBase logger = logging.getLogger("NeoLogger") DEPOSIT_AMOUNT = 30 diff --git a/pytest_tests/testsuites/services/http_gate/test_http_bearer.py b/pytest_tests/testsuites/services/http_gate/test_http_bearer.py index 493d78ff..1b779d06 100644 --- a/pytest_tests/testsuites/services/http_gate/test_http_bearer.py +++ b/pytest_tests/testsuites/services/http_gate/test_http_bearer.py @@ -2,11 +2,9 @@ import logging import allure import pytest -from container import create_container -from file_helper import generate_file from frostfs_testlib.resources.common import PUBLIC_ACL -from http_gate import get_object_and_verify_hashes, upload_via_http_gate_curl -from python_keywords.acl import ( + +from pytest_tests.helpers.acl import ( EACLAccess, EACLOperation, EACLRole, @@ -18,8 +16,10 @@ from python_keywords.acl import ( sign_bearer, wait_for_cache_expired, ) - -from steps.cluster_test_base import ClusterTestBase +from pytest_tests.helpers.container import create_container +from pytest_tests.helpers.file_helper import generate_file +from pytest_tests.helpers.http_gate import get_object_and_verify_hashes, upload_via_http_gate_curl +from pytest_tests.steps.cluster_test_base import ClusterTestBase logger = logging.getLogger("NeoLogger") diff --git a/pytest_tests/testsuites/services/http_gate/test_http_gate.py b/pytest_tests/testsuites/services/http_gate/test_http_gate.py index e88bbda2..6db28dbc 100644 --- a/pytest_tests/testsuites/services/http_gate/test_http_gate.py +++ b/pytest_tests/testsuites/services/http_gate/test_http_gate.py @@ -3,12 +3,13 @@ import os import allure import pytest -from epoch import get_epoch -from file_helper import generate_file, get_file_hash from frostfs_testlib.resources.common import PUBLIC_ACL -from python_keywords.container import create_container -from python_keywords.frostfs_verbs import put_object_to_random_node -from python_keywords.http_gate import ( + +from pytest_tests.helpers.container import create_container +from pytest_tests.helpers.epoch import get_epoch +from pytest_tests.helpers.file_helper import generate_file, get_file_hash +from pytest_tests.helpers.frostfs_verbs import put_object_to_random_node +from pytest_tests.helpers.http_gate import ( attr_into_header, get_object_and_verify_hashes, get_object_by_attr_and_verify_hashes, @@ -19,9 +20,8 @@ from python_keywords.http_gate import ( upload_via_http_gate, upload_via_http_gate_curl, ) -from utility import wait_for_gc_pass_on_storage_nodes - -from steps.cluster_test_base import ClusterTestBase +from pytest_tests.helpers.utility import wait_for_gc_pass_on_storage_nodes +from pytest_tests.steps.cluster_test_base import ClusterTestBase logger = logging.getLogger("NeoLogger") OBJECT_NOT_FOUND_ERROR = "not found" diff --git a/pytest_tests/testsuites/services/http_gate/test_http_headers.py b/pytest_tests/testsuites/services/http_gate/test_http_headers.py index b2b8fb0b..ab9a03b5 100644 --- a/pytest_tests/testsuites/services/http_gate/test_http_headers.py +++ b/pytest_tests/testsuites/services/http_gate/test_http_headers.py @@ -3,26 +3,26 @@ import os import allure import pytest -from container import ( +from frostfs_testlib.resources.common import PUBLIC_ACL +from pytest import FixtureRequest + +from pytest_tests.helpers.container import ( create_container, delete_container, list_containers, wait_for_container_deletion, ) -from file_helper import generate_file -from frostfs_testlib.resources.common import PUBLIC_ACL -from http_gate import ( +from pytest_tests.helpers.file_helper import generate_file +from pytest_tests.helpers.frostfs_verbs import delete_object +from pytest_tests.helpers.http_gate import ( attr_into_str_header_curl, get_object_by_attr_and_verify_hashes, try_to_get_object_and_expect_error, try_to_get_object_via_passed_request_and_expect_error, upload_via_http_gate_curl, ) -from pytest import FixtureRequest -from python_keywords.frostfs_verbs import delete_object - -from helpers.storage_object_info import StorageObjectInfo -from steps.cluster_test_base import ClusterTestBase +from pytest_tests.helpers.storage_object_info import StorageObjectInfo +from pytest_tests.steps.cluster_test_base import ClusterTestBase OBJECT_ALREADY_REMOVED_ERROR = "object already removed" logger = logging.getLogger("NeoLogger") diff --git a/pytest_tests/testsuites/services/http_gate/test_http_object.py b/pytest_tests/testsuites/services/http_gate/test_http_object.py index cc7459b6..80eddf2c 100644 --- a/pytest_tests/testsuites/services/http_gate/test_http_object.py +++ b/pytest_tests/testsuites/services/http_gate/test_http_object.py @@ -2,17 +2,17 @@ import logging import allure import pytest -from container import create_container -from file_helper import generate_file from frostfs_testlib.resources.common import PUBLIC_ACL -from http_gate import ( + +from pytest_tests.helpers.container import create_container +from pytest_tests.helpers.file_helper import generate_file +from pytest_tests.helpers.frostfs_verbs import put_object_to_random_node +from pytest_tests.helpers.http_gate import ( get_object_and_verify_hashes, get_object_by_attr_and_verify_hashes, try_to_get_object_via_passed_request_and_expect_error, ) -from python_keywords.frostfs_verbs import put_object_to_random_node - -from steps.cluster_test_base import ClusterTestBase +from pytest_tests.steps.cluster_test_base import ClusterTestBase logger = logging.getLogger("NeoLogger") diff --git a/pytest_tests/testsuites/services/http_gate/test_http_streaming.py b/pytest_tests/testsuites/services/http_gate/test_http_streaming.py index f30bb2c3..f569b9d2 100644 --- a/pytest_tests/testsuites/services/http_gate/test_http_streaming.py +++ b/pytest_tests/testsuites/services/http_gate/test_http_streaming.py @@ -2,12 +2,12 @@ import logging import allure import pytest -from container import create_container -from file_helper import generate_file from frostfs_testlib.resources.common import PUBLIC_ACL -from http_gate import get_object_and_verify_hashes, upload_via_http_gate_curl -from steps.cluster_test_base import ClusterTestBase +from pytest_tests.helpers.container import create_container +from pytest_tests.helpers.file_helper import generate_file +from pytest_tests.helpers.http_gate import get_object_and_verify_hashes, upload_via_http_gate_curl +from pytest_tests.steps.cluster_test_base import ClusterTestBase logger = logging.getLogger("NeoLogger") diff --git a/pytest_tests/testsuites/services/http_gate/test_http_system_header.py b/pytest_tests/testsuites/services/http_gate/test_http_system_header.py index e6d2db16..4cfde2a0 100644 --- a/pytest_tests/testsuites/services/http_gate/test_http_system_header.py +++ b/pytest_tests/testsuites/services/http_gate/test_http_system_header.py @@ -5,23 +5,23 @@ from typing import Optional import allure import pytest -from container import create_container -from epoch import get_epoch, wait_for_epochs_align -from file_helper import generate_file from frostfs_testlib.resources.common import OBJECT_NOT_FOUND, PUBLIC_ACL -from http_gate import ( + +from pytest_tests.helpers.container import create_container +from pytest_tests.helpers.epoch import get_epoch, wait_for_epochs_align +from pytest_tests.helpers.file_helper import generate_file +from pytest_tests.helpers.frostfs_verbs import ( + get_netmap_netinfo, + get_object_from_random_node, + head_object, +) +from pytest_tests.helpers.http_gate import ( attr_into_str_header_curl, get_object_and_verify_hashes, try_to_get_object_and_expect_error, upload_via_http_gate_curl, ) -from python_keywords.frostfs_verbs import ( - get_netmap_netinfo, - get_object_from_random_node, - head_object, -) - -from steps.cluster_test_base import ClusterTestBase +from pytest_tests.steps.cluster_test_base import ClusterTestBase logger = logging.getLogger("NeoLogger") EXPIRATION_TIMESTAMP_HEADER = "__FROSRFS__EXPIRATION_TIMESTAMP" diff --git a/pytest_tests/testsuites/services/s3_gate/test_s3_ACL.py b/pytest_tests/testsuites/services/s3_gate/test_s3_ACL.py index e85d931d..d2761fb2 100644 --- a/pytest_tests/testsuites/services/s3_gate/test_s3_ACL.py +++ b/pytest_tests/testsuites/services/s3_gate/test_s3_ACL.py @@ -1,10 +1,10 @@ import allure import pytest -from file_helper import generate_file -from s3_helper import assert_s3_acl, object_key_from_file_path -from steps import s3_gate_bucket, s3_gate_object -from steps.s3_gate_base import TestS3GateBase +from pytest_tests.helpers.file_helper import generate_file +from pytest_tests.helpers.s3_helper import assert_s3_acl, object_key_from_file_path +from pytest_tests.steps import s3_gate_bucket, s3_gate_object +from pytest_tests.steps.s3_gate_base import TestS3GateBase def pytest_generate_tests(metafunc): diff --git a/pytest_tests/testsuites/services/s3_gate/test_s3_bucket.py b/pytest_tests/testsuites/services/s3_gate/test_s3_bucket.py index be9d6c51..37f87ca5 100644 --- a/pytest_tests/testsuites/services/s3_gate/test_s3_bucket.py +++ b/pytest_tests/testsuites/services/s3_gate/test_s3_bucket.py @@ -2,16 +2,16 @@ from datetime import datetime, timedelta import allure import pytest -from file_helper import generate_file -from s3_helper import ( + +from pytest_tests.helpers.file_helper import generate_file +from pytest_tests.helpers.s3_helper import ( assert_object_lock_mode, assert_s3_acl, check_objects_in_bucket, object_key_from_file_path, ) - -from steps import s3_gate_bucket, s3_gate_object -from steps.s3_gate_base import TestS3GateBase +from pytest_tests.steps import s3_gate_bucket, s3_gate_object +from pytest_tests.steps.s3_gate_base import TestS3GateBase def pytest_generate_tests(metafunc): diff --git a/pytest_tests/testsuites/services/s3_gate/test_s3_gate.py b/pytest_tests/testsuites/services/s3_gate/test_s3_gate.py index b89e1a2b..54308288 100644 --- a/pytest_tests/testsuites/services/s3_gate/test_s3_gate.py +++ b/pytest_tests/testsuites/services/s3_gate/test_s3_gate.py @@ -4,26 +4,26 @@ from random import choice, choices import allure import pytest -from aws_cli_client import AwsCliClient -from common import ASSETS_DIR -from epoch import tick_epoch -from file_helper import ( + +from pytest_tests.helpers.aws_cli_client import AwsCliClient +from pytest_tests.helpers.epoch import tick_epoch +from pytest_tests.helpers.file_helper import ( generate_file, generate_file_with_content, get_file_content, get_file_hash, split_file, ) -from s3_helper import ( +from pytest_tests.helpers.s3_helper import ( check_objects_in_bucket, check_tags_by_bucket, check_tags_by_object, set_bucket_versioning, try_to_get_objects_and_expect_error, ) - -from steps import s3_gate_bucket, s3_gate_object -from steps.s3_gate_base import TestS3GateBase +from pytest_tests.resources.common import ASSETS_DIR +from pytest_tests.steps import s3_gate_bucket, s3_gate_object +from pytest_tests.steps.s3_gate_base import TestS3GateBase logger = logging.getLogger("NeoLogger") diff --git a/pytest_tests/testsuites/services/s3_gate/test_s3_locking.py b/pytest_tests/testsuites/services/s3_gate/test_s3_locking.py index 758e77c7..90ecaabb 100644 --- a/pytest_tests/testsuites/services/s3_gate/test_s3_locking.py +++ b/pytest_tests/testsuites/services/s3_gate/test_s3_locking.py @@ -3,11 +3,15 @@ from datetime import datetime, timedelta import allure import pytest -from file_helper import generate_file, generate_file_with_content -from s3_helper import assert_object_lock_mode, check_objects_in_bucket, object_key_from_file_path -from steps import s3_gate_bucket, s3_gate_object -from steps.s3_gate_base import TestS3GateBase +from pytest_tests.helpers.file_helper import generate_file, generate_file_with_content +from pytest_tests.helpers.s3_helper import ( + assert_object_lock_mode, + check_objects_in_bucket, + object_key_from_file_path, +) +from pytest_tests.steps import s3_gate_bucket, s3_gate_object +from pytest_tests.steps.s3_gate_base import TestS3GateBase def pytest_generate_tests(metafunc): diff --git a/pytest_tests/testsuites/services/s3_gate/test_s3_multipart.py b/pytest_tests/testsuites/services/s3_gate/test_s3_multipart.py index 4d691807..7e811033 100644 --- a/pytest_tests/testsuites/services/s3_gate/test_s3_multipart.py +++ b/pytest_tests/testsuites/services/s3_gate/test_s3_multipart.py @@ -1,10 +1,14 @@ import allure import pytest -from file_helper import generate_file, get_file_hash, split_file -from s3_helper import check_objects_in_bucket, object_key_from_file_path, set_bucket_versioning -from steps import s3_gate_bucket, s3_gate_object -from steps.s3_gate_base import TestS3GateBase +from pytest_tests.helpers.file_helper import generate_file, get_file_hash, split_file +from pytest_tests.helpers.s3_helper import ( + check_objects_in_bucket, + object_key_from_file_path, + set_bucket_versioning, +) +from pytest_tests.steps import s3_gate_bucket, s3_gate_object +from pytest_tests.steps.s3_gate_base import TestS3GateBase PART_SIZE = 5 * 1024 * 1024 diff --git a/pytest_tests/testsuites/services/s3_gate/test_s3_object.py b/pytest_tests/testsuites/services/s3_gate/test_s3_object.py index 6dc89865..56977353 100644 --- a/pytest_tests/testsuites/services/s3_gate/test_s3_object.py +++ b/pytest_tests/testsuites/services/s3_gate/test_s3_object.py @@ -6,20 +6,25 @@ from random import choices, sample import allure import pytest -from aws_cli_client import AwsCliClient -from common import ASSETS_DIR, FREE_STORAGE, WALLET_PASS -from file_helper import concat_files, generate_file, generate_file_with_content, get_file_hash from frostfs_testlib.utils import wallet_utils -from python_keywords.payment_neogo import deposit_gas, transfer_gas -from s3_helper import ( + +from pytest_tests.helpers.aws_cli_client import AwsCliClient +from pytest_tests.helpers.file_helper import ( + concat_files, + generate_file, + generate_file_with_content, + get_file_hash, +) +from pytest_tests.helpers.payment_neogo import deposit_gas, transfer_gas +from pytest_tests.helpers.s3_helper import ( assert_object_lock_mode, assert_s3_acl, check_objects_in_bucket, set_bucket_versioning, ) - -from steps import s3_gate_bucket, s3_gate_object -from steps.s3_gate_base import TestS3GateBase +from pytest_tests.resources.common import ASSETS_DIR, FREE_STORAGE, WALLET_PASS +from pytest_tests.steps import s3_gate_bucket, s3_gate_object +from pytest_tests.steps.s3_gate_base import TestS3GateBase def pytest_generate_tests(metafunc): diff --git a/pytest_tests/testsuites/services/s3_gate/test_s3_policy.py b/pytest_tests/testsuites/services/s3_gate/test_s3_policy.py index 3933bfeb..cc78d625 100644 --- a/pytest_tests/testsuites/services/s3_gate/test_s3_policy.py +++ b/pytest_tests/testsuites/services/s3_gate/test_s3_policy.py @@ -2,17 +2,21 @@ import os import allure import pytest -from file_helper import generate_file -from python_keywords.container import search_container_by_name -from python_keywords.storage_policy import get_simple_object_copies -from s3_helper import check_objects_in_bucket, object_key_from_file_path, set_bucket_versioning -from steps import s3_gate_bucket, s3_gate_object -from steps.s3_gate_base import TestS3GateBase +from pytest_tests.helpers.container import search_container_by_name +from pytest_tests.helpers.file_helper import generate_file +from pytest_tests.helpers.s3_helper import ( + check_objects_in_bucket, + object_key_from_file_path, + set_bucket_versioning, +) +from pytest_tests.helpers.storage_policy import get_simple_object_copies +from pytest_tests.steps import s3_gate_bucket, s3_gate_object +from pytest_tests.steps.s3_gate_base import TestS3GateBase def pytest_generate_tests(metafunc): - policy = f"{os.getcwd()}/robot/resources/files/policy.json" + policy = f"{os.getcwd()}/pytest_tests/resources/files/policy.json" if "s3_client" in metafunc.fixturenames: metafunc.parametrize( "s3_client", @@ -95,7 +99,7 @@ class TestS3GatePolicy(TestS3GateBase): s3_gate_bucket.get_bucket_policy(self.s3_client, bucket) with allure.step("Put new policy"): - custom_policy = f"file://{os.getcwd()}/robot/resources/files/bucket_policy.json" + custom_policy = f"file://{os.getcwd()}/pytest_tests/resources/files/bucket_policy.json" custom_policy = { "Version": "2008-10-17", "Id": "aaaa-bbbb-cccc-dddd", diff --git a/pytest_tests/testsuites/services/s3_gate/test_s3_tagging.py b/pytest_tests/testsuites/services/s3_gate/test_s3_tagging.py index 2379a28a..6f2ac289 100644 --- a/pytest_tests/testsuites/services/s3_gate/test_s3_tagging.py +++ b/pytest_tests/testsuites/services/s3_gate/test_s3_tagging.py @@ -1,16 +1,18 @@ -import os -import uuid from random import choice from string import ascii_letters from typing import Tuple import allure import pytest -from file_helper import generate_file -from s3_helper import check_tags_by_bucket, check_tags_by_object, object_key_from_file_path -from steps import s3_gate_bucket, s3_gate_object -from steps.s3_gate_base import TestS3GateBase +from pytest_tests.helpers.file_helper import generate_file +from pytest_tests.helpers.s3_helper import ( + check_tags_by_bucket, + check_tags_by_object, + object_key_from_file_path, +) +from pytest_tests.steps import s3_gate_bucket, s3_gate_object +from pytest_tests.steps.s3_gate_base import TestS3GateBase def pytest_generate_tests(metafunc): diff --git a/pytest_tests/testsuites/services/s3_gate/test_s3_versioning.py b/pytest_tests/testsuites/services/s3_gate/test_s3_versioning.py index a9ac5b85..2b99e358 100644 --- a/pytest_tests/testsuites/services/s3_gate/test_s3_versioning.py +++ b/pytest_tests/testsuites/services/s3_gate/test_s3_versioning.py @@ -2,11 +2,11 @@ import os import allure import pytest -from file_helper import generate_file, generate_file_with_content -from s3_helper import set_bucket_versioning -from steps import s3_gate_bucket, s3_gate_object -from steps.s3_gate_base import TestS3GateBase +from pytest_tests.helpers.file_helper import generate_file, generate_file_with_content +from pytest_tests.helpers.s3_helper import set_bucket_versioning +from pytest_tests.steps import s3_gate_bucket, s3_gate_object +from pytest_tests.steps.s3_gate_base import TestS3GateBase def pytest_generate_tests(metafunc): diff --git a/pytest_tests/testsuites/services/test_binaries.py b/pytest_tests/testsuites/services/test_binaries.py index 3e3fae00..78392106 100644 --- a/pytest_tests/testsuites/services/test_binaries.py +++ b/pytest_tests/testsuites/services/test_binaries.py @@ -5,11 +5,12 @@ from re import match import allure import pytest import requests -from binary_version_helper import get_remote_binaries_versions -from common import BIN_VERSIONS_FILE -from env_properties import read_env_properties, save_env_properties from frostfs_testlib.hosting import Hosting +from pytest_tests.helpers.binary_version import get_remote_binaries_versions +from pytest_tests.helpers.env_properties import read_env_properties, save_env_properties +from pytest_tests.resources.common import BIN_VERSIONS_FILE + logger = logging.getLogger("NeoLogger") diff --git a/pytest_tests/testsuites/session_token/conftest.py b/pytest_tests/testsuites/session_token/conftest.py index a80aab53..8678234a 100644 --- a/pytest_tests/testsuites/session_token/conftest.py +++ b/pytest_tests/testsuites/session_token/conftest.py @@ -1,5 +1,6 @@ import pytest -from wallet import WalletFactory, WalletFile + +from pytest_tests.helpers.wallet import WalletFactory, WalletFile @pytest.fixture(scope="module") diff --git a/pytest_tests/testsuites/session_token/test_object_session_token.py b/pytest_tests/testsuites/session_token/test_object_session_token.py index 3ef261ab..b1c29156 100644 --- a/pytest_tests/testsuites/session_token/test_object_session_token.py +++ b/pytest_tests/testsuites/session_token/test_object_session_token.py @@ -2,15 +2,15 @@ import random import allure import pytest -from cluster_test_base import ClusterTestBase -from common import WALLET_PASS -from file_helper import generate_file from frostfs_testlib.resources.common import SESSION_NOT_FOUND from frostfs_testlib.utils import wallet_utils -from python_keywords.container import create_container -from python_keywords.frostfs_verbs import delete_object, put_object, put_object_to_random_node -from steps.session_token import create_session_token +from pytest_tests.helpers.container import create_container +from pytest_tests.helpers.file_helper import generate_file +from pytest_tests.helpers.frostfs_verbs import delete_object, put_object, put_object_to_random_node +from pytest_tests.resources.common import WALLET_PASS +from pytest_tests.steps.cluster_test_base import ClusterTestBase +from pytest_tests.steps.session_token import create_session_token @pytest.mark.sanity diff --git a/pytest_tests/testsuites/session_token/test_static_object_session_token.py b/pytest_tests/testsuites/session_token/test_static_object_session_token.py index f9cdf832..840d71ad 100644 --- a/pytest_tests/testsuites/session_token/test_static_object_session_token.py +++ b/pytest_tests/testsuites/session_token/test_static_object_session_token.py @@ -2,10 +2,6 @@ import logging import allure import pytest -from cluster import Cluster -from cluster_test_base import ClusterTestBase -from epoch import ensure_fresh_epoch -from file_helper import generate_file from frostfs_testlib.resources.common import ( EXPIRED_SESSION_TOKEN, MALFORMED_REQUEST, @@ -14,8 +10,12 @@ from frostfs_testlib.resources.common import ( ) from frostfs_testlib.shell import Shell from pytest import FixtureRequest -from python_keywords.container import create_container -from python_keywords.frostfs_verbs import ( + +from pytest_tests.helpers.cluster import Cluster +from pytest_tests.helpers.container import create_container +from pytest_tests.helpers.epoch import ensure_fresh_epoch +from pytest_tests.helpers.file_helper import generate_file +from pytest_tests.helpers.frostfs_verbs import ( delete_object, get_object, get_object_from_random_node, @@ -25,11 +25,11 @@ from python_keywords.frostfs_verbs import ( put_object_to_random_node, search_object, ) -from test_control import expect_not_raises -from wallet import WalletFile - -from helpers.storage_object_info import StorageObjectInfo -from steps.session_token import ( +from pytest_tests.helpers.storage_object_info import StorageObjectInfo +from pytest_tests.helpers.test_control import expect_not_raises +from pytest_tests.helpers.wallet import WalletFile +from pytest_tests.steps.cluster_test_base import ClusterTestBase +from pytest_tests.steps.session_token import ( INVALID_SIGNATURE, UNRELATED_CONTAINER, UNRELATED_KEY, @@ -41,7 +41,7 @@ from steps.session_token import ( get_object_signed_token, sign_session_token, ) -from steps.storage_object import delete_objects +from pytest_tests.steps.storage_object import delete_objects logger = logging.getLogger("NeoLogger") diff --git a/pytest_tests/testsuites/session_token/test_static_session_token_container.py b/pytest_tests/testsuites/session_token/test_static_session_token_container.py index 591f4670..afc33cfd 100644 --- a/pytest_tests/testsuites/session_token/test_static_session_token_container.py +++ b/pytest_tests/testsuites/session_token/test_static_session_token_container.py @@ -1,9 +1,9 @@ import allure import pytest -from file_helper import generate_file from frostfs_testlib.resources.common import PUBLIC_ACL from frostfs_testlib.shell import Shell -from python_keywords.acl import ( + +from pytest_tests.helpers.acl import ( EACLAccess, EACLOperation, EACLRole, @@ -12,17 +12,17 @@ from python_keywords.acl import ( set_eacl, wait_for_cache_expired, ) -from python_keywords.container import ( +from pytest_tests.helpers.container import ( create_container, delete_container, get_container, list_containers, ) -from python_keywords.object_access import can_put_object -from wallet import WalletFile - -from steps.cluster_test_base import ClusterTestBase -from steps.session_token import ContainerVerb, get_container_signed_token +from pytest_tests.helpers.file_helper import generate_file +from pytest_tests.helpers.object_access import can_put_object +from pytest_tests.helpers.wallet import WalletFile +from pytest_tests.steps.cluster_test_base import ClusterTestBase +from pytest_tests.steps.session_token import ContainerVerb, get_container_signed_token @pytest.mark.static_session_container diff --git a/pytest_tests/testsuites/shard/test_control_shard.py b/pytest_tests/testsuites/shard/test_control_shard.py index 22f89cd1..f3f0a747 100644 --- a/pytest_tests/testsuites/shard/test_control_shard.py +++ b/pytest_tests/testsuites/shard/test_control_shard.py @@ -7,11 +7,12 @@ from io import StringIO import allure import pytest import yaml -from cluster import Cluster, StorageNode -from common import WALLET_CONFIG from configobj import ConfigObj from frostfs_testlib.cli import FrostfsCli +from pytest_tests.helpers.cluster import Cluster, StorageNode +from pytest_tests.resources.common import WALLET_CONFIG + SHARD_PREFIX = "FROSTFS_STORAGE_SHARD_" BLOBSTOR_PREFIX = "_BLOBSTOR_" diff --git a/pytest_tests/wallet_pass.yml b/pytest_tests/wallet_pass.yml deleted file mode 100644 index 1986cf9e..00000000 --- a/pytest_tests/wallet_pass.yml +++ /dev/null @@ -1 +0,0 @@ -password: "" diff --git a/requirements.txt b/requirements.txt index 7b5e93a3..830cdb3f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -50,7 +50,6 @@ pytest-lazy-fixture==0.6.3 python-dateutil==2.8.2 pyyaml==6.0 requests==2.28.0 -robotframework==4.1.2 s3transfer==0.3.7 six==1.16.0 tenacity==8.0.1 diff --git a/robot/resources/lib/python_keywords/container.py b/robot/resources/lib/python_keywords/container.py deleted file mode 100644 index 26cf2be7..00000000 --- a/robot/resources/lib/python_keywords/container.py +++ /dev/null @@ -1,239 +0,0 @@ -#!/usr/bin/python3.9 - -""" - This module contains keywords that utilize `frostfs-cli container` commands. -""" - -import json -import logging -from time import sleep -from typing import Optional, Union - -import allure -from common import FROSTFS_CLI_EXEC, WALLET_CONFIG -from frostfs_testlib.cli import FrostfsCli -from frostfs_testlib.shell import Shell -from frostfs_testlib.utils import json_utils - -logger = logging.getLogger("NeoLogger") - -DEFAULT_PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 4 FROM * AS X" -SINGLE_PLACEMENT_RULE = "REP 1 IN X CBF 1 SELECT 4 FROM * AS X" -REP_2_FOR_3_NODES_PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 3 FROM * AS X" - - -@allure.step("Create Container") -def create_container( - wallet: str, - shell: Shell, - endpoint: str, - rule: str = DEFAULT_PLACEMENT_RULE, - basic_acl: str = "", - attributes: Optional[dict] = None, - session_token: str = "", - session_wallet: str = "", - name: str = None, - options: dict = None, - await_mode: bool = True, - wait_for_creation: bool = True, -) -> str: - """ - A wrapper for `frostfs-cli container create` call. - - Args: - wallet (str): a wallet on whose behalf a container is created - rule (optional, str): placement rule for container - basic_acl (optional, str): an ACL for container, will be - appended to `--basic-acl` key - attributes (optional, dict): container attributes , will be - appended to `--attributes` key - session_token (optional, str): a path to session token file - session_wallet(optional, str): a path to the wallet which signed - the session token; this parameter makes sense - when paired with `session_token` - shell: executor for cli command - endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key - options (optional, dict): any other options to pass to the call - name (optional, str): container name attribute - await_mode (bool): block execution until container is persisted - wait_for_creation (): Wait for container shows in container list - - Returns: - (str): CID of the created container - """ - - cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, WALLET_CONFIG) - result = cli.container.create( - rpc_endpoint=endpoint, - wallet=session_wallet if session_wallet else wallet, - policy=rule, - basic_acl=basic_acl, - attributes=attributes, - name=name, - session=session_token, - await_mode=await_mode, - **options or {}, - ) - - cid = _parse_cid(result.stdout) - - logger.info("Container created; waiting until it is persisted in the sidechain") - - if wait_for_creation: - wait_for_container_creation(wallet, cid, shell, endpoint) - - return cid - - -def wait_for_container_creation( - wallet: str, cid: str, shell: Shell, endpoint: str, attempts: int = 15, sleep_interval: int = 1 -): - for _ in range(attempts): - containers = list_containers(wallet, shell, endpoint) - if cid in containers: - return - logger.info(f"There is no {cid} in {containers} yet; sleep {sleep_interval} and continue") - sleep(sleep_interval) - raise RuntimeError( - f"After {attempts * sleep_interval} seconds container {cid} hasn't been persisted; exiting" - ) - - -def wait_for_container_deletion( - wallet: str, cid: str, shell: Shell, endpoint: str, attempts: int = 30, sleep_interval: int = 1 -): - for _ in range(attempts): - try: - get_container(wallet, cid, shell=shell, endpoint=endpoint) - sleep(sleep_interval) - continue - except Exception as err: - if "container not found" not in str(err): - raise AssertionError(f'Expected "container not found" in error, got\n{err}') - return - raise AssertionError(f"Expected container deleted during {attempts * sleep_interval} sec.") - - -@allure.step("List Containers") -def list_containers(wallet: str, shell: Shell, endpoint: str) -> list[str]: - """ - A wrapper for `frostfs-cli container list` call. It returns all the - available containers for the given wallet. - Args: - wallet (str): a wallet on whose behalf we list the containers - shell: executor for cli command - endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key - Returns: - (list): list of containers - """ - cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, WALLET_CONFIG) - result = cli.container.list(rpc_endpoint=endpoint, wallet=wallet) - logger.info(f"Containers: \n{result}") - return result.stdout.split() - - -@allure.step("Get Container") -def get_container( - wallet: str, - cid: str, - shell: Shell, - endpoint: str, - json_mode: bool = True, -) -> Union[dict, str]: - """ - A wrapper for `frostfs-cli container get` call. It extracts container's - attributes and rearranges them into a more compact view. - Args: - wallet (str): path to a wallet on whose behalf we get the container - cid (str): ID of the container to get - shell: executor for cli command - endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key - json_mode (bool): return container in JSON format - Returns: - (dict, str): dict of container attributes - """ - - cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, WALLET_CONFIG) - result = cli.container.get(rpc_endpoint=endpoint, wallet=wallet, cid=cid, json_mode=json_mode) - - if not json_mode: - return result.stdout - - container_info = json.loads(result.stdout) - attributes = dict() - for attr in container_info["attributes"]: - attributes[attr["key"]] = attr["value"] - container_info["attributes"] = attributes - container_info["ownerID"] = json_utils.json_reencode(container_info["ownerID"]["value"]) - return container_info - - -@allure.step("Delete Container") -# TODO: make the error message about a non-found container more user-friendly -# https://github.com/nspcc-dev/frostfs-contract/issues/121 -def delete_container( - wallet: str, - cid: str, - shell: Shell, - endpoint: str, - force: bool = False, - session_token: Optional[str] = None, - await_mode: bool = False, -) -> None: - """ - A wrapper for `frostfs-cli container delete` call. - Args: - wallet (str): path to a wallet on whose behalf we delete the container - cid (str): ID of the container to delete - shell: executor for cli command - endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key - force (bool): do not check whether container contains locks and remove immediately - session_token: a path to session token file - This function doesn't return anything. - """ - - cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, WALLET_CONFIG) - cli.container.delete( - wallet=wallet, - cid=cid, - rpc_endpoint=endpoint, - force=force, - session=session_token, - await_mode=await_mode, - ) - - -def _parse_cid(output: str) -> str: - """ - Parses container ID from a given CLI output. The input string we expect: - container ID: 2tz86kVTDpJxWHrhw3h6PbKMwkLtBEwoqhHQCKTre1FN - awaiting... - container has been persisted on sidechain - We want to take 'container ID' value from the string. - - Args: - output (str): CLI output to parse - - Returns: - (str): extracted CID - """ - try: - # taking first line from command's output - first_line = output.split("\n")[0] - except Exception: - first_line = "" - logger.error(f"Got empty output: {output}") - splitted = first_line.split(": ") - if len(splitted) != 2: - raise ValueError(f"no CID was parsed from command output: \t{first_line}") - return splitted[1] - - -@allure.step("Search container by name") -def search_container_by_name(wallet: str, name: str, shell: Shell, endpoint: str): - list_cids = list_containers(wallet, shell, endpoint) - for cid in list_cids: - cont_info = get_container(wallet, cid, shell, endpoint, True) - if cont_info.get("attributes").get("Name", None) == name: - return cid - return None diff --git a/robot/variables/eacl_object_filters.py b/robot/variables/eacl_object_filters.py deleted file mode 100644 index d953a1e3..00000000 --- a/robot/variables/eacl_object_filters.py +++ /dev/null @@ -1,23 +0,0 @@ -EACL_OBJ_FILTERS = { - "$Object:objectID": "objectID", - "$Object:containerID": "containerID", - "$Object:ownerID": "ownerID", - "$Object:creationEpoch": "creationEpoch", - "$Object:payloadLength": "payloadLength", - "$Object:payloadHash": "payloadHash", - "$Object:objectType": "objectType", - "$Object:homomorphicHash": "homomorphicHash", - "$Object:version": "version", -} - -VERB_FILTER_DEP = { - "$Object:objectID": ["GET", "HEAD", "DELETE", "RANGE", "RANGEHASH"], - "$Object:containerID": ["GET", "PUT", "HEAD", "DELETE", "SEARCH", "RANGE", "RANGEHASH"], - "$Object:ownerID": ["GET", "HEAD"], - "$Object:creationEpoch": ["GET", "PUT", "HEAD"], - "$Object:payloadLength": ["GET", "PUT", "HEAD"], - "$Object:payloadHash": ["GET", "PUT", "HEAD"], - "$Object:objectType": ["GET", "PUT", "HEAD"], - "$Object:homomorphicHash": ["GET", "PUT", "HEAD"], - "$Object:version": ["GET", "PUT", "HEAD"], -} diff --git a/venv/local-pytest/environment.sh b/venv/local-pytest/environment.sh index 4a08be59..d809193d 100755 --- a/venv/local-pytest/environment.sh +++ b/venv/local-pytest/environment.sh @@ -3,6 +3,4 @@ export NEOFS_MORPH_DISABLE_CACHE=true export DEVENV_PATH="${DEVENV_PATH:-${VIRTUAL_ENV}/../../frostfs-dev-env}" pushd $DEVENV_PATH > /dev/null export `make env` -popd > /dev/null - -export PYTHONPATH=${PYTHONPATH}:${VIRTUAL_ENV}/../robot/resources/lib/:${VIRTUAL_ENV}/../robot/resources/lib/python_keywords:${VIRTUAL_ENV}/../robot/resources/lib/robot:${VIRTUAL_ENV}/../robot/variables:${VIRTUAL_ENV}/../pytest_tests/helpers:${VIRTUAL_ENV}/../pytest_tests/steps:${VIRTUAL_ENV}/../pytest_tests/resources +popd > /dev/null \ No newline at end of file