From 77126f270657f08cfcb2e9cbf89b4b3b6884c140 Mon Sep 17 00:00:00 2001 From: "a.berezin" Date: Thu, 17 Oct 2024 15:07:53 +0300 Subject: [PATCH 1/5] [#311] add new pattern --- pytest_tests/testsuites/special/test_frostfs_logs.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pytest_tests/testsuites/special/test_frostfs_logs.py b/pytest_tests/testsuites/special/test_frostfs_logs.py index 4f4acb50..8af9d513 100644 --- a/pytest_tests/testsuites/special/test_frostfs_logs.py +++ b/pytest_tests/testsuites/special/test_frostfs_logs.py @@ -31,7 +31,17 @@ class TestLogs: if not os.path.exists(logs_dir): os.makedirs(logs_dir) - issues_regex = r"\bpanic\b|\boom\b|too many|insufficient funds|insufficient amount of gas|cannot assign requested address|\bunable to process\b" + regexes = [ + r"\bpanic\b", + r"\boom\b", + r"too many", + r"insufficient funds", + r"insufficient amount of gas", + r"cannot assign requested address", + r"\bunable to process\b", + r"\bmaximum number of subscriptions is reached\b", + ] + issues_regex = "|".join(regexes) exclude_filter = r"too many requests" log_level_priority = "3" # will include 0-3 priority logs (0: emergency 1: alerts 2: critical 3: errors) From 6b83a89b94968bbb10e528d7937464b40dc1b0f3 Mon Sep 17 00:00:00 2001 From: Ilyas Niyazov Date: Thu, 24 Oct 2024 16:00:10 +0300 Subject: [PATCH 2/5] [#312] Extend container metrics tests --- .../metrics/test_container_metrics.py | 54 +++++++++++++++++-- requirements.txt | 2 +- 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/pytest_tests/testsuites/metrics/test_container_metrics.py b/pytest_tests/testsuites/metrics/test_container_metrics.py index b061bb78..86f5ba25 100644 --- a/pytest_tests/testsuites/metrics/test_container_metrics.py +++ b/pytest_tests/testsuites/metrics/test_container_metrics.py @@ -4,11 +4,11 @@ import allure from frostfs_testlib.testing.parallel import parallel import pytest from frostfs_testlib import reporter -from frostfs_testlib.steps.cli.container import create_container, search_nodes_with_container +from frostfs_testlib.steps.cli.container import create_container, delete_container, search_nodes_with_container, wait_for_container_deletion from frostfs_testlib.steps.cli.object import delete_object, head_object, put_object_to_random_node -from frostfs_testlib.steps.metrics import check_metrics_counter, get_metrics_value +from frostfs_testlib.steps.metrics import calc_metrics_count_from_stdout, check_metrics_counter, get_metrics_value from frostfs_testlib.steps.storage_policy import get_nodes_with_object -from frostfs_testlib.storage.cluster import Cluster +from frostfs_testlib.storage.cluster import Cluster, ClusterNode from frostfs_testlib.storage.dataclasses.object_size import ObjectSize from frostfs_testlib.storage.dataclasses.wallet import WalletInfo from frostfs_testlib.testing.cluster_test_base import ClusterTestBase @@ -24,6 +24,14 @@ class TestContainerMetrics(ClusterTestBase): def put_object_parallel(self, file_path: str, wallet: WalletInfo, cid: str): oid = put_object_to_random_node(wallet, file_path, cid, self.shell, self.cluster) return oid + + @reporter.step("Get metrics value from node") + def get_metrics_search_by_greps_parallel(self, node: ClusterNode, **greps): + try: + content_stdout = node.metrics.storage.get_metrics_search_by_greps(greps) + return calc_metrics_count_from_stdout(content_stdout) + except Exception as e: + return None @allure.title("Container metrics (obj_size={object_size},policy={policy})") @pytest.mark.parametrize("placement_policy, policy", [("REP 2 IN X CBF 2 SELECT 2 FROM * AS X", "REP"), ("EC 1.1 CBF 1", "EC")]) @@ -164,3 +172,43 @@ class TestContainerMetrics(ClusterTestBase): for act_metric in metrics_value_nodes: assert act_metric >= 0, "Metrics value is negative" assert sum(metrics_value_nodes) // len(self.cluster.cluster_nodes) == tombstones_size, "tomstone size of objects not correct" + + + @allure.title("Container metrics (policy={policy})") + @pytest.mark.parametrize("placement_policy, policy", [("REP 2 IN X CBF 2 SELECT 2 FROM * AS X", "REP"), ("EC 1.1 CBF 1", "EC")]) + def test_container_metrics_delete_complex_objects( + self, + complex_object_size: ObjectSize, + default_wallet: WalletInfo, + cluster: Cluster, + placement_policy: str, + policy: str + ): + copies = 2 if policy == "REP" else 1 + objects_count = 2 + metric_name = "frostfs_node_engine_container_objects_total" + with reporter.step(f"Create container"): + cid = create_container(default_wallet, self.shell, cluster.default_rpc_endpoint, rule=placement_policy) + + with reporter.step(f"Put {objects_count} objects"): + files_path = [generate_file(complex_object_size.value) for _ in range(objects_count)] + futures = parallel(self.put_object_parallel, files_path, wallet=default_wallet, cid=cid) + oids = [future.result() for future in futures] + + with reporter.step(f"Check metrics value in each nodes, should be {objects_count} for 'user'"): + check_metrics_counter(cluster.cluster_nodes, counter_exp=objects_count * copies, command=metric_name, cid=cid, type="user") + + with reporter.step("Delete objects and container"): + for oid in oids: + delete_object(default_wallet, cid, oid, self.shell, cluster.default_rpc_endpoint) + + delete_container(default_wallet, cid, self.shell, cluster.default_rpc_endpoint) + + with reporter.step("Tick epoch and check container was deleted"): + self.tick_epoch() + wait_for_container_deletion(default_wallet, cid, shell=self.shell, endpoint=cluster.default_rpc_endpoint) + + with reporter.step(f"Check metrics value in each nodes, should not be show any result"): + futures = parallel(self.get_metrics_search_by_greps_parallel, cluster.cluster_nodes, command=metric_name, cid=cid) + metrics_results = [future.result() for future in futures if future.result() is not None] + assert len(metrics_results) == 0, f"Metrics value is not empty in Prometheus, actual value in nodes: {metrics_results}" diff --git a/requirements.txt b/requirements.txt index 97e6d78d..30a005ad 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ allure-pytest==2.13.2 allure-python-commons==2.13.2 base58==2.1.0 -boto3==1.16.33 +boto3==1.35.30 botocore==1.19.33 configobj==5.0.6 neo-mamba==1.0.0 From 75508cc70c4ff33c2032765d65da26d76b8aabff Mon Sep 17 00:00:00 2001 From: anurindm Date: Wed, 23 Oct 2024 13:58:40 +0300 Subject: [PATCH 3/5] Added nearest node count calculation for choosing ec policy Signed-off-by: Dmitry Anurin --- pytest_tests/testsuites/replication/test_ec_replication.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pytest_tests/testsuites/replication/test_ec_replication.py b/pytest_tests/testsuites/replication/test_ec_replication.py index 8e243a8d..21ce932e 100644 --- a/pytest_tests/testsuites/replication/test_ec_replication.py +++ b/pytest_tests/testsuites/replication/test_ec_replication.py @@ -39,7 +39,8 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: 100: ["EC 12.4", "EC 8.4", "EC 5.3", "EC 4.4"], } - metafunc.parametrize("ec_policy, node_count", ((ec_policy, node_count) for ec_policy in ec_map[node_count])) + nearest_node_count = ([4] + (list(filter(lambda x: x <= node_count, ec_map.keys()))))[-1] + metafunc.parametrize("ec_policy, node_count", ((ec_policy, node_count) for ec_policy in ec_map[nearest_node_count])) @allure.title("Initialized remote FrostfsAdm") From b7669fc96f874cfd8e7d137829e9814fbf1990d2 Mon Sep 17 00:00:00 2001 From: "a.berezin" Date: Tue, 29 Oct 2024 13:32:07 +0300 Subject: [PATCH 4/5] [#315] Use relpath for files Signed-off-by: a.berezin --- __init__.py | 3 ++ pytest_tests/helpers/container_access.py | 2 +- pytest_tests/resources/common.py | 3 ++ .../testsuites/access/acl/test_acl.py | 3 +- .../testsuites/access/ape/test_ape.py | 3 +- .../testsuites/access/ape/test_ape_filters.py | 7 ++-- .../testsuites/access/ape/test_bearer.py | 4 +-- pytest_tests/testsuites/conftest.py | 2 +- pytest_tests/testsuites/container/__init__.py | 0 .../testsuites/container/test_container.py | 2 +- .../testsuites/container/test_policy.py | 4 +-- pytest_tests/testsuites/failovers/__init__.py | 0 .../failovers/test_failover_storage.py | 32 +++++++------------ .../testsuites/management/__init__.py | 0 .../management/test_node_management.py | 2 +- pytest_tests/testsuites/object/__init__.py | 0 .../object/test_object_api_bearer.py | 4 +-- .../testsuites/object/test_object_lifetime.py | 2 +- .../testsuites/object/test_object_lock.py | 2 +- .../testsuites/replication/__init__.py | 0 .../replication/test_ec_replication.py | 6 ++-- pytest_tests/testsuites/services/__init__.py | 0 .../testsuites/services/http_gate/__init__.py | 0 .../services/http_gate/test_http_bearer.py | 2 +- .../services/http_gate/test_http_gate.py | 2 +- .../testsuites/services/s3_gate/__init__.py | 0 .../services/s3_gate/test_s3_policy.py | 6 ++-- 27 files changed, 43 insertions(+), 48 deletions(-) create mode 100644 __init__.py create mode 100644 pytest_tests/testsuites/container/__init__.py create mode 100644 pytest_tests/testsuites/failovers/__init__.py create mode 100644 pytest_tests/testsuites/management/__init__.py create mode 100644 pytest_tests/testsuites/object/__init__.py create mode 100644 pytest_tests/testsuites/replication/__init__.py create mode 100644 pytest_tests/testsuites/services/__init__.py create mode 100644 pytest_tests/testsuites/services/http_gate/__init__.py create mode 100644 pytest_tests/testsuites/services/s3_gate/__init__.py diff --git a/__init__.py b/__init__.py new file mode 100644 index 00000000..4a2630d8 --- /dev/null +++ b/__init__.py @@ -0,0 +1,3 @@ +import os + +TESTS_BASE_PATH = os.path.dirname(os.path.relpath(__file__)) diff --git a/pytest_tests/helpers/container_access.py b/pytest_tests/helpers/container_access.py index f19ab9b7..59194d5c 100644 --- a/pytest_tests/helpers/container_access.py +++ b/pytest_tests/helpers/container_access.py @@ -6,7 +6,7 @@ from frostfs_testlib.storage.cluster import Cluster from frostfs_testlib.storage.dataclasses import ape from frostfs_testlib.storage.dataclasses.wallet import WalletInfo -from pytest_tests.helpers.object_access import ( +from ..helpers.object_access import ( can_delete_object, can_get_head_object, can_get_object, diff --git a/pytest_tests/resources/common.py b/pytest_tests/resources/common.py index ebede363..38cef385 100644 --- a/pytest_tests/resources/common.py +++ b/pytest_tests/resources/common.py @@ -1,5 +1,8 @@ import os +from ... import TESTS_BASE_PATH + TEST_CYCLES_COUNT = int(os.getenv("TEST_CYCLES_COUNT", "1")) DEVENV_PATH = os.getenv("DEVENV_PATH", os.path.join("..", "frostfs-dev-env")) +S3_POLICY_FILE_LOCATION = os.path.join(TESTS_BASE_PATH, "pytest_tests/resources/files/policy.json") diff --git a/pytest_tests/testsuites/access/acl/test_acl.py b/pytest_tests/testsuites/access/acl/test_acl.py index a8d9ebb5..af64d0cf 100644 --- a/pytest_tests/testsuites/access/acl/test_acl.py +++ b/pytest_tests/testsuites/access/acl/test_acl.py @@ -8,8 +8,7 @@ from frostfs_testlib.storage.cluster import Cluster from frostfs_testlib.storage.dataclasses.wallet import WalletInfo from frostfs_testlib.testing.cluster_test_base import ClusterTestBase -from pytest_tests.helpers.container_access import assert_full_access_to_container, assert_no_access_to_container, assert_read_only_container - +from ....helpers.container_access import assert_full_access_to_container, assert_no_access_to_container, assert_read_only_container from ....helpers.container_spec import ContainerSpec diff --git a/pytest_tests/testsuites/access/ape/test_ape.py b/pytest_tests/testsuites/access/ape/test_ape.py index 56874075..f36d038e 100644 --- a/pytest_tests/testsuites/access/ape/test_ape.py +++ b/pytest_tests/testsuites/access/ape/test_ape.py @@ -12,13 +12,12 @@ from frostfs_testlib.utils import wallet_utils from frostfs_testlib.utils.failover_utils import wait_object_replication from frostfs_testlib.utils.file_utils import TestFile -from pytest_tests.helpers.container_access import ( +from ....helpers.container_access import ( ALL_OBJECT_OPERATIONS, assert_access_to_container, assert_full_access_to_container, assert_no_access_to_container, ) - from ....helpers.container_spec import ContainerSpec diff --git a/pytest_tests/testsuites/access/ape/test_ape_filters.py b/pytest_tests/testsuites/access/ape/test_ape_filters.py index 062f3f2d..561ed81a 100644 --- a/pytest_tests/testsuites/access/ape/test_ape_filters.py +++ b/pytest_tests/testsuites/access/ape/test_ape_filters.py @@ -10,17 +10,16 @@ from frostfs_testlib.testing.cluster_test_base import ClusterTestBase from frostfs_testlib.testing.test_control import expect_not_raises from frostfs_testlib.utils.file_utils import TestFile -from pytest_tests.helpers.bearer_token import create_bearer_token -from pytest_tests.helpers.container_access import ( +from ....helpers.bearer_token import create_bearer_token +from ....helpers.container_access import ( ALL_OBJECT_OPERATIONS, FULL_ACCESS, assert_access_to_container, assert_full_access_to_container, assert_no_access_to_container, ) -from pytest_tests.helpers.object_access import OBJECT_ACCESS_DENIED - from ....helpers.container_spec import ContainerSpec +from ....helpers.object_access import OBJECT_ACCESS_DENIED @pytest.mark.nightly diff --git a/pytest_tests/testsuites/access/ape/test_bearer.py b/pytest_tests/testsuites/access/ape/test_bearer.py index 66045ebb..64a89ffd 100644 --- a/pytest_tests/testsuites/access/ape/test_bearer.py +++ b/pytest_tests/testsuites/access/ape/test_bearer.py @@ -7,8 +7,8 @@ from frostfs_testlib.storage.dataclasses.wallet import WalletInfo from frostfs_testlib.testing.cluster_test_base import ClusterTestBase from frostfs_testlib.utils.file_utils import TestFile -from pytest_tests.helpers.bearer_token import create_bearer_token -from pytest_tests.helpers.container_access import ( +from ....helpers.bearer_token import create_bearer_token +from ....helpers.container_access import ( ALL_OBJECT_OPERATIONS, assert_access_to_container, assert_full_access_to_container, diff --git a/pytest_tests/testsuites/conftest.py b/pytest_tests/testsuites/conftest.py index 9dedc53d..31900ff2 100644 --- a/pytest_tests/testsuites/conftest.py +++ b/pytest_tests/testsuites/conftest.py @@ -35,7 +35,7 @@ from frostfs_testlib.testing.test_control import run_optionally, wait_for_succes from frostfs_testlib.utils import env_utils, string_utils, version_utils from frostfs_testlib.utils.file_utils import TestFile, generate_file -from pytest_tests.resources.common import TEST_CYCLES_COUNT +from ..resources.common import TEST_CYCLES_COUNT logger = logging.getLogger("NeoLogger") diff --git a/pytest_tests/testsuites/container/__init__.py b/pytest_tests/testsuites/container/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pytest_tests/testsuites/container/test_container.py b/pytest_tests/testsuites/container/test_container.py index d34a109e..ce1f760d 100644 --- a/pytest_tests/testsuites/container/test_container.py +++ b/pytest_tests/testsuites/container/test_container.py @@ -13,7 +13,7 @@ from frostfs_testlib.steps.cli.container import ( from frostfs_testlib.storage.dataclasses.wallet import WalletInfo from frostfs_testlib.testing.cluster_test_base import ClusterTestBase -from pytest_tests.helpers.utility import placement_policy_from_container +from ...helpers.utility import placement_policy_from_container @pytest.mark.nightly diff --git a/pytest_tests/testsuites/container/test_policy.py b/pytest_tests/testsuites/container/test_policy.py index 3def93c4..f0b9dd28 100644 --- a/pytest_tests/testsuites/container/test_policy.py +++ b/pytest_tests/testsuites/container/test_policy.py @@ -16,8 +16,8 @@ from frostfs_testlib.testing.test_control import wait_for_success from frostfs_testlib.utils.cli_utils import parse_netmap_output from frostfs_testlib.utils.file_utils import generate_file -from pytest_tests.helpers.utility import placement_policy_from_container -from pytest_tests.resources.policy_error_patterns import NOT_ENOUGH_TO_SELECT, NOT_FOUND_FILTER, NOT_FOUND_SELECTOR, NOT_PARSE_POLICY +from ...helpers.utility import placement_policy_from_container +from ...resources.policy_error_patterns import NOT_ENOUGH_TO_SELECT, NOT_FOUND_FILTER, NOT_FOUND_SELECTOR, NOT_PARSE_POLICY @pytest.mark.nightly diff --git a/pytest_tests/testsuites/failovers/__init__.py b/pytest_tests/testsuites/failovers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pytest_tests/testsuites/failovers/test_failover_storage.py b/pytest_tests/testsuites/failovers/test_failover_storage.py index 5f9eba51..f99333f8 100644 --- a/pytest_tests/testsuites/failovers/test_failover_storage.py +++ b/pytest_tests/testsuites/failovers/test_failover_storage.py @@ -34,6 +34,8 @@ from frostfs_testlib.utils.failover_utils import wait_object_replication from frostfs_testlib.utils.file_keeper import FileKeeper from frostfs_testlib.utils.file_utils import generate_file, get_file_hash +from ...resources.common import S3_POLICY_FILE_LOCATION + logger = logging.getLogger("NeoLogger") stopped_nodes: list[StorageNode] = [] @@ -95,9 +97,7 @@ class TestFailoverStorage(ClusterTestBase): ) with reporter.step("Check object data is not corrupted"): - got_file_path = get_object( - wallet, cid, oid, endpoint=replicated_nodes[0].get_rpc_endpoint(), shell=self.shell - ) + got_file_path = get_object(wallet, cid, oid, endpoint=replicated_nodes[0].get_rpc_endpoint(), shell=self.shell) assert get_file_hash(source_file_path) == get_file_hash(got_file_path) with reporter.step("Return all hosts"): @@ -105,12 +105,10 @@ class TestFailoverStorage(ClusterTestBase): with reporter.step("Check object data is not corrupted"): replicated_nodes = wait_object_replication(cid, oid, 2, shell=self.shell, nodes=self.cluster.storage_nodes) - got_file_path = get_object( - wallet, cid, oid, shell=self.shell, endpoint=replicated_nodes[0].get_rpc_endpoint() - ) + got_file_path = get_object(wallet, cid, oid, shell=self.shell, endpoint=replicated_nodes[0].get_rpc_endpoint()) assert get_file_hash(source_file_path) == get_file_hash(got_file_path) - @pytest.mark.parametrize("s3_policy", ["pytest_tests/resources/files/policy.json"], indirect=True) + @pytest.mark.parametrize("s3_policy", [S3_POLICY_FILE_LOCATION], indirect=True) @allure.title("Do not ignore unhealthy tree endpoints (s3_client={s3_client})") def test_unhealthy_tree( self, @@ -151,7 +149,7 @@ class TestFailoverStorage(ClusterTestBase): wallet=default_wallet, shell=self.shell, endpoint=self.cluster.storage_nodes[0].get_rpc_endpoint(), - bucket_container_resolver=bucket_container_resolver + bucket_container_resolver=bucket_container_resolver, )[0] with reporter.step("Turn off all storage nodes except bucket node"): @@ -282,9 +280,7 @@ class TestEmptyMap(ClusterTestBase): cluster_state_controller.stop_services_of_type(StorageNode) with reporter.step("Remove all nodes from network map"): - remove_nodes_from_map_morph( - shell=self.shell, cluster=self.cluster, remove_nodes=self.cluster.services(StorageNode) - ) + remove_nodes_from_map_morph(shell=self.shell, cluster=self.cluster, remove_nodes=self.cluster.services(StorageNode)) with reporter.step("Return all storage nodes to network map"): self.return_nodes_after_stop_with_check_empty_map(cluster_state_controller) @@ -465,9 +461,7 @@ class TestStorageDataLoss(ClusterTestBase): s3_client.put_object(bucket, complex_object_path) with reporter.step("Check objects are in bucket"): - s3_helper.check_objects_in_bucket( - s3_client, bucket, expected_objects=[simple_object_key, complex_object_key] - ) + s3_helper.check_objects_in_bucket(s3_client, bucket, expected_objects=[simple_object_key, complex_object_key]) with reporter.step("Stop storage services on all nodes"): cluster_state_controller.stop_services_of_type(StorageNode) @@ -581,17 +575,13 @@ class TestStorageDataLoss(ClusterTestBase): exception_messages.append(f"Shard {shard} changed status to {status}") with reporter.step("No related errors should be in log"): - if node_under_test.host.is_message_in_logs( - message_regex=r"\Wno such file or directory\W", since=test_start_time - ): + if node_under_test.host.is_message_in_logs(message_regex=r"\Wno such file or directory\W", since=test_start_time): exception_messages.append(f"Node {node_under_test} have shard errors in logs") with reporter.step("Pass test if no errors found"): assert not exception_messages, "\n".join(exception_messages) - @allure.title( - "Loss of one node should trigger use of tree and storage service in another node (s3_client={s3_client})" - ) + @allure.title("Loss of one node should trigger use of tree and storage service in another node (s3_client={s3_client})") def test_s3_one_endpoint_loss( self, bucket, @@ -613,7 +603,7 @@ class TestStorageDataLoss(ClusterTestBase): put_object = s3_client.put_object(bucket, file_path) s3_helper.check_objects_in_bucket(s3_client, bucket, expected_objects=[file_name]) - @pytest.mark.parametrize("s3_policy", ["pytest_tests/resources/files/policy.json"], indirect=True) + @pytest.mark.parametrize("s3_policy", [S3_POLICY_FILE_LOCATION], indirect=True) @allure.title("After Pilorama.db loss on one node object is retrievable (s3_client={s3_client})") def test_s3_one_pilorama_loss( self, diff --git a/pytest_tests/testsuites/management/__init__.py b/pytest_tests/testsuites/management/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pytest_tests/testsuites/management/test_node_management.py b/pytest_tests/testsuites/management/test_node_management.py index a6854862..0948f609 100644 --- a/pytest_tests/testsuites/management/test_node_management.py +++ b/pytest_tests/testsuites/management/test_node_management.py @@ -44,7 +44,7 @@ from frostfs_testlib.utils import string_utils from frostfs_testlib.utils.failover_utils import wait_object_replication from frostfs_testlib.utils.file_utils import generate_file -from pytest_tests.helpers.utility import wait_for_gc_pass_on_storage_nodes +from ...helpers.utility import wait_for_gc_pass_on_storage_nodes logger = logging.getLogger("NeoLogger") check_nodes: list[StorageNode] = [] diff --git a/pytest_tests/testsuites/object/__init__.py b/pytest_tests/testsuites/object/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pytest_tests/testsuites/object/test_object_api_bearer.py b/pytest_tests/testsuites/object/test_object_api_bearer.py index 694dcefc..f6f9d404 100644 --- a/pytest_tests/testsuites/object/test_object_api_bearer.py +++ b/pytest_tests/testsuites/object/test_object_api_bearer.py @@ -21,8 +21,8 @@ from frostfs_testlib.testing.cluster_test_base import ClusterTestBase from frostfs_testlib.testing.test_control import expect_not_raises from pytest import FixtureRequest -from pytest_tests.helpers.bearer_token import create_bearer_token -from pytest_tests.helpers.container_access import assert_full_access_to_container +from ...helpers.bearer_token import create_bearer_token +from ...helpers.container_access import assert_full_access_to_container @pytest.fixture(scope="session") diff --git a/pytest_tests/testsuites/object/test_object_lifetime.py b/pytest_tests/testsuites/object/test_object_lifetime.py index ce4d40ad..7e326e89 100644 --- a/pytest_tests/testsuites/object/test_object_lifetime.py +++ b/pytest_tests/testsuites/object/test_object_lifetime.py @@ -12,7 +12,7 @@ from frostfs_testlib.storage.dataclasses.wallet import WalletInfo from frostfs_testlib.testing.cluster_test_base import ClusterTestBase from frostfs_testlib.utils.file_utils import generate_file, get_file_hash -from pytest_tests.helpers.utility import wait_for_gc_pass_on_storage_nodes +from ...helpers.utility import wait_for_gc_pass_on_storage_nodes logger = logging.getLogger("NeoLogger") diff --git a/pytest_tests/testsuites/object/test_object_lock.py b/pytest_tests/testsuites/object/test_object_lock.py index d1fc3ebc..56591464 100755 --- a/pytest_tests/testsuites/object/test_object_lock.py +++ b/pytest_tests/testsuites/object/test_object_lock.py @@ -31,7 +31,7 @@ from frostfs_testlib.testing.cluster_test_base import ClusterTestBase from frostfs_testlib.testing.test_control import expect_not_raises, wait_for_success from frostfs_testlib.utils import datetime_utils, string_utils -from pytest_tests.helpers.utility import wait_for_gc_pass_on_storage_nodes +from ...helpers.utility import wait_for_gc_pass_on_storage_nodes logger = logging.getLogger("NeoLogger") diff --git a/pytest_tests/testsuites/replication/__init__.py b/pytest_tests/testsuites/replication/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pytest_tests/testsuites/replication/test_ec_replication.py b/pytest_tests/testsuites/replication/test_ec_replication.py index 21ce932e..0877da59 100644 --- a/pytest_tests/testsuites/replication/test_ec_replication.py +++ b/pytest_tests/testsuites/replication/test_ec_replication.py @@ -22,6 +22,8 @@ from frostfs_testlib.testing.test_control import wait_for_success from frostfs_testlib.utils import datetime_utils from frostfs_testlib.utils.file_utils import generate_file, get_file_hash +from ...resources.common import S3_POLICY_FILE_LOCATION + def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: if "ec_policy" not in metafunc.fixturenames: @@ -678,7 +680,7 @@ class TestECReplication(ClusterTestBase): ) @allure.title("Create bucket with EC policy (s3_client={s3_client})") - @pytest.mark.parametrize("s3_policy, s3_client", [("pytest_tests/resources/files/policy.json", AwsCliClient)], indirect=True) + @pytest.mark.parametrize("s3_policy, s3_client", [(S3_POLICY_FILE_LOCATION, AwsCliClient)], indirect=True) def test_create_bucket_with_ec_location( self, s3_client: S3ClientWrapper, bucket_container_resolver: BucketContainerResolver, grpc_client: GrpcClientWrapper ) -> None: @@ -693,7 +695,7 @@ class TestECReplication(ClusterTestBase): assert container @allure.title("Bucket object count chunks (s3_client={s3_client}, size={object_size})") - @pytest.mark.parametrize("s3_policy, s3_client", [("pytest_tests/resources/files/policy.json", AwsCliClient)], indirect=True) + @pytest.mark.parametrize("s3_policy, s3_client", [(S3_POLICY_FILE_LOCATION, AwsCliClient)], indirect=True) def test_count_chunks_bucket_with_ec_location( self, s3_client: S3ClientWrapper, diff --git a/pytest_tests/testsuites/services/__init__.py b/pytest_tests/testsuites/services/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pytest_tests/testsuites/services/http_gate/__init__.py b/pytest_tests/testsuites/services/http_gate/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pytest_tests/testsuites/services/http_gate/test_http_bearer.py b/pytest_tests/testsuites/services/http_gate/test_http_bearer.py index 7d5db791..4e200229 100644 --- a/pytest_tests/testsuites/services/http_gate/test_http_bearer.py +++ b/pytest_tests/testsuites/services/http_gate/test_http_bearer.py @@ -15,7 +15,7 @@ from frostfs_testlib.storage.dataclasses.wallet import WalletInfo from frostfs_testlib.testing.cluster_test_base import ClusterTestBase from frostfs_testlib.utils.file_utils import generate_file -from pytest_tests.helpers.bearer_token import create_bearer_token +from ....helpers.bearer_token import create_bearer_token logger = logging.getLogger("NeoLogger") diff --git a/pytest_tests/testsuites/services/http_gate/test_http_gate.py b/pytest_tests/testsuites/services/http_gate/test_http_gate.py index 1bd736dd..0561f5a8 100644 --- a/pytest_tests/testsuites/services/http_gate/test_http_gate.py +++ b/pytest_tests/testsuites/services/http_gate/test_http_gate.py @@ -20,7 +20,7 @@ from frostfs_testlib.storage.dataclasses.object_size import ObjectSize from frostfs_testlib.testing.cluster_test_base import ClusterTestBase from frostfs_testlib.utils.file_utils import generate_file, get_file_hash -from pytest_tests.helpers.utility import wait_for_gc_pass_on_storage_nodes +from ....helpers.utility import wait_for_gc_pass_on_storage_nodes OBJECT_NOT_FOUND_ERROR = "not found" diff --git a/pytest_tests/testsuites/services/s3_gate/__init__.py b/pytest_tests/testsuites/services/s3_gate/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pytest_tests/testsuites/services/s3_gate/test_s3_policy.py b/pytest_tests/testsuites/services/s3_gate/test_s3_policy.py index 3c6803bb..d5334173 100644 --- a/pytest_tests/testsuites/services/s3_gate/test_s3_policy.py +++ b/pytest_tests/testsuites/services/s3_gate/test_s3_policy.py @@ -1,5 +1,4 @@ import json -import os import allure import pytest @@ -15,10 +14,12 @@ from frostfs_testlib.testing.cluster_test_base import ClusterTestBase from frostfs_testlib.testing.test_control import expect_not_raises from frostfs_testlib.utils.file_utils import generate_file +from ....resources.common import S3_POLICY_FILE_LOCATION + @pytest.mark.nightly @pytest.mark.s3_gate -@pytest.mark.parametrize("s3_policy", ["pytest_tests/resources/files/policy.json"], indirect=True) +@pytest.mark.parametrize("s3_policy", [S3_POLICY_FILE_LOCATION], indirect=True) class TestS3GatePolicy(ClusterTestBase): @allure.title("Bucket creation with retention policy applied (s3_client={s3_client})") def test_s3_bucket_location( @@ -100,7 +101,6 @@ class TestS3GatePolicy(ClusterTestBase): s3_client.get_bucket_policy(bucket) with reporter.step("Put new policy"): - custom_policy = f"file://{os.getcwd()}/pytest_tests/resources/files/bucket_policy.json" custom_policy = { "Version": "2012-10-17", "Id": "aaaa-bbbb-cccc-dddd", From 09acd6f283e3de78ff1f68589fc5935b85e0c28f Mon Sep 17 00:00:00 2001 From: "a.berezin" Date: Wed, 30 Oct 2024 14:19:26 +0300 Subject: [PATCH 5/5] [#316] Move variable under package for single repo runs Signed-off-by: a.berezin --- __init__.py | 3 --- pytest_tests/__init__.py | 3 +++ pytest_tests/resources/common.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/__init__.py b/__init__.py index 4a2630d8..e69de29b 100644 --- a/__init__.py +++ b/__init__.py @@ -1,3 +0,0 @@ -import os - -TESTS_BASE_PATH = os.path.dirname(os.path.relpath(__file__)) diff --git a/pytest_tests/__init__.py b/pytest_tests/__init__.py index e69de29b..4a2630d8 100644 --- a/pytest_tests/__init__.py +++ b/pytest_tests/__init__.py @@ -0,0 +1,3 @@ +import os + +TESTS_BASE_PATH = os.path.dirname(os.path.relpath(__file__)) diff --git a/pytest_tests/resources/common.py b/pytest_tests/resources/common.py index 38cef385..83e8e837 100644 --- a/pytest_tests/resources/common.py +++ b/pytest_tests/resources/common.py @@ -1,8 +1,8 @@ import os -from ... import TESTS_BASE_PATH +from .. import TESTS_BASE_PATH TEST_CYCLES_COUNT = int(os.getenv("TEST_CYCLES_COUNT", "1")) DEVENV_PATH = os.getenv("DEVENV_PATH", os.path.join("..", "frostfs-dev-env")) -S3_POLICY_FILE_LOCATION = os.path.join(TESTS_BASE_PATH, "pytest_tests/resources/files/policy.json") +S3_POLICY_FILE_LOCATION = os.path.join(TESTS_BASE_PATH, "resources/files/policy.json")