[#315] Use abspath for files #315

Merged
abereziny merged 1 commit from abereziny/frostfs-testcases:feature-abspath-for-files into master 2024-10-29 12:25:37 +00:00
27 changed files with 43 additions and 48 deletions
Showing only changes of commit b7669fc96f - Show all commits

3
__init__.py Normal file
View file

@ -0,0 +1,3 @@
import os
TESTS_BASE_PATH = os.path.dirname(os.path.relpath(__file__))

View file

@ -6,7 +6,7 @@ from frostfs_testlib.storage.cluster import Cluster
from frostfs_testlib.storage.dataclasses import ape from frostfs_testlib.storage.dataclasses import ape
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from pytest_tests.helpers.object_access import ( from ..helpers.object_access import (
can_delete_object, can_delete_object,
can_get_head_object, can_get_head_object,
can_get_object, can_get_object,

View file

@ -1,5 +1,8 @@
import os import os
from ... import TESTS_BASE_PATH
TEST_CYCLES_COUNT = int(os.getenv("TEST_CYCLES_COUNT", "1")) TEST_CYCLES_COUNT = int(os.getenv("TEST_CYCLES_COUNT", "1"))
DEVENV_PATH = os.getenv("DEVENV_PATH", os.path.join("..", "frostfs-dev-env")) DEVENV_PATH = os.getenv("DEVENV_PATH", os.path.join("..", "frostfs-dev-env"))
S3_POLICY_FILE_LOCATION = os.path.join(TESTS_BASE_PATH, "pytest_tests/resources/files/policy.json")

View file

@ -8,8 +8,7 @@ from frostfs_testlib.storage.cluster import Cluster
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from pytest_tests.helpers.container_access import assert_full_access_to_container, assert_no_access_to_container, assert_read_only_container from ....helpers.container_access import assert_full_access_to_container, assert_no_access_to_container, assert_read_only_container
from ....helpers.container_spec import ContainerSpec from ....helpers.container_spec import ContainerSpec

View file

@ -12,13 +12,12 @@ from frostfs_testlib.utils import wallet_utils
from frostfs_testlib.utils.failover_utils import wait_object_replication from frostfs_testlib.utils.failover_utils import wait_object_replication
from frostfs_testlib.utils.file_utils import TestFile from frostfs_testlib.utils.file_utils import TestFile
from pytest_tests.helpers.container_access import ( from ....helpers.container_access import (
ALL_OBJECT_OPERATIONS, ALL_OBJECT_OPERATIONS,
assert_access_to_container, assert_access_to_container,
assert_full_access_to_container, assert_full_access_to_container,
assert_no_access_to_container, assert_no_access_to_container,
) )
from ....helpers.container_spec import ContainerSpec from ....helpers.container_spec import ContainerSpec

View file

@ -10,17 +10,16 @@ from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.testing.test_control import expect_not_raises from frostfs_testlib.testing.test_control import expect_not_raises
from frostfs_testlib.utils.file_utils import TestFile from frostfs_testlib.utils.file_utils import TestFile
from pytest_tests.helpers.bearer_token import create_bearer_token from ....helpers.bearer_token import create_bearer_token
from pytest_tests.helpers.container_access import ( from ....helpers.container_access import (
ALL_OBJECT_OPERATIONS, ALL_OBJECT_OPERATIONS,
FULL_ACCESS, FULL_ACCESS,
assert_access_to_container, assert_access_to_container,
assert_full_access_to_container, assert_full_access_to_container,
assert_no_access_to_container, assert_no_access_to_container,
) )
from pytest_tests.helpers.object_access import OBJECT_ACCESS_DENIED
from ....helpers.container_spec import ContainerSpec from ....helpers.container_spec import ContainerSpec
from ....helpers.object_access import OBJECT_ACCESS_DENIED
@pytest.mark.nightly @pytest.mark.nightly

View file

@ -7,8 +7,8 @@ from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.utils.file_utils import TestFile from frostfs_testlib.utils.file_utils import TestFile
from pytest_tests.helpers.bearer_token import create_bearer_token from ....helpers.bearer_token import create_bearer_token
from pytest_tests.helpers.container_access import ( from ....helpers.container_access import (
ALL_OBJECT_OPERATIONS, ALL_OBJECT_OPERATIONS,
assert_access_to_container, assert_access_to_container,
assert_full_access_to_container, assert_full_access_to_container,

View file

@ -35,7 +35,7 @@ from frostfs_testlib.testing.test_control import run_optionally, wait_for_succes
from frostfs_testlib.utils import env_utils, string_utils, version_utils from frostfs_testlib.utils import env_utils, string_utils, version_utils
from frostfs_testlib.utils.file_utils import TestFile, generate_file from frostfs_testlib.utils.file_utils import TestFile, generate_file
from pytest_tests.resources.common import TEST_CYCLES_COUNT from ..resources.common import TEST_CYCLES_COUNT
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")

View file

@ -13,7 +13,7 @@ from frostfs_testlib.steps.cli.container import (
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from pytest_tests.helpers.utility import placement_policy_from_container from ...helpers.utility import placement_policy_from_container
@pytest.mark.nightly @pytest.mark.nightly

View file

@ -16,8 +16,8 @@ from frostfs_testlib.testing.test_control import wait_for_success
from frostfs_testlib.utils.cli_utils import parse_netmap_output from frostfs_testlib.utils.cli_utils import parse_netmap_output
from frostfs_testlib.utils.file_utils import generate_file from frostfs_testlib.utils.file_utils import generate_file
from pytest_tests.helpers.utility import placement_policy_from_container from ...helpers.utility import placement_policy_from_container
from pytest_tests.resources.policy_error_patterns import NOT_ENOUGH_TO_SELECT, NOT_FOUND_FILTER, NOT_FOUND_SELECTOR, NOT_PARSE_POLICY from ...resources.policy_error_patterns import NOT_ENOUGH_TO_SELECT, NOT_FOUND_FILTER, NOT_FOUND_SELECTOR, NOT_PARSE_POLICY
@pytest.mark.nightly @pytest.mark.nightly

View file

@ -34,6 +34,8 @@ from frostfs_testlib.utils.failover_utils import wait_object_replication
from frostfs_testlib.utils.file_keeper import FileKeeper from frostfs_testlib.utils.file_keeper import FileKeeper
from frostfs_testlib.utils.file_utils import generate_file, get_file_hash from frostfs_testlib.utils.file_utils import generate_file, get_file_hash
from ...resources.common import S3_POLICY_FILE_LOCATION
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
stopped_nodes: list[StorageNode] = [] stopped_nodes: list[StorageNode] = []
@ -95,9 +97,7 @@ class TestFailoverStorage(ClusterTestBase):
) )
with reporter.step("Check object data is not corrupted"): with reporter.step("Check object data is not corrupted"):
got_file_path = get_object( got_file_path = get_object(wallet, cid, oid, endpoint=replicated_nodes[0].get_rpc_endpoint(), shell=self.shell)
wallet, cid, oid, endpoint=replicated_nodes[0].get_rpc_endpoint(), shell=self.shell
)
assert get_file_hash(source_file_path) == get_file_hash(got_file_path) assert get_file_hash(source_file_path) == get_file_hash(got_file_path)
with reporter.step("Return all hosts"): with reporter.step("Return all hosts"):
@ -105,12 +105,10 @@ class TestFailoverStorage(ClusterTestBase):
with reporter.step("Check object data is not corrupted"): with reporter.step("Check object data is not corrupted"):
replicated_nodes = wait_object_replication(cid, oid, 2, shell=self.shell, nodes=self.cluster.storage_nodes) replicated_nodes = wait_object_replication(cid, oid, 2, shell=self.shell, nodes=self.cluster.storage_nodes)
got_file_path = get_object( got_file_path = get_object(wallet, cid, oid, shell=self.shell, endpoint=replicated_nodes[0].get_rpc_endpoint())
wallet, cid, oid, shell=self.shell, endpoint=replicated_nodes[0].get_rpc_endpoint()
)
assert get_file_hash(source_file_path) == get_file_hash(got_file_path) assert get_file_hash(source_file_path) == get_file_hash(got_file_path)
@pytest.mark.parametrize("s3_policy", ["pytest_tests/resources/files/policy.json"], indirect=True) @pytest.mark.parametrize("s3_policy", [S3_POLICY_FILE_LOCATION], indirect=True)
@allure.title("Do not ignore unhealthy tree endpoints (s3_client={s3_client})") @allure.title("Do not ignore unhealthy tree endpoints (s3_client={s3_client})")
def test_unhealthy_tree( def test_unhealthy_tree(
self, self,
@ -151,7 +149,7 @@ class TestFailoverStorage(ClusterTestBase):
wallet=default_wallet, wallet=default_wallet,
shell=self.shell, shell=self.shell,
endpoint=self.cluster.storage_nodes[0].get_rpc_endpoint(), endpoint=self.cluster.storage_nodes[0].get_rpc_endpoint(),
bucket_container_resolver=bucket_container_resolver bucket_container_resolver=bucket_container_resolver,
)[0] )[0]
with reporter.step("Turn off all storage nodes except bucket node"): with reporter.step("Turn off all storage nodes except bucket node"):
@ -282,9 +280,7 @@ class TestEmptyMap(ClusterTestBase):
cluster_state_controller.stop_services_of_type(StorageNode) cluster_state_controller.stop_services_of_type(StorageNode)
with reporter.step("Remove all nodes from network map"): with reporter.step("Remove all nodes from network map"):
remove_nodes_from_map_morph( remove_nodes_from_map_morph(shell=self.shell, cluster=self.cluster, remove_nodes=self.cluster.services(StorageNode))
shell=self.shell, cluster=self.cluster, remove_nodes=self.cluster.services(StorageNode)
)
with reporter.step("Return all storage nodes to network map"): with reporter.step("Return all storage nodes to network map"):
self.return_nodes_after_stop_with_check_empty_map(cluster_state_controller) self.return_nodes_after_stop_with_check_empty_map(cluster_state_controller)
@ -465,9 +461,7 @@ class TestStorageDataLoss(ClusterTestBase):
s3_client.put_object(bucket, complex_object_path) s3_client.put_object(bucket, complex_object_path)
with reporter.step("Check objects are in bucket"): with reporter.step("Check objects are in bucket"):
s3_helper.check_objects_in_bucket( s3_helper.check_objects_in_bucket(s3_client, bucket, expected_objects=[simple_object_key, complex_object_key])
s3_client, bucket, expected_objects=[simple_object_key, complex_object_key]
)
with reporter.step("Stop storage services on all nodes"): with reporter.step("Stop storage services on all nodes"):
cluster_state_controller.stop_services_of_type(StorageNode) cluster_state_controller.stop_services_of_type(StorageNode)
@ -581,17 +575,13 @@ class TestStorageDataLoss(ClusterTestBase):
exception_messages.append(f"Shard {shard} changed status to {status}") exception_messages.append(f"Shard {shard} changed status to {status}")
with reporter.step("No related errors should be in log"): with reporter.step("No related errors should be in log"):
if node_under_test.host.is_message_in_logs( if node_under_test.host.is_message_in_logs(message_regex=r"\Wno such file or directory\W", since=test_start_time):
message_regex=r"\Wno such file or directory\W", since=test_start_time
):
exception_messages.append(f"Node {node_under_test} have shard errors in logs") exception_messages.append(f"Node {node_under_test} have shard errors in logs")
with reporter.step("Pass test if no errors found"): with reporter.step("Pass test if no errors found"):
assert not exception_messages, "\n".join(exception_messages) assert not exception_messages, "\n".join(exception_messages)
@allure.title( @allure.title("Loss of one node should trigger use of tree and storage service in another node (s3_client={s3_client})")
"Loss of one node should trigger use of tree and storage service in another node (s3_client={s3_client})"
)
def test_s3_one_endpoint_loss( def test_s3_one_endpoint_loss(
self, self,
bucket, bucket,
@ -613,7 +603,7 @@ class TestStorageDataLoss(ClusterTestBase):
put_object = s3_client.put_object(bucket, file_path) put_object = s3_client.put_object(bucket, file_path)
s3_helper.check_objects_in_bucket(s3_client, bucket, expected_objects=[file_name]) s3_helper.check_objects_in_bucket(s3_client, bucket, expected_objects=[file_name])
@pytest.mark.parametrize("s3_policy", ["pytest_tests/resources/files/policy.json"], indirect=True) @pytest.mark.parametrize("s3_policy", [S3_POLICY_FILE_LOCATION], indirect=True)
@allure.title("After Pilorama.db loss on one node object is retrievable (s3_client={s3_client})") @allure.title("After Pilorama.db loss on one node object is retrievable (s3_client={s3_client})")
def test_s3_one_pilorama_loss( def test_s3_one_pilorama_loss(
self, self,

View file

@ -44,7 +44,7 @@ from frostfs_testlib.utils import string_utils
from frostfs_testlib.utils.failover_utils import wait_object_replication from frostfs_testlib.utils.failover_utils import wait_object_replication
from frostfs_testlib.utils.file_utils import generate_file from frostfs_testlib.utils.file_utils import generate_file
from pytest_tests.helpers.utility import wait_for_gc_pass_on_storage_nodes from ...helpers.utility import wait_for_gc_pass_on_storage_nodes
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
check_nodes: list[StorageNode] = [] check_nodes: list[StorageNode] = []

View file

@ -21,8 +21,8 @@ from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.testing.test_control import expect_not_raises from frostfs_testlib.testing.test_control import expect_not_raises
from pytest import FixtureRequest from pytest import FixtureRequest
from pytest_tests.helpers.bearer_token import create_bearer_token from ...helpers.bearer_token import create_bearer_token
from pytest_tests.helpers.container_access import assert_full_access_to_container from ...helpers.container_access import assert_full_access_to_container
@pytest.fixture(scope="session") @pytest.fixture(scope="session")

View file

@ -12,7 +12,7 @@ from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.utils.file_utils import generate_file, get_file_hash from frostfs_testlib.utils.file_utils import generate_file, get_file_hash
from pytest_tests.helpers.utility import wait_for_gc_pass_on_storage_nodes from ...helpers.utility import wait_for_gc_pass_on_storage_nodes
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")

View file

@ -31,7 +31,7 @@ from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.testing.test_control import expect_not_raises, wait_for_success from frostfs_testlib.testing.test_control import expect_not_raises, wait_for_success
from frostfs_testlib.utils import datetime_utils, string_utils from frostfs_testlib.utils import datetime_utils, string_utils
from pytest_tests.helpers.utility import wait_for_gc_pass_on_storage_nodes from ...helpers.utility import wait_for_gc_pass_on_storage_nodes
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")

View file

@ -22,6 +22,8 @@ from frostfs_testlib.testing.test_control import wait_for_success
from frostfs_testlib.utils import datetime_utils from frostfs_testlib.utils import datetime_utils
from frostfs_testlib.utils.file_utils import generate_file, get_file_hash from frostfs_testlib.utils.file_utils import generate_file, get_file_hash
from ...resources.common import S3_POLICY_FILE_LOCATION
def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
if "ec_policy" not in metafunc.fixturenames: if "ec_policy" not in metafunc.fixturenames:
@ -678,7 +680,7 @@ class TestECReplication(ClusterTestBase):
) )
@allure.title("Create bucket with EC policy (s3_client={s3_client})") @allure.title("Create bucket with EC policy (s3_client={s3_client})")
@pytest.mark.parametrize("s3_policy, s3_client", [("pytest_tests/resources/files/policy.json", AwsCliClient)], indirect=True) @pytest.mark.parametrize("s3_policy, s3_client", [(S3_POLICY_FILE_LOCATION, AwsCliClient)], indirect=True)
def test_create_bucket_with_ec_location( def test_create_bucket_with_ec_location(
self, s3_client: S3ClientWrapper, bucket_container_resolver: BucketContainerResolver, grpc_client: GrpcClientWrapper self, s3_client: S3ClientWrapper, bucket_container_resolver: BucketContainerResolver, grpc_client: GrpcClientWrapper
) -> None: ) -> None:
@ -693,7 +695,7 @@ class TestECReplication(ClusterTestBase):
assert container assert container
@allure.title("Bucket object count chunks (s3_client={s3_client}, size={object_size})") @allure.title("Bucket object count chunks (s3_client={s3_client}, size={object_size})")
@pytest.mark.parametrize("s3_policy, s3_client", [("pytest_tests/resources/files/policy.json", AwsCliClient)], indirect=True) @pytest.mark.parametrize("s3_policy, s3_client", [(S3_POLICY_FILE_LOCATION, AwsCliClient)], indirect=True)
def test_count_chunks_bucket_with_ec_location( def test_count_chunks_bucket_with_ec_location(
self, self,
s3_client: S3ClientWrapper, s3_client: S3ClientWrapper,

View file

@ -15,7 +15,7 @@ from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.utils.file_utils import generate_file from frostfs_testlib.utils.file_utils import generate_file
from pytest_tests.helpers.bearer_token import create_bearer_token from ....helpers.bearer_token import create_bearer_token
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")

View file

@ -20,7 +20,7 @@ from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.utils.file_utils import generate_file, get_file_hash from frostfs_testlib.utils.file_utils import generate_file, get_file_hash
from pytest_tests.helpers.utility import wait_for_gc_pass_on_storage_nodes from ....helpers.utility import wait_for_gc_pass_on_storage_nodes
OBJECT_NOT_FOUND_ERROR = "not found" OBJECT_NOT_FOUND_ERROR = "not found"

View file

@ -1,5 +1,4 @@
import json import json
import os
import allure import allure
import pytest import pytest
@ -15,10 +14,12 @@ from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.testing.test_control import expect_not_raises from frostfs_testlib.testing.test_control import expect_not_raises
from frostfs_testlib.utils.file_utils import generate_file from frostfs_testlib.utils.file_utils import generate_file
from ....resources.common import S3_POLICY_FILE_LOCATION
@pytest.mark.nightly @pytest.mark.nightly
@pytest.mark.s3_gate @pytest.mark.s3_gate
@pytest.mark.parametrize("s3_policy", ["pytest_tests/resources/files/policy.json"], indirect=True) @pytest.mark.parametrize("s3_policy", [S3_POLICY_FILE_LOCATION], indirect=True)
class TestS3GatePolicy(ClusterTestBase): class TestS3GatePolicy(ClusterTestBase):
@allure.title("Bucket creation with retention policy applied (s3_client={s3_client})") @allure.title("Bucket creation with retention policy applied (s3_client={s3_client})")
def test_s3_bucket_location( def test_s3_bucket_location(
@ -100,7 +101,6 @@ class TestS3GatePolicy(ClusterTestBase):
s3_client.get_bucket_policy(bucket) s3_client.get_bucket_policy(bucket)
with reporter.step("Put new policy"): with reporter.step("Put new policy"):
custom_policy = f"file://{os.getcwd()}/pytest_tests/resources/files/bucket_policy.json"
custom_policy = { custom_policy = {
"Version": "2012-10-17", "Version": "2012-10-17",
"Id": "aaaa-bbbb-cccc-dddd", "Id": "aaaa-bbbb-cccc-dddd",