From 06dc226ef84aab86647cd216e36feba42a83debe Mon Sep 17 00:00:00 2001 From: Andrey Berezin Date: Thu, 9 Mar 2023 14:41:38 +0300 Subject: [PATCH] Add timeout for cli commands Signed-off-by: Andrey Berezin --- .../helpers/complex_object_actions.py | 49 ++++++++++++++----- pytest_tests/helpers/epoch.py | 3 +- pytest_tests/helpers/object_access.py | 9 ++++ pytest_tests/helpers/storage_group.py | 19 ++++++- .../testsuites/shard/test_control_shard.py | 3 +- 5 files changed, 69 insertions(+), 14 deletions(-) diff --git a/pytest_tests/helpers/complex_object_actions.py b/pytest_tests/helpers/complex_object_actions.py index 5dc58405..c90f3802 100644 --- a/pytest_tests/helpers/complex_object_actions.py +++ b/pytest_tests/helpers/complex_object_actions.py @@ -20,21 +20,25 @@ from pytest_tests.helpers import frostfs_verbs from pytest_tests.helpers.cluster import Cluster, StorageNode from pytest_tests.helpers.frostfs_verbs import head_object from pytest_tests.helpers.storage_object_info import StorageObjectInfo -from pytest_tests.resources.common import WALLET_CONFIG +from pytest_tests.resources.common import CLI_DEFAULT_TIMEOUT, WALLET_CONFIG logger = logging.getLogger("NeoLogger") def get_storage_object_chunks( - storage_object: StorageObjectInfo, shell: Shell, cluster: Cluster + storage_object: StorageObjectInfo, + shell: Shell, + cluster: Cluster, + timeout: Optional[str] = CLI_DEFAULT_TIMEOUT, ) -> list[str]: """ Get complex object split objects ids (no linker object) Args: - storage_object: storage_object to get it's chunks - shell: client shell to do cmd requests - cluster: cluster object under test + storage_object: storage_object to get it's chunks + shell: client shell to do cmd requests + cluster: cluster object under test + timeout: Timeout for an operation. Returns: list of object ids of complex object chunks @@ -48,6 +52,7 @@ def get_storage_object_chunks( shell, cluster.storage_nodes, is_direct=False, + timeout=timeout, ) head = head_object( storage_object.wallet_file_path, @@ -55,6 +60,7 @@ def get_storage_object_chunks( split_object_id, shell, cluster.default_rpc_endpoint, + timeout=timeout, ) chunks_object_ids = [] @@ -65,7 +71,10 @@ def get_storage_object_chunks( def get_complex_object_split_ranges( - storage_object: StorageObjectInfo, shell: Shell, cluster: Cluster + storage_object: StorageObjectInfo, + shell: Shell, + cluster: Cluster, + timeout: Optional[str] = CLI_DEFAULT_TIMEOUT, ) -> list[Tuple[int, int]]: """ @@ -75,9 +84,10 @@ def get_complex_object_split_ranges( [(0, 30), (30, 30), (60, 30), (90, 10)] Args: - storage_object: storage_object to get it's chunks - shell: client shell to do cmd requests - cluster: cluster object under test + storage_object: storage_object to get it's chunks + shell: client shell to do cmd requests + cluster: cluster object under test + timeout: Timeout for an operation. Returns: list of object ids of complex object chunks @@ -93,6 +103,7 @@ def get_complex_object_split_ranges( chunk_id, shell, cluster.default_rpc_endpoint, + timeout=timeout, ) length = int(head["header"]["payloadLength"]) @@ -113,6 +124,7 @@ def get_link_object( bearer: str = "", wallet_config: str = WALLET_CONFIG, is_direct: bool = True, + timeout: Optional[str] = CLI_DEFAULT_TIMEOUT, ): """ Args: @@ -126,6 +138,7 @@ def get_link_object( wallet_config (optional, str): path to the frostfs-cli config file is_direct: send request directly to the node or not; this flag turns into `--ttl 1` key + timeout: Timeout for an operation. Returns: (str): Link Object ID When no Link Object ID is found after all Storage Nodes polling, @@ -144,6 +157,7 @@ def get_link_object( is_direct=is_direct, bearer=bearer, wallet_config=wallet_config, + timeout=timeout, ) if resp["link"]: return resp["link"] @@ -155,7 +169,12 @@ def get_link_object( @allure.step("Get Last Object") def get_last_object( - wallet: str, cid: str, oid: str, shell: Shell, nodes: list[StorageNode] + wallet: str, + cid: str, + oid: str, + shell: Shell, + nodes: list[StorageNode], + timeout: Optional[str] = CLI_DEFAULT_TIMEOUT, ) -> Optional[str]: """ Args: @@ -165,6 +184,7 @@ def get_last_object( oid (str): Large Object ID shell: executor for cli command nodes: list of nodes to do search on + timeout: Timeout for an operation. Returns: (str): Last Object ID When no Last Object ID is found after all Storage Nodes polling, @@ -174,7 +194,14 @@ def get_last_object( endpoint = node.get_rpc_endpoint() try: resp = frostfs_verbs.head_object( - wallet, cid, oid, shell=shell, endpoint=endpoint, is_raw=True, is_direct=True + wallet, + cid, + oid, + shell=shell, + endpoint=endpoint, + is_raw=True, + is_direct=True, + timeout=timeout, ) if resp["lastPart"]: return resp["lastPart"] diff --git a/pytest_tests/helpers/epoch.py b/pytest_tests/helpers/epoch.py index 1eed15f1..4016af87 100644 --- a/pytest_tests/helpers/epoch.py +++ b/pytest_tests/helpers/epoch.py @@ -11,6 +11,7 @@ from pytest_tests.helpers.cluster import Cluster, StorageNode from pytest_tests.helpers.payment_neogo import get_contract_hash from pytest_tests.helpers.test_control import wait_for_success from pytest_tests.resources.common import ( + CLI_DEFAULT_TIMEOUT, FROSTFS_ADM_CONFIG_PATH, FROSTFS_ADM_EXEC, FROSTFS_CLI_EXEC, @@ -55,7 +56,7 @@ def get_epoch(shell: Shell, cluster: Cluster, alive_node: Optional[StorageNode] cli = FrostfsCli(shell=shell, frostfs_cli_exec_path=FROSTFS_CLI_EXEC, config_file=wallet_config) - epoch = cli.netmap.epoch(endpoint, wallet_path) + epoch = cli.netmap.epoch(endpoint, wallet_path, timeout=CLI_DEFAULT_TIMEOUT) return int(epoch.stdout) diff --git a/pytest_tests/helpers/object_access.py b/pytest_tests/helpers/object_access.py index 905487f5..8f56b947 100644 --- a/pytest_tests/helpers/object_access.py +++ b/pytest_tests/helpers/object_access.py @@ -16,6 +16,7 @@ from pytest_tests.helpers.frostfs_verbs import ( put_object_to_random_node, search_object, ) +from pytest_tests.resources.common import CLI_DEFAULT_TIMEOUT OPERATION_ERROR_TYPE = RuntimeError @@ -123,6 +124,7 @@ def can_get_head_object( bearer: Optional[str] = None, wallet_config: Optional[str] = None, xhdr: Optional[dict] = None, + timeout: Optional[str] = CLI_DEFAULT_TIMEOUT, ) -> bool: with allure.step("Try get head of object"): try: @@ -135,6 +137,7 @@ def can_get_head_object( xhdr=xhdr, shell=shell, endpoint=endpoint, + timeout=timeout, ) except OPERATION_ERROR_TYPE as err: assert string_utils.is_str_match_pattern( @@ -153,6 +156,7 @@ def can_get_range_of_object( bearer: Optional[str] = None, wallet_config: Optional[str] = None, xhdr: Optional[dict] = None, + timeout: Optional[str] = CLI_DEFAULT_TIMEOUT, ) -> bool: with allure.step("Try get range of object"): try: @@ -166,6 +170,7 @@ def can_get_range_of_object( xhdr=xhdr, shell=shell, endpoint=endpoint, + timeout=timeout, ) except OPERATION_ERROR_TYPE as err: assert string_utils.is_str_match_pattern( @@ -184,6 +189,7 @@ def can_get_range_hash_of_object( bearer: Optional[str] = None, wallet_config: Optional[str] = None, xhdr: Optional[dict] = None, + timeout: Optional[str] = CLI_DEFAULT_TIMEOUT, ) -> bool: with allure.step("Try get range hash of object"): try: @@ -197,6 +203,7 @@ def can_get_range_hash_of_object( xhdr=xhdr, shell=shell, endpoint=endpoint, + timeout=timeout, ) except OPERATION_ERROR_TYPE as err: assert string_utils.is_str_match_pattern( @@ -215,6 +222,7 @@ def can_search_object( bearer: Optional[str] = None, wallet_config: Optional[str] = None, xhdr: Optional[dict] = None, + timeout: Optional[str] = CLI_DEFAULT_TIMEOUT, ) -> bool: with allure.step("Try search object in container"): try: @@ -226,6 +234,7 @@ def can_search_object( xhdr=xhdr, shell=shell, endpoint=endpoint, + timeout=timeout, ) except OPERATION_ERROR_TYPE as err: assert string_utils.is_str_match_pattern( diff --git a/pytest_tests/helpers/storage_group.py b/pytest_tests/helpers/storage_group.py index cce22717..3e04441e 100644 --- a/pytest_tests/helpers/storage_group.py +++ b/pytest_tests/helpers/storage_group.py @@ -12,7 +12,7 @@ from frostfs_testlib.shell import Shell from pytest_tests.helpers.cluster import Cluster from pytest_tests.helpers.complex_object_actions import get_link_object from pytest_tests.helpers.frostfs_verbs import head_object -from pytest_tests.resources.common import FROSTFS_CLI_EXEC, WALLET_CONFIG +from pytest_tests.resources.common import CLI_DEFAULT_TIMEOUT, FROSTFS_CLI_EXEC, WALLET_CONFIG logger = logging.getLogger("NeoLogger") @@ -27,6 +27,7 @@ def put_storagegroup( bearer: Optional[str] = None, wallet_config: str = WALLET_CONFIG, lifetime: int = 10, + timeout: Optional[str] = CLI_DEFAULT_TIMEOUT, ) -> str: """ Wrapper for `frostfs-cli storagegroup put`. Before the SG is created, @@ -40,6 +41,7 @@ def put_storagegroup( objects: List of Object IDs to include into the SG. bearer: Path to Bearer token file. wallet_config: Path to frostfs-cli config file. + timeout: Timeout for an operation. Returns: Object ID of created Storage Group. """ @@ -66,6 +68,7 @@ def list_storagegroup( cid: str, bearer: Optional[str] = None, wallet_config: str = WALLET_CONFIG, + timeout: Optional[str] = CLI_DEFAULT_TIMEOUT, ) -> list: """ Wrapper for `frostfs-cli storagegroup list`. This operation @@ -76,6 +79,7 @@ def list_storagegroup( cid: ID of Container to list. bearer: Path to Bearer token file. wallet_config: Path to frostfs-cli config file. + timeout: Timeout for an operation. Returns: Object IDs of found Storage Groups. """ @@ -87,6 +91,7 @@ def list_storagegroup( cid=cid, bearer=bearer, rpc_endpoint=endpoint, + timeout=timeout, ) # throwing off the first string of output found_objects = result.stdout.split("\n")[1:] @@ -102,6 +107,7 @@ def get_storagegroup( gid: str, bearer: str = "", wallet_config: str = WALLET_CONFIG, + timeout: Optional[str] = CLI_DEFAULT_TIMEOUT, ) -> dict: """ Wrapper for `frostfs-cli storagegroup get`. @@ -112,6 +118,7 @@ def get_storagegroup( gid: ID of the Storage Group. bearer: Path to Bearer token file. wallet_config: Path to frostfs-cli config file. + timeout: Timeout for an operation. Returns: Detailed information on the Storage Group. """ @@ -124,6 +131,7 @@ def get_storagegroup( bearer=bearer, id=gid, rpc_endpoint=endpoint, + timeout=timeout, ) # TODO: temporary solution for parsing output. Needs to be replaced with @@ -153,6 +161,7 @@ def delete_storagegroup( gid: str, bearer: str = "", wallet_config: str = WALLET_CONFIG, + timeout: Optional[str] = CLI_DEFAULT_TIMEOUT, ) -> str: """ Wrapper for `frostfs-cli storagegroup delete`. @@ -163,6 +172,7 @@ def delete_storagegroup( gid: ID of the Storage Group. bearer: Path to Bearer token file. wallet_config: Path to frostfs-cli config file. + timeout: Timeout for an operation. Returns: Tombstone ID of the deleted Storage Group. """ @@ -175,6 +185,7 @@ def delete_storagegroup( bearer=bearer, id=gid, rpc_endpoint=endpoint, + timeout=timeout, ) tombstone_id = result.stdout.strip().split("\n")[1].split(": ")[1] return tombstone_id @@ -189,6 +200,7 @@ def verify_list_storage_group( gid: str, bearer: str = None, wallet_config: str = WALLET_CONFIG, + timeout: Optional[str] = CLI_DEFAULT_TIMEOUT, ): storage_groups = list_storagegroup( shell=shell, @@ -197,6 +209,7 @@ def verify_list_storage_group( cid=cid, bearer=bearer, wallet_config=wallet_config, + timeout=timeout, ) assert gid in storage_groups @@ -213,6 +226,7 @@ def verify_get_storage_group( max_object_size: int, bearer: str = None, wallet_config: str = WALLET_CONFIG, + timeout: Optional[str] = CLI_DEFAULT_TIMEOUT, ): obj_parts = [] endpoint = cluster.default_rpc_endpoint @@ -226,6 +240,7 @@ def verify_get_storage_group( nodes=cluster.storage_nodes, bearer=bearer, wallet_config=wallet_config, + timeout=timeout, ) obj_head = head_object( wallet=wallet, @@ -236,6 +251,7 @@ def verify_get_storage_group( is_raw=True, bearer=bearer, wallet_config=wallet_config, + timeout=timeout, ) obj_parts = obj_head["header"]["split"]["children"] @@ -248,6 +264,7 @@ def verify_get_storage_group( gid=gid, bearer=bearer, wallet_config=wallet_config, + timeout=timeout, ) exp_size = object_size * obj_num if object_size < max_object_size: diff --git a/pytest_tests/testsuites/shard/test_control_shard.py b/pytest_tests/testsuites/shard/test_control_shard.py index f3f0a747..0676d61b 100644 --- a/pytest_tests/testsuites/shard/test_control_shard.py +++ b/pytest_tests/testsuites/shard/test_control_shard.py @@ -11,7 +11,7 @@ from configobj import ConfigObj from frostfs_testlib.cli import FrostfsCli from pytest_tests.helpers.cluster import Cluster, StorageNode -from pytest_tests.resources.common import WALLET_CONFIG +from pytest_tests.resources.common import CLI_DEFAULT_TIMEOUT, WALLET_CONFIG SHARD_PREFIX = "FROSTFS_STORAGE_SHARD_" BLOBSTOR_PREFIX = "_BLOBSTOR_" @@ -143,6 +143,7 @@ class TestControlShard: wallet=wallet_path, wallet_password=wallet_password, json_mode=True, + timeout=CLI_DEFAULT_TIMEOUT, ) return [Shard.from_object(shard) for shard in json.loads(result.stdout.split(">", 1)[1])]