Add timeout for cli commands

Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
This commit is contained in:
Andrey Berezin 2023-03-09 14:41:38 +03:00
parent ac7dae0d2d
commit 06dc226ef8
5 changed files with 69 additions and 14 deletions

View file

@ -20,13 +20,16 @@ from pytest_tests.helpers import frostfs_verbs
from pytest_tests.helpers.cluster import Cluster, StorageNode from pytest_tests.helpers.cluster import Cluster, StorageNode
from pytest_tests.helpers.frostfs_verbs import head_object from pytest_tests.helpers.frostfs_verbs import head_object
from pytest_tests.helpers.storage_object_info import StorageObjectInfo from pytest_tests.helpers.storage_object_info import StorageObjectInfo
from pytest_tests.resources.common import WALLET_CONFIG from pytest_tests.resources.common import CLI_DEFAULT_TIMEOUT, WALLET_CONFIG
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
def get_storage_object_chunks( def get_storage_object_chunks(
storage_object: StorageObjectInfo, shell: Shell, cluster: Cluster storage_object: StorageObjectInfo,
shell: Shell,
cluster: Cluster,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> list[str]: ) -> list[str]:
""" """
Get complex object split objects ids (no linker object) Get complex object split objects ids (no linker object)
@ -35,6 +38,7 @@ def get_storage_object_chunks(
storage_object: storage_object to get it's chunks storage_object: storage_object to get it's chunks
shell: client shell to do cmd requests shell: client shell to do cmd requests
cluster: cluster object under test cluster: cluster object under test
timeout: Timeout for an operation.
Returns: Returns:
list of object ids of complex object chunks list of object ids of complex object chunks
@ -48,6 +52,7 @@ def get_storage_object_chunks(
shell, shell,
cluster.storage_nodes, cluster.storage_nodes,
is_direct=False, is_direct=False,
timeout=timeout,
) )
head = head_object( head = head_object(
storage_object.wallet_file_path, storage_object.wallet_file_path,
@ -55,6 +60,7 @@ def get_storage_object_chunks(
split_object_id, split_object_id,
shell, shell,
cluster.default_rpc_endpoint, cluster.default_rpc_endpoint,
timeout=timeout,
) )
chunks_object_ids = [] chunks_object_ids = []
@ -65,7 +71,10 @@ def get_storage_object_chunks(
def get_complex_object_split_ranges( def get_complex_object_split_ranges(
storage_object: StorageObjectInfo, shell: Shell, cluster: Cluster storage_object: StorageObjectInfo,
shell: Shell,
cluster: Cluster,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> list[Tuple[int, int]]: ) -> list[Tuple[int, int]]:
""" """
@ -78,6 +87,7 @@ def get_complex_object_split_ranges(
storage_object: storage_object to get it's chunks storage_object: storage_object to get it's chunks
shell: client shell to do cmd requests shell: client shell to do cmd requests
cluster: cluster object under test cluster: cluster object under test
timeout: Timeout for an operation.
Returns: Returns:
list of object ids of complex object chunks list of object ids of complex object chunks
@ -93,6 +103,7 @@ def get_complex_object_split_ranges(
chunk_id, chunk_id,
shell, shell,
cluster.default_rpc_endpoint, cluster.default_rpc_endpoint,
timeout=timeout,
) )
length = int(head["header"]["payloadLength"]) length = int(head["header"]["payloadLength"])
@ -113,6 +124,7 @@ def get_link_object(
bearer: str = "", bearer: str = "",
wallet_config: str = WALLET_CONFIG, wallet_config: str = WALLET_CONFIG,
is_direct: bool = True, is_direct: bool = True,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
): ):
""" """
Args: Args:
@ -126,6 +138,7 @@ def get_link_object(
wallet_config (optional, str): path to the frostfs-cli config file wallet_config (optional, str): path to the frostfs-cli config file
is_direct: send request directly to the node or not; this flag is_direct: send request directly to the node or not; this flag
turns into `--ttl 1` key turns into `--ttl 1` key
timeout: Timeout for an operation.
Returns: Returns:
(str): Link Object ID (str): Link Object ID
When no Link Object ID is found after all Storage Nodes polling, When no Link Object ID is found after all Storage Nodes polling,
@ -144,6 +157,7 @@ def get_link_object(
is_direct=is_direct, is_direct=is_direct,
bearer=bearer, bearer=bearer,
wallet_config=wallet_config, wallet_config=wallet_config,
timeout=timeout,
) )
if resp["link"]: if resp["link"]:
return resp["link"] return resp["link"]
@ -155,7 +169,12 @@ def get_link_object(
@allure.step("Get Last Object") @allure.step("Get Last Object")
def get_last_object( def get_last_object(
wallet: str, cid: str, oid: str, shell: Shell, nodes: list[StorageNode] wallet: str,
cid: str,
oid: str,
shell: Shell,
nodes: list[StorageNode],
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> Optional[str]: ) -> Optional[str]:
""" """
Args: Args:
@ -165,6 +184,7 @@ def get_last_object(
oid (str): Large Object ID oid (str): Large Object ID
shell: executor for cli command shell: executor for cli command
nodes: list of nodes to do search on nodes: list of nodes to do search on
timeout: Timeout for an operation.
Returns: Returns:
(str): Last Object ID (str): Last Object ID
When no Last Object ID is found after all Storage Nodes polling, When no Last Object ID is found after all Storage Nodes polling,
@ -174,7 +194,14 @@ def get_last_object(
endpoint = node.get_rpc_endpoint() endpoint = node.get_rpc_endpoint()
try: try:
resp = frostfs_verbs.head_object( resp = frostfs_verbs.head_object(
wallet, cid, oid, shell=shell, endpoint=endpoint, is_raw=True, is_direct=True wallet,
cid,
oid,
shell=shell,
endpoint=endpoint,
is_raw=True,
is_direct=True,
timeout=timeout,
) )
if resp["lastPart"]: if resp["lastPart"]:
return resp["lastPart"] return resp["lastPart"]

View file

@ -11,6 +11,7 @@ from pytest_tests.helpers.cluster import Cluster, StorageNode
from pytest_tests.helpers.payment_neogo import get_contract_hash from pytest_tests.helpers.payment_neogo import get_contract_hash
from pytest_tests.helpers.test_control import wait_for_success from pytest_tests.helpers.test_control import wait_for_success
from pytest_tests.resources.common import ( from pytest_tests.resources.common import (
CLI_DEFAULT_TIMEOUT,
FROSTFS_ADM_CONFIG_PATH, FROSTFS_ADM_CONFIG_PATH,
FROSTFS_ADM_EXEC, FROSTFS_ADM_EXEC,
FROSTFS_CLI_EXEC, FROSTFS_CLI_EXEC,
@ -55,7 +56,7 @@ def get_epoch(shell: Shell, cluster: Cluster, alive_node: Optional[StorageNode]
cli = FrostfsCli(shell=shell, frostfs_cli_exec_path=FROSTFS_CLI_EXEC, config_file=wallet_config) cli = FrostfsCli(shell=shell, frostfs_cli_exec_path=FROSTFS_CLI_EXEC, config_file=wallet_config)
epoch = cli.netmap.epoch(endpoint, wallet_path) epoch = cli.netmap.epoch(endpoint, wallet_path, timeout=CLI_DEFAULT_TIMEOUT)
return int(epoch.stdout) return int(epoch.stdout)

View file

@ -16,6 +16,7 @@ from pytest_tests.helpers.frostfs_verbs import (
put_object_to_random_node, put_object_to_random_node,
search_object, search_object,
) )
from pytest_tests.resources.common import CLI_DEFAULT_TIMEOUT
OPERATION_ERROR_TYPE = RuntimeError OPERATION_ERROR_TYPE = RuntimeError
@ -123,6 +124,7 @@ def can_get_head_object(
bearer: Optional[str] = None, bearer: Optional[str] = None,
wallet_config: Optional[str] = None, wallet_config: Optional[str] = None,
xhdr: Optional[dict] = None, xhdr: Optional[dict] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> bool: ) -> bool:
with allure.step("Try get head of object"): with allure.step("Try get head of object"):
try: try:
@ -135,6 +137,7 @@ def can_get_head_object(
xhdr=xhdr, xhdr=xhdr,
shell=shell, shell=shell,
endpoint=endpoint, endpoint=endpoint,
timeout=timeout,
) )
except OPERATION_ERROR_TYPE as err: except OPERATION_ERROR_TYPE as err:
assert string_utils.is_str_match_pattern( assert string_utils.is_str_match_pattern(
@ -153,6 +156,7 @@ def can_get_range_of_object(
bearer: Optional[str] = None, bearer: Optional[str] = None,
wallet_config: Optional[str] = None, wallet_config: Optional[str] = None,
xhdr: Optional[dict] = None, xhdr: Optional[dict] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> bool: ) -> bool:
with allure.step("Try get range of object"): with allure.step("Try get range of object"):
try: try:
@ -166,6 +170,7 @@ def can_get_range_of_object(
xhdr=xhdr, xhdr=xhdr,
shell=shell, shell=shell,
endpoint=endpoint, endpoint=endpoint,
timeout=timeout,
) )
except OPERATION_ERROR_TYPE as err: except OPERATION_ERROR_TYPE as err:
assert string_utils.is_str_match_pattern( assert string_utils.is_str_match_pattern(
@ -184,6 +189,7 @@ def can_get_range_hash_of_object(
bearer: Optional[str] = None, bearer: Optional[str] = None,
wallet_config: Optional[str] = None, wallet_config: Optional[str] = None,
xhdr: Optional[dict] = None, xhdr: Optional[dict] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> bool: ) -> bool:
with allure.step("Try get range hash of object"): with allure.step("Try get range hash of object"):
try: try:
@ -197,6 +203,7 @@ def can_get_range_hash_of_object(
xhdr=xhdr, xhdr=xhdr,
shell=shell, shell=shell,
endpoint=endpoint, endpoint=endpoint,
timeout=timeout,
) )
except OPERATION_ERROR_TYPE as err: except OPERATION_ERROR_TYPE as err:
assert string_utils.is_str_match_pattern( assert string_utils.is_str_match_pattern(
@ -215,6 +222,7 @@ def can_search_object(
bearer: Optional[str] = None, bearer: Optional[str] = None,
wallet_config: Optional[str] = None, wallet_config: Optional[str] = None,
xhdr: Optional[dict] = None, xhdr: Optional[dict] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> bool: ) -> bool:
with allure.step("Try search object in container"): with allure.step("Try search object in container"):
try: try:
@ -226,6 +234,7 @@ def can_search_object(
xhdr=xhdr, xhdr=xhdr,
shell=shell, shell=shell,
endpoint=endpoint, endpoint=endpoint,
timeout=timeout,
) )
except OPERATION_ERROR_TYPE as err: except OPERATION_ERROR_TYPE as err:
assert string_utils.is_str_match_pattern( assert string_utils.is_str_match_pattern(

View file

@ -12,7 +12,7 @@ from frostfs_testlib.shell import Shell
from pytest_tests.helpers.cluster import Cluster from pytest_tests.helpers.cluster import Cluster
from pytest_tests.helpers.complex_object_actions import get_link_object from pytest_tests.helpers.complex_object_actions import get_link_object
from pytest_tests.helpers.frostfs_verbs import head_object from pytest_tests.helpers.frostfs_verbs import head_object
from pytest_tests.resources.common import FROSTFS_CLI_EXEC, WALLET_CONFIG from pytest_tests.resources.common import CLI_DEFAULT_TIMEOUT, FROSTFS_CLI_EXEC, WALLET_CONFIG
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
@ -27,6 +27,7 @@ def put_storagegroup(
bearer: Optional[str] = None, bearer: Optional[str] = None,
wallet_config: str = WALLET_CONFIG, wallet_config: str = WALLET_CONFIG,
lifetime: int = 10, lifetime: int = 10,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> str: ) -> str:
""" """
Wrapper for `frostfs-cli storagegroup put`. Before the SG is created, Wrapper for `frostfs-cli storagegroup put`. Before the SG is created,
@ -40,6 +41,7 @@ def put_storagegroup(
objects: List of Object IDs to include into the SG. objects: List of Object IDs to include into the SG.
bearer: Path to Bearer token file. bearer: Path to Bearer token file.
wallet_config: Path to frostfs-cli config file. wallet_config: Path to frostfs-cli config file.
timeout: Timeout for an operation.
Returns: Returns:
Object ID of created Storage Group. Object ID of created Storage Group.
""" """
@ -66,6 +68,7 @@ def list_storagegroup(
cid: str, cid: str,
bearer: Optional[str] = None, bearer: Optional[str] = None,
wallet_config: str = WALLET_CONFIG, wallet_config: str = WALLET_CONFIG,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> list: ) -> list:
""" """
Wrapper for `frostfs-cli storagegroup list`. This operation Wrapper for `frostfs-cli storagegroup list`. This operation
@ -76,6 +79,7 @@ def list_storagegroup(
cid: ID of Container to list. cid: ID of Container to list.
bearer: Path to Bearer token file. bearer: Path to Bearer token file.
wallet_config: Path to frostfs-cli config file. wallet_config: Path to frostfs-cli config file.
timeout: Timeout for an operation.
Returns: Returns:
Object IDs of found Storage Groups. Object IDs of found Storage Groups.
""" """
@ -87,6 +91,7 @@ def list_storagegroup(
cid=cid, cid=cid,
bearer=bearer, bearer=bearer,
rpc_endpoint=endpoint, rpc_endpoint=endpoint,
timeout=timeout,
) )
# throwing off the first string of output # throwing off the first string of output
found_objects = result.stdout.split("\n")[1:] found_objects = result.stdout.split("\n")[1:]
@ -102,6 +107,7 @@ def get_storagegroup(
gid: str, gid: str,
bearer: str = "", bearer: str = "",
wallet_config: str = WALLET_CONFIG, wallet_config: str = WALLET_CONFIG,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> dict: ) -> dict:
""" """
Wrapper for `frostfs-cli storagegroup get`. Wrapper for `frostfs-cli storagegroup get`.
@ -112,6 +118,7 @@ def get_storagegroup(
gid: ID of the Storage Group. gid: ID of the Storage Group.
bearer: Path to Bearer token file. bearer: Path to Bearer token file.
wallet_config: Path to frostfs-cli config file. wallet_config: Path to frostfs-cli config file.
timeout: Timeout for an operation.
Returns: Returns:
Detailed information on the Storage Group. Detailed information on the Storage Group.
""" """
@ -124,6 +131,7 @@ def get_storagegroup(
bearer=bearer, bearer=bearer,
id=gid, id=gid,
rpc_endpoint=endpoint, rpc_endpoint=endpoint,
timeout=timeout,
) )
# TODO: temporary solution for parsing output. Needs to be replaced with # TODO: temporary solution for parsing output. Needs to be replaced with
@ -153,6 +161,7 @@ def delete_storagegroup(
gid: str, gid: str,
bearer: str = "", bearer: str = "",
wallet_config: str = WALLET_CONFIG, wallet_config: str = WALLET_CONFIG,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> str: ) -> str:
""" """
Wrapper for `frostfs-cli storagegroup delete`. Wrapper for `frostfs-cli storagegroup delete`.
@ -163,6 +172,7 @@ def delete_storagegroup(
gid: ID of the Storage Group. gid: ID of the Storage Group.
bearer: Path to Bearer token file. bearer: Path to Bearer token file.
wallet_config: Path to frostfs-cli config file. wallet_config: Path to frostfs-cli config file.
timeout: Timeout for an operation.
Returns: Returns:
Tombstone ID of the deleted Storage Group. Tombstone ID of the deleted Storage Group.
""" """
@ -175,6 +185,7 @@ def delete_storagegroup(
bearer=bearer, bearer=bearer,
id=gid, id=gid,
rpc_endpoint=endpoint, rpc_endpoint=endpoint,
timeout=timeout,
) )
tombstone_id = result.stdout.strip().split("\n")[1].split(": ")[1] tombstone_id = result.stdout.strip().split("\n")[1].split(": ")[1]
return tombstone_id return tombstone_id
@ -189,6 +200,7 @@ def verify_list_storage_group(
gid: str, gid: str,
bearer: str = None, bearer: str = None,
wallet_config: str = WALLET_CONFIG, wallet_config: str = WALLET_CONFIG,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
): ):
storage_groups = list_storagegroup( storage_groups = list_storagegroup(
shell=shell, shell=shell,
@ -197,6 +209,7 @@ def verify_list_storage_group(
cid=cid, cid=cid,
bearer=bearer, bearer=bearer,
wallet_config=wallet_config, wallet_config=wallet_config,
timeout=timeout,
) )
assert gid in storage_groups assert gid in storage_groups
@ -213,6 +226,7 @@ def verify_get_storage_group(
max_object_size: int, max_object_size: int,
bearer: str = None, bearer: str = None,
wallet_config: str = WALLET_CONFIG, wallet_config: str = WALLET_CONFIG,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
): ):
obj_parts = [] obj_parts = []
endpoint = cluster.default_rpc_endpoint endpoint = cluster.default_rpc_endpoint
@ -226,6 +240,7 @@ def verify_get_storage_group(
nodes=cluster.storage_nodes, nodes=cluster.storage_nodes,
bearer=bearer, bearer=bearer,
wallet_config=wallet_config, wallet_config=wallet_config,
timeout=timeout,
) )
obj_head = head_object( obj_head = head_object(
wallet=wallet, wallet=wallet,
@ -236,6 +251,7 @@ def verify_get_storage_group(
is_raw=True, is_raw=True,
bearer=bearer, bearer=bearer,
wallet_config=wallet_config, wallet_config=wallet_config,
timeout=timeout,
) )
obj_parts = obj_head["header"]["split"]["children"] obj_parts = obj_head["header"]["split"]["children"]
@ -248,6 +264,7 @@ def verify_get_storage_group(
gid=gid, gid=gid,
bearer=bearer, bearer=bearer,
wallet_config=wallet_config, wallet_config=wallet_config,
timeout=timeout,
) )
exp_size = object_size * obj_num exp_size = object_size * obj_num
if object_size < max_object_size: if object_size < max_object_size:

View file

@ -11,7 +11,7 @@ from configobj import ConfigObj
from frostfs_testlib.cli import FrostfsCli from frostfs_testlib.cli import FrostfsCli
from pytest_tests.helpers.cluster import Cluster, StorageNode from pytest_tests.helpers.cluster import Cluster, StorageNode
from pytest_tests.resources.common import WALLET_CONFIG from pytest_tests.resources.common import CLI_DEFAULT_TIMEOUT, WALLET_CONFIG
SHARD_PREFIX = "FROSTFS_STORAGE_SHARD_" SHARD_PREFIX = "FROSTFS_STORAGE_SHARD_"
BLOBSTOR_PREFIX = "_BLOBSTOR_" BLOBSTOR_PREFIX = "_BLOBSTOR_"
@ -143,6 +143,7 @@ class TestControlShard:
wallet=wallet_path, wallet=wallet_path,
wallet_password=wallet_password, wallet_password=wallet_password,
json_mode=True, json_mode=True,
timeout=CLI_DEFAULT_TIMEOUT,
) )
return [Shard.from_object(shard) for shard in json.loads(result.stdout.split(">", 1)[1])] return [Shard.from_object(shard) for shard in json.loads(result.stdout.split(">", 1)[1])]