dstepanov-yadro
/
frostfs-testcases
forked from TrueCloudLab/frostfs-testcases
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add timeout for cli commands 

Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
fix/compare_sorted_object_id_lists
Andrey Berezin 2023-03-09 14:13:51 +03:00
parent 3802df25fe
commit ac7dae0d2d
5 changed files with 108 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
Makefile
View File

@ -1,28 +1,29 @@
#!/usr/bin/make -f
SHELL := /bin/bash
PYTHON_VERSION := 3.10
VENV_NAME = frostfs-testcases-${PYTHON_VERSION}
VENV_DIR := venv.${VENV_NAME}
.DEFAULT_GOAL := help
current_dir := $(shell pwd)
SHELL ?= bash
venv: create requirements paths precommit
@echo Ready
VENVS = $(shell ls -1d venv/*/ | sort -u | xargs basename -a)
precommit:
@echo Isntalling pre-commit hooks
. ${VENV_DIR}/bin/activate && pre-commit install
.PHONY: all
all: venvs
paths:
@echo Append paths for project
@echo Virtual environment: ${VENV_DIR}
@sudo rm -rf ${VENV_DIR}/lib/python${PYTHON_VERSION}/site-packages/_paths.pth
@sudo touch ${VENV_DIR}/lib/python${PYTHON_VERSION}/site-packages/_paths.pth
@echo ${current_dir} | sudo tee ${VENV_DIR}/lib/python${PYTHON_VERSION}/site-packages/_paths.pth
include venv_template.mk
create:
@echo Create virtual environment for
virtualenv --python=python${PYTHON_VERSION} --prompt=${VENV_NAME} ${VENV_DIR}
.PHONY: venvs
venvs:
$(foreach venv,$(VENVS),venv.$(venv))
$(foreach venv,$(VENVS),$(eval $(call VENV_template,$(venv))))
clean:
rm -rf venv.*
pytest-local:
@echo "⇒ Run Pytest"
python -m pytest pytest_tests/testsuites/
help:
@echo "⇒ run Run testcases ${R}"
requirements:
@echo Isntalling pip requirements
. ${VENV_DIR}/bin/activate && pip install -e ../frostfs-testlib
. ${VENV_DIR}/bin/activate && pip install -Ur pytest_tests/requirements.txt

28
Makefile.old 100644
View File

@ -0,0 +1,28 @@
#!/usr/bin/make -f
.DEFAULT_GOAL := help
SHELL ?= bash
VENVS = $(shell ls -1d venv/*/ | sort -u | xargs basename -a)
.PHONY: all
all: venvs
include venv_template.mk
.PHONY: venvs
venvs:
$(foreach venv,$(VENVS),venv.$(venv))
$(foreach venv,$(VENVS),$(eval $(call VENV_template,$(venv))))
clean:
rm -rf venv.*
pytest-local:
@echo "⇒ Run Pytest"
python -m pytest pytest_tests/testsuites/
help:
@echo "⇒ run Run testcases ${R}"

21
pytest_tests/helpers/container.py
View File

@ -14,7 +14,7 @@ from pytest_tests.helpers.file_helper import generate_file, get_file_hash
from pytest_tests.helpers.frostfs_verbs import put_object, put_object_to_random_node
from pytest_tests.helpers.storage_object_info import StorageObjectInfo
from pytest_tests.helpers.wallet import WalletFile
from pytest_tests.resources.common import FROSTFS_CLI_EXEC, WALLET_CONFIG
from pytest_tests.resources.common import CLI_DEFAULT_TIMEOUT, FROSTFS_CLI_EXEC, WALLET_CONFIG
logger = logging.getLogger("NeoLogger")
@ -115,6 +115,7 @@ def create_container(
options: dict = None,
await_mode: bool = True,
wait_for_creation: bool = True,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> str:
"""
A wrapper for `frostfs-cli container create` call.
@ -136,6 +137,7 @@ def create_container(
name (optional, str): container name attribute
await_mode (bool): block execution until container is persisted
wait_for_creation (): Wait for container shows in container list
timeout: Timeout for the operation.
Returns:
(str): CID of the created container
@ -151,6 +153,7 @@ def create_container(
name=name,
session=session_token,
await_mode=await_mode,
timeout=timeout,
**options or {},
)
@ -194,7 +197,9 @@ def wait_for_container_deletion(
@allure.step("List Containers")
def list_containers(wallet: str, shell: Shell, endpoint: str) -> list[str]:
def list_containers(
wallet: str, shell: Shell, endpoint: str, timeout: Optional[str] = CLI_DEFAULT_TIMEOUT
) -> list[str]:
"""
A wrapper for `frostfs-cli container list` call. It returns all the
available containers for the given wallet.
@ -202,11 +207,12 @@ def list_containers(wallet: str, shell: Shell, endpoint: str) -> list[str]:
wallet (str): a wallet on whose behalf we list the containers
shell: executor for cli command
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
timeout: Timeout for the operation.
Returns:
(list): list of containers
"""
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, WALLET_CONFIG)
result = cli.container.list(rpc_endpoint=endpoint, wallet=wallet)
result = cli.container.list(rpc_endpoint=endpoint, wallet=wallet, timeout=timeout)
logger.info(f"Containers: \n{result}")
return result.stdout.split()
@ -218,6 +224,7 @@ def get_container(
shell: Shell,
endpoint: str,
json_mode: bool = True,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> Union[dict, str]:
"""
A wrapper for `frostfs-cli container get` call. It extracts container's
@ -228,12 +235,15 @@ def get_container(
shell: executor for cli command
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
json_mode (bool): return container in JSON format
timeout: Timeout for the operation.
Returns:
(dict, str): dict of container attributes
"""
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, WALLET_CONFIG)
result = cli.container.get(rpc_endpoint=endpoint, wallet=wallet, cid=cid, json_mode=json_mode)
result = cli.container.get(
rpc_endpoint=endpoint, wallet=wallet, cid=cid, json_mode=json_mode, timeout=timeout
)
if not json_mode:
return result.stdout
@ -258,6 +268,7 @@ def delete_container(
force: bool = False,
session_token: Optional[str] = None,
await_mode: bool = False,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> None:
"""
A wrapper for `frostfs-cli container delete` call.
@ -268,6 +279,7 @@ def delete_container(
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
force (bool): do not check whether container contains locks and remove immediately
session_token: a path to session token file
timeout: Timeout for the operation.
This function doesn't return anything.
"""
@ -279,6 +291,7 @@ def delete_container(
force=force,
session=session_token,
await_mode=await_mode,
timeout=timeout,
)

39
pytest_tests/helpers/frostfs_verbs.py
View File

@ -11,7 +11,12 @@ from frostfs_testlib.shell import Shell
from frostfs_testlib.utils import json_utils
from pytest_tests.helpers.cluster import Cluster
from pytest_tests.resources.common import ASSETS_DIR, FROSTFS_CLI_EXEC, WALLET_CONFIG
from pytest_tests.resources.common import (
ASSETS_DIR,
CLI_DEFAULT_TIMEOUT,
FROSTFS_CLI_EXEC,
WALLET_CONFIG,
)
logger = logging.getLogger("NeoLogger")
@ -29,6 +34,7 @@ def get_object_from_random_node(
wallet_config: Optional[str] = None,
no_progress: bool = True,
session: Optional[str] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> str:
"""
GET from FrostFS random storage node
@ -45,6 +51,7 @@ def get_object_from_random_node(
no_progress(optional, bool): do not show progress bar
xhdr (optional, dict): Request X-Headers in form of Key=Value
session (optional, dict): path to a JSON-encoded container session token
timeout: Timeout for the operation.
Returns:
(str): path to downloaded file
"""
@ -61,6 +68,7 @@ def get_object_from_random_node(
wallet_config,
no_progress,
session,
timeout,
)
@ -77,6 +85,7 @@ def get_object(
wallet_config: Optional[str] = None,
no_progress: bool = True,
session: Optional[str] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> str:
"""
GET from FrostFS.
@ -93,6 +102,7 @@ def get_object(
no_progress(optional, bool): do not show progress bar
xhdr (optional, dict): Request X-Headers in form of Key=Value
session (optional, dict): path to a JSON-encoded container session token
timeout: Timeout for the operation.
Returns:
(str): path to downloaded file
"""
@ -112,6 +122,7 @@ def get_object(
no_progress=no_progress,
xhdr=xhdr,
session=session,
timeout=timeout,
)
return file_path
@ -129,6 +140,7 @@ def get_range_hash(
wallet_config: Optional[str] = None,
xhdr: Optional[dict] = None,
session: Optional[str] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
):
"""
GETRANGEHASH of given Object.
@ -145,6 +157,7 @@ def get_range_hash(
wallet_config: path to the wallet config
xhdr: Request X-Headers in form of Key=Values
session: Filepath to a JSON- or binary-encoded token of the object RANGEHASH session.
timeout: Timeout for the operation.
Returns:
None
"""
@ -158,6 +171,7 @@ def get_range_hash(
bearer=bearer,
xhdr=xhdr,
session=session,
timeout=timeout,
)
# cutting off output about range offset and length
@ -178,6 +192,7 @@ def put_object_to_random_node(
expire_at: Optional[int] = None,
no_progress: bool = True,
session: Optional[str] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
):
"""
PUT of given file to a random storage node.
@ -196,6 +211,7 @@ def put_object_to_random_node(
expire_at: Last epoch in the life of the object
xhdr: Request X-Headers in form of Key=Value
session: path to a JSON-encoded container session token
timeout: Timeout for the operation.
Returns:
ID of uploaded Object
"""
@ -214,6 +230,7 @@ def put_object_to_random_node(
expire_at,
no_progress,
session,
timeout,
)
@ -231,6 +248,7 @@ def put_object(
expire_at: Optional[int] = None,
no_progress: bool = True,
session: Optional[str] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
):
"""
PUT of given file.
@ -248,6 +266,7 @@ def put_object(
expire_at: Last epoch in the life of the object
xhdr: Request X-Headers in form of Key=Value
session: path to a JSON-encoded container session token
timeout: Timeout for the operation.
Returns:
(str): ID of uploaded Object
"""
@ -264,6 +283,7 @@ def put_object(
no_progress=no_progress,
xhdr=xhdr,
session=session,
timeout=timeout,
)
# splitting CLI output to lines and taking the penultimate line
@ -283,6 +303,7 @@ def delete_object(
wallet_config: Optional[str] = None,
xhdr: Optional[dict] = None,
session: Optional[str] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
):
"""
DELETE an Object.
@ -297,6 +318,7 @@ def delete_object(
wallet_config: path to the wallet config
xhdr: Request X-Headers in form of Key=Value
session: path to a JSON-encoded container session token
timeout: Timeout for the operation.
Returns:
(str): Tombstone ID
"""
@ -310,6 +332,7 @@ def delete_object(
bearer=bearer,
xhdr=xhdr,
session=session,
timeout=timeout,
)
id_str = result.stdout.split("\n")[1]
@ -329,6 +352,7 @@ def get_range(
bearer: str = "",
xhdr: Optional[dict] = None,
session: Optional[str] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
):
"""
GETRANGE an Object.
@ -344,6 +368,7 @@ def get_range(
wallet_config: path to the wallet config
xhdr: Request X-Headers in form of Key=Value
session: path to a JSON-encoded container session token
timeout: Timeout for the operation.
Returns:
(str, bytes) - path to the file with range content and content of this file as bytes
"""
@ -360,6 +385,7 @@ def get_range(
bearer=bearer,
xhdr=xhdr,
session=session,
timeout=timeout,
)
with open(range_file_path, "rb") as file:
@ -382,6 +408,7 @@ def lock_object(
wallet_config: Optional[str] = None,
ttl: Optional[int] = None,
xhdr: Optional[dict] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> str:
"""
Lock object in container.
@ -399,6 +426,7 @@ def lock_object(
ttl: TTL value in request meta header (default 2).
wallet: WIF (NEP-2) string or path to the wallet or binary key.
xhdr: Dict with request X-Headers.
timeout: Timeout for the operation.
Returns:
Lock object ID
@ -417,6 +445,7 @@ def lock_object(
xhdr=xhdr,
session=session,
ttl=ttl,
timeout=timeout,
)
# splitting CLI output to lines and taking the penultimate line
@ -439,6 +468,7 @@ def search_object(
session: Optional[str] = None,
phy: bool = False,
root: bool = False,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> list:
"""
SEARCH an Object.
@ -456,6 +486,7 @@ def search_object(
session: path to a JSON-encoded container session token
phy: Search physically stored objects.
root: Search for user objects.
timeout: Timeout for the operation.
Returns:
list of found ObjectIDs
@ -474,6 +505,7 @@ def search_object(
session=session,
phy=phy,
root=root,
timeout=timeout,
)
found_objects = re.findall(r"(\w{43,44})", result.stdout)
@ -502,6 +534,7 @@ def get_netmap_netinfo(
address: Optional[str] = None,
ttl: Optional[int] = None,
xhdr: Optional[dict] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> dict[str, Any]:
"""
Get netmap netinfo output from node
@ -526,6 +559,7 @@ def get_netmap_netinfo(
address=address,
ttl=ttl,
xhdr=xhdr,
timeout=timeout,
)
settings = dict()
@ -556,6 +590,7 @@ def head_object(
is_direct: bool = False,
wallet_config: Optional[str] = None,
session: Optional[str] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
):
"""
HEAD an Object.
@ -576,6 +611,7 @@ def head_object(
wallet_config(optional, str): path to the wallet config
xhdr (optional, dict): Request X-Headers in form of Key=Value
session (optional, dict): path to a JSON-encoded container session token
timeout: Timeout for the operation.
Returns:
depending on the `json_output` parameter value, the function returns
(dict): HEAD response in JSON format
@ -595,6 +631,7 @@ def head_object(
ttl=1 if is_direct else None,
xhdr=xhdr,
session=session,
timeout=timeout,
)
if not json_output:

2
pytest_tests/resources/common.py
View File

@ -45,6 +45,8 @@ STORAGE_NODE_SERVICE_NAME_REGEX = r"s\d\d"
HTTP_GATE_SERVICE_NAME_REGEX = r"http-gate\d\d"
S3_GATE_SERVICE_NAME_REGEX = r"s3-gate\d\d"
CLI_DEFAULT_TIMEOUT = os.getenv("CLI_DEFAULT_TIMEOUT", None)
# Generate wallet configs
# TODO: we should move all info about wallet configs to fixtures
WALLET_CONFIG = os.path.join(os.getcwd(), "wallet_config.yml")