diff --git a/Makefile b/Makefile index db276b74..e9a37dfc 100644 --- a/Makefile +++ b/Makefile @@ -1,28 +1,29 @@ -#!/usr/bin/make -f +SHELL := /bin/bash +PYTHON_VERSION := 3.10 +VENV_NAME = frostfs-testcases-${PYTHON_VERSION} +VENV_DIR := venv.${VENV_NAME} -.DEFAULT_GOAL := help +current_dir := $(shell pwd) -SHELL ?= bash +venv: create requirements paths precommit + @echo Ready -VENVS = $(shell ls -1d venv/*/ | sort -u | xargs basename -a) +precommit: + @echo Isntalling pre-commit hooks + . ${VENV_DIR}/bin/activate && pre-commit install -.PHONY: all -all: venvs +paths: + @echo Append paths for project + @echo Virtual environment: ${VENV_DIR} + @sudo rm -rf ${VENV_DIR}/lib/python${PYTHON_VERSION}/site-packages/_paths.pth + @sudo touch ${VENV_DIR}/lib/python${PYTHON_VERSION}/site-packages/_paths.pth + @echo ${current_dir} | sudo tee ${VENV_DIR}/lib/python${PYTHON_VERSION}/site-packages/_paths.pth -include venv_template.mk +create: + @echo Create virtual environment for + virtualenv --python=python${PYTHON_VERSION} --prompt=${VENV_NAME} ${VENV_DIR} -.PHONY: venvs -venvs: - $(foreach venv,$(VENVS),venv.$(venv)) - -$(foreach venv,$(VENVS),$(eval $(call VENV_template,$(venv)))) - -clean: - rm -rf venv.* - -pytest-local: - @echo "⇒ Run Pytest" - python -m pytest pytest_tests/testsuites/ - -help: - @echo "⇒ run Run testcases ${R}" +requirements: + @echo Isntalling pip requirements + . ${VENV_DIR}/bin/activate && pip install -e ../frostfs-testlib + . ${VENV_DIR}/bin/activate && pip install -Ur pytest_tests/requirements.txt \ No newline at end of file diff --git a/Makefile.old b/Makefile.old new file mode 100644 index 00000000..db276b74 --- /dev/null +++ b/Makefile.old @@ -0,0 +1,28 @@ +#!/usr/bin/make -f + +.DEFAULT_GOAL := help + +SHELL ?= bash + +VENVS = $(shell ls -1d venv/*/ | sort -u | xargs basename -a) + +.PHONY: all +all: venvs + +include venv_template.mk + +.PHONY: venvs +venvs: + $(foreach venv,$(VENVS),venv.$(venv)) + +$(foreach venv,$(VENVS),$(eval $(call VENV_template,$(venv)))) + +clean: + rm -rf venv.* + +pytest-local: + @echo "⇒ Run Pytest" + python -m pytest pytest_tests/testsuites/ + +help: + @echo "⇒ run Run testcases ${R}" diff --git a/pytest_tests/helpers/container.py b/pytest_tests/helpers/container.py index eba48929..93b0d3cc 100644 --- a/pytest_tests/helpers/container.py +++ b/pytest_tests/helpers/container.py @@ -14,7 +14,7 @@ from pytest_tests.helpers.file_helper import generate_file, get_file_hash from pytest_tests.helpers.frostfs_verbs import put_object, put_object_to_random_node from pytest_tests.helpers.storage_object_info import StorageObjectInfo from pytest_tests.helpers.wallet import WalletFile -from pytest_tests.resources.common import FROSTFS_CLI_EXEC, WALLET_CONFIG +from pytest_tests.resources.common import CLI_DEFAULT_TIMEOUT, FROSTFS_CLI_EXEC, WALLET_CONFIG logger = logging.getLogger("NeoLogger") @@ -115,6 +115,7 @@ def create_container( options: dict = None, await_mode: bool = True, wait_for_creation: bool = True, + timeout: Optional[str] = CLI_DEFAULT_TIMEOUT, ) -> str: """ A wrapper for `frostfs-cli container create` call. @@ -136,6 +137,7 @@ def create_container( name (optional, str): container name attribute await_mode (bool): block execution until container is persisted wait_for_creation (): Wait for container shows in container list + timeout: Timeout for the operation. Returns: (str): CID of the created container @@ -151,6 +153,7 @@ def create_container( name=name, session=session_token, await_mode=await_mode, + timeout=timeout, **options or {}, ) @@ -194,7 +197,9 @@ def wait_for_container_deletion( @allure.step("List Containers") -def list_containers(wallet: str, shell: Shell, endpoint: str) -> list[str]: +def list_containers( + wallet: str, shell: Shell, endpoint: str, timeout: Optional[str] = CLI_DEFAULT_TIMEOUT +) -> list[str]: """ A wrapper for `frostfs-cli container list` call. It returns all the available containers for the given wallet. @@ -202,11 +207,12 @@ def list_containers(wallet: str, shell: Shell, endpoint: str) -> list[str]: wallet (str): a wallet on whose behalf we list the containers shell: executor for cli command endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key + timeout: Timeout for the operation. Returns: (list): list of containers """ cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, WALLET_CONFIG) - result = cli.container.list(rpc_endpoint=endpoint, wallet=wallet) + result = cli.container.list(rpc_endpoint=endpoint, wallet=wallet, timeout=timeout) logger.info(f"Containers: \n{result}") return result.stdout.split() @@ -218,6 +224,7 @@ def get_container( shell: Shell, endpoint: str, json_mode: bool = True, + timeout: Optional[str] = CLI_DEFAULT_TIMEOUT, ) -> Union[dict, str]: """ A wrapper for `frostfs-cli container get` call. It extracts container's @@ -228,12 +235,15 @@ def get_container( shell: executor for cli command endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key json_mode (bool): return container in JSON format + timeout: Timeout for the operation. Returns: (dict, str): dict of container attributes """ cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, WALLET_CONFIG) - result = cli.container.get(rpc_endpoint=endpoint, wallet=wallet, cid=cid, json_mode=json_mode) + result = cli.container.get( + rpc_endpoint=endpoint, wallet=wallet, cid=cid, json_mode=json_mode, timeout=timeout + ) if not json_mode: return result.stdout @@ -258,6 +268,7 @@ def delete_container( force: bool = False, session_token: Optional[str] = None, await_mode: bool = False, + timeout: Optional[str] = CLI_DEFAULT_TIMEOUT, ) -> None: """ A wrapper for `frostfs-cli container delete` call. @@ -268,6 +279,7 @@ def delete_container( endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key force (bool): do not check whether container contains locks and remove immediately session_token: a path to session token file + timeout: Timeout for the operation. This function doesn't return anything. """ @@ -279,6 +291,7 @@ def delete_container( force=force, session=session_token, await_mode=await_mode, + timeout=timeout, ) diff --git a/pytest_tests/helpers/frostfs_verbs.py b/pytest_tests/helpers/frostfs_verbs.py index 21b52294..aa58132b 100644 --- a/pytest_tests/helpers/frostfs_verbs.py +++ b/pytest_tests/helpers/frostfs_verbs.py @@ -11,7 +11,12 @@ from frostfs_testlib.shell import Shell from frostfs_testlib.utils import json_utils from pytest_tests.helpers.cluster import Cluster -from pytest_tests.resources.common import ASSETS_DIR, FROSTFS_CLI_EXEC, WALLET_CONFIG +from pytest_tests.resources.common import ( + ASSETS_DIR, + CLI_DEFAULT_TIMEOUT, + FROSTFS_CLI_EXEC, + WALLET_CONFIG, +) logger = logging.getLogger("NeoLogger") @@ -29,6 +34,7 @@ def get_object_from_random_node( wallet_config: Optional[str] = None, no_progress: bool = True, session: Optional[str] = None, + timeout: Optional[str] = CLI_DEFAULT_TIMEOUT, ) -> str: """ GET from FrostFS random storage node @@ -45,6 +51,7 @@ def get_object_from_random_node( no_progress(optional, bool): do not show progress bar xhdr (optional, dict): Request X-Headers in form of Key=Value session (optional, dict): path to a JSON-encoded container session token + timeout: Timeout for the operation. Returns: (str): path to downloaded file """ @@ -61,6 +68,7 @@ def get_object_from_random_node( wallet_config, no_progress, session, + timeout, ) @@ -77,6 +85,7 @@ def get_object( wallet_config: Optional[str] = None, no_progress: bool = True, session: Optional[str] = None, + timeout: Optional[str] = CLI_DEFAULT_TIMEOUT, ) -> str: """ GET from FrostFS. @@ -93,6 +102,7 @@ def get_object( no_progress(optional, bool): do not show progress bar xhdr (optional, dict): Request X-Headers in form of Key=Value session (optional, dict): path to a JSON-encoded container session token + timeout: Timeout for the operation. Returns: (str): path to downloaded file """ @@ -112,6 +122,7 @@ def get_object( no_progress=no_progress, xhdr=xhdr, session=session, + timeout=timeout, ) return file_path @@ -129,6 +140,7 @@ def get_range_hash( wallet_config: Optional[str] = None, xhdr: Optional[dict] = None, session: Optional[str] = None, + timeout: Optional[str] = CLI_DEFAULT_TIMEOUT, ): """ GETRANGEHASH of given Object. @@ -145,6 +157,7 @@ def get_range_hash( wallet_config: path to the wallet config xhdr: Request X-Headers in form of Key=Values session: Filepath to a JSON- or binary-encoded token of the object RANGEHASH session. + timeout: Timeout for the operation. Returns: None """ @@ -158,6 +171,7 @@ def get_range_hash( bearer=bearer, xhdr=xhdr, session=session, + timeout=timeout, ) # cutting off output about range offset and length @@ -178,6 +192,7 @@ def put_object_to_random_node( expire_at: Optional[int] = None, no_progress: bool = True, session: Optional[str] = None, + timeout: Optional[str] = CLI_DEFAULT_TIMEOUT, ): """ PUT of given file to a random storage node. @@ -196,6 +211,7 @@ def put_object_to_random_node( expire_at: Last epoch in the life of the object xhdr: Request X-Headers in form of Key=Value session: path to a JSON-encoded container session token + timeout: Timeout for the operation. Returns: ID of uploaded Object """ @@ -214,6 +230,7 @@ def put_object_to_random_node( expire_at, no_progress, session, + timeout, ) @@ -231,6 +248,7 @@ def put_object( expire_at: Optional[int] = None, no_progress: bool = True, session: Optional[str] = None, + timeout: Optional[str] = CLI_DEFAULT_TIMEOUT, ): """ PUT of given file. @@ -248,6 +266,7 @@ def put_object( expire_at: Last epoch in the life of the object xhdr: Request X-Headers in form of Key=Value session: path to a JSON-encoded container session token + timeout: Timeout for the operation. Returns: (str): ID of uploaded Object """ @@ -264,6 +283,7 @@ def put_object( no_progress=no_progress, xhdr=xhdr, session=session, + timeout=timeout, ) # splitting CLI output to lines and taking the penultimate line @@ -283,6 +303,7 @@ def delete_object( wallet_config: Optional[str] = None, xhdr: Optional[dict] = None, session: Optional[str] = None, + timeout: Optional[str] = CLI_DEFAULT_TIMEOUT, ): """ DELETE an Object. @@ -297,6 +318,7 @@ def delete_object( wallet_config: path to the wallet config xhdr: Request X-Headers in form of Key=Value session: path to a JSON-encoded container session token + timeout: Timeout for the operation. Returns: (str): Tombstone ID """ @@ -310,6 +332,7 @@ def delete_object( bearer=bearer, xhdr=xhdr, session=session, + timeout=timeout, ) id_str = result.stdout.split("\n")[1] @@ -329,6 +352,7 @@ def get_range( bearer: str = "", xhdr: Optional[dict] = None, session: Optional[str] = None, + timeout: Optional[str] = CLI_DEFAULT_TIMEOUT, ): """ GETRANGE an Object. @@ -344,6 +368,7 @@ def get_range( wallet_config: path to the wallet config xhdr: Request X-Headers in form of Key=Value session: path to a JSON-encoded container session token + timeout: Timeout for the operation. Returns: (str, bytes) - path to the file with range content and content of this file as bytes """ @@ -360,6 +385,7 @@ def get_range( bearer=bearer, xhdr=xhdr, session=session, + timeout=timeout, ) with open(range_file_path, "rb") as file: @@ -382,6 +408,7 @@ def lock_object( wallet_config: Optional[str] = None, ttl: Optional[int] = None, xhdr: Optional[dict] = None, + timeout: Optional[str] = CLI_DEFAULT_TIMEOUT, ) -> str: """ Lock object in container. @@ -399,6 +426,7 @@ def lock_object( ttl: TTL value in request meta header (default 2). wallet: WIF (NEP-2) string or path to the wallet or binary key. xhdr: Dict with request X-Headers. + timeout: Timeout for the operation. Returns: Lock object ID @@ -417,6 +445,7 @@ def lock_object( xhdr=xhdr, session=session, ttl=ttl, + timeout=timeout, ) # splitting CLI output to lines and taking the penultimate line @@ -439,6 +468,7 @@ def search_object( session: Optional[str] = None, phy: bool = False, root: bool = False, + timeout: Optional[str] = CLI_DEFAULT_TIMEOUT, ) -> list: """ SEARCH an Object. @@ -456,6 +486,7 @@ def search_object( session: path to a JSON-encoded container session token phy: Search physically stored objects. root: Search for user objects. + timeout: Timeout for the operation. Returns: list of found ObjectIDs @@ -474,6 +505,7 @@ def search_object( session=session, phy=phy, root=root, + timeout=timeout, ) found_objects = re.findall(r"(\w{43,44})", result.stdout) @@ -502,6 +534,7 @@ def get_netmap_netinfo( address: Optional[str] = None, ttl: Optional[int] = None, xhdr: Optional[dict] = None, + timeout: Optional[str] = CLI_DEFAULT_TIMEOUT, ) -> dict[str, Any]: """ Get netmap netinfo output from node @@ -526,6 +559,7 @@ def get_netmap_netinfo( address=address, ttl=ttl, xhdr=xhdr, + timeout=timeout, ) settings = dict() @@ -556,6 +590,7 @@ def head_object( is_direct: bool = False, wallet_config: Optional[str] = None, session: Optional[str] = None, + timeout: Optional[str] = CLI_DEFAULT_TIMEOUT, ): """ HEAD an Object. @@ -576,6 +611,7 @@ def head_object( wallet_config(optional, str): path to the wallet config xhdr (optional, dict): Request X-Headers in form of Key=Value session (optional, dict): path to a JSON-encoded container session token + timeout: Timeout for the operation. Returns: depending on the `json_output` parameter value, the function returns (dict): HEAD response in JSON format @@ -595,6 +631,7 @@ def head_object( ttl=1 if is_direct else None, xhdr=xhdr, session=session, + timeout=timeout, ) if not json_output: diff --git a/pytest_tests/resources/common.py b/pytest_tests/resources/common.py index 1f81786c..b616c00b 100644 --- a/pytest_tests/resources/common.py +++ b/pytest_tests/resources/common.py @@ -45,6 +45,8 @@ STORAGE_NODE_SERVICE_NAME_REGEX = r"s\d\d" HTTP_GATE_SERVICE_NAME_REGEX = r"http-gate\d\d" S3_GATE_SERVICE_NAME_REGEX = r"s3-gate\d\d" +CLI_DEFAULT_TIMEOUT = os.getenv("CLI_DEFAULT_TIMEOUT", None) + # Generate wallet configs # TODO: we should move all info about wallet configs to fixtures WALLET_CONFIG = os.path.join(os.getcwd(), "wallet_config.yml")