ChEktn/frostfs-testlib
1
0
Fork 0
forked from TrueCloudLab/frostfs-testlib
Code Pull requests Activity

Add GenericCli utility

Browse source 
Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
This commit is contained in:
Andrey Berezin 2024-02-14 16:16:59 +03:00
parent 4f3814690e
commit 751381cd60
8 changed files with 80 additions and 71 deletions
Download patch file Download diff file Expand all files Collapse all files

6
src/frostfs_testlib/steps/cli/container.py
View file

@ -8,7 +8,7 @@ from typing import Optional, Union
import requests
from frostfs_testlib import reporter
from frostfs_testlib.cli import FrostfsCli
from frostfs_testlib.cli import FrostfsCli, GenericCli
from frostfs_testlib.resources.cli import CLI_DEFAULT_TIMEOUT, FROSTFS_CLI_EXEC
from frostfs_testlib.resources.common import DEFAULT_WALLET_CONFIG
from frostfs_testlib.shell import Shell
@ -345,8 +345,8 @@ def _parse_cid(output: str) -> str:
@reporter.step("Search container by name")
def search_container_by_name(name: str, node: ClusterNode):
node_shell = node.host.get_shell()
output = node_shell.exec(f"curl -I HEAD http://127.0.0.1:8084/{name}")
curl = GenericCli("curl", node.host)
output = curl(f"-I http://127.0.0.1:8084/{name}")
pattern = r"X-Container-Id: (\S+)"
cid = re.findall(pattern, output.stdout)
if cid:

74
src/frostfs_testlib/steps/http/http_gate.py
View file

@ -11,13 +11,14 @@ from urllib.parse import quote_plus
import requests
from frostfs_testlib import reporter
from frostfs_testlib.cli import GenericCli
from frostfs_testlib.resources.common import SIMPLE_OBJECT_SIZE
from frostfs_testlib.s3.aws_cli_client import command_options
from frostfs_testlib.shell import Shell
from frostfs_testlib.shell.local_shell import LocalShell
from frostfs_testlib.steps.cli.object import get_object
from frostfs_testlib.steps.storage_policy import get_nodes_without_object
from frostfs_testlib.storage.cluster import StorageNode
from frostfs_testlib.storage.cluster import ClusterNode, StorageNode
from frostfs_testlib.testing.test_control import retry
from frostfs_testlib.utils.file_utils import get_file_hash
@ -31,8 +32,7 @@ local_shell = LocalShell()
def get_via_http_gate(
cid: str,
oid: str,
endpoint: str,
http_hostname: str,
node: ClusterNode,
request_path: Optional[str] = None,
timeout: Optional[int] = 300,
):
@ -40,18 +40,19 @@ def get_via_http_gate(
This function gets given object from HTTP gate
cid: container id to get object from
oid: object ID
endpoint: http gate endpoint
http_hostname: http host name on the node
node: node to make request
request_path: (optional) http request, if ommited - use default [{endpoint}/get/{cid}/{oid}]
"""
# if `request_path` parameter omitted, use default
if request_path is None:
request = f"{endpoint}/get/{cid}/{oid}"
request = f"{node.http_gate.get_endpoint()}/get/{cid}/{oid}"
else:
request = f"{endpoint}{request_path}"
request = f"{node.http_gate.get_endpoint()}{request_path}"
resp = requests.get(request, headers={"Host": http_hostname}, stream=True, timeout=timeout, verify=False)
resp = requests.get(
request, headers={"Host": node.storage_node.get_http_hostname()[0]}, stream=True, timeout=timeout, verify=False
)
if not resp.ok:
raise Exception(
@ -72,15 +73,14 @@ def get_via_http_gate(
@reporter.step("Get via Zip HTTP Gate")
def get_via_zip_http_gate(cid: str, prefix: str, endpoint: str, http_hostname: str, timeout: Optional[int] = 300):
def get_via_zip_http_gate(cid: str, prefix: str, node: ClusterNode, timeout: Optional[int] = 300):
"""
This function gets given object from HTTP gate
cid: container id to get object from
prefix: common prefix
endpoint: http gate endpoint
http_hostname: http host name on the node
node: node to make request
"""
request = f"{endpoint}/zip/{cid}/{prefix}"
request = f"{node.http_gate.get_endpoint()}/zip/{cid}/{prefix}"
resp = requests.get(request, stream=True, timeout=timeout, verify=False)
if not resp.ok:
@ -109,8 +109,7 @@ def get_via_zip_http_gate(cid: str, prefix: str, endpoint: str, http_hostname: s
def get_via_http_gate_by_attribute(
cid: str,
attribute: dict,
endpoint: str,
http_hostname: str,
node: ClusterNode,
request_path: Optional[str] = None,
timeout: Optional[int] = 300,
):
@ -126,11 +125,13 @@ def get_via_http_gate_by_attribute(
attr_value = quote_plus(str(attribute.get(attr_name)))
# if `request_path` parameter ommited, use default
if request_path is None:
request = f"{endpoint}/get_by_attribute/{cid}/{quote_plus(str(attr_name))}/{attr_value}"
request = f"{node.http_gate.get_endpoint()}/get_by_attribute/{cid}/{quote_plus(str(attr_name))}/{attr_value}"
else:
request = f"{endpoint}{request_path}"
request = f"{node.http_gate.get_endpoint()}{request_path}"
resp = requests.get(request, stream=True, timeout=timeout, verify=False, headers={"Host": http_hostname})
resp = requests.get(
request, stream=True, timeout=timeout, verify=False, headers={"Host": node.storage_node.get_http_hostname()[0]}
)
if not resp.ok:
raise Exception(
@ -247,19 +248,18 @@ def upload_via_http_gate_curl(
@retry(max_attempts=3, sleep_interval=1)
@reporter.step("Get via HTTP Gate using Curl")
def get_via_http_curl(cid: str, oid: str, endpoint: str, http_hostname: str) -> str:
def get_via_http_curl(cid: str, oid: str, node: ClusterNode) -> str:
"""
This function gets given object from HTTP gate using curl utility.
cid: CID to get object from
oid: object OID
endpoint: http gate endpoint
http_hostname: http host name of the node
node: node for request
"""
request = f"{endpoint}/get/{cid}/{oid}"
request = f"{node.http_gate.get_endpoint()}/get/{cid}/{oid}"
file_path = os.path.join(os.getcwd(), ASSETS_DIR, f"{cid}_{oid}_{str(uuid.uuid4())}")
cmd = f'curl -k -H "Host: {http_hostname}" {request} > {file_path}'
local_shell.exec(cmd)
curl = GenericCli("curl", node.host)
curl(f'-k -H "Host: {node.storage_node.get_http_hostname()[0]}"', f"{request} > {file_path}", shell=local_shell)
return file_path
@ -274,12 +274,11 @@ def _attach_allure_step(request: str, status_code: int, req_type="GET"):
def try_to_get_object_and_expect_error(
cid: str,
oid: str,
node: ClusterNode,
error_pattern: str,
endpoint: str,
http_hostname: str,
) -> None:
try:
get_via_http_gate(cid=cid, oid=oid, endpoint=endpoint, http_hostname=http_hostname)
get_via_http_gate(cid=cid, oid=oid, node=node)
raise AssertionError(f"Expected error on getting object with cid: {cid}")
except Exception as err:
match = error_pattern.casefold() in str(err).casefold()
@ -292,13 +291,10 @@ def get_object_by_attr_and_verify_hashes(
file_name: str,
cid: str,
attrs: dict,
endpoint: str,
http_hostname: str,
node: ClusterNode,
) -> None:
got_file_path_http = get_via_http_gate(cid=cid, oid=oid, endpoint=endpoint, http_hostname=http_hostname)
got_file_path_http_attr = get_via_http_gate_by_attribute(
cid=cid, attribute=attrs, endpoint=endpoint, http_hostname=http_hostname
)
got_file_path_http = get_via_http_gate(cid=cid, oid=oid, node=node)
got_file_path_http_attr = get_via_http_gate_by_attribute(cid=cid, attribute=attrs, node=node)
assert_hashes_are_equal(file_name, got_file_path_http, got_file_path_http_attr)
@ -309,8 +305,7 @@ def verify_object_hash(
cid: str,
shell: Shell,
nodes: list[StorageNode],
endpoint: str,
http_hostname: str,
request_node: ClusterNode,
object_getter=None,
) -> None:
@ -336,7 +331,7 @@ def verify_object_hash(
shell=shell,
endpoint=random_node.get_rpc_endpoint(),
)
got_file_path_http = object_getter(cid=cid, oid=oid, endpoint=endpoint, http_hostname=http_hostname)
got_file_path_http = object_getter(cid=cid, oid=oid, node=request_node)
assert_hashes_are_equal(file_name, got_file_path, got_file_path_http)
@ -365,10 +360,9 @@ def attr_into_str_header_curl(attrs: dict) -> list:
def try_to_get_object_via_passed_request_and_expect_error(
cid: str,
oid: str,
node: ClusterNode,
error_pattern: str,
endpoint: str,
http_request_path: str,
http_hostname: str,
attrs: Optional[dict] = None,
) -> None:
try:
@ -376,17 +370,15 @@ def try_to_get_object_via_passed_request_and_expect_error(
get_via_http_gate(
cid=cid,
oid=oid,
endpoint=endpoint,
node=node,
request_path=http_request_path,
http_hostname=http_hostname,
)
else:
get_via_http_gate_by_attribute(
cid=cid,
attribute=attrs,
endpoint=endpoint,
node=node,
request_path=http_request_path,
http_hostname=http_hostname,
)
raise AssertionError(f"Expected error on getting object with cid: {cid}")
except Exception as err: