forked from TrueCloudLab/frostfs-testlib
Add local shell and small fix
Signed-off-by: Dmitriy Zayakin <d.zayakin@yadro.com>
This commit is contained in:
parent
f3c160f313
commit
137fd21561
4 changed files with 113 additions and 122 deletions
|
|
@ -12,12 +12,13 @@ import requests
|
|||
|
||||
from frostfs_testlib.reporter import get_reporter
|
||||
from frostfs_testlib.resources.common import SIMPLE_OBJECT_SIZE
|
||||
from frostfs_testlib.s3.aws_cli_client import LONG_TIMEOUT
|
||||
from frostfs_testlib.s3.aws_cli_client import command_options
|
||||
from frostfs_testlib.shell import Shell
|
||||
from frostfs_testlib.shell.local_shell import LocalShell
|
||||
from frostfs_testlib.steps.cli.object import get_object
|
||||
from frostfs_testlib.steps.storage_policy import get_nodes_without_object
|
||||
from frostfs_testlib.storage.cluster import StorageNode
|
||||
from frostfs_testlib.utils.cli_utils import _cmd_run
|
||||
from frostfs_testlib.testing.test_control import retry
|
||||
from frostfs_testlib.utils.file_utils import get_file_hash
|
||||
|
||||
reporter = get_reporter()
|
||||
|
|
@ -25,6 +26,7 @@ reporter = get_reporter()
|
|||
logger = logging.getLogger("NeoLogger")
|
||||
|
||||
ASSETS_DIR = os.getenv("ASSETS_DIR", "TemporaryDir/")
|
||||
local_shell = LocalShell()
|
||||
|
||||
|
||||
@reporter.step_deco("Get via HTTP Gate")
|
||||
|
|
@ -51,7 +53,9 @@ def get_via_http_gate(
|
|||
else:
|
||||
request = f"{endpoint}{request_path}"
|
||||
|
||||
resp = requests.get(request, headers={"Host": http_hostname}, stream=True, timeout=timeout, verify=False)
|
||||
resp = requests.get(
|
||||
request, headers={"Host": http_hostname}, stream=True, timeout=timeout, verify=False
|
||||
)
|
||||
|
||||
if not resp.ok:
|
||||
raise Exception(
|
||||
|
|
@ -72,7 +76,9 @@ def get_via_http_gate(
|
|||
|
||||
|
||||
@reporter.step_deco("Get via Zip HTTP Gate")
|
||||
def get_via_zip_http_gate(cid: str, prefix: str, endpoint: str, http_hostname: str, timeout: Optional[int] = 300):
|
||||
def get_via_zip_http_gate(
|
||||
cid: str, prefix: str, endpoint: str, http_hostname: str, timeout: Optional[int] = 300
|
||||
):
|
||||
"""
|
||||
This function gets given object from HTTP gate
|
||||
cid: container id to get object from
|
||||
|
|
@ -130,7 +136,9 @@ def get_via_http_gate_by_attribute(
|
|||
else:
|
||||
request = f"{endpoint}{request_path}"
|
||||
|
||||
resp = requests.get(request, stream=True, timeout=timeout, verify=False, headers={"Host": http_hostname})
|
||||
resp = requests.get(
|
||||
request, stream=True, timeout=timeout, verify=False, headers={"Host": http_hostname}
|
||||
)
|
||||
|
||||
if not resp.ok:
|
||||
raise Exception(
|
||||
|
|
@ -165,7 +173,9 @@ def upload_via_http_gate(
|
|||
request = f"{endpoint}/upload/{cid}"
|
||||
files = {"upload_file": open(path, "rb")}
|
||||
body = {"filename": path}
|
||||
resp = requests.post(request, files=files, data=body, headers=headers, timeout=timeout, verify=False)
|
||||
resp = requests.post(
|
||||
request, files=files, data=body, headers=headers, timeout=timeout, verify=False
|
||||
)
|
||||
|
||||
if not resp.ok:
|
||||
raise Exception(
|
||||
|
|
@ -223,16 +233,16 @@ def upload_via_http_gate_curl(
|
|||
large_object = is_object_large(filepath)
|
||||
if large_object:
|
||||
# pre-clean
|
||||
_cmd_run("rm pipe -f")
|
||||
local_shell.exec("rm pipe -f")
|
||||
files = f"file=@pipe;filename={os.path.basename(filepath)}"
|
||||
cmd = f"mkfifo pipe;cat {filepath} > pipe & curl -k --no-buffer -F '{files}' {attributes} {request}"
|
||||
output = _cmd_run(cmd, LONG_TIMEOUT)
|
||||
output = local_shell.exec(cmd, command_options)
|
||||
# clean up pipe
|
||||
_cmd_run("rm pipe")
|
||||
local_shell.exec("rm pipe")
|
||||
else:
|
||||
files = f"file=@{filepath};filename={os.path.basename(filepath)}"
|
||||
cmd = f"curl -k -F '{files}' {attributes} {request}"
|
||||
output = _cmd_run(cmd)
|
||||
output = local_shell.exec(cmd)
|
||||
|
||||
if error_pattern:
|
||||
match = error_pattern.casefold() in str(output).casefold()
|
||||
|
|
@ -245,6 +255,7 @@ def upload_via_http_gate_curl(
|
|||
return oid_re.group(1)
|
||||
|
||||
|
||||
@retry(max_attempts=3, sleep_interval=1)
|
||||
@reporter.step_deco("Get via HTTP Gate using Curl")
|
||||
def get_via_http_curl(cid: str, oid: str, endpoint: str, http_hostname: str) -> str:
|
||||
"""
|
||||
|
|
@ -257,8 +268,8 @@ def get_via_http_curl(cid: str, oid: str, endpoint: str, http_hostname: str) ->
|
|||
request = f"{endpoint}/get/{cid}/{oid}"
|
||||
file_path = os.path.join(os.getcwd(), ASSETS_DIR, f"{cid}_{oid}_{str(uuid.uuid4())}")
|
||||
|
||||
cmd = f"curl -k -H \"Host: {http_hostname}\" {request} > {file_path}"
|
||||
_cmd_run(cmd)
|
||||
cmd = f'curl -k -H "Host: {http_hostname}" {request} > {file_path}'
|
||||
local_shell.exec(cmd)
|
||||
|
||||
return file_path
|
||||
|
||||
|
|
@ -271,7 +282,11 @@ def _attach_allure_step(request: str, status_code: int, req_type="GET"):
|
|||
|
||||
@reporter.step_deco("Try to get object and expect error")
|
||||
def try_to_get_object_and_expect_error(
|
||||
cid: str, oid: str, error_pattern: str, endpoint: str, http_hostname: str,
|
||||
cid: str,
|
||||
oid: str,
|
||||
error_pattern: str,
|
||||
endpoint: str,
|
||||
http_hostname: str,
|
||||
) -> None:
|
||||
try:
|
||||
get_via_http_gate(cid=cid, oid=oid, endpoint=endpoint, http_hostname=http_hostname)
|
||||
|
|
@ -283,9 +298,16 @@ def try_to_get_object_and_expect_error(
|
|||
|
||||
@reporter.step_deco("Verify object can be get using HTTP header attribute")
|
||||
def get_object_by_attr_and_verify_hashes(
|
||||
oid: str, file_name: str, cid: str, attrs: dict, endpoint: str, http_hostname: str,
|
||||
oid: str,
|
||||
file_name: str,
|
||||
cid: str,
|
||||
attrs: dict,
|
||||
endpoint: str,
|
||||
http_hostname: str,
|
||||
) -> None:
|
||||
got_file_path_http = get_via_http_gate(cid=cid, oid=oid, endpoint=endpoint, http_hostname=http_hostname)
|
||||
got_file_path_http = get_via_http_gate(
|
||||
cid=cid, oid=oid, endpoint=endpoint, http_hostname=http_hostname
|
||||
)
|
||||
got_file_path_http_attr = get_via_http_gate_by_attribute(
|
||||
cid=cid, attribute=attrs, endpoint=endpoint, http_hostname=http_hostname
|
||||
)
|
||||
|
|
@ -326,7 +348,9 @@ def verify_object_hash(
|
|||
shell=shell,
|
||||
endpoint=random_node.get_rpc_endpoint(),
|
||||
)
|
||||
got_file_path_http = object_getter(cid=cid, oid=oid, endpoint=endpoint, http_hostname=http_hostname)
|
||||
got_file_path_http = object_getter(
|
||||
cid=cid, oid=oid, endpoint=endpoint, http_hostname=http_hostname
|
||||
)
|
||||
|
||||
assert_hashes_are_equal(file_name, got_file_path, got_file_path_http)
|
||||
|
||||
|
|
@ -369,10 +393,20 @@ def try_to_get_object_via_passed_request_and_expect_error(
|
|||
) -> None:
|
||||
try:
|
||||
if attrs is None:
|
||||
get_via_http_gate(cid=cid, oid=oid, endpoint=endpoint, request_path=http_request_path, http_hostname=http_hostname)
|
||||
get_via_http_gate(
|
||||
cid=cid,
|
||||
oid=oid,
|
||||
endpoint=endpoint,
|
||||
request_path=http_request_path,
|
||||
http_hostname=http_hostname,
|
||||
)
|
||||
else:
|
||||
get_via_http_gate_by_attribute(
|
||||
cid=cid, attribute=attrs, endpoint=endpoint, request_path=http_request_path, http_hostname=http_hostname
|
||||
cid=cid,
|
||||
attribute=attrs,
|
||||
endpoint=endpoint,
|
||||
request_path=http_request_path,
|
||||
http_hostname=http_hostname,
|
||||
)
|
||||
raise AssertionError(f"Expected error on getting object with cid: {cid}")
|
||||
except Exception as err:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue