From b856e820080fba80ce7abf6fb46798b68b66145a Mon Sep 17 00:00:00 2001 From: Yaroslava Lukoyanova Date: Mon, 24 Jul 2023 10:21:11 +0300 Subject: [PATCH] Added http hostname as a header to all http calls --- src/frostfs_testlib/steps/http/http_gate.py | 39 ++++++++++++------- src/frostfs_testlib/storage/cluster.py | 4 ++ src/frostfs_testlib/storage/constants.py | 2 + .../storage/dataclasses/frostfs_services.py | 6 +++ 4 files changed, 38 insertions(+), 13 deletions(-) diff --git a/src/frostfs_testlib/steps/http/http_gate.py b/src/frostfs_testlib/steps/http/http_gate.py index efc5258..8080689 100644 --- a/src/frostfs_testlib/steps/http/http_gate.py +++ b/src/frostfs_testlib/steps/http/http_gate.py @@ -32,6 +32,7 @@ def get_via_http_gate( cid: str, oid: str, endpoint: str, + http_hostname: str, request_path: Optional[str] = None, timeout: Optional[int] = 300, ): @@ -40,6 +41,7 @@ def get_via_http_gate( cid: container id to get object from oid: object ID endpoint: http gate endpoint + http_hostname: http host name on the node request_path: (optional) http request, if ommited - use default [{endpoint}/get/{cid}/{oid}] """ @@ -49,13 +51,14 @@ def get_via_http_gate( else: request = f"{endpoint}{request_path}" - resp = requests.get(request, stream=True, timeout=timeout, verify=False) + resp = requests.get(request, headers={"Host": http_hostname}, stream=True, timeout=timeout, verify=False) if not resp.ok: raise Exception( f"""Failed to get object via HTTP gate: request: {resp.request.path_url}, response: {resp.text}, + headers: {resp.headers}, status code: {resp.status_code} {resp.reason}""" ) @@ -69,12 +72,13 @@ def get_via_http_gate( @reporter.step_deco("Get via Zip HTTP Gate") -def get_via_zip_http_gate(cid: str, prefix: str, endpoint: str, timeout: Optional[int] = 300): +def get_via_zip_http_gate(cid: str, prefix: str, endpoint: str, http_hostname: str, timeout: Optional[int] = 300): """ This function gets given object from HTTP gate cid: container id to get object from prefix: common prefix endpoint: http gate endpoint + http_hostname: http host name on the node """ request = f"{endpoint}/zip/{cid}/{prefix}" resp = requests.get(request, stream=True, timeout=timeout, verify=False) @@ -84,6 +88,7 @@ def get_via_zip_http_gate(cid: str, prefix: str, endpoint: str, timeout: Optiona f"""Failed to get object via HTTP gate: request: {resp.request.path_url}, response: {resp.text}, + headers: {resp.headers}, status code: {resp.status_code} {resp.reason}""" ) @@ -105,6 +110,7 @@ def get_via_http_gate_by_attribute( cid: str, attribute: dict, endpoint: str, + http_hostname: str, request_path: Optional[str] = None, timeout: Optional[int] = 300, ): @@ -113,6 +119,7 @@ def get_via_http_gate_by_attribute( cid: CID to get object from attribute: attribute {name: attribute} value pair endpoint: http gate endpoint + http_hostname: http host name on the node request_path: (optional) http request path, if ommited - use default [{endpoint}/get_by_attribute/{Key}/{Value}] """ attr_name = list(attribute.keys())[0] @@ -123,13 +130,14 @@ def get_via_http_gate_by_attribute( else: request = f"{endpoint}{request_path}" - resp = requests.get(request, stream=True, timeout=timeout, verify=False) + resp = requests.get(request, stream=True, timeout=timeout, verify=False, headers={"Host": http_hostname}) if not resp.ok: raise Exception( f"""Failed to get object via HTTP gate: request: {resp.request.path_url}, response: {resp.text}, + headers: {resp.headers}, status code: {resp.status_code} {resp.reason}""" ) @@ -142,6 +150,7 @@ def get_via_http_gate_by_attribute( return file_path +# TODO: pass http_hostname as a header @reporter.step_deco("Upload via HTTP Gate") def upload_via_http_gate( cid: str, path: str, endpoint: str, headers: Optional[dict] = None, timeout: Optional[int] = 300 @@ -188,6 +197,7 @@ def is_object_large(filepath: str) -> bool: return False +# TODO: pass http_hostname as a header @reporter.step_deco("Upload via HTTP Gate using Curl") def upload_via_http_gate_curl( cid: str, @@ -236,17 +246,18 @@ def upload_via_http_gate_curl( @reporter.step_deco("Get via HTTP Gate using Curl") -def get_via_http_curl(cid: str, oid: str, endpoint: str) -> str: +def get_via_http_curl(cid: str, oid: str, endpoint: str, http_hostname: str) -> str: """ This function gets given object from HTTP gate using curl utility. cid: CID to get object from oid: object OID endpoint: http gate endpoint + http_hostname: http host name of the node """ request = f"{endpoint}/get/{cid}/{oid}" file_path = os.path.join(os.getcwd(), ASSETS_DIR, f"{cid}_{oid}_{str(uuid.uuid4())}") - cmd = f"curl -k {request} > {file_path}" + cmd = f"curl -k -H \"Host: {http_hostname}\" {request} > {file_path}" _cmd_run(cmd) return file_path @@ -260,10 +271,10 @@ def _attach_allure_step(request: str, status_code: int, req_type="GET"): @reporter.step_deco("Try to get object and expect error") def try_to_get_object_and_expect_error( - cid: str, oid: str, error_pattern: str, endpoint: str + cid: str, oid: str, error_pattern: str, endpoint: str, http_hostname: str, ) -> None: try: - get_via_http_gate(cid=cid, oid=oid, endpoint=endpoint) + get_via_http_gate(cid=cid, oid=oid, endpoint=endpoint, http_hostname=http_hostname) raise AssertionError(f"Expected error on getting object with cid: {cid}") except Exception as err: match = error_pattern.casefold() in str(err).casefold() @@ -272,11 +283,11 @@ def try_to_get_object_and_expect_error( @reporter.step_deco("Verify object can be get using HTTP header attribute") def get_object_by_attr_and_verify_hashes( - oid: str, file_name: str, cid: str, attrs: dict, endpoint: str + oid: str, file_name: str, cid: str, attrs: dict, endpoint: str, http_hostname: str, ) -> None: - got_file_path_http = get_via_http_gate(cid=cid, oid=oid, endpoint=endpoint) + got_file_path_http = get_via_http_gate(cid=cid, oid=oid, endpoint=endpoint, http_hostname=http_hostname) got_file_path_http_attr = get_via_http_gate_by_attribute( - cid=cid, attribute=attrs, endpoint=endpoint + cid=cid, attribute=attrs, endpoint=endpoint, http_hostname=http_hostname ) assert_hashes_are_equal(file_name, got_file_path_http, got_file_path_http_attr) @@ -289,6 +300,7 @@ def verify_object_hash( shell: Shell, nodes: list[StorageNode], endpoint: str, + http_hostname: str, object_getter=None, ) -> None: @@ -314,7 +326,7 @@ def verify_object_hash( shell=shell, endpoint=random_node.get_rpc_endpoint(), ) - got_file_path_http = object_getter(cid=cid, oid=oid, endpoint=endpoint) + got_file_path_http = object_getter(cid=cid, oid=oid, endpoint=endpoint, http_hostname=http_hostname) assert_hashes_are_equal(file_name, got_file_path, got_file_path_http) @@ -352,14 +364,15 @@ def try_to_get_object_via_passed_request_and_expect_error( error_pattern: str, endpoint: str, http_request_path: str, + http_hostname: str, attrs: Optional[dict] = None, ) -> None: try: if attrs is None: - get_via_http_gate(cid=cid, oid=oid, endpoint=endpoint, request_path=http_request_path) + get_via_http_gate(cid=cid, oid=oid, endpoint=endpoint, request_path=http_request_path, http_hostname=http_hostname) else: get_via_http_gate_by_attribute( - cid=cid, attribute=attrs, endpoint=endpoint, request_path=http_request_path + cid=cid, attribute=attrs, endpoint=endpoint, request_path=http_request_path, http_hostname=http_hostname ) raise AssertionError(f"Expected error on getting object with cid: {cid}") except Exception as err: diff --git a/src/frostfs_testlib/storage/cluster.py b/src/frostfs_testlib/storage/cluster.py index 91487c9..0e24ebb 100644 --- a/src/frostfs_testlib/storage/cluster.py +++ b/src/frostfs_testlib/storage/cluster.py @@ -130,6 +130,8 @@ class Cluster: default_rpc_endpoint: str default_s3_gate_endpoint: str default_http_gate_endpoint: str + default_http_hostname: str + default_s3_hostname: str def __init__(self, hosting: Hosting) -> None: self._hosting = hosting @@ -138,6 +140,8 @@ class Cluster: self.default_rpc_endpoint = self.services(StorageNode)[0].get_rpc_endpoint() self.default_s3_gate_endpoint = self.services(S3Gate)[0].get_endpoint() self.default_http_gate_endpoint = self.services(HTTPGate)[0].get_endpoint() + self.default_http_hostname = self.services(StorageNode)[0].get_http_hostname() + self.default_s3_hostname = self.services(StorageNode)[0].get_s3_hostname() @property def hosts(self) -> list[Host]: diff --git a/src/frostfs_testlib/storage/constants.py b/src/frostfs_testlib/storage/constants.py index 95ea3f2..2f9d8a8 100644 --- a/src/frostfs_testlib/storage/constants.py +++ b/src/frostfs_testlib/storage/constants.py @@ -11,6 +11,8 @@ class ConfigAttributes: ENDPOINT_INTERNAL = "endpoint_internal0" CONTROL_ENDPOINT = "control_endpoint" UN_LOCODE = "un_locode" + HTTP_HOSTNAME = "http_hostname" + S3_HOSTNAME = "s3_hostname" class _FrostfsServicesNames: diff --git a/src/frostfs_testlib/storage/dataclasses/frostfs_services.py b/src/frostfs_testlib/storage/dataclasses/frostfs_services.py index 23e3335..944837a 100644 --- a/src/frostfs_testlib/storage/dataclasses/frostfs_services.py +++ b/src/frostfs_testlib/storage/dataclasses/frostfs_services.py @@ -170,6 +170,12 @@ class StorageNode(NodeBase): def get_data_directory(self) -> str: return self.host.get_data_directory(self.name) + + def get_http_hostname(self) -> str: + return self._get_attribute(ConfigAttributes.HTTP_HOSTNAME) + + def get_s3_hostname(self) -> str: + return self._get_attribute(ConfigAttributes.S3_HOSTNAME) def delete_blobovnicza(self): self.host.delete_blobovnicza(self.name)