Added http hostname as a header to all http calls

This commit is contained in:
Yaroslava Lukoyanova 2023-07-24 10:21:11 +03:00 committed by ylukoyan
parent 2240be09d2
commit b856e82008
4 changed files with 38 additions and 13 deletions

View file

@ -32,6 +32,7 @@ def get_via_http_gate(
cid: str, cid: str,
oid: str, oid: str,
endpoint: str, endpoint: str,
http_hostname: str,
request_path: Optional[str] = None, request_path: Optional[str] = None,
timeout: Optional[int] = 300, timeout: Optional[int] = 300,
): ):
@ -40,6 +41,7 @@ def get_via_http_gate(
cid: container id to get object from cid: container id to get object from
oid: object ID oid: object ID
endpoint: http gate endpoint endpoint: http gate endpoint
http_hostname: http host name on the node
request_path: (optional) http request, if ommited - use default [{endpoint}/get/{cid}/{oid}] request_path: (optional) http request, if ommited - use default [{endpoint}/get/{cid}/{oid}]
""" """
@ -49,13 +51,14 @@ def get_via_http_gate(
else: else:
request = f"{endpoint}{request_path}" request = f"{endpoint}{request_path}"
resp = requests.get(request, stream=True, timeout=timeout, verify=False) resp = requests.get(request, headers={"Host": http_hostname}, stream=True, timeout=timeout, verify=False)
if not resp.ok: if not resp.ok:
raise Exception( raise Exception(
f"""Failed to get object via HTTP gate: f"""Failed to get object via HTTP gate:
request: {resp.request.path_url}, request: {resp.request.path_url},
response: {resp.text}, response: {resp.text},
headers: {resp.headers},
status code: {resp.status_code} {resp.reason}""" status code: {resp.status_code} {resp.reason}"""
) )
@ -69,12 +72,13 @@ def get_via_http_gate(
@reporter.step_deco("Get via Zip HTTP Gate") @reporter.step_deco("Get via Zip HTTP Gate")
def get_via_zip_http_gate(cid: str, prefix: str, endpoint: str, timeout: Optional[int] = 300): def get_via_zip_http_gate(cid: str, prefix: str, endpoint: str, http_hostname: str, timeout: Optional[int] = 300):
""" """
This function gets given object from HTTP gate This function gets given object from HTTP gate
cid: container id to get object from cid: container id to get object from
prefix: common prefix prefix: common prefix
endpoint: http gate endpoint endpoint: http gate endpoint
http_hostname: http host name on the node
""" """
request = f"{endpoint}/zip/{cid}/{prefix}" request = f"{endpoint}/zip/{cid}/{prefix}"
resp = requests.get(request, stream=True, timeout=timeout, verify=False) resp = requests.get(request, stream=True, timeout=timeout, verify=False)
@ -84,6 +88,7 @@ def get_via_zip_http_gate(cid: str, prefix: str, endpoint: str, timeout: Optiona
f"""Failed to get object via HTTP gate: f"""Failed to get object via HTTP gate:
request: {resp.request.path_url}, request: {resp.request.path_url},
response: {resp.text}, response: {resp.text},
headers: {resp.headers},
status code: {resp.status_code} {resp.reason}""" status code: {resp.status_code} {resp.reason}"""
) )
@ -105,6 +110,7 @@ def get_via_http_gate_by_attribute(
cid: str, cid: str,
attribute: dict, attribute: dict,
endpoint: str, endpoint: str,
http_hostname: str,
request_path: Optional[str] = None, request_path: Optional[str] = None,
timeout: Optional[int] = 300, timeout: Optional[int] = 300,
): ):
@ -113,6 +119,7 @@ def get_via_http_gate_by_attribute(
cid: CID to get object from cid: CID to get object from
attribute: attribute {name: attribute} value pair attribute: attribute {name: attribute} value pair
endpoint: http gate endpoint endpoint: http gate endpoint
http_hostname: http host name on the node
request_path: (optional) http request path, if ommited - use default [{endpoint}/get_by_attribute/{Key}/{Value}] request_path: (optional) http request path, if ommited - use default [{endpoint}/get_by_attribute/{Key}/{Value}]
""" """
attr_name = list(attribute.keys())[0] attr_name = list(attribute.keys())[0]
@ -123,13 +130,14 @@ def get_via_http_gate_by_attribute(
else: else:
request = f"{endpoint}{request_path}" request = f"{endpoint}{request_path}"
resp = requests.get(request, stream=True, timeout=timeout, verify=False) resp = requests.get(request, stream=True, timeout=timeout, verify=False, headers={"Host": http_hostname})
if not resp.ok: if not resp.ok:
raise Exception( raise Exception(
f"""Failed to get object via HTTP gate: f"""Failed to get object via HTTP gate:
request: {resp.request.path_url}, request: {resp.request.path_url},
response: {resp.text}, response: {resp.text},
headers: {resp.headers},
status code: {resp.status_code} {resp.reason}""" status code: {resp.status_code} {resp.reason}"""
) )
@ -142,6 +150,7 @@ def get_via_http_gate_by_attribute(
return file_path return file_path
# TODO: pass http_hostname as a header
@reporter.step_deco("Upload via HTTP Gate") @reporter.step_deco("Upload via HTTP Gate")
def upload_via_http_gate( def upload_via_http_gate(
cid: str, path: str, endpoint: str, headers: Optional[dict] = None, timeout: Optional[int] = 300 cid: str, path: str, endpoint: str, headers: Optional[dict] = None, timeout: Optional[int] = 300
@ -188,6 +197,7 @@ def is_object_large(filepath: str) -> bool:
return False return False
# TODO: pass http_hostname as a header
@reporter.step_deco("Upload via HTTP Gate using Curl") @reporter.step_deco("Upload via HTTP Gate using Curl")
def upload_via_http_gate_curl( def upload_via_http_gate_curl(
cid: str, cid: str,
@ -236,17 +246,18 @@ def upload_via_http_gate_curl(
@reporter.step_deco("Get via HTTP Gate using Curl") @reporter.step_deco("Get via HTTP Gate using Curl")
def get_via_http_curl(cid: str, oid: str, endpoint: str) -> str: def get_via_http_curl(cid: str, oid: str, endpoint: str, http_hostname: str) -> str:
""" """
This function gets given object from HTTP gate using curl utility. This function gets given object from HTTP gate using curl utility.
cid: CID to get object from cid: CID to get object from
oid: object OID oid: object OID
endpoint: http gate endpoint endpoint: http gate endpoint
http_hostname: http host name of the node
""" """
request = f"{endpoint}/get/{cid}/{oid}" request = f"{endpoint}/get/{cid}/{oid}"
file_path = os.path.join(os.getcwd(), ASSETS_DIR, f"{cid}_{oid}_{str(uuid.uuid4())}") file_path = os.path.join(os.getcwd(), ASSETS_DIR, f"{cid}_{oid}_{str(uuid.uuid4())}")
cmd = f"curl -k {request} > {file_path}" cmd = f"curl -k -H \"Host: {http_hostname}\" {request} > {file_path}"
_cmd_run(cmd) _cmd_run(cmd)
return file_path return file_path
@ -260,10 +271,10 @@ def _attach_allure_step(request: str, status_code: int, req_type="GET"):
@reporter.step_deco("Try to get object and expect error") @reporter.step_deco("Try to get object and expect error")
def try_to_get_object_and_expect_error( def try_to_get_object_and_expect_error(
cid: str, oid: str, error_pattern: str, endpoint: str cid: str, oid: str, error_pattern: str, endpoint: str, http_hostname: str,
) -> None: ) -> None:
try: try:
get_via_http_gate(cid=cid, oid=oid, endpoint=endpoint) get_via_http_gate(cid=cid, oid=oid, endpoint=endpoint, http_hostname=http_hostname)
raise AssertionError(f"Expected error on getting object with cid: {cid}") raise AssertionError(f"Expected error on getting object with cid: {cid}")
except Exception as err: except Exception as err:
match = error_pattern.casefold() in str(err).casefold() match = error_pattern.casefold() in str(err).casefold()
@ -272,11 +283,11 @@ def try_to_get_object_and_expect_error(
@reporter.step_deco("Verify object can be get using HTTP header attribute") @reporter.step_deco("Verify object can be get using HTTP header attribute")
def get_object_by_attr_and_verify_hashes( def get_object_by_attr_and_verify_hashes(
oid: str, file_name: str, cid: str, attrs: dict, endpoint: str oid: str, file_name: str, cid: str, attrs: dict, endpoint: str, http_hostname: str,
) -> None: ) -> None:
got_file_path_http = get_via_http_gate(cid=cid, oid=oid, endpoint=endpoint) got_file_path_http = get_via_http_gate(cid=cid, oid=oid, endpoint=endpoint, http_hostname=http_hostname)
got_file_path_http_attr = get_via_http_gate_by_attribute( got_file_path_http_attr = get_via_http_gate_by_attribute(
cid=cid, attribute=attrs, endpoint=endpoint cid=cid, attribute=attrs, endpoint=endpoint, http_hostname=http_hostname
) )
assert_hashes_are_equal(file_name, got_file_path_http, got_file_path_http_attr) assert_hashes_are_equal(file_name, got_file_path_http, got_file_path_http_attr)
@ -289,6 +300,7 @@ def verify_object_hash(
shell: Shell, shell: Shell,
nodes: list[StorageNode], nodes: list[StorageNode],
endpoint: str, endpoint: str,
http_hostname: str,
object_getter=None, object_getter=None,
) -> None: ) -> None:
@ -314,7 +326,7 @@ def verify_object_hash(
shell=shell, shell=shell,
endpoint=random_node.get_rpc_endpoint(), endpoint=random_node.get_rpc_endpoint(),
) )
got_file_path_http = object_getter(cid=cid, oid=oid, endpoint=endpoint) got_file_path_http = object_getter(cid=cid, oid=oid, endpoint=endpoint, http_hostname=http_hostname)
assert_hashes_are_equal(file_name, got_file_path, got_file_path_http) assert_hashes_are_equal(file_name, got_file_path, got_file_path_http)
@ -352,14 +364,15 @@ def try_to_get_object_via_passed_request_and_expect_error(
error_pattern: str, error_pattern: str,
endpoint: str, endpoint: str,
http_request_path: str, http_request_path: str,
http_hostname: str,
attrs: Optional[dict] = None, attrs: Optional[dict] = None,
) -> None: ) -> None:
try: try:
if attrs is None: if attrs is None:
get_via_http_gate(cid=cid, oid=oid, endpoint=endpoint, request_path=http_request_path) get_via_http_gate(cid=cid, oid=oid, endpoint=endpoint, request_path=http_request_path, http_hostname=http_hostname)
else: else:
get_via_http_gate_by_attribute( get_via_http_gate_by_attribute(
cid=cid, attribute=attrs, endpoint=endpoint, request_path=http_request_path cid=cid, attribute=attrs, endpoint=endpoint, request_path=http_request_path, http_hostname=http_hostname
) )
raise AssertionError(f"Expected error on getting object with cid: {cid}") raise AssertionError(f"Expected error on getting object with cid: {cid}")
except Exception as err: except Exception as err:

View file

@ -130,6 +130,8 @@ class Cluster:
default_rpc_endpoint: str default_rpc_endpoint: str
default_s3_gate_endpoint: str default_s3_gate_endpoint: str
default_http_gate_endpoint: str default_http_gate_endpoint: str
default_http_hostname: str
default_s3_hostname: str
def __init__(self, hosting: Hosting) -> None: def __init__(self, hosting: Hosting) -> None:
self._hosting = hosting self._hosting = hosting
@ -138,6 +140,8 @@ class Cluster:
self.default_rpc_endpoint = self.services(StorageNode)[0].get_rpc_endpoint() self.default_rpc_endpoint = self.services(StorageNode)[0].get_rpc_endpoint()
self.default_s3_gate_endpoint = self.services(S3Gate)[0].get_endpoint() self.default_s3_gate_endpoint = self.services(S3Gate)[0].get_endpoint()
self.default_http_gate_endpoint = self.services(HTTPGate)[0].get_endpoint() self.default_http_gate_endpoint = self.services(HTTPGate)[0].get_endpoint()
self.default_http_hostname = self.services(StorageNode)[0].get_http_hostname()
self.default_s3_hostname = self.services(StorageNode)[0].get_s3_hostname()
@property @property
def hosts(self) -> list[Host]: def hosts(self) -> list[Host]:

View file

@ -11,6 +11,8 @@ class ConfigAttributes:
ENDPOINT_INTERNAL = "endpoint_internal0" ENDPOINT_INTERNAL = "endpoint_internal0"
CONTROL_ENDPOINT = "control_endpoint" CONTROL_ENDPOINT = "control_endpoint"
UN_LOCODE = "un_locode" UN_LOCODE = "un_locode"
HTTP_HOSTNAME = "http_hostname"
S3_HOSTNAME = "s3_hostname"
class _FrostfsServicesNames: class _FrostfsServicesNames:

View file

@ -170,6 +170,12 @@ class StorageNode(NodeBase):
def get_data_directory(self) -> str: def get_data_directory(self) -> str:
return self.host.get_data_directory(self.name) return self.host.get_data_directory(self.name)
def get_http_hostname(self) -> str:
return self._get_attribute(ConfigAttributes.HTTP_HOSTNAME)
def get_s3_hostname(self) -> str:
return self._get_attribute(ConfigAttributes.S3_HOSTNAME)
def delete_blobovnicza(self): def delete_blobovnicza(self):
self.host.delete_blobovnicza(self.name) self.host.delete_blobovnicza(self.name)