anurindm/frostfs-testlib
1
0
Fork 0
forked from TrueCloudLab/frostfs-testlib
Code Pull requests Activity

Added http hostname as a header to all http calls

Browse source
This commit is contained in:
Yaroslava Lukoyanova 2023-07-24 10:21:11 +03:00 committed by ylukoyan
parent 2240be09d2
commit b856e82008
4 changed files with 38 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

39
src/frostfs_testlib/steps/http/http_gate.py
View file

@ -32,6 +32,7 @@ def get_via_http_gate(
cid: str,
oid: str,
endpoint: str,
http_hostname: str,
request_path: Optional[str] = None,
timeout: Optional[int] = 300,
):
@ -40,6 +41,7 @@ def get_via_http_gate(
cid: container id to get object from
oid: object ID
endpoint: http gate endpoint
http_hostname: http host name on the node
request_path: (optional) http request, if ommited - use default [{endpoint}/get/{cid}/{oid}]
"""
@ -49,13 +51,14 @@ def get_via_http_gate(
else:
request = f"{endpoint}{request_path}"
resp = requests.get(request, stream=True, timeout=timeout, verify=False)
resp = requests.get(request, headers={"Host": http_hostname}, stream=True, timeout=timeout, verify=False)
if not resp.ok:
raise Exception(
f"""Failed to get object via HTTP gate:
request: {resp.request.path_url},
response: {resp.text},
headers: {resp.headers},
status code: {resp.status_code} {resp.reason}"""
)
@ -69,12 +72,13 @@ def get_via_http_gate(
@reporter.step_deco("Get via Zip HTTP Gate")
def get_via_zip_http_gate(cid: str, prefix: str, endpoint: str, timeout: Optional[int] = 300):
def get_via_zip_http_gate(cid: str, prefix: str, endpoint: str, http_hostname: str, timeout: Optional[int] = 300):
"""
This function gets given object from HTTP gate
cid: container id to get object from
prefix: common prefix
endpoint: http gate endpoint
http_hostname: http host name on the node
"""
request = f"{endpoint}/zip/{cid}/{prefix}"
resp = requests.get(request, stream=True, timeout=timeout, verify=False)
@ -84,6 +88,7 @@ def get_via_zip_http_gate(cid: str, prefix: str, endpoint: str, timeout: Optiona
f"""Failed to get object via HTTP gate:
request: {resp.request.path_url},
response: {resp.text},
headers: {resp.headers},
status code: {resp.status_code} {resp.reason}"""
)
@ -105,6 +110,7 @@ def get_via_http_gate_by_attribute(
cid: str,
attribute: dict,
endpoint: str,
http_hostname: str,
request_path: Optional[str] = None,
timeout: Optional[int] = 300,
):
@ -113,6 +119,7 @@ def get_via_http_gate_by_attribute(
cid: CID to get object from
attribute: attribute {name: attribute} value pair
endpoint: http gate endpoint
http_hostname: http host name on the node
request_path: (optional) http request path, if ommited - use default [{endpoint}/get_by_attribute/{Key}/{Value}]
"""
attr_name = list(attribute.keys())[0]
@ -123,13 +130,14 @@ def get_via_http_gate_by_attribute(
else:
request = f"{endpoint}{request_path}"
resp = requests.get(request, stream=True, timeout=timeout, verify=False)
resp = requests.get(request, stream=True, timeout=timeout, verify=False, headers={"Host": http_hostname})
if not resp.ok:
raise Exception(
f"""Failed to get object via HTTP gate:
request: {resp.request.path_url},
response: {resp.text},
headers: {resp.headers},
status code: {resp.status_code} {resp.reason}"""
)
@ -142,6 +150,7 @@ def get_via_http_gate_by_attribute(
return file_path
# TODO: pass http_hostname as a header
@reporter.step_deco("Upload via HTTP Gate")
def upload_via_http_gate(
cid: str, path: str, endpoint: str, headers: Optional[dict] = None, timeout: Optional[int] = 300
@ -188,6 +197,7 @@ def is_object_large(filepath: str) -> bool:
return False
# TODO: pass http_hostname as a header
@reporter.step_deco("Upload via HTTP Gate using Curl")
def upload_via_http_gate_curl(
cid: str,
@ -236,17 +246,18 @@ def upload_via_http_gate_curl(
@reporter.step_deco("Get via HTTP Gate using Curl")
def get_via_http_curl(cid: str, oid: str, endpoint: str) -> str:
def get_via_http_curl(cid: str, oid: str, endpoint: str, http_hostname: str) -> str:
"""
This function gets given object from HTTP gate using curl utility.
cid: CID to get object from
oid: object OID
endpoint: http gate endpoint
http_hostname: http host name of the node
"""
request = f"{endpoint}/get/{cid}/{oid}"
file_path = os.path.join(os.getcwd(), ASSETS_DIR, f"{cid}_{oid}_{str(uuid.uuid4())}")
cmd = f"curl -k {request} > {file_path}"
cmd = f"curl -k -H \"Host: {http_hostname}\" {request} > {file_path}"
_cmd_run(cmd)
return file_path
@ -260,10 +271,10 @@ def _attach_allure_step(request: str, status_code: int, req_type="GET"):
@reporter.step_deco("Try to get object and expect error")
def try_to_get_object_and_expect_error(
cid: str, oid: str, error_pattern: str, endpoint: str
cid: str, oid: str, error_pattern: str, endpoint: str, http_hostname: str,
) -> None:
try:
get_via_http_gate(cid=cid, oid=oid, endpoint=endpoint)
get_via_http_gate(cid=cid, oid=oid, endpoint=endpoint, http_hostname=http_hostname)
raise AssertionError(f"Expected error on getting object with cid: {cid}")
except Exception as err:
match = error_pattern.casefold() in str(err).casefold()
@ -272,11 +283,11 @@ def try_to_get_object_and_expect_error(
@reporter.step_deco("Verify object can be get using HTTP header attribute")
def get_object_by_attr_and_verify_hashes(
oid: str, file_name: str, cid: str, attrs: dict, endpoint: str
oid: str, file_name: str, cid: str, attrs: dict, endpoint: str, http_hostname: str,
) -> None:
got_file_path_http = get_via_http_gate(cid=cid, oid=oid, endpoint=endpoint)
got_file_path_http = get_via_http_gate(cid=cid, oid=oid, endpoint=endpoint, http_hostname=http_hostname)
got_file_path_http_attr = get_via_http_gate_by_attribute(
cid=cid, attribute=attrs, endpoint=endpoint
cid=cid, attribute=attrs, endpoint=endpoint, http_hostname=http_hostname
)
assert_hashes_are_equal(file_name, got_file_path_http, got_file_path_http_attr)
@ -289,6 +300,7 @@ def verify_object_hash(
shell: Shell,
nodes: list[StorageNode],
endpoint: str,
http_hostname: str,
object_getter=None,
) -> None:
@ -314,7 +326,7 @@ def verify_object_hash(
shell=shell,
endpoint=random_node.get_rpc_endpoint(),
)
got_file_path_http = object_getter(cid=cid, oid=oid, endpoint=endpoint)
got_file_path_http = object_getter(cid=cid, oid=oid, endpoint=endpoint, http_hostname=http_hostname)
assert_hashes_are_equal(file_name, got_file_path, got_file_path_http)
@ -352,14 +364,15 @@ def try_to_get_object_via_passed_request_and_expect_error(
error_pattern: str,
endpoint: str,
http_request_path: str,
http_hostname: str,
attrs: Optional[dict] = None,
) -> None:
try:
if attrs is None:
get_via_http_gate(cid=cid, oid=oid, endpoint=endpoint, request_path=http_request_path)
get_via_http_gate(cid=cid, oid=oid, endpoint=endpoint, request_path=http_request_path, http_hostname=http_hostname)
else:
get_via_http_gate_by_attribute(
cid=cid, attribute=attrs, endpoint=endpoint, request_path=http_request_path
cid=cid, attribute=attrs, endpoint=endpoint, request_path=http_request_path, http_hostname=http_hostname
)
raise AssertionError(f"Expected error on getting object with cid: {cid}")
except Exception as err:

4
src/frostfs_testlib/storage/cluster.py
View file

@ -130,6 +130,8 @@ class Cluster:
default_rpc_endpoint: str
default_s3_gate_endpoint: str
default_http_gate_endpoint: str
default_http_hostname: str
default_s3_hostname: str
def __init__(self, hosting: Hosting) -> None:
self._hosting = hosting
@ -138,6 +140,8 @@ class Cluster:
self.default_rpc_endpoint = self.services(StorageNode)[0].get_rpc_endpoint()
self.default_s3_gate_endpoint = self.services(S3Gate)[0].get_endpoint()
self.default_http_gate_endpoint = self.services(HTTPGate)[0].get_endpoint()
self.default_http_hostname = self.services(StorageNode)[0].get_http_hostname()
self.default_s3_hostname = self.services(StorageNode)[0].get_s3_hostname()
@property
def hosts(self) -> list[Host]:

2
src/frostfs_testlib/storage/constants.py
View file

@ -11,6 +11,8 @@ class ConfigAttributes:
ENDPOINT_INTERNAL = "endpoint_internal0"
CONTROL_ENDPOINT = "control_endpoint"
UN_LOCODE = "un_locode"
HTTP_HOSTNAME = "http_hostname"
S3_HOSTNAME = "s3_hostname"
class _FrostfsServicesNames:

6
src/frostfs_testlib/storage/dataclasses/frostfs_services.py
View file

@ -170,6 +170,12 @@ class StorageNode(NodeBase):
def get_data_directory(self) -> str:
return self.host.get_data_directory(self.name)
def get_http_hostname(self) -> str:
return self._get_attribute(ConfigAttributes.HTTP_HOSTNAME)
def get_s3_hostname(self) -> str:
return self._get_attribute(ConfigAttributes.S3_HOSTNAME)
def delete_blobovnicza(self):
self.host.delete_blobovnicza(self.name)