forked from TrueCloudLab/frostfs-testlib
Added http hostname as a header to all http calls
This commit is contained in:
parent
2240be09d2
commit
b856e82008
4 changed files with 38 additions and 13 deletions
|
@ -32,6 +32,7 @@ def get_via_http_gate(
|
||||||
cid: str,
|
cid: str,
|
||||||
oid: str,
|
oid: str,
|
||||||
endpoint: str,
|
endpoint: str,
|
||||||
|
http_hostname: str,
|
||||||
request_path: Optional[str] = None,
|
request_path: Optional[str] = None,
|
||||||
timeout: Optional[int] = 300,
|
timeout: Optional[int] = 300,
|
||||||
):
|
):
|
||||||
|
@ -40,6 +41,7 @@ def get_via_http_gate(
|
||||||
cid: container id to get object from
|
cid: container id to get object from
|
||||||
oid: object ID
|
oid: object ID
|
||||||
endpoint: http gate endpoint
|
endpoint: http gate endpoint
|
||||||
|
http_hostname: http host name on the node
|
||||||
request_path: (optional) http request, if ommited - use default [{endpoint}/get/{cid}/{oid}]
|
request_path: (optional) http request, if ommited - use default [{endpoint}/get/{cid}/{oid}]
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
@ -49,13 +51,14 @@ def get_via_http_gate(
|
||||||
else:
|
else:
|
||||||
request = f"{endpoint}{request_path}"
|
request = f"{endpoint}{request_path}"
|
||||||
|
|
||||||
resp = requests.get(request, stream=True, timeout=timeout, verify=False)
|
resp = requests.get(request, headers={"Host": http_hostname}, stream=True, timeout=timeout, verify=False)
|
||||||
|
|
||||||
if not resp.ok:
|
if not resp.ok:
|
||||||
raise Exception(
|
raise Exception(
|
||||||
f"""Failed to get object via HTTP gate:
|
f"""Failed to get object via HTTP gate:
|
||||||
request: {resp.request.path_url},
|
request: {resp.request.path_url},
|
||||||
response: {resp.text},
|
response: {resp.text},
|
||||||
|
headers: {resp.headers},
|
||||||
status code: {resp.status_code} {resp.reason}"""
|
status code: {resp.status_code} {resp.reason}"""
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -69,12 +72,13 @@ def get_via_http_gate(
|
||||||
|
|
||||||
|
|
||||||
@reporter.step_deco("Get via Zip HTTP Gate")
|
@reporter.step_deco("Get via Zip HTTP Gate")
|
||||||
def get_via_zip_http_gate(cid: str, prefix: str, endpoint: str, timeout: Optional[int] = 300):
|
def get_via_zip_http_gate(cid: str, prefix: str, endpoint: str, http_hostname: str, timeout: Optional[int] = 300):
|
||||||
"""
|
"""
|
||||||
This function gets given object from HTTP gate
|
This function gets given object from HTTP gate
|
||||||
cid: container id to get object from
|
cid: container id to get object from
|
||||||
prefix: common prefix
|
prefix: common prefix
|
||||||
endpoint: http gate endpoint
|
endpoint: http gate endpoint
|
||||||
|
http_hostname: http host name on the node
|
||||||
"""
|
"""
|
||||||
request = f"{endpoint}/zip/{cid}/{prefix}"
|
request = f"{endpoint}/zip/{cid}/{prefix}"
|
||||||
resp = requests.get(request, stream=True, timeout=timeout, verify=False)
|
resp = requests.get(request, stream=True, timeout=timeout, verify=False)
|
||||||
|
@ -84,6 +88,7 @@ def get_via_zip_http_gate(cid: str, prefix: str, endpoint: str, timeout: Optiona
|
||||||
f"""Failed to get object via HTTP gate:
|
f"""Failed to get object via HTTP gate:
|
||||||
request: {resp.request.path_url},
|
request: {resp.request.path_url},
|
||||||
response: {resp.text},
|
response: {resp.text},
|
||||||
|
headers: {resp.headers},
|
||||||
status code: {resp.status_code} {resp.reason}"""
|
status code: {resp.status_code} {resp.reason}"""
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -105,6 +110,7 @@ def get_via_http_gate_by_attribute(
|
||||||
cid: str,
|
cid: str,
|
||||||
attribute: dict,
|
attribute: dict,
|
||||||
endpoint: str,
|
endpoint: str,
|
||||||
|
http_hostname: str,
|
||||||
request_path: Optional[str] = None,
|
request_path: Optional[str] = None,
|
||||||
timeout: Optional[int] = 300,
|
timeout: Optional[int] = 300,
|
||||||
):
|
):
|
||||||
|
@ -113,6 +119,7 @@ def get_via_http_gate_by_attribute(
|
||||||
cid: CID to get object from
|
cid: CID to get object from
|
||||||
attribute: attribute {name: attribute} value pair
|
attribute: attribute {name: attribute} value pair
|
||||||
endpoint: http gate endpoint
|
endpoint: http gate endpoint
|
||||||
|
http_hostname: http host name on the node
|
||||||
request_path: (optional) http request path, if ommited - use default [{endpoint}/get_by_attribute/{Key}/{Value}]
|
request_path: (optional) http request path, if ommited - use default [{endpoint}/get_by_attribute/{Key}/{Value}]
|
||||||
"""
|
"""
|
||||||
attr_name = list(attribute.keys())[0]
|
attr_name = list(attribute.keys())[0]
|
||||||
|
@ -123,13 +130,14 @@ def get_via_http_gate_by_attribute(
|
||||||
else:
|
else:
|
||||||
request = f"{endpoint}{request_path}"
|
request = f"{endpoint}{request_path}"
|
||||||
|
|
||||||
resp = requests.get(request, stream=True, timeout=timeout, verify=False)
|
resp = requests.get(request, stream=True, timeout=timeout, verify=False, headers={"Host": http_hostname})
|
||||||
|
|
||||||
if not resp.ok:
|
if not resp.ok:
|
||||||
raise Exception(
|
raise Exception(
|
||||||
f"""Failed to get object via HTTP gate:
|
f"""Failed to get object via HTTP gate:
|
||||||
request: {resp.request.path_url},
|
request: {resp.request.path_url},
|
||||||
response: {resp.text},
|
response: {resp.text},
|
||||||
|
headers: {resp.headers},
|
||||||
status code: {resp.status_code} {resp.reason}"""
|
status code: {resp.status_code} {resp.reason}"""
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -142,6 +150,7 @@ def get_via_http_gate_by_attribute(
|
||||||
return file_path
|
return file_path
|
||||||
|
|
||||||
|
|
||||||
|
# TODO: pass http_hostname as a header
|
||||||
@reporter.step_deco("Upload via HTTP Gate")
|
@reporter.step_deco("Upload via HTTP Gate")
|
||||||
def upload_via_http_gate(
|
def upload_via_http_gate(
|
||||||
cid: str, path: str, endpoint: str, headers: Optional[dict] = None, timeout: Optional[int] = 300
|
cid: str, path: str, endpoint: str, headers: Optional[dict] = None, timeout: Optional[int] = 300
|
||||||
|
@ -188,6 +197,7 @@ def is_object_large(filepath: str) -> bool:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
# TODO: pass http_hostname as a header
|
||||||
@reporter.step_deco("Upload via HTTP Gate using Curl")
|
@reporter.step_deco("Upload via HTTP Gate using Curl")
|
||||||
def upload_via_http_gate_curl(
|
def upload_via_http_gate_curl(
|
||||||
cid: str,
|
cid: str,
|
||||||
|
@ -236,17 +246,18 @@ def upload_via_http_gate_curl(
|
||||||
|
|
||||||
|
|
||||||
@reporter.step_deco("Get via HTTP Gate using Curl")
|
@reporter.step_deco("Get via HTTP Gate using Curl")
|
||||||
def get_via_http_curl(cid: str, oid: str, endpoint: str) -> str:
|
def get_via_http_curl(cid: str, oid: str, endpoint: str, http_hostname: str) -> str:
|
||||||
"""
|
"""
|
||||||
This function gets given object from HTTP gate using curl utility.
|
This function gets given object from HTTP gate using curl utility.
|
||||||
cid: CID to get object from
|
cid: CID to get object from
|
||||||
oid: object OID
|
oid: object OID
|
||||||
endpoint: http gate endpoint
|
endpoint: http gate endpoint
|
||||||
|
http_hostname: http host name of the node
|
||||||
"""
|
"""
|
||||||
request = f"{endpoint}/get/{cid}/{oid}"
|
request = f"{endpoint}/get/{cid}/{oid}"
|
||||||
file_path = os.path.join(os.getcwd(), ASSETS_DIR, f"{cid}_{oid}_{str(uuid.uuid4())}")
|
file_path = os.path.join(os.getcwd(), ASSETS_DIR, f"{cid}_{oid}_{str(uuid.uuid4())}")
|
||||||
|
|
||||||
cmd = f"curl -k {request} > {file_path}"
|
cmd = f"curl -k -H \"Host: {http_hostname}\" {request} > {file_path}"
|
||||||
_cmd_run(cmd)
|
_cmd_run(cmd)
|
||||||
|
|
||||||
return file_path
|
return file_path
|
||||||
|
@ -260,10 +271,10 @@ def _attach_allure_step(request: str, status_code: int, req_type="GET"):
|
||||||
|
|
||||||
@reporter.step_deco("Try to get object and expect error")
|
@reporter.step_deco("Try to get object and expect error")
|
||||||
def try_to_get_object_and_expect_error(
|
def try_to_get_object_and_expect_error(
|
||||||
cid: str, oid: str, error_pattern: str, endpoint: str
|
cid: str, oid: str, error_pattern: str, endpoint: str, http_hostname: str,
|
||||||
) -> None:
|
) -> None:
|
||||||
try:
|
try:
|
||||||
get_via_http_gate(cid=cid, oid=oid, endpoint=endpoint)
|
get_via_http_gate(cid=cid, oid=oid, endpoint=endpoint, http_hostname=http_hostname)
|
||||||
raise AssertionError(f"Expected error on getting object with cid: {cid}")
|
raise AssertionError(f"Expected error on getting object with cid: {cid}")
|
||||||
except Exception as err:
|
except Exception as err:
|
||||||
match = error_pattern.casefold() in str(err).casefold()
|
match = error_pattern.casefold() in str(err).casefold()
|
||||||
|
@ -272,11 +283,11 @@ def try_to_get_object_and_expect_error(
|
||||||
|
|
||||||
@reporter.step_deco("Verify object can be get using HTTP header attribute")
|
@reporter.step_deco("Verify object can be get using HTTP header attribute")
|
||||||
def get_object_by_attr_and_verify_hashes(
|
def get_object_by_attr_and_verify_hashes(
|
||||||
oid: str, file_name: str, cid: str, attrs: dict, endpoint: str
|
oid: str, file_name: str, cid: str, attrs: dict, endpoint: str, http_hostname: str,
|
||||||
) -> None:
|
) -> None:
|
||||||
got_file_path_http = get_via_http_gate(cid=cid, oid=oid, endpoint=endpoint)
|
got_file_path_http = get_via_http_gate(cid=cid, oid=oid, endpoint=endpoint, http_hostname=http_hostname)
|
||||||
got_file_path_http_attr = get_via_http_gate_by_attribute(
|
got_file_path_http_attr = get_via_http_gate_by_attribute(
|
||||||
cid=cid, attribute=attrs, endpoint=endpoint
|
cid=cid, attribute=attrs, endpoint=endpoint, http_hostname=http_hostname
|
||||||
)
|
)
|
||||||
assert_hashes_are_equal(file_name, got_file_path_http, got_file_path_http_attr)
|
assert_hashes_are_equal(file_name, got_file_path_http, got_file_path_http_attr)
|
||||||
|
|
||||||
|
@ -289,6 +300,7 @@ def verify_object_hash(
|
||||||
shell: Shell,
|
shell: Shell,
|
||||||
nodes: list[StorageNode],
|
nodes: list[StorageNode],
|
||||||
endpoint: str,
|
endpoint: str,
|
||||||
|
http_hostname: str,
|
||||||
object_getter=None,
|
object_getter=None,
|
||||||
) -> None:
|
) -> None:
|
||||||
|
|
||||||
|
@ -314,7 +326,7 @@ def verify_object_hash(
|
||||||
shell=shell,
|
shell=shell,
|
||||||
endpoint=random_node.get_rpc_endpoint(),
|
endpoint=random_node.get_rpc_endpoint(),
|
||||||
)
|
)
|
||||||
got_file_path_http = object_getter(cid=cid, oid=oid, endpoint=endpoint)
|
got_file_path_http = object_getter(cid=cid, oid=oid, endpoint=endpoint, http_hostname=http_hostname)
|
||||||
|
|
||||||
assert_hashes_are_equal(file_name, got_file_path, got_file_path_http)
|
assert_hashes_are_equal(file_name, got_file_path, got_file_path_http)
|
||||||
|
|
||||||
|
@ -352,14 +364,15 @@ def try_to_get_object_via_passed_request_and_expect_error(
|
||||||
error_pattern: str,
|
error_pattern: str,
|
||||||
endpoint: str,
|
endpoint: str,
|
||||||
http_request_path: str,
|
http_request_path: str,
|
||||||
|
http_hostname: str,
|
||||||
attrs: Optional[dict] = None,
|
attrs: Optional[dict] = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
try:
|
try:
|
||||||
if attrs is None:
|
if attrs is None:
|
||||||
get_via_http_gate(cid=cid, oid=oid, endpoint=endpoint, request_path=http_request_path)
|
get_via_http_gate(cid=cid, oid=oid, endpoint=endpoint, request_path=http_request_path, http_hostname=http_hostname)
|
||||||
else:
|
else:
|
||||||
get_via_http_gate_by_attribute(
|
get_via_http_gate_by_attribute(
|
||||||
cid=cid, attribute=attrs, endpoint=endpoint, request_path=http_request_path
|
cid=cid, attribute=attrs, endpoint=endpoint, request_path=http_request_path, http_hostname=http_hostname
|
||||||
)
|
)
|
||||||
raise AssertionError(f"Expected error on getting object with cid: {cid}")
|
raise AssertionError(f"Expected error on getting object with cid: {cid}")
|
||||||
except Exception as err:
|
except Exception as err:
|
||||||
|
|
|
@ -130,6 +130,8 @@ class Cluster:
|
||||||
default_rpc_endpoint: str
|
default_rpc_endpoint: str
|
||||||
default_s3_gate_endpoint: str
|
default_s3_gate_endpoint: str
|
||||||
default_http_gate_endpoint: str
|
default_http_gate_endpoint: str
|
||||||
|
default_http_hostname: str
|
||||||
|
default_s3_hostname: str
|
||||||
|
|
||||||
def __init__(self, hosting: Hosting) -> None:
|
def __init__(self, hosting: Hosting) -> None:
|
||||||
self._hosting = hosting
|
self._hosting = hosting
|
||||||
|
@ -138,6 +140,8 @@ class Cluster:
|
||||||
self.default_rpc_endpoint = self.services(StorageNode)[0].get_rpc_endpoint()
|
self.default_rpc_endpoint = self.services(StorageNode)[0].get_rpc_endpoint()
|
||||||
self.default_s3_gate_endpoint = self.services(S3Gate)[0].get_endpoint()
|
self.default_s3_gate_endpoint = self.services(S3Gate)[0].get_endpoint()
|
||||||
self.default_http_gate_endpoint = self.services(HTTPGate)[0].get_endpoint()
|
self.default_http_gate_endpoint = self.services(HTTPGate)[0].get_endpoint()
|
||||||
|
self.default_http_hostname = self.services(StorageNode)[0].get_http_hostname()
|
||||||
|
self.default_s3_hostname = self.services(StorageNode)[0].get_s3_hostname()
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def hosts(self) -> list[Host]:
|
def hosts(self) -> list[Host]:
|
||||||
|
|
|
@ -11,6 +11,8 @@ class ConfigAttributes:
|
||||||
ENDPOINT_INTERNAL = "endpoint_internal0"
|
ENDPOINT_INTERNAL = "endpoint_internal0"
|
||||||
CONTROL_ENDPOINT = "control_endpoint"
|
CONTROL_ENDPOINT = "control_endpoint"
|
||||||
UN_LOCODE = "un_locode"
|
UN_LOCODE = "un_locode"
|
||||||
|
HTTP_HOSTNAME = "http_hostname"
|
||||||
|
S3_HOSTNAME = "s3_hostname"
|
||||||
|
|
||||||
|
|
||||||
class _FrostfsServicesNames:
|
class _FrostfsServicesNames:
|
||||||
|
|
|
@ -171,6 +171,12 @@ class StorageNode(NodeBase):
|
||||||
def get_data_directory(self) -> str:
|
def get_data_directory(self) -> str:
|
||||||
return self.host.get_data_directory(self.name)
|
return self.host.get_data_directory(self.name)
|
||||||
|
|
||||||
|
def get_http_hostname(self) -> str:
|
||||||
|
return self._get_attribute(ConfigAttributes.HTTP_HOSTNAME)
|
||||||
|
|
||||||
|
def get_s3_hostname(self) -> str:
|
||||||
|
return self._get_attribute(ConfigAttributes.S3_HOSTNAME)
|
||||||
|
|
||||||
def delete_blobovnicza(self):
|
def delete_blobovnicza(self):
|
||||||
self.host.delete_blobovnicza(self.name)
|
self.host.delete_blobovnicza(self.name)
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue