Compare commits
No commits in common. "master" and "fix-failing-cat" have entirely different histories.
master
...
fix-failin
30 changed files with 522 additions and 908 deletions
|
@ -10,7 +10,6 @@ tenacity==8.0.1
|
||||||
pytest==7.1.2
|
pytest==7.1.2
|
||||||
boto3==1.35.30
|
boto3==1.35.30
|
||||||
boto3-stubs[essential]==1.35.30
|
boto3-stubs[essential]==1.35.30
|
||||||
pydantic==2.10.6
|
|
||||||
|
|
||||||
# Dev dependencies
|
# Dev dependencies
|
||||||
black==22.8.0
|
black==22.8.0
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
__version__ = "2.0.1"
|
__version__ = "2.0.1"
|
||||||
|
|
||||||
from .fixtures import configure_testlib, hosting, temp_directory
|
from .fixtures import configure_testlib, hosting, temp_directory
|
||||||
from .hooks import pytest_add_frostfs_marker, pytest_collection_modifyitems
|
from .hooks import pytest_collection_modifyitems
|
||||||
|
|
|
@ -24,7 +24,9 @@ class CliCommand:
|
||||||
def __init__(self, shell: Shell, cli_exec_path: str, **base_params):
|
def __init__(self, shell: Shell, cli_exec_path: str, **base_params):
|
||||||
self.shell = shell
|
self.shell = shell
|
||||||
self.cli_exec_path = cli_exec_path
|
self.cli_exec_path = cli_exec_path
|
||||||
self.__base_params = " ".join([f"--{param} {value}" for param, value in base_params.items() if value])
|
self.__base_params = " ".join(
|
||||||
|
[f"--{param} {value}" for param, value in base_params.items() if value]
|
||||||
|
)
|
||||||
|
|
||||||
def _format_command(self, command: str, **params) -> str:
|
def _format_command(self, command: str, **params) -> str:
|
||||||
param_str = []
|
param_str = []
|
||||||
|
@ -46,7 +48,9 @@ class CliCommand:
|
||||||
val_str = str(value_item).replace("'", "\\'")
|
val_str = str(value_item).replace("'", "\\'")
|
||||||
param_str.append(f"--{param} '{val_str}'")
|
param_str.append(f"--{param} '{val_str}'")
|
||||||
elif isinstance(value, dict):
|
elif isinstance(value, dict):
|
||||||
param_str.append(f'--{param} \'{",".join(f"{key}={val}" for key, val in value.items())}\'')
|
param_str.append(
|
||||||
|
f'--{param} \'{",".join(f"{key}={val}" for key, val in value.items())}\''
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
if "'" in str(value):
|
if "'" in str(value):
|
||||||
value_str = str(value).replace('"', '\\"')
|
value_str = str(value).replace('"', '\\"')
|
||||||
|
@ -59,18 +63,12 @@ class CliCommand:
|
||||||
return f"{self.cli_exec_path} {self.__base_params} {command or ''} {param_str}"
|
return f"{self.cli_exec_path} {self.__base_params} {command or ''} {param_str}"
|
||||||
|
|
||||||
def _execute(self, command: Optional[str], **params) -> CommandResult:
|
def _execute(self, command: Optional[str], **params) -> CommandResult:
|
||||||
timeout = int(params["timeout"].rstrip("s")) if params.get("timeout") else None
|
return self.shell.exec(self._format_command(command, **params))
|
||||||
return self.shell.exec(
|
|
||||||
self._format_command(command, **params),
|
|
||||||
CommandOptions(timeout=timeout),
|
|
||||||
)
|
|
||||||
|
|
||||||
def _execute_with_password(self, command: Optional[str], password, **params) -> CommandResult:
|
def _execute_with_password(self, command: Optional[str], password, **params) -> CommandResult:
|
||||||
timeout = int(params["timeout"].rstrip("s")) if params.get("timeout") else None
|
|
||||||
return self.shell.exec(
|
return self.shell.exec(
|
||||||
self._format_command(command, **params),
|
self._format_command(command, **params),
|
||||||
CommandOptions(
|
options=CommandOptions(
|
||||||
interactive_inputs=[InteractiveInput(prompt_pattern="assword", input=password)],
|
interactive_inputs=[InteractiveInput(prompt_pattern="assword", input=password)]
|
||||||
timeout=timeout,
|
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
|
@ -12,7 +12,6 @@ class FrostfsCliNetmap(CliCommand):
|
||||||
address: Optional[str] = None,
|
address: Optional[str] = None,
|
||||||
generate_key: bool = False,
|
generate_key: bool = False,
|
||||||
ttl: Optional[int] = None,
|
ttl: Optional[int] = None,
|
||||||
trace: Optional[bool] = False,
|
|
||||||
xhdr: Optional[dict] = None,
|
xhdr: Optional[dict] = None,
|
||||||
timeout: Optional[str] = None,
|
timeout: Optional[str] = None,
|
||||||
) -> CommandResult:
|
) -> CommandResult:
|
||||||
|
@ -43,7 +42,6 @@ class FrostfsCliNetmap(CliCommand):
|
||||||
address: Optional[str] = None,
|
address: Optional[str] = None,
|
||||||
generate_key: bool = False,
|
generate_key: bool = False,
|
||||||
ttl: Optional[int] = None,
|
ttl: Optional[int] = None,
|
||||||
trace: Optional[bool] = False,
|
|
||||||
xhdr: Optional[dict] = None,
|
xhdr: Optional[dict] = None,
|
||||||
timeout: Optional[str] = None,
|
timeout: Optional[str] = None,
|
||||||
) -> CommandResult:
|
) -> CommandResult:
|
||||||
|
@ -75,7 +73,6 @@ class FrostfsCliNetmap(CliCommand):
|
||||||
generate_key: bool = False,
|
generate_key: bool = False,
|
||||||
json: bool = False,
|
json: bool = False,
|
||||||
ttl: Optional[int] = None,
|
ttl: Optional[int] = None,
|
||||||
trace: Optional[bool] = False,
|
|
||||||
xhdr: Optional[dict] = None,
|
xhdr: Optional[dict] = None,
|
||||||
timeout: Optional[str] = None,
|
timeout: Optional[str] = None,
|
||||||
) -> CommandResult:
|
) -> CommandResult:
|
||||||
|
@ -107,7 +104,6 @@ class FrostfsCliNetmap(CliCommand):
|
||||||
address: Optional[str] = None,
|
address: Optional[str] = None,
|
||||||
generate_key: bool = False,
|
generate_key: bool = False,
|
||||||
ttl: Optional[int] = None,
|
ttl: Optional[int] = None,
|
||||||
trace: Optional[bool] = False,
|
|
||||||
xhdr: Optional[dict] = None,
|
xhdr: Optional[dict] = None,
|
||||||
timeout: Optional[str] = None,
|
timeout: Optional[str] = None,
|
||||||
) -> CommandResult:
|
) -> CommandResult:
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
import re
|
import re
|
||||||
|
|
||||||
from frostfs_testlib.storage.cluster import ClusterNode
|
from frostfs_testlib.storage.cluster import ClusterNode
|
||||||
from frostfs_testlib.storage.dataclasses.storage_object_info import Interfaces, NodeInfo, NodeNetInfo, NodeNetmapInfo, NodeStatus
|
from frostfs_testlib.storage.dataclasses.storage_object_info import Interfaces, NodeNetInfo, NodeNetmapInfo, NodeStatus
|
||||||
|
|
||||||
|
|
||||||
class NetmapParser:
|
class NetmapParser:
|
||||||
|
@ -20,6 +20,8 @@ class NetmapParser:
|
||||||
"withdrawal_fee": r"Withdrawal fee: (?P<withdrawal_fee>\d+)",
|
"withdrawal_fee": r"Withdrawal fee: (?P<withdrawal_fee>\d+)",
|
||||||
"homomorphic_hashing_disabled": r"Homomorphic hashing disabled: (?P<homomorphic_hashing_disabled>true|false)",
|
"homomorphic_hashing_disabled": r"Homomorphic hashing disabled: (?P<homomorphic_hashing_disabled>true|false)",
|
||||||
"maintenance_mode_allowed": r"Maintenance mode allowed: (?P<maintenance_mode_allowed>true|false)",
|
"maintenance_mode_allowed": r"Maintenance mode allowed: (?P<maintenance_mode_allowed>true|false)",
|
||||||
|
"eigen_trust_alpha": r"EigenTrustAlpha: (?P<eigen_trust_alpha>\d+\w+$)",
|
||||||
|
"eigen_trust_iterations": r"EigenTrustIterations: (?P<eigen_trust_iterations>\d+)",
|
||||||
}
|
}
|
||||||
parse_result = {}
|
parse_result = {}
|
||||||
|
|
||||||
|
@ -62,7 +64,7 @@ class NetmapParser:
|
||||||
for node in netmap_nodes:
|
for node in netmap_nodes:
|
||||||
for key, regex in regexes.items():
|
for key, regex in regexes.items():
|
||||||
search_result = re.search(regex, node, flags=re.MULTILINE)
|
search_result = re.search(regex, node, flags=re.MULTILINE)
|
||||||
if search_result is None:
|
if search_result == None:
|
||||||
result_netmap[key] = None
|
result_netmap[key] = None
|
||||||
continue
|
continue
|
||||||
if key == "node_data_ips":
|
if key == "node_data_ips":
|
||||||
|
@ -81,22 +83,9 @@ class NetmapParser:
|
||||||
return dataclasses_netmap
|
return dataclasses_netmap
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def snapshot_one_node(output: str, rpc_endpoint: str) -> NodeNetmapInfo | None:
|
def snapshot_one_node(output: str, cluster_node: ClusterNode) -> NodeNetmapInfo | None:
|
||||||
snapshot_nodes = NetmapParser.snapshot_all_nodes(output=output)
|
snapshot_nodes = NetmapParser.snapshot_all_nodes(output=output)
|
||||||
for snapshot in snapshot_nodes:
|
snapshot_node = [node for node in snapshot_nodes if node.node == cluster_node.get_interface(Interfaces.MGMT)]
|
||||||
for endpoint in snapshot.external_address:
|
if not snapshot_node:
|
||||||
if rpc_endpoint.split(":")[0] in endpoint:
|
return None
|
||||||
return snapshot
|
return snapshot_node[0]
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def node_info(output: dict) -> NodeNetmapInfo:
|
|
||||||
data_dict = {"attributes": {}}
|
|
||||||
|
|
||||||
for key, value in output.items():
|
|
||||||
if key != "attributes":
|
|
||||||
data_dict[key] = value
|
|
||||||
|
|
||||||
for attribute in output["attributes"]:
|
|
||||||
data_dict["attributes"][attribute["key"]] = attribute["value"]
|
|
||||||
|
|
||||||
return NodeInfo(**data_dict)
|
|
||||||
|
|
|
@ -1 +0,0 @@
|
||||||
from frostfs_testlib.clients.http.http_client import HttpClient
|
|
|
@ -134,10 +134,9 @@ class HttpClient:
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def _create_curl_request(cls, url: str, method: str, headers: httpx.Headers, data: str, files: dict) -> str:
|
def _create_curl_request(cls, url: str, method: str, headers: httpx.Headers, data: str, files: dict) -> str:
|
||||||
excluded_headers = {"Accept-Encoding", "Connection", "User-Agent", "Content-Length"}
|
headers = " ".join(f'-H "{name.title()}: {value}"' for name, value in headers.items())
|
||||||
headers = " ".join(f"-H '{header.title()}: {value}'" for header, value in headers.items() if header.title() not in excluded_headers)
|
|
||||||
|
|
||||||
data = f" -d '{data}'" if data else ""
|
data = f" -d '{data}'" if data else ""
|
||||||
|
|
||||||
for name, path in files.items():
|
for name, path in files.items():
|
||||||
data += f' -F "{name}=@{path}"'
|
data += f' -F "{name}=@{path}"'
|
||||||
|
|
||||||
|
|
|
@ -1,3 +1 @@
|
||||||
from frostfs_testlib.clients.s3.aws_cli_client import AwsCliClient
|
from frostfs_testlib.clients.s3.interfaces import BucketContainerResolver, S3ClientWrapper, VersioningStatus
|
||||||
from frostfs_testlib.clients.s3.boto3_client import Boto3ClientWrapper
|
|
||||||
from frostfs_testlib.clients.s3.interfaces import ACL, BucketContainerResolver, S3ClientWrapper, VersioningStatus
|
|
||||||
|
|
|
@ -33,14 +33,12 @@ class AwsCliClient(S3ClientWrapper):
|
||||||
self, access_key_id: str, secret_access_key: str, s3gate_endpoint: str, profile: str = "default", region: str = "us-east-1"
|
self, access_key_id: str, secret_access_key: str, s3gate_endpoint: str, profile: str = "default", region: str = "us-east-1"
|
||||||
) -> None:
|
) -> None:
|
||||||
self.s3gate_endpoint = s3gate_endpoint
|
self.s3gate_endpoint = s3gate_endpoint
|
||||||
self.iam_endpoint = None
|
|
||||||
|
|
||||||
self.access_key_id: str = access_key_id
|
self.access_key_id: str = access_key_id
|
||||||
self.secret_access_key: str = secret_access_key
|
self.secret_access_key: str = secret_access_key
|
||||||
self.profile = profile
|
self.profile = profile
|
||||||
self.region = region
|
|
||||||
|
|
||||||
self.local_shell = LocalShell()
|
self.local_shell = LocalShell()
|
||||||
|
self.region = region
|
||||||
|
self.iam_endpoint = None
|
||||||
try:
|
try:
|
||||||
_configure_aws_cli(f"aws configure --profile {profile}", access_key_id, secret_access_key, region)
|
_configure_aws_cli(f"aws configure --profile {profile}", access_key_id, secret_access_key, region)
|
||||||
self.local_shell.exec(f"aws configure set max_attempts {MAX_REQUEST_ATTEMPTS} --profile {profile}")
|
self.local_shell.exec(f"aws configure set max_attempts {MAX_REQUEST_ATTEMPTS} --profile {profile}")
|
||||||
|
@ -1229,7 +1227,7 @@ class AwsCliClient(S3ClientWrapper):
|
||||||
output = self.local_shell.exec(cmd).stdout
|
output = self.local_shell.exec(cmd).stdout
|
||||||
response = self._to_json(output)
|
response = self._to_json(output)
|
||||||
|
|
||||||
assert "AttachedPolicies" in response.keys(), f"Expected AttachedPolicies in response:\n{response}"
|
assert response.get("AttachedPolicies"), f"Expected AttachedPolicies in response:\n{response}"
|
||||||
|
|
||||||
return response
|
return response
|
||||||
|
|
||||||
|
@ -1241,7 +1239,7 @@ class AwsCliClient(S3ClientWrapper):
|
||||||
output = self.local_shell.exec(cmd).stdout
|
output = self.local_shell.exec(cmd).stdout
|
||||||
response = self._to_json(output)
|
response = self._to_json(output)
|
||||||
|
|
||||||
assert "AttachedPolicies" in response.keys(), f"Expected AttachedPolicies in response:\n{response}"
|
assert response.get("AttachedPolicies"), f"Expected AttachedPolicies in response:\n{response}"
|
||||||
|
|
||||||
return response
|
return response
|
||||||
|
|
||||||
|
@ -1266,7 +1264,7 @@ class AwsCliClient(S3ClientWrapper):
|
||||||
output = self.local_shell.exec(cmd).stdout
|
output = self.local_shell.exec(cmd).stdout
|
||||||
response = self._to_json(output)
|
response = self._to_json(output)
|
||||||
|
|
||||||
assert "PolicyNames" in response.keys(), f"Expected PolicyNames in response:\n{response}"
|
assert response.get("PolicyNames"), f"Expected PolicyNames in response:\n{response}"
|
||||||
|
|
||||||
return response
|
return response
|
||||||
|
|
||||||
|
@ -1278,7 +1276,7 @@ class AwsCliClient(S3ClientWrapper):
|
||||||
output = self.local_shell.exec(cmd).stdout
|
output = self.local_shell.exec(cmd).stdout
|
||||||
response = self._to_json(output)
|
response = self._to_json(output)
|
||||||
|
|
||||||
assert "Groups" in response.keys(), f"Expected Groups in response:\n{response}"
|
assert response.get("Groups"), f"Expected Groups in response:\n{response}"
|
||||||
|
|
||||||
return response
|
return response
|
||||||
|
|
||||||
|
@ -1290,7 +1288,7 @@ class AwsCliClient(S3ClientWrapper):
|
||||||
output = self.local_shell.exec(cmd).stdout
|
output = self.local_shell.exec(cmd).stdout
|
||||||
response = self._to_json(output)
|
response = self._to_json(output)
|
||||||
|
|
||||||
assert "Groups" in response.keys(), f"Expected Groups in response:\n{response}"
|
assert response.get("Groups"), f"Expected Groups in response:\n{response}"
|
||||||
|
|
||||||
return response
|
return response
|
||||||
|
|
||||||
|
@ -1326,7 +1324,7 @@ class AwsCliClient(S3ClientWrapper):
|
||||||
output = self.local_shell.exec(cmd).stdout
|
output = self.local_shell.exec(cmd).stdout
|
||||||
response = self._to_json(output)
|
response = self._to_json(output)
|
||||||
|
|
||||||
assert "PolicyNames" in response.keys(), f"Expected PolicyNames in response:\n{response}"
|
assert response.get("PolicyNames"), f"Expected PolicyNames in response:\n{response}"
|
||||||
|
|
||||||
return response
|
return response
|
||||||
|
|
||||||
|
|
|
@ -35,20 +35,26 @@ class Boto3ClientWrapper(S3ClientWrapper):
|
||||||
def __init__(
|
def __init__(
|
||||||
self, access_key_id: str, secret_access_key: str, s3gate_endpoint: str, profile: str = "default", region: str = "us-east-1"
|
self, access_key_id: str, secret_access_key: str, s3gate_endpoint: str, profile: str = "default", region: str = "us-east-1"
|
||||||
) -> None:
|
) -> None:
|
||||||
self.s3gate_endpoint: str = ""
|
|
||||||
self.boto3_client: S3Client = None
|
self.boto3_client: S3Client = None
|
||||||
|
self.s3gate_endpoint: str = ""
|
||||||
|
|
||||||
self.iam_endpoint: str = ""
|
|
||||||
self.boto3_iam_client: S3Client = None
|
self.boto3_iam_client: S3Client = None
|
||||||
|
self.iam_endpoint: str = ""
|
||||||
|
|
||||||
self.boto3_sts_client: S3Client = None
|
self.boto3_sts_client: S3Client = None
|
||||||
|
|
||||||
self.access_key_id = access_key_id
|
self.access_key_id: str = access_key_id
|
||||||
self.secret_access_key = secret_access_key
|
self.secret_access_key: str = secret_access_key
|
||||||
self.profile = profile
|
self.profile = profile
|
||||||
self.region = region
|
self.region = region
|
||||||
|
|
||||||
self.session = boto3.Session()
|
self.session = boto3.Session()
|
||||||
self.config = Config(retries={"max_attempts": MAX_REQUEST_ATTEMPTS, "mode": RETRY_MODE})
|
self.config = Config(
|
||||||
|
retries={
|
||||||
|
"max_attempts": MAX_REQUEST_ATTEMPTS,
|
||||||
|
"mode": RETRY_MODE,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
self.set_endpoint(s3gate_endpoint)
|
self.set_endpoint(s3gate_endpoint)
|
||||||
|
|
||||||
|
@ -84,7 +90,7 @@ class Boto3ClientWrapper(S3ClientWrapper):
|
||||||
endpoint_url=self.iam_endpoint,
|
endpoint_url=self.iam_endpoint,
|
||||||
verify=False,
|
verify=False,
|
||||||
)
|
)
|
||||||
# since the STS does not have an endpoint, IAM is used
|
# since the STS does not have an enpoint, IAM is used
|
||||||
self.boto3_sts_client = self.session.client(
|
self.boto3_sts_client = self.session.client(
|
||||||
service_name="sts",
|
service_name="sts",
|
||||||
aws_access_key_id=self.access_key_id,
|
aws_access_key_id=self.access_key_id,
|
||||||
|
@ -139,7 +145,6 @@ class Boto3ClientWrapper(S3ClientWrapper):
|
||||||
params = {"Bucket": bucket}
|
params = {"Bucket": bucket}
|
||||||
if object_lock_enabled_for_bucket is not None:
|
if object_lock_enabled_for_bucket is not None:
|
||||||
params.update({"ObjectLockEnabledForBucket": object_lock_enabled_for_bucket})
|
params.update({"ObjectLockEnabledForBucket": object_lock_enabled_for_bucket})
|
||||||
|
|
||||||
if acl is not None:
|
if acl is not None:
|
||||||
params.update({"ACL": acl})
|
params.update({"ACL": acl})
|
||||||
elif grant_write or grant_read or grant_full_control:
|
elif grant_write or grant_read or grant_full_control:
|
||||||
|
@ -149,7 +154,6 @@ class Boto3ClientWrapper(S3ClientWrapper):
|
||||||
params.update({"GrantRead": grant_read})
|
params.update({"GrantRead": grant_read})
|
||||||
elif grant_full_control:
|
elif grant_full_control:
|
||||||
params.update({"GrantFullControl": grant_full_control})
|
params.update({"GrantFullControl": grant_full_control})
|
||||||
|
|
||||||
if location_constraint:
|
if location_constraint:
|
||||||
params.update({"CreateBucketConfiguration": {"LocationConstraint": location_constraint}})
|
params.update({"CreateBucketConfiguration": {"LocationConstraint": location_constraint}})
|
||||||
|
|
||||||
|
@ -1087,7 +1091,7 @@ class Boto3ClientWrapper(S3ClientWrapper):
|
||||||
endpoint=self.iam_endpoint,
|
endpoint=self.iam_endpoint,
|
||||||
profile=self.profile,
|
profile=self.profile,
|
||||||
)
|
)
|
||||||
assert "AttachedPolicies" in response.keys(), f"Expected AttachedPolicies in response:\n{response}"
|
assert response.get("AttachedPolicies"), f"Expected AttachedPolicies in response:\n{response}"
|
||||||
return response
|
return response
|
||||||
|
|
||||||
@reporter.step("Lists all managed policies that are attached to the specified IAM user")
|
@reporter.step("Lists all managed policies that are attached to the specified IAM user")
|
||||||
|
@ -1098,7 +1102,7 @@ class Boto3ClientWrapper(S3ClientWrapper):
|
||||||
endpoint=self.iam_endpoint,
|
endpoint=self.iam_endpoint,
|
||||||
profile=self.profile,
|
profile=self.profile,
|
||||||
)
|
)
|
||||||
assert "AttachedPolicies" in response.keys(), f"Expected AttachedPolicies in response:\n{response}"
|
assert response.get("AttachedPolicies"), f"Expected AttachedPolicies in response:\n{response}"
|
||||||
return response
|
return response
|
||||||
|
|
||||||
@reporter.step("Lists all IAM users, groups, and roles that the specified managed policy is attached to")
|
@reporter.step("Lists all IAM users, groups, and roles that the specified managed policy is attached to")
|
||||||
|
@ -1123,7 +1127,7 @@ class Boto3ClientWrapper(S3ClientWrapper):
|
||||||
endpoint=self.iam_endpoint,
|
endpoint=self.iam_endpoint,
|
||||||
profile=self.profile,
|
profile=self.profile,
|
||||||
)
|
)
|
||||||
assert "PolicyNames" in response.keys(), f"Expected PolicyNames in response:\n{response}"
|
assert response.get("PolicyNames"), f"Expected PolicyNames in response:\n{response}"
|
||||||
return response
|
return response
|
||||||
|
|
||||||
@reporter.step("Lists the IAM groups")
|
@reporter.step("Lists the IAM groups")
|
||||||
|
@ -1133,7 +1137,7 @@ class Boto3ClientWrapper(S3ClientWrapper):
|
||||||
endpoint=self.iam_endpoint,
|
endpoint=self.iam_endpoint,
|
||||||
profile=self.profile,
|
profile=self.profile,
|
||||||
)
|
)
|
||||||
assert "Groups" in response.keys(), f"Expected Groups in response:\n{response}"
|
assert response.get("Groups"), f"Expected Groups in response:\n{response}"
|
||||||
return response
|
return response
|
||||||
|
|
||||||
@reporter.step("Lists the IAM groups that the specified IAM user belongs to")
|
@reporter.step("Lists the IAM groups that the specified IAM user belongs to")
|
||||||
|
@ -1144,7 +1148,7 @@ class Boto3ClientWrapper(S3ClientWrapper):
|
||||||
endpoint=self.iam_endpoint,
|
endpoint=self.iam_endpoint,
|
||||||
profile=self.profile,
|
profile=self.profile,
|
||||||
)
|
)
|
||||||
assert "Groups" in response.keys(), f"Expected Groups in response:\n{response}"
|
assert response.get("Groups"), f"Expected Groups in response:\n{response}"
|
||||||
return response
|
return response
|
||||||
|
|
||||||
@reporter.step("Lists all the managed policies that are available in your AWS account")
|
@reporter.step("Lists all the managed policies that are available in your AWS account")
|
||||||
|
@ -1176,7 +1180,7 @@ class Boto3ClientWrapper(S3ClientWrapper):
|
||||||
endpoint=self.iam_endpoint,
|
endpoint=self.iam_endpoint,
|
||||||
profile=self.profile,
|
profile=self.profile,
|
||||||
)
|
)
|
||||||
assert "PolicyNames" in response.keys(), f"Expected PolicyNames in response:\n{response}"
|
assert response.get("PolicyNames"), f"Expected PolicyNames in response:\n{response}"
|
||||||
return response
|
return response
|
||||||
|
|
||||||
@reporter.step("Lists the IAM users")
|
@reporter.step("Lists the IAM users")
|
||||||
|
|
|
@ -22,15 +22,15 @@ class VersioningStatus(HumanReadableEnum):
|
||||||
SUSPENDED = "Suspended"
|
SUSPENDED = "Suspended"
|
||||||
|
|
||||||
|
|
||||||
class ACL:
|
ACL_COPY = [
|
||||||
PRIVATE = "private"
|
"private",
|
||||||
PUBLIC_READ = "public-read"
|
"public-read",
|
||||||
PUBLIC_READ_WRITE = "public-read-write"
|
"public-read-write",
|
||||||
AUTHENTICATED_READ = "authenticated-read"
|
"authenticated-read",
|
||||||
AWS_EXEC_READ = "aws-exec-read"
|
"aws-exec-read",
|
||||||
BUCKET_OWNER_READ = "bucket-owner-read"
|
"bucket-owner-read",
|
||||||
BUCKET_OWNER_FULL_CONTROL = "bucket-owner-full-control"
|
"bucket-owner-full-control",
|
||||||
LOG_DELIVERY_WRITE = "log-delivery-write"
|
]
|
||||||
|
|
||||||
|
|
||||||
class BucketContainerResolver(ABC):
|
class BucketContainerResolver(ABC):
|
||||||
|
@ -50,14 +50,6 @@ class BucketContainerResolver(ABC):
|
||||||
|
|
||||||
|
|
||||||
class S3ClientWrapper(HumanReadableABC):
|
class S3ClientWrapper(HumanReadableABC):
|
||||||
access_key_id: str
|
|
||||||
secret_access_key: str
|
|
||||||
profile: str
|
|
||||||
region: str
|
|
||||||
|
|
||||||
s3gate_endpoint: str
|
|
||||||
iam_endpoint: str
|
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def __init__(self, access_key_id: str, secret_access_key: str, s3gate_endpoint: str, profile: str, region: str) -> None:
|
def __init__(self, access_key_id: str, secret_access_key: str, s3gate_endpoint: str, profile: str, region: str) -> None:
|
||||||
pass
|
pass
|
||||||
|
|
|
@ -1,8 +1,8 @@
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
@pytest.hookimpl(specname="pytest_collection_modifyitems")
|
@pytest.hookimpl
|
||||||
def pytest_add_frostfs_marker(items: list[pytest.Item]):
|
def pytest_collection_modifyitems(items: list[pytest.Item]):
|
||||||
# All tests which reside in frostfs nodeid are granted with frostfs marker, excluding
|
# All tests which reside in frostfs nodeid are granted with frostfs marker, excluding
|
||||||
# nodeid = full path of the test
|
# nodeid = full path of the test
|
||||||
# 1. plugins
|
# 1. plugins
|
||||||
|
@ -11,18 +11,3 @@ def pytest_add_frostfs_marker(items: list[pytest.Item]):
|
||||||
location = item.location[0]
|
location = item.location[0]
|
||||||
if "frostfs" in location and "plugin" not in location and "testlib" not in location:
|
if "frostfs" in location and "plugin" not in location and "testlib" not in location:
|
||||||
item.add_marker("frostfs")
|
item.add_marker("frostfs")
|
||||||
|
|
||||||
|
|
||||||
# pytest hook. Do not rename
|
|
||||||
@pytest.hookimpl(trylast=True)
|
|
||||||
def pytest_collection_modifyitems(items: list[pytest.Item]):
|
|
||||||
# Change order of tests based on @pytest.mark.order(<int>) marker
|
|
||||||
def order(item: pytest.Item) -> int:
|
|
||||||
order_marker = item.get_closest_marker("order")
|
|
||||||
if order_marker and (len(order_marker.args) != 1 or not isinstance(order_marker.args[0], int)):
|
|
||||||
raise RuntimeError("Incorrect usage of pytest.mark.order")
|
|
||||||
|
|
||||||
order_value = order_marker.args[0] if order_marker else 0
|
|
||||||
return order_value
|
|
||||||
|
|
||||||
items.sort(key=lambda item: order(item))
|
|
||||||
|
|
|
@ -182,10 +182,8 @@ class Preset(MetaConfig):
|
||||||
pregen_json: Optional[str] = metadata_field(all_load_scenarios, "out", "PREGEN_JSON", False)
|
pregen_json: Optional[str] = metadata_field(all_load_scenarios, "out", "PREGEN_JSON", False)
|
||||||
# Workers count for preset
|
# Workers count for preset
|
||||||
workers: Optional[int] = metadata_field(all_load_scenarios, "workers", None, False)
|
workers: Optional[int] = metadata_field(all_load_scenarios, "workers", None, False)
|
||||||
# TODO: Deprecated. Acl for container/buckets
|
# Acl for container/buckets
|
||||||
acl: Optional[str] = metadata_field(all_load_scenarios, "acl", None, False)
|
acl: Optional[str] = metadata_field(all_load_scenarios, "acl", None, False)
|
||||||
# APE rule for containers instead of deprecated ACL
|
|
||||||
rule: Optional[list[str]] = metadata_field(grpc_preset_scenarios, "rule", None, False, formatter=force_list)
|
|
||||||
|
|
||||||
# ------ GRPC ------
|
# ------ GRPC ------
|
||||||
# Amount of containers which should be created
|
# Amount of containers which should be created
|
||||||
|
|
|
@ -1,6 +1,5 @@
|
||||||
# Regex patterns of status codes of Container service
|
# Regex patterns of status codes of Container service
|
||||||
CONTAINER_NOT_FOUND = "code = 3072.*message = container not found"
|
CONTAINER_NOT_FOUND = "code = 3072.*message = container not found"
|
||||||
SUBJECT_NOT_FOUND = "code = 1024.*message = frostfs error: chain/client.*subject not found.*"
|
|
||||||
|
|
||||||
# Regex patterns of status codes of Object service
|
# Regex patterns of status codes of Object service
|
||||||
MALFORMED_REQUEST = "code = 1024.*message = malformed request"
|
MALFORMED_REQUEST = "code = 1024.*message = malformed request"
|
||||||
|
|
|
@ -5,7 +5,6 @@ class ConfigAttributes:
|
||||||
WALLET_CONFIG = "wallet_config"
|
WALLET_CONFIG = "wallet_config"
|
||||||
CONFIG_DIR = "service_config_dir"
|
CONFIG_DIR = "service_config_dir"
|
||||||
CONFIG_PATH = "config_path"
|
CONFIG_PATH = "config_path"
|
||||||
WORKING_DIR = "working_dir"
|
|
||||||
SHARD_CONFIG_PATH = "shard_config_path"
|
SHARD_CONFIG_PATH = "shard_config_path"
|
||||||
LOGGER_CONFIG_PATH = "logger_config_path"
|
LOGGER_CONFIG_PATH = "logger_config_path"
|
||||||
LOCAL_WALLET_PATH = "local_wallet_path"
|
LOCAL_WALLET_PATH = "local_wallet_path"
|
||||||
|
@ -16,7 +15,6 @@ class ConfigAttributes:
|
||||||
ENDPOINT_DATA_0_NS = "endpoint_data0_namespace"
|
ENDPOINT_DATA_0_NS = "endpoint_data0_namespace"
|
||||||
ENDPOINT_INTERNAL = "endpoint_internal0"
|
ENDPOINT_INTERNAL = "endpoint_internal0"
|
||||||
ENDPOINT_PROMETHEUS = "endpoint_prometheus"
|
ENDPOINT_PROMETHEUS = "endpoint_prometheus"
|
||||||
ENDPOINT_PPROF = "endpoint_pprof"
|
|
||||||
CONTROL_ENDPOINT = "control_endpoint"
|
CONTROL_ENDPOINT = "control_endpoint"
|
||||||
UN_LOCODE = "un_locode"
|
UN_LOCODE = "un_locode"
|
||||||
|
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
|
import datetime
|
||||||
import itertools
|
import itertools
|
||||||
import logging
|
import logging
|
||||||
import time
|
import time
|
||||||
from datetime import datetime, timezone
|
|
||||||
from typing import TypeVar
|
from typing import TypeVar
|
||||||
|
|
||||||
import frostfs_testlib.resources.optionals as optionals
|
import frostfs_testlib.resources.optionals as optionals
|
||||||
|
@ -390,23 +390,31 @@ class ClusterStateController:
|
||||||
@reporter.step("Get node time")
|
@reporter.step("Get node time")
|
||||||
def get_node_date(self, node: ClusterNode) -> datetime:
|
def get_node_date(self, node: ClusterNode) -> datetime:
|
||||||
shell = node.host.get_shell()
|
shell = node.host.get_shell()
|
||||||
return datetime.strptime(shell.exec('date +"%Y-%m-%d %H:%M:%S"').stdout.strip(), "%Y-%m-%d %H:%M:%S")
|
return datetime.datetime.strptime(shell.exec("hwclock -r").stdout.strip(), "%Y-%m-%d %H:%M:%S.%f%z")
|
||||||
|
|
||||||
@reporter.step("Set node time to {in_date}")
|
@reporter.step("Set node time to {in_date}")
|
||||||
def change_node_date(self, node: ClusterNode, in_date: datetime) -> None:
|
def change_node_date(self, node: ClusterNode, in_date: datetime) -> None:
|
||||||
shell = node.host.get_shell()
|
shell = node.host.get_shell()
|
||||||
in_date_frmt = in_date.strftime("%Y-%m-%d %H:%M:%S")
|
shell.exec(f"date -s @{time.mktime(in_date.timetuple())}")
|
||||||
shell.exec(f"timedatectl set-time '{in_date_frmt}'")
|
shell.exec("hwclock --systohc")
|
||||||
node_time = self.get_node_date(node)
|
node_time = self.get_node_date(node)
|
||||||
with reporter.step(f"Verify difference between {node_time} and {in_date} is less than a minute"):
|
with reporter.step(f"Verify difference between {node_time} and {in_date} is less than a minute"):
|
||||||
assert (node_time - in_date).total_seconds() < 60
|
assert (self.get_node_date(node) - in_date) < datetime.timedelta(minutes=1)
|
||||||
|
|
||||||
@reporter.step("Restore time")
|
@reporter.step(f"Restore time")
|
||||||
def restore_node_date(self, node: ClusterNode) -> None:
|
def restore_node_date(self, node: ClusterNode) -> None:
|
||||||
shell = node.host.get_shell()
|
shell = node.host.get_shell()
|
||||||
now_time = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S")
|
now_time = datetime.datetime.now(datetime.timezone.utc)
|
||||||
with reporter.step(f"Set {now_time} time"):
|
with reporter.step(f"Set {now_time} time"):
|
||||||
shell.exec(f"timedatectl set-time '{now_time}'")
|
shell.exec(f"date -s @{time.mktime(now_time.timetuple())}")
|
||||||
|
shell.exec("hwclock --systohc")
|
||||||
|
|
||||||
|
@reporter.step("Change the synchronizer status to {status}")
|
||||||
|
def set_sync_date_all_nodes(self, status: str):
|
||||||
|
if status == "active":
|
||||||
|
parallel(self._enable_date_synchronizer, self.cluster.cluster_nodes)
|
||||||
|
return
|
||||||
|
parallel(self._disable_date_synchronizer, self.cluster.cluster_nodes)
|
||||||
|
|
||||||
@reporter.step("Set MaintenanceModeAllowed - {status}")
|
@reporter.step("Set MaintenanceModeAllowed - {status}")
|
||||||
def set_maintenance_mode_allowed(self, status: str, cluster_node: ClusterNode) -> None:
|
def set_maintenance_mode_allowed(self, status: str, cluster_node: ClusterNode) -> None:
|
||||||
|
@ -492,6 +500,16 @@ class ClusterStateController:
|
||||||
frostfs_cli_remote = FrostfsCli(shell=shell, frostfs_cli_exec_path=FROSTFS_CLI_EXEC, config_file=wallet_config_path)
|
frostfs_cli_remote = FrostfsCli(shell=shell, frostfs_cli_exec_path=FROSTFS_CLI_EXEC, config_file=wallet_config_path)
|
||||||
return frostfs_adm, frostfs_cli, frostfs_cli_remote
|
return frostfs_adm, frostfs_cli, frostfs_cli_remote
|
||||||
|
|
||||||
|
def _enable_date_synchronizer(self, cluster_node: ClusterNode):
|
||||||
|
shell = cluster_node.host.get_shell()
|
||||||
|
shell.exec("timedatectl set-ntp true")
|
||||||
|
cluster_node.host.wait_for_service_to_be_in_state("systemd-timesyncd", "active", 15)
|
||||||
|
|
||||||
|
def _disable_date_synchronizer(self, cluster_node: ClusterNode):
|
||||||
|
shell = cluster_node.host.get_shell()
|
||||||
|
shell.exec("timedatectl set-ntp false")
|
||||||
|
cluster_node.host.wait_for_service_to_be_in_state("systemd-timesyncd", "inactive", 15)
|
||||||
|
|
||||||
def _get_disk_controller(self, node: StorageNode, device: str, mountpoint: str) -> DiskController:
|
def _get_disk_controller(self, node: StorageNode, device: str, mountpoint: str) -> DiskController:
|
||||||
disk_controller_id = DiskController.get_id(node, device)
|
disk_controller_id = DiskController.get_id(node, device)
|
||||||
if disk_controller_id in self.detached_disks.keys():
|
if disk_controller_id in self.detached_disks.keys():
|
||||||
|
|
|
@ -82,9 +82,6 @@ class NodeBase(HumanReadableABC):
|
||||||
def get_metrics_endpoint(self) -> str:
|
def get_metrics_endpoint(self) -> str:
|
||||||
return self._get_attribute(ConfigAttributes.ENDPOINT_PROMETHEUS)
|
return self._get_attribute(ConfigAttributes.ENDPOINT_PROMETHEUS)
|
||||||
|
|
||||||
def get_pprof_endpoint(self) -> str:
|
|
||||||
return self._get_attribute(ConfigAttributes.ENDPOINT_PPROF)
|
|
||||||
|
|
||||||
def stop_service(self, mask: bool = True):
|
def stop_service(self, mask: bool = True):
|
||||||
if mask:
|
if mask:
|
||||||
with reporter.step(f"Mask {self.name} service on {self.host.config.address}"):
|
with reporter.step(f"Mask {self.name} service on {self.host.config.address}"):
|
||||||
|
@ -147,13 +144,6 @@ class NodeBase(HumanReadableABC):
|
||||||
else None
|
else None
|
||||||
)
|
)
|
||||||
|
|
||||||
def get_working_dir_path(self) -> Optional[str]:
|
|
||||||
"""
|
|
||||||
Returns working directory path located on remote host
|
|
||||||
"""
|
|
||||||
config_attributes = self.host.get_service_config(self.name)
|
|
||||||
return self._get_attribute(ConfigAttributes.WORKING_DIR) if ConfigAttributes.WORKING_DIR in config_attributes.attributes else None
|
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def config_dir(self) -> str:
|
def config_dir(self) -> str:
|
||||||
return self._get_attribute(ConfigAttributes.CONFIG_DIR)
|
return self._get_attribute(ConfigAttributes.CONFIG_DIR)
|
||||||
|
|
|
@ -1,9 +1,6 @@
|
||||||
import re
|
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
||||||
from pydantic import BaseModel, Field, field_validator
|
|
||||||
|
|
||||||
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
|
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
|
||||||
from frostfs_testlib.testing.readable import HumanReadableEnum
|
from frostfs_testlib.testing.readable import HumanReadableEnum
|
||||||
|
|
||||||
|
@ -78,37 +75,8 @@ class NodeNetInfo:
|
||||||
withdrawal_fee: str = None
|
withdrawal_fee: str = None
|
||||||
homomorphic_hashing_disabled: str = None
|
homomorphic_hashing_disabled: str = None
|
||||||
maintenance_mode_allowed: str = None
|
maintenance_mode_allowed: str = None
|
||||||
|
eigen_trust_alpha: str = None
|
||||||
|
eigen_trust_iterations: str = None
|
||||||
class Attributes(BaseModel):
|
|
||||||
cluster_name: str = Field(alias="ClusterName")
|
|
||||||
continent: str = Field(alias="Continent")
|
|
||||||
country: str = Field(alias="Country")
|
|
||||||
country_code: str = Field(alias="CountryCode")
|
|
||||||
external_addr: list[str] = Field(alias="ExternalAddr")
|
|
||||||
location: str = Field(alias="Location")
|
|
||||||
node: str = Field(alias="Node")
|
|
||||||
subdiv: str = Field(alias="SubDiv")
|
|
||||||
subdiv_code: str = Field(alias="SubDivCode")
|
|
||||||
un_locode: str = Field(alias="UN-LOCODE")
|
|
||||||
role: str = Field(alias="role")
|
|
||||||
|
|
||||||
@field_validator("external_addr", mode="before")
|
|
||||||
@classmethod
|
|
||||||
def convert_external_addr(cls, value: str) -> list[str]:
|
|
||||||
return [f"{ip}:{port}" for ip, port in re.findall(r"/ip4/([\d\.]+)/(?:tcp|tls)/(\d+)", value)]
|
|
||||||
|
|
||||||
|
|
||||||
class NodeInfo(BaseModel):
|
|
||||||
public_key: str = Field(alias="publicKey")
|
|
||||||
addresses: list[str] = Field(alias="addresses")
|
|
||||||
state: str = Field(alias="state")
|
|
||||||
attributes: Attributes = Field(alias="attributes")
|
|
||||||
|
|
||||||
@field_validator("addresses", mode="before")
|
|
||||||
@classmethod
|
|
||||||
def convert_external_addr(cls, value: str) -> list[str]:
|
|
||||||
return [f"{ip}:{port}" for ip, port in re.findall(r"/ip4/([\d\.]+)/(?:tcp|tls)/(\d+)", ",".join(value))]
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
|
|
|
@ -1,14 +1,14 @@
|
||||||
from frostfs_testlib.cli.frostfs_cli.cli import FrostfsCli
|
from frostfs_testlib.cli.frostfs_cli.cli import FrostfsCli
|
||||||
from frostfs_testlib.storage.grpc_operations import implementations, interfaces, interfaces_wrapper
|
from frostfs_testlib.storage.grpc_operations import interfaces
|
||||||
|
from frostfs_testlib.storage.grpc_operations.implementations import container, object
|
||||||
|
|
||||||
|
|
||||||
class CliClientWrapper(interfaces_wrapper.GrpcClientWrapper):
|
class CliClientWrapper(interfaces.GrpcClientWrapper):
|
||||||
def __init__(self, cli: FrostfsCli) -> None:
|
def __init__(self, cli: FrostfsCli) -> None:
|
||||||
self.cli = cli
|
self.cli = cli
|
||||||
self.object: interfaces.ObjectInterface = implementations.ObjectOperations(self.cli)
|
self.object: interfaces.ObjectInterface = object.ObjectOperations(self.cli)
|
||||||
self.container: interfaces.ContainerInterface = implementations.ContainerOperations(self.cli)
|
self.container: interfaces.ContainerInterface = container.ContainerOperations(self.cli)
|
||||||
self.netmap: interfaces.NetmapInterface = implementations.NetmapOperations(self.cli)
|
|
||||||
|
|
||||||
|
|
||||||
class RpcClientWrapper(interfaces_wrapper.GrpcClientWrapper):
|
class RpcClientWrapper(interfaces.GrpcClientWrapper):
|
||||||
pass # The next series
|
pass # The next series
|
||||||
|
|
|
@ -1,4 +0,0 @@
|
||||||
from .chunks import ChunksOperations
|
|
||||||
from .container import ContainerOperations
|
|
||||||
from .netmap import NetmapOperations
|
|
||||||
from .object import ObjectOperations
|
|
|
@ -1,171 +0,0 @@
|
||||||
import json as module_json
|
|
||||||
from typing import List, Optional
|
|
||||||
|
|
||||||
from frostfs_testlib.cli.frostfs_cli.cli import FrostfsCli
|
|
||||||
from frostfs_testlib.cli.netmap_parser import NetmapParser
|
|
||||||
from frostfs_testlib.resources.cli import CLI_DEFAULT_TIMEOUT
|
|
||||||
from frostfs_testlib.storage.dataclasses.storage_object_info import NodeNetInfo, NodeNetmapInfo
|
|
||||||
|
|
||||||
from .. import interfaces
|
|
||||||
|
|
||||||
|
|
||||||
class NetmapOperations(interfaces.NetmapInterface):
|
|
||||||
def __init__(self, cli: FrostfsCli) -> None:
|
|
||||||
self.cli = cli
|
|
||||||
|
|
||||||
def epoch(
|
|
||||||
self,
|
|
||||||
rpc_endpoint: str,
|
|
||||||
wallet: Optional[str] = None,
|
|
||||||
address: Optional[str] = None,
|
|
||||||
generate_key: bool = False,
|
|
||||||
ttl: Optional[int] = None,
|
|
||||||
trace: Optional[bool] = True,
|
|
||||||
xhdr: Optional[dict] = None,
|
|
||||||
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
|
||||||
) -> int:
|
|
||||||
"""
|
|
||||||
Get current epoch number.
|
|
||||||
"""
|
|
||||||
output = (
|
|
||||||
self.cli.netmap.epoch(
|
|
||||||
rpc_endpoint=rpc_endpoint,
|
|
||||||
wallet=wallet,
|
|
||||||
address=address,
|
|
||||||
generate_key=generate_key,
|
|
||||||
ttl=ttl,
|
|
||||||
trace=trace,
|
|
||||||
xhdr=xhdr,
|
|
||||||
timeout=timeout,
|
|
||||||
)
|
|
||||||
.stdout.split("Trace ID")[0]
|
|
||||||
.strip()
|
|
||||||
)
|
|
||||||
|
|
||||||
return int(output)
|
|
||||||
|
|
||||||
def netinfo(
|
|
||||||
self,
|
|
||||||
rpc_endpoint: str,
|
|
||||||
wallet: Optional[str] = None,
|
|
||||||
address: Optional[str] = None,
|
|
||||||
generate_key: bool = False,
|
|
||||||
ttl: Optional[int] = None,
|
|
||||||
trace: Optional[bool] = True,
|
|
||||||
xhdr: Optional[dict] = None,
|
|
||||||
timeout: Optional[str] = None,
|
|
||||||
) -> NodeNetInfo:
|
|
||||||
"""
|
|
||||||
Get target node info.
|
|
||||||
"""
|
|
||||||
output = (
|
|
||||||
self.cli.netmap.netinfo(
|
|
||||||
rpc_endpoint=rpc_endpoint,
|
|
||||||
wallet=wallet,
|
|
||||||
address=address,
|
|
||||||
generate_key=generate_key,
|
|
||||||
ttl=ttl,
|
|
||||||
trace=trace,
|
|
||||||
xhdr=xhdr,
|
|
||||||
timeout=timeout,
|
|
||||||
)
|
|
||||||
.stdout.split("Trace ID")[0]
|
|
||||||
.strip()
|
|
||||||
)
|
|
||||||
|
|
||||||
return NetmapParser.netinfo(output)
|
|
||||||
|
|
||||||
def nodeinfo(
|
|
||||||
self,
|
|
||||||
rpc_endpoint: str,
|
|
||||||
wallet: Optional[str] = None,
|
|
||||||
address: Optional[str] = None,
|
|
||||||
generate_key: bool = False,
|
|
||||||
json: bool = True,
|
|
||||||
ttl: Optional[int] = None,
|
|
||||||
trace: Optional[bool] = True,
|
|
||||||
xhdr: Optional[dict] = None,
|
|
||||||
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
|
||||||
) -> NodeNetmapInfo:
|
|
||||||
"""
|
|
||||||
Get target node info.
|
|
||||||
"""
|
|
||||||
output = (
|
|
||||||
self.cli.netmap.nodeinfo(
|
|
||||||
rpc_endpoint=rpc_endpoint,
|
|
||||||
wallet=wallet,
|
|
||||||
address=address,
|
|
||||||
generate_key=generate_key,
|
|
||||||
json=json,
|
|
||||||
ttl=ttl,
|
|
||||||
trace=trace,
|
|
||||||
xhdr=xhdr,
|
|
||||||
timeout=timeout,
|
|
||||||
)
|
|
||||||
.stdout.split("Trace ID")[0]
|
|
||||||
.strip()
|
|
||||||
)
|
|
||||||
|
|
||||||
return NetmapParser.node_info(module_json.loads(output))
|
|
||||||
|
|
||||||
def snapshot(
|
|
||||||
self,
|
|
||||||
rpc_endpoint: str,
|
|
||||||
wallet: Optional[str] = None,
|
|
||||||
address: Optional[str] = None,
|
|
||||||
generate_key: bool = False,
|
|
||||||
ttl: Optional[int] = None,
|
|
||||||
trace: Optional[bool] = True,
|
|
||||||
xhdr: Optional[dict] = None,
|
|
||||||
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
|
||||||
) -> List[NodeNetmapInfo]:
|
|
||||||
"""
|
|
||||||
Get target node info.
|
|
||||||
"""
|
|
||||||
output = (
|
|
||||||
self.cli.netmap.snapshot(
|
|
||||||
rpc_endpoint=rpc_endpoint,
|
|
||||||
wallet=wallet,
|
|
||||||
address=address,
|
|
||||||
generate_key=generate_key,
|
|
||||||
ttl=ttl,
|
|
||||||
trace=trace,
|
|
||||||
xhdr=xhdr,
|
|
||||||
timeout=timeout,
|
|
||||||
)
|
|
||||||
.stdout.split("Trace ID")[0]
|
|
||||||
.strip()
|
|
||||||
)
|
|
||||||
|
|
||||||
return NetmapParser.snapshot_all_nodes(output)
|
|
||||||
|
|
||||||
def snapshot_one_node(
|
|
||||||
self,
|
|
||||||
rpc_endpoint: str,
|
|
||||||
wallet: Optional[str] = None,
|
|
||||||
address: Optional[str] = None,
|
|
||||||
generate_key: bool = False,
|
|
||||||
ttl: Optional[int] = None,
|
|
||||||
trace: Optional[bool] = True,
|
|
||||||
xhdr: Optional[dict] = None,
|
|
||||||
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
|
||||||
) -> List[NodeNetmapInfo]:
|
|
||||||
"""
|
|
||||||
Get target one node info.
|
|
||||||
"""
|
|
||||||
output = (
|
|
||||||
self.cli.netmap.snapshot(
|
|
||||||
rpc_endpoint=rpc_endpoint,
|
|
||||||
wallet=wallet,
|
|
||||||
address=address,
|
|
||||||
generate_key=generate_key,
|
|
||||||
ttl=ttl,
|
|
||||||
trace=trace,
|
|
||||||
xhdr=xhdr,
|
|
||||||
timeout=timeout,
|
|
||||||
)
|
|
||||||
.stdout.split("Trace ID")[0]
|
|
||||||
.strip()
|
|
||||||
)
|
|
||||||
|
|
||||||
return NetmapParser.snapshot_one_node(output, rpc_endpoint)
|
|
424
src/frostfs_testlib/storage/grpc_operations/interfaces.py
Normal file
424
src/frostfs_testlib/storage/grpc_operations/interfaces.py
Normal file
|
@ -0,0 +1,424 @@
|
||||||
|
from abc import ABC, abstractmethod
|
||||||
|
from typing import Any, List, Optional
|
||||||
|
|
||||||
|
from frostfs_testlib.shell.interfaces import CommandResult
|
||||||
|
from frostfs_testlib.storage.cluster import Cluster, ClusterNode
|
||||||
|
from frostfs_testlib.storage.constants import PlacementRule
|
||||||
|
from frostfs_testlib.storage.dataclasses.storage_object_info import Chunk, NodeNetmapInfo
|
||||||
|
from frostfs_testlib.utils import file_utils
|
||||||
|
|
||||||
|
|
||||||
|
class ChunksInterface(ABC):
|
||||||
|
@abstractmethod
|
||||||
|
def search_node_without_chunks(self, chunks: list[Chunk], cluster: Cluster, endpoint: str = None) -> list[ClusterNode]:
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def get_chunk_node(self, cluster: Cluster, chunk: Chunk) -> tuple[ClusterNode, NodeNetmapInfo]:
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def get_shard_chunk(self, node: ClusterNode, chunk: Chunk) -> str:
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def get_all(
|
||||||
|
self,
|
||||||
|
rpc_endpoint: str,
|
||||||
|
cid: str,
|
||||||
|
oid: str,
|
||||||
|
wallet: Optional[str] = None,
|
||||||
|
address: Optional[str] = None,
|
||||||
|
bearer: Optional[str] = None,
|
||||||
|
generate_key: Optional[bool] = None,
|
||||||
|
trace: bool = False,
|
||||||
|
root: bool = False,
|
||||||
|
verify_presence_all: bool = False,
|
||||||
|
json: bool = True,
|
||||||
|
ttl: Optional[int] = None,
|
||||||
|
xhdr: Optional[dict] = None,
|
||||||
|
timeout: Optional[str] = None,
|
||||||
|
) -> list[Chunk]:
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def get_parity(
|
||||||
|
self,
|
||||||
|
rpc_endpoint: str,
|
||||||
|
cid: str,
|
||||||
|
wallet: Optional[str] = None,
|
||||||
|
address: Optional[str] = None,
|
||||||
|
bearer: Optional[str] = None,
|
||||||
|
generate_key: Optional[bool] = None,
|
||||||
|
oid: Optional[str] = None,
|
||||||
|
trace: bool = False,
|
||||||
|
root: bool = False,
|
||||||
|
verify_presence_all: bool = False,
|
||||||
|
json: bool = True,
|
||||||
|
ttl: Optional[int] = None,
|
||||||
|
xhdr: Optional[dict] = None,
|
||||||
|
timeout: Optional[str] = None,
|
||||||
|
) -> Chunk:
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def get_first_data(
|
||||||
|
self,
|
||||||
|
rpc_endpoint: str,
|
||||||
|
cid: str,
|
||||||
|
wallet: Optional[str] = None,
|
||||||
|
address: Optional[str] = None,
|
||||||
|
bearer: Optional[str] = None,
|
||||||
|
generate_key: Optional[bool] = None,
|
||||||
|
oid: Optional[str] = None,
|
||||||
|
trace: bool = False,
|
||||||
|
root: bool = False,
|
||||||
|
verify_presence_all: bool = False,
|
||||||
|
json: bool = True,
|
||||||
|
ttl: Optional[int] = None,
|
||||||
|
xhdr: Optional[dict] = None,
|
||||||
|
timeout: Optional[str] = None,
|
||||||
|
) -> Chunk:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class ObjectInterface(ABC):
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.chunks: ChunksInterface
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def delete(
|
||||||
|
self,
|
||||||
|
cid: str,
|
||||||
|
oid: str,
|
||||||
|
endpoint: str,
|
||||||
|
bearer: str = "",
|
||||||
|
xhdr: Optional[dict] = None,
|
||||||
|
session: Optional[str] = None,
|
||||||
|
timeout: Optional[str] = None,
|
||||||
|
) -> str:
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def get(
|
||||||
|
self,
|
||||||
|
cid: str,
|
||||||
|
oid: str,
|
||||||
|
endpoint: str,
|
||||||
|
bearer: Optional[str] = None,
|
||||||
|
write_object: Optional[str] = None,
|
||||||
|
xhdr: Optional[dict] = None,
|
||||||
|
no_progress: bool = True,
|
||||||
|
session: Optional[str] = None,
|
||||||
|
timeout: Optional[str] = None,
|
||||||
|
) -> file_utils.TestFile:
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def get_from_random_node(
|
||||||
|
self,
|
||||||
|
cid: str,
|
||||||
|
oid: str,
|
||||||
|
cluster: Cluster,
|
||||||
|
bearer: Optional[str] = None,
|
||||||
|
write_object: Optional[str] = None,
|
||||||
|
xhdr: Optional[dict] = None,
|
||||||
|
no_progress: bool = True,
|
||||||
|
session: Optional[str] = None,
|
||||||
|
timeout: Optional[str] = None,
|
||||||
|
) -> str:
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def hash(
|
||||||
|
self,
|
||||||
|
endpoint: str,
|
||||||
|
cid: str,
|
||||||
|
oid: str,
|
||||||
|
address: Optional[str] = None,
|
||||||
|
bearer: Optional[str] = None,
|
||||||
|
generate_key: Optional[bool] = None,
|
||||||
|
range: Optional[str] = None,
|
||||||
|
salt: Optional[str] = None,
|
||||||
|
ttl: Optional[int] = None,
|
||||||
|
session: Optional[str] = None,
|
||||||
|
hash_type: Optional[str] = None,
|
||||||
|
xhdr: Optional[dict] = None,
|
||||||
|
timeout: Optional[str] = None,
|
||||||
|
) -> str:
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def head(
|
||||||
|
self,
|
||||||
|
cid: str,
|
||||||
|
oid: str,
|
||||||
|
endpoint: str,
|
||||||
|
bearer: str = "",
|
||||||
|
xhdr: Optional[dict] = None,
|
||||||
|
json_output: bool = True,
|
||||||
|
is_raw: bool = False,
|
||||||
|
is_direct: bool = False,
|
||||||
|
session: Optional[str] = None,
|
||||||
|
timeout: Optional[str] = None,
|
||||||
|
) -> CommandResult | Any:
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def lock(
|
||||||
|
self,
|
||||||
|
cid: str,
|
||||||
|
oid: str,
|
||||||
|
endpoint: str,
|
||||||
|
lifetime: Optional[int] = None,
|
||||||
|
expire_at: Optional[int] = None,
|
||||||
|
address: Optional[str] = None,
|
||||||
|
bearer: Optional[str] = None,
|
||||||
|
session: Optional[str] = None,
|
||||||
|
ttl: Optional[int] = None,
|
||||||
|
xhdr: Optional[dict] = None,
|
||||||
|
timeout: Optional[str] = None,
|
||||||
|
) -> str:
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def put(
|
||||||
|
self,
|
||||||
|
path: str,
|
||||||
|
cid: str,
|
||||||
|
endpoint: str,
|
||||||
|
bearer: Optional[str] = None,
|
||||||
|
copies_number: Optional[int] = None,
|
||||||
|
attributes: Optional[dict] = None,
|
||||||
|
xhdr: Optional[dict] = None,
|
||||||
|
expire_at: Optional[int] = None,
|
||||||
|
no_progress: bool = True,
|
||||||
|
session: Optional[str] = None,
|
||||||
|
timeout: Optional[str] = None,
|
||||||
|
) -> str:
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def patch(
|
||||||
|
self,
|
||||||
|
cid: str,
|
||||||
|
oid: str,
|
||||||
|
endpoint: str,
|
||||||
|
ranges: Optional[list[str]] = None,
|
||||||
|
payloads: Optional[list[str]] = None,
|
||||||
|
new_attrs: Optional[str] = None,
|
||||||
|
replace_attrs: bool = False,
|
||||||
|
bearer: Optional[str] = None,
|
||||||
|
xhdr: Optional[dict] = None,
|
||||||
|
session: Optional[str] = None,
|
||||||
|
timeout: Optional[str] = None,
|
||||||
|
trace: bool = False,
|
||||||
|
) -> str:
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def put_to_random_node(
|
||||||
|
self,
|
||||||
|
path: str,
|
||||||
|
cid: str,
|
||||||
|
cluster: Cluster,
|
||||||
|
bearer: Optional[str] = None,
|
||||||
|
copies_number: Optional[int] = None,
|
||||||
|
attributes: Optional[dict] = None,
|
||||||
|
xhdr: Optional[dict] = None,
|
||||||
|
expire_at: Optional[int] = None,
|
||||||
|
no_progress: bool = True,
|
||||||
|
session: Optional[str] = None,
|
||||||
|
timeout: Optional[str] = None,
|
||||||
|
) -> str:
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def range(
|
||||||
|
self,
|
||||||
|
cid: str,
|
||||||
|
oid: str,
|
||||||
|
range_cut: str,
|
||||||
|
endpoint: str,
|
||||||
|
bearer: str = "",
|
||||||
|
xhdr: Optional[dict] = None,
|
||||||
|
session: Optional[str] = None,
|
||||||
|
timeout: Optional[str] = None,
|
||||||
|
) -> tuple[file_utils.TestFile, bytes]:
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def search(
|
||||||
|
self,
|
||||||
|
cid: str,
|
||||||
|
endpoint: str,
|
||||||
|
bearer: str = "",
|
||||||
|
oid: Optional[str] = None,
|
||||||
|
filters: Optional[dict] = None,
|
||||||
|
expected_objects_list: Optional[list] = None,
|
||||||
|
xhdr: Optional[dict] = None,
|
||||||
|
session: Optional[str] = None,
|
||||||
|
phy: bool = False,
|
||||||
|
root: bool = False,
|
||||||
|
timeout: Optional[str] = None,
|
||||||
|
address: Optional[str] = None,
|
||||||
|
generate_key: Optional[bool] = None,
|
||||||
|
ttl: Optional[int] = None,
|
||||||
|
) -> List:
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def nodes(
|
||||||
|
self,
|
||||||
|
cluster: Cluster,
|
||||||
|
cid: str,
|
||||||
|
oid: str,
|
||||||
|
alive_node: ClusterNode,
|
||||||
|
bearer: str = "",
|
||||||
|
xhdr: Optional[dict] = None,
|
||||||
|
is_direct: bool = False,
|
||||||
|
verify_presence_all: bool = False,
|
||||||
|
timeout: Optional[str] = None,
|
||||||
|
) -> List[ClusterNode]:
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def parts(
|
||||||
|
self,
|
||||||
|
cid: str,
|
||||||
|
oid: str,
|
||||||
|
alive_node: ClusterNode,
|
||||||
|
bearer: str = "",
|
||||||
|
xhdr: Optional[dict] = None,
|
||||||
|
is_direct: bool = False,
|
||||||
|
verify_presence_all: bool = False,
|
||||||
|
timeout: Optional[str] = None,
|
||||||
|
) -> List[str]:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class ContainerInterface(ABC):
|
||||||
|
@abstractmethod
|
||||||
|
def create(
|
||||||
|
self,
|
||||||
|
endpoint: str,
|
||||||
|
nns_zone: Optional[str] = None,
|
||||||
|
nns_name: Optional[str] = None,
|
||||||
|
address: Optional[str] = None,
|
||||||
|
attributes: Optional[dict] = None,
|
||||||
|
basic_acl: Optional[str] = None,
|
||||||
|
await_mode: bool = False,
|
||||||
|
disable_timestamp: bool = False,
|
||||||
|
force: bool = False,
|
||||||
|
trace: bool = False,
|
||||||
|
name: Optional[str] = None,
|
||||||
|
nonce: Optional[str] = None,
|
||||||
|
policy: Optional[str] = None,
|
||||||
|
session: Optional[str] = None,
|
||||||
|
subnet: Optional[str] = None,
|
||||||
|
ttl: Optional[int] = None,
|
||||||
|
xhdr: Optional[dict] = None,
|
||||||
|
timeout: Optional[str] = None,
|
||||||
|
) -> str:
|
||||||
|
"""
|
||||||
|
Create a new container and register it in the FrostFS.
|
||||||
|
It will be stored in the sidechain when the Inner Ring accepts it.
|
||||||
|
"""
|
||||||
|
raise NotImplementedError("No implemethed method create")
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def delete(
|
||||||
|
self,
|
||||||
|
endpoint: str,
|
||||||
|
cid: str,
|
||||||
|
address: Optional[str] = None,
|
||||||
|
await_mode: bool = False,
|
||||||
|
session: Optional[str] = None,
|
||||||
|
ttl: Optional[int] = None,
|
||||||
|
xhdr: Optional[dict] = None,
|
||||||
|
force: bool = False,
|
||||||
|
trace: bool = False,
|
||||||
|
) -> List[str]:
|
||||||
|
"""
|
||||||
|
Delete an existing container.
|
||||||
|
Only the owner of the container has permission to remove the container.
|
||||||
|
"""
|
||||||
|
raise NotImplementedError("No implemethed method delete")
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def get(
|
||||||
|
self,
|
||||||
|
endpoint: str,
|
||||||
|
cid: str,
|
||||||
|
address: Optional[str] = None,
|
||||||
|
generate_key: Optional[bool] = None,
|
||||||
|
await_mode: bool = False,
|
||||||
|
to: Optional[str] = None,
|
||||||
|
json_mode: bool = True,
|
||||||
|
trace: bool = False,
|
||||||
|
ttl: Optional[int] = None,
|
||||||
|
xhdr: Optional[dict] = None,
|
||||||
|
timeout: Optional[str] = None,
|
||||||
|
) -> List[str]:
|
||||||
|
"""Get container field info."""
|
||||||
|
raise NotImplementedError("No implemethed method get")
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def get_eacl(
|
||||||
|
self,
|
||||||
|
endpoint: str,
|
||||||
|
cid: str,
|
||||||
|
address: Optional[str] = None,
|
||||||
|
generate_key: Optional[bool] = None,
|
||||||
|
await_mode: bool = False,
|
||||||
|
json_mode: bool = True,
|
||||||
|
trace: bool = False,
|
||||||
|
to: Optional[str] = None,
|
||||||
|
session: Optional[str] = None,
|
||||||
|
ttl: Optional[int] = None,
|
||||||
|
xhdr: Optional[dict] = None,
|
||||||
|
timeout: Optional[str] = None,
|
||||||
|
) -> List[str]:
|
||||||
|
"""Get extended ACL table of container."""
|
||||||
|
raise NotImplementedError("No implemethed method get-eacl")
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def list(
|
||||||
|
self,
|
||||||
|
endpoint: str,
|
||||||
|
name: Optional[str] = None,
|
||||||
|
address: Optional[str] = None,
|
||||||
|
generate_key: Optional[bool] = None,
|
||||||
|
trace: bool = False,
|
||||||
|
owner: Optional[str] = None,
|
||||||
|
ttl: Optional[int] = None,
|
||||||
|
xhdr: Optional[dict] = None,
|
||||||
|
timeout: Optional[str] = None,
|
||||||
|
**params,
|
||||||
|
) -> List[str]:
|
||||||
|
"""List all created containers."""
|
||||||
|
raise NotImplementedError("No implemethed method list")
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def nodes(
|
||||||
|
self,
|
||||||
|
endpoint: str,
|
||||||
|
cid: str,
|
||||||
|
cluster: Cluster,
|
||||||
|
address: Optional[str] = None,
|
||||||
|
ttl: Optional[int] = None,
|
||||||
|
from_file: Optional[str] = None,
|
||||||
|
trace: bool = False,
|
||||||
|
short: Optional[bool] = True,
|
||||||
|
xhdr: Optional[dict] = None,
|
||||||
|
generate_key: Optional[bool] = None,
|
||||||
|
timeout: Optional[str] = None,
|
||||||
|
) -> List[ClusterNode]:
|
||||||
|
"""Show the nodes participating in the container in the current epoch."""
|
||||||
|
raise NotImplementedError("No implemethed method nodes")
|
||||||
|
|
||||||
|
|
||||||
|
class GrpcClientWrapper(ABC):
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.object: ObjectInterface
|
||||||
|
self.container: ContainerInterface
|
|
@ -1,4 +0,0 @@
|
||||||
from .chunks import ChunksInterface
|
|
||||||
from .container import ContainerInterface
|
|
||||||
from .netmap import NetmapInterface
|
|
||||||
from .object import ObjectInterface
|
|
|
@ -1,79 +0,0 @@
|
||||||
from abc import ABC, abstractmethod
|
|
||||||
from typing import Optional
|
|
||||||
|
|
||||||
from frostfs_testlib.storage.cluster import Cluster, ClusterNode
|
|
||||||
from frostfs_testlib.storage.dataclasses.storage_object_info import Chunk, NodeNetmapInfo
|
|
||||||
|
|
||||||
|
|
||||||
class ChunksInterface(ABC):
|
|
||||||
@abstractmethod
|
|
||||||
def search_node_without_chunks(self, chunks: list[Chunk], cluster: Cluster, endpoint: str = None) -> list[ClusterNode]:
|
|
||||||
pass
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def get_chunk_node(self, cluster: Cluster, chunk: Chunk) -> tuple[ClusterNode, NodeNetmapInfo]:
|
|
||||||
pass
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def get_shard_chunk(self, node: ClusterNode, chunk: Chunk) -> str:
|
|
||||||
pass
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def get_all(
|
|
||||||
self,
|
|
||||||
rpc_endpoint: str,
|
|
||||||
cid: str,
|
|
||||||
oid: str,
|
|
||||||
wallet: Optional[str] = None,
|
|
||||||
address: Optional[str] = None,
|
|
||||||
bearer: Optional[str] = None,
|
|
||||||
generate_key: Optional[bool] = None,
|
|
||||||
trace: bool = False,
|
|
||||||
root: bool = False,
|
|
||||||
verify_presence_all: bool = False,
|
|
||||||
json: bool = True,
|
|
||||||
ttl: Optional[int] = None,
|
|
||||||
xhdr: Optional[dict] = None,
|
|
||||||
timeout: Optional[str] = None,
|
|
||||||
) -> list[Chunk]:
|
|
||||||
pass
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def get_parity(
|
|
||||||
self,
|
|
||||||
rpc_endpoint: str,
|
|
||||||
cid: str,
|
|
||||||
wallet: Optional[str] = None,
|
|
||||||
address: Optional[str] = None,
|
|
||||||
bearer: Optional[str] = None,
|
|
||||||
generate_key: Optional[bool] = None,
|
|
||||||
oid: Optional[str] = None,
|
|
||||||
trace: bool = False,
|
|
||||||
root: bool = False,
|
|
||||||
verify_presence_all: bool = False,
|
|
||||||
json: bool = True,
|
|
||||||
ttl: Optional[int] = None,
|
|
||||||
xhdr: Optional[dict] = None,
|
|
||||||
timeout: Optional[str] = None,
|
|
||||||
) -> Chunk:
|
|
||||||
pass
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def get_first_data(
|
|
||||||
self,
|
|
||||||
rpc_endpoint: str,
|
|
||||||
cid: str,
|
|
||||||
wallet: Optional[str] = None,
|
|
||||||
address: Optional[str] = None,
|
|
||||||
bearer: Optional[str] = None,
|
|
||||||
generate_key: Optional[bool] = None,
|
|
||||||
oid: Optional[str] = None,
|
|
||||||
trace: bool = False,
|
|
||||||
root: bool = False,
|
|
||||||
verify_presence_all: bool = False,
|
|
||||||
json: bool = True,
|
|
||||||
ttl: Optional[int] = None,
|
|
||||||
xhdr: Optional[dict] = None,
|
|
||||||
timeout: Optional[str] = None,
|
|
||||||
) -> Chunk:
|
|
||||||
pass
|
|
|
@ -1,125 +0,0 @@
|
||||||
from abc import ABC, abstractmethod
|
|
||||||
from typing import List, Optional
|
|
||||||
|
|
||||||
from frostfs_testlib.storage.cluster import Cluster, ClusterNode
|
|
||||||
|
|
||||||
|
|
||||||
class ContainerInterface(ABC):
|
|
||||||
@abstractmethod
|
|
||||||
def create(
|
|
||||||
self,
|
|
||||||
endpoint: str,
|
|
||||||
nns_zone: Optional[str] = None,
|
|
||||||
nns_name: Optional[str] = None,
|
|
||||||
address: Optional[str] = None,
|
|
||||||
attributes: Optional[dict] = None,
|
|
||||||
basic_acl: Optional[str] = None,
|
|
||||||
await_mode: bool = False,
|
|
||||||
disable_timestamp: bool = False,
|
|
||||||
force: bool = False,
|
|
||||||
trace: bool = False,
|
|
||||||
name: Optional[str] = None,
|
|
||||||
nonce: Optional[str] = None,
|
|
||||||
policy: Optional[str] = None,
|
|
||||||
session: Optional[str] = None,
|
|
||||||
subnet: Optional[str] = None,
|
|
||||||
ttl: Optional[int] = None,
|
|
||||||
xhdr: Optional[dict] = None,
|
|
||||||
timeout: Optional[str] = None,
|
|
||||||
) -> str:
|
|
||||||
"""
|
|
||||||
Create a new container and register it in the FrostFS.
|
|
||||||
It will be stored in the sidechain when the Inner Ring accepts it.
|
|
||||||
"""
|
|
||||||
raise NotImplementedError("No implemethed method create")
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def delete(
|
|
||||||
self,
|
|
||||||
endpoint: str,
|
|
||||||
cid: str,
|
|
||||||
address: Optional[str] = None,
|
|
||||||
await_mode: bool = False,
|
|
||||||
session: Optional[str] = None,
|
|
||||||
ttl: Optional[int] = None,
|
|
||||||
xhdr: Optional[dict] = None,
|
|
||||||
force: bool = False,
|
|
||||||
trace: bool = False,
|
|
||||||
) -> List[str]:
|
|
||||||
"""
|
|
||||||
Delete an existing container.
|
|
||||||
Only the owner of the container has permission to remove the container.
|
|
||||||
"""
|
|
||||||
raise NotImplementedError("No implemethed method delete")
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def get(
|
|
||||||
self,
|
|
||||||
endpoint: str,
|
|
||||||
cid: str,
|
|
||||||
address: Optional[str] = None,
|
|
||||||
generate_key: Optional[bool] = None,
|
|
||||||
await_mode: bool = False,
|
|
||||||
to: Optional[str] = None,
|
|
||||||
json_mode: bool = True,
|
|
||||||
trace: bool = False,
|
|
||||||
ttl: Optional[int] = None,
|
|
||||||
xhdr: Optional[dict] = None,
|
|
||||||
timeout: Optional[str] = None,
|
|
||||||
) -> List[str]:
|
|
||||||
"""Get container field info."""
|
|
||||||
raise NotImplementedError("No implemethed method get")
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def get_eacl(
|
|
||||||
self,
|
|
||||||
endpoint: str,
|
|
||||||
cid: str,
|
|
||||||
address: Optional[str] = None,
|
|
||||||
generate_key: Optional[bool] = None,
|
|
||||||
await_mode: bool = False,
|
|
||||||
json_mode: bool = True,
|
|
||||||
trace: bool = False,
|
|
||||||
to: Optional[str] = None,
|
|
||||||
session: Optional[str] = None,
|
|
||||||
ttl: Optional[int] = None,
|
|
||||||
xhdr: Optional[dict] = None,
|
|
||||||
timeout: Optional[str] = None,
|
|
||||||
) -> List[str]:
|
|
||||||
"""Get extended ACL table of container."""
|
|
||||||
raise NotImplementedError("No implemethed method get-eacl")
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def list(
|
|
||||||
self,
|
|
||||||
endpoint: str,
|
|
||||||
name: Optional[str] = None,
|
|
||||||
address: Optional[str] = None,
|
|
||||||
generate_key: Optional[bool] = None,
|
|
||||||
trace: bool = False,
|
|
||||||
owner: Optional[str] = None,
|
|
||||||
ttl: Optional[int] = None,
|
|
||||||
xhdr: Optional[dict] = None,
|
|
||||||
timeout: Optional[str] = None,
|
|
||||||
**params,
|
|
||||||
) -> List[str]:
|
|
||||||
"""List all created containers."""
|
|
||||||
raise NotImplementedError("No implemethed method list")
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def nodes(
|
|
||||||
self,
|
|
||||||
endpoint: str,
|
|
||||||
cid: str,
|
|
||||||
cluster: Cluster,
|
|
||||||
address: Optional[str] = None,
|
|
||||||
ttl: Optional[int] = None,
|
|
||||||
from_file: Optional[str] = None,
|
|
||||||
trace: bool = False,
|
|
||||||
short: Optional[bool] = True,
|
|
||||||
xhdr: Optional[dict] = None,
|
|
||||||
generate_key: Optional[bool] = None,
|
|
||||||
timeout: Optional[str] = None,
|
|
||||||
) -> List[ClusterNode]:
|
|
||||||
"""Show the nodes participating in the container in the current epoch."""
|
|
||||||
raise NotImplementedError("No implemethed method nodes")
|
|
|
@ -1,89 +0,0 @@
|
||||||
from abc import ABC, abstractmethod
|
|
||||||
from typing import List, Optional
|
|
||||||
|
|
||||||
from frostfs_testlib.storage.dataclasses.storage_object_info import NodeNetInfo, NodeNetmapInfo
|
|
||||||
|
|
||||||
|
|
||||||
class NetmapInterface(ABC):
|
|
||||||
@abstractmethod
|
|
||||||
def epoch(
|
|
||||||
self,
|
|
||||||
rpc_endpoint: str,
|
|
||||||
wallet: Optional[str] = None,
|
|
||||||
address: Optional[str] = None,
|
|
||||||
generate_key: bool = False,
|
|
||||||
ttl: Optional[int] = None,
|
|
||||||
trace: Optional[bool] = False,
|
|
||||||
xhdr: Optional[dict] = None,
|
|
||||||
timeout: Optional[str] = None,
|
|
||||||
) -> int:
|
|
||||||
"""
|
|
||||||
Get current epoch number.
|
|
||||||
"""
|
|
||||||
raise NotImplementedError("No implemethed method epoch")
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def netinfo(
|
|
||||||
self,
|
|
||||||
rpc_endpoint: str,
|
|
||||||
wallet: Optional[str] = None,
|
|
||||||
address: Optional[str] = None,
|
|
||||||
generate_key: bool = False,
|
|
||||||
json: bool = True,
|
|
||||||
ttl: Optional[int] = None,
|
|
||||||
xhdr: Optional[dict] = None,
|
|
||||||
timeout: Optional[str] = None,
|
|
||||||
) -> NodeNetInfo:
|
|
||||||
"""
|
|
||||||
Get target node info.
|
|
||||||
"""
|
|
||||||
raise NotImplementedError("No implemethed method netinfo")
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def nodeinfo(
|
|
||||||
self,
|
|
||||||
rpc_endpoint: str,
|
|
||||||
wallet: Optional[str] = None,
|
|
||||||
address: Optional[str] = None,
|
|
||||||
generate_key: bool = False,
|
|
||||||
json: bool = True,
|
|
||||||
ttl: Optional[int] = None,
|
|
||||||
xhdr: Optional[dict] = None,
|
|
||||||
timeout: Optional[str] = None,
|
|
||||||
) -> NodeNetmapInfo:
|
|
||||||
"""
|
|
||||||
Get target node info.
|
|
||||||
"""
|
|
||||||
raise NotImplementedError("No implemethed method nodeinfo")
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def snapshot(
|
|
||||||
self,
|
|
||||||
rpc_endpoint: str,
|
|
||||||
wallet: Optional[str] = None,
|
|
||||||
address: Optional[str] = None,
|
|
||||||
generate_key: bool = False,
|
|
||||||
ttl: Optional[int] = None,
|
|
||||||
xhdr: Optional[dict] = None,
|
|
||||||
timeout: Optional[str] = None,
|
|
||||||
) -> List[NodeNetmapInfo]:
|
|
||||||
"""
|
|
||||||
Get target node info.
|
|
||||||
"""
|
|
||||||
raise NotImplementedError("No implemethed method snapshot")
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def snapshot_one_node(
|
|
||||||
self,
|
|
||||||
rpc_endpoint: str,
|
|
||||||
wallet: Optional[str] = None,
|
|
||||||
address: Optional[str] = None,
|
|
||||||
generate_key: bool = False,
|
|
||||||
ttl: Optional[int] = None,
|
|
||||||
xhdr: Optional[dict] = None,
|
|
||||||
timeout: Optional[str] = None,
|
|
||||||
) -> List[NodeNetmapInfo]:
|
|
||||||
"""
|
|
||||||
Get target one node info.
|
|
||||||
"""
|
|
||||||
raise NotImplementedError("No implemethed method snapshot")
|
|
|
@ -1,223 +0,0 @@
|
||||||
from abc import ABC, abstractmethod
|
|
||||||
from typing import Any, List, Optional
|
|
||||||
|
|
||||||
from frostfs_testlib.shell.interfaces import CommandResult
|
|
||||||
from frostfs_testlib.storage.cluster import Cluster, ClusterNode
|
|
||||||
from frostfs_testlib.utils import file_utils
|
|
||||||
|
|
||||||
from .chunks import ChunksInterface
|
|
||||||
|
|
||||||
|
|
||||||
class ObjectInterface(ABC):
|
|
||||||
def __init__(self) -> None:
|
|
||||||
self.chunks: ChunksInterface
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def delete(
|
|
||||||
self,
|
|
||||||
cid: str,
|
|
||||||
oid: str,
|
|
||||||
endpoint: str,
|
|
||||||
bearer: str = "",
|
|
||||||
xhdr: Optional[dict] = None,
|
|
||||||
session: Optional[str] = None,
|
|
||||||
timeout: Optional[str] = None,
|
|
||||||
) -> str:
|
|
||||||
pass
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def get(
|
|
||||||
self,
|
|
||||||
cid: str,
|
|
||||||
oid: str,
|
|
||||||
endpoint: str,
|
|
||||||
bearer: Optional[str] = None,
|
|
||||||
write_object: Optional[str] = None,
|
|
||||||
xhdr: Optional[dict] = None,
|
|
||||||
no_progress: bool = True,
|
|
||||||
session: Optional[str] = None,
|
|
||||||
timeout: Optional[str] = None,
|
|
||||||
) -> file_utils.TestFile:
|
|
||||||
pass
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def get_from_random_node(
|
|
||||||
self,
|
|
||||||
cid: str,
|
|
||||||
oid: str,
|
|
||||||
cluster: Cluster,
|
|
||||||
bearer: Optional[str] = None,
|
|
||||||
write_object: Optional[str] = None,
|
|
||||||
xhdr: Optional[dict] = None,
|
|
||||||
no_progress: bool = True,
|
|
||||||
session: Optional[str] = None,
|
|
||||||
timeout: Optional[str] = None,
|
|
||||||
) -> str:
|
|
||||||
pass
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def hash(
|
|
||||||
self,
|
|
||||||
endpoint: str,
|
|
||||||
cid: str,
|
|
||||||
oid: str,
|
|
||||||
address: Optional[str] = None,
|
|
||||||
bearer: Optional[str] = None,
|
|
||||||
generate_key: Optional[bool] = None,
|
|
||||||
range: Optional[str] = None,
|
|
||||||
salt: Optional[str] = None,
|
|
||||||
ttl: Optional[int] = None,
|
|
||||||
session: Optional[str] = None,
|
|
||||||
hash_type: Optional[str] = None,
|
|
||||||
xhdr: Optional[dict] = None,
|
|
||||||
timeout: Optional[str] = None,
|
|
||||||
) -> str:
|
|
||||||
pass
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def head(
|
|
||||||
self,
|
|
||||||
cid: str,
|
|
||||||
oid: str,
|
|
||||||
endpoint: str,
|
|
||||||
bearer: str = "",
|
|
||||||
xhdr: Optional[dict] = None,
|
|
||||||
json_output: bool = True,
|
|
||||||
is_raw: bool = False,
|
|
||||||
is_direct: bool = False,
|
|
||||||
session: Optional[str] = None,
|
|
||||||
timeout: Optional[str] = None,
|
|
||||||
) -> CommandResult | Any:
|
|
||||||
pass
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def lock(
|
|
||||||
self,
|
|
||||||
cid: str,
|
|
||||||
oid: str,
|
|
||||||
endpoint: str,
|
|
||||||
lifetime: Optional[int] = None,
|
|
||||||
expire_at: Optional[int] = None,
|
|
||||||
address: Optional[str] = None,
|
|
||||||
bearer: Optional[str] = None,
|
|
||||||
session: Optional[str] = None,
|
|
||||||
ttl: Optional[int] = None,
|
|
||||||
xhdr: Optional[dict] = None,
|
|
||||||
timeout: Optional[str] = None,
|
|
||||||
) -> str:
|
|
||||||
pass
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def put(
|
|
||||||
self,
|
|
||||||
path: str,
|
|
||||||
cid: str,
|
|
||||||
endpoint: str,
|
|
||||||
bearer: Optional[str] = None,
|
|
||||||
copies_number: Optional[int] = None,
|
|
||||||
attributes: Optional[dict] = None,
|
|
||||||
xhdr: Optional[dict] = None,
|
|
||||||
expire_at: Optional[int] = None,
|
|
||||||
no_progress: bool = True,
|
|
||||||
session: Optional[str] = None,
|
|
||||||
timeout: Optional[str] = None,
|
|
||||||
) -> str:
|
|
||||||
pass
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def patch(
|
|
||||||
self,
|
|
||||||
cid: str,
|
|
||||||
oid: str,
|
|
||||||
endpoint: str,
|
|
||||||
ranges: Optional[list[str]] = None,
|
|
||||||
payloads: Optional[list[str]] = None,
|
|
||||||
new_attrs: Optional[str] = None,
|
|
||||||
replace_attrs: bool = False,
|
|
||||||
bearer: Optional[str] = None,
|
|
||||||
xhdr: Optional[dict] = None,
|
|
||||||
session: Optional[str] = None,
|
|
||||||
timeout: Optional[str] = None,
|
|
||||||
trace: bool = False,
|
|
||||||
) -> str:
|
|
||||||
pass
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def put_to_random_node(
|
|
||||||
self,
|
|
||||||
path: str,
|
|
||||||
cid: str,
|
|
||||||
cluster: Cluster,
|
|
||||||
bearer: Optional[str] = None,
|
|
||||||
copies_number: Optional[int] = None,
|
|
||||||
attributes: Optional[dict] = None,
|
|
||||||
xhdr: Optional[dict] = None,
|
|
||||||
expire_at: Optional[int] = None,
|
|
||||||
no_progress: bool = True,
|
|
||||||
session: Optional[str] = None,
|
|
||||||
timeout: Optional[str] = None,
|
|
||||||
) -> str:
|
|
||||||
pass
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def range(
|
|
||||||
self,
|
|
||||||
cid: str,
|
|
||||||
oid: str,
|
|
||||||
range_cut: str,
|
|
||||||
endpoint: str,
|
|
||||||
bearer: str = "",
|
|
||||||
xhdr: Optional[dict] = None,
|
|
||||||
session: Optional[str] = None,
|
|
||||||
timeout: Optional[str] = None,
|
|
||||||
) -> tuple[file_utils.TestFile, bytes]:
|
|
||||||
pass
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def search(
|
|
||||||
self,
|
|
||||||
cid: str,
|
|
||||||
endpoint: str,
|
|
||||||
bearer: str = "",
|
|
||||||
oid: Optional[str] = None,
|
|
||||||
filters: Optional[dict] = None,
|
|
||||||
expected_objects_list: Optional[list] = None,
|
|
||||||
xhdr: Optional[dict] = None,
|
|
||||||
session: Optional[str] = None,
|
|
||||||
phy: bool = False,
|
|
||||||
root: bool = False,
|
|
||||||
timeout: Optional[str] = None,
|
|
||||||
address: Optional[str] = None,
|
|
||||||
generate_key: Optional[bool] = None,
|
|
||||||
ttl: Optional[int] = None,
|
|
||||||
) -> List:
|
|
||||||
pass
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def nodes(
|
|
||||||
self,
|
|
||||||
cluster: Cluster,
|
|
||||||
cid: str,
|
|
||||||
oid: str,
|
|
||||||
alive_node: ClusterNode,
|
|
||||||
bearer: str = "",
|
|
||||||
xhdr: Optional[dict] = None,
|
|
||||||
is_direct: bool = False,
|
|
||||||
verify_presence_all: bool = False,
|
|
||||||
timeout: Optional[str] = None,
|
|
||||||
) -> List[ClusterNode]:
|
|
||||||
pass
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def parts(
|
|
||||||
self,
|
|
||||||
cid: str,
|
|
||||||
oid: str,
|
|
||||||
alive_node: ClusterNode,
|
|
||||||
bearer: str = "",
|
|
||||||
xhdr: Optional[dict] = None,
|
|
||||||
is_direct: bool = False,
|
|
||||||
verify_presence_all: bool = False,
|
|
||||||
timeout: Optional[str] = None,
|
|
||||||
) -> List[str]:
|
|
||||||
pass
|
|
|
@ -1,10 +0,0 @@
|
||||||
from abc import ABC
|
|
||||||
|
|
||||||
from . import interfaces
|
|
||||||
|
|
||||||
|
|
||||||
class GrpcClientWrapper(ABC):
|
|
||||||
def __init__(self) -> None:
|
|
||||||
self.object: interfaces.ObjectInterface
|
|
||||||
self.container: interfaces.ContainerInterface
|
|
||||||
self.netmap: interfaces.NetmapInterface
|
|
|
@ -64,7 +64,7 @@ def parallel_binary_verions(host: Host) -> dict[str, str]:
|
||||||
try:
|
try:
|
||||||
result = shell.exec(f"{binary_path} {binary['param']}")
|
result = shell.exec(f"{binary_path} {binary['param']}")
|
||||||
version = parse_version(result.stdout) or parse_version(result.stderr) or "Unknown"
|
version = parse_version(result.stdout) or parse_version(result.stderr) or "Unknown"
|
||||||
versions_at_host[binary_name] = version.strip()
|
versions_at_host[binary_name] = version
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
logger.error(f"Cannot get version for {binary_path} because of\n{exc}")
|
logger.error(f"Cannot get version for {binary_path} because of\n{exc}")
|
||||||
versions_at_host[binary_name] = "Unknown"
|
versions_at_host[binary_name] = "Unknown"
|
||||||
|
|
|
@ -6,7 +6,10 @@ import pytest
|
||||||
from frostfs_testlib.load.load_config import EndpointSelectionStrategy, LoadParams, LoadScenario, LoadType, Preset, ReadFrom
|
from frostfs_testlib.load.load_config import EndpointSelectionStrategy, LoadParams, LoadScenario, LoadType, Preset, ReadFrom
|
||||||
from frostfs_testlib.load.runners import DefaultRunner
|
from frostfs_testlib.load.runners import DefaultRunner
|
||||||
from frostfs_testlib.resources.load_params import BACKGROUND_LOAD_DEFAULT_VU_INIT_TIME
|
from frostfs_testlib.resources.load_params import BACKGROUND_LOAD_DEFAULT_VU_INIT_TIME
|
||||||
|
from frostfs_testlib.storage.cluster import ClusterNode
|
||||||
from frostfs_testlib.storage.controllers.background_load_controller import BackgroundLoadController
|
from frostfs_testlib.storage.controllers.background_load_controller import BackgroundLoadController
|
||||||
|
from frostfs_testlib.storage.dataclasses.frostfs_services import StorageNode
|
||||||
|
from frostfs_testlib.storage.dataclasses.node_base import NodeBase
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
|
@ -126,8 +129,6 @@ class TestLoadConfig:
|
||||||
"--size '11'",
|
"--size '11'",
|
||||||
"--acl 'acl'",
|
"--acl 'acl'",
|
||||||
"--preload_obj '13'",
|
"--preload_obj '13'",
|
||||||
"--retry '24'",
|
|
||||||
"--rule 'rule' --rule 'rule_2'",
|
|
||||||
"--out 'pregen_json'",
|
"--out 'pregen_json'",
|
||||||
"--workers '7'",
|
"--workers '7'",
|
||||||
"--containers '16'",
|
"--containers '16'",
|
||||||
|
@ -160,8 +161,6 @@ class TestLoadConfig:
|
||||||
expected_preset_args = [
|
expected_preset_args = [
|
||||||
"--size '11'",
|
"--size '11'",
|
||||||
"--preload_obj '13'",
|
"--preload_obj '13'",
|
||||||
"--retry '24'",
|
|
||||||
"--rule 'rule' --rule 'rule_2'",
|
|
||||||
"--out 'pregen_json'",
|
"--out 'pregen_json'",
|
||||||
"--workers '7'",
|
"--workers '7'",
|
||||||
"--containers '16'",
|
"--containers '16'",
|
||||||
|
@ -318,8 +317,6 @@ class TestLoadConfig:
|
||||||
"--no-verify-ssl",
|
"--no-verify-ssl",
|
||||||
"--size '11'",
|
"--size '11'",
|
||||||
"--preload_obj '13'",
|
"--preload_obj '13'",
|
||||||
"--retry '24'",
|
|
||||||
"--rule 'rule' --rule 'rule_2'",
|
|
||||||
"--out 'pregen_json'",
|
"--out 'pregen_json'",
|
||||||
"--workers '7'",
|
"--workers '7'",
|
||||||
"--containers '16'",
|
"--containers '16'",
|
||||||
|
@ -353,8 +350,6 @@ class TestLoadConfig:
|
||||||
expected_preset_args = [
|
expected_preset_args = [
|
||||||
"--size '11'",
|
"--size '11'",
|
||||||
"--preload_obj '13'",
|
"--preload_obj '13'",
|
||||||
"--retry '24'",
|
|
||||||
"--rule 'rule' --rule 'rule_2'",
|
|
||||||
"--out 'pregen_json'",
|
"--out 'pregen_json'",
|
||||||
"--workers '7'",
|
"--workers '7'",
|
||||||
"--containers '16'",
|
"--containers '16'",
|
||||||
|
@ -420,26 +415,6 @@ class TestLoadConfig:
|
||||||
|
|
||||||
self._check_preset_params(load_params, params)
|
self._check_preset_params(load_params, params)
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
|
||||||
"load_type, input, value, params",
|
|
||||||
[
|
|
||||||
(LoadType.gRPC, ["A C ", " B"], ["A C", "B"], [f"--rule 'A C' --rule 'B'"]),
|
|
||||||
(LoadType.gRPC, " A ", ["A"], ["--rule 'A'"]),
|
|
||||||
(LoadType.gRPC, " A , B ", ["A , B"], ["--rule 'A , B'"]),
|
|
||||||
(LoadType.gRPC, [" A", "B "], ["A", "B"], ["--rule 'A' --rule 'B'"]),
|
|
||||||
(LoadType.gRPC, None, None, []),
|
|
||||||
(LoadType.S3, ["A C ", " B"], ["A C", "B"], []),
|
|
||||||
(LoadType.S3, None, None, []),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
def test_ape_list_parsing_formatter(self, load_type, input, value, params):
|
|
||||||
load_params = LoadParams(load_type)
|
|
||||||
load_params.preset = Preset()
|
|
||||||
load_params.preset.rule = input
|
|
||||||
assert load_params.preset.rule == value
|
|
||||||
|
|
||||||
self._check_preset_params(load_params, params)
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("load_params, load_type", [(LoadScenario.VERIFY, LoadType.S3)], indirect=True)
|
@pytest.mark.parametrize("load_params, load_type", [(LoadScenario.VERIFY, LoadType.S3)], indirect=True)
|
||||||
def test_argument_parsing_for_s3_verify_scenario(self, load_params: LoadParams):
|
def test_argument_parsing_for_s3_verify_scenario(self, load_params: LoadParams):
|
||||||
expected_env_vars = {
|
expected_env_vars = {
|
||||||
|
@ -469,8 +444,6 @@ class TestLoadConfig:
|
||||||
expected_preset_args = [
|
expected_preset_args = [
|
||||||
"--size '0'",
|
"--size '0'",
|
||||||
"--preload_obj '0'",
|
"--preload_obj '0'",
|
||||||
"--retry '0'",
|
|
||||||
"--rule ''",
|
|
||||||
"--out ''",
|
"--out ''",
|
||||||
"--workers '0'",
|
"--workers '0'",
|
||||||
"--containers '0'",
|
"--containers '0'",
|
||||||
|
@ -502,8 +475,6 @@ class TestLoadConfig:
|
||||||
expected_preset_args = [
|
expected_preset_args = [
|
||||||
"--size '0'",
|
"--size '0'",
|
||||||
"--preload_obj '0'",
|
"--preload_obj '0'",
|
||||||
"--retry '0'",
|
|
||||||
"--rule ''",
|
|
||||||
"--out ''",
|
"--out ''",
|
||||||
"--workers '0'",
|
"--workers '0'",
|
||||||
"--containers '0'",
|
"--containers '0'",
|
||||||
|
@ -611,8 +582,6 @@ class TestLoadConfig:
|
||||||
expected_preset_args = [
|
expected_preset_args = [
|
||||||
"--size '0'",
|
"--size '0'",
|
||||||
"--preload_obj '0'",
|
"--preload_obj '0'",
|
||||||
"--retry '0'",
|
|
||||||
"--rule ''",
|
|
||||||
"--out ''",
|
"--out ''",
|
||||||
"--workers '0'",
|
"--workers '0'",
|
||||||
"--containers '0'",
|
"--containers '0'",
|
||||||
|
@ -644,8 +613,6 @@ class TestLoadConfig:
|
||||||
expected_preset_args = [
|
expected_preset_args = [
|
||||||
"--size '0'",
|
"--size '0'",
|
||||||
"--preload_obj '0'",
|
"--preload_obj '0'",
|
||||||
"--retry '0'",
|
|
||||||
"--rule ''",
|
|
||||||
"--out ''",
|
"--out ''",
|
||||||
"--workers '0'",
|
"--workers '0'",
|
||||||
"--containers '0'",
|
"--containers '0'",
|
||||||
|
|
Loading…
Add table
Reference in a new issue