Compare commits

..

No commits in common. "master" and "fix-failing-cat" have entirely different histories.

30 changed files with 522 additions and 908 deletions

View file

@ -10,7 +10,6 @@ tenacity==8.0.1
pytest==7.1.2 pytest==7.1.2
boto3==1.35.30 boto3==1.35.30
boto3-stubs[essential]==1.35.30 boto3-stubs[essential]==1.35.30
pydantic==2.10.6
# Dev dependencies # Dev dependencies
black==22.8.0 black==22.8.0

View file

@ -1,4 +1,4 @@
__version__ = "2.0.1" __version__ = "2.0.1"
from .fixtures import configure_testlib, hosting, temp_directory from .fixtures import configure_testlib, hosting, temp_directory
from .hooks import pytest_add_frostfs_marker, pytest_collection_modifyitems from .hooks import pytest_collection_modifyitems

View file

@ -24,7 +24,9 @@ class CliCommand:
def __init__(self, shell: Shell, cli_exec_path: str, **base_params): def __init__(self, shell: Shell, cli_exec_path: str, **base_params):
self.shell = shell self.shell = shell
self.cli_exec_path = cli_exec_path self.cli_exec_path = cli_exec_path
self.__base_params = " ".join([f"--{param} {value}" for param, value in base_params.items() if value]) self.__base_params = " ".join(
[f"--{param} {value}" for param, value in base_params.items() if value]
)
def _format_command(self, command: str, **params) -> str: def _format_command(self, command: str, **params) -> str:
param_str = [] param_str = []
@ -46,7 +48,9 @@ class CliCommand:
val_str = str(value_item).replace("'", "\\'") val_str = str(value_item).replace("'", "\\'")
param_str.append(f"--{param} '{val_str}'") param_str.append(f"--{param} '{val_str}'")
elif isinstance(value, dict): elif isinstance(value, dict):
param_str.append(f'--{param} \'{",".join(f"{key}={val}" for key, val in value.items())}\'') param_str.append(
f'--{param} \'{",".join(f"{key}={val}" for key, val in value.items())}\''
)
else: else:
if "'" in str(value): if "'" in str(value):
value_str = str(value).replace('"', '\\"') value_str = str(value).replace('"', '\\"')
@ -59,18 +63,12 @@ class CliCommand:
return f"{self.cli_exec_path} {self.__base_params} {command or ''} {param_str}" return f"{self.cli_exec_path} {self.__base_params} {command or ''} {param_str}"
def _execute(self, command: Optional[str], **params) -> CommandResult: def _execute(self, command: Optional[str], **params) -> CommandResult:
timeout = int(params["timeout"].rstrip("s")) if params.get("timeout") else None return self.shell.exec(self._format_command(command, **params))
return self.shell.exec(
self._format_command(command, **params),
CommandOptions(timeout=timeout),
)
def _execute_with_password(self, command: Optional[str], password, **params) -> CommandResult: def _execute_with_password(self, command: Optional[str], password, **params) -> CommandResult:
timeout = int(params["timeout"].rstrip("s")) if params.get("timeout") else None
return self.shell.exec( return self.shell.exec(
self._format_command(command, **params), self._format_command(command, **params),
CommandOptions( options=CommandOptions(
interactive_inputs=[InteractiveInput(prompt_pattern="assword", input=password)], interactive_inputs=[InteractiveInput(prompt_pattern="assword", input=password)]
timeout=timeout,
), ),
) )

View file

@ -12,7 +12,6 @@ class FrostfsCliNetmap(CliCommand):
address: Optional[str] = None, address: Optional[str] = None,
generate_key: bool = False, generate_key: bool = False,
ttl: Optional[int] = None, ttl: Optional[int] = None,
trace: Optional[bool] = False,
xhdr: Optional[dict] = None, xhdr: Optional[dict] = None,
timeout: Optional[str] = None, timeout: Optional[str] = None,
) -> CommandResult: ) -> CommandResult:
@ -43,7 +42,6 @@ class FrostfsCliNetmap(CliCommand):
address: Optional[str] = None, address: Optional[str] = None,
generate_key: bool = False, generate_key: bool = False,
ttl: Optional[int] = None, ttl: Optional[int] = None,
trace: Optional[bool] = False,
xhdr: Optional[dict] = None, xhdr: Optional[dict] = None,
timeout: Optional[str] = None, timeout: Optional[str] = None,
) -> CommandResult: ) -> CommandResult:
@ -75,7 +73,6 @@ class FrostfsCliNetmap(CliCommand):
generate_key: bool = False, generate_key: bool = False,
json: bool = False, json: bool = False,
ttl: Optional[int] = None, ttl: Optional[int] = None,
trace: Optional[bool] = False,
xhdr: Optional[dict] = None, xhdr: Optional[dict] = None,
timeout: Optional[str] = None, timeout: Optional[str] = None,
) -> CommandResult: ) -> CommandResult:
@ -107,7 +104,6 @@ class FrostfsCliNetmap(CliCommand):
address: Optional[str] = None, address: Optional[str] = None,
generate_key: bool = False, generate_key: bool = False,
ttl: Optional[int] = None, ttl: Optional[int] = None,
trace: Optional[bool] = False,
xhdr: Optional[dict] = None, xhdr: Optional[dict] = None,
timeout: Optional[str] = None, timeout: Optional[str] = None,
) -> CommandResult: ) -> CommandResult:

View file

@ -1,7 +1,7 @@
import re import re
from frostfs_testlib.storage.cluster import ClusterNode from frostfs_testlib.storage.cluster import ClusterNode
from frostfs_testlib.storage.dataclasses.storage_object_info import Interfaces, NodeInfo, NodeNetInfo, NodeNetmapInfo, NodeStatus from frostfs_testlib.storage.dataclasses.storage_object_info import Interfaces, NodeNetInfo, NodeNetmapInfo, NodeStatus
class NetmapParser: class NetmapParser:
@ -20,6 +20,8 @@ class NetmapParser:
"withdrawal_fee": r"Withdrawal fee: (?P<withdrawal_fee>\d+)", "withdrawal_fee": r"Withdrawal fee: (?P<withdrawal_fee>\d+)",
"homomorphic_hashing_disabled": r"Homomorphic hashing disabled: (?P<homomorphic_hashing_disabled>true|false)", "homomorphic_hashing_disabled": r"Homomorphic hashing disabled: (?P<homomorphic_hashing_disabled>true|false)",
"maintenance_mode_allowed": r"Maintenance mode allowed: (?P<maintenance_mode_allowed>true|false)", "maintenance_mode_allowed": r"Maintenance mode allowed: (?P<maintenance_mode_allowed>true|false)",
"eigen_trust_alpha": r"EigenTrustAlpha: (?P<eigen_trust_alpha>\d+\w+$)",
"eigen_trust_iterations": r"EigenTrustIterations: (?P<eigen_trust_iterations>\d+)",
} }
parse_result = {} parse_result = {}
@ -62,7 +64,7 @@ class NetmapParser:
for node in netmap_nodes: for node in netmap_nodes:
for key, regex in regexes.items(): for key, regex in regexes.items():
search_result = re.search(regex, node, flags=re.MULTILINE) search_result = re.search(regex, node, flags=re.MULTILINE)
if search_result is None: if search_result == None:
result_netmap[key] = None result_netmap[key] = None
continue continue
if key == "node_data_ips": if key == "node_data_ips":
@ -81,22 +83,9 @@ class NetmapParser:
return dataclasses_netmap return dataclasses_netmap
@staticmethod @staticmethod
def snapshot_one_node(output: str, rpc_endpoint: str) -> NodeNetmapInfo | None: def snapshot_one_node(output: str, cluster_node: ClusterNode) -> NodeNetmapInfo | None:
snapshot_nodes = NetmapParser.snapshot_all_nodes(output=output) snapshot_nodes = NetmapParser.snapshot_all_nodes(output=output)
for snapshot in snapshot_nodes: snapshot_node = [node for node in snapshot_nodes if node.node == cluster_node.get_interface(Interfaces.MGMT)]
for endpoint in snapshot.external_address: if not snapshot_node:
if rpc_endpoint.split(":")[0] in endpoint: return None
return snapshot return snapshot_node[0]
@staticmethod
def node_info(output: dict) -> NodeNetmapInfo:
data_dict = {"attributes": {}}
for key, value in output.items():
if key != "attributes":
data_dict[key] = value
for attribute in output["attributes"]:
data_dict["attributes"][attribute["key"]] = attribute["value"]
return NodeInfo(**data_dict)

View file

@ -1 +0,0 @@
from frostfs_testlib.clients.http.http_client import HttpClient

View file

@ -134,10 +134,9 @@ class HttpClient:
@classmethod @classmethod
def _create_curl_request(cls, url: str, method: str, headers: httpx.Headers, data: str, files: dict) -> str: def _create_curl_request(cls, url: str, method: str, headers: httpx.Headers, data: str, files: dict) -> str:
excluded_headers = {"Accept-Encoding", "Connection", "User-Agent", "Content-Length"} headers = " ".join(f'-H "{name.title()}: {value}"' for name, value in headers.items())
headers = " ".join(f"-H '{header.title()}: {value}'" for header, value in headers.items() if header.title() not in excluded_headers)
data = f" -d '{data}'" if data else "" data = f" -d '{data}'" if data else ""
for name, path in files.items(): for name, path in files.items():
data += f' -F "{name}=@{path}"' data += f' -F "{name}=@{path}"'

View file

@ -1,3 +1 @@
from frostfs_testlib.clients.s3.aws_cli_client import AwsCliClient from frostfs_testlib.clients.s3.interfaces import BucketContainerResolver, S3ClientWrapper, VersioningStatus
from frostfs_testlib.clients.s3.boto3_client import Boto3ClientWrapper
from frostfs_testlib.clients.s3.interfaces import ACL, BucketContainerResolver, S3ClientWrapper, VersioningStatus

View file

@ -33,14 +33,12 @@ class AwsCliClient(S3ClientWrapper):
self, access_key_id: str, secret_access_key: str, s3gate_endpoint: str, profile: str = "default", region: str = "us-east-1" self, access_key_id: str, secret_access_key: str, s3gate_endpoint: str, profile: str = "default", region: str = "us-east-1"
) -> None: ) -> None:
self.s3gate_endpoint = s3gate_endpoint self.s3gate_endpoint = s3gate_endpoint
self.iam_endpoint = None
self.access_key_id: str = access_key_id self.access_key_id: str = access_key_id
self.secret_access_key: str = secret_access_key self.secret_access_key: str = secret_access_key
self.profile = profile self.profile = profile
self.region = region
self.local_shell = LocalShell() self.local_shell = LocalShell()
self.region = region
self.iam_endpoint = None
try: try:
_configure_aws_cli(f"aws configure --profile {profile}", access_key_id, secret_access_key, region) _configure_aws_cli(f"aws configure --profile {profile}", access_key_id, secret_access_key, region)
self.local_shell.exec(f"aws configure set max_attempts {MAX_REQUEST_ATTEMPTS} --profile {profile}") self.local_shell.exec(f"aws configure set max_attempts {MAX_REQUEST_ATTEMPTS} --profile {profile}")
@ -1229,7 +1227,7 @@ class AwsCliClient(S3ClientWrapper):
output = self.local_shell.exec(cmd).stdout output = self.local_shell.exec(cmd).stdout
response = self._to_json(output) response = self._to_json(output)
assert "AttachedPolicies" in response.keys(), f"Expected AttachedPolicies in response:\n{response}" assert response.get("AttachedPolicies"), f"Expected AttachedPolicies in response:\n{response}"
return response return response
@ -1241,7 +1239,7 @@ class AwsCliClient(S3ClientWrapper):
output = self.local_shell.exec(cmd).stdout output = self.local_shell.exec(cmd).stdout
response = self._to_json(output) response = self._to_json(output)
assert "AttachedPolicies" in response.keys(), f"Expected AttachedPolicies in response:\n{response}" assert response.get("AttachedPolicies"), f"Expected AttachedPolicies in response:\n{response}"
return response return response
@ -1266,7 +1264,7 @@ class AwsCliClient(S3ClientWrapper):
output = self.local_shell.exec(cmd).stdout output = self.local_shell.exec(cmd).stdout
response = self._to_json(output) response = self._to_json(output)
assert "PolicyNames" in response.keys(), f"Expected PolicyNames in response:\n{response}" assert response.get("PolicyNames"), f"Expected PolicyNames in response:\n{response}"
return response return response
@ -1278,7 +1276,7 @@ class AwsCliClient(S3ClientWrapper):
output = self.local_shell.exec(cmd).stdout output = self.local_shell.exec(cmd).stdout
response = self._to_json(output) response = self._to_json(output)
assert "Groups" in response.keys(), f"Expected Groups in response:\n{response}" assert response.get("Groups"), f"Expected Groups in response:\n{response}"
return response return response
@ -1290,7 +1288,7 @@ class AwsCliClient(S3ClientWrapper):
output = self.local_shell.exec(cmd).stdout output = self.local_shell.exec(cmd).stdout
response = self._to_json(output) response = self._to_json(output)
assert "Groups" in response.keys(), f"Expected Groups in response:\n{response}" assert response.get("Groups"), f"Expected Groups in response:\n{response}"
return response return response
@ -1326,7 +1324,7 @@ class AwsCliClient(S3ClientWrapper):
output = self.local_shell.exec(cmd).stdout output = self.local_shell.exec(cmd).stdout
response = self._to_json(output) response = self._to_json(output)
assert "PolicyNames" in response.keys(), f"Expected PolicyNames in response:\n{response}" assert response.get("PolicyNames"), f"Expected PolicyNames in response:\n{response}"
return response return response

View file

@ -35,20 +35,26 @@ class Boto3ClientWrapper(S3ClientWrapper):
def __init__( def __init__(
self, access_key_id: str, secret_access_key: str, s3gate_endpoint: str, profile: str = "default", region: str = "us-east-1" self, access_key_id: str, secret_access_key: str, s3gate_endpoint: str, profile: str = "default", region: str = "us-east-1"
) -> None: ) -> None:
self.s3gate_endpoint: str = ""
self.boto3_client: S3Client = None self.boto3_client: S3Client = None
self.s3gate_endpoint: str = ""
self.iam_endpoint: str = ""
self.boto3_iam_client: S3Client = None self.boto3_iam_client: S3Client = None
self.iam_endpoint: str = ""
self.boto3_sts_client: S3Client = None self.boto3_sts_client: S3Client = None
self.access_key_id = access_key_id self.access_key_id: str = access_key_id
self.secret_access_key = secret_access_key self.secret_access_key: str = secret_access_key
self.profile = profile self.profile = profile
self.region = region self.region = region
self.session = boto3.Session() self.session = boto3.Session()
self.config = Config(retries={"max_attempts": MAX_REQUEST_ATTEMPTS, "mode": RETRY_MODE}) self.config = Config(
retries={
"max_attempts": MAX_REQUEST_ATTEMPTS,
"mode": RETRY_MODE,
}
)
self.set_endpoint(s3gate_endpoint) self.set_endpoint(s3gate_endpoint)
@ -84,7 +90,7 @@ class Boto3ClientWrapper(S3ClientWrapper):
endpoint_url=self.iam_endpoint, endpoint_url=self.iam_endpoint,
verify=False, verify=False,
) )
# since the STS does not have an endpoint, IAM is used # since the STS does not have an enpoint, IAM is used
self.boto3_sts_client = self.session.client( self.boto3_sts_client = self.session.client(
service_name="sts", service_name="sts",
aws_access_key_id=self.access_key_id, aws_access_key_id=self.access_key_id,
@ -139,7 +145,6 @@ class Boto3ClientWrapper(S3ClientWrapper):
params = {"Bucket": bucket} params = {"Bucket": bucket}
if object_lock_enabled_for_bucket is not None: if object_lock_enabled_for_bucket is not None:
params.update({"ObjectLockEnabledForBucket": object_lock_enabled_for_bucket}) params.update({"ObjectLockEnabledForBucket": object_lock_enabled_for_bucket})
if acl is not None: if acl is not None:
params.update({"ACL": acl}) params.update({"ACL": acl})
elif grant_write or grant_read or grant_full_control: elif grant_write or grant_read or grant_full_control:
@ -149,7 +154,6 @@ class Boto3ClientWrapper(S3ClientWrapper):
params.update({"GrantRead": grant_read}) params.update({"GrantRead": grant_read})
elif grant_full_control: elif grant_full_control:
params.update({"GrantFullControl": grant_full_control}) params.update({"GrantFullControl": grant_full_control})
if location_constraint: if location_constraint:
params.update({"CreateBucketConfiguration": {"LocationConstraint": location_constraint}}) params.update({"CreateBucketConfiguration": {"LocationConstraint": location_constraint}})
@ -1087,7 +1091,7 @@ class Boto3ClientWrapper(S3ClientWrapper):
endpoint=self.iam_endpoint, endpoint=self.iam_endpoint,
profile=self.profile, profile=self.profile,
) )
assert "AttachedPolicies" in response.keys(), f"Expected AttachedPolicies in response:\n{response}" assert response.get("AttachedPolicies"), f"Expected AttachedPolicies in response:\n{response}"
return response return response
@reporter.step("Lists all managed policies that are attached to the specified IAM user") @reporter.step("Lists all managed policies that are attached to the specified IAM user")
@ -1098,7 +1102,7 @@ class Boto3ClientWrapper(S3ClientWrapper):
endpoint=self.iam_endpoint, endpoint=self.iam_endpoint,
profile=self.profile, profile=self.profile,
) )
assert "AttachedPolicies" in response.keys(), f"Expected AttachedPolicies in response:\n{response}" assert response.get("AttachedPolicies"), f"Expected AttachedPolicies in response:\n{response}"
return response return response
@reporter.step("Lists all IAM users, groups, and roles that the specified managed policy is attached to") @reporter.step("Lists all IAM users, groups, and roles that the specified managed policy is attached to")
@ -1123,7 +1127,7 @@ class Boto3ClientWrapper(S3ClientWrapper):
endpoint=self.iam_endpoint, endpoint=self.iam_endpoint,
profile=self.profile, profile=self.profile,
) )
assert "PolicyNames" in response.keys(), f"Expected PolicyNames in response:\n{response}" assert response.get("PolicyNames"), f"Expected PolicyNames in response:\n{response}"
return response return response
@reporter.step("Lists the IAM groups") @reporter.step("Lists the IAM groups")
@ -1133,7 +1137,7 @@ class Boto3ClientWrapper(S3ClientWrapper):
endpoint=self.iam_endpoint, endpoint=self.iam_endpoint,
profile=self.profile, profile=self.profile,
) )
assert "Groups" in response.keys(), f"Expected Groups in response:\n{response}" assert response.get("Groups"), f"Expected Groups in response:\n{response}"
return response return response
@reporter.step("Lists the IAM groups that the specified IAM user belongs to") @reporter.step("Lists the IAM groups that the specified IAM user belongs to")
@ -1144,7 +1148,7 @@ class Boto3ClientWrapper(S3ClientWrapper):
endpoint=self.iam_endpoint, endpoint=self.iam_endpoint,
profile=self.profile, profile=self.profile,
) )
assert "Groups" in response.keys(), f"Expected Groups in response:\n{response}" assert response.get("Groups"), f"Expected Groups in response:\n{response}"
return response return response
@reporter.step("Lists all the managed policies that are available in your AWS account") @reporter.step("Lists all the managed policies that are available in your AWS account")
@ -1176,7 +1180,7 @@ class Boto3ClientWrapper(S3ClientWrapper):
endpoint=self.iam_endpoint, endpoint=self.iam_endpoint,
profile=self.profile, profile=self.profile,
) )
assert "PolicyNames" in response.keys(), f"Expected PolicyNames in response:\n{response}" assert response.get("PolicyNames"), f"Expected PolicyNames in response:\n{response}"
return response return response
@reporter.step("Lists the IAM users") @reporter.step("Lists the IAM users")

View file

@ -22,15 +22,15 @@ class VersioningStatus(HumanReadableEnum):
SUSPENDED = "Suspended" SUSPENDED = "Suspended"
class ACL: ACL_COPY = [
PRIVATE = "private" "private",
PUBLIC_READ = "public-read" "public-read",
PUBLIC_READ_WRITE = "public-read-write" "public-read-write",
AUTHENTICATED_READ = "authenticated-read" "authenticated-read",
AWS_EXEC_READ = "aws-exec-read" "aws-exec-read",
BUCKET_OWNER_READ = "bucket-owner-read" "bucket-owner-read",
BUCKET_OWNER_FULL_CONTROL = "bucket-owner-full-control" "bucket-owner-full-control",
LOG_DELIVERY_WRITE = "log-delivery-write" ]
class BucketContainerResolver(ABC): class BucketContainerResolver(ABC):
@ -50,14 +50,6 @@ class BucketContainerResolver(ABC):
class S3ClientWrapper(HumanReadableABC): class S3ClientWrapper(HumanReadableABC):
access_key_id: str
secret_access_key: str
profile: str
region: str
s3gate_endpoint: str
iam_endpoint: str
@abstractmethod @abstractmethod
def __init__(self, access_key_id: str, secret_access_key: str, s3gate_endpoint: str, profile: str, region: str) -> None: def __init__(self, access_key_id: str, secret_access_key: str, s3gate_endpoint: str, profile: str, region: str) -> None:
pass pass

View file

@ -1,8 +1,8 @@
import pytest import pytest
@pytest.hookimpl(specname="pytest_collection_modifyitems") @pytest.hookimpl
def pytest_add_frostfs_marker(items: list[pytest.Item]): def pytest_collection_modifyitems(items: list[pytest.Item]):
# All tests which reside in frostfs nodeid are granted with frostfs marker, excluding # All tests which reside in frostfs nodeid are granted with frostfs marker, excluding
# nodeid = full path of the test # nodeid = full path of the test
# 1. plugins # 1. plugins
@ -11,18 +11,3 @@ def pytest_add_frostfs_marker(items: list[pytest.Item]):
location = item.location[0] location = item.location[0]
if "frostfs" in location and "plugin" not in location and "testlib" not in location: if "frostfs" in location and "plugin" not in location and "testlib" not in location:
item.add_marker("frostfs") item.add_marker("frostfs")
# pytest hook. Do not rename
@pytest.hookimpl(trylast=True)
def pytest_collection_modifyitems(items: list[pytest.Item]):
# Change order of tests based on @pytest.mark.order(<int>) marker
def order(item: pytest.Item) -> int:
order_marker = item.get_closest_marker("order")
if order_marker and (len(order_marker.args) != 1 or not isinstance(order_marker.args[0], int)):
raise RuntimeError("Incorrect usage of pytest.mark.order")
order_value = order_marker.args[0] if order_marker else 0
return order_value
items.sort(key=lambda item: order(item))

View file

@ -182,10 +182,8 @@ class Preset(MetaConfig):
pregen_json: Optional[str] = metadata_field(all_load_scenarios, "out", "PREGEN_JSON", False) pregen_json: Optional[str] = metadata_field(all_load_scenarios, "out", "PREGEN_JSON", False)
# Workers count for preset # Workers count for preset
workers: Optional[int] = metadata_field(all_load_scenarios, "workers", None, False) workers: Optional[int] = metadata_field(all_load_scenarios, "workers", None, False)
# TODO: Deprecated. Acl for container/buckets # Acl for container/buckets
acl: Optional[str] = metadata_field(all_load_scenarios, "acl", None, False) acl: Optional[str] = metadata_field(all_load_scenarios, "acl", None, False)
# APE rule for containers instead of deprecated ACL
rule: Optional[list[str]] = metadata_field(grpc_preset_scenarios, "rule", None, False, formatter=force_list)
# ------ GRPC ------ # ------ GRPC ------
# Amount of containers which should be created # Amount of containers which should be created

View file

@ -1,6 +1,5 @@
# Regex patterns of status codes of Container service # Regex patterns of status codes of Container service
CONTAINER_NOT_FOUND = "code = 3072.*message = container not found" CONTAINER_NOT_FOUND = "code = 3072.*message = container not found"
SUBJECT_NOT_FOUND = "code = 1024.*message = frostfs error: chain/client.*subject not found.*"
# Regex patterns of status codes of Object service # Regex patterns of status codes of Object service
MALFORMED_REQUEST = "code = 1024.*message = malformed request" MALFORMED_REQUEST = "code = 1024.*message = malformed request"

View file

@ -5,7 +5,6 @@ class ConfigAttributes:
WALLET_CONFIG = "wallet_config" WALLET_CONFIG = "wallet_config"
CONFIG_DIR = "service_config_dir" CONFIG_DIR = "service_config_dir"
CONFIG_PATH = "config_path" CONFIG_PATH = "config_path"
WORKING_DIR = "working_dir"
SHARD_CONFIG_PATH = "shard_config_path" SHARD_CONFIG_PATH = "shard_config_path"
LOGGER_CONFIG_PATH = "logger_config_path" LOGGER_CONFIG_PATH = "logger_config_path"
LOCAL_WALLET_PATH = "local_wallet_path" LOCAL_WALLET_PATH = "local_wallet_path"
@ -16,7 +15,6 @@ class ConfigAttributes:
ENDPOINT_DATA_0_NS = "endpoint_data0_namespace" ENDPOINT_DATA_0_NS = "endpoint_data0_namespace"
ENDPOINT_INTERNAL = "endpoint_internal0" ENDPOINT_INTERNAL = "endpoint_internal0"
ENDPOINT_PROMETHEUS = "endpoint_prometheus" ENDPOINT_PROMETHEUS = "endpoint_prometheus"
ENDPOINT_PPROF = "endpoint_pprof"
CONTROL_ENDPOINT = "control_endpoint" CONTROL_ENDPOINT = "control_endpoint"
UN_LOCODE = "un_locode" UN_LOCODE = "un_locode"

View file

@ -1,7 +1,7 @@
import datetime
import itertools import itertools
import logging import logging
import time import time
from datetime import datetime, timezone
from typing import TypeVar from typing import TypeVar
import frostfs_testlib.resources.optionals as optionals import frostfs_testlib.resources.optionals as optionals
@ -390,23 +390,31 @@ class ClusterStateController:
@reporter.step("Get node time") @reporter.step("Get node time")
def get_node_date(self, node: ClusterNode) -> datetime: def get_node_date(self, node: ClusterNode) -> datetime:
shell = node.host.get_shell() shell = node.host.get_shell()
return datetime.strptime(shell.exec('date +"%Y-%m-%d %H:%M:%S"').stdout.strip(), "%Y-%m-%d %H:%M:%S") return datetime.datetime.strptime(shell.exec("hwclock -r").stdout.strip(), "%Y-%m-%d %H:%M:%S.%f%z")
@reporter.step("Set node time to {in_date}") @reporter.step("Set node time to {in_date}")
def change_node_date(self, node: ClusterNode, in_date: datetime) -> None: def change_node_date(self, node: ClusterNode, in_date: datetime) -> None:
shell = node.host.get_shell() shell = node.host.get_shell()
in_date_frmt = in_date.strftime("%Y-%m-%d %H:%M:%S") shell.exec(f"date -s @{time.mktime(in_date.timetuple())}")
shell.exec(f"timedatectl set-time '{in_date_frmt}'") shell.exec("hwclock --systohc")
node_time = self.get_node_date(node) node_time = self.get_node_date(node)
with reporter.step(f"Verify difference between {node_time} and {in_date} is less than a minute"): with reporter.step(f"Verify difference between {node_time} and {in_date} is less than a minute"):
assert (node_time - in_date).total_seconds() < 60 assert (self.get_node_date(node) - in_date) < datetime.timedelta(minutes=1)
@reporter.step("Restore time") @reporter.step(f"Restore time")
def restore_node_date(self, node: ClusterNode) -> None: def restore_node_date(self, node: ClusterNode) -> None:
shell = node.host.get_shell() shell = node.host.get_shell()
now_time = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S") now_time = datetime.datetime.now(datetime.timezone.utc)
with reporter.step(f"Set {now_time} time"): with reporter.step(f"Set {now_time} time"):
shell.exec(f"timedatectl set-time '{now_time}'") shell.exec(f"date -s @{time.mktime(now_time.timetuple())}")
shell.exec("hwclock --systohc")
@reporter.step("Change the synchronizer status to {status}")
def set_sync_date_all_nodes(self, status: str):
if status == "active":
parallel(self._enable_date_synchronizer, self.cluster.cluster_nodes)
return
parallel(self._disable_date_synchronizer, self.cluster.cluster_nodes)
@reporter.step("Set MaintenanceModeAllowed - {status}") @reporter.step("Set MaintenanceModeAllowed - {status}")
def set_maintenance_mode_allowed(self, status: str, cluster_node: ClusterNode) -> None: def set_maintenance_mode_allowed(self, status: str, cluster_node: ClusterNode) -> None:
@ -492,6 +500,16 @@ class ClusterStateController:
frostfs_cli_remote = FrostfsCli(shell=shell, frostfs_cli_exec_path=FROSTFS_CLI_EXEC, config_file=wallet_config_path) frostfs_cli_remote = FrostfsCli(shell=shell, frostfs_cli_exec_path=FROSTFS_CLI_EXEC, config_file=wallet_config_path)
return frostfs_adm, frostfs_cli, frostfs_cli_remote return frostfs_adm, frostfs_cli, frostfs_cli_remote
def _enable_date_synchronizer(self, cluster_node: ClusterNode):
shell = cluster_node.host.get_shell()
shell.exec("timedatectl set-ntp true")
cluster_node.host.wait_for_service_to_be_in_state("systemd-timesyncd", "active", 15)
def _disable_date_synchronizer(self, cluster_node: ClusterNode):
shell = cluster_node.host.get_shell()
shell.exec("timedatectl set-ntp false")
cluster_node.host.wait_for_service_to_be_in_state("systemd-timesyncd", "inactive", 15)
def _get_disk_controller(self, node: StorageNode, device: str, mountpoint: str) -> DiskController: def _get_disk_controller(self, node: StorageNode, device: str, mountpoint: str) -> DiskController:
disk_controller_id = DiskController.get_id(node, device) disk_controller_id = DiskController.get_id(node, device)
if disk_controller_id in self.detached_disks.keys(): if disk_controller_id in self.detached_disks.keys():

View file

@ -82,9 +82,6 @@ class NodeBase(HumanReadableABC):
def get_metrics_endpoint(self) -> str: def get_metrics_endpoint(self) -> str:
return self._get_attribute(ConfigAttributes.ENDPOINT_PROMETHEUS) return self._get_attribute(ConfigAttributes.ENDPOINT_PROMETHEUS)
def get_pprof_endpoint(self) -> str:
return self._get_attribute(ConfigAttributes.ENDPOINT_PPROF)
def stop_service(self, mask: bool = True): def stop_service(self, mask: bool = True):
if mask: if mask:
with reporter.step(f"Mask {self.name} service on {self.host.config.address}"): with reporter.step(f"Mask {self.name} service on {self.host.config.address}"):
@ -147,13 +144,6 @@ class NodeBase(HumanReadableABC):
else None else None
) )
def get_working_dir_path(self) -> Optional[str]:
"""
Returns working directory path located on remote host
"""
config_attributes = self.host.get_service_config(self.name)
return self._get_attribute(ConfigAttributes.WORKING_DIR) if ConfigAttributes.WORKING_DIR in config_attributes.attributes else None
@property @property
def config_dir(self) -> str: def config_dir(self) -> str:
return self._get_attribute(ConfigAttributes.CONFIG_DIR) return self._get_attribute(ConfigAttributes.CONFIG_DIR)

View file

@ -1,9 +1,6 @@
import re
from dataclasses import dataclass from dataclasses import dataclass
from typing import Optional from typing import Optional
from pydantic import BaseModel, Field, field_validator
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.readable import HumanReadableEnum from frostfs_testlib.testing.readable import HumanReadableEnum
@ -78,37 +75,8 @@ class NodeNetInfo:
withdrawal_fee: str = None withdrawal_fee: str = None
homomorphic_hashing_disabled: str = None homomorphic_hashing_disabled: str = None
maintenance_mode_allowed: str = None maintenance_mode_allowed: str = None
eigen_trust_alpha: str = None
eigen_trust_iterations: str = None
class Attributes(BaseModel):
cluster_name: str = Field(alias="ClusterName")
continent: str = Field(alias="Continent")
country: str = Field(alias="Country")
country_code: str = Field(alias="CountryCode")
external_addr: list[str] = Field(alias="ExternalAddr")
location: str = Field(alias="Location")
node: str = Field(alias="Node")
subdiv: str = Field(alias="SubDiv")
subdiv_code: str = Field(alias="SubDivCode")
un_locode: str = Field(alias="UN-LOCODE")
role: str = Field(alias="role")
@field_validator("external_addr", mode="before")
@classmethod
def convert_external_addr(cls, value: str) -> list[str]:
return [f"{ip}:{port}" for ip, port in re.findall(r"/ip4/([\d\.]+)/(?:tcp|tls)/(\d+)", value)]
class NodeInfo(BaseModel):
public_key: str = Field(alias="publicKey")
addresses: list[str] = Field(alias="addresses")
state: str = Field(alias="state")
attributes: Attributes = Field(alias="attributes")
@field_validator("addresses", mode="before")
@classmethod
def convert_external_addr(cls, value: str) -> list[str]:
return [f"{ip}:{port}" for ip, port in re.findall(r"/ip4/([\d\.]+)/(?:tcp|tls)/(\d+)", ",".join(value))]
@dataclass @dataclass

View file

@ -1,14 +1,14 @@
from frostfs_testlib.cli.frostfs_cli.cli import FrostfsCli from frostfs_testlib.cli.frostfs_cli.cli import FrostfsCli
from frostfs_testlib.storage.grpc_operations import implementations, interfaces, interfaces_wrapper from frostfs_testlib.storage.grpc_operations import interfaces
from frostfs_testlib.storage.grpc_operations.implementations import container, object
class CliClientWrapper(interfaces_wrapper.GrpcClientWrapper): class CliClientWrapper(interfaces.GrpcClientWrapper):
def __init__(self, cli: FrostfsCli) -> None: def __init__(self, cli: FrostfsCli) -> None:
self.cli = cli self.cli = cli
self.object: interfaces.ObjectInterface = implementations.ObjectOperations(self.cli) self.object: interfaces.ObjectInterface = object.ObjectOperations(self.cli)
self.container: interfaces.ContainerInterface = implementations.ContainerOperations(self.cli) self.container: interfaces.ContainerInterface = container.ContainerOperations(self.cli)
self.netmap: interfaces.NetmapInterface = implementations.NetmapOperations(self.cli)
class RpcClientWrapper(interfaces_wrapper.GrpcClientWrapper): class RpcClientWrapper(interfaces.GrpcClientWrapper):
pass # The next series pass # The next series

View file

@ -1,4 +0,0 @@
from .chunks import ChunksOperations
from .container import ContainerOperations
from .netmap import NetmapOperations
from .object import ObjectOperations

View file

@ -1,171 +0,0 @@
import json as module_json
from typing import List, Optional
from frostfs_testlib.cli.frostfs_cli.cli import FrostfsCli
from frostfs_testlib.cli.netmap_parser import NetmapParser
from frostfs_testlib.resources.cli import CLI_DEFAULT_TIMEOUT
from frostfs_testlib.storage.dataclasses.storage_object_info import NodeNetInfo, NodeNetmapInfo
from .. import interfaces
class NetmapOperations(interfaces.NetmapInterface):
def __init__(self, cli: FrostfsCli) -> None:
self.cli = cli
def epoch(
self,
rpc_endpoint: str,
wallet: Optional[str] = None,
address: Optional[str] = None,
generate_key: bool = False,
ttl: Optional[int] = None,
trace: Optional[bool] = True,
xhdr: Optional[dict] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> int:
"""
Get current epoch number.
"""
output = (
self.cli.netmap.epoch(
rpc_endpoint=rpc_endpoint,
wallet=wallet,
address=address,
generate_key=generate_key,
ttl=ttl,
trace=trace,
xhdr=xhdr,
timeout=timeout,
)
.stdout.split("Trace ID")[0]
.strip()
)
return int(output)
def netinfo(
self,
rpc_endpoint: str,
wallet: Optional[str] = None,
address: Optional[str] = None,
generate_key: bool = False,
ttl: Optional[int] = None,
trace: Optional[bool] = True,
xhdr: Optional[dict] = None,
timeout: Optional[str] = None,
) -> NodeNetInfo:
"""
Get target node info.
"""
output = (
self.cli.netmap.netinfo(
rpc_endpoint=rpc_endpoint,
wallet=wallet,
address=address,
generate_key=generate_key,
ttl=ttl,
trace=trace,
xhdr=xhdr,
timeout=timeout,
)
.stdout.split("Trace ID")[0]
.strip()
)
return NetmapParser.netinfo(output)
def nodeinfo(
self,
rpc_endpoint: str,
wallet: Optional[str] = None,
address: Optional[str] = None,
generate_key: bool = False,
json: bool = True,
ttl: Optional[int] = None,
trace: Optional[bool] = True,
xhdr: Optional[dict] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> NodeNetmapInfo:
"""
Get target node info.
"""
output = (
self.cli.netmap.nodeinfo(
rpc_endpoint=rpc_endpoint,
wallet=wallet,
address=address,
generate_key=generate_key,
json=json,
ttl=ttl,
trace=trace,
xhdr=xhdr,
timeout=timeout,
)
.stdout.split("Trace ID")[0]
.strip()
)
return NetmapParser.node_info(module_json.loads(output))
def snapshot(
self,
rpc_endpoint: str,
wallet: Optional[str] = None,
address: Optional[str] = None,
generate_key: bool = False,
ttl: Optional[int] = None,
trace: Optional[bool] = True,
xhdr: Optional[dict] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> List[NodeNetmapInfo]:
"""
Get target node info.
"""
output = (
self.cli.netmap.snapshot(
rpc_endpoint=rpc_endpoint,
wallet=wallet,
address=address,
generate_key=generate_key,
ttl=ttl,
trace=trace,
xhdr=xhdr,
timeout=timeout,
)
.stdout.split("Trace ID")[0]
.strip()
)
return NetmapParser.snapshot_all_nodes(output)
def snapshot_one_node(
self,
rpc_endpoint: str,
wallet: Optional[str] = None,
address: Optional[str] = None,
generate_key: bool = False,
ttl: Optional[int] = None,
trace: Optional[bool] = True,
xhdr: Optional[dict] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> List[NodeNetmapInfo]:
"""
Get target one node info.
"""
output = (
self.cli.netmap.snapshot(
rpc_endpoint=rpc_endpoint,
wallet=wallet,
address=address,
generate_key=generate_key,
ttl=ttl,
trace=trace,
xhdr=xhdr,
timeout=timeout,
)
.stdout.split("Trace ID")[0]
.strip()
)
return NetmapParser.snapshot_one_node(output, rpc_endpoint)

View file

@ -0,0 +1,424 @@
from abc import ABC, abstractmethod
from typing import Any, List, Optional
from frostfs_testlib.shell.interfaces import CommandResult
from frostfs_testlib.storage.cluster import Cluster, ClusterNode
from frostfs_testlib.storage.constants import PlacementRule
from frostfs_testlib.storage.dataclasses.storage_object_info import Chunk, NodeNetmapInfo
from frostfs_testlib.utils import file_utils
class ChunksInterface(ABC):
@abstractmethod
def search_node_without_chunks(self, chunks: list[Chunk], cluster: Cluster, endpoint: str = None) -> list[ClusterNode]:
pass
@abstractmethod
def get_chunk_node(self, cluster: Cluster, chunk: Chunk) -> tuple[ClusterNode, NodeNetmapInfo]:
pass
@abstractmethod
def get_shard_chunk(self, node: ClusterNode, chunk: Chunk) -> str:
pass
@abstractmethod
def get_all(
self,
rpc_endpoint: str,
cid: str,
oid: str,
wallet: Optional[str] = None,
address: Optional[str] = None,
bearer: Optional[str] = None,
generate_key: Optional[bool] = None,
trace: bool = False,
root: bool = False,
verify_presence_all: bool = False,
json: bool = True,
ttl: Optional[int] = None,
xhdr: Optional[dict] = None,
timeout: Optional[str] = None,
) -> list[Chunk]:
pass
@abstractmethod
def get_parity(
self,
rpc_endpoint: str,
cid: str,
wallet: Optional[str] = None,
address: Optional[str] = None,
bearer: Optional[str] = None,
generate_key: Optional[bool] = None,
oid: Optional[str] = None,
trace: bool = False,
root: bool = False,
verify_presence_all: bool = False,
json: bool = True,
ttl: Optional[int] = None,
xhdr: Optional[dict] = None,
timeout: Optional[str] = None,
) -> Chunk:
pass
@abstractmethod
def get_first_data(
self,
rpc_endpoint: str,
cid: str,
wallet: Optional[str] = None,
address: Optional[str] = None,
bearer: Optional[str] = None,
generate_key: Optional[bool] = None,
oid: Optional[str] = None,
trace: bool = False,
root: bool = False,
verify_presence_all: bool = False,
json: bool = True,
ttl: Optional[int] = None,
xhdr: Optional[dict] = None,
timeout: Optional[str] = None,
) -> Chunk:
pass
class ObjectInterface(ABC):
def __init__(self) -> None:
self.chunks: ChunksInterface
@abstractmethod
def delete(
self,
cid: str,
oid: str,
endpoint: str,
bearer: str = "",
xhdr: Optional[dict] = None,
session: Optional[str] = None,
timeout: Optional[str] = None,
) -> str:
pass
@abstractmethod
def get(
self,
cid: str,
oid: str,
endpoint: str,
bearer: Optional[str] = None,
write_object: Optional[str] = None,
xhdr: Optional[dict] = None,
no_progress: bool = True,
session: Optional[str] = None,
timeout: Optional[str] = None,
) -> file_utils.TestFile:
pass
@abstractmethod
def get_from_random_node(
self,
cid: str,
oid: str,
cluster: Cluster,
bearer: Optional[str] = None,
write_object: Optional[str] = None,
xhdr: Optional[dict] = None,
no_progress: bool = True,
session: Optional[str] = None,
timeout: Optional[str] = None,
) -> str:
pass
@abstractmethod
def hash(
self,
endpoint: str,
cid: str,
oid: str,
address: Optional[str] = None,
bearer: Optional[str] = None,
generate_key: Optional[bool] = None,
range: Optional[str] = None,
salt: Optional[str] = None,
ttl: Optional[int] = None,
session: Optional[str] = None,
hash_type: Optional[str] = None,
xhdr: Optional[dict] = None,
timeout: Optional[str] = None,
) -> str:
pass
@abstractmethod
def head(
self,
cid: str,
oid: str,
endpoint: str,
bearer: str = "",
xhdr: Optional[dict] = None,
json_output: bool = True,
is_raw: bool = False,
is_direct: bool = False,
session: Optional[str] = None,
timeout: Optional[str] = None,
) -> CommandResult | Any:
pass
@abstractmethod
def lock(
self,
cid: str,
oid: str,
endpoint: str,
lifetime: Optional[int] = None,
expire_at: Optional[int] = None,
address: Optional[str] = None,
bearer: Optional[str] = None,
session: Optional[str] = None,
ttl: Optional[int] = None,
xhdr: Optional[dict] = None,
timeout: Optional[str] = None,
) -> str:
pass
@abstractmethod
def put(
self,
path: str,
cid: str,
endpoint: str,
bearer: Optional[str] = None,
copies_number: Optional[int] = None,
attributes: Optional[dict] = None,
xhdr: Optional[dict] = None,
expire_at: Optional[int] = None,
no_progress: bool = True,
session: Optional[str] = None,
timeout: Optional[str] = None,
) -> str:
pass
@abstractmethod
def patch(
self,
cid: str,
oid: str,
endpoint: str,
ranges: Optional[list[str]] = None,
payloads: Optional[list[str]] = None,
new_attrs: Optional[str] = None,
replace_attrs: bool = False,
bearer: Optional[str] = None,
xhdr: Optional[dict] = None,
session: Optional[str] = None,
timeout: Optional[str] = None,
trace: bool = False,
) -> str:
pass
@abstractmethod
def put_to_random_node(
self,
path: str,
cid: str,
cluster: Cluster,
bearer: Optional[str] = None,
copies_number: Optional[int] = None,
attributes: Optional[dict] = None,
xhdr: Optional[dict] = None,
expire_at: Optional[int] = None,
no_progress: bool = True,
session: Optional[str] = None,
timeout: Optional[str] = None,
) -> str:
pass
@abstractmethod
def range(
self,
cid: str,
oid: str,
range_cut: str,
endpoint: str,
bearer: str = "",
xhdr: Optional[dict] = None,
session: Optional[str] = None,
timeout: Optional[str] = None,
) -> tuple[file_utils.TestFile, bytes]:
pass
@abstractmethod
def search(
self,
cid: str,
endpoint: str,
bearer: str = "",
oid: Optional[str] = None,
filters: Optional[dict] = None,
expected_objects_list: Optional[list] = None,
xhdr: Optional[dict] = None,
session: Optional[str] = None,
phy: bool = False,
root: bool = False,
timeout: Optional[str] = None,
address: Optional[str] = None,
generate_key: Optional[bool] = None,
ttl: Optional[int] = None,
) -> List:
pass
@abstractmethod
def nodes(
self,
cluster: Cluster,
cid: str,
oid: str,
alive_node: ClusterNode,
bearer: str = "",
xhdr: Optional[dict] = None,
is_direct: bool = False,
verify_presence_all: bool = False,
timeout: Optional[str] = None,
) -> List[ClusterNode]:
pass
@abstractmethod
def parts(
self,
cid: str,
oid: str,
alive_node: ClusterNode,
bearer: str = "",
xhdr: Optional[dict] = None,
is_direct: bool = False,
verify_presence_all: bool = False,
timeout: Optional[str] = None,
) -> List[str]:
pass
class ContainerInterface(ABC):
@abstractmethod
def create(
self,
endpoint: str,
nns_zone: Optional[str] = None,
nns_name: Optional[str] = None,
address: Optional[str] = None,
attributes: Optional[dict] = None,
basic_acl: Optional[str] = None,
await_mode: bool = False,
disable_timestamp: bool = False,
force: bool = False,
trace: bool = False,
name: Optional[str] = None,
nonce: Optional[str] = None,
policy: Optional[str] = None,
session: Optional[str] = None,
subnet: Optional[str] = None,
ttl: Optional[int] = None,
xhdr: Optional[dict] = None,
timeout: Optional[str] = None,
) -> str:
"""
Create a new container and register it in the FrostFS.
It will be stored in the sidechain when the Inner Ring accepts it.
"""
raise NotImplementedError("No implemethed method create")
@abstractmethod
def delete(
self,
endpoint: str,
cid: str,
address: Optional[str] = None,
await_mode: bool = False,
session: Optional[str] = None,
ttl: Optional[int] = None,
xhdr: Optional[dict] = None,
force: bool = False,
trace: bool = False,
) -> List[str]:
"""
Delete an existing container.
Only the owner of the container has permission to remove the container.
"""
raise NotImplementedError("No implemethed method delete")
@abstractmethod
def get(
self,
endpoint: str,
cid: str,
address: Optional[str] = None,
generate_key: Optional[bool] = None,
await_mode: bool = False,
to: Optional[str] = None,
json_mode: bool = True,
trace: bool = False,
ttl: Optional[int] = None,
xhdr: Optional[dict] = None,
timeout: Optional[str] = None,
) -> List[str]:
"""Get container field info."""
raise NotImplementedError("No implemethed method get")
@abstractmethod
def get_eacl(
self,
endpoint: str,
cid: str,
address: Optional[str] = None,
generate_key: Optional[bool] = None,
await_mode: bool = False,
json_mode: bool = True,
trace: bool = False,
to: Optional[str] = None,
session: Optional[str] = None,
ttl: Optional[int] = None,
xhdr: Optional[dict] = None,
timeout: Optional[str] = None,
) -> List[str]:
"""Get extended ACL table of container."""
raise NotImplementedError("No implemethed method get-eacl")
@abstractmethod
def list(
self,
endpoint: str,
name: Optional[str] = None,
address: Optional[str] = None,
generate_key: Optional[bool] = None,
trace: bool = False,
owner: Optional[str] = None,
ttl: Optional[int] = None,
xhdr: Optional[dict] = None,
timeout: Optional[str] = None,
**params,
) -> List[str]:
"""List all created containers."""
raise NotImplementedError("No implemethed method list")
@abstractmethod
def nodes(
self,
endpoint: str,
cid: str,
cluster: Cluster,
address: Optional[str] = None,
ttl: Optional[int] = None,
from_file: Optional[str] = None,
trace: bool = False,
short: Optional[bool] = True,
xhdr: Optional[dict] = None,
generate_key: Optional[bool] = None,
timeout: Optional[str] = None,
) -> List[ClusterNode]:
"""Show the nodes participating in the container in the current epoch."""
raise NotImplementedError("No implemethed method nodes")
class GrpcClientWrapper(ABC):
def __init__(self) -> None:
self.object: ObjectInterface
self.container: ContainerInterface

View file

@ -1,4 +0,0 @@
from .chunks import ChunksInterface
from .container import ContainerInterface
from .netmap import NetmapInterface
from .object import ObjectInterface

View file

@ -1,79 +0,0 @@
from abc import ABC, abstractmethod
from typing import Optional
from frostfs_testlib.storage.cluster import Cluster, ClusterNode
from frostfs_testlib.storage.dataclasses.storage_object_info import Chunk, NodeNetmapInfo
class ChunksInterface(ABC):
@abstractmethod
def search_node_without_chunks(self, chunks: list[Chunk], cluster: Cluster, endpoint: str = None) -> list[ClusterNode]:
pass
@abstractmethod
def get_chunk_node(self, cluster: Cluster, chunk: Chunk) -> tuple[ClusterNode, NodeNetmapInfo]:
pass
@abstractmethod
def get_shard_chunk(self, node: ClusterNode, chunk: Chunk) -> str:
pass
@abstractmethod
def get_all(
self,
rpc_endpoint: str,
cid: str,
oid: str,
wallet: Optional[str] = None,
address: Optional[str] = None,
bearer: Optional[str] = None,
generate_key: Optional[bool] = None,
trace: bool = False,
root: bool = False,
verify_presence_all: bool = False,
json: bool = True,
ttl: Optional[int] = None,
xhdr: Optional[dict] = None,
timeout: Optional[str] = None,
) -> list[Chunk]:
pass
@abstractmethod
def get_parity(
self,
rpc_endpoint: str,
cid: str,
wallet: Optional[str] = None,
address: Optional[str] = None,
bearer: Optional[str] = None,
generate_key: Optional[bool] = None,
oid: Optional[str] = None,
trace: bool = False,
root: bool = False,
verify_presence_all: bool = False,
json: bool = True,
ttl: Optional[int] = None,
xhdr: Optional[dict] = None,
timeout: Optional[str] = None,
) -> Chunk:
pass
@abstractmethod
def get_first_data(
self,
rpc_endpoint: str,
cid: str,
wallet: Optional[str] = None,
address: Optional[str] = None,
bearer: Optional[str] = None,
generate_key: Optional[bool] = None,
oid: Optional[str] = None,
trace: bool = False,
root: bool = False,
verify_presence_all: bool = False,
json: bool = True,
ttl: Optional[int] = None,
xhdr: Optional[dict] = None,
timeout: Optional[str] = None,
) -> Chunk:
pass

View file

@ -1,125 +0,0 @@
from abc import ABC, abstractmethod
from typing import List, Optional
from frostfs_testlib.storage.cluster import Cluster, ClusterNode
class ContainerInterface(ABC):
@abstractmethod
def create(
self,
endpoint: str,
nns_zone: Optional[str] = None,
nns_name: Optional[str] = None,
address: Optional[str] = None,
attributes: Optional[dict] = None,
basic_acl: Optional[str] = None,
await_mode: bool = False,
disable_timestamp: bool = False,
force: bool = False,
trace: bool = False,
name: Optional[str] = None,
nonce: Optional[str] = None,
policy: Optional[str] = None,
session: Optional[str] = None,
subnet: Optional[str] = None,
ttl: Optional[int] = None,
xhdr: Optional[dict] = None,
timeout: Optional[str] = None,
) -> str:
"""
Create a new container and register it in the FrostFS.
It will be stored in the sidechain when the Inner Ring accepts it.
"""
raise NotImplementedError("No implemethed method create")
@abstractmethod
def delete(
self,
endpoint: str,
cid: str,
address: Optional[str] = None,
await_mode: bool = False,
session: Optional[str] = None,
ttl: Optional[int] = None,
xhdr: Optional[dict] = None,
force: bool = False,
trace: bool = False,
) -> List[str]:
"""
Delete an existing container.
Only the owner of the container has permission to remove the container.
"""
raise NotImplementedError("No implemethed method delete")
@abstractmethod
def get(
self,
endpoint: str,
cid: str,
address: Optional[str] = None,
generate_key: Optional[bool] = None,
await_mode: bool = False,
to: Optional[str] = None,
json_mode: bool = True,
trace: bool = False,
ttl: Optional[int] = None,
xhdr: Optional[dict] = None,
timeout: Optional[str] = None,
) -> List[str]:
"""Get container field info."""
raise NotImplementedError("No implemethed method get")
@abstractmethod
def get_eacl(
self,
endpoint: str,
cid: str,
address: Optional[str] = None,
generate_key: Optional[bool] = None,
await_mode: bool = False,
json_mode: bool = True,
trace: bool = False,
to: Optional[str] = None,
session: Optional[str] = None,
ttl: Optional[int] = None,
xhdr: Optional[dict] = None,
timeout: Optional[str] = None,
) -> List[str]:
"""Get extended ACL table of container."""
raise NotImplementedError("No implemethed method get-eacl")
@abstractmethod
def list(
self,
endpoint: str,
name: Optional[str] = None,
address: Optional[str] = None,
generate_key: Optional[bool] = None,
trace: bool = False,
owner: Optional[str] = None,
ttl: Optional[int] = None,
xhdr: Optional[dict] = None,
timeout: Optional[str] = None,
**params,
) -> List[str]:
"""List all created containers."""
raise NotImplementedError("No implemethed method list")
@abstractmethod
def nodes(
self,
endpoint: str,
cid: str,
cluster: Cluster,
address: Optional[str] = None,
ttl: Optional[int] = None,
from_file: Optional[str] = None,
trace: bool = False,
short: Optional[bool] = True,
xhdr: Optional[dict] = None,
generate_key: Optional[bool] = None,
timeout: Optional[str] = None,
) -> List[ClusterNode]:
"""Show the nodes participating in the container in the current epoch."""
raise NotImplementedError("No implemethed method nodes")

View file

@ -1,89 +0,0 @@
from abc import ABC, abstractmethod
from typing import List, Optional
from frostfs_testlib.storage.dataclasses.storage_object_info import NodeNetInfo, NodeNetmapInfo
class NetmapInterface(ABC):
@abstractmethod
def epoch(
self,
rpc_endpoint: str,
wallet: Optional[str] = None,
address: Optional[str] = None,
generate_key: bool = False,
ttl: Optional[int] = None,
trace: Optional[bool] = False,
xhdr: Optional[dict] = None,
timeout: Optional[str] = None,
) -> int:
"""
Get current epoch number.
"""
raise NotImplementedError("No implemethed method epoch")
@abstractmethod
def netinfo(
self,
rpc_endpoint: str,
wallet: Optional[str] = None,
address: Optional[str] = None,
generate_key: bool = False,
json: bool = True,
ttl: Optional[int] = None,
xhdr: Optional[dict] = None,
timeout: Optional[str] = None,
) -> NodeNetInfo:
"""
Get target node info.
"""
raise NotImplementedError("No implemethed method netinfo")
@abstractmethod
def nodeinfo(
self,
rpc_endpoint: str,
wallet: Optional[str] = None,
address: Optional[str] = None,
generate_key: bool = False,
json: bool = True,
ttl: Optional[int] = None,
xhdr: Optional[dict] = None,
timeout: Optional[str] = None,
) -> NodeNetmapInfo:
"""
Get target node info.
"""
raise NotImplementedError("No implemethed method nodeinfo")
@abstractmethod
def snapshot(
self,
rpc_endpoint: str,
wallet: Optional[str] = None,
address: Optional[str] = None,
generate_key: bool = False,
ttl: Optional[int] = None,
xhdr: Optional[dict] = None,
timeout: Optional[str] = None,
) -> List[NodeNetmapInfo]:
"""
Get target node info.
"""
raise NotImplementedError("No implemethed method snapshot")
@abstractmethod
def snapshot_one_node(
self,
rpc_endpoint: str,
wallet: Optional[str] = None,
address: Optional[str] = None,
generate_key: bool = False,
ttl: Optional[int] = None,
xhdr: Optional[dict] = None,
timeout: Optional[str] = None,
) -> List[NodeNetmapInfo]:
"""
Get target one node info.
"""
raise NotImplementedError("No implemethed method snapshot")

View file

@ -1,223 +0,0 @@
from abc import ABC, abstractmethod
from typing import Any, List, Optional
from frostfs_testlib.shell.interfaces import CommandResult
from frostfs_testlib.storage.cluster import Cluster, ClusterNode
from frostfs_testlib.utils import file_utils
from .chunks import ChunksInterface
class ObjectInterface(ABC):
def __init__(self) -> None:
self.chunks: ChunksInterface
@abstractmethod
def delete(
self,
cid: str,
oid: str,
endpoint: str,
bearer: str = "",
xhdr: Optional[dict] = None,
session: Optional[str] = None,
timeout: Optional[str] = None,
) -> str:
pass
@abstractmethod
def get(
self,
cid: str,
oid: str,
endpoint: str,
bearer: Optional[str] = None,
write_object: Optional[str] = None,
xhdr: Optional[dict] = None,
no_progress: bool = True,
session: Optional[str] = None,
timeout: Optional[str] = None,
) -> file_utils.TestFile:
pass
@abstractmethod
def get_from_random_node(
self,
cid: str,
oid: str,
cluster: Cluster,
bearer: Optional[str] = None,
write_object: Optional[str] = None,
xhdr: Optional[dict] = None,
no_progress: bool = True,
session: Optional[str] = None,
timeout: Optional[str] = None,
) -> str:
pass
@abstractmethod
def hash(
self,
endpoint: str,
cid: str,
oid: str,
address: Optional[str] = None,
bearer: Optional[str] = None,
generate_key: Optional[bool] = None,
range: Optional[str] = None,
salt: Optional[str] = None,
ttl: Optional[int] = None,
session: Optional[str] = None,
hash_type: Optional[str] = None,
xhdr: Optional[dict] = None,
timeout: Optional[str] = None,
) -> str:
pass
@abstractmethod
def head(
self,
cid: str,
oid: str,
endpoint: str,
bearer: str = "",
xhdr: Optional[dict] = None,
json_output: bool = True,
is_raw: bool = False,
is_direct: bool = False,
session: Optional[str] = None,
timeout: Optional[str] = None,
) -> CommandResult | Any:
pass
@abstractmethod
def lock(
self,
cid: str,
oid: str,
endpoint: str,
lifetime: Optional[int] = None,
expire_at: Optional[int] = None,
address: Optional[str] = None,
bearer: Optional[str] = None,
session: Optional[str] = None,
ttl: Optional[int] = None,
xhdr: Optional[dict] = None,
timeout: Optional[str] = None,
) -> str:
pass
@abstractmethod
def put(
self,
path: str,
cid: str,
endpoint: str,
bearer: Optional[str] = None,
copies_number: Optional[int] = None,
attributes: Optional[dict] = None,
xhdr: Optional[dict] = None,
expire_at: Optional[int] = None,
no_progress: bool = True,
session: Optional[str] = None,
timeout: Optional[str] = None,
) -> str:
pass
@abstractmethod
def patch(
self,
cid: str,
oid: str,
endpoint: str,
ranges: Optional[list[str]] = None,
payloads: Optional[list[str]] = None,
new_attrs: Optional[str] = None,
replace_attrs: bool = False,
bearer: Optional[str] = None,
xhdr: Optional[dict] = None,
session: Optional[str] = None,
timeout: Optional[str] = None,
trace: bool = False,
) -> str:
pass
@abstractmethod
def put_to_random_node(
self,
path: str,
cid: str,
cluster: Cluster,
bearer: Optional[str] = None,
copies_number: Optional[int] = None,
attributes: Optional[dict] = None,
xhdr: Optional[dict] = None,
expire_at: Optional[int] = None,
no_progress: bool = True,
session: Optional[str] = None,
timeout: Optional[str] = None,
) -> str:
pass
@abstractmethod
def range(
self,
cid: str,
oid: str,
range_cut: str,
endpoint: str,
bearer: str = "",
xhdr: Optional[dict] = None,
session: Optional[str] = None,
timeout: Optional[str] = None,
) -> tuple[file_utils.TestFile, bytes]:
pass
@abstractmethod
def search(
self,
cid: str,
endpoint: str,
bearer: str = "",
oid: Optional[str] = None,
filters: Optional[dict] = None,
expected_objects_list: Optional[list] = None,
xhdr: Optional[dict] = None,
session: Optional[str] = None,
phy: bool = False,
root: bool = False,
timeout: Optional[str] = None,
address: Optional[str] = None,
generate_key: Optional[bool] = None,
ttl: Optional[int] = None,
) -> List:
pass
@abstractmethod
def nodes(
self,
cluster: Cluster,
cid: str,
oid: str,
alive_node: ClusterNode,
bearer: str = "",
xhdr: Optional[dict] = None,
is_direct: bool = False,
verify_presence_all: bool = False,
timeout: Optional[str] = None,
) -> List[ClusterNode]:
pass
@abstractmethod
def parts(
self,
cid: str,
oid: str,
alive_node: ClusterNode,
bearer: str = "",
xhdr: Optional[dict] = None,
is_direct: bool = False,
verify_presence_all: bool = False,
timeout: Optional[str] = None,
) -> List[str]:
pass

View file

@ -1,10 +0,0 @@
from abc import ABC
from . import interfaces
class GrpcClientWrapper(ABC):
def __init__(self) -> None:
self.object: interfaces.ObjectInterface
self.container: interfaces.ContainerInterface
self.netmap: interfaces.NetmapInterface

View file

@ -64,7 +64,7 @@ def parallel_binary_verions(host: Host) -> dict[str, str]:
try: try:
result = shell.exec(f"{binary_path} {binary['param']}") result = shell.exec(f"{binary_path} {binary['param']}")
version = parse_version(result.stdout) or parse_version(result.stderr) or "Unknown" version = parse_version(result.stdout) or parse_version(result.stderr) or "Unknown"
versions_at_host[binary_name] = version.strip() versions_at_host[binary_name] = version
except Exception as exc: except Exception as exc:
logger.error(f"Cannot get version for {binary_path} because of\n{exc}") logger.error(f"Cannot get version for {binary_path} because of\n{exc}")
versions_at_host[binary_name] = "Unknown" versions_at_host[binary_name] = "Unknown"

View file

@ -6,7 +6,10 @@ import pytest
from frostfs_testlib.load.load_config import EndpointSelectionStrategy, LoadParams, LoadScenario, LoadType, Preset, ReadFrom from frostfs_testlib.load.load_config import EndpointSelectionStrategy, LoadParams, LoadScenario, LoadType, Preset, ReadFrom
from frostfs_testlib.load.runners import DefaultRunner from frostfs_testlib.load.runners import DefaultRunner
from frostfs_testlib.resources.load_params import BACKGROUND_LOAD_DEFAULT_VU_INIT_TIME from frostfs_testlib.resources.load_params import BACKGROUND_LOAD_DEFAULT_VU_INIT_TIME
from frostfs_testlib.storage.cluster import ClusterNode
from frostfs_testlib.storage.controllers.background_load_controller import BackgroundLoadController from frostfs_testlib.storage.controllers.background_load_controller import BackgroundLoadController
from frostfs_testlib.storage.dataclasses.frostfs_services import StorageNode
from frostfs_testlib.storage.dataclasses.node_base import NodeBase
@dataclass @dataclass
@ -126,8 +129,6 @@ class TestLoadConfig:
"--size '11'", "--size '11'",
"--acl 'acl'", "--acl 'acl'",
"--preload_obj '13'", "--preload_obj '13'",
"--retry '24'",
"--rule 'rule' --rule 'rule_2'",
"--out 'pregen_json'", "--out 'pregen_json'",
"--workers '7'", "--workers '7'",
"--containers '16'", "--containers '16'",
@ -160,8 +161,6 @@ class TestLoadConfig:
expected_preset_args = [ expected_preset_args = [
"--size '11'", "--size '11'",
"--preload_obj '13'", "--preload_obj '13'",
"--retry '24'",
"--rule 'rule' --rule 'rule_2'",
"--out 'pregen_json'", "--out 'pregen_json'",
"--workers '7'", "--workers '7'",
"--containers '16'", "--containers '16'",
@ -318,8 +317,6 @@ class TestLoadConfig:
"--no-verify-ssl", "--no-verify-ssl",
"--size '11'", "--size '11'",
"--preload_obj '13'", "--preload_obj '13'",
"--retry '24'",
"--rule 'rule' --rule 'rule_2'",
"--out 'pregen_json'", "--out 'pregen_json'",
"--workers '7'", "--workers '7'",
"--containers '16'", "--containers '16'",
@ -353,8 +350,6 @@ class TestLoadConfig:
expected_preset_args = [ expected_preset_args = [
"--size '11'", "--size '11'",
"--preload_obj '13'", "--preload_obj '13'",
"--retry '24'",
"--rule 'rule' --rule 'rule_2'",
"--out 'pregen_json'", "--out 'pregen_json'",
"--workers '7'", "--workers '7'",
"--containers '16'", "--containers '16'",
@ -420,26 +415,6 @@ class TestLoadConfig:
self._check_preset_params(load_params, params) self._check_preset_params(load_params, params)
@pytest.mark.parametrize(
"load_type, input, value, params",
[
(LoadType.gRPC, ["A C ", " B"], ["A C", "B"], [f"--rule 'A C' --rule 'B'"]),
(LoadType.gRPC, " A ", ["A"], ["--rule 'A'"]),
(LoadType.gRPC, " A , B ", ["A , B"], ["--rule 'A , B'"]),
(LoadType.gRPC, [" A", "B "], ["A", "B"], ["--rule 'A' --rule 'B'"]),
(LoadType.gRPC, None, None, []),
(LoadType.S3, ["A C ", " B"], ["A C", "B"], []),
(LoadType.S3, None, None, []),
],
)
def test_ape_list_parsing_formatter(self, load_type, input, value, params):
load_params = LoadParams(load_type)
load_params.preset = Preset()
load_params.preset.rule = input
assert load_params.preset.rule == value
self._check_preset_params(load_params, params)
@pytest.mark.parametrize("load_params, load_type", [(LoadScenario.VERIFY, LoadType.S3)], indirect=True) @pytest.mark.parametrize("load_params, load_type", [(LoadScenario.VERIFY, LoadType.S3)], indirect=True)
def test_argument_parsing_for_s3_verify_scenario(self, load_params: LoadParams): def test_argument_parsing_for_s3_verify_scenario(self, load_params: LoadParams):
expected_env_vars = { expected_env_vars = {
@ -469,8 +444,6 @@ class TestLoadConfig:
expected_preset_args = [ expected_preset_args = [
"--size '0'", "--size '0'",
"--preload_obj '0'", "--preload_obj '0'",
"--retry '0'",
"--rule ''",
"--out ''", "--out ''",
"--workers '0'", "--workers '0'",
"--containers '0'", "--containers '0'",
@ -502,8 +475,6 @@ class TestLoadConfig:
expected_preset_args = [ expected_preset_args = [
"--size '0'", "--size '0'",
"--preload_obj '0'", "--preload_obj '0'",
"--retry '0'",
"--rule ''",
"--out ''", "--out ''",
"--workers '0'", "--workers '0'",
"--containers '0'", "--containers '0'",
@ -611,8 +582,6 @@ class TestLoadConfig:
expected_preset_args = [ expected_preset_args = [
"--size '0'", "--size '0'",
"--preload_obj '0'", "--preload_obj '0'",
"--retry '0'",
"--rule ''",
"--out ''", "--out ''",
"--workers '0'", "--workers '0'",
"--containers '0'", "--containers '0'",
@ -644,8 +613,6 @@ class TestLoadConfig:
expected_preset_args = [ expected_preset_args = [
"--size '0'", "--size '0'",
"--preload_obj '0'", "--preload_obj '0'",
"--retry '0'",
"--rule ''",
"--out ''", "--out ''",
"--workers '0'", "--workers '0'",
"--containers '0'", "--containers '0'",