ChEktn/frostfs-testlib
1
0
Fork 0
forked from TrueCloudLab/frostfs-testlib
Code Pull requests Activity

[#191] Credentials work overhaul

Browse source 
Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
This commit is contained in:
Andrey Berezin 2024-03-11 19:23:10 +03:00
parent 09a7f66d1e
commit 25925c637b
31 changed files with 370 additions and 485 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
pyproject.toml
View file

@ -58,15 +58,19 @@ neo-go = "frostfs_testlib.storage.dataclasses.frostfs_services:MorphChain"
frostfs-ir = "frostfs_testlib.storage.dataclasses.frostfs_services:InnerRing"
[project.entry-points."frostfs.testlib.credentials_providers"]
authmate = "frostfs_testlib.credentials.authmate_s3:AuthmateS3CredentialsProvider"
authmate = "frostfs_testlib.credentials.authmate_s3_provider:AuthmateS3CredentialsProvider"
wallet_factory = "frostfs_testlib.credentials.wallet_factory_provider:WalletFactoryProvider"
[project.entry-points."frostfs.testlib.bucket_cid_resolver"]
frostfs = "frostfs_testlib.s3.curl_bucket_resolver:CurlBucketContainerResolver"
[tool.isort]
profile = "black"
src_paths = ["src", "tests"]
line_length = 120
line_length = 140
[tool.black]
line-length = 120
line-length = 140
target-version = ["py310"]
[tool.bumpver]

16
src/frostfs_testlib/cli/frostfs_cli/container.py
View file

@ -8,7 +8,7 @@ class FrostfsCliContainer(CliCommand):
def create(
self,
rpc_endpoint: str,
wallet: str,
wallet: Optional[str] = None,
address: Optional[str] = None,
attributes: Optional[dict] = None,
basic_acl: Optional[str] = None,
@ -57,8 +57,8 @@ class FrostfsCliContainer(CliCommand):
def delete(
self,
rpc_endpoint: str,
wallet: str,
cid: str,
wallet: Optional[str] = None,
address: Optional[str] = None,
await_mode: bool = False,
session: Optional[str] = None,
@ -93,8 +93,8 @@ class FrostfsCliContainer(CliCommand):
def get(
self,
rpc_endpoint: str,
wallet: str,
cid: str,
wallet: Optional[str] = None,
address: Optional[str] = None,
await_mode: bool = False,
to: Optional[str] = None,
@ -129,8 +129,8 @@ class FrostfsCliContainer(CliCommand):
def get_eacl(
self,
rpc_endpoint: str,
wallet: str,
cid: str,
wallet: Optional[str] = None,
address: Optional[str] = None,
await_mode: bool = False,
to: Optional[str] = None,
@ -166,7 +166,7 @@ class FrostfsCliContainer(CliCommand):
def list(
self,
rpc_endpoint: str,
wallet: str,
wallet: Optional[str] = None,
address: Optional[str] = None,
owner: Optional[str] = None,
ttl: Optional[int] = None,
@ -197,8 +197,8 @@ class FrostfsCliContainer(CliCommand):
def list_objects(
self,
rpc_endpoint: str,
wallet: str,
cid: str,
wallet: Optional[str] = None,
address: Optional[str] = None,
ttl: Optional[int] = None,
xhdr: Optional[dict] = None,
@ -227,8 +227,8 @@ class FrostfsCliContainer(CliCommand):
def set_eacl(
self,
rpc_endpoint: str,
wallet: str,
cid: str,
wallet: Optional[str] = None,
address: Optional[str] = None,
await_mode: bool = False,
table: Optional[str] = None,
@ -264,8 +264,8 @@ class FrostfsCliContainer(CliCommand):
def search_node(
self,
rpc_endpoint: str,
wallet: str,
cid: str,
wallet: Optional[str] = None,
address: Optional[str] = None,
ttl: Optional[int] = None,
from_file: Optional[str] = None,

8
src/frostfs_testlib/cli/frostfs_cli/netmap.py
View file

@ -8,7 +8,7 @@ class FrostfsCliNetmap(CliCommand):
def epoch(
self,
rpc_endpoint: str,
wallet: str,
wallet: Optional[str] = None,
address: Optional[str] = None,
generate_key: bool = False,
ttl: Optional[int] = None,
@ -38,7 +38,7 @@ class FrostfsCliNetmap(CliCommand):
def netinfo(
self,
rpc_endpoint: str,
wallet: str,
wallet: Optional[str] = None,
address: Optional[str] = None,
generate_key: bool = False,
ttl: Optional[int] = None,
@ -68,7 +68,7 @@ class FrostfsCliNetmap(CliCommand):
def nodeinfo(
self,
rpc_endpoint: str,
wallet: str,
wallet: Optional[str] = None,
address: Optional[str] = None,
generate_key: bool = False,
json: bool = False,
@ -100,7 +100,7 @@ class FrostfsCliNetmap(CliCommand):
def snapshot(
self,
rpc_endpoint: str,
wallet: str,
wallet: Optional[str] = None,
address: Optional[str] = None,
generate_key: bool = False,
ttl: Optional[int] = None,

16
src/frostfs_testlib/cli/frostfs_cli/object.py
View file

@ -8,9 +8,9 @@ class FrostfsCliObject(CliCommand):
def delete(
self,
rpc_endpoint: str,
wallet: str,
cid: str,
oid: str,
wallet: Optional[str] = None,
address: Optional[str] = None,
bearer: Optional[str] = None,
session: Optional[str] = None,
@ -44,9 +44,9 @@ class FrostfsCliObject(CliCommand):
def get(
self,
rpc_endpoint: str,
wallet: str,
cid: str,
oid: str,
wallet: Optional[str] = None,
address: Optional[str] = None,
bearer: Optional[str] = None,
file: Optional[str] = None,
@ -88,9 +88,9 @@ class FrostfsCliObject(CliCommand):
def hash(
self,
rpc_endpoint: str,
wallet: str,
cid: str,
oid: str,
wallet: Optional[str] = None,
address: Optional[str] = None,
bearer: Optional[str] = None,
range: Optional[str] = None,
@ -130,9 +130,9 @@ class FrostfsCliObject(CliCommand):
def head(
self,
rpc_endpoint: str,
wallet: str,
cid: str,
oid: str,
wallet: Optional[str] = None,
address: Optional[str] = None,
bearer: Optional[str] = None,
file: Optional[str] = None,
@ -176,9 +176,9 @@ class FrostfsCliObject(CliCommand):
def lock(
self,
rpc_endpoint: str,
wallet: str,
cid: str,
oid: str,
wallet: Optional[str] = None,
lifetime: Optional[int] = None,
expire_at: Optional[int] = None,
address: Optional[str] = None,
@ -216,9 +216,9 @@ class FrostfsCliObject(CliCommand):
def put(
self,
rpc_endpoint: str,
wallet: str,
cid: str,
file: str,
wallet: Optional[str] = None,
address: Optional[str] = None,
attributes: Optional[dict] = None,
bearer: Optional[str] = None,
@ -267,10 +267,10 @@ class FrostfsCliObject(CliCommand):
def range(
self,
rpc_endpoint: str,
wallet: str,
cid: str,
oid: str,
range: str,
wallet: Optional[str] = None,
address: Optional[str] = None,
bearer: Optional[str] = None,
file: Optional[str] = None,
@ -311,8 +311,8 @@ class FrostfsCliObject(CliCommand):
def search(
self,
rpc_endpoint: str,
wallet: str,
cid: str,
wallet: Optional[str] = None,
address: Optional[str] = None,
bearer: Optional[str] = None,
filters: Optional[list] = None,

10
src/frostfs_testlib/cli/frostfs_cli/session.py
View file

@ -9,7 +9,6 @@ class FrostfsCliSession(CliCommand):
self,
rpc_endpoint: str,
wallet: str,
wallet_password: str,
out: str,
lifetime: Optional[int] = None,
address: Optional[str] = None,
@ -30,12 +29,7 @@ class FrostfsCliSession(CliCommand):
Returns:
Command's result.
"""
return self._execute_with_password(
return self._execute(
"session create",
wallet_password,
**{
param: value
for param, value in locals().items()
if param not in ["self", "wallet_password"]
},
**{param: value for param, value in locals().items() if param not in ["self"]},
)

4
src/frostfs_testlib/cli/frostfs_cli/util.py
View file

@ -7,9 +7,9 @@ from frostfs_testlib.shell import CommandResult
class FrostfsCliUtil(CliCommand):
def sign_bearer_token(
self,
wallet: str,
from_file: str,
to_file: str,
wallet: Optional[str] = None,
address: Optional[str] = None,
json: Optional[bool] = False,
) -> CommandResult:
@ -33,9 +33,9 @@ class FrostfsCliUtil(CliCommand):
def sign_session_token(
self,
wallet: str,
from_file: str,
to_file: str,
wallet: Optional[str] = None,
address: Optional[str] = None,
) -> CommandResult:
"""

32
src/frostfs_testlib/credentials/authmate_s3.py → src/frostfs_testlib/credentials/authmate_s3_provider.py
View file

@ -1,25 +1,26 @@
import re
from datetime import datetime
from typing import Optional
from frostfs_testlib import reporter
from frostfs_testlib.cli import FrostfsAuthmate
from frostfs_testlib.credentials.interfaces import S3CredentialsProvider
from frostfs_testlib.credentials.interfaces import S3Credentials, S3CredentialsProvider, User
from frostfs_testlib.resources.cli import FROSTFS_AUTHMATE_EXEC
from frostfs_testlib.shell import Shell
from frostfs_testlib.shell import LocalShell
from frostfs_testlib.steps.cli.container import list_containers
from frostfs_testlib.storage.cluster import Cluster, ClusterNode
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.storage.cluster import ClusterNode
from frostfs_testlib.storage.dataclasses.frostfs_services import S3Gate
class AuthmateS3CredentialsProvider(S3CredentialsProvider):
@reporter.step("Init S3 Credentials using Authmate CLI")
def provide(self, cluster_node: ClusterNode) -> tuple[str, str]:
cluster: Cluster = self.stash["cluster"]
shell: Shell = self.stash["shell"]
wallet: WalletInfo = self.stash["wallet"]
def provide(self, user: User, cluster_node: ClusterNode, location_constraints: Optional[str] = None) -> S3Credentials:
cluster_nodes: list[ClusterNode] = self.cluster.cluster_nodes
shell = LocalShell()
wallet = user.wallet
endpoint = cluster_node.storage_node.get_rpc_endpoint()
gate_public_keys = [s3gate.get_wallet_public_key() for s3gate in cluster.s3_gates]
gate_public_keys = [node.service(S3Gate).get_wallet_public_key() for node in cluster_nodes]
# unique short bucket name
bucket = f"bucket_{hex(int(datetime.now().timestamp()*1000000))}"
@ -29,21 +30,18 @@ class AuthmateS3CredentialsProvider(S3CredentialsProvider):
peer=endpoint,
gate_public_key=gate_public_keys,
wallet_password=wallet.password,
container_policy=self.stash.get("location_constraints"),
container_policy=location_constraints,
container_friendly_name=bucket,
).stdout
aws_access_key_id = str(
re.search(r"access_key_id.*:\s.(?P<aws_access_key_id>\w*)", issue_secret_output).group("aws_access_key_id")
)
aws_access_key_id = str(re.search(r"access_key_id.*:\s.(?P<aws_access_key_id>\w*)", issue_secret_output).group("aws_access_key_id"))
aws_secret_access_key = str(
re.search(r"secret_access_key.*:\s.(?P<aws_secret_access_key>\w*)", issue_secret_output).group(
"aws_secret_access_key"
)
re.search(r"secret_access_key.*:\s.(?P<aws_secret_access_key>\w*)", issue_secret_output).group("aws_secret_access_key")
)
cid = str(re.search(r"container_id.*:\s.(?P<container_id>\w*)", issue_secret_output).group("container_id"))
containers_list = list_containers(wallet.path, shell, endpoint)
assert cid in containers_list, f"Expected cid {cid} in {containers_list}"
return aws_access_key_id, aws_secret_access_key
user.s3_credentials = S3Credentials(aws_access_key_id, aws_secret_access_key)
return user.s3_credentials

50
src/frostfs_testlib/credentials/interfaces.py
View file

@ -1,25 +1,51 @@
from abc import abstractmethod
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from typing import Any, Optional
from frostfs_testlib.plugins import load_plugin
from frostfs_testlib.storage.cluster import ClusterNode
from frostfs_testlib.storage.cluster import Cluster, ClusterNode
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
class S3CredentialsProvider(object):
stash: dict
@dataclass
class S3Credentials:
access_key: str
secret_key: str
def __init__(self, stash: dict) -> None:
self.stash = stash
@dataclass
class User:
name: str
attributes: dict[str, Any] = field(default_factory=dict)
wallet: WalletInfo | None = None
s3_credentials: S3Credentials | None = None
class S3CredentialsProvider(ABC):
def __init__(self, cluster: Cluster) -> None:
self.cluster = cluster
@abstractmethod
def provide(self, cluster_node: ClusterNode) -> tuple[str, str]:
def provide(self, user: User, cluster_node: ClusterNode, location_constraints: Optional[str] = None) -> S3Credentials:
raise NotImplementedError("Directly called abstract class?")
class GrpcCredentialsProvider(ABC):
def __init__(self, cluster: Cluster) -> None:
self.cluster = cluster
@abstractmethod
def provide(self, user: User, cluster_node: ClusterNode) -> WalletInfo:
raise NotImplementedError("Directly called abstract class?")
class CredentialsProvider(object):
stash: dict
S3: S3CredentialsProvider
GRPC: GrpcCredentialsProvider
def __init__(self, s3_plugin_name: str) -> None:
self.stash = {}
s3cls = load_plugin("frostfs.testlib.credentials_providers", s3_plugin_name)
self.S3 = s3cls(self.stash)
def __init__(self, cluster: Cluster) -> None:
config = cluster.cluster_nodes[0].host.config
s3_cls = load_plugin("frostfs.testlib.credentials_providers", config.s3_creds_plugin_name)
self.S3 = s3_cls(cluster)
grpc_cls = load_plugin("frostfs.testlib.credentials_providers", config.grpc_creds_plugin_name)
self.GRPC = grpc_cls(cluster)

14
src/frostfs_testlib/credentials/wallet_factory_provider.py Normal file
View file

@ -0,0 +1,14 @@
from frostfs_testlib import reporter
from frostfs_testlib.credentials.interfaces import GrpcCredentialsProvider, User
from frostfs_testlib.resources.common import ASSETS_DIR, DEFAULT_WALLET_PASS
from frostfs_testlib.shell.local_shell import LocalShell
from frostfs_testlib.storage.cluster import ClusterNode
from frostfs_testlib.storage.dataclasses.wallet import WalletFactory, WalletInfo
class WalletFactoryProvider(GrpcCredentialsProvider):
@reporter.step("Init gRPC Credentials using wallet generation")
def provide(self, user: User, cluster_node: ClusterNode) -> WalletInfo:
wallet_factory = WalletFactory(ASSETS_DIR, LocalShell())
user.wallet = wallet_factory.create_wallet(file_name=user, password=DEFAULT_WALLET_PASS)
return user.wallet

2
src/frostfs_testlib/hosting/config.py
View file

@ -63,6 +63,8 @@ class HostConfig:
healthcheck_plugin_name: str
address: str
s3_creds_plugin_name: str = field(default="authmate")
grpc_creds_plugin_name: str = field(default="wallet_factory")
product: str = field(default="frostfs")
services: list[ServiceConfig] = field(default_factory=list)
clis: list[CLIConfig] = field(default_factory=list)
attributes: dict[str, str] = field(default_factory=dict)

34
src/frostfs_testlib/load/k6.py
View file

@ -9,13 +9,13 @@ from typing import Any
from urllib.parse import urlparse
from frostfs_testlib import reporter
from frostfs_testlib.credentials.interfaces import User
from frostfs_testlib.load.interfaces.loader import Loader
from frostfs_testlib.load.load_config import K6ProcessAllocationStrategy, LoadParams, LoadScenario, LoadType
from frostfs_testlib.processes.remote_process import RemoteProcess
from frostfs_testlib.resources.common import STORAGE_USER_NAME
from frostfs_testlib.resources.load_params import K6_STOP_SIGNAL_TIMEOUT, K6_TEARDOWN_PERIOD
from frostfs_testlib.shell import Shell
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.test_control import wait_for_success
EXIT_RESULT_CODE = 0
@ -42,16 +42,16 @@ class K6:
k6_dir: str,
shell: Shell,
loader: Loader,
wallet: WalletInfo,
user: User,
):
if load_params.scenario is None:
raise RuntimeError("Scenario should not be none")
self.load_params: LoadParams = load_params
self.load_params = load_params
self.endpoints = endpoints
self.loader: Loader = loader
self.shell: Shell = shell
self.wallet = wallet
self.loader = loader
self.shell = shell
self.user = user
self.preset_output: str = ""
self.summary_json: str = os.path.join(
self.load_params.working_dir,
@ -64,13 +64,9 @@ class K6:
f"{self._generate_env_variables()}{self._k6_dir}/k6 run {self._generate_k6_variables()} "
f"{self._k6_dir}/scenarios/{self.load_params.scenario.value}.js"
)
user = STORAGE_USER_NAME if self.load_params.scenario == LoadScenario.LOCAL else None
process_id = (
self.load_params.load_id
if self.load_params.scenario != LoadScenario.VERIFY
else f"{self.load_params.load_id}_verify"
)
self._k6_process = RemoteProcess.create(command, self.shell, self.load_params.working_dir, user, process_id)
remote_user = STORAGE_USER_NAME if self.load_params.scenario == LoadScenario.LOCAL else None
process_id = self.load_params.load_id if self.load_params.scenario != LoadScenario.VERIFY else f"{self.load_params.load_id}_verify"
self._k6_process = RemoteProcess.create(command, self.shell, self.load_params.working_dir, remote_user, process_id)
def _get_fill_percents(self):
fill_percents = self.shell.exec("df -H --output=source,pcent,target | grep frostfs").stdout.split("\n")
@ -103,8 +99,8 @@ class K6:
preset_grpc: [
preset_grpc,
f"--endpoint {','.join(self.endpoints)}",
f"--wallet {self.wallet.path} ",
f"--config {self.wallet.config_path} ",
f"--wallet {self.user.wallet.path} ",
f"--config {self.user.wallet.config_path} ",
],
preset_s3: [
preset_s3,
@ -167,9 +163,7 @@ class K6:
remaining_time = timeout - working_time
setup_teardown_time = (
int(K6_TEARDOWN_PERIOD)
+ self.load_params.get_init_time()
+ int(self.load_params.setup_timeout.replace("s", "").strip())
int(K6_TEARDOWN_PERIOD) + self.load_params.get_init_time() + int(self.load_params.setup_timeout.replace("s", "").strip())
)
remaining_time_including_setup_and_teardown = remaining_time + setup_teardown_time
timeout = remaining_time_including_setup_and_teardown
@ -201,9 +195,7 @@ class K6:
if not self.load_params.fill_percent is None:
with reporter.step(f"Check the percentage of filling of all data disks on the node"):
if self.check_fill_percent():
logger.info(
f"Stopping load on because disks is filled more then {self.load_params.fill_percent}%"
)
logger.info(f"Stopping load on because disks is filled more then {self.load_params.fill_percent}%")
event.set()
self.stop()
return

114
src/frostfs_testlib/load/runners.py
View file

@ -1,24 +1,20 @@
import copy
import itertools
import math
import re
import time
from dataclasses import fields
from threading import Event
from typing import Optional
from urllib.parse import urlparse
import yaml
from frostfs_testlib import reporter
from frostfs_testlib.cli.frostfs_authmate.authmate import FrostfsAuthmate
from frostfs_testlib.credentials.interfaces import S3Credentials, User
from frostfs_testlib.load.interfaces.loader import Loader
from frostfs_testlib.load.interfaces.scenario_runner import ScenarioRunner
from frostfs_testlib.load.k6 import K6
from frostfs_testlib.load.load_config import K6ProcessAllocationStrategy, LoadParams, LoadType
from frostfs_testlib.load.loaders import NodeLoader, RemoteLoader
from frostfs_testlib.resources import optionals
from frostfs_testlib.resources.cli import FROSTFS_AUTHMATE_EXEC
from frostfs_testlib.resources.common import STORAGE_USER_NAME
from frostfs_testlib.resources.load_params import BACKGROUND_LOAD_VUS_COUNT_DIVISOR, LOAD_NODE_SSH_USER, LOAD_NODES
from frostfs_testlib.shell.command_inspectors import SuInspector
@ -26,7 +22,6 @@ from frostfs_testlib.shell.interfaces import CommandOptions, InteractiveInput
from frostfs_testlib.storage.cluster import ClusterNode
from frostfs_testlib.storage.controllers.cluster_state_controller import ClusterStateController
from frostfs_testlib.storage.dataclasses.frostfs_services import S3Gate, StorageNode
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing import parallel, run_optionally
from frostfs_testlib.testing.test_control import retry
from frostfs_testlib.utils import datetime_utils
@ -57,17 +52,17 @@ class RunnerBase(ScenarioRunner):
class DefaultRunner(RunnerBase):
loaders: list[Loader]
loaders_wallet: WalletInfo
user: User
def __init__(
self,
loaders_wallet: WalletInfo,
user: User,
load_ip_list: Optional[list[str]] = None,
) -> None:
if load_ip_list is None:
load_ip_list = LOAD_NODES
self.loaders = RemoteLoader.from_ip_list(load_ip_list)
self.loaders_wallet = loaders_wallet
self.user = user
@run_optionally(optionals.OPTIONAL_BACKGROUND_LOAD_ENABLED)
@reporter.step("Preparation steps")
@ -86,55 +81,27 @@ class DefaultRunner(RunnerBase):
return
with reporter.step("Init s3 client on loaders"):
storage_node = nodes_under_load[0].service(StorageNode)
s3_public_keys = [node.service(S3Gate).get_wallet_public_key() for node in cluster_nodes]
grpc_peer = storage_node.get_rpc_endpoint()
parallel(self._prepare_loader, self.loaders, load_params, grpc_peer, s3_public_keys, k6_dir)
s3_credentials = self.user.s3_credentials
parallel(self._aws_configure_on_loader, self.loaders, s3_credentials)
def _force_fresh_registry(self, loader: Loader, load_params: LoadParams):
with reporter.step(f"Forcing fresh registry on {loader.ip}"):
shell = loader.get_shell()
shell.exec(f"rm -f {load_params.registry_file}")
def _prepare_loader(
def _aws_configure_on_loader(
self,
loader: Loader,
load_params: LoadParams,
grpc_peer: str,
s3_public_keys: list[str],
k6_dir: str,
s3_credentials: S3Credentials,
):
with reporter.step(f"Init s3 client on {loader.ip}"):
shell = loader.get_shell()
frostfs_authmate_exec: FrostfsAuthmate = FrostfsAuthmate(shell, FROSTFS_AUTHMATE_EXEC)
issue_secret_output = frostfs_authmate_exec.secret.issue(
wallet=self.loaders_wallet.path,
peer=grpc_peer,
gate_public_key=s3_public_keys,
container_placement_policy=load_params.preset.container_placement_policy,
container_policy=f"{k6_dir}/scenarios/files/policy.json",
wallet_password=self.loaders_wallet.password,
).stdout
aws_access_key_id = str(
re.search(r"access_key_id.*:\s.(?P<aws_access_key_id>\w*)", issue_secret_output).group(
"aws_access_key_id"
)
)
aws_secret_access_key = str(
re.search(
r"secret_access_key.*:\s.(?P<aws_secret_access_key>\w*)",
issue_secret_output,
).group("aws_secret_access_key")
)
with reporter.step(f"Aws configure on {loader.ip}"):
configure_input = [
InteractiveInput(prompt_pattern=r"AWS Access Key ID.*", input=aws_access_key_id),
InteractiveInput(prompt_pattern=r"AWS Secret Access Key.*", input=aws_secret_access_key),
InteractiveInput(prompt_pattern=r"AWS Access Key ID.*", input=s3_credentials.access_key),
InteractiveInput(prompt_pattern=r"AWS Secret Access Key.*", input=s3_credentials.secret_key),
InteractiveInput(prompt_pattern=r".*", input=""),
InteractiveInput(prompt_pattern=r".*", input=""),
]
shell.exec("aws configure", CommandOptions(interactive_inputs=configure_input))
loader.get_shell().exec("aws configure", CommandOptions(interactive_inputs=configure_input))
@reporter.step("Init k6 instances")
def init_k6_instances(self, load_params: LoadParams, endpoints: list[str], k6_dir: str):
@ -176,12 +143,10 @@ class DefaultRunner(RunnerBase):
k6_dir,
shell,
loader,
self.loaders_wallet,
self.user,
)
def _get_distributed_load_params_list(
self, original_load_params: LoadParams, workers_count: int
) -> list[LoadParams]:
def _get_distributed_load_params_list(self, original_load_params: LoadParams, workers_count: int) -> list[LoadParams]:
divisor = int(BACKGROUND_LOAD_VUS_COUNT_DIVISOR)
distributed_load_params: list[LoadParams] = []
@ -266,18 +231,20 @@ class LocalRunner(RunnerBase):
loaders: list[Loader]
cluster_state_controller: ClusterStateController
file_keeper: FileKeeper
wallet: WalletInfo
user: User
def __init__(
self,
cluster_state_controller: ClusterStateController,
file_keeper: FileKeeper,
nodes_under_load: list[ClusterNode],
user: User,
) -> None:
self.cluster_state_controller = cluster_state_controller
self.file_keeper = file_keeper
self.loaders = [NodeLoader(node) for node in nodes_under_load]
self.nodes_under_load = nodes_under_load
self.user = user
@run_optionally(optionals.OPTIONAL_BACKGROUND_LOAD_ENABLED)
@reporter.step("Preparation steps")
@ -326,11 +293,9 @@ class LocalRunner(RunnerBase):
shell.exec(f"sudo tar xf {k6_dir}/k6.tar.gz --strip-components 2 -C {k6_dir}")
shell.exec(f"sudo chmod -R 777 {k6_dir}")
with reporter.step("Create empty_passwd"):
self.wallet = WalletInfo(f"{k6_dir}/scenarios/files/wallet.json", "", "/tmp/empty_passwd.yml")
content = yaml.dump({"password": ""})
shell.exec(f'echo "{content}" | sudo tee {self.wallet.config_path}')
shell.exec(f"sudo chmod -R 777 {self.wallet.config_path}")
with reporter.step("chmod 777 wallet related files on loader"):
shell.exec(f"sudo chmod -R 777 {self.user.wallet.config_path}")
shell.exec(f"sudo chmod -R 777 {self.user.wallet.path}")
@reporter.step("Init k6 instances")
def init_k6_instances(self, load_params: LoadParams, endpoints: list[str], k6_dir: str):
@ -363,7 +328,7 @@ class LocalRunner(RunnerBase):
k6_dir,
shell,
loader,
self.wallet,
self.user,
)
def start(self):
@ -453,7 +418,7 @@ class S3LocalRunner(LocalRunner):
k6_dir,
shell,
loader,
self.wallet,
self.user,
)
@run_optionally(optionals.OPTIONAL_BACKGROUND_LOAD_ENABLED)
@ -466,17 +431,10 @@ class S3LocalRunner(LocalRunner):
k6_dir: str,
):
self.k6_dir = k6_dir
with reporter.step("Init s3 client on loaders"):
storage_node = nodes_under_load[0].service(StorageNode)
s3_public_keys = [node.service(S3Gate).get_wallet_public_key() for node in cluster_nodes]
grpc_peer = storage_node.get_rpc_endpoint()
parallel(self.prepare_node, nodes_under_load, k6_dir, load_params, s3_public_keys, grpc_peer)
parallel(self.prepare_node, nodes_under_load, k6_dir, load_params, cluster_nodes)
@reporter.step("Prepare node {cluster_node}")
def prepare_node(
self, cluster_node: ClusterNode, k6_dir: str, load_params: LoadParams, s3_public_keys: list[str], grpc_peer: str
):
def prepare_node(self, cluster_node: ClusterNode, k6_dir: str, load_params: LoadParams, cluster_nodes: list[ClusterNode]):
LocalRunner.prepare_node(self, cluster_node, k6_dir, load_params)
self.endpoints = cluster_node.s3_gate.get_all_endpoints()
shell = cluster_node.host.get_shell()
@ -497,29 +455,9 @@ class S3LocalRunner(LocalRunner):
shell.exec(f"sudo python3 -m pip install -I {k6_dir}/requests.tar.gz")
with reporter.step(f"Init s3 client on {cluster_node.host_ip}"):
frostfs_authmate_exec: FrostfsAuthmate = FrostfsAuthmate(shell, FROSTFS_AUTHMATE_EXEC)
issue_secret_output = frostfs_authmate_exec.secret.issue(
wallet=self.wallet.path,
peer=grpc_peer,
gate_public_key=s3_public_keys,
container_placement_policy=load_params.preset.container_placement_policy,
container_policy=f"{k6_dir}/scenarios/files/policy.json",
wallet_password=self.wallet.password,
).stdout
aws_access_key_id = str(
re.search(r"access_key_id.*:\s.(?P<aws_access_key_id>\w*)", issue_secret_output).group(
"aws_access_key_id"
)
)
aws_secret_access_key = str(
re.search(
r"secret_access_key.*:\s.(?P<aws_secret_access_key>\w*)",
issue_secret_output,
).group("aws_secret_access_key")
)
configure_input = [
InteractiveInput(prompt_pattern=r"AWS Access Key ID.*", input=aws_access_key_id),
InteractiveInput(prompt_pattern=r"AWS Secret Access Key.*", input=aws_secret_access_key),
InteractiveInput(prompt_pattern=r"AWS Access Key ID.*", input=self.user.s3_credentials.access_key),
InteractiveInput(prompt_pattern=r"AWS Secret Access Key.*", input=self.user.s3_credentials.secret_key),
InteractiveInput(prompt_pattern=r".*", input=""),
InteractiveInput(prompt_pattern=r".*", input=""),
]

16
src/frostfs_testlib/s3/curl_bucket_resolver.py Normal file
View file

@ -0,0 +1,16 @@
import re
from frostfs_testlib.cli.generic_cli import GenericCli
from frostfs_testlib.s3.interfaces import BucketContainerResolver
from frostfs_testlib.storage.cluster import ClusterNode
class CurlBucketContainerResolver(BucketContainerResolver):
def resolve(self, node: ClusterNode, bucket_name: str, **kwargs: dict) -> str:
curl = GenericCli("curl", node.host)
output = curl(f"-I http://127.0.0.1:8084/{bucket_name}")
pattern = r"X-Container-Id: (\S+)"
cid = re.findall(pattern, output.stdout)
if cid:
return cid[0]
return None

27
src/frostfs_testlib/s3/interfaces.py
View file

@ -1,7 +1,8 @@
from abc import abstractmethod
from abc import ABC, abstractmethod
from datetime import datetime
from typing import Literal, Optional, Union
from frostfs_testlib.storage.cluster import ClusterNode
from frostfs_testlib.testing.readable import HumanReadableABC, HumanReadableEnum
@ -31,6 +32,22 @@ ACL_COPY = [
]
class BucketContainerResolver(ABC):
@abstractmethod
def resolve(self, node: ClusterNode, bucket_name: str, **kwargs: dict) -> str:
"""
Resolve Container ID from bucket name
Args:
node: node from where we want to resolve
bucket_name: name of the bucket
**kwargs: any other required params
Returns: Container ID
"""
raise NotImplementedError("Call from abstract class")
class S3ClientWrapper(HumanReadableABC):
@abstractmethod
def __init__(self, access_key_id: str, secret_access_key: str, s3gate_endpoint: str, profile: str) -> None:
@ -296,15 +313,11 @@ class S3ClientWrapper(HumanReadableABC):
abort a given multipart upload multiple times in order to completely free all storage consumed by all parts."""
@abstractmethod
def upload_part(
self, bucket: str, key: str, upload_id: str, part_num: int, filepath: str
) -> str:
def upload_part(self, bucket: str, key: str, upload_id: str, part_num: int, filepath: str) -> str:
"""Uploads a part in a multipart upload."""
@abstractmethod
def upload_part_copy(
self, bucket: str, key: str, upload_id: str, part_num: int, copy_source: str
) -> str:
def upload_part_copy(self, bucket: str, key: str, upload_id: str, part_num: int, copy_source: str) -> str:
"""Uploads a part by copying data from an existing object as data source."""
@abstractmethod

42
src/frostfs_testlib/steps/acl.py
View file

@ -11,25 +11,20 @@ import base58
from frostfs_testlib import reporter
from frostfs_testlib.cli import FrostfsCli
from frostfs_testlib.resources.cli import FROSTFS_CLI_EXEC
from frostfs_testlib.resources.common import ASSETS_DIR, DEFAULT_WALLET_CONFIG
from frostfs_testlib.resources.common import ASSETS_DIR
from frostfs_testlib.shell import Shell
from frostfs_testlib.storage.dataclasses.acl import (
EACL_LIFETIME,
FROSTFS_CONTRACT_CACHE_TIMEOUT,
EACLPubKey,
EACLRole,
EACLRule,
)
from frostfs_testlib.storage.dataclasses.acl import EACL_LIFETIME, FROSTFS_CONTRACT_CACHE_TIMEOUT, EACLPubKey, EACLRole, EACLRule
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.utils import wallet_utils
logger = logging.getLogger("NeoLogger")
@reporter.step("Get extended ACL")
def get_eacl(wallet_path: str, cid: str, shell: Shell, endpoint: str) -> Optional[str]:
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, DEFAULT_WALLET_CONFIG)
def get_eacl(wallet: WalletInfo, cid: str, shell: Shell, endpoint: str) -> Optional[str]:
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
try:
result = cli.container.get_eacl(wallet=wallet_path, rpc_endpoint=endpoint, cid=cid)
result = cli.container.get_eacl(rpc_endpoint=endpoint, cid=cid)
except RuntimeError as exc:
logger.info("Extended ACL table is not set for this container")
logger.info(f"Got exception while getting eacl: {exc}")
@ -41,16 +36,15 @@ def get_eacl(wallet_path: str, cid: str, shell: Shell, endpoint: str) -> Optiona
@reporter.step("Set extended ACL")
def set_eacl(
wallet_path: str,
wallet: WalletInfo,
cid: str,
eacl_table_path: str,
shell: Shell,
endpoint: str,
session_token: Optional[str] = None,
) -> None:
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, DEFAULT_WALLET_CONFIG)
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
cli.container.set_eacl(
wallet=wallet_path,
rpc_endpoint=endpoint,
cid=cid,
table=eacl_table_path,
@ -66,7 +60,7 @@ def _encode_cid_for_eacl(cid: str) -> str:
def create_eacl(cid: str, rules_list: List[EACLRule], shell: Shell) -> str:
table_file_path = os.path.join(os.getcwd(), ASSETS_DIR, f"eacl_table_{str(uuid.uuid4())}.json")
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, DEFAULT_WALLET_CONFIG)
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC)
cli.acl.extended_create(cid=cid, out=table_file_path, rule=rules_list)
with open(table_file_path, "r") as file:
@ -77,7 +71,7 @@ def create_eacl(cid: str, rules_list: List[EACLRule], shell: Shell) -> str:
def form_bearertoken_file(
wif: str,
wallet: WalletInfo,
cid: str,
eacl_rule_list: List[Union[EACLRule, EACLPubKey]],
shell: Shell,
@ -92,7 +86,7 @@ def form_bearertoken_file(
enc_cid = _encode_cid_for_eacl(cid) if cid else None
file_path = os.path.join(os.getcwd(), ASSETS_DIR, str(uuid.uuid4()))
eacl = get_eacl(wif, cid, shell, endpoint)
eacl = get_eacl(wallet, cid, shell, endpoint)
json_eacl = dict()
if eacl:
eacl = eacl.replace("eACL: ", "").split("Signature")[0]
@ -133,7 +127,7 @@ def form_bearertoken_file(
if sign:
sign_bearer(
shell=shell,
wallet_path=wif,
wallet=wallet,
eacl_rules_file_from=file_path,
eacl_rules_file_to=file_path,
json=True,
@ -164,11 +158,9 @@ def eacl_rules(access: str, verbs: list, user: str) -> list[str]:
return rules
def sign_bearer(shell: Shell, wallet_path: str, eacl_rules_file_from: str, eacl_rules_file_to: str, json: bool) -> None:
frostfscli = FrostfsCli(shell=shell, frostfs_cli_exec_path=FROSTFS_CLI_EXEC, config_file=DEFAULT_WALLET_CONFIG)
frostfscli.util.sign_bearer_token(
wallet=wallet_path, from_file=eacl_rules_file_from, to_file=eacl_rules_file_to, json=json
)
def sign_bearer(shell: Shell, wallet: WalletInfo, eacl_rules_file_from: str, eacl_rules_file_to: str, json: bool) -> None:
frostfscli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
frostfscli.util.sign_bearer_token(eacl_rules_file_from, eacl_rules_file_to, json=json)
@reporter.step("Wait for eACL cache expired")
@ -178,9 +170,7 @@ def wait_for_cache_expired():
@reporter.step("Return bearer token in base64 to caller")
def bearer_token_base64_from_file(
bearer_path: str,
) -> str:
def bearer_token_base64_from_file(bearer_path: str) -> str:
with open(bearer_path, "rb") as file:
signed = file.read()
return base64.b64encode(signed).decode("utf-8")

92
src/frostfs_testlib/steps/cli/container.py
View file

@ -5,12 +5,11 @@ from dataclasses import dataclass
from time import sleep
from typing import Optional, Union
import requests
from frostfs_testlib import reporter
from frostfs_testlib.cli import FrostfsCli, GenericCli
from frostfs_testlib.cli import FrostfsCli
from frostfs_testlib.plugins import load_plugin
from frostfs_testlib.resources.cli import CLI_DEFAULT_TIMEOUT, FROSTFS_CLI_EXEC
from frostfs_testlib.resources.common import DEFAULT_WALLET_CONFIG
from frostfs_testlib.s3.interfaces import BucketContainerResolver
from frostfs_testlib.shell import Shell
from frostfs_testlib.steps.cli.object import put_object, put_object_to_random_node
from frostfs_testlib.storage.cluster import Cluster, ClusterNode
@ -25,7 +24,7 @@ logger = logging.getLogger("NeoLogger")
@dataclass
class StorageContainerInfo:
id: str
wallet_file: WalletInfo
wallet: WalletInfo
class StorageContainer:
@ -42,11 +41,8 @@ class StorageContainer:
def get_id(self) -> str:
return self.storage_container_info.id
def get_wallet_path(self) -> str:
return self.storage_container_info.wallet_file.path
def get_wallet_config_path(self) -> str:
return self.storage_container_info.wallet_file.config_path
def get_wallet(self) -> str:
return self.storage_container_info.wallet
@reporter.step("Generate new object and put in container")
def generate_object(
@ -61,37 +57,34 @@ class StorageContainer:
file_hash = get_file_hash(file_path)
container_id = self.get_id()
wallet_path = self.get_wallet_path()
wallet_config = self.get_wallet_config_path()
wallet = self.get_wallet()
with reporter.step(f"Put object with size {size} to container {container_id}"):
if endpoint:
object_id = put_object(
wallet=wallet_path,
wallet=wallet,
path=file_path,
cid=container_id,
expire_at=expire_at,
shell=self.shell,
endpoint=endpoint,
bearer=bearer_token,
wallet_config=wallet_config,
)
else:
object_id = put_object_to_random_node(
wallet=wallet_path,
wallet=wallet,
path=file_path,
cid=container_id,
expire_at=expire_at,
shell=self.shell,
cluster=self.cluster,
bearer=bearer_token,
wallet_config=wallet_config,
)
storage_object = StorageObjectInfo(
container_id,
object_id,
size=size,
wallet_file_path=wallet_path,
wallet=wallet,
file_path=file_path,
file_hash=file_hash,
)
@ -106,14 +99,13 @@ REP_2_FOR_3_NODES_PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 3 FROM * AS X"
@reporter.step("Create Container")
def create_container(
wallet: str,
wallet: WalletInfo,
shell: Shell,
endpoint: str,
rule: str = DEFAULT_PLACEMENT_RULE,
basic_acl: str = "",
attributes: Optional[dict] = None,
session_token: str = "",
session_wallet: str = "",
name: Optional[str] = None,
options: Optional[dict] = None,
await_mode: bool = True,
@ -124,7 +116,7 @@ def create_container(
A wrapper for `frostfs-cli container create` call.
Args:
wallet (str): a wallet on whose behalf a container is created
wallet (WalletInfo): a wallet on whose behalf a container is created
rule (optional, str): placement rule for container
basic_acl (optional, str): an ACL for container, will be
appended to `--basic-acl` key
@ -146,10 +138,9 @@ def create_container(
(str): CID of the created container
"""
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, DEFAULT_WALLET_CONFIG)
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
result = cli.container.create(
rpc_endpoint=endpoint,
wallet=session_wallet if session_wallet else wallet,
policy=rule,
basic_acl=basic_acl,
attributes=attributes,
@ -170,9 +161,7 @@ def create_container(
return cid
def wait_for_container_creation(
wallet: str, cid: str, shell: Shell, endpoint: str, attempts: int = 15, sleep_interval: int = 1
):
def wait_for_container_creation(wallet: WalletInfo, cid: str, shell: Shell, endpoint: str, attempts: int = 15, sleep_interval: int = 1):
for _ in range(attempts):
containers = list_containers(wallet, shell, endpoint)
if cid in containers:
@ -182,9 +171,7 @@ def wait_for_container_creation(
raise RuntimeError(f"After {attempts * sleep_interval} seconds container {cid} hasn't been persisted; exiting")
def wait_for_container_deletion(
wallet: str, cid: str, shell: Shell, endpoint: str, attempts: int = 30, sleep_interval: int = 1
):
def wait_for_container_deletion(wallet: WalletInfo, cid: str, shell: Shell, endpoint: str, attempts: int = 30, sleep_interval: int = 1):
for _ in range(attempts):
try:
get_container(wallet, cid, shell=shell, endpoint=endpoint)
@ -198,29 +185,27 @@ def wait_for_container_deletion(
@reporter.step("List Containers")
def list_containers(
wallet: str, shell: Shell, endpoint: str, timeout: Optional[str] = CLI_DEFAULT_TIMEOUT
) -> list[str]:
def list_containers(wallet: WalletInfo, shell: Shell, endpoint: str, timeout: Optional[str] = CLI_DEFAULT_TIMEOUT) -> list[str]:
"""
A wrapper for `frostfs-cli container list` call. It returns all the
available containers for the given wallet.
Args:
wallet (str): a wallet on whose behalf we list the containers
wallet (WalletInfo): a wallet on whose behalf we list the containers
shell: executor for cli command
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
timeout: Timeout for the operation.
Returns:
(list): list of containers
"""
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, DEFAULT_WALLET_CONFIG)
result = cli.container.list(rpc_endpoint=endpoint, wallet=wallet, timeout=timeout)
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
result = cli.container.list(rpc_endpoint=endpoint, timeout=timeout)
logger.info(f"Containers: \n{result}")
return result.stdout.split()
@reporter.step("List Objects in container")
def list_objects(
wallet: str,
wallet: WalletInfo,
shell: Shell,
container_id: str,
endpoint: str,
@ -230,7 +215,7 @@ def list_objects(
A wrapper for `frostfs-cli container list-objects` call. It returns all the
available objects in container.
Args:
wallet (str): a wallet on whose behalf we list the containers objects
wallet (WalletInfo): a wallet on whose behalf we list the containers objects
shell: executor for cli command
container_id: cid of container
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
@ -238,15 +223,15 @@ def list_objects(
Returns:
(list): list of containers
"""
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, DEFAULT_WALLET_CONFIG)
result = cli.container.list_objects(rpc_endpoint=endpoint, wallet=wallet, cid=container_id, timeout=timeout)
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
result = cli.container.list_objects(rpc_endpoint=endpoint, cid=container_id, timeout=timeout)
logger.info(f"Container objects: \n{result}")
return result.stdout.split()
@reporter.step("Get Container")
def get_container(
wallet: str,
wallet: WalletInfo,
cid: str,
shell: Shell,
endpoint: str,
@ -257,7 +242,7 @@ def get_container(
A wrapper for `frostfs-cli container get` call. It extracts container's
attributes and rearranges them into a more compact view.
Args:
wallet (str): path to a wallet on whose behalf we get the container
wallet (WalletInfo): path to a wallet on whose behalf we get the container
cid (str): ID of the container to get
shell: executor for cli command
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
@ -267,8 +252,8 @@ def get_container(
(dict, str): dict of container attributes
"""
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, DEFAULT_WALLET_CONFIG)
result = cli.container.get(rpc_endpoint=endpoint, wallet=wallet, cid=cid, json_mode=json_mode, timeout=timeout)
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
result = cli.container.get(rpc_endpoint=endpoint, cid=cid, json_mode=json_mode, timeout=timeout)
if not json_mode:
return result.stdout
@ -285,7 +270,7 @@ def get_container(
@reporter.step("Delete Container")
# TODO: make the error message about a non-found container more user-friendly
def delete_container(
wallet: str,
wallet: WalletInfo,
cid: str,
shell: Shell,
endpoint: str,
@ -297,7 +282,7 @@ def delete_container(
A wrapper for `frostfs-cli container delete` call.
Args:
await_mode: Block execution until container is removed.
wallet (str): path to a wallet on whose behalf we delete the container
wallet (WalletInfo): path to a wallet on whose behalf we delete the container
cid (str): ID of the container to delete
shell: executor for cli command
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
@ -306,9 +291,8 @@ def delete_container(
This function doesn't return anything.
"""
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, DEFAULT_WALLET_CONFIG)
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
cli.container.delete(
wallet=wallet,
cid=cid,
rpc_endpoint=endpoint,
force=force,
@ -345,26 +329,22 @@ def _parse_cid(output: str) -> str:
@reporter.step("Search container by name")
def search_container_by_name(name: str, node: ClusterNode):
curl = GenericCli("curl", node.host)
output = curl(f"-I http://127.0.0.1:8084/{name}")
pattern = r"X-Container-Id: (\S+)"
cid = re.findall(pattern, output.stdout)
if cid:
return cid[0]
return None
resolver_cls = load_plugin("frostfs.testlib.bucket_cid_resolver", node.host.config.product)
resolver: BucketContainerResolver = resolver_cls()
return resolver.resolve(node, name)
@reporter.step("Search for nodes with a container")
def search_nodes_with_container(
wallet: str,
wallet: WalletInfo,
cid: str,
shell: Shell,
endpoint: str,
cluster: Cluster,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> list[ClusterNode]:
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, DEFAULT_WALLET_CONFIG)
result = cli.container.search_node(rpc_endpoint=endpoint, wallet=wallet, cid=cid, timeout=timeout)
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
result = cli.container.search_node(rpc_endpoint=endpoint, cid=cid, timeout=timeout)
pattern = r"[0-9]+(?:\.[0-9]+){3}"
nodes_ip = list(set(re.findall(pattern, result.stdout)))

99
src/frostfs_testlib/steps/cli/object.py
View file

@ -9,9 +9,10 @@ from frostfs_testlib import reporter
from frostfs_testlib.cli import FrostfsCli
from frostfs_testlib.cli.neogo import NeoGo
from frostfs_testlib.resources.cli import CLI_DEFAULT_TIMEOUT, FROSTFS_CLI_EXEC, NEOGO_EXECUTABLE
from frostfs_testlib.resources.common import ASSETS_DIR, DEFAULT_WALLET_CONFIG
from frostfs_testlib.resources.common import ASSETS_DIR
from frostfs_testlib.shell import Shell
from frostfs_testlib.storage.cluster import Cluster, ClusterNode
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.utils import json_utils
from frostfs_testlib.utils.cli_utils import parse_cmd_table, parse_netmap_output
@ -20,7 +21,7 @@ logger = logging.getLogger("NeoLogger")
@reporter.step("Get object from random node")
def get_object_from_random_node(
wallet: str,
wallet: WalletInfo,
cid: str,
oid: str,
shell: Shell,
@ -28,7 +29,6 @@ def get_object_from_random_node(
bearer: Optional[str] = None,
write_object: Optional[str] = None,
xhdr: Optional[dict] = None,
wallet_config: Optional[str] = None,
no_progress: bool = True,
session: Optional[str] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
@ -44,7 +44,6 @@ def get_object_from_random_node(
cluster: cluster object
bearer (optional, str): path to Bearer Token file, appends to `--bearer` key
write_object (optional, str): path to downloaded file, appends to `--file` key
wallet_config(optional, str): path to the wallet config
no_progress(optional, bool): do not show progress bar
xhdr (optional, dict): Request X-Headers in form of Key=Value
session (optional, dict): path to a JSON-encoded container session token
@ -62,7 +61,6 @@ def get_object_from_random_node(
bearer,
write_object,
xhdr,
wallet_config,
no_progress,
session,
timeout,
@ -71,7 +69,7 @@ def get_object_from_random_node(
@reporter.step("Get object from {endpoint}")
def get_object(
wallet: str,
wallet: WalletInfo,
cid: str,
oid: str,
shell: Shell,
@ -79,7 +77,6 @@ def get_object(
bearer: Optional[str] = None,
write_object: Optional[str] = None,
xhdr: Optional[dict] = None,
wallet_config: Optional[str] = None,
no_progress: bool = True,
session: Optional[str] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
@ -88,14 +85,13 @@ def get_object(
GET from FrostFS.
Args:
wallet (str): wallet on whose behalf GET is done
wallet (WalletInfo): wallet on whose behalf GET is done
cid (str): ID of Container where we get the Object from
oid (str): Object ID
shell: executor for cli command
bearer: path to Bearer Token file, appends to `--bearer` key
write_object: path to downloaded file, appends to `--file` key
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
wallet_config(optional, str): path to the wallet config
no_progress(optional, bool): do not show progress bar
xhdr (optional, dict): Request X-Headers in form of Key=Value
session (optional, dict): path to a JSON-encoded container session token
@ -108,10 +104,9 @@ def get_object(
write_object = str(uuid.uuid4())
file_path = os.path.join(ASSETS_DIR, write_object)
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or DEFAULT_WALLET_CONFIG)
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
cli.object.get(
rpc_endpoint=endpoint,
wallet=wallet,
cid=cid,
oid=oid,
file=file_path,
@ -127,14 +122,13 @@ def get_object(
@reporter.step("Get Range Hash from {endpoint}")
def get_range_hash(
wallet: str,
wallet: WalletInfo,
cid: str,
oid: str,
range_cut: str,
shell: Shell,
endpoint: str,
bearer: Optional[str] = None,
wallet_config: Optional[str] = None,
xhdr: Optional[dict] = None,
session: Optional[str] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
@ -151,17 +145,15 @@ def get_range_hash(
range_cut: Range to take hash from in the form offset1:length1,...,
value to pass to the `--range` parameter
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
wallet_config: path to the wallet config
xhdr: Request X-Headers in form of Key=Values
session: Filepath to a JSON- or binary-encoded token of the object RANGEHASH session.
timeout: Timeout for the operation.
Returns:
None
"""
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or DEFAULT_WALLET_CONFIG)
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
result = cli.object.hash(
rpc_endpoint=endpoint,
wallet=wallet,
cid=cid,
oid=oid,
range=range_cut,
@ -177,7 +169,7 @@ def get_range_hash(
@reporter.step("Put object to random node")
def put_object_to_random_node(
wallet: str,
wallet: WalletInfo,
path: str,
cid: str,
shell: Shell,
@ -186,7 +178,6 @@ def put_object_to_random_node(
copies_number: Optional[int] = None,
attributes: Optional[dict] = None,
xhdr: Optional[dict] = None,
wallet_config: Optional[str] = None,
expire_at: Optional[int] = None,
no_progress: bool = True,
session: Optional[str] = None,
@ -205,7 +196,6 @@ def put_object_to_random_node(
copies_number: Number of copies of the object to store within the RPC call
attributes: User attributes in form of Key1=Value1,Key2=Value2
cluster: cluster under test
wallet_config: path to the wallet config
no_progress: do not show progress bar
expire_at: Last epoch in the life of the object
xhdr: Request X-Headers in form of Key=Value
@ -226,7 +216,6 @@ def put_object_to_random_node(
copies_number,
attributes,
xhdr,
wallet_config,
expire_at,
no_progress,
session,
@ -236,7 +225,7 @@ def put_object_to_random_node(
@reporter.step("Put object at {endpoint} in container {cid}")
def put_object(
wallet: str,
wallet: WalletInfo,
path: str,
cid: str,
shell: Shell,
@ -245,7 +234,6 @@ def put_object(
copies_number: Optional[int] = None,
attributes: Optional[dict] = None,
xhdr: Optional[dict] = None,
wallet_config: Optional[str] = None,
expire_at: Optional[int] = None,
no_progress: bool = True,
session: Optional[str] = None,
@ -263,7 +251,6 @@ def put_object(
copies_number: Number of copies of the object to store within the RPC call
attributes: User attributes in form of Key1=Value1,Key2=Value2
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
wallet_config: path to the wallet config
no_progress: do not show progress bar
expire_at: Last epoch in the life of the object
xhdr: Request X-Headers in form of Key=Value
@ -273,10 +260,9 @@ def put_object(
(str): ID of uploaded Object
"""
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or DEFAULT_WALLET_CONFIG)
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
result = cli.object.put(
rpc_endpoint=endpoint,
wallet=wallet,
file=path,
cid=cid,
attributes=attributes,
@ -297,13 +283,12 @@ def put_object(
@reporter.step("Delete object {cid}/{oid} from {endpoint}")
def delete_object(
wallet: str,
wallet: WalletInfo,
cid: str,
oid: str,
shell: Shell,
endpoint: str,
bearer: str = "",
wallet_config: Optional[str] = None,
xhdr: Optional[dict] = None,
session: Optional[str] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
@ -318,7 +303,6 @@ def delete_object(
shell: executor for cli command
bearer: path to Bearer Token file, appends to `--bearer` key
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
wallet_config: path to the wallet config
xhdr: Request X-Headers in form of Key=Value
session: path to a JSON-encoded container session token
timeout: Timeout for the operation.
@ -326,10 +310,9 @@ def delete_object(
(str): Tombstone ID
"""
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or DEFAULT_WALLET_CONFIG)
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
result = cli.object.delete(
rpc_endpoint=endpoint,
wallet=wallet,
cid=cid,
oid=oid,
bearer=bearer,
@ -345,13 +328,12 @@ def delete_object(
@reporter.step("Get Range")
def get_range(
wallet: str,
wallet: WalletInfo,
cid: str,
oid: str,
range_cut: str,
shell: Shell,
endpoint: str,
wallet_config: Optional[str] = None,
bearer: str = "",
xhdr: Optional[dict] = None,
session: Optional[str] = None,
@ -368,7 +350,6 @@ def get_range(
shell: executor for cli command
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
bearer: path to Bearer Token file, appends to `--bearer` key
wallet_config: path to the wallet config
xhdr: Request X-Headers in form of Key=Value
session: path to a JSON-encoded container session token
timeout: Timeout for the operation.
@ -377,10 +358,9 @@ def get_range(
"""
range_file_path = os.path.join(ASSETS_DIR, str(uuid.uuid4()))
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or DEFAULT_WALLET_CONFIG)
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
cli.object.range(
rpc_endpoint=endpoint,
wallet=wallet,
cid=cid,
oid=oid,
range=range_cut,
@ -398,7 +378,7 @@ def get_range(
@reporter.step("Lock Object")
def lock_object(
wallet: str,
wallet: WalletInfo,
cid: str,
oid: str,
shell: Shell,
@ -408,7 +388,6 @@ def lock_object(
address: Optional[str] = None,
bearer: Optional[str] = None,
session: Optional[str] = None,
wallet_config: Optional[str] = None,
ttl: Optional[int] = None,
xhdr: Optional[dict] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
@ -435,13 +414,12 @@ def lock_object(
Lock object ID
"""
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or DEFAULT_WALLET_CONFIG)
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
result = cli.object.lock(
rpc_endpoint=endpoint,
lifetime=lifetime,
expire_at=expire_at,
address=address,
wallet=wallet,
cid=cid,
oid=oid,
bearer=bearer,
@ -459,14 +437,13 @@ def lock_object(
@reporter.step("Search object")
def search_object(
wallet: str,
wallet: WalletInfo,
cid: str,
shell: Shell,
endpoint: str,
bearer: str = "",
filters: Optional[dict] = None,
expected_objects_list: Optional[list] = None,
wallet_config: Optional[str] = None,
xhdr: Optional[dict] = None,
session: Optional[str] = None,
phy: bool = False,
@ -484,7 +461,6 @@ def search_object(
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
filters: key=value pairs to filter Objects
expected_objects_list: a list of ObjectIDs to compare found Objects with
wallet_config: path to the wallet config
xhdr: Request X-Headers in form of Key=Value
session: path to a JSON-encoded container session token
phy: Search physically stored objects.
@ -495,10 +471,9 @@ def search_object(
list of found ObjectIDs
"""
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or DEFAULT_WALLET_CONFIG)
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
result = cli.object.search(
rpc_endpoint=endpoint,
wallet=wallet,
cid=cid,
bearer=bearer,
xhdr=xhdr,
@ -513,23 +488,18 @@ def search_object(
if expected_objects_list:
if sorted(found_objects) == sorted(expected_objects_list):
logger.info(
f"Found objects list '{found_objects}' " f"is equal for expected list '{expected_objects_list}'"
)
logger.info(f"Found objects list '{found_objects}' " f"is equal for expected list '{expected_objects_list}'")
else:
logger.warning(
f"Found object list {found_objects} " f"is not equal to expected list '{expected_objects_list}'"
)
logger.warning(f"Found object list {found_objects} " f"is not equal to expected list '{expected_objects_list}'")
return found_objects
@reporter.step("Get netmap netinfo")
def get_netmap_netinfo(
wallet: str,
wallet: WalletInfo,
shell: Shell,
endpoint: str,
wallet_config: Optional[str] = None,
address: Optional[str] = None,
ttl: Optional[int] = None,
xhdr: Optional[dict] = None,
@ -539,7 +509,7 @@ def get_netmap_netinfo(
Get netmap netinfo output from node
Args:
wallet (str): wallet on whose behalf request is done
wallet (WalletInfo): wallet on whose behalf request is done
shell: executor for cli command
endpoint (optional, str): FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
address: Address of wallet account
@ -552,9 +522,8 @@ def get_netmap_netinfo(
(dict): dict of parsed command output
"""
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or DEFAULT_WALLET_CONFIG)
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
output = cli.netmap.netinfo(
wallet=wallet,
rpc_endpoint=endpoint,
address=address,
ttl=ttl,
@ -578,7 +547,7 @@ def get_netmap_netinfo(
@reporter.step("Head object")
def head_object(
wallet: str,
wallet: WalletInfo,
cid: str,
oid: str,
shell: Shell,
@ -588,7 +557,6 @@ def head_object(
json_output: bool = True,
is_raw: bool = False,
is_direct: bool = False,
wallet_config: Optional[str] = None,
session: Optional[str] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
):
@ -596,7 +564,7 @@ def head_object(
HEAD an Object.
Args:
wallet (str): wallet on whose behalf HEAD is done
wallet (WalletInfo): wallet on whose behalf HEAD is done
cid (str): ID of Container where we get the Object from
oid (str): ObjectID to HEAD
shell: executor for cli command
@ -608,7 +576,6 @@ def head_object(
turns into `--raw` key
is_direct(optional, bool): send request directly to the node or not; this flag
turns into `--ttl 1` key
wallet_config(optional, str): path to the wallet config
xhdr (optional, dict): Request X-Headers in form of Key=Value
session (optional, dict): path to a JSON-encoded container session token
timeout: Timeout for the operation.
@ -619,10 +586,9 @@ def head_object(
(str): HEAD response as a plain text
"""
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or DEFAULT_WALLET_CONFIG)
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
result = cli.object.head(
rpc_endpoint=endpoint,
wallet=wallet,
cid=cid,
oid=oid,
bearer=bearer,
@ -673,7 +639,7 @@ def head_object(
@reporter.step("Run neo-go dump-keys")
def neo_go_dump_keys(shell: Shell, wallet: str) -> dict:
def neo_go_dump_keys(shell: Shell, wallet: WalletInfo) -> dict:
"""
Run neo-go dump keys command
@ -761,9 +727,7 @@ def get_object_nodes(
parsing_output = parse_cmd_table(result_object_nodes.stdout, "|")
list_object_nodes = [
node
for node in parsing_output
if node["should_contain_object"] == "true" and node["actually_contains_object"] == "true"
node for node in parsing_output if node["should_contain_object"] == "true" and node["actually_contains_object"] == "true"
]
netmap_nodes_list = parse_netmap_output(
@ -780,10 +744,7 @@ def get_object_nodes(
]
result = [
cluster_node
for netmap_node in netmap_nodes
for cluster_node in cluster.cluster_nodes
if netmap_node.node == cluster_node.host_ip
cluster_node for netmap_node in netmap_nodes for cluster_node in cluster.cluster_nodes if netmap_node.node == cluster_node.host_ip
]
return result

14
src/frostfs_testlib/steps/complex_object_actions.py
View file

@ -14,11 +14,11 @@ from typing import Optional, Tuple
from frostfs_testlib import reporter
from frostfs_testlib.resources.cli import CLI_DEFAULT_TIMEOUT
from frostfs_testlib.resources.common import DEFAULT_WALLET_CONFIG
from frostfs_testlib.shell import Shell
from frostfs_testlib.steps.cli.object import head_object
from frostfs_testlib.storage.cluster import Cluster, StorageNode
from frostfs_testlib.storage.dataclasses.storage_object_info import StorageObjectInfo
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
logger = logging.getLogger("NeoLogger")
@ -44,7 +44,7 @@ def get_storage_object_chunks(
with reporter.step(f"Get complex object chunks (f{storage_object.oid})"):
split_object_id = get_link_object(
storage_object.wallet_file_path,
storage_object.wallet,
storage_object.cid,
storage_object.oid,
shell,
@ -53,7 +53,7 @@ def get_storage_object_chunks(
timeout=timeout,
)
head = head_object(
storage_object.wallet_file_path,
storage_object.wallet,
storage_object.cid,
split_object_id,
shell,
@ -96,7 +96,7 @@ def get_complex_object_split_ranges(
chunks_ids = get_storage_object_chunks(storage_object, shell, cluster)
for chunk_id in chunks_ids:
head = head_object(
storage_object.wallet_file_path,
storage_object.wallet,
storage_object.cid,
chunk_id,
shell,
@ -114,13 +114,12 @@ def get_complex_object_split_ranges(
@reporter.step("Get Link Object")
def get_link_object(
wallet: str,
wallet: WalletInfo,
cid: str,
oid: str,
shell: Shell,
nodes: list[StorageNode],
bearer: str = "",
wallet_config: str = DEFAULT_WALLET_CONFIG,
is_direct: bool = True,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
):
@ -154,7 +153,6 @@ def get_link_object(
is_raw=True,
is_direct=is_direct,
bearer=bearer,
wallet_config=wallet_config,
timeout=timeout,
)
if resp["link"]:
@ -167,7 +165,7 @@ def get_link_object(
@reporter.step("Get Last Object")
def get_last_object(
wallet: str,
wallet: WalletInfo,
cid: str,
oid: str,
shell: Shell,

8
src/frostfs_testlib/steps/epoch.py
View file

@ -4,13 +4,7 @@ from typing import Optional
from frostfs_testlib import reporter
from frostfs_testlib.cli import FrostfsAdm, FrostfsCli, NeoGo
from frostfs_testlib.resources.cli import (
CLI_DEFAULT_TIMEOUT,
FROSTFS_ADM_CONFIG_PATH,
FROSTFS_ADM_EXEC,
FROSTFS_CLI_EXEC,
NEOGO_EXECUTABLE,
)
from frostfs_testlib.resources.cli import CLI_DEFAULT_TIMEOUT, FROSTFS_ADM_CONFIG_PATH, FROSTFS_ADM_EXEC, FROSTFS_CLI_EXEC, NEOGO_EXECUTABLE
from frostfs_testlib.resources.common import MORPH_BLOCK_TIME
from frostfs_testlib.shell import Shell
from frostfs_testlib.steps.payment_neogo import get_contract_hash

11
src/frostfs_testlib/steps/s3/s3_helper.py
View file

@ -10,6 +10,7 @@ from frostfs_testlib.s3 import S3ClientWrapper, VersioningStatus
from frostfs_testlib.shell import Shell
from frostfs_testlib.steps.cli.container import search_container_by_name, search_nodes_with_container
from frostfs_testlib.storage.cluster import Cluster, ClusterNode
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
logger = logging.getLogger("NeoLogger")
@ -28,9 +29,7 @@ def check_objects_in_bucket(
assert bucket_object in bucket_objects, f"Expected object {bucket_object} in objects list {bucket_objects}"
for bucket_object in unexpected_objects:
assert (
bucket_object not in bucket_objects
), f"Expected object {bucket_object} not in objects list {bucket_objects}"
assert bucket_object not in bucket_objects, f"Expected object {bucket_object} not in objects list {bucket_objects}"
@reporter.step("Try to get object and got error")
@ -58,9 +57,7 @@ def object_key_from_file_path(full_path: str) -> str:
return os.path.basename(full_path)
def assert_tags(
actual_tags: list, expected_tags: Optional[list] = None, unexpected_tags: Optional[list] = None
) -> None:
def assert_tags(actual_tags: list, expected_tags: Optional[list] = None, unexpected_tags: Optional[list] = None) -> None:
expected_tags = [{"Key": key, "Value": value} for key, value in expected_tags] if expected_tags else []
unexpected_tags = [{"Key": key, "Value": value} for key, value in unexpected_tags] if unexpected_tags else []
if expected_tags == []:
@ -180,7 +177,7 @@ def delete_bucket_with_objects(s3_client: S3ClientWrapper, bucket: str):
def search_nodes_with_bucket(
cluster: Cluster,
bucket_name: str,
wallet: str,
wallet: WalletInfo,
shell: Shell,
endpoint: str,
) -> list[ClusterNode]:

17
src/frostfs_testlib/steps/session_token.py
View file

@ -4,13 +4,12 @@ import logging
import os
import uuid
from dataclasses import dataclass
from enum import Enum
from typing import Any, Optional
from frostfs_testlib import reporter
from frostfs_testlib.cli import FrostfsCli
from frostfs_testlib.resources.cli import FROSTFS_CLI_EXEC
from frostfs_testlib.resources.common import ASSETS_DIR, DEFAULT_WALLET_CONFIG
from frostfs_testlib.resources.common import ASSETS_DIR
from frostfs_testlib.shell import Shell
from frostfs_testlib.storage.dataclasses.storage_object_info import StorageObjectInfo
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
@ -231,8 +230,7 @@ def get_object_signed_token(
def create_session_token(
shell: Shell,
owner: str,
wallet_path: str,
wallet_password: str,
wallet: WalletInfo,
rpc_endpoint: str,
) -> str:
"""
@ -247,19 +245,18 @@ def create_session_token(
The path to the generated session token file.
"""
session_token = os.path.join(os.getcwd(), ASSETS_DIR, str(uuid.uuid4()))
frostfscli = FrostfsCli(shell=shell, frostfs_cli_exec_path=FROSTFS_CLI_EXEC)
frostfscli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
frostfscli.session.create(
rpc_endpoint=rpc_endpoint,
address=owner,
wallet=wallet_path,
wallet_password=wallet_password,
out=session_token,
wallet=wallet.path,
)
return session_token
@reporter.step("Sign Session Token")
def sign_session_token(shell: Shell, session_token_file: str, wlt: WalletInfo) -> str:
def sign_session_token(shell: Shell, session_token_file: str, wallet: WalletInfo) -> str:
"""
This function signs the session token by the given wallet.
@ -272,6 +269,6 @@ def sign_session_token(shell: Shell, session_token_file: str, wlt: WalletInfo) -
The path to the signed token.
"""
signed_token_file = os.path.join(os.getcwd(), ASSETS_DIR, str(uuid.uuid4()))
frostfscli = FrostfsCli(shell=shell, frostfs_cli_exec_path=FROSTFS_CLI_EXEC, config_file=DEFAULT_WALLET_CONFIG)
frostfscli.util.sign_session_token(wallet=wlt.path, from_file=session_token_file, to_file=signed_token_file)
frostfscli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
frostfscli.util.sign_session_token(session_token_file, signed_token_file)
return signed_token_file

6
src/frostfs_testlib/steps/storage_object.py
View file

@ -30,14 +30,14 @@ def delete_objects(storage_objects: list[StorageObjectInfo], shell: Shell, clust
with reporter.step("Delete objects"):
for storage_object in storage_objects:
storage_object.tombstone = delete_object(
storage_object.wallet_file_path,
storage_object.wallet,
storage_object.cid,
storage_object.oid,
shell=shell,
endpoint=cluster.default_rpc_endpoint,
)
verify_head_tombstone(
wallet_path=storage_object.wallet_file_path,
wallet=storage_object.wallet,
cid=storage_object.cid,
oid_ts=storage_object.tombstone,
oid=storage_object.oid,
@ -52,7 +52,7 @@ def delete_objects(storage_objects: list[StorageObjectInfo], shell: Shell, clust
for storage_object in storage_objects:
with pytest.raises(Exception, match=OBJECT_ALREADY_REMOVED):
get_object(
storage_object.wallet_file_path,
storage_object.wallet,
storage_object.cid,
storage_object.oid,
shell=shell,

16
src/frostfs_testlib/steps/storage_policy.py
View file

@ -12,13 +12,15 @@ from frostfs_testlib.shell import Shell
from frostfs_testlib.steps.cli.object import head_object
from frostfs_testlib.steps.complex_object_actions import get_last_object
from frostfs_testlib.storage.cluster import StorageNode
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.utils import string_utils
logger = logging.getLogger("NeoLogger")
# TODO: Unused, remove or make use of
@reporter.step("Get Object Copies")
def get_object_copies(complexity: str, wallet: str, cid: str, oid: str, shell: Shell, nodes: list[StorageNode]) -> int:
def get_object_copies(complexity: str, wallet: WalletInfo, cid: str, oid: str, shell: Shell, nodes: list[StorageNode]) -> int:
"""
The function performs requests to all nodes of the container and
finds out if they store a copy of the object. The procedure is
@ -43,7 +45,7 @@ def get_object_copies(complexity: str, wallet: str, cid: str, oid: str, shell: S
@reporter.step("Get Simple Object Copies")
def get_simple_object_copies(wallet: str, cid: str, oid: str, shell: Shell, nodes: list[StorageNode]) -> int:
def get_simple_object_copies(wallet: WalletInfo, cid: str, oid: str, shell: Shell, nodes: list[StorageNode]) -> int:
"""
To figure out the number of a simple object copies, only direct
HEAD requests should be made to the every node of the container.
@ -72,7 +74,7 @@ def get_simple_object_copies(wallet: str, cid: str, oid: str, shell: Shell, node
@reporter.step("Get Complex Object Copies")
def get_complex_object_copies(wallet: str, cid: str, oid: str, shell: Shell, nodes: list[StorageNode]) -> int:
def get_complex_object_copies(wallet: WalletInfo, cid: str, oid: str, shell: Shell, nodes: list[StorageNode]) -> int:
"""
To figure out the number of a complex object copies, we firstly
need to retrieve its Last object. We consider that the number of
@ -109,8 +111,7 @@ def get_nodes_with_object(cid: str, oid: str, shell: Shell, nodes: list[StorageN
nodes_list = []
for node in nodes:
wallet = node.get_wallet_path()
wallet_config = node.get_wallet_config_path()
wallet = WalletInfo.from_node(node)
try:
res = head_object(
wallet,
@ -119,7 +120,6 @@ def get_nodes_with_object(cid: str, oid: str, shell: Shell, nodes: list[StorageN
shell=shell,
endpoint=node.get_rpc_endpoint(),
is_direct=True,
wallet_config=wallet_config,
)
if res is not None:
logger.info(f"Found object {oid} on node {node}")
@ -131,9 +131,7 @@ def get_nodes_with_object(cid: str, oid: str, shell: Shell, nodes: list[StorageN
@reporter.step("Get Nodes Without Object")
def get_nodes_without_object(
wallet: str, cid: str, oid: str, shell: Shell, nodes: list[StorageNode]
) -> list[StorageNode]:
def get_nodes_without_object(wallet: WalletInfo, cid: str, oid: str, shell: Shell, nodes: list[StorageNode]) -> list[StorageNode]:
"""
The function returns list of nodes which do not store
the given object.

16
src/frostfs_testlib/steps/tombstone.py
View file

@ -1,31 +1,23 @@
import json
import logging
from neo3.wallet import wallet
from frostfs_testlib import reporter
from frostfs_testlib.shell import Shell
from frostfs_testlib.steps.cli.object import head_object
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
logger = logging.getLogger("NeoLogger")
@reporter.step("Verify Head Tombstone")
def verify_head_tombstone(wallet_path: str, cid: str, oid_ts: str, oid: str, shell: Shell, endpoint: str):
header = head_object(wallet_path, cid, oid_ts, shell=shell, endpoint=endpoint)["header"]
def verify_head_tombstone(wallet: WalletInfo, cid: str, oid_ts: str, oid: str, shell: Shell, endpoint: str):
header = head_object(wallet, cid, oid_ts, shell=shell, endpoint=endpoint)["header"]
s_oid = header["sessionToken"]["body"]["object"]["target"]["objects"]
logger.info(f"Header Session OIDs is {s_oid}")
logger.info(f"OID is {oid}")
assert header["containerID"] == cid, "Tombstone Header CID is wrong"
with open(wallet_path, "r") as file:
wlt_data = json.loads(file.read())
wlt = wallet.Wallet.from_json(wlt_data, password="")
addr = wlt.accounts[0].address
assert header["ownerID"] == addr, "Tombstone Owner ID is wrong"
assert header["ownerID"] == wallet.get_address_from_json(0), "Tombstone Owner ID is wrong"
assert header["objectType"] == "TOMBSTONE", "Header Type isn't Tombstone"
assert header["sessionToken"]["body"]["object"]["verb"] == "DELETE", "Header Session Type isn't DELETE"
assert header["sessionToken"]["body"]["object"]["target"]["container"] == cid, "Header Session ID is wrong"

10
src/frostfs_testlib/storage/controllers/background_load_controller.py
View file

@ -1,6 +1,5 @@
import copy
from datetime import datetime
from typing import Optional
import frostfs_testlib.resources.optionals as optionals
from frostfs_testlib import reporter
@ -10,7 +9,6 @@ from frostfs_testlib.load.load_report import LoadReport
from frostfs_testlib.load.load_verifiers import LoadVerifier
from frostfs_testlib.storage.cluster import ClusterNode
from frostfs_testlib.storage.dataclasses.frostfs_services import S3Gate, StorageNode
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.parallel import parallel
from frostfs_testlib.testing.test_control import run_optionally
@ -23,7 +21,6 @@ class BackgroundLoadController:
cluster_nodes: list[ClusterNode]
nodes_under_load: list[ClusterNode]
load_counter: int
loaders_wallet: WalletInfo
load_summaries: dict
endpoints: list[str]
runner: ScenarioRunner
@ -34,7 +31,6 @@ class BackgroundLoadController:
self,
k6_dir: str,
load_params: LoadParams,
loaders_wallet: WalletInfo,
cluster_nodes: list[ClusterNode],
nodes_under_load: list[ClusterNode],
runner: ScenarioRunner,
@ -45,7 +41,6 @@ class BackgroundLoadController:
self.cluster_nodes = cluster_nodes
self.nodes_under_load = nodes_under_load
self.load_counter = 1
self.loaders_wallet = loaders_wallet
self.runner = runner
self.started = False
self.load_reporters = []
@ -64,10 +59,7 @@ class BackgroundLoadController:
)
),
EndpointSelectionStrategy.FIRST: list(
set(
node_under_load.service(StorageNode).get_rpc_endpoint()
for node_under_load in self.nodes_under_load
)
set(node_under_load.service(StorageNode).get_rpc_endpoint() for node_under_load in self.nodes_under_load)
),
},
# for some reason xk6 appends http protocol on its own

37
src/frostfs_testlib/storage/controllers/cluster_state_controller.py
View file

@ -11,12 +11,13 @@ from frostfs_testlib.healthcheck.interfaces import Healthcheck
from frostfs_testlib.hosting.interfaces import HostStatus
from frostfs_testlib.plugins import load_all
from frostfs_testlib.resources.cli import FROSTFS_ADM_CONFIG_PATH, FROSTFS_ADM_EXEC, FROSTFS_CLI_EXEC
from frostfs_testlib.resources.common import DEFAULT_WALLET_CONFIG, MORPH_BLOCK_TIME
from frostfs_testlib.resources.common import MORPH_BLOCK_TIME
from frostfs_testlib.shell import CommandOptions, Shell, SshConnectionProvider
from frostfs_testlib.steps.network import IpHelper
from frostfs_testlib.storage.cluster import Cluster, ClusterNode, S3Gate, StorageNode
from frostfs_testlib.storage.controllers.disk_controller import DiskController
from frostfs_testlib.storage.dataclasses.node_base import NodeBase, ServiceClass
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing import parallel
from frostfs_testlib.testing.test_control import retry, run_optionally, wait_for_success
from frostfs_testlib.utils.datetime_utils import parse_time
@ -413,12 +414,12 @@ class ClusterStateController:
frostfs_adm.morph.set_config(set_key_value=f"MaintenanceModeAllowed={status}")
@reporter.step("Set mode node to {status}")
def set_mode_node(self, cluster_node: ClusterNode, wallet: str, status: str, await_tick: bool = True) -> None:
def set_mode_node(self, cluster_node: ClusterNode, wallet: WalletInfo, status: str, await_tick: bool = True) -> None:
rpc_endpoint = cluster_node.storage_node.get_rpc_endpoint()
control_endpoint = cluster_node.service(StorageNode).get_control_endpoint()
frostfs_adm, frostfs_cli, frostfs_cli_remote = self._get_cli(local_shell=self.shell, cluster_node=cluster_node)
node_netinfo = NetmapParser.netinfo(frostfs_cli.netmap.netinfo(rpc_endpoint=rpc_endpoint, wallet=wallet).stdout)
frostfs_adm, frostfs_cli, frostfs_cli_remote = self._get_cli(local_shell=self.shell, local_wallet=wallet, cluster_node=cluster_node)
node_netinfo = NetmapParser.netinfo(frostfs_cli.netmap.netinfo(rpc_endpoint=rpc_endpoint).stdout)
with reporter.step("If status maintenance, then check that the option is enabled"):
if node_netinfo.maintenance_mode_allowed == "false":
@ -437,12 +438,10 @@ class ClusterStateController:
self.check_node_status(status=status, wallet=wallet, cluster_node=cluster_node)
@wait_for_success(80, 8, title="Wait for storage status become {status}")
def check_node_status(self, status: str, wallet: str, cluster_node: ClusterNode):
frostfs_cli = FrostfsCli(
shell=self.shell, frostfs_cli_exec_path=FROSTFS_CLI_EXEC, config_file=DEFAULT_WALLET_CONFIG
)
def check_node_status(self, status: str, wallet: WalletInfo, cluster_node: ClusterNode):
frostfs_cli = FrostfsCli(self.shell, FROSTFS_CLI_EXEC, wallet.config_path)
netmap = NetmapParser.snapshot_all_nodes(
frostfs_cli.netmap.snapshot(rpc_endpoint=cluster_node.storage_node.get_rpc_endpoint(), wallet=wallet).stdout
frostfs_cli.netmap.snapshot(rpc_endpoint=cluster_node.storage_node.get_rpc_endpoint()).stdout
)
netmap = [node for node in netmap if cluster_node.host_ip == node.node]
if status == "offline":
@ -450,7 +449,9 @@ class ClusterStateController:
else:
assert netmap[0].node_status == status.upper(), f"Node state - {netmap[0].node_status} != {status} expect"
def _get_cli(self, local_shell: Shell, cluster_node: ClusterNode) -> tuple[FrostfsAdm, FrostfsCli, FrostfsCli]:
def _get_cli(
self, local_shell: Shell, local_wallet: WalletInfo, cluster_node: ClusterNode
) -> tuple[FrostfsAdm, FrostfsCli, FrostfsCli]:
# TODO Move to service config
host = cluster_node.host
service_config = host.get_service_config(cluster_node.storage_node.name)
@ -462,12 +463,8 @@ class ClusterStateController:
wallet_config = f'wallet: {wallet_path}\npassword: "{wallet_password}"'
shell.exec(f"echo '{wallet_config}' > {wallet_config_path}")
frostfs_adm = FrostfsAdm(
shell=shell, frostfs_adm_exec_path=FROSTFS_ADM_EXEC, config_file=FROSTFS_ADM_CONFIG_PATH
)
frostfs_cli = FrostfsCli(
shell=local_shell, frostfs_cli_exec_path=FROSTFS_CLI_EXEC, config_file=DEFAULT_WALLET_CONFIG
)
frostfs_adm = FrostfsAdm(shell=shell, frostfs_adm_exec_path=FROSTFS_ADM_EXEC, config_file=FROSTFS_ADM_CONFIG_PATH)
frostfs_cli = FrostfsCli(local_shell, FROSTFS_CLI_EXEC, local_wallet.config_path)
frostfs_cli_remote = FrostfsCli(
shell=shell,
frostfs_cli_exec_path=FROSTFS_CLI_EXEC,
@ -511,9 +508,7 @@ class ClusterStateController:
options = CommandOptions(check=False)
return self.shell.exec(f"ping {node.host.config.address} -c 1", options).return_code
@retry(
max_attempts=60, sleep_interval=10, expected_result=HostStatus.ONLINE, title="Waiting for {node} to go online"
)
@retry(max_attempts=60, sleep_interval=10, expected_result=HostStatus.ONLINE, title="Waiting for {node} to go online")
def _wait_for_host_online(self, node: ClusterNode):
try:
ping_result = self._ping_host(node)
@ -524,9 +519,7 @@ class ClusterStateController:
logger.warning(f"Host ping fails with error {err}")
return HostStatus.OFFLINE
@retry(
max_attempts=60, sleep_interval=10, expected_result=HostStatus.OFFLINE, title="Waiting for {node} to go offline"
)
@retry(max_attempts=60, sleep_interval=10, expected_result=HostStatus.OFFLINE, title="Waiting for {node} to go offline")
def _wait_for_host_offline(self, node: ClusterNode):
try:
ping_result = self._ping_host(node)

20
src/frostfs_testlib/storage/dataclasses/acl.py
View file

@ -1,8 +1,8 @@
import logging
from dataclasses import dataclass
from enum import Enum
from typing import Any, Dict, List, Optional, Union
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.readable import HumanReadableEnum
from frostfs_testlib.utils import wallet_utils
@ -65,11 +65,7 @@ class EACLFilters:
def __str__(self):
return ",".join(
[
f"{filter.header_type.value}:"
f"{filter.key}{filter.match_type.value}{filter.value}"
for filter in self.filters
]
[f"{filter.header_type.value}:" f"{filter.key}{filter.match_type.value}{filter.value}" for filter in self.filters]
if self.filters
else []
)
@ -84,7 +80,7 @@ class EACLPubKey:
class EACLRule:
operation: Optional[EACLOperation] = None
access: Optional[EACLAccess] = None
role: Optional[Union[EACLRole, str]] = None
role: Optional[Union[EACLRole, WalletInfo]] = None
filters: Optional[EACLFilters] = None
def to_dict(self) -> Dict[str, Any]:
@ -96,9 +92,9 @@ class EACLRule:
}
def __str__(self):
role = (
self.role.value
if isinstance(self.role, EACLRole)
else f'pubkey:{wallet_utils.get_wallet_public_key(self.role, "")}'
)
role = ""
if isinstance(self.role, EACLRole):
role = self.role.value
if isinstance(self.role, WalletInfo):
role = f"pubkey:{wallet_utils.get_wallet_public_key(self.role.path, self.role.password)}"
return f'{self.access.value} {self.operation.value} {self.filters or ""} {role}'

3
src/frostfs_testlib/storage/dataclasses/storage_object_info.py
View file

@ -1,6 +1,7 @@
from dataclasses import dataclass
from typing import Optional
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.readable import HumanReadableEnum
@ -19,7 +20,7 @@ class LockObjectInfo(ObjectRef):
@dataclass
class StorageObjectInfo(ObjectRef):
size: Optional[int] = None
wallet_file_path: Optional[str] = None
wallet: Optional[WalletInfo] = None
file_path: Optional[str] = None
file_hash: Optional[str] = None
attributes: Optional[list[dict[str, str]]] = None

35
src/frostfs_testlib/storage/dataclasses/wallet.py
View file

@ -1,13 +1,15 @@
import json
import logging
import os
import uuid
from dataclasses import dataclass
from typing import Optional
from frostfs_testlib.resources.common import DEFAULT_WALLET_CONFIG, DEFAULT_WALLET_PASS
import yaml
from frostfs_testlib import reporter
from frostfs_testlib.resources.common import ASSETS_DIR, DEFAULT_WALLET_CONFIG, DEFAULT_WALLET_PASS
from frostfs_testlib.shell import Shell
from frostfs_testlib.storage.cluster import Cluster, NodeBase
from frostfs_testlib.storage.cluster import NodeBase
from frostfs_testlib.utils.wallet_utils import get_last_address_from_wallet, init_wallet
logger = logging.getLogger("frostfs.testlib.utils")
@ -21,9 +23,13 @@ class WalletInfo:
@staticmethod
def from_node(node: NodeBase):
return WalletInfo(
node.get_wallet_path(), node.get_wallet_password(), node.get_wallet_config_path()
)
wallet_path = node.get_wallet_path()
wallet_password = node.get_wallet_password()
wallet_config_file = os.path.join(ASSETS_DIR, os.path.basename(node.get_wallet_config_path()))
with open(wallet_config_file, "w") as file:
file.write(yaml.dump({"wallet": wallet_path, "password": wallet_password}))
return WalletInfo(wallet_path, wallet_password, wallet_config_file)
def get_address(self) -> str:
"""
@ -47,22 +53,17 @@ class WalletInfo:
"""
with open(self.path, "r") as wallet:
wallet_json = json.load(wallet)
assert abs(account_id) + 1 <= len(
wallet_json["accounts"]
), f"There is no index '{account_id}' in wallet: {wallet_json}"
assert abs(account_id) + 1 <= len(wallet_json["accounts"]), f"There is no index '{account_id}' in wallet: {wallet_json}"
return wallet_json["accounts"][account_id]["address"]
class WalletFactory:
def __init__(self, wallets_dir: str, shell: Shell, cluster: Cluster) -> None:
def __init__(self, wallets_dir: str, shell: Shell) -> None:
self.shell = shell
self.wallets_dir = wallets_dir
self.cluster = cluster
def create_wallet(
self, file_name: Optional[str] = None, password: Optional[str] = None
) -> WalletInfo:
def create_wallet(self, file_name: str, password: Optional[str] = None) -> WalletInfo:
"""
Creates new default wallet.
@ -74,8 +75,6 @@ class WalletFactory:
WalletInfo object of new wallet.
"""
if file_name is None:
file_name = str(uuid.uuid4())
if password is None:
password = ""
@ -85,6 +84,8 @@ class WalletFactory:
init_wallet(wallet_path, password)
with open(wallet_config_path, "w") as config_file:
config_file.write(f'password: "{password}"')
config_file.write(f'wallet: {wallet_path}\npassword: "{password}"')
reporter.attach(wallet_path, os.path.basename(wallet_path))
return WalletInfo(wallet_path, password, wallet_config_path)

18
src/frostfs_testlib/utils/version_utils.py
View file

@ -4,7 +4,6 @@ import re
from frostfs_testlib.cli import FrostfsAdm, FrostfsCli
from frostfs_testlib.hosting import Host, Hosting
from frostfs_testlib.resources.cli import FROSTFS_ADM_EXEC, FROSTFS_AUTHMATE_EXEC, FROSTFS_CLI_EXEC, NEOGO_EXECUTABLE
from frostfs_testlib.resources.common import DEFAULT_WALLET_CONFIG
from frostfs_testlib.shell import Shell
from frostfs_testlib.testing.parallel import parallel
@ -18,7 +17,7 @@ def get_local_binaries_versions(shell: Shell) -> dict[str, str]:
out = shell.exec(f"{binary} --version").stdout
versions[binary] = _parse_version(out)
frostfs_cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, DEFAULT_WALLET_CONFIG)
frostfs_cli = FrostfsCli(shell, FROSTFS_CLI_EXEC)
versions[FROSTFS_CLI_EXEC] = _parse_version(frostfs_cli.version.get().stdout)
try:
@ -83,12 +82,12 @@ def get_remote_binaries_versions(hosting: Hosting) -> dict[str, str]:
for host, binary_versions in versions_by_host.items():
for name, binary in binary_versions.items():
version = binary["version"]
if not cheak_versions.get(f'{name[:-2]}', None):
captured_version = cheak_versions.get(f'{name[:-2]}',{}).get(host, {}).get(captured_version)
cheak_versions[f'{name[:-2]}'] = {host: {version: name}}
if not cheak_versions.get(f"{name[:-2]}", None):
captured_version = cheak_versions.get(f"{name[:-2]}", {}).get(host, {}).get(captured_version)
cheak_versions[f"{name[:-2]}"] = {host: {version: name}}
else:
captured_version = list(cheak_versions.get(f'{name[:-2]}',{}).get(previous_host).keys())[0]
cheak_versions[f'{name[:-2]}'].update({host:{version:name}})
captured_version = list(cheak_versions.get(f"{name[:-2]}", {}).get(previous_host).keys())[0]
cheak_versions[f"{name[:-2]}"].update({host: {version: name}})
if captured_version and captured_version != version:
exception.add(name[:-2])
@ -99,10 +98,11 @@ def get_remote_binaries_versions(hosting: Hosting) -> dict[str, str]:
for i in exception:
for host in versions_by_host.keys():
for version, name in cheak_versions.get(i).get(host).items():
exсeptions.append(f'Binary {name} has inconsistent version {version} on host {host}')
exсeptions.append('\n')
exсeptions.append(f"Binary {name} has inconsistent version {version} on host {host}")
exсeptions.append("\n")
return versions, exсeptions
def _parse_version(version_output: str) -> str:
version = re.search(r"version[:\s]*v?(.+)", version_output, re.IGNORECASE)
return version.group(1).strip() if version else version_output

40
src/frostfs_testlib/utils/wallet_utils.py
View file

@ -9,6 +9,16 @@ from neo3.wallet import wallet as neo3_wallet
logger = logging.getLogger("frostfs.testlib.utils")
def __fix_wallet_schema(wallet: dict) -> None:
# Temporary function to fix wallets that do not conform to the schema
# TODO: get rid of it once issue is solved
if "name" not in wallet:
wallet["name"] = None
for account in wallet["accounts"]:
if "extra" not in account:
account["extra"] = None
def init_wallet(wallet_path: str, wallet_password: str):
"""
Create new wallet and new account.
@ -33,29 +43,15 @@ def get_last_address_from_wallet(wallet_path: str, wallet_password: str):
Returns:
The address for the wallet.
"""
with open(wallet_path) as wallet_file:
wallet = neo3_wallet.Wallet.from_json(json.load(wallet_file), password=wallet_password)
wallet = load_wallet(wallet_path, wallet_password)
address = wallet.accounts[-1].address
logger.info(f"got address: {address}")
return address
def get_wallet_public_key(wallet_path: str, wallet_password: str, format: str = "hex") -> str:
def __fix_wallet_schema(wallet: dict) -> None:
# Temporary function to fix wallets that do not conform to the schema
# TODO: get rid of it once issue is solved
if "name" not in wallet:
wallet["name"] = None
for account in wallet["accounts"]:
if "extra" not in account:
account["extra"] = None
# Get public key from wallet file
with open(wallet_path, "r") as file:
wallet_content = json.load(file)
__fix_wallet_schema(wallet_content)
wallet_from_json = neo3_wallet.Wallet.from_json(wallet_content, password=wallet_password)
public_key_hex = str(wallet_from_json.accounts[0].public_key)
wallet = load_wallet(wallet_path, wallet_password)
public_key_hex = str(wallet.accounts[0].public_key)
# Convert public key to specified format
if format == "hex":
@ -69,7 +65,9 @@ def get_wallet_public_key(wallet_path: str, wallet_password: str, format: str =
raise ValueError(f"Invalid public key format: {format}")
def load_wallet(path: str, passwd: str = "") -> neo3_wallet.Wallet:
with open(path, "r") as wallet_file:
wlt_data = wallet_file.read()
return neo3_wallet.Wallet.from_json(json.loads(wlt_data), password=passwd)
def load_wallet(wallet_path: str, wallet_password: str) -> neo3_wallet.Wallet:
with open(wallet_path) as wallet_file:
wallet_content = json.load(wallet_file)
__fix_wallet_schema(wallet_content)
return neo3_wallet.Wallet.from_json(wallet_content, password=wallet_password)