[#191] Credentials work overhaul

Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
This commit is contained in:
Andrey Berezin 2024-03-11 19:23:10 +03:00
parent 09a7f66d1e
commit 25925c637b
31 changed files with 370 additions and 485 deletions

View file

@ -58,15 +58,19 @@ neo-go = "frostfs_testlib.storage.dataclasses.frostfs_services:MorphChain"
frostfs-ir = "frostfs_testlib.storage.dataclasses.frostfs_services:InnerRing" frostfs-ir = "frostfs_testlib.storage.dataclasses.frostfs_services:InnerRing"
[project.entry-points."frostfs.testlib.credentials_providers"] [project.entry-points."frostfs.testlib.credentials_providers"]
authmate = "frostfs_testlib.credentials.authmate_s3:AuthmateS3CredentialsProvider" authmate = "frostfs_testlib.credentials.authmate_s3_provider:AuthmateS3CredentialsProvider"
wallet_factory = "frostfs_testlib.credentials.wallet_factory_provider:WalletFactoryProvider"
[project.entry-points."frostfs.testlib.bucket_cid_resolver"]
frostfs = "frostfs_testlib.s3.curl_bucket_resolver:CurlBucketContainerResolver"
[tool.isort] [tool.isort]
profile = "black" profile = "black"
src_paths = ["src", "tests"] src_paths = ["src", "tests"]
line_length = 120 line_length = 140
[tool.black] [tool.black]
line-length = 120 line-length = 140
target-version = ["py310"] target-version = ["py310"]
[tool.bumpver] [tool.bumpver]

View file

@ -8,7 +8,7 @@ class FrostfsCliContainer(CliCommand):
def create( def create(
self, self,
rpc_endpoint: str, rpc_endpoint: str,
wallet: str, wallet: Optional[str] = None,
address: Optional[str] = None, address: Optional[str] = None,
attributes: Optional[dict] = None, attributes: Optional[dict] = None,
basic_acl: Optional[str] = None, basic_acl: Optional[str] = None,
@ -57,8 +57,8 @@ class FrostfsCliContainer(CliCommand):
def delete( def delete(
self, self,
rpc_endpoint: str, rpc_endpoint: str,
wallet: str,
cid: str, cid: str,
wallet: Optional[str] = None,
address: Optional[str] = None, address: Optional[str] = None,
await_mode: bool = False, await_mode: bool = False,
session: Optional[str] = None, session: Optional[str] = None,
@ -93,8 +93,8 @@ class FrostfsCliContainer(CliCommand):
def get( def get(
self, self,
rpc_endpoint: str, rpc_endpoint: str,
wallet: str,
cid: str, cid: str,
wallet: Optional[str] = None,
address: Optional[str] = None, address: Optional[str] = None,
await_mode: bool = False, await_mode: bool = False,
to: Optional[str] = None, to: Optional[str] = None,
@ -129,8 +129,8 @@ class FrostfsCliContainer(CliCommand):
def get_eacl( def get_eacl(
self, self,
rpc_endpoint: str, rpc_endpoint: str,
wallet: str,
cid: str, cid: str,
wallet: Optional[str] = None,
address: Optional[str] = None, address: Optional[str] = None,
await_mode: bool = False, await_mode: bool = False,
to: Optional[str] = None, to: Optional[str] = None,
@ -166,7 +166,7 @@ class FrostfsCliContainer(CliCommand):
def list( def list(
self, self,
rpc_endpoint: str, rpc_endpoint: str,
wallet: str, wallet: Optional[str] = None,
address: Optional[str] = None, address: Optional[str] = None,
owner: Optional[str] = None, owner: Optional[str] = None,
ttl: Optional[int] = None, ttl: Optional[int] = None,
@ -197,8 +197,8 @@ class FrostfsCliContainer(CliCommand):
def list_objects( def list_objects(
self, self,
rpc_endpoint: str, rpc_endpoint: str,
wallet: str,
cid: str, cid: str,
wallet: Optional[str] = None,
address: Optional[str] = None, address: Optional[str] = None,
ttl: Optional[int] = None, ttl: Optional[int] = None,
xhdr: Optional[dict] = None, xhdr: Optional[dict] = None,
@ -227,8 +227,8 @@ class FrostfsCliContainer(CliCommand):
def set_eacl( def set_eacl(
self, self,
rpc_endpoint: str, rpc_endpoint: str,
wallet: str,
cid: str, cid: str,
wallet: Optional[str] = None,
address: Optional[str] = None, address: Optional[str] = None,
await_mode: bool = False, await_mode: bool = False,
table: Optional[str] = None, table: Optional[str] = None,
@ -264,8 +264,8 @@ class FrostfsCliContainer(CliCommand):
def search_node( def search_node(
self, self,
rpc_endpoint: str, rpc_endpoint: str,
wallet: str,
cid: str, cid: str,
wallet: Optional[str] = None,
address: Optional[str] = None, address: Optional[str] = None,
ttl: Optional[int] = None, ttl: Optional[int] = None,
from_file: Optional[str] = None, from_file: Optional[str] = None,

View file

@ -8,7 +8,7 @@ class FrostfsCliNetmap(CliCommand):
def epoch( def epoch(
self, self,
rpc_endpoint: str, rpc_endpoint: str,
wallet: str, wallet: Optional[str] = None,
address: Optional[str] = None, address: Optional[str] = None,
generate_key: bool = False, generate_key: bool = False,
ttl: Optional[int] = None, ttl: Optional[int] = None,
@ -38,7 +38,7 @@ class FrostfsCliNetmap(CliCommand):
def netinfo( def netinfo(
self, self,
rpc_endpoint: str, rpc_endpoint: str,
wallet: str, wallet: Optional[str] = None,
address: Optional[str] = None, address: Optional[str] = None,
generate_key: bool = False, generate_key: bool = False,
ttl: Optional[int] = None, ttl: Optional[int] = None,
@ -68,7 +68,7 @@ class FrostfsCliNetmap(CliCommand):
def nodeinfo( def nodeinfo(
self, self,
rpc_endpoint: str, rpc_endpoint: str,
wallet: str, wallet: Optional[str] = None,
address: Optional[str] = None, address: Optional[str] = None,
generate_key: bool = False, generate_key: bool = False,
json: bool = False, json: bool = False,
@ -100,7 +100,7 @@ class FrostfsCliNetmap(CliCommand):
def snapshot( def snapshot(
self, self,
rpc_endpoint: str, rpc_endpoint: str,
wallet: str, wallet: Optional[str] = None,
address: Optional[str] = None, address: Optional[str] = None,
generate_key: bool = False, generate_key: bool = False,
ttl: Optional[int] = None, ttl: Optional[int] = None,

View file

@ -8,9 +8,9 @@ class FrostfsCliObject(CliCommand):
def delete( def delete(
self, self,
rpc_endpoint: str, rpc_endpoint: str,
wallet: str,
cid: str, cid: str,
oid: str, oid: str,
wallet: Optional[str] = None,
address: Optional[str] = None, address: Optional[str] = None,
bearer: Optional[str] = None, bearer: Optional[str] = None,
session: Optional[str] = None, session: Optional[str] = None,
@ -44,9 +44,9 @@ class FrostfsCliObject(CliCommand):
def get( def get(
self, self,
rpc_endpoint: str, rpc_endpoint: str,
wallet: str,
cid: str, cid: str,
oid: str, oid: str,
wallet: Optional[str] = None,
address: Optional[str] = None, address: Optional[str] = None,
bearer: Optional[str] = None, bearer: Optional[str] = None,
file: Optional[str] = None, file: Optional[str] = None,
@ -88,9 +88,9 @@ class FrostfsCliObject(CliCommand):
def hash( def hash(
self, self,
rpc_endpoint: str, rpc_endpoint: str,
wallet: str,
cid: str, cid: str,
oid: str, oid: str,
wallet: Optional[str] = None,
address: Optional[str] = None, address: Optional[str] = None,
bearer: Optional[str] = None, bearer: Optional[str] = None,
range: Optional[str] = None, range: Optional[str] = None,
@ -130,9 +130,9 @@ class FrostfsCliObject(CliCommand):
def head( def head(
self, self,
rpc_endpoint: str, rpc_endpoint: str,
wallet: str,
cid: str, cid: str,
oid: str, oid: str,
wallet: Optional[str] = None,
address: Optional[str] = None, address: Optional[str] = None,
bearer: Optional[str] = None, bearer: Optional[str] = None,
file: Optional[str] = None, file: Optional[str] = None,
@ -176,9 +176,9 @@ class FrostfsCliObject(CliCommand):
def lock( def lock(
self, self,
rpc_endpoint: str, rpc_endpoint: str,
wallet: str,
cid: str, cid: str,
oid: str, oid: str,
wallet: Optional[str] = None,
lifetime: Optional[int] = None, lifetime: Optional[int] = None,
expire_at: Optional[int] = None, expire_at: Optional[int] = None,
address: Optional[str] = None, address: Optional[str] = None,
@ -216,9 +216,9 @@ class FrostfsCliObject(CliCommand):
def put( def put(
self, self,
rpc_endpoint: str, rpc_endpoint: str,
wallet: str,
cid: str, cid: str,
file: str, file: str,
wallet: Optional[str] = None,
address: Optional[str] = None, address: Optional[str] = None,
attributes: Optional[dict] = None, attributes: Optional[dict] = None,
bearer: Optional[str] = None, bearer: Optional[str] = None,
@ -267,10 +267,10 @@ class FrostfsCliObject(CliCommand):
def range( def range(
self, self,
rpc_endpoint: str, rpc_endpoint: str,
wallet: str,
cid: str, cid: str,
oid: str, oid: str,
range: str, range: str,
wallet: Optional[str] = None,
address: Optional[str] = None, address: Optional[str] = None,
bearer: Optional[str] = None, bearer: Optional[str] = None,
file: Optional[str] = None, file: Optional[str] = None,
@ -311,8 +311,8 @@ class FrostfsCliObject(CliCommand):
def search( def search(
self, self,
rpc_endpoint: str, rpc_endpoint: str,
wallet: str,
cid: str, cid: str,
wallet: Optional[str] = None,
address: Optional[str] = None, address: Optional[str] = None,
bearer: Optional[str] = None, bearer: Optional[str] = None,
filters: Optional[list] = None, filters: Optional[list] = None,

View file

@ -9,7 +9,6 @@ class FrostfsCliSession(CliCommand):
self, self,
rpc_endpoint: str, rpc_endpoint: str,
wallet: str, wallet: str,
wallet_password: str,
out: str, out: str,
lifetime: Optional[int] = None, lifetime: Optional[int] = None,
address: Optional[str] = None, address: Optional[str] = None,
@ -30,12 +29,7 @@ class FrostfsCliSession(CliCommand):
Returns: Returns:
Command's result. Command's result.
""" """
return self._execute_with_password( return self._execute(
"session create", "session create",
wallet_password, **{param: value for param, value in locals().items() if param not in ["self"]},
**{
param: value
for param, value in locals().items()
if param not in ["self", "wallet_password"]
},
) )

View file

@ -6,12 +6,12 @@ from frostfs_testlib.shell import CommandResult
class FrostfsCliUtil(CliCommand): class FrostfsCliUtil(CliCommand):
def sign_bearer_token( def sign_bearer_token(
self, self,
wallet: str, from_file: str,
from_file: str, to_file: str,
to_file: str, wallet: Optional[str] = None,
address: Optional[str] = None, address: Optional[str] = None,
json: Optional[bool] = False, json: Optional[bool] = False,
) -> CommandResult: ) -> CommandResult:
""" """
Sign bearer token to use it in requests. Sign bearer token to use it in requests.
@ -33,9 +33,9 @@ class FrostfsCliUtil(CliCommand):
def sign_session_token( def sign_session_token(
self, self,
wallet: str,
from_file: str, from_file: str,
to_file: str, to_file: str,
wallet: Optional[str] = None,
address: Optional[str] = None, address: Optional[str] = None,
) -> CommandResult: ) -> CommandResult:
""" """

View file

@ -1,25 +1,26 @@
import re import re
from datetime import datetime from datetime import datetime
from typing import Optional
from frostfs_testlib import reporter from frostfs_testlib import reporter
from frostfs_testlib.cli import FrostfsAuthmate from frostfs_testlib.cli import FrostfsAuthmate
from frostfs_testlib.credentials.interfaces import S3CredentialsProvider from frostfs_testlib.credentials.interfaces import S3Credentials, S3CredentialsProvider, User
from frostfs_testlib.resources.cli import FROSTFS_AUTHMATE_EXEC from frostfs_testlib.resources.cli import FROSTFS_AUTHMATE_EXEC
from frostfs_testlib.shell import Shell from frostfs_testlib.shell import LocalShell
from frostfs_testlib.steps.cli.container import list_containers from frostfs_testlib.steps.cli.container import list_containers
from frostfs_testlib.storage.cluster import Cluster, ClusterNode from frostfs_testlib.storage.cluster import ClusterNode
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo from frostfs_testlib.storage.dataclasses.frostfs_services import S3Gate
class AuthmateS3CredentialsProvider(S3CredentialsProvider): class AuthmateS3CredentialsProvider(S3CredentialsProvider):
@reporter.step("Init S3 Credentials using Authmate CLI") @reporter.step("Init S3 Credentials using Authmate CLI")
def provide(self, cluster_node: ClusterNode) -> tuple[str, str]: def provide(self, user: User, cluster_node: ClusterNode, location_constraints: Optional[str] = None) -> S3Credentials:
cluster: Cluster = self.stash["cluster"] cluster_nodes: list[ClusterNode] = self.cluster.cluster_nodes
shell: Shell = self.stash["shell"] shell = LocalShell()
wallet: WalletInfo = self.stash["wallet"] wallet = user.wallet
endpoint = cluster_node.storage_node.get_rpc_endpoint() endpoint = cluster_node.storage_node.get_rpc_endpoint()
gate_public_keys = [s3gate.get_wallet_public_key() for s3gate in cluster.s3_gates] gate_public_keys = [node.service(S3Gate).get_wallet_public_key() for node in cluster_nodes]
# unique short bucket name # unique short bucket name
bucket = f"bucket_{hex(int(datetime.now().timestamp()*1000000))}" bucket = f"bucket_{hex(int(datetime.now().timestamp()*1000000))}"
@ -29,21 +30,18 @@ class AuthmateS3CredentialsProvider(S3CredentialsProvider):
peer=endpoint, peer=endpoint,
gate_public_key=gate_public_keys, gate_public_key=gate_public_keys,
wallet_password=wallet.password, wallet_password=wallet.password,
container_policy=self.stash.get("location_constraints"), container_policy=location_constraints,
container_friendly_name=bucket, container_friendly_name=bucket,
).stdout ).stdout
aws_access_key_id = str( aws_access_key_id = str(re.search(r"access_key_id.*:\s.(?P<aws_access_key_id>\w*)", issue_secret_output).group("aws_access_key_id"))
re.search(r"access_key_id.*:\s.(?P<aws_access_key_id>\w*)", issue_secret_output).group("aws_access_key_id")
)
aws_secret_access_key = str( aws_secret_access_key = str(
re.search(r"secret_access_key.*:\s.(?P<aws_secret_access_key>\w*)", issue_secret_output).group( re.search(r"secret_access_key.*:\s.(?P<aws_secret_access_key>\w*)", issue_secret_output).group("aws_secret_access_key")
"aws_secret_access_key"
)
) )
cid = str(re.search(r"container_id.*:\s.(?P<container_id>\w*)", issue_secret_output).group("container_id")) cid = str(re.search(r"container_id.*:\s.(?P<container_id>\w*)", issue_secret_output).group("container_id"))
containers_list = list_containers(wallet.path, shell, endpoint) containers_list = list_containers(wallet.path, shell, endpoint)
assert cid in containers_list, f"Expected cid {cid} in {containers_list}" assert cid in containers_list, f"Expected cid {cid} in {containers_list}"
return aws_access_key_id, aws_secret_access_key user.s3_credentials = S3Credentials(aws_access_key_id, aws_secret_access_key)
return user.s3_credentials

View file

@ -1,25 +1,51 @@
from abc import abstractmethod from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from typing import Any, Optional
from frostfs_testlib.plugins import load_plugin from frostfs_testlib.plugins import load_plugin
from frostfs_testlib.storage.cluster import ClusterNode from frostfs_testlib.storage.cluster import Cluster, ClusterNode
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
class S3CredentialsProvider(object): @dataclass
stash: dict class S3Credentials:
access_key: str
secret_key: str
def __init__(self, stash: dict) -> None:
self.stash = stash @dataclass
class User:
name: str
attributes: dict[str, Any] = field(default_factory=dict)
wallet: WalletInfo | None = None
s3_credentials: S3Credentials | None = None
class S3CredentialsProvider(ABC):
def __init__(self, cluster: Cluster) -> None:
self.cluster = cluster
@abstractmethod @abstractmethod
def provide(self, cluster_node: ClusterNode) -> tuple[str, str]: def provide(self, user: User, cluster_node: ClusterNode, location_constraints: Optional[str] = None) -> S3Credentials:
raise NotImplementedError("Directly called abstract class?")
class GrpcCredentialsProvider(ABC):
def __init__(self, cluster: Cluster) -> None:
self.cluster = cluster
@abstractmethod
def provide(self, user: User, cluster_node: ClusterNode) -> WalletInfo:
raise NotImplementedError("Directly called abstract class?") raise NotImplementedError("Directly called abstract class?")
class CredentialsProvider(object): class CredentialsProvider(object):
stash: dict
S3: S3CredentialsProvider S3: S3CredentialsProvider
GRPC: GrpcCredentialsProvider
def __init__(self, s3_plugin_name: str) -> None: def __init__(self, cluster: Cluster) -> None:
self.stash = {} config = cluster.cluster_nodes[0].host.config
s3cls = load_plugin("frostfs.testlib.credentials_providers", s3_plugin_name) s3_cls = load_plugin("frostfs.testlib.credentials_providers", config.s3_creds_plugin_name)
self.S3 = s3cls(self.stash) self.S3 = s3_cls(cluster)
grpc_cls = load_plugin("frostfs.testlib.credentials_providers", config.grpc_creds_plugin_name)
self.GRPC = grpc_cls(cluster)

View file

@ -0,0 +1,14 @@
from frostfs_testlib import reporter
from frostfs_testlib.credentials.interfaces import GrpcCredentialsProvider, User
from frostfs_testlib.resources.common import ASSETS_DIR, DEFAULT_WALLET_PASS
from frostfs_testlib.shell.local_shell import LocalShell
from frostfs_testlib.storage.cluster import ClusterNode
from frostfs_testlib.storage.dataclasses.wallet import WalletFactory, WalletInfo
class WalletFactoryProvider(GrpcCredentialsProvider):
@reporter.step("Init gRPC Credentials using wallet generation")
def provide(self, user: User, cluster_node: ClusterNode) -> WalletInfo:
wallet_factory = WalletFactory(ASSETS_DIR, LocalShell())
user.wallet = wallet_factory.create_wallet(file_name=user, password=DEFAULT_WALLET_PASS)
return user.wallet

View file

@ -63,6 +63,8 @@ class HostConfig:
healthcheck_plugin_name: str healthcheck_plugin_name: str
address: str address: str
s3_creds_plugin_name: str = field(default="authmate") s3_creds_plugin_name: str = field(default="authmate")
grpc_creds_plugin_name: str = field(default="wallet_factory")
product: str = field(default="frostfs")
services: list[ServiceConfig] = field(default_factory=list) services: list[ServiceConfig] = field(default_factory=list)
clis: list[CLIConfig] = field(default_factory=list) clis: list[CLIConfig] = field(default_factory=list)
attributes: dict[str, str] = field(default_factory=dict) attributes: dict[str, str] = field(default_factory=dict)

View file

@ -9,13 +9,13 @@ from typing import Any
from urllib.parse import urlparse from urllib.parse import urlparse
from frostfs_testlib import reporter from frostfs_testlib import reporter
from frostfs_testlib.credentials.interfaces import User
from frostfs_testlib.load.interfaces.loader import Loader from frostfs_testlib.load.interfaces.loader import Loader
from frostfs_testlib.load.load_config import K6ProcessAllocationStrategy, LoadParams, LoadScenario, LoadType from frostfs_testlib.load.load_config import K6ProcessAllocationStrategy, LoadParams, LoadScenario, LoadType
from frostfs_testlib.processes.remote_process import RemoteProcess from frostfs_testlib.processes.remote_process import RemoteProcess
from frostfs_testlib.resources.common import STORAGE_USER_NAME from frostfs_testlib.resources.common import STORAGE_USER_NAME
from frostfs_testlib.resources.load_params import K6_STOP_SIGNAL_TIMEOUT, K6_TEARDOWN_PERIOD from frostfs_testlib.resources.load_params import K6_STOP_SIGNAL_TIMEOUT, K6_TEARDOWN_PERIOD
from frostfs_testlib.shell import Shell from frostfs_testlib.shell import Shell
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.test_control import wait_for_success from frostfs_testlib.testing.test_control import wait_for_success
EXIT_RESULT_CODE = 0 EXIT_RESULT_CODE = 0
@ -42,16 +42,16 @@ class K6:
k6_dir: str, k6_dir: str,
shell: Shell, shell: Shell,
loader: Loader, loader: Loader,
wallet: WalletInfo, user: User,
): ):
if load_params.scenario is None: if load_params.scenario is None:
raise RuntimeError("Scenario should not be none") raise RuntimeError("Scenario should not be none")
self.load_params: LoadParams = load_params self.load_params = load_params
self.endpoints = endpoints self.endpoints = endpoints
self.loader: Loader = loader self.loader = loader
self.shell: Shell = shell self.shell = shell
self.wallet = wallet self.user = user
self.preset_output: str = "" self.preset_output: str = ""
self.summary_json: str = os.path.join( self.summary_json: str = os.path.join(
self.load_params.working_dir, self.load_params.working_dir,
@ -64,13 +64,9 @@ class K6:
f"{self._generate_env_variables()}{self._k6_dir}/k6 run {self._generate_k6_variables()} " f"{self._generate_env_variables()}{self._k6_dir}/k6 run {self._generate_k6_variables()} "
f"{self._k6_dir}/scenarios/{self.load_params.scenario.value}.js" f"{self._k6_dir}/scenarios/{self.load_params.scenario.value}.js"
) )
user = STORAGE_USER_NAME if self.load_params.scenario == LoadScenario.LOCAL else None remote_user = STORAGE_USER_NAME if self.load_params.scenario == LoadScenario.LOCAL else None
process_id = ( process_id = self.load_params.load_id if self.load_params.scenario != LoadScenario.VERIFY else f"{self.load_params.load_id}_verify"
self.load_params.load_id self._k6_process = RemoteProcess.create(command, self.shell, self.load_params.working_dir, remote_user, process_id)
if self.load_params.scenario != LoadScenario.VERIFY
else f"{self.load_params.load_id}_verify"
)
self._k6_process = RemoteProcess.create(command, self.shell, self.load_params.working_dir, user, process_id)
def _get_fill_percents(self): def _get_fill_percents(self):
fill_percents = self.shell.exec("df -H --output=source,pcent,target | grep frostfs").stdout.split("\n") fill_percents = self.shell.exec("df -H --output=source,pcent,target | grep frostfs").stdout.split("\n")
@ -103,8 +99,8 @@ class K6:
preset_grpc: [ preset_grpc: [
preset_grpc, preset_grpc,
f"--endpoint {','.join(self.endpoints)}", f"--endpoint {','.join(self.endpoints)}",
f"--wallet {self.wallet.path} ", f"--wallet {self.user.wallet.path} ",
f"--config {self.wallet.config_path} ", f"--config {self.user.wallet.config_path} ",
], ],
preset_s3: [ preset_s3: [
preset_s3, preset_s3,
@ -167,9 +163,7 @@ class K6:
remaining_time = timeout - working_time remaining_time = timeout - working_time
setup_teardown_time = ( setup_teardown_time = (
int(K6_TEARDOWN_PERIOD) int(K6_TEARDOWN_PERIOD) + self.load_params.get_init_time() + int(self.load_params.setup_timeout.replace("s", "").strip())
+ self.load_params.get_init_time()
+ int(self.load_params.setup_timeout.replace("s", "").strip())
) )
remaining_time_including_setup_and_teardown = remaining_time + setup_teardown_time remaining_time_including_setup_and_teardown = remaining_time + setup_teardown_time
timeout = remaining_time_including_setup_and_teardown timeout = remaining_time_including_setup_and_teardown
@ -201,9 +195,7 @@ class K6:
if not self.load_params.fill_percent is None: if not self.load_params.fill_percent is None:
with reporter.step(f"Check the percentage of filling of all data disks on the node"): with reporter.step(f"Check the percentage of filling of all data disks on the node"):
if self.check_fill_percent(): if self.check_fill_percent():
logger.info( logger.info(f"Stopping load on because disks is filled more then {self.load_params.fill_percent}%")
f"Stopping load on because disks is filled more then {self.load_params.fill_percent}%"
)
event.set() event.set()
self.stop() self.stop()
return return

View file

@ -1,24 +1,20 @@
import copy import copy
import itertools import itertools
import math import math
import re
import time import time
from dataclasses import fields from dataclasses import fields
from threading import Event from threading import Event
from typing import Optional from typing import Optional
from urllib.parse import urlparse from urllib.parse import urlparse
import yaml
from frostfs_testlib import reporter from frostfs_testlib import reporter
from frostfs_testlib.cli.frostfs_authmate.authmate import FrostfsAuthmate from frostfs_testlib.credentials.interfaces import S3Credentials, User
from frostfs_testlib.load.interfaces.loader import Loader from frostfs_testlib.load.interfaces.loader import Loader
from frostfs_testlib.load.interfaces.scenario_runner import ScenarioRunner from frostfs_testlib.load.interfaces.scenario_runner import ScenarioRunner
from frostfs_testlib.load.k6 import K6 from frostfs_testlib.load.k6 import K6
from frostfs_testlib.load.load_config import K6ProcessAllocationStrategy, LoadParams, LoadType from frostfs_testlib.load.load_config import K6ProcessAllocationStrategy, LoadParams, LoadType
from frostfs_testlib.load.loaders import NodeLoader, RemoteLoader from frostfs_testlib.load.loaders import NodeLoader, RemoteLoader
from frostfs_testlib.resources import optionals from frostfs_testlib.resources import optionals
from frostfs_testlib.resources.cli import FROSTFS_AUTHMATE_EXEC
from frostfs_testlib.resources.common import STORAGE_USER_NAME from frostfs_testlib.resources.common import STORAGE_USER_NAME
from frostfs_testlib.resources.load_params import BACKGROUND_LOAD_VUS_COUNT_DIVISOR, LOAD_NODE_SSH_USER, LOAD_NODES from frostfs_testlib.resources.load_params import BACKGROUND_LOAD_VUS_COUNT_DIVISOR, LOAD_NODE_SSH_USER, LOAD_NODES
from frostfs_testlib.shell.command_inspectors import SuInspector from frostfs_testlib.shell.command_inspectors import SuInspector
@ -26,7 +22,6 @@ from frostfs_testlib.shell.interfaces import CommandOptions, InteractiveInput
from frostfs_testlib.storage.cluster import ClusterNode from frostfs_testlib.storage.cluster import ClusterNode
from frostfs_testlib.storage.controllers.cluster_state_controller import ClusterStateController from frostfs_testlib.storage.controllers.cluster_state_controller import ClusterStateController
from frostfs_testlib.storage.dataclasses.frostfs_services import S3Gate, StorageNode from frostfs_testlib.storage.dataclasses.frostfs_services import S3Gate, StorageNode
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing import parallel, run_optionally from frostfs_testlib.testing import parallel, run_optionally
from frostfs_testlib.testing.test_control import retry from frostfs_testlib.testing.test_control import retry
from frostfs_testlib.utils import datetime_utils from frostfs_testlib.utils import datetime_utils
@ -57,17 +52,17 @@ class RunnerBase(ScenarioRunner):
class DefaultRunner(RunnerBase): class DefaultRunner(RunnerBase):
loaders: list[Loader] loaders: list[Loader]
loaders_wallet: WalletInfo user: User
def __init__( def __init__(
self, self,
loaders_wallet: WalletInfo, user: User,
load_ip_list: Optional[list[str]] = None, load_ip_list: Optional[list[str]] = None,
) -> None: ) -> None:
if load_ip_list is None: if load_ip_list is None:
load_ip_list = LOAD_NODES load_ip_list = LOAD_NODES
self.loaders = RemoteLoader.from_ip_list(load_ip_list) self.loaders = RemoteLoader.from_ip_list(load_ip_list)
self.loaders_wallet = loaders_wallet self.user = user
@run_optionally(optionals.OPTIONAL_BACKGROUND_LOAD_ENABLED) @run_optionally(optionals.OPTIONAL_BACKGROUND_LOAD_ENABLED)
@reporter.step("Preparation steps") @reporter.step("Preparation steps")
@ -86,55 +81,27 @@ class DefaultRunner(RunnerBase):
return return
with reporter.step("Init s3 client on loaders"): with reporter.step("Init s3 client on loaders"):
storage_node = nodes_under_load[0].service(StorageNode) s3_credentials = self.user.s3_credentials
s3_public_keys = [node.service(S3Gate).get_wallet_public_key() for node in cluster_nodes] parallel(self._aws_configure_on_loader, self.loaders, s3_credentials)
grpc_peer = storage_node.get_rpc_endpoint()
parallel(self._prepare_loader, self.loaders, load_params, grpc_peer, s3_public_keys, k6_dir)
def _force_fresh_registry(self, loader: Loader, load_params: LoadParams): def _force_fresh_registry(self, loader: Loader, load_params: LoadParams):
with reporter.step(f"Forcing fresh registry on {loader.ip}"): with reporter.step(f"Forcing fresh registry on {loader.ip}"):
shell = loader.get_shell() shell = loader.get_shell()
shell.exec(f"rm -f {load_params.registry_file}") shell.exec(f"rm -f {load_params.registry_file}")
def _prepare_loader( def _aws_configure_on_loader(
self, self,
loader: Loader, loader: Loader,
load_params: LoadParams, s3_credentials: S3Credentials,
grpc_peer: str,
s3_public_keys: list[str],
k6_dir: str,
): ):
with reporter.step(f"Init s3 client on {loader.ip}"): with reporter.step(f"Aws configure on {loader.ip}"):
shell = loader.get_shell()
frostfs_authmate_exec: FrostfsAuthmate = FrostfsAuthmate(shell, FROSTFS_AUTHMATE_EXEC)
issue_secret_output = frostfs_authmate_exec.secret.issue(
wallet=self.loaders_wallet.path,
peer=grpc_peer,
gate_public_key=s3_public_keys,
container_placement_policy=load_params.preset.container_placement_policy,
container_policy=f"{k6_dir}/scenarios/files/policy.json",
wallet_password=self.loaders_wallet.password,
).stdout
aws_access_key_id = str(
re.search(r"access_key_id.*:\s.(?P<aws_access_key_id>\w*)", issue_secret_output).group(
"aws_access_key_id"
)
)
aws_secret_access_key = str(
re.search(
r"secret_access_key.*:\s.(?P<aws_secret_access_key>\w*)",
issue_secret_output,
).group("aws_secret_access_key")
)
configure_input = [ configure_input = [
InteractiveInput(prompt_pattern=r"AWS Access Key ID.*", input=aws_access_key_id), InteractiveInput(prompt_pattern=r"AWS Access Key ID.*", input=s3_credentials.access_key),
InteractiveInput(prompt_pattern=r"AWS Secret Access Key.*", input=aws_secret_access_key), InteractiveInput(prompt_pattern=r"AWS Secret Access Key.*", input=s3_credentials.secret_key),
InteractiveInput(prompt_pattern=r".*", input=""), InteractiveInput(prompt_pattern=r".*", input=""),
InteractiveInput(prompt_pattern=r".*", input=""), InteractiveInput(prompt_pattern=r".*", input=""),
] ]
shell.exec("aws configure", CommandOptions(interactive_inputs=configure_input)) loader.get_shell().exec("aws configure", CommandOptions(interactive_inputs=configure_input))
@reporter.step("Init k6 instances") @reporter.step("Init k6 instances")
def init_k6_instances(self, load_params: LoadParams, endpoints: list[str], k6_dir: str): def init_k6_instances(self, load_params: LoadParams, endpoints: list[str], k6_dir: str):
@ -176,12 +143,10 @@ class DefaultRunner(RunnerBase):
k6_dir, k6_dir,
shell, shell,
loader, loader,
self.loaders_wallet, self.user,
) )
def _get_distributed_load_params_list( def _get_distributed_load_params_list(self, original_load_params: LoadParams, workers_count: int) -> list[LoadParams]:
self, original_load_params: LoadParams, workers_count: int
) -> list[LoadParams]:
divisor = int(BACKGROUND_LOAD_VUS_COUNT_DIVISOR) divisor = int(BACKGROUND_LOAD_VUS_COUNT_DIVISOR)
distributed_load_params: list[LoadParams] = [] distributed_load_params: list[LoadParams] = []
@ -266,18 +231,20 @@ class LocalRunner(RunnerBase):
loaders: list[Loader] loaders: list[Loader]
cluster_state_controller: ClusterStateController cluster_state_controller: ClusterStateController
file_keeper: FileKeeper file_keeper: FileKeeper
wallet: WalletInfo user: User
def __init__( def __init__(
self, self,
cluster_state_controller: ClusterStateController, cluster_state_controller: ClusterStateController,
file_keeper: FileKeeper, file_keeper: FileKeeper,
nodes_under_load: list[ClusterNode], nodes_under_load: list[ClusterNode],
user: User,
) -> None: ) -> None:
self.cluster_state_controller = cluster_state_controller self.cluster_state_controller = cluster_state_controller
self.file_keeper = file_keeper self.file_keeper = file_keeper
self.loaders = [NodeLoader(node) for node in nodes_under_load] self.loaders = [NodeLoader(node) for node in nodes_under_load]
self.nodes_under_load = nodes_under_load self.nodes_under_load = nodes_under_load
self.user = user
@run_optionally(optionals.OPTIONAL_BACKGROUND_LOAD_ENABLED) @run_optionally(optionals.OPTIONAL_BACKGROUND_LOAD_ENABLED)
@reporter.step("Preparation steps") @reporter.step("Preparation steps")
@ -326,11 +293,9 @@ class LocalRunner(RunnerBase):
shell.exec(f"sudo tar xf {k6_dir}/k6.tar.gz --strip-components 2 -C {k6_dir}") shell.exec(f"sudo tar xf {k6_dir}/k6.tar.gz --strip-components 2 -C {k6_dir}")
shell.exec(f"sudo chmod -R 777 {k6_dir}") shell.exec(f"sudo chmod -R 777 {k6_dir}")
with reporter.step("Create empty_passwd"): with reporter.step("chmod 777 wallet related files on loader"):
self.wallet = WalletInfo(f"{k6_dir}/scenarios/files/wallet.json", "", "/tmp/empty_passwd.yml") shell.exec(f"sudo chmod -R 777 {self.user.wallet.config_path}")
content = yaml.dump({"password": ""}) shell.exec(f"sudo chmod -R 777 {self.user.wallet.path}")
shell.exec(f'echo "{content}" | sudo tee {self.wallet.config_path}')
shell.exec(f"sudo chmod -R 777 {self.wallet.config_path}")
@reporter.step("Init k6 instances") @reporter.step("Init k6 instances")
def init_k6_instances(self, load_params: LoadParams, endpoints: list[str], k6_dir: str): def init_k6_instances(self, load_params: LoadParams, endpoints: list[str], k6_dir: str):
@ -363,7 +328,7 @@ class LocalRunner(RunnerBase):
k6_dir, k6_dir,
shell, shell,
loader, loader,
self.wallet, self.user,
) )
def start(self): def start(self):
@ -453,7 +418,7 @@ class S3LocalRunner(LocalRunner):
k6_dir, k6_dir,
shell, shell,
loader, loader,
self.wallet, self.user,
) )
@run_optionally(optionals.OPTIONAL_BACKGROUND_LOAD_ENABLED) @run_optionally(optionals.OPTIONAL_BACKGROUND_LOAD_ENABLED)
@ -466,17 +431,10 @@ class S3LocalRunner(LocalRunner):
k6_dir: str, k6_dir: str,
): ):
self.k6_dir = k6_dir self.k6_dir = k6_dir
with reporter.step("Init s3 client on loaders"): parallel(self.prepare_node, nodes_under_load, k6_dir, load_params, cluster_nodes)
storage_node = nodes_under_load[0].service(StorageNode)
s3_public_keys = [node.service(S3Gate).get_wallet_public_key() for node in cluster_nodes]
grpc_peer = storage_node.get_rpc_endpoint()
parallel(self.prepare_node, nodes_under_load, k6_dir, load_params, s3_public_keys, grpc_peer)
@reporter.step("Prepare node {cluster_node}") @reporter.step("Prepare node {cluster_node}")
def prepare_node( def prepare_node(self, cluster_node: ClusterNode, k6_dir: str, load_params: LoadParams, cluster_nodes: list[ClusterNode]):
self, cluster_node: ClusterNode, k6_dir: str, load_params: LoadParams, s3_public_keys: list[str], grpc_peer: str
):
LocalRunner.prepare_node(self, cluster_node, k6_dir, load_params) LocalRunner.prepare_node(self, cluster_node, k6_dir, load_params)
self.endpoints = cluster_node.s3_gate.get_all_endpoints() self.endpoints = cluster_node.s3_gate.get_all_endpoints()
shell = cluster_node.host.get_shell() shell = cluster_node.host.get_shell()
@ -497,29 +455,9 @@ class S3LocalRunner(LocalRunner):
shell.exec(f"sudo python3 -m pip install -I {k6_dir}/requests.tar.gz") shell.exec(f"sudo python3 -m pip install -I {k6_dir}/requests.tar.gz")
with reporter.step(f"Init s3 client on {cluster_node.host_ip}"): with reporter.step(f"Init s3 client on {cluster_node.host_ip}"):
frostfs_authmate_exec: FrostfsAuthmate = FrostfsAuthmate(shell, FROSTFS_AUTHMATE_EXEC)
issue_secret_output = frostfs_authmate_exec.secret.issue(
wallet=self.wallet.path,
peer=grpc_peer,
gate_public_key=s3_public_keys,
container_placement_policy=load_params.preset.container_placement_policy,
container_policy=f"{k6_dir}/scenarios/files/policy.json",
wallet_password=self.wallet.password,
).stdout
aws_access_key_id = str(
re.search(r"access_key_id.*:\s.(?P<aws_access_key_id>\w*)", issue_secret_output).group(
"aws_access_key_id"
)
)
aws_secret_access_key = str(
re.search(
r"secret_access_key.*:\s.(?P<aws_secret_access_key>\w*)",
issue_secret_output,
).group("aws_secret_access_key")
)
configure_input = [ configure_input = [
InteractiveInput(prompt_pattern=r"AWS Access Key ID.*", input=aws_access_key_id), InteractiveInput(prompt_pattern=r"AWS Access Key ID.*", input=self.user.s3_credentials.access_key),
InteractiveInput(prompt_pattern=r"AWS Secret Access Key.*", input=aws_secret_access_key), InteractiveInput(prompt_pattern=r"AWS Secret Access Key.*", input=self.user.s3_credentials.secret_key),
InteractiveInput(prompt_pattern=r".*", input=""), InteractiveInput(prompt_pattern=r".*", input=""),
InteractiveInput(prompt_pattern=r".*", input=""), InteractiveInput(prompt_pattern=r".*", input=""),
] ]

View file

@ -0,0 +1,16 @@
import re
from frostfs_testlib.cli.generic_cli import GenericCli
from frostfs_testlib.s3.interfaces import BucketContainerResolver
from frostfs_testlib.storage.cluster import ClusterNode
class CurlBucketContainerResolver(BucketContainerResolver):
def resolve(self, node: ClusterNode, bucket_name: str, **kwargs: dict) -> str:
curl = GenericCli("curl", node.host)
output = curl(f"-I http://127.0.0.1:8084/{bucket_name}")
pattern = r"X-Container-Id: (\S+)"
cid = re.findall(pattern, output.stdout)
if cid:
return cid[0]
return None

View file

@ -1,7 +1,8 @@
from abc import abstractmethod from abc import ABC, abstractmethod
from datetime import datetime from datetime import datetime
from typing import Literal, Optional, Union from typing import Literal, Optional, Union
from frostfs_testlib.storage.cluster import ClusterNode
from frostfs_testlib.testing.readable import HumanReadableABC, HumanReadableEnum from frostfs_testlib.testing.readable import HumanReadableABC, HumanReadableEnum
@ -31,6 +32,22 @@ ACL_COPY = [
] ]
class BucketContainerResolver(ABC):
@abstractmethod
def resolve(self, node: ClusterNode, bucket_name: str, **kwargs: dict) -> str:
"""
Resolve Container ID from bucket name
Args:
node: node from where we want to resolve
bucket_name: name of the bucket
**kwargs: any other required params
Returns: Container ID
"""
raise NotImplementedError("Call from abstract class")
class S3ClientWrapper(HumanReadableABC): class S3ClientWrapper(HumanReadableABC):
@abstractmethod @abstractmethod
def __init__(self, access_key_id: str, secret_access_key: str, s3gate_endpoint: str, profile: str) -> None: def __init__(self, access_key_id: str, secret_access_key: str, s3gate_endpoint: str, profile: str) -> None:
@ -296,15 +313,11 @@ class S3ClientWrapper(HumanReadableABC):
abort a given multipart upload multiple times in order to completely free all storage consumed by all parts.""" abort a given multipart upload multiple times in order to completely free all storage consumed by all parts."""
@abstractmethod @abstractmethod
def upload_part( def upload_part(self, bucket: str, key: str, upload_id: str, part_num: int, filepath: str) -> str:
self, bucket: str, key: str, upload_id: str, part_num: int, filepath: str
) -> str:
"""Uploads a part in a multipart upload.""" """Uploads a part in a multipart upload."""
@abstractmethod @abstractmethod
def upload_part_copy( def upload_part_copy(self, bucket: str, key: str, upload_id: str, part_num: int, copy_source: str) -> str:
self, bucket: str, key: str, upload_id: str, part_num: int, copy_source: str
) -> str:
"""Uploads a part by copying data from an existing object as data source.""" """Uploads a part by copying data from an existing object as data source."""
@abstractmethod @abstractmethod

View file

@ -11,25 +11,20 @@ import base58
from frostfs_testlib import reporter from frostfs_testlib import reporter
from frostfs_testlib.cli import FrostfsCli from frostfs_testlib.cli import FrostfsCli
from frostfs_testlib.resources.cli import FROSTFS_CLI_EXEC from frostfs_testlib.resources.cli import FROSTFS_CLI_EXEC
from frostfs_testlib.resources.common import ASSETS_DIR, DEFAULT_WALLET_CONFIG from frostfs_testlib.resources.common import ASSETS_DIR
from frostfs_testlib.shell import Shell from frostfs_testlib.shell import Shell
from frostfs_testlib.storage.dataclasses.acl import ( from frostfs_testlib.storage.dataclasses.acl import EACL_LIFETIME, FROSTFS_CONTRACT_CACHE_TIMEOUT, EACLPubKey, EACLRole, EACLRule
EACL_LIFETIME, from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
FROSTFS_CONTRACT_CACHE_TIMEOUT,
EACLPubKey,
EACLRole,
EACLRule,
)
from frostfs_testlib.utils import wallet_utils from frostfs_testlib.utils import wallet_utils
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
@reporter.step("Get extended ACL") @reporter.step("Get extended ACL")
def get_eacl(wallet_path: str, cid: str, shell: Shell, endpoint: str) -> Optional[str]: def get_eacl(wallet: WalletInfo, cid: str, shell: Shell, endpoint: str) -> Optional[str]:
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, DEFAULT_WALLET_CONFIG) cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
try: try:
result = cli.container.get_eacl(wallet=wallet_path, rpc_endpoint=endpoint, cid=cid) result = cli.container.get_eacl(rpc_endpoint=endpoint, cid=cid)
except RuntimeError as exc: except RuntimeError as exc:
logger.info("Extended ACL table is not set for this container") logger.info("Extended ACL table is not set for this container")
logger.info(f"Got exception while getting eacl: {exc}") logger.info(f"Got exception while getting eacl: {exc}")
@ -41,16 +36,15 @@ def get_eacl(wallet_path: str, cid: str, shell: Shell, endpoint: str) -> Optiona
@reporter.step("Set extended ACL") @reporter.step("Set extended ACL")
def set_eacl( def set_eacl(
wallet_path: str, wallet: WalletInfo,
cid: str, cid: str,
eacl_table_path: str, eacl_table_path: str,
shell: Shell, shell: Shell,
endpoint: str, endpoint: str,
session_token: Optional[str] = None, session_token: Optional[str] = None,
) -> None: ) -> None:
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, DEFAULT_WALLET_CONFIG) cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
cli.container.set_eacl( cli.container.set_eacl(
wallet=wallet_path,
rpc_endpoint=endpoint, rpc_endpoint=endpoint,
cid=cid, cid=cid,
table=eacl_table_path, table=eacl_table_path,
@ -66,7 +60,7 @@ def _encode_cid_for_eacl(cid: str) -> str:
def create_eacl(cid: str, rules_list: List[EACLRule], shell: Shell) -> str: def create_eacl(cid: str, rules_list: List[EACLRule], shell: Shell) -> str:
table_file_path = os.path.join(os.getcwd(), ASSETS_DIR, f"eacl_table_{str(uuid.uuid4())}.json") table_file_path = os.path.join(os.getcwd(), ASSETS_DIR, f"eacl_table_{str(uuid.uuid4())}.json")
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, DEFAULT_WALLET_CONFIG) cli = FrostfsCli(shell, FROSTFS_CLI_EXEC)
cli.acl.extended_create(cid=cid, out=table_file_path, rule=rules_list) cli.acl.extended_create(cid=cid, out=table_file_path, rule=rules_list)
with open(table_file_path, "r") as file: with open(table_file_path, "r") as file:
@ -77,7 +71,7 @@ def create_eacl(cid: str, rules_list: List[EACLRule], shell: Shell) -> str:
def form_bearertoken_file( def form_bearertoken_file(
wif: str, wallet: WalletInfo,
cid: str, cid: str,
eacl_rule_list: List[Union[EACLRule, EACLPubKey]], eacl_rule_list: List[Union[EACLRule, EACLPubKey]],
shell: Shell, shell: Shell,
@ -92,7 +86,7 @@ def form_bearertoken_file(
enc_cid = _encode_cid_for_eacl(cid) if cid else None enc_cid = _encode_cid_for_eacl(cid) if cid else None
file_path = os.path.join(os.getcwd(), ASSETS_DIR, str(uuid.uuid4())) file_path = os.path.join(os.getcwd(), ASSETS_DIR, str(uuid.uuid4()))
eacl = get_eacl(wif, cid, shell, endpoint) eacl = get_eacl(wallet, cid, shell, endpoint)
json_eacl = dict() json_eacl = dict()
if eacl: if eacl:
eacl = eacl.replace("eACL: ", "").split("Signature")[0] eacl = eacl.replace("eACL: ", "").split("Signature")[0]
@ -133,7 +127,7 @@ def form_bearertoken_file(
if sign: if sign:
sign_bearer( sign_bearer(
shell=shell, shell=shell,
wallet_path=wif, wallet=wallet,
eacl_rules_file_from=file_path, eacl_rules_file_from=file_path,
eacl_rules_file_to=file_path, eacl_rules_file_to=file_path,
json=True, json=True,
@ -164,11 +158,9 @@ def eacl_rules(access: str, verbs: list, user: str) -> list[str]:
return rules return rules
def sign_bearer(shell: Shell, wallet_path: str, eacl_rules_file_from: str, eacl_rules_file_to: str, json: bool) -> None: def sign_bearer(shell: Shell, wallet: WalletInfo, eacl_rules_file_from: str, eacl_rules_file_to: str, json: bool) -> None:
frostfscli = FrostfsCli(shell=shell, frostfs_cli_exec_path=FROSTFS_CLI_EXEC, config_file=DEFAULT_WALLET_CONFIG) frostfscli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
frostfscli.util.sign_bearer_token( frostfscli.util.sign_bearer_token(eacl_rules_file_from, eacl_rules_file_to, json=json)
wallet=wallet_path, from_file=eacl_rules_file_from, to_file=eacl_rules_file_to, json=json
)
@reporter.step("Wait for eACL cache expired") @reporter.step("Wait for eACL cache expired")
@ -178,9 +170,7 @@ def wait_for_cache_expired():
@reporter.step("Return bearer token in base64 to caller") @reporter.step("Return bearer token in base64 to caller")
def bearer_token_base64_from_file( def bearer_token_base64_from_file(bearer_path: str) -> str:
bearer_path: str,
) -> str:
with open(bearer_path, "rb") as file: with open(bearer_path, "rb") as file:
signed = file.read() signed = file.read()
return base64.b64encode(signed).decode("utf-8") return base64.b64encode(signed).decode("utf-8")

View file

@ -5,12 +5,11 @@ from dataclasses import dataclass
from time import sleep from time import sleep
from typing import Optional, Union from typing import Optional, Union
import requests
from frostfs_testlib import reporter from frostfs_testlib import reporter
from frostfs_testlib.cli import FrostfsCli, GenericCli from frostfs_testlib.cli import FrostfsCli
from frostfs_testlib.plugins import load_plugin
from frostfs_testlib.resources.cli import CLI_DEFAULT_TIMEOUT, FROSTFS_CLI_EXEC from frostfs_testlib.resources.cli import CLI_DEFAULT_TIMEOUT, FROSTFS_CLI_EXEC
from frostfs_testlib.resources.common import DEFAULT_WALLET_CONFIG from frostfs_testlib.s3.interfaces import BucketContainerResolver
from frostfs_testlib.shell import Shell from frostfs_testlib.shell import Shell
from frostfs_testlib.steps.cli.object import put_object, put_object_to_random_node from frostfs_testlib.steps.cli.object import put_object, put_object_to_random_node
from frostfs_testlib.storage.cluster import Cluster, ClusterNode from frostfs_testlib.storage.cluster import Cluster, ClusterNode
@ -25,7 +24,7 @@ logger = logging.getLogger("NeoLogger")
@dataclass @dataclass
class StorageContainerInfo: class StorageContainerInfo:
id: str id: str
wallet_file: WalletInfo wallet: WalletInfo
class StorageContainer: class StorageContainer:
@ -42,11 +41,8 @@ class StorageContainer:
def get_id(self) -> str: def get_id(self) -> str:
return self.storage_container_info.id return self.storage_container_info.id
def get_wallet_path(self) -> str: def get_wallet(self) -> str:
return self.storage_container_info.wallet_file.path return self.storage_container_info.wallet
def get_wallet_config_path(self) -> str:
return self.storage_container_info.wallet_file.config_path
@reporter.step("Generate new object and put in container") @reporter.step("Generate new object and put in container")
def generate_object( def generate_object(
@ -61,37 +57,34 @@ class StorageContainer:
file_hash = get_file_hash(file_path) file_hash = get_file_hash(file_path)
container_id = self.get_id() container_id = self.get_id()
wallet_path = self.get_wallet_path() wallet = self.get_wallet()
wallet_config = self.get_wallet_config_path()
with reporter.step(f"Put object with size {size} to container {container_id}"): with reporter.step(f"Put object with size {size} to container {container_id}"):
if endpoint: if endpoint:
object_id = put_object( object_id = put_object(
wallet=wallet_path, wallet=wallet,
path=file_path, path=file_path,
cid=container_id, cid=container_id,
expire_at=expire_at, expire_at=expire_at,
shell=self.shell, shell=self.shell,
endpoint=endpoint, endpoint=endpoint,
bearer=bearer_token, bearer=bearer_token,
wallet_config=wallet_config,
) )
else: else:
object_id = put_object_to_random_node( object_id = put_object_to_random_node(
wallet=wallet_path, wallet=wallet,
path=file_path, path=file_path,
cid=container_id, cid=container_id,
expire_at=expire_at, expire_at=expire_at,
shell=self.shell, shell=self.shell,
cluster=self.cluster, cluster=self.cluster,
bearer=bearer_token, bearer=bearer_token,
wallet_config=wallet_config,
) )
storage_object = StorageObjectInfo( storage_object = StorageObjectInfo(
container_id, container_id,
object_id, object_id,
size=size, size=size,
wallet_file_path=wallet_path, wallet=wallet,
file_path=file_path, file_path=file_path,
file_hash=file_hash, file_hash=file_hash,
) )
@ -106,14 +99,13 @@ REP_2_FOR_3_NODES_PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 3 FROM * AS X"
@reporter.step("Create Container") @reporter.step("Create Container")
def create_container( def create_container(
wallet: str, wallet: WalletInfo,
shell: Shell, shell: Shell,
endpoint: str, endpoint: str,
rule: str = DEFAULT_PLACEMENT_RULE, rule: str = DEFAULT_PLACEMENT_RULE,
basic_acl: str = "", basic_acl: str = "",
attributes: Optional[dict] = None, attributes: Optional[dict] = None,
session_token: str = "", session_token: str = "",
session_wallet: str = "",
name: Optional[str] = None, name: Optional[str] = None,
options: Optional[dict] = None, options: Optional[dict] = None,
await_mode: bool = True, await_mode: bool = True,
@ -124,7 +116,7 @@ def create_container(
A wrapper for `frostfs-cli container create` call. A wrapper for `frostfs-cli container create` call.
Args: Args:
wallet (str): a wallet on whose behalf a container is created wallet (WalletInfo): a wallet on whose behalf a container is created
rule (optional, str): placement rule for container rule (optional, str): placement rule for container
basic_acl (optional, str): an ACL for container, will be basic_acl (optional, str): an ACL for container, will be
appended to `--basic-acl` key appended to `--basic-acl` key
@ -146,10 +138,9 @@ def create_container(
(str): CID of the created container (str): CID of the created container
""" """
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, DEFAULT_WALLET_CONFIG) cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
result = cli.container.create( result = cli.container.create(
rpc_endpoint=endpoint, rpc_endpoint=endpoint,
wallet=session_wallet if session_wallet else wallet,
policy=rule, policy=rule,
basic_acl=basic_acl, basic_acl=basic_acl,
attributes=attributes, attributes=attributes,
@ -170,9 +161,7 @@ def create_container(
return cid return cid
def wait_for_container_creation( def wait_for_container_creation(wallet: WalletInfo, cid: str, shell: Shell, endpoint: str, attempts: int = 15, sleep_interval: int = 1):
wallet: str, cid: str, shell: Shell, endpoint: str, attempts: int = 15, sleep_interval: int = 1
):
for _ in range(attempts): for _ in range(attempts):
containers = list_containers(wallet, shell, endpoint) containers = list_containers(wallet, shell, endpoint)
if cid in containers: if cid in containers:
@ -182,9 +171,7 @@ def wait_for_container_creation(
raise RuntimeError(f"After {attempts * sleep_interval} seconds container {cid} hasn't been persisted; exiting") raise RuntimeError(f"After {attempts * sleep_interval} seconds container {cid} hasn't been persisted; exiting")
def wait_for_container_deletion( def wait_for_container_deletion(wallet: WalletInfo, cid: str, shell: Shell, endpoint: str, attempts: int = 30, sleep_interval: int = 1):
wallet: str, cid: str, shell: Shell, endpoint: str, attempts: int = 30, sleep_interval: int = 1
):
for _ in range(attempts): for _ in range(attempts):
try: try:
get_container(wallet, cid, shell=shell, endpoint=endpoint) get_container(wallet, cid, shell=shell, endpoint=endpoint)
@ -198,29 +185,27 @@ def wait_for_container_deletion(
@reporter.step("List Containers") @reporter.step("List Containers")
def list_containers( def list_containers(wallet: WalletInfo, shell: Shell, endpoint: str, timeout: Optional[str] = CLI_DEFAULT_TIMEOUT) -> list[str]:
wallet: str, shell: Shell, endpoint: str, timeout: Optional[str] = CLI_DEFAULT_TIMEOUT
) -> list[str]:
""" """
A wrapper for `frostfs-cli container list` call. It returns all the A wrapper for `frostfs-cli container list` call. It returns all the
available containers for the given wallet. available containers for the given wallet.
Args: Args:
wallet (str): a wallet on whose behalf we list the containers wallet (WalletInfo): a wallet on whose behalf we list the containers
shell: executor for cli command shell: executor for cli command
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
timeout: Timeout for the operation. timeout: Timeout for the operation.
Returns: Returns:
(list): list of containers (list): list of containers
""" """
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, DEFAULT_WALLET_CONFIG) cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
result = cli.container.list(rpc_endpoint=endpoint, wallet=wallet, timeout=timeout) result = cli.container.list(rpc_endpoint=endpoint, timeout=timeout)
logger.info(f"Containers: \n{result}") logger.info(f"Containers: \n{result}")
return result.stdout.split() return result.stdout.split()
@reporter.step("List Objects in container") @reporter.step("List Objects in container")
def list_objects( def list_objects(
wallet: str, wallet: WalletInfo,
shell: Shell, shell: Shell,
container_id: str, container_id: str,
endpoint: str, endpoint: str,
@ -230,7 +215,7 @@ def list_objects(
A wrapper for `frostfs-cli container list-objects` call. It returns all the A wrapper for `frostfs-cli container list-objects` call. It returns all the
available objects in container. available objects in container.
Args: Args:
wallet (str): a wallet on whose behalf we list the containers objects wallet (WalletInfo): a wallet on whose behalf we list the containers objects
shell: executor for cli command shell: executor for cli command
container_id: cid of container container_id: cid of container
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
@ -238,15 +223,15 @@ def list_objects(
Returns: Returns:
(list): list of containers (list): list of containers
""" """
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, DEFAULT_WALLET_CONFIG) cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
result = cli.container.list_objects(rpc_endpoint=endpoint, wallet=wallet, cid=container_id, timeout=timeout) result = cli.container.list_objects(rpc_endpoint=endpoint, cid=container_id, timeout=timeout)
logger.info(f"Container objects: \n{result}") logger.info(f"Container objects: \n{result}")
return result.stdout.split() return result.stdout.split()
@reporter.step("Get Container") @reporter.step("Get Container")
def get_container( def get_container(
wallet: str, wallet: WalletInfo,
cid: str, cid: str,
shell: Shell, shell: Shell,
endpoint: str, endpoint: str,
@ -257,7 +242,7 @@ def get_container(
A wrapper for `frostfs-cli container get` call. It extracts container's A wrapper for `frostfs-cli container get` call. It extracts container's
attributes and rearranges them into a more compact view. attributes and rearranges them into a more compact view.
Args: Args:
wallet (str): path to a wallet on whose behalf we get the container wallet (WalletInfo): path to a wallet on whose behalf we get the container
cid (str): ID of the container to get cid (str): ID of the container to get
shell: executor for cli command shell: executor for cli command
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
@ -267,8 +252,8 @@ def get_container(
(dict, str): dict of container attributes (dict, str): dict of container attributes
""" """
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, DEFAULT_WALLET_CONFIG) cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
result = cli.container.get(rpc_endpoint=endpoint, wallet=wallet, cid=cid, json_mode=json_mode, timeout=timeout) result = cli.container.get(rpc_endpoint=endpoint, cid=cid, json_mode=json_mode, timeout=timeout)
if not json_mode: if not json_mode:
return result.stdout return result.stdout
@ -285,7 +270,7 @@ def get_container(
@reporter.step("Delete Container") @reporter.step("Delete Container")
# TODO: make the error message about a non-found container more user-friendly # TODO: make the error message about a non-found container more user-friendly
def delete_container( def delete_container(
wallet: str, wallet: WalletInfo,
cid: str, cid: str,
shell: Shell, shell: Shell,
endpoint: str, endpoint: str,
@ -297,7 +282,7 @@ def delete_container(
A wrapper for `frostfs-cli container delete` call. A wrapper for `frostfs-cli container delete` call.
Args: Args:
await_mode: Block execution until container is removed. await_mode: Block execution until container is removed.
wallet (str): path to a wallet on whose behalf we delete the container wallet (WalletInfo): path to a wallet on whose behalf we delete the container
cid (str): ID of the container to delete cid (str): ID of the container to delete
shell: executor for cli command shell: executor for cli command
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
@ -306,9 +291,8 @@ def delete_container(
This function doesn't return anything. This function doesn't return anything.
""" """
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, DEFAULT_WALLET_CONFIG) cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
cli.container.delete( cli.container.delete(
wallet=wallet,
cid=cid, cid=cid,
rpc_endpoint=endpoint, rpc_endpoint=endpoint,
force=force, force=force,
@ -345,26 +329,22 @@ def _parse_cid(output: str) -> str:
@reporter.step("Search container by name") @reporter.step("Search container by name")
def search_container_by_name(name: str, node: ClusterNode): def search_container_by_name(name: str, node: ClusterNode):
curl = GenericCli("curl", node.host) resolver_cls = load_plugin("frostfs.testlib.bucket_cid_resolver", node.host.config.product)
output = curl(f"-I http://127.0.0.1:8084/{name}") resolver: BucketContainerResolver = resolver_cls()
pattern = r"X-Container-Id: (\S+)" return resolver.resolve(node, name)
cid = re.findall(pattern, output.stdout)
if cid:
return cid[0]
return None
@reporter.step("Search for nodes with a container") @reporter.step("Search for nodes with a container")
def search_nodes_with_container( def search_nodes_with_container(
wallet: str, wallet: WalletInfo,
cid: str, cid: str,
shell: Shell, shell: Shell,
endpoint: str, endpoint: str,
cluster: Cluster, cluster: Cluster,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT, timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
) -> list[ClusterNode]: ) -> list[ClusterNode]:
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, DEFAULT_WALLET_CONFIG) cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
result = cli.container.search_node(rpc_endpoint=endpoint, wallet=wallet, cid=cid, timeout=timeout) result = cli.container.search_node(rpc_endpoint=endpoint, cid=cid, timeout=timeout)
pattern = r"[0-9]+(?:\.[0-9]+){3}" pattern = r"[0-9]+(?:\.[0-9]+){3}"
nodes_ip = list(set(re.findall(pattern, result.stdout))) nodes_ip = list(set(re.findall(pattern, result.stdout)))

View file

@ -9,9 +9,10 @@ from frostfs_testlib import reporter
from frostfs_testlib.cli import FrostfsCli from frostfs_testlib.cli import FrostfsCli
from frostfs_testlib.cli.neogo import NeoGo from frostfs_testlib.cli.neogo import NeoGo
from frostfs_testlib.resources.cli import CLI_DEFAULT_TIMEOUT, FROSTFS_CLI_EXEC, NEOGO_EXECUTABLE from frostfs_testlib.resources.cli import CLI_DEFAULT_TIMEOUT, FROSTFS_CLI_EXEC, NEOGO_EXECUTABLE
from frostfs_testlib.resources.common import ASSETS_DIR, DEFAULT_WALLET_CONFIG from frostfs_testlib.resources.common import ASSETS_DIR
from frostfs_testlib.shell import Shell from frostfs_testlib.shell import Shell
from frostfs_testlib.storage.cluster import Cluster, ClusterNode from frostfs_testlib.storage.cluster import Cluster, ClusterNode
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.utils import json_utils from frostfs_testlib.utils import json_utils
from frostfs_testlib.utils.cli_utils import parse_cmd_table, parse_netmap_output from frostfs_testlib.utils.cli_utils import parse_cmd_table, parse_netmap_output
@ -20,7 +21,7 @@ logger = logging.getLogger("NeoLogger")
@reporter.step("Get object from random node") @reporter.step("Get object from random node")
def get_object_from_random_node( def get_object_from_random_node(
wallet: str, wallet: WalletInfo,
cid: str, cid: str,
oid: str, oid: str,
shell: Shell, shell: Shell,
@ -28,7 +29,6 @@ def get_object_from_random_node(
bearer: Optional[str] = None, bearer: Optional[str] = None,
write_object: Optional[str] = None, write_object: Optional[str] = None,
xhdr: Optional[dict] = None, xhdr: Optional[dict] = None,
wallet_config: Optional[str] = None,
no_progress: bool = True, no_progress: bool = True,
session: Optional[str] = None, session: Optional[str] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT, timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
@ -44,7 +44,6 @@ def get_object_from_random_node(
cluster: cluster object cluster: cluster object
bearer (optional, str): path to Bearer Token file, appends to `--bearer` key bearer (optional, str): path to Bearer Token file, appends to `--bearer` key
write_object (optional, str): path to downloaded file, appends to `--file` key write_object (optional, str): path to downloaded file, appends to `--file` key
wallet_config(optional, str): path to the wallet config
no_progress(optional, bool): do not show progress bar no_progress(optional, bool): do not show progress bar
xhdr (optional, dict): Request X-Headers in form of Key=Value xhdr (optional, dict): Request X-Headers in form of Key=Value
session (optional, dict): path to a JSON-encoded container session token session (optional, dict): path to a JSON-encoded container session token
@ -62,7 +61,6 @@ def get_object_from_random_node(
bearer, bearer,
write_object, write_object,
xhdr, xhdr,
wallet_config,
no_progress, no_progress,
session, session,
timeout, timeout,
@ -71,7 +69,7 @@ def get_object_from_random_node(
@reporter.step("Get object from {endpoint}") @reporter.step("Get object from {endpoint}")
def get_object( def get_object(
wallet: str, wallet: WalletInfo,
cid: str, cid: str,
oid: str, oid: str,
shell: Shell, shell: Shell,
@ -79,7 +77,6 @@ def get_object(
bearer: Optional[str] = None, bearer: Optional[str] = None,
write_object: Optional[str] = None, write_object: Optional[str] = None,
xhdr: Optional[dict] = None, xhdr: Optional[dict] = None,
wallet_config: Optional[str] = None,
no_progress: bool = True, no_progress: bool = True,
session: Optional[str] = None, session: Optional[str] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT, timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
@ -88,14 +85,13 @@ def get_object(
GET from FrostFS. GET from FrostFS.
Args: Args:
wallet (str): wallet on whose behalf GET is done wallet (WalletInfo): wallet on whose behalf GET is done
cid (str): ID of Container where we get the Object from cid (str): ID of Container where we get the Object from
oid (str): Object ID oid (str): Object ID
shell: executor for cli command shell: executor for cli command
bearer: path to Bearer Token file, appends to `--bearer` key bearer: path to Bearer Token file, appends to `--bearer` key
write_object: path to downloaded file, appends to `--file` key write_object: path to downloaded file, appends to `--file` key
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
wallet_config(optional, str): path to the wallet config
no_progress(optional, bool): do not show progress bar no_progress(optional, bool): do not show progress bar
xhdr (optional, dict): Request X-Headers in form of Key=Value xhdr (optional, dict): Request X-Headers in form of Key=Value
session (optional, dict): path to a JSON-encoded container session token session (optional, dict): path to a JSON-encoded container session token
@ -108,10 +104,9 @@ def get_object(
write_object = str(uuid.uuid4()) write_object = str(uuid.uuid4())
file_path = os.path.join(ASSETS_DIR, write_object) file_path = os.path.join(ASSETS_DIR, write_object)
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or DEFAULT_WALLET_CONFIG) cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
cli.object.get( cli.object.get(
rpc_endpoint=endpoint, rpc_endpoint=endpoint,
wallet=wallet,
cid=cid, cid=cid,
oid=oid, oid=oid,
file=file_path, file=file_path,
@ -127,14 +122,13 @@ def get_object(
@reporter.step("Get Range Hash from {endpoint}") @reporter.step("Get Range Hash from {endpoint}")
def get_range_hash( def get_range_hash(
wallet: str, wallet: WalletInfo,
cid: str, cid: str,
oid: str, oid: str,
range_cut: str, range_cut: str,
shell: Shell, shell: Shell,
endpoint: str, endpoint: str,
bearer: Optional[str] = None, bearer: Optional[str] = None,
wallet_config: Optional[str] = None,
xhdr: Optional[dict] = None, xhdr: Optional[dict] = None,
session: Optional[str] = None, session: Optional[str] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT, timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
@ -151,17 +145,15 @@ def get_range_hash(
range_cut: Range to take hash from in the form offset1:length1,..., range_cut: Range to take hash from in the form offset1:length1,...,
value to pass to the `--range` parameter value to pass to the `--range` parameter
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
wallet_config: path to the wallet config
xhdr: Request X-Headers in form of Key=Values xhdr: Request X-Headers in form of Key=Values
session: Filepath to a JSON- or binary-encoded token of the object RANGEHASH session. session: Filepath to a JSON- or binary-encoded token of the object RANGEHASH session.
timeout: Timeout for the operation. timeout: Timeout for the operation.
Returns: Returns:
None None
""" """
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or DEFAULT_WALLET_CONFIG) cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
result = cli.object.hash( result = cli.object.hash(
rpc_endpoint=endpoint, rpc_endpoint=endpoint,
wallet=wallet,
cid=cid, cid=cid,
oid=oid, oid=oid,
range=range_cut, range=range_cut,
@ -177,7 +169,7 @@ def get_range_hash(
@reporter.step("Put object to random node") @reporter.step("Put object to random node")
def put_object_to_random_node( def put_object_to_random_node(
wallet: str, wallet: WalletInfo,
path: str, path: str,
cid: str, cid: str,
shell: Shell, shell: Shell,
@ -186,7 +178,6 @@ def put_object_to_random_node(
copies_number: Optional[int] = None, copies_number: Optional[int] = None,
attributes: Optional[dict] = None, attributes: Optional[dict] = None,
xhdr: Optional[dict] = None, xhdr: Optional[dict] = None,
wallet_config: Optional[str] = None,
expire_at: Optional[int] = None, expire_at: Optional[int] = None,
no_progress: bool = True, no_progress: bool = True,
session: Optional[str] = None, session: Optional[str] = None,
@ -205,7 +196,6 @@ def put_object_to_random_node(
copies_number: Number of copies of the object to store within the RPC call copies_number: Number of copies of the object to store within the RPC call
attributes: User attributes in form of Key1=Value1,Key2=Value2 attributes: User attributes in form of Key1=Value1,Key2=Value2
cluster: cluster under test cluster: cluster under test
wallet_config: path to the wallet config
no_progress: do not show progress bar no_progress: do not show progress bar
expire_at: Last epoch in the life of the object expire_at: Last epoch in the life of the object
xhdr: Request X-Headers in form of Key=Value xhdr: Request X-Headers in form of Key=Value
@ -226,7 +216,6 @@ def put_object_to_random_node(
copies_number, copies_number,
attributes, attributes,
xhdr, xhdr,
wallet_config,
expire_at, expire_at,
no_progress, no_progress,
session, session,
@ -236,7 +225,7 @@ def put_object_to_random_node(
@reporter.step("Put object at {endpoint} in container {cid}") @reporter.step("Put object at {endpoint} in container {cid}")
def put_object( def put_object(
wallet: str, wallet: WalletInfo,
path: str, path: str,
cid: str, cid: str,
shell: Shell, shell: Shell,
@ -245,7 +234,6 @@ def put_object(
copies_number: Optional[int] = None, copies_number: Optional[int] = None,
attributes: Optional[dict] = None, attributes: Optional[dict] = None,
xhdr: Optional[dict] = None, xhdr: Optional[dict] = None,
wallet_config: Optional[str] = None,
expire_at: Optional[int] = None, expire_at: Optional[int] = None,
no_progress: bool = True, no_progress: bool = True,
session: Optional[str] = None, session: Optional[str] = None,
@ -263,7 +251,6 @@ def put_object(
copies_number: Number of copies of the object to store within the RPC call copies_number: Number of copies of the object to store within the RPC call
attributes: User attributes in form of Key1=Value1,Key2=Value2 attributes: User attributes in form of Key1=Value1,Key2=Value2
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
wallet_config: path to the wallet config
no_progress: do not show progress bar no_progress: do not show progress bar
expire_at: Last epoch in the life of the object expire_at: Last epoch in the life of the object
xhdr: Request X-Headers in form of Key=Value xhdr: Request X-Headers in form of Key=Value
@ -273,10 +260,9 @@ def put_object(
(str): ID of uploaded Object (str): ID of uploaded Object
""" """
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or DEFAULT_WALLET_CONFIG) cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
result = cli.object.put( result = cli.object.put(
rpc_endpoint=endpoint, rpc_endpoint=endpoint,
wallet=wallet,
file=path, file=path,
cid=cid, cid=cid,
attributes=attributes, attributes=attributes,
@ -297,13 +283,12 @@ def put_object(
@reporter.step("Delete object {cid}/{oid} from {endpoint}") @reporter.step("Delete object {cid}/{oid} from {endpoint}")
def delete_object( def delete_object(
wallet: str, wallet: WalletInfo,
cid: str, cid: str,
oid: str, oid: str,
shell: Shell, shell: Shell,
endpoint: str, endpoint: str,
bearer: str = "", bearer: str = "",
wallet_config: Optional[str] = None,
xhdr: Optional[dict] = None, xhdr: Optional[dict] = None,
session: Optional[str] = None, session: Optional[str] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT, timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
@ -318,7 +303,6 @@ def delete_object(
shell: executor for cli command shell: executor for cli command
bearer: path to Bearer Token file, appends to `--bearer` key bearer: path to Bearer Token file, appends to `--bearer` key
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
wallet_config: path to the wallet config
xhdr: Request X-Headers in form of Key=Value xhdr: Request X-Headers in form of Key=Value
session: path to a JSON-encoded container session token session: path to a JSON-encoded container session token
timeout: Timeout for the operation. timeout: Timeout for the operation.
@ -326,10 +310,9 @@ def delete_object(
(str): Tombstone ID (str): Tombstone ID
""" """
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or DEFAULT_WALLET_CONFIG) cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
result = cli.object.delete( result = cli.object.delete(
rpc_endpoint=endpoint, rpc_endpoint=endpoint,
wallet=wallet,
cid=cid, cid=cid,
oid=oid, oid=oid,
bearer=bearer, bearer=bearer,
@ -345,13 +328,12 @@ def delete_object(
@reporter.step("Get Range") @reporter.step("Get Range")
def get_range( def get_range(
wallet: str, wallet: WalletInfo,
cid: str, cid: str,
oid: str, oid: str,
range_cut: str, range_cut: str,
shell: Shell, shell: Shell,
endpoint: str, endpoint: str,
wallet_config: Optional[str] = None,
bearer: str = "", bearer: str = "",
xhdr: Optional[dict] = None, xhdr: Optional[dict] = None,
session: Optional[str] = None, session: Optional[str] = None,
@ -368,7 +350,6 @@ def get_range(
shell: executor for cli command shell: executor for cli command
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
bearer: path to Bearer Token file, appends to `--bearer` key bearer: path to Bearer Token file, appends to `--bearer` key
wallet_config: path to the wallet config
xhdr: Request X-Headers in form of Key=Value xhdr: Request X-Headers in form of Key=Value
session: path to a JSON-encoded container session token session: path to a JSON-encoded container session token
timeout: Timeout for the operation. timeout: Timeout for the operation.
@ -377,10 +358,9 @@ def get_range(
""" """
range_file_path = os.path.join(ASSETS_DIR, str(uuid.uuid4())) range_file_path = os.path.join(ASSETS_DIR, str(uuid.uuid4()))
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or DEFAULT_WALLET_CONFIG) cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
cli.object.range( cli.object.range(
rpc_endpoint=endpoint, rpc_endpoint=endpoint,
wallet=wallet,
cid=cid, cid=cid,
oid=oid, oid=oid,
range=range_cut, range=range_cut,
@ -398,7 +378,7 @@ def get_range(
@reporter.step("Lock Object") @reporter.step("Lock Object")
def lock_object( def lock_object(
wallet: str, wallet: WalletInfo,
cid: str, cid: str,
oid: str, oid: str,
shell: Shell, shell: Shell,
@ -408,7 +388,6 @@ def lock_object(
address: Optional[str] = None, address: Optional[str] = None,
bearer: Optional[str] = None, bearer: Optional[str] = None,
session: Optional[str] = None, session: Optional[str] = None,
wallet_config: Optional[str] = None,
ttl: Optional[int] = None, ttl: Optional[int] = None,
xhdr: Optional[dict] = None, xhdr: Optional[dict] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT, timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
@ -435,13 +414,12 @@ def lock_object(
Lock object ID Lock object ID
""" """
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or DEFAULT_WALLET_CONFIG) cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
result = cli.object.lock( result = cli.object.lock(
rpc_endpoint=endpoint, rpc_endpoint=endpoint,
lifetime=lifetime, lifetime=lifetime,
expire_at=expire_at, expire_at=expire_at,
address=address, address=address,
wallet=wallet,
cid=cid, cid=cid,
oid=oid, oid=oid,
bearer=bearer, bearer=bearer,
@ -459,14 +437,13 @@ def lock_object(
@reporter.step("Search object") @reporter.step("Search object")
def search_object( def search_object(
wallet: str, wallet: WalletInfo,
cid: str, cid: str,
shell: Shell, shell: Shell,
endpoint: str, endpoint: str,
bearer: str = "", bearer: str = "",
filters: Optional[dict] = None, filters: Optional[dict] = None,
expected_objects_list: Optional[list] = None, expected_objects_list: Optional[list] = None,
wallet_config: Optional[str] = None,
xhdr: Optional[dict] = None, xhdr: Optional[dict] = None,
session: Optional[str] = None, session: Optional[str] = None,
phy: bool = False, phy: bool = False,
@ -484,7 +461,6 @@ def search_object(
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
filters: key=value pairs to filter Objects filters: key=value pairs to filter Objects
expected_objects_list: a list of ObjectIDs to compare found Objects with expected_objects_list: a list of ObjectIDs to compare found Objects with
wallet_config: path to the wallet config
xhdr: Request X-Headers in form of Key=Value xhdr: Request X-Headers in form of Key=Value
session: path to a JSON-encoded container session token session: path to a JSON-encoded container session token
phy: Search physically stored objects. phy: Search physically stored objects.
@ -495,10 +471,9 @@ def search_object(
list of found ObjectIDs list of found ObjectIDs
""" """
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or DEFAULT_WALLET_CONFIG) cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
result = cli.object.search( result = cli.object.search(
rpc_endpoint=endpoint, rpc_endpoint=endpoint,
wallet=wallet,
cid=cid, cid=cid,
bearer=bearer, bearer=bearer,
xhdr=xhdr, xhdr=xhdr,
@ -513,23 +488,18 @@ def search_object(
if expected_objects_list: if expected_objects_list:
if sorted(found_objects) == sorted(expected_objects_list): if sorted(found_objects) == sorted(expected_objects_list):
logger.info( logger.info(f"Found objects list '{found_objects}' " f"is equal for expected list '{expected_objects_list}'")
f"Found objects list '{found_objects}' " f"is equal for expected list '{expected_objects_list}'"
)
else: else:
logger.warning( logger.warning(f"Found object list {found_objects} " f"is not equal to expected list '{expected_objects_list}'")
f"Found object list {found_objects} " f"is not equal to expected list '{expected_objects_list}'"
)
return found_objects return found_objects
@reporter.step("Get netmap netinfo") @reporter.step("Get netmap netinfo")
def get_netmap_netinfo( def get_netmap_netinfo(
wallet: str, wallet: WalletInfo,
shell: Shell, shell: Shell,
endpoint: str, endpoint: str,
wallet_config: Optional[str] = None,
address: Optional[str] = None, address: Optional[str] = None,
ttl: Optional[int] = None, ttl: Optional[int] = None,
xhdr: Optional[dict] = None, xhdr: Optional[dict] = None,
@ -539,7 +509,7 @@ def get_netmap_netinfo(
Get netmap netinfo output from node Get netmap netinfo output from node
Args: Args:
wallet (str): wallet on whose behalf request is done wallet (WalletInfo): wallet on whose behalf request is done
shell: executor for cli command shell: executor for cli command
endpoint (optional, str): FrostFS endpoint to send request to, appends to `--rpc-endpoint` key endpoint (optional, str): FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
address: Address of wallet account address: Address of wallet account
@ -552,9 +522,8 @@ def get_netmap_netinfo(
(dict): dict of parsed command output (dict): dict of parsed command output
""" """
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or DEFAULT_WALLET_CONFIG) cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
output = cli.netmap.netinfo( output = cli.netmap.netinfo(
wallet=wallet,
rpc_endpoint=endpoint, rpc_endpoint=endpoint,
address=address, address=address,
ttl=ttl, ttl=ttl,
@ -578,7 +547,7 @@ def get_netmap_netinfo(
@reporter.step("Head object") @reporter.step("Head object")
def head_object( def head_object(
wallet: str, wallet: WalletInfo,
cid: str, cid: str,
oid: str, oid: str,
shell: Shell, shell: Shell,
@ -588,7 +557,6 @@ def head_object(
json_output: bool = True, json_output: bool = True,
is_raw: bool = False, is_raw: bool = False,
is_direct: bool = False, is_direct: bool = False,
wallet_config: Optional[str] = None,
session: Optional[str] = None, session: Optional[str] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT, timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
): ):
@ -596,7 +564,7 @@ def head_object(
HEAD an Object. HEAD an Object.
Args: Args:
wallet (str): wallet on whose behalf HEAD is done wallet (WalletInfo): wallet on whose behalf HEAD is done
cid (str): ID of Container where we get the Object from cid (str): ID of Container where we get the Object from
oid (str): ObjectID to HEAD oid (str): ObjectID to HEAD
shell: executor for cli command shell: executor for cli command
@ -608,7 +576,6 @@ def head_object(
turns into `--raw` key turns into `--raw` key
is_direct(optional, bool): send request directly to the node or not; this flag is_direct(optional, bool): send request directly to the node or not; this flag
turns into `--ttl 1` key turns into `--ttl 1` key
wallet_config(optional, str): path to the wallet config
xhdr (optional, dict): Request X-Headers in form of Key=Value xhdr (optional, dict): Request X-Headers in form of Key=Value
session (optional, dict): path to a JSON-encoded container session token session (optional, dict): path to a JSON-encoded container session token
timeout: Timeout for the operation. timeout: Timeout for the operation.
@ -619,10 +586,9 @@ def head_object(
(str): HEAD response as a plain text (str): HEAD response as a plain text
""" """
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or DEFAULT_WALLET_CONFIG) cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
result = cli.object.head( result = cli.object.head(
rpc_endpoint=endpoint, rpc_endpoint=endpoint,
wallet=wallet,
cid=cid, cid=cid,
oid=oid, oid=oid,
bearer=bearer, bearer=bearer,
@ -673,7 +639,7 @@ def head_object(
@reporter.step("Run neo-go dump-keys") @reporter.step("Run neo-go dump-keys")
def neo_go_dump_keys(shell: Shell, wallet: str) -> dict: def neo_go_dump_keys(shell: Shell, wallet: WalletInfo) -> dict:
""" """
Run neo-go dump keys command Run neo-go dump keys command
@ -761,9 +727,7 @@ def get_object_nodes(
parsing_output = parse_cmd_table(result_object_nodes.stdout, "|") parsing_output = parse_cmd_table(result_object_nodes.stdout, "|")
list_object_nodes = [ list_object_nodes = [
node node for node in parsing_output if node["should_contain_object"] == "true" and node["actually_contains_object"] == "true"
for node in parsing_output
if node["should_contain_object"] == "true" and node["actually_contains_object"] == "true"
] ]
netmap_nodes_list = parse_netmap_output( netmap_nodes_list = parse_netmap_output(
@ -780,10 +744,7 @@ def get_object_nodes(
] ]
result = [ result = [
cluster_node cluster_node for netmap_node in netmap_nodes for cluster_node in cluster.cluster_nodes if netmap_node.node == cluster_node.host_ip
for netmap_node in netmap_nodes
for cluster_node in cluster.cluster_nodes
if netmap_node.node == cluster_node.host_ip
] ]
return result return result

View file

@ -14,11 +14,11 @@ from typing import Optional, Tuple
from frostfs_testlib import reporter from frostfs_testlib import reporter
from frostfs_testlib.resources.cli import CLI_DEFAULT_TIMEOUT from frostfs_testlib.resources.cli import CLI_DEFAULT_TIMEOUT
from frostfs_testlib.resources.common import DEFAULT_WALLET_CONFIG
from frostfs_testlib.shell import Shell from frostfs_testlib.shell import Shell
from frostfs_testlib.steps.cli.object import head_object from frostfs_testlib.steps.cli.object import head_object
from frostfs_testlib.storage.cluster import Cluster, StorageNode from frostfs_testlib.storage.cluster import Cluster, StorageNode
from frostfs_testlib.storage.dataclasses.storage_object_info import StorageObjectInfo from frostfs_testlib.storage.dataclasses.storage_object_info import StorageObjectInfo
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
@ -44,7 +44,7 @@ def get_storage_object_chunks(
with reporter.step(f"Get complex object chunks (f{storage_object.oid})"): with reporter.step(f"Get complex object chunks (f{storage_object.oid})"):
split_object_id = get_link_object( split_object_id = get_link_object(
storage_object.wallet_file_path, storage_object.wallet,
storage_object.cid, storage_object.cid,
storage_object.oid, storage_object.oid,
shell, shell,
@ -53,7 +53,7 @@ def get_storage_object_chunks(
timeout=timeout, timeout=timeout,
) )
head = head_object( head = head_object(
storage_object.wallet_file_path, storage_object.wallet,
storage_object.cid, storage_object.cid,
split_object_id, split_object_id,
shell, shell,
@ -96,7 +96,7 @@ def get_complex_object_split_ranges(
chunks_ids = get_storage_object_chunks(storage_object, shell, cluster) chunks_ids = get_storage_object_chunks(storage_object, shell, cluster)
for chunk_id in chunks_ids: for chunk_id in chunks_ids:
head = head_object( head = head_object(
storage_object.wallet_file_path, storage_object.wallet,
storage_object.cid, storage_object.cid,
chunk_id, chunk_id,
shell, shell,
@ -114,13 +114,12 @@ def get_complex_object_split_ranges(
@reporter.step("Get Link Object") @reporter.step("Get Link Object")
def get_link_object( def get_link_object(
wallet: str, wallet: WalletInfo,
cid: str, cid: str,
oid: str, oid: str,
shell: Shell, shell: Shell,
nodes: list[StorageNode], nodes: list[StorageNode],
bearer: str = "", bearer: str = "",
wallet_config: str = DEFAULT_WALLET_CONFIG,
is_direct: bool = True, is_direct: bool = True,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT, timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
): ):
@ -154,7 +153,6 @@ def get_link_object(
is_raw=True, is_raw=True,
is_direct=is_direct, is_direct=is_direct,
bearer=bearer, bearer=bearer,
wallet_config=wallet_config,
timeout=timeout, timeout=timeout,
) )
if resp["link"]: if resp["link"]:
@ -167,7 +165,7 @@ def get_link_object(
@reporter.step("Get Last Object") @reporter.step("Get Last Object")
def get_last_object( def get_last_object(
wallet: str, wallet: WalletInfo,
cid: str, cid: str,
oid: str, oid: str,
shell: Shell, shell: Shell,

View file

@ -4,13 +4,7 @@ from typing import Optional
from frostfs_testlib import reporter from frostfs_testlib import reporter
from frostfs_testlib.cli import FrostfsAdm, FrostfsCli, NeoGo from frostfs_testlib.cli import FrostfsAdm, FrostfsCli, NeoGo
from frostfs_testlib.resources.cli import ( from frostfs_testlib.resources.cli import CLI_DEFAULT_TIMEOUT, FROSTFS_ADM_CONFIG_PATH, FROSTFS_ADM_EXEC, FROSTFS_CLI_EXEC, NEOGO_EXECUTABLE
CLI_DEFAULT_TIMEOUT,
FROSTFS_ADM_CONFIG_PATH,
FROSTFS_ADM_EXEC,
FROSTFS_CLI_EXEC,
NEOGO_EXECUTABLE,
)
from frostfs_testlib.resources.common import MORPH_BLOCK_TIME from frostfs_testlib.resources.common import MORPH_BLOCK_TIME
from frostfs_testlib.shell import Shell from frostfs_testlib.shell import Shell
from frostfs_testlib.steps.payment_neogo import get_contract_hash from frostfs_testlib.steps.payment_neogo import get_contract_hash

View file

@ -10,6 +10,7 @@ from frostfs_testlib.s3 import S3ClientWrapper, VersioningStatus
from frostfs_testlib.shell import Shell from frostfs_testlib.shell import Shell
from frostfs_testlib.steps.cli.container import search_container_by_name, search_nodes_with_container from frostfs_testlib.steps.cli.container import search_container_by_name, search_nodes_with_container
from frostfs_testlib.storage.cluster import Cluster, ClusterNode from frostfs_testlib.storage.cluster import Cluster, ClusterNode
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
@ -28,9 +29,7 @@ def check_objects_in_bucket(
assert bucket_object in bucket_objects, f"Expected object {bucket_object} in objects list {bucket_objects}" assert bucket_object in bucket_objects, f"Expected object {bucket_object} in objects list {bucket_objects}"
for bucket_object in unexpected_objects: for bucket_object in unexpected_objects:
assert ( assert bucket_object not in bucket_objects, f"Expected object {bucket_object} not in objects list {bucket_objects}"
bucket_object not in bucket_objects
), f"Expected object {bucket_object} not in objects list {bucket_objects}"
@reporter.step("Try to get object and got error") @reporter.step("Try to get object and got error")
@ -58,9 +57,7 @@ def object_key_from_file_path(full_path: str) -> str:
return os.path.basename(full_path) return os.path.basename(full_path)
def assert_tags( def assert_tags(actual_tags: list, expected_tags: Optional[list] = None, unexpected_tags: Optional[list] = None) -> None:
actual_tags: list, expected_tags: Optional[list] = None, unexpected_tags: Optional[list] = None
) -> None:
expected_tags = [{"Key": key, "Value": value} for key, value in expected_tags] if expected_tags else [] expected_tags = [{"Key": key, "Value": value} for key, value in expected_tags] if expected_tags else []
unexpected_tags = [{"Key": key, "Value": value} for key, value in unexpected_tags] if unexpected_tags else [] unexpected_tags = [{"Key": key, "Value": value} for key, value in unexpected_tags] if unexpected_tags else []
if expected_tags == []: if expected_tags == []:
@ -180,7 +177,7 @@ def delete_bucket_with_objects(s3_client: S3ClientWrapper, bucket: str):
def search_nodes_with_bucket( def search_nodes_with_bucket(
cluster: Cluster, cluster: Cluster,
bucket_name: str, bucket_name: str,
wallet: str, wallet: WalletInfo,
shell: Shell, shell: Shell,
endpoint: str, endpoint: str,
) -> list[ClusterNode]: ) -> list[ClusterNode]:

View file

@ -4,13 +4,12 @@ import logging
import os import os
import uuid import uuid
from dataclasses import dataclass from dataclasses import dataclass
from enum import Enum
from typing import Any, Optional from typing import Any, Optional
from frostfs_testlib import reporter from frostfs_testlib import reporter
from frostfs_testlib.cli import FrostfsCli from frostfs_testlib.cli import FrostfsCli
from frostfs_testlib.resources.cli import FROSTFS_CLI_EXEC from frostfs_testlib.resources.cli import FROSTFS_CLI_EXEC
from frostfs_testlib.resources.common import ASSETS_DIR, DEFAULT_WALLET_CONFIG from frostfs_testlib.resources.common import ASSETS_DIR
from frostfs_testlib.shell import Shell from frostfs_testlib.shell import Shell
from frostfs_testlib.storage.dataclasses.storage_object_info import StorageObjectInfo from frostfs_testlib.storage.dataclasses.storage_object_info import StorageObjectInfo
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
@ -231,8 +230,7 @@ def get_object_signed_token(
def create_session_token( def create_session_token(
shell: Shell, shell: Shell,
owner: str, owner: str,
wallet_path: str, wallet: WalletInfo,
wallet_password: str,
rpc_endpoint: str, rpc_endpoint: str,
) -> str: ) -> str:
""" """
@ -247,19 +245,18 @@ def create_session_token(
The path to the generated session token file. The path to the generated session token file.
""" """
session_token = os.path.join(os.getcwd(), ASSETS_DIR, str(uuid.uuid4())) session_token = os.path.join(os.getcwd(), ASSETS_DIR, str(uuid.uuid4()))
frostfscli = FrostfsCli(shell=shell, frostfs_cli_exec_path=FROSTFS_CLI_EXEC) frostfscli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
frostfscli.session.create( frostfscli.session.create(
rpc_endpoint=rpc_endpoint, rpc_endpoint=rpc_endpoint,
address=owner, address=owner,
wallet=wallet_path,
wallet_password=wallet_password,
out=session_token, out=session_token,
wallet=wallet.path,
) )
return session_token return session_token
@reporter.step("Sign Session Token") @reporter.step("Sign Session Token")
def sign_session_token(shell: Shell, session_token_file: str, wlt: WalletInfo) -> str: def sign_session_token(shell: Shell, session_token_file: str, wallet: WalletInfo) -> str:
""" """
This function signs the session token by the given wallet. This function signs the session token by the given wallet.
@ -272,6 +269,6 @@ def sign_session_token(shell: Shell, session_token_file: str, wlt: WalletInfo) -
The path to the signed token. The path to the signed token.
""" """
signed_token_file = os.path.join(os.getcwd(), ASSETS_DIR, str(uuid.uuid4())) signed_token_file = os.path.join(os.getcwd(), ASSETS_DIR, str(uuid.uuid4()))
frostfscli = FrostfsCli(shell=shell, frostfs_cli_exec_path=FROSTFS_CLI_EXEC, config_file=DEFAULT_WALLET_CONFIG) frostfscli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
frostfscli.util.sign_session_token(wallet=wlt.path, from_file=session_token_file, to_file=signed_token_file) frostfscli.util.sign_session_token(session_token_file, signed_token_file)
return signed_token_file return signed_token_file

View file

@ -30,14 +30,14 @@ def delete_objects(storage_objects: list[StorageObjectInfo], shell: Shell, clust
with reporter.step("Delete objects"): with reporter.step("Delete objects"):
for storage_object in storage_objects: for storage_object in storage_objects:
storage_object.tombstone = delete_object( storage_object.tombstone = delete_object(
storage_object.wallet_file_path, storage_object.wallet,
storage_object.cid, storage_object.cid,
storage_object.oid, storage_object.oid,
shell=shell, shell=shell,
endpoint=cluster.default_rpc_endpoint, endpoint=cluster.default_rpc_endpoint,
) )
verify_head_tombstone( verify_head_tombstone(
wallet_path=storage_object.wallet_file_path, wallet=storage_object.wallet,
cid=storage_object.cid, cid=storage_object.cid,
oid_ts=storage_object.tombstone, oid_ts=storage_object.tombstone,
oid=storage_object.oid, oid=storage_object.oid,
@ -52,7 +52,7 @@ def delete_objects(storage_objects: list[StorageObjectInfo], shell: Shell, clust
for storage_object in storage_objects: for storage_object in storage_objects:
with pytest.raises(Exception, match=OBJECT_ALREADY_REMOVED): with pytest.raises(Exception, match=OBJECT_ALREADY_REMOVED):
get_object( get_object(
storage_object.wallet_file_path, storage_object.wallet,
storage_object.cid, storage_object.cid,
storage_object.oid, storage_object.oid,
shell=shell, shell=shell,

View file

@ -12,13 +12,15 @@ from frostfs_testlib.shell import Shell
from frostfs_testlib.steps.cli.object import head_object from frostfs_testlib.steps.cli.object import head_object
from frostfs_testlib.steps.complex_object_actions import get_last_object from frostfs_testlib.steps.complex_object_actions import get_last_object
from frostfs_testlib.storage.cluster import StorageNode from frostfs_testlib.storage.cluster import StorageNode
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.utils import string_utils from frostfs_testlib.utils import string_utils
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
# TODO: Unused, remove or make use of
@reporter.step("Get Object Copies") @reporter.step("Get Object Copies")
def get_object_copies(complexity: str, wallet: str, cid: str, oid: str, shell: Shell, nodes: list[StorageNode]) -> int: def get_object_copies(complexity: str, wallet: WalletInfo, cid: str, oid: str, shell: Shell, nodes: list[StorageNode]) -> int:
""" """
The function performs requests to all nodes of the container and The function performs requests to all nodes of the container and
finds out if they store a copy of the object. The procedure is finds out if they store a copy of the object. The procedure is
@ -43,7 +45,7 @@ def get_object_copies(complexity: str, wallet: str, cid: str, oid: str, shell: S
@reporter.step("Get Simple Object Copies") @reporter.step("Get Simple Object Copies")
def get_simple_object_copies(wallet: str, cid: str, oid: str, shell: Shell, nodes: list[StorageNode]) -> int: def get_simple_object_copies(wallet: WalletInfo, cid: str, oid: str, shell: Shell, nodes: list[StorageNode]) -> int:
""" """
To figure out the number of a simple object copies, only direct To figure out the number of a simple object copies, only direct
HEAD requests should be made to the every node of the container. HEAD requests should be made to the every node of the container.
@ -72,7 +74,7 @@ def get_simple_object_copies(wallet: str, cid: str, oid: str, shell: Shell, node
@reporter.step("Get Complex Object Copies") @reporter.step("Get Complex Object Copies")
def get_complex_object_copies(wallet: str, cid: str, oid: str, shell: Shell, nodes: list[StorageNode]) -> int: def get_complex_object_copies(wallet: WalletInfo, cid: str, oid: str, shell: Shell, nodes: list[StorageNode]) -> int:
""" """
To figure out the number of a complex object copies, we firstly To figure out the number of a complex object copies, we firstly
need to retrieve its Last object. We consider that the number of need to retrieve its Last object. We consider that the number of
@ -109,8 +111,7 @@ def get_nodes_with_object(cid: str, oid: str, shell: Shell, nodes: list[StorageN
nodes_list = [] nodes_list = []
for node in nodes: for node in nodes:
wallet = node.get_wallet_path() wallet = WalletInfo.from_node(node)
wallet_config = node.get_wallet_config_path()
try: try:
res = head_object( res = head_object(
wallet, wallet,
@ -119,7 +120,6 @@ def get_nodes_with_object(cid: str, oid: str, shell: Shell, nodes: list[StorageN
shell=shell, shell=shell,
endpoint=node.get_rpc_endpoint(), endpoint=node.get_rpc_endpoint(),
is_direct=True, is_direct=True,
wallet_config=wallet_config,
) )
if res is not None: if res is not None:
logger.info(f"Found object {oid} on node {node}") logger.info(f"Found object {oid} on node {node}")
@ -131,9 +131,7 @@ def get_nodes_with_object(cid: str, oid: str, shell: Shell, nodes: list[StorageN
@reporter.step("Get Nodes Without Object") @reporter.step("Get Nodes Without Object")
def get_nodes_without_object( def get_nodes_without_object(wallet: WalletInfo, cid: str, oid: str, shell: Shell, nodes: list[StorageNode]) -> list[StorageNode]:
wallet: str, cid: str, oid: str, shell: Shell, nodes: list[StorageNode]
) -> list[StorageNode]:
""" """
The function returns list of nodes which do not store The function returns list of nodes which do not store
the given object. the given object.

View file

@ -1,31 +1,23 @@
import json
import logging import logging
from neo3.wallet import wallet
from frostfs_testlib import reporter from frostfs_testlib import reporter
from frostfs_testlib.shell import Shell from frostfs_testlib.shell import Shell
from frostfs_testlib.steps.cli.object import head_object from frostfs_testlib.steps.cli.object import head_object
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
@reporter.step("Verify Head Tombstone") @reporter.step("Verify Head Tombstone")
def verify_head_tombstone(wallet_path: str, cid: str, oid_ts: str, oid: str, shell: Shell, endpoint: str): def verify_head_tombstone(wallet: WalletInfo, cid: str, oid_ts: str, oid: str, shell: Shell, endpoint: str):
header = head_object(wallet_path, cid, oid_ts, shell=shell, endpoint=endpoint)["header"] header = head_object(wallet, cid, oid_ts, shell=shell, endpoint=endpoint)["header"]
s_oid = header["sessionToken"]["body"]["object"]["target"]["objects"] s_oid = header["sessionToken"]["body"]["object"]["target"]["objects"]
logger.info(f"Header Session OIDs is {s_oid}") logger.info(f"Header Session OIDs is {s_oid}")
logger.info(f"OID is {oid}") logger.info(f"OID is {oid}")
assert header["containerID"] == cid, "Tombstone Header CID is wrong" assert header["containerID"] == cid, "Tombstone Header CID is wrong"
assert header["ownerID"] == wallet.get_address_from_json(0), "Tombstone Owner ID is wrong"
with open(wallet_path, "r") as file:
wlt_data = json.loads(file.read())
wlt = wallet.Wallet.from_json(wlt_data, password="")
addr = wlt.accounts[0].address
assert header["ownerID"] == addr, "Tombstone Owner ID is wrong"
assert header["objectType"] == "TOMBSTONE", "Header Type isn't Tombstone" assert header["objectType"] == "TOMBSTONE", "Header Type isn't Tombstone"
assert header["sessionToken"]["body"]["object"]["verb"] == "DELETE", "Header Session Type isn't DELETE" assert header["sessionToken"]["body"]["object"]["verb"] == "DELETE", "Header Session Type isn't DELETE"
assert header["sessionToken"]["body"]["object"]["target"]["container"] == cid, "Header Session ID is wrong" assert header["sessionToken"]["body"]["object"]["target"]["container"] == cid, "Header Session ID is wrong"

View file

@ -1,6 +1,5 @@
import copy import copy
from datetime import datetime from datetime import datetime
from typing import Optional
import frostfs_testlib.resources.optionals as optionals import frostfs_testlib.resources.optionals as optionals
from frostfs_testlib import reporter from frostfs_testlib import reporter
@ -10,7 +9,6 @@ from frostfs_testlib.load.load_report import LoadReport
from frostfs_testlib.load.load_verifiers import LoadVerifier from frostfs_testlib.load.load_verifiers import LoadVerifier
from frostfs_testlib.storage.cluster import ClusterNode from frostfs_testlib.storage.cluster import ClusterNode
from frostfs_testlib.storage.dataclasses.frostfs_services import S3Gate, StorageNode from frostfs_testlib.storage.dataclasses.frostfs_services import S3Gate, StorageNode
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.parallel import parallel from frostfs_testlib.testing.parallel import parallel
from frostfs_testlib.testing.test_control import run_optionally from frostfs_testlib.testing.test_control import run_optionally
@ -23,7 +21,6 @@ class BackgroundLoadController:
cluster_nodes: list[ClusterNode] cluster_nodes: list[ClusterNode]
nodes_under_load: list[ClusterNode] nodes_under_load: list[ClusterNode]
load_counter: int load_counter: int
loaders_wallet: WalletInfo
load_summaries: dict load_summaries: dict
endpoints: list[str] endpoints: list[str]
runner: ScenarioRunner runner: ScenarioRunner
@ -34,7 +31,6 @@ class BackgroundLoadController:
self, self,
k6_dir: str, k6_dir: str,
load_params: LoadParams, load_params: LoadParams,
loaders_wallet: WalletInfo,
cluster_nodes: list[ClusterNode], cluster_nodes: list[ClusterNode],
nodes_under_load: list[ClusterNode], nodes_under_load: list[ClusterNode],
runner: ScenarioRunner, runner: ScenarioRunner,
@ -45,7 +41,6 @@ class BackgroundLoadController:
self.cluster_nodes = cluster_nodes self.cluster_nodes = cluster_nodes
self.nodes_under_load = nodes_under_load self.nodes_under_load = nodes_under_load
self.load_counter = 1 self.load_counter = 1
self.loaders_wallet = loaders_wallet
self.runner = runner self.runner = runner
self.started = False self.started = False
self.load_reporters = [] self.load_reporters = []
@ -64,10 +59,7 @@ class BackgroundLoadController:
) )
), ),
EndpointSelectionStrategy.FIRST: list( EndpointSelectionStrategy.FIRST: list(
set( set(node_under_load.service(StorageNode).get_rpc_endpoint() for node_under_load in self.nodes_under_load)
node_under_load.service(StorageNode).get_rpc_endpoint()
for node_under_load in self.nodes_under_load
)
), ),
}, },
# for some reason xk6 appends http protocol on its own # for some reason xk6 appends http protocol on its own

View file

@ -11,12 +11,13 @@ from frostfs_testlib.healthcheck.interfaces import Healthcheck
from frostfs_testlib.hosting.interfaces import HostStatus from frostfs_testlib.hosting.interfaces import HostStatus
from frostfs_testlib.plugins import load_all from frostfs_testlib.plugins import load_all
from frostfs_testlib.resources.cli import FROSTFS_ADM_CONFIG_PATH, FROSTFS_ADM_EXEC, FROSTFS_CLI_EXEC from frostfs_testlib.resources.cli import FROSTFS_ADM_CONFIG_PATH, FROSTFS_ADM_EXEC, FROSTFS_CLI_EXEC
from frostfs_testlib.resources.common import DEFAULT_WALLET_CONFIG, MORPH_BLOCK_TIME from frostfs_testlib.resources.common import MORPH_BLOCK_TIME
from frostfs_testlib.shell import CommandOptions, Shell, SshConnectionProvider from frostfs_testlib.shell import CommandOptions, Shell, SshConnectionProvider
from frostfs_testlib.steps.network import IpHelper from frostfs_testlib.steps.network import IpHelper
from frostfs_testlib.storage.cluster import Cluster, ClusterNode, S3Gate, StorageNode from frostfs_testlib.storage.cluster import Cluster, ClusterNode, S3Gate, StorageNode
from frostfs_testlib.storage.controllers.disk_controller import DiskController from frostfs_testlib.storage.controllers.disk_controller import DiskController
from frostfs_testlib.storage.dataclasses.node_base import NodeBase, ServiceClass from frostfs_testlib.storage.dataclasses.node_base import NodeBase, ServiceClass
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing import parallel from frostfs_testlib.testing import parallel
from frostfs_testlib.testing.test_control import retry, run_optionally, wait_for_success from frostfs_testlib.testing.test_control import retry, run_optionally, wait_for_success
from frostfs_testlib.utils.datetime_utils import parse_time from frostfs_testlib.utils.datetime_utils import parse_time
@ -413,12 +414,12 @@ class ClusterStateController:
frostfs_adm.morph.set_config(set_key_value=f"MaintenanceModeAllowed={status}") frostfs_adm.morph.set_config(set_key_value=f"MaintenanceModeAllowed={status}")
@reporter.step("Set mode node to {status}") @reporter.step("Set mode node to {status}")
def set_mode_node(self, cluster_node: ClusterNode, wallet: str, status: str, await_tick: bool = True) -> None: def set_mode_node(self, cluster_node: ClusterNode, wallet: WalletInfo, status: str, await_tick: bool = True) -> None:
rpc_endpoint = cluster_node.storage_node.get_rpc_endpoint() rpc_endpoint = cluster_node.storage_node.get_rpc_endpoint()
control_endpoint = cluster_node.service(StorageNode).get_control_endpoint() control_endpoint = cluster_node.service(StorageNode).get_control_endpoint()
frostfs_adm, frostfs_cli, frostfs_cli_remote = self._get_cli(local_shell=self.shell, cluster_node=cluster_node) frostfs_adm, frostfs_cli, frostfs_cli_remote = self._get_cli(local_shell=self.shell, local_wallet=wallet, cluster_node=cluster_node)
node_netinfo = NetmapParser.netinfo(frostfs_cli.netmap.netinfo(rpc_endpoint=rpc_endpoint, wallet=wallet).stdout) node_netinfo = NetmapParser.netinfo(frostfs_cli.netmap.netinfo(rpc_endpoint=rpc_endpoint).stdout)
with reporter.step("If status maintenance, then check that the option is enabled"): with reporter.step("If status maintenance, then check that the option is enabled"):
if node_netinfo.maintenance_mode_allowed == "false": if node_netinfo.maintenance_mode_allowed == "false":
@ -437,12 +438,10 @@ class ClusterStateController:
self.check_node_status(status=status, wallet=wallet, cluster_node=cluster_node) self.check_node_status(status=status, wallet=wallet, cluster_node=cluster_node)
@wait_for_success(80, 8, title="Wait for storage status become {status}") @wait_for_success(80, 8, title="Wait for storage status become {status}")
def check_node_status(self, status: str, wallet: str, cluster_node: ClusterNode): def check_node_status(self, status: str, wallet: WalletInfo, cluster_node: ClusterNode):
frostfs_cli = FrostfsCli( frostfs_cli = FrostfsCli(self.shell, FROSTFS_CLI_EXEC, wallet.config_path)
shell=self.shell, frostfs_cli_exec_path=FROSTFS_CLI_EXEC, config_file=DEFAULT_WALLET_CONFIG
)
netmap = NetmapParser.snapshot_all_nodes( netmap = NetmapParser.snapshot_all_nodes(
frostfs_cli.netmap.snapshot(rpc_endpoint=cluster_node.storage_node.get_rpc_endpoint(), wallet=wallet).stdout frostfs_cli.netmap.snapshot(rpc_endpoint=cluster_node.storage_node.get_rpc_endpoint()).stdout
) )
netmap = [node for node in netmap if cluster_node.host_ip == node.node] netmap = [node for node in netmap if cluster_node.host_ip == node.node]
if status == "offline": if status == "offline":
@ -450,7 +449,9 @@ class ClusterStateController:
else: else:
assert netmap[0].node_status == status.upper(), f"Node state - {netmap[0].node_status} != {status} expect" assert netmap[0].node_status == status.upper(), f"Node state - {netmap[0].node_status} != {status} expect"
def _get_cli(self, local_shell: Shell, cluster_node: ClusterNode) -> tuple[FrostfsAdm, FrostfsCli, FrostfsCli]: def _get_cli(
self, local_shell: Shell, local_wallet: WalletInfo, cluster_node: ClusterNode
) -> tuple[FrostfsAdm, FrostfsCli, FrostfsCli]:
# TODO Move to service config # TODO Move to service config
host = cluster_node.host host = cluster_node.host
service_config = host.get_service_config(cluster_node.storage_node.name) service_config = host.get_service_config(cluster_node.storage_node.name)
@ -462,12 +463,8 @@ class ClusterStateController:
wallet_config = f'wallet: {wallet_path}\npassword: "{wallet_password}"' wallet_config = f'wallet: {wallet_path}\npassword: "{wallet_password}"'
shell.exec(f"echo '{wallet_config}' > {wallet_config_path}") shell.exec(f"echo '{wallet_config}' > {wallet_config_path}")
frostfs_adm = FrostfsAdm( frostfs_adm = FrostfsAdm(shell=shell, frostfs_adm_exec_path=FROSTFS_ADM_EXEC, config_file=FROSTFS_ADM_CONFIG_PATH)
shell=shell, frostfs_adm_exec_path=FROSTFS_ADM_EXEC, config_file=FROSTFS_ADM_CONFIG_PATH frostfs_cli = FrostfsCli(local_shell, FROSTFS_CLI_EXEC, local_wallet.config_path)
)
frostfs_cli = FrostfsCli(
shell=local_shell, frostfs_cli_exec_path=FROSTFS_CLI_EXEC, config_file=DEFAULT_WALLET_CONFIG
)
frostfs_cli_remote = FrostfsCli( frostfs_cli_remote = FrostfsCli(
shell=shell, shell=shell,
frostfs_cli_exec_path=FROSTFS_CLI_EXEC, frostfs_cli_exec_path=FROSTFS_CLI_EXEC,
@ -511,9 +508,7 @@ class ClusterStateController:
options = CommandOptions(check=False) options = CommandOptions(check=False)
return self.shell.exec(f"ping {node.host.config.address} -c 1", options).return_code return self.shell.exec(f"ping {node.host.config.address} -c 1", options).return_code
@retry( @retry(max_attempts=60, sleep_interval=10, expected_result=HostStatus.ONLINE, title="Waiting for {node} to go online")
max_attempts=60, sleep_interval=10, expected_result=HostStatus.ONLINE, title="Waiting for {node} to go online"
)
def _wait_for_host_online(self, node: ClusterNode): def _wait_for_host_online(self, node: ClusterNode):
try: try:
ping_result = self._ping_host(node) ping_result = self._ping_host(node)
@ -524,9 +519,7 @@ class ClusterStateController:
logger.warning(f"Host ping fails with error {err}") logger.warning(f"Host ping fails with error {err}")
return HostStatus.OFFLINE return HostStatus.OFFLINE
@retry( @retry(max_attempts=60, sleep_interval=10, expected_result=HostStatus.OFFLINE, title="Waiting for {node} to go offline")
max_attempts=60, sleep_interval=10, expected_result=HostStatus.OFFLINE, title="Waiting for {node} to go offline"
)
def _wait_for_host_offline(self, node: ClusterNode): def _wait_for_host_offline(self, node: ClusterNode):
try: try:
ping_result = self._ping_host(node) ping_result = self._ping_host(node)

View file

@ -1,8 +1,8 @@
import logging import logging
from dataclasses import dataclass from dataclasses import dataclass
from enum import Enum
from typing import Any, Dict, List, Optional, Union from typing import Any, Dict, List, Optional, Union
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.readable import HumanReadableEnum from frostfs_testlib.testing.readable import HumanReadableEnum
from frostfs_testlib.utils import wallet_utils from frostfs_testlib.utils import wallet_utils
@ -65,11 +65,7 @@ class EACLFilters:
def __str__(self): def __str__(self):
return ",".join( return ",".join(
[ [f"{filter.header_type.value}:" f"{filter.key}{filter.match_type.value}{filter.value}" for filter in self.filters]
f"{filter.header_type.value}:"
f"{filter.key}{filter.match_type.value}{filter.value}"
for filter in self.filters
]
if self.filters if self.filters
else [] else []
) )
@ -84,7 +80,7 @@ class EACLPubKey:
class EACLRule: class EACLRule:
operation: Optional[EACLOperation] = None operation: Optional[EACLOperation] = None
access: Optional[EACLAccess] = None access: Optional[EACLAccess] = None
role: Optional[Union[EACLRole, str]] = None role: Optional[Union[EACLRole, WalletInfo]] = None
filters: Optional[EACLFilters] = None filters: Optional[EACLFilters] = None
def to_dict(self) -> Dict[str, Any]: def to_dict(self) -> Dict[str, Any]:
@ -96,9 +92,9 @@ class EACLRule:
} }
def __str__(self): def __str__(self):
role = ( role = ""
self.role.value if isinstance(self.role, EACLRole):
if isinstance(self.role, EACLRole) role = self.role.value
else f'pubkey:{wallet_utils.get_wallet_public_key(self.role, "")}' if isinstance(self.role, WalletInfo):
) role = f"pubkey:{wallet_utils.get_wallet_public_key(self.role.path, self.role.password)}"
return f'{self.access.value} {self.operation.value} {self.filters or ""} {role}' return f'{self.access.value} {self.operation.value} {self.filters or ""} {role}'

View file

@ -1,6 +1,7 @@
from dataclasses import dataclass from dataclasses import dataclass
from typing import Optional from typing import Optional
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.readable import HumanReadableEnum from frostfs_testlib.testing.readable import HumanReadableEnum
@ -19,7 +20,7 @@ class LockObjectInfo(ObjectRef):
@dataclass @dataclass
class StorageObjectInfo(ObjectRef): class StorageObjectInfo(ObjectRef):
size: Optional[int] = None size: Optional[int] = None
wallet_file_path: Optional[str] = None wallet: Optional[WalletInfo] = None
file_path: Optional[str] = None file_path: Optional[str] = None
file_hash: Optional[str] = None file_hash: Optional[str] = None
attributes: Optional[list[dict[str, str]]] = None attributes: Optional[list[dict[str, str]]] = None

View file

@ -1,13 +1,15 @@
import json import json
import logging import logging
import os import os
import uuid
from dataclasses import dataclass from dataclasses import dataclass
from typing import Optional from typing import Optional
from frostfs_testlib.resources.common import DEFAULT_WALLET_CONFIG, DEFAULT_WALLET_PASS import yaml
from frostfs_testlib import reporter
from frostfs_testlib.resources.common import ASSETS_DIR, DEFAULT_WALLET_CONFIG, DEFAULT_WALLET_PASS
from frostfs_testlib.shell import Shell from frostfs_testlib.shell import Shell
from frostfs_testlib.storage.cluster import Cluster, NodeBase from frostfs_testlib.storage.cluster import NodeBase
from frostfs_testlib.utils.wallet_utils import get_last_address_from_wallet, init_wallet from frostfs_testlib.utils.wallet_utils import get_last_address_from_wallet, init_wallet
logger = logging.getLogger("frostfs.testlib.utils") logger = logging.getLogger("frostfs.testlib.utils")
@ -21,9 +23,13 @@ class WalletInfo:
@staticmethod @staticmethod
def from_node(node: NodeBase): def from_node(node: NodeBase):
return WalletInfo( wallet_path = node.get_wallet_path()
node.get_wallet_path(), node.get_wallet_password(), node.get_wallet_config_path() wallet_password = node.get_wallet_password()
) wallet_config_file = os.path.join(ASSETS_DIR, os.path.basename(node.get_wallet_config_path()))
with open(wallet_config_file, "w") as file:
file.write(yaml.dump({"wallet": wallet_path, "password": wallet_password}))
return WalletInfo(wallet_path, wallet_password, wallet_config_file)
def get_address(self) -> str: def get_address(self) -> str:
""" """
@ -47,22 +53,17 @@ class WalletInfo:
""" """
with open(self.path, "r") as wallet: with open(self.path, "r") as wallet:
wallet_json = json.load(wallet) wallet_json = json.load(wallet)
assert abs(account_id) + 1 <= len( assert abs(account_id) + 1 <= len(wallet_json["accounts"]), f"There is no index '{account_id}' in wallet: {wallet_json}"
wallet_json["accounts"]
), f"There is no index '{account_id}' in wallet: {wallet_json}"
return wallet_json["accounts"][account_id]["address"] return wallet_json["accounts"][account_id]["address"]
class WalletFactory: class WalletFactory:
def __init__(self, wallets_dir: str, shell: Shell, cluster: Cluster) -> None: def __init__(self, wallets_dir: str, shell: Shell) -> None:
self.shell = shell self.shell = shell
self.wallets_dir = wallets_dir self.wallets_dir = wallets_dir
self.cluster = cluster
def create_wallet( def create_wallet(self, file_name: str, password: Optional[str] = None) -> WalletInfo:
self, file_name: Optional[str] = None, password: Optional[str] = None
) -> WalletInfo:
""" """
Creates new default wallet. Creates new default wallet.
@ -74,8 +75,6 @@ class WalletFactory:
WalletInfo object of new wallet. WalletInfo object of new wallet.
""" """
if file_name is None:
file_name = str(uuid.uuid4())
if password is None: if password is None:
password = "" password = ""
@ -85,6 +84,8 @@ class WalletFactory:
init_wallet(wallet_path, password) init_wallet(wallet_path, password)
with open(wallet_config_path, "w") as config_file: with open(wallet_config_path, "w") as config_file:
config_file.write(f'password: "{password}"') config_file.write(f'wallet: {wallet_path}\npassword: "{password}"')
reporter.attach(wallet_path, os.path.basename(wallet_path))
return WalletInfo(wallet_path, password, wallet_config_path) return WalletInfo(wallet_path, password, wallet_config_path)

View file

@ -4,7 +4,6 @@ import re
from frostfs_testlib.cli import FrostfsAdm, FrostfsCli from frostfs_testlib.cli import FrostfsAdm, FrostfsCli
from frostfs_testlib.hosting import Host, Hosting from frostfs_testlib.hosting import Host, Hosting
from frostfs_testlib.resources.cli import FROSTFS_ADM_EXEC, FROSTFS_AUTHMATE_EXEC, FROSTFS_CLI_EXEC, NEOGO_EXECUTABLE from frostfs_testlib.resources.cli import FROSTFS_ADM_EXEC, FROSTFS_AUTHMATE_EXEC, FROSTFS_CLI_EXEC, NEOGO_EXECUTABLE
from frostfs_testlib.resources.common import DEFAULT_WALLET_CONFIG
from frostfs_testlib.shell import Shell from frostfs_testlib.shell import Shell
from frostfs_testlib.testing.parallel import parallel from frostfs_testlib.testing.parallel import parallel
@ -18,7 +17,7 @@ def get_local_binaries_versions(shell: Shell) -> dict[str, str]:
out = shell.exec(f"{binary} --version").stdout out = shell.exec(f"{binary} --version").stdout
versions[binary] = _parse_version(out) versions[binary] = _parse_version(out)
frostfs_cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, DEFAULT_WALLET_CONFIG) frostfs_cli = FrostfsCli(shell, FROSTFS_CLI_EXEC)
versions[FROSTFS_CLI_EXEC] = _parse_version(frostfs_cli.version.get().stdout) versions[FROSTFS_CLI_EXEC] = _parse_version(frostfs_cli.version.get().stdout)
try: try:
@ -36,7 +35,7 @@ def get_local_binaries_versions(shell: Shell) -> dict[str, str]:
def parallel_binary_verions(host: Host) -> dict[str, str]: def parallel_binary_verions(host: Host) -> dict[str, str]:
versions_by_host = {} versions_by_host = {}
binary_path_by_name = {} # Maps binary name to executable path binary_path_by_name = {} # Maps binary name to executable path
for service_config in host.config.services: for service_config in host.config.services:
exec_path = service_config.attributes.get("exec_path") exec_path = service_config.attributes.get("exec_path")
@ -65,7 +64,7 @@ def parallel_binary_verions(host: Host) -> dict[str, str]:
versions_at_host[binary_name] = {"version": "Unknown", "check": binary["check"]} versions_at_host[binary_name] = {"version": "Unknown", "check": binary["check"]}
versions_by_host[host.config.address] = versions_at_host versions_by_host[host.config.address] = versions_at_host
return versions_by_host return versions_by_host
def get_remote_binaries_versions(hosting: Hosting) -> dict[str, str]: def get_remote_binaries_versions(hosting: Hosting) -> dict[str, str]:
versions_by_host = {} versions_by_host = {}
@ -83,26 +82,27 @@ def get_remote_binaries_versions(hosting: Hosting) -> dict[str, str]:
for host, binary_versions in versions_by_host.items(): for host, binary_versions in versions_by_host.items():
for name, binary in binary_versions.items(): for name, binary in binary_versions.items():
version = binary["version"] version = binary["version"]
if not cheak_versions.get(f'{name[:-2]}', None): if not cheak_versions.get(f"{name[:-2]}", None):
captured_version = cheak_versions.get(f'{name[:-2]}',{}).get(host, {}).get(captured_version) captured_version = cheak_versions.get(f"{name[:-2]}", {}).get(host, {}).get(captured_version)
cheak_versions[f'{name[:-2]}'] = {host: {version: name}} cheak_versions[f"{name[:-2]}"] = {host: {version: name}}
else: else:
captured_version = list(cheak_versions.get(f'{name[:-2]}',{}).get(previous_host).keys())[0] captured_version = list(cheak_versions.get(f"{name[:-2]}", {}).get(previous_host).keys())[0]
cheak_versions[f'{name[:-2]}'].update({host:{version:name}}) cheak_versions[f"{name[:-2]}"].update({host: {version: name}})
if captured_version and captured_version != version: if captured_version and captured_version != version:
exception.add(name[:-2]) exception.add(name[:-2])
versions[name] = {"version": version, "check": binary["check"]} versions[name] = {"version": version, "check": binary["check"]}
previous_host = host previous_host = host
if exception: if exception:
for i in exception: for i in exception:
for host in versions_by_host.keys(): for host in versions_by_host.keys():
for version, name in cheak_versions.get(i).get(host).items(): for version, name in cheak_versions.get(i).get(host).items():
exсeptions.append(f'Binary {name} has inconsistent version {version} on host {host}') exсeptions.append(f"Binary {name} has inconsistent version {version} on host {host}")
exсeptions.append('\n') exсeptions.append("\n")
return versions, exсeptions return versions, exсeptions
def _parse_version(version_output: str) -> str: def _parse_version(version_output: str) -> str:
version = re.search(r"version[:\s]*v?(.+)", version_output, re.IGNORECASE) version = re.search(r"version[:\s]*v?(.+)", version_output, re.IGNORECASE)
return version.group(1).strip() if version else version_output return version.group(1).strip() if version else version_output

View file

@ -9,6 +9,16 @@ from neo3.wallet import wallet as neo3_wallet
logger = logging.getLogger("frostfs.testlib.utils") logger = logging.getLogger("frostfs.testlib.utils")
def __fix_wallet_schema(wallet: dict) -> None:
# Temporary function to fix wallets that do not conform to the schema
# TODO: get rid of it once issue is solved
if "name" not in wallet:
wallet["name"] = None
for account in wallet["accounts"]:
if "extra" not in account:
account["extra"] = None
def init_wallet(wallet_path: str, wallet_password: str): def init_wallet(wallet_path: str, wallet_password: str):
""" """
Create new wallet and new account. Create new wallet and new account.
@ -33,29 +43,15 @@ def get_last_address_from_wallet(wallet_path: str, wallet_password: str):
Returns: Returns:
The address for the wallet. The address for the wallet.
""" """
with open(wallet_path) as wallet_file: wallet = load_wallet(wallet_path, wallet_password)
wallet = neo3_wallet.Wallet.from_json(json.load(wallet_file), password=wallet_password)
address = wallet.accounts[-1].address address = wallet.accounts[-1].address
logger.info(f"got address: {address}") logger.info(f"got address: {address}")
return address return address
def get_wallet_public_key(wallet_path: str, wallet_password: str, format: str = "hex") -> str: def get_wallet_public_key(wallet_path: str, wallet_password: str, format: str = "hex") -> str:
def __fix_wallet_schema(wallet: dict) -> None: wallet = load_wallet(wallet_path, wallet_password)
# Temporary function to fix wallets that do not conform to the schema public_key_hex = str(wallet.accounts[0].public_key)
# TODO: get rid of it once issue is solved
if "name" not in wallet:
wallet["name"] = None
for account in wallet["accounts"]:
if "extra" not in account:
account["extra"] = None
# Get public key from wallet file
with open(wallet_path, "r") as file:
wallet_content = json.load(file)
__fix_wallet_schema(wallet_content)
wallet_from_json = neo3_wallet.Wallet.from_json(wallet_content, password=wallet_password)
public_key_hex = str(wallet_from_json.accounts[0].public_key)
# Convert public key to specified format # Convert public key to specified format
if format == "hex": if format == "hex":
@ -69,7 +65,9 @@ def get_wallet_public_key(wallet_path: str, wallet_password: str, format: str =
raise ValueError(f"Invalid public key format: {format}") raise ValueError(f"Invalid public key format: {format}")
def load_wallet(path: str, passwd: str = "") -> neo3_wallet.Wallet: def load_wallet(wallet_path: str, wallet_password: str) -> neo3_wallet.Wallet:
with open(path, "r") as wallet_file: with open(wallet_path) as wallet_file:
wlt_data = wallet_file.read() wallet_content = json.load(wallet_file)
return neo3_wallet.Wallet.from_json(json.loads(wlt_data), password=passwd)
__fix_wallet_schema(wallet_content)
return neo3_wallet.Wallet.from_json(wallet_content, password=wallet_password)