forked from TrueCloudLab/frostfs-testlib
[#191] Credentials work overhaul
Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
This commit is contained in:
parent
09a7f66d1e
commit
25925c637b
31 changed files with 370 additions and 485 deletions
|
|
@ -5,12 +5,11 @@ from dataclasses import dataclass
|
|||
from time import sleep
|
||||
from typing import Optional, Union
|
||||
|
||||
import requests
|
||||
|
||||
from frostfs_testlib import reporter
|
||||
from frostfs_testlib.cli import FrostfsCli, GenericCli
|
||||
from frostfs_testlib.cli import FrostfsCli
|
||||
from frostfs_testlib.plugins import load_plugin
|
||||
from frostfs_testlib.resources.cli import CLI_DEFAULT_TIMEOUT, FROSTFS_CLI_EXEC
|
||||
from frostfs_testlib.resources.common import DEFAULT_WALLET_CONFIG
|
||||
from frostfs_testlib.s3.interfaces import BucketContainerResolver
|
||||
from frostfs_testlib.shell import Shell
|
||||
from frostfs_testlib.steps.cli.object import put_object, put_object_to_random_node
|
||||
from frostfs_testlib.storage.cluster import Cluster, ClusterNode
|
||||
|
|
@ -25,7 +24,7 @@ logger = logging.getLogger("NeoLogger")
|
|||
@dataclass
|
||||
class StorageContainerInfo:
|
||||
id: str
|
||||
wallet_file: WalletInfo
|
||||
wallet: WalletInfo
|
||||
|
||||
|
||||
class StorageContainer:
|
||||
|
|
@ -42,11 +41,8 @@ class StorageContainer:
|
|||
def get_id(self) -> str:
|
||||
return self.storage_container_info.id
|
||||
|
||||
def get_wallet_path(self) -> str:
|
||||
return self.storage_container_info.wallet_file.path
|
||||
|
||||
def get_wallet_config_path(self) -> str:
|
||||
return self.storage_container_info.wallet_file.config_path
|
||||
def get_wallet(self) -> str:
|
||||
return self.storage_container_info.wallet
|
||||
|
||||
@reporter.step("Generate new object and put in container")
|
||||
def generate_object(
|
||||
|
|
@ -61,37 +57,34 @@ class StorageContainer:
|
|||
file_hash = get_file_hash(file_path)
|
||||
|
||||
container_id = self.get_id()
|
||||
wallet_path = self.get_wallet_path()
|
||||
wallet_config = self.get_wallet_config_path()
|
||||
wallet = self.get_wallet()
|
||||
with reporter.step(f"Put object with size {size} to container {container_id}"):
|
||||
if endpoint:
|
||||
object_id = put_object(
|
||||
wallet=wallet_path,
|
||||
wallet=wallet,
|
||||
path=file_path,
|
||||
cid=container_id,
|
||||
expire_at=expire_at,
|
||||
shell=self.shell,
|
||||
endpoint=endpoint,
|
||||
bearer=bearer_token,
|
||||
wallet_config=wallet_config,
|
||||
)
|
||||
else:
|
||||
object_id = put_object_to_random_node(
|
||||
wallet=wallet_path,
|
||||
wallet=wallet,
|
||||
path=file_path,
|
||||
cid=container_id,
|
||||
expire_at=expire_at,
|
||||
shell=self.shell,
|
||||
cluster=self.cluster,
|
||||
bearer=bearer_token,
|
||||
wallet_config=wallet_config,
|
||||
)
|
||||
|
||||
storage_object = StorageObjectInfo(
|
||||
container_id,
|
||||
object_id,
|
||||
size=size,
|
||||
wallet_file_path=wallet_path,
|
||||
wallet=wallet,
|
||||
file_path=file_path,
|
||||
file_hash=file_hash,
|
||||
)
|
||||
|
|
@ -106,14 +99,13 @@ REP_2_FOR_3_NODES_PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 3 FROM * AS X"
|
|||
|
||||
@reporter.step("Create Container")
|
||||
def create_container(
|
||||
wallet: str,
|
||||
wallet: WalletInfo,
|
||||
shell: Shell,
|
||||
endpoint: str,
|
||||
rule: str = DEFAULT_PLACEMENT_RULE,
|
||||
basic_acl: str = "",
|
||||
attributes: Optional[dict] = None,
|
||||
session_token: str = "",
|
||||
session_wallet: str = "",
|
||||
name: Optional[str] = None,
|
||||
options: Optional[dict] = None,
|
||||
await_mode: bool = True,
|
||||
|
|
@ -124,7 +116,7 @@ def create_container(
|
|||
A wrapper for `frostfs-cli container create` call.
|
||||
|
||||
Args:
|
||||
wallet (str): a wallet on whose behalf a container is created
|
||||
wallet (WalletInfo): a wallet on whose behalf a container is created
|
||||
rule (optional, str): placement rule for container
|
||||
basic_acl (optional, str): an ACL for container, will be
|
||||
appended to `--basic-acl` key
|
||||
|
|
@ -146,10 +138,9 @@ def create_container(
|
|||
(str): CID of the created container
|
||||
"""
|
||||
|
||||
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, DEFAULT_WALLET_CONFIG)
|
||||
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
|
||||
result = cli.container.create(
|
||||
rpc_endpoint=endpoint,
|
||||
wallet=session_wallet if session_wallet else wallet,
|
||||
policy=rule,
|
||||
basic_acl=basic_acl,
|
||||
attributes=attributes,
|
||||
|
|
@ -170,9 +161,7 @@ def create_container(
|
|||
return cid
|
||||
|
||||
|
||||
def wait_for_container_creation(
|
||||
wallet: str, cid: str, shell: Shell, endpoint: str, attempts: int = 15, sleep_interval: int = 1
|
||||
):
|
||||
def wait_for_container_creation(wallet: WalletInfo, cid: str, shell: Shell, endpoint: str, attempts: int = 15, sleep_interval: int = 1):
|
||||
for _ in range(attempts):
|
||||
containers = list_containers(wallet, shell, endpoint)
|
||||
if cid in containers:
|
||||
|
|
@ -182,9 +171,7 @@ def wait_for_container_creation(
|
|||
raise RuntimeError(f"After {attempts * sleep_interval} seconds container {cid} hasn't been persisted; exiting")
|
||||
|
||||
|
||||
def wait_for_container_deletion(
|
||||
wallet: str, cid: str, shell: Shell, endpoint: str, attempts: int = 30, sleep_interval: int = 1
|
||||
):
|
||||
def wait_for_container_deletion(wallet: WalletInfo, cid: str, shell: Shell, endpoint: str, attempts: int = 30, sleep_interval: int = 1):
|
||||
for _ in range(attempts):
|
||||
try:
|
||||
get_container(wallet, cid, shell=shell, endpoint=endpoint)
|
||||
|
|
@ -198,29 +185,27 @@ def wait_for_container_deletion(
|
|||
|
||||
|
||||
@reporter.step("List Containers")
|
||||
def list_containers(
|
||||
wallet: str, shell: Shell, endpoint: str, timeout: Optional[str] = CLI_DEFAULT_TIMEOUT
|
||||
) -> list[str]:
|
||||
def list_containers(wallet: WalletInfo, shell: Shell, endpoint: str, timeout: Optional[str] = CLI_DEFAULT_TIMEOUT) -> list[str]:
|
||||
"""
|
||||
A wrapper for `frostfs-cli container list` call. It returns all the
|
||||
available containers for the given wallet.
|
||||
Args:
|
||||
wallet (str): a wallet on whose behalf we list the containers
|
||||
wallet (WalletInfo): a wallet on whose behalf we list the containers
|
||||
shell: executor for cli command
|
||||
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
|
||||
timeout: Timeout for the operation.
|
||||
Returns:
|
||||
(list): list of containers
|
||||
"""
|
||||
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, DEFAULT_WALLET_CONFIG)
|
||||
result = cli.container.list(rpc_endpoint=endpoint, wallet=wallet, timeout=timeout)
|
||||
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
|
||||
result = cli.container.list(rpc_endpoint=endpoint, timeout=timeout)
|
||||
logger.info(f"Containers: \n{result}")
|
||||
return result.stdout.split()
|
||||
|
||||
|
||||
@reporter.step("List Objects in container")
|
||||
def list_objects(
|
||||
wallet: str,
|
||||
wallet: WalletInfo,
|
||||
shell: Shell,
|
||||
container_id: str,
|
||||
endpoint: str,
|
||||
|
|
@ -230,7 +215,7 @@ def list_objects(
|
|||
A wrapper for `frostfs-cli container list-objects` call. It returns all the
|
||||
available objects in container.
|
||||
Args:
|
||||
wallet (str): a wallet on whose behalf we list the containers objects
|
||||
wallet (WalletInfo): a wallet on whose behalf we list the containers objects
|
||||
shell: executor for cli command
|
||||
container_id: cid of container
|
||||
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
|
||||
|
|
@ -238,15 +223,15 @@ def list_objects(
|
|||
Returns:
|
||||
(list): list of containers
|
||||
"""
|
||||
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, DEFAULT_WALLET_CONFIG)
|
||||
result = cli.container.list_objects(rpc_endpoint=endpoint, wallet=wallet, cid=container_id, timeout=timeout)
|
||||
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
|
||||
result = cli.container.list_objects(rpc_endpoint=endpoint, cid=container_id, timeout=timeout)
|
||||
logger.info(f"Container objects: \n{result}")
|
||||
return result.stdout.split()
|
||||
|
||||
|
||||
@reporter.step("Get Container")
|
||||
def get_container(
|
||||
wallet: str,
|
||||
wallet: WalletInfo,
|
||||
cid: str,
|
||||
shell: Shell,
|
||||
endpoint: str,
|
||||
|
|
@ -257,7 +242,7 @@ def get_container(
|
|||
A wrapper for `frostfs-cli container get` call. It extracts container's
|
||||
attributes and rearranges them into a more compact view.
|
||||
Args:
|
||||
wallet (str): path to a wallet on whose behalf we get the container
|
||||
wallet (WalletInfo): path to a wallet on whose behalf we get the container
|
||||
cid (str): ID of the container to get
|
||||
shell: executor for cli command
|
||||
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
|
||||
|
|
@ -267,8 +252,8 @@ def get_container(
|
|||
(dict, str): dict of container attributes
|
||||
"""
|
||||
|
||||
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, DEFAULT_WALLET_CONFIG)
|
||||
result = cli.container.get(rpc_endpoint=endpoint, wallet=wallet, cid=cid, json_mode=json_mode, timeout=timeout)
|
||||
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
|
||||
result = cli.container.get(rpc_endpoint=endpoint, cid=cid, json_mode=json_mode, timeout=timeout)
|
||||
|
||||
if not json_mode:
|
||||
return result.stdout
|
||||
|
|
@ -285,7 +270,7 @@ def get_container(
|
|||
@reporter.step("Delete Container")
|
||||
# TODO: make the error message about a non-found container more user-friendly
|
||||
def delete_container(
|
||||
wallet: str,
|
||||
wallet: WalletInfo,
|
||||
cid: str,
|
||||
shell: Shell,
|
||||
endpoint: str,
|
||||
|
|
@ -297,7 +282,7 @@ def delete_container(
|
|||
A wrapper for `frostfs-cli container delete` call.
|
||||
Args:
|
||||
await_mode: Block execution until container is removed.
|
||||
wallet (str): path to a wallet on whose behalf we delete the container
|
||||
wallet (WalletInfo): path to a wallet on whose behalf we delete the container
|
||||
cid (str): ID of the container to delete
|
||||
shell: executor for cli command
|
||||
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
|
||||
|
|
@ -306,9 +291,8 @@ def delete_container(
|
|||
This function doesn't return anything.
|
||||
"""
|
||||
|
||||
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, DEFAULT_WALLET_CONFIG)
|
||||
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
|
||||
cli.container.delete(
|
||||
wallet=wallet,
|
||||
cid=cid,
|
||||
rpc_endpoint=endpoint,
|
||||
force=force,
|
||||
|
|
@ -345,26 +329,22 @@ def _parse_cid(output: str) -> str:
|
|||
|
||||
@reporter.step("Search container by name")
|
||||
def search_container_by_name(name: str, node: ClusterNode):
|
||||
curl = GenericCli("curl", node.host)
|
||||
output = curl(f"-I http://127.0.0.1:8084/{name}")
|
||||
pattern = r"X-Container-Id: (\S+)"
|
||||
cid = re.findall(pattern, output.stdout)
|
||||
if cid:
|
||||
return cid[0]
|
||||
return None
|
||||
resolver_cls = load_plugin("frostfs.testlib.bucket_cid_resolver", node.host.config.product)
|
||||
resolver: BucketContainerResolver = resolver_cls()
|
||||
return resolver.resolve(node, name)
|
||||
|
||||
|
||||
@reporter.step("Search for nodes with a container")
|
||||
def search_nodes_with_container(
|
||||
wallet: str,
|
||||
wallet: WalletInfo,
|
||||
cid: str,
|
||||
shell: Shell,
|
||||
endpoint: str,
|
||||
cluster: Cluster,
|
||||
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
|
||||
) -> list[ClusterNode]:
|
||||
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, DEFAULT_WALLET_CONFIG)
|
||||
result = cli.container.search_node(rpc_endpoint=endpoint, wallet=wallet, cid=cid, timeout=timeout)
|
||||
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
|
||||
result = cli.container.search_node(rpc_endpoint=endpoint, cid=cid, timeout=timeout)
|
||||
|
||||
pattern = r"[0-9]+(?:\.[0-9]+){3}"
|
||||
nodes_ip = list(set(re.findall(pattern, result.stdout)))
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue