abereziny/frostfs-testlib
1
0
Fork 0
forked from TrueCloudLab/frostfs-testlib
Code Issues Pull requests Projects Releases Packages Wiki Activity

[#191] Credentials work overhaul

Browse source 
Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
This commit is contained in:
Andrey Berezin 2024-03-11 19:23:10 +03:00
parent 09a7f66d1e
commit 25925c637b
31 changed files with 370 additions and 485 deletions
Download patch file Download diff file Expand all files Collapse all files

99
src/frostfs_testlib/steps/cli/object.py
View file

@ -9,9 +9,10 @@ from frostfs_testlib import reporter
from frostfs_testlib.cli import FrostfsCli
from frostfs_testlib.cli.neogo import NeoGo
from frostfs_testlib.resources.cli import CLI_DEFAULT_TIMEOUT, FROSTFS_CLI_EXEC, NEOGO_EXECUTABLE
from frostfs_testlib.resources.common import ASSETS_DIR, DEFAULT_WALLET_CONFIG
from frostfs_testlib.resources.common import ASSETS_DIR
from frostfs_testlib.shell import Shell
from frostfs_testlib.storage.cluster import Cluster, ClusterNode
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.utils import json_utils
from frostfs_testlib.utils.cli_utils import parse_cmd_table, parse_netmap_output
@ -20,7 +21,7 @@ logger = logging.getLogger("NeoLogger")
@reporter.step("Get object from random node")
def get_object_from_random_node(
wallet: str,
wallet: WalletInfo,
cid: str,
oid: str,
shell: Shell,
@ -28,7 +29,6 @@ def get_object_from_random_node(
bearer: Optional[str] = None,
write_object: Optional[str] = None,
xhdr: Optional[dict] = None,
wallet_config: Optional[str] = None,
no_progress: bool = True,
session: Optional[str] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
@ -44,7 +44,6 @@ def get_object_from_random_node(
cluster: cluster object
bearer (optional, str): path to Bearer Token file, appends to `--bearer` key
write_object (optional, str): path to downloaded file, appends to `--file` key
wallet_config(optional, str): path to the wallet config
no_progress(optional, bool): do not show progress bar
xhdr (optional, dict): Request X-Headers in form of Key=Value
session (optional, dict): path to a JSON-encoded container session token
@ -62,7 +61,6 @@ def get_object_from_random_node(
bearer,
write_object,
xhdr,
wallet_config,
no_progress,
session,
timeout,
@ -71,7 +69,7 @@ def get_object_from_random_node(
@reporter.step("Get object from {endpoint}")
def get_object(
wallet: str,
wallet: WalletInfo,
cid: str,
oid: str,
shell: Shell,
@ -79,7 +77,6 @@ def get_object(
bearer: Optional[str] = None,
write_object: Optional[str] = None,
xhdr: Optional[dict] = None,
wallet_config: Optional[str] = None,
no_progress: bool = True,
session: Optional[str] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
@ -88,14 +85,13 @@ def get_object(
GET from FrostFS.
Args:
wallet (str): wallet on whose behalf GET is done
wallet (WalletInfo): wallet on whose behalf GET is done
cid (str): ID of Container where we get the Object from
oid (str): Object ID
shell: executor for cli command
bearer: path to Bearer Token file, appends to `--bearer` key
write_object: path to downloaded file, appends to `--file` key
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
wallet_config(optional, str): path to the wallet config
no_progress(optional, bool): do not show progress bar
xhdr (optional, dict): Request X-Headers in form of Key=Value
session (optional, dict): path to a JSON-encoded container session token
@ -108,10 +104,9 @@ def get_object(
write_object = str(uuid.uuid4())
file_path = os.path.join(ASSETS_DIR, write_object)
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or DEFAULT_WALLET_CONFIG)
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
cli.object.get(
rpc_endpoint=endpoint,
wallet=wallet,
cid=cid,
oid=oid,
file=file_path,
@ -127,14 +122,13 @@ def get_object(
@reporter.step("Get Range Hash from {endpoint}")
def get_range_hash(
wallet: str,
wallet: WalletInfo,
cid: str,
oid: str,
range_cut: str,
shell: Shell,
endpoint: str,
bearer: Optional[str] = None,
wallet_config: Optional[str] = None,
xhdr: Optional[dict] = None,
session: Optional[str] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
@ -151,17 +145,15 @@ def get_range_hash(
range_cut: Range to take hash from in the form offset1:length1,...,
value to pass to the `--range` parameter
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
wallet_config: path to the wallet config
xhdr: Request X-Headers in form of Key=Values
session: Filepath to a JSON- or binary-encoded token of the object RANGEHASH session.
timeout: Timeout for the operation.
Returns:
None
"""
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or DEFAULT_WALLET_CONFIG)
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
result = cli.object.hash(
rpc_endpoint=endpoint,
wallet=wallet,
cid=cid,
oid=oid,
range=range_cut,
@ -177,7 +169,7 @@ def get_range_hash(
@reporter.step("Put object to random node")
def put_object_to_random_node(
wallet: str,
wallet: WalletInfo,
path: str,
cid: str,
shell: Shell,
@ -186,7 +178,6 @@ def put_object_to_random_node(
copies_number: Optional[int] = None,
attributes: Optional[dict] = None,
xhdr: Optional[dict] = None,
wallet_config: Optional[str] = None,
expire_at: Optional[int] = None,
no_progress: bool = True,
session: Optional[str] = None,
@ -205,7 +196,6 @@ def put_object_to_random_node(
copies_number: Number of copies of the object to store within the RPC call
attributes: User attributes in form of Key1=Value1,Key2=Value2
cluster: cluster under test
wallet_config: path to the wallet config
no_progress: do not show progress bar
expire_at: Last epoch in the life of the object
xhdr: Request X-Headers in form of Key=Value
@ -226,7 +216,6 @@ def put_object_to_random_node(
copies_number,
attributes,
xhdr,
wallet_config,
expire_at,
no_progress,
session,
@ -236,7 +225,7 @@ def put_object_to_random_node(
@reporter.step("Put object at {endpoint} in container {cid}")
def put_object(
wallet: str,
wallet: WalletInfo,
path: str,
cid: str,
shell: Shell,
@ -245,7 +234,6 @@ def put_object(
copies_number: Optional[int] = None,
attributes: Optional[dict] = None,
xhdr: Optional[dict] = None,
wallet_config: Optional[str] = None,
expire_at: Optional[int] = None,
no_progress: bool = True,
session: Optional[str] = None,
@ -263,7 +251,6 @@ def put_object(
copies_number: Number of copies of the object to store within the RPC call
attributes: User attributes in form of Key1=Value1,Key2=Value2
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
wallet_config: path to the wallet config
no_progress: do not show progress bar
expire_at: Last epoch in the life of the object
xhdr: Request X-Headers in form of Key=Value
@ -273,10 +260,9 @@ def put_object(
(str): ID of uploaded Object
"""
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or DEFAULT_WALLET_CONFIG)
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
result = cli.object.put(
rpc_endpoint=endpoint,
wallet=wallet,
file=path,
cid=cid,
attributes=attributes,
@ -297,13 +283,12 @@ def put_object(
@reporter.step("Delete object {cid}/{oid} from {endpoint}")
def delete_object(
wallet: str,
wallet: WalletInfo,
cid: str,
oid: str,
shell: Shell,
endpoint: str,
bearer: str = "",
wallet_config: Optional[str] = None,
xhdr: Optional[dict] = None,
session: Optional[str] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
@ -318,7 +303,6 @@ def delete_object(
shell: executor for cli command
bearer: path to Bearer Token file, appends to `--bearer` key
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
wallet_config: path to the wallet config
xhdr: Request X-Headers in form of Key=Value
session: path to a JSON-encoded container session token
timeout: Timeout for the operation.
@ -326,10 +310,9 @@ def delete_object(
(str): Tombstone ID
"""
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or DEFAULT_WALLET_CONFIG)
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
result = cli.object.delete(
rpc_endpoint=endpoint,
wallet=wallet,
cid=cid,
oid=oid,
bearer=bearer,
@ -345,13 +328,12 @@ def delete_object(
@reporter.step("Get Range")
def get_range(
wallet: str,
wallet: WalletInfo,
cid: str,
oid: str,
range_cut: str,
shell: Shell,
endpoint: str,
wallet_config: Optional[str] = None,
bearer: str = "",
xhdr: Optional[dict] = None,
session: Optional[str] = None,
@ -368,7 +350,6 @@ def get_range(
shell: executor for cli command
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
bearer: path to Bearer Token file, appends to `--bearer` key
wallet_config: path to the wallet config
xhdr: Request X-Headers in form of Key=Value
session: path to a JSON-encoded container session token
timeout: Timeout for the operation.
@ -377,10 +358,9 @@ def get_range(
"""
range_file_path = os.path.join(ASSETS_DIR, str(uuid.uuid4()))
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or DEFAULT_WALLET_CONFIG)
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
cli.object.range(
rpc_endpoint=endpoint,
wallet=wallet,
cid=cid,
oid=oid,
range=range_cut,
@ -398,7 +378,7 @@ def get_range(
@reporter.step("Lock Object")
def lock_object(
wallet: str,
wallet: WalletInfo,
cid: str,
oid: str,
shell: Shell,
@ -408,7 +388,6 @@ def lock_object(
address: Optional[str] = None,
bearer: Optional[str] = None,
session: Optional[str] = None,
wallet_config: Optional[str] = None,
ttl: Optional[int] = None,
xhdr: Optional[dict] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
@ -435,13 +414,12 @@ def lock_object(
Lock object ID
"""
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or DEFAULT_WALLET_CONFIG)
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
result = cli.object.lock(
rpc_endpoint=endpoint,
lifetime=lifetime,
expire_at=expire_at,
address=address,
wallet=wallet,
cid=cid,
oid=oid,
bearer=bearer,
@ -459,14 +437,13 @@ def lock_object(
@reporter.step("Search object")
def search_object(
wallet: str,
wallet: WalletInfo,
cid: str,
shell: Shell,
endpoint: str,
bearer: str = "",
filters: Optional[dict] = None,
expected_objects_list: Optional[list] = None,
wallet_config: Optional[str] = None,
xhdr: Optional[dict] = None,
session: Optional[str] = None,
phy: bool = False,
@ -484,7 +461,6 @@ def search_object(
endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
filters: key=value pairs to filter Objects
expected_objects_list: a list of ObjectIDs to compare found Objects with
wallet_config: path to the wallet config
xhdr: Request X-Headers in form of Key=Value
session: path to a JSON-encoded container session token
phy: Search physically stored objects.
@ -495,10 +471,9 @@ def search_object(
list of found ObjectIDs
"""
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or DEFAULT_WALLET_CONFIG)
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
result = cli.object.search(
rpc_endpoint=endpoint,
wallet=wallet,
cid=cid,
bearer=bearer,
xhdr=xhdr,
@ -513,23 +488,18 @@ def search_object(
if expected_objects_list:
if sorted(found_objects) == sorted(expected_objects_list):
logger.info(
f"Found objects list '{found_objects}' " f"is equal for expected list '{expected_objects_list}'"
)
logger.info(f"Found objects list '{found_objects}' " f"is equal for expected list '{expected_objects_list}'")
else:
logger.warning(
f"Found object list {found_objects} " f"is not equal to expected list '{expected_objects_list}'"
)
logger.warning(f"Found object list {found_objects} " f"is not equal to expected list '{expected_objects_list}'")
return found_objects
@reporter.step("Get netmap netinfo")
def get_netmap_netinfo(
wallet: str,
wallet: WalletInfo,
shell: Shell,
endpoint: str,
wallet_config: Optional[str] = None,
address: Optional[str] = None,
ttl: Optional[int] = None,
xhdr: Optional[dict] = None,
@ -539,7 +509,7 @@ def get_netmap_netinfo(
Get netmap netinfo output from node
Args:
wallet (str): wallet on whose behalf request is done
wallet (WalletInfo): wallet on whose behalf request is done
shell: executor for cli command
endpoint (optional, str): FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
address: Address of wallet account
@ -552,9 +522,8 @@ def get_netmap_netinfo(
(dict): dict of parsed command output
"""
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or DEFAULT_WALLET_CONFIG)
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
output = cli.netmap.netinfo(
wallet=wallet,
rpc_endpoint=endpoint,
address=address,
ttl=ttl,
@ -578,7 +547,7 @@ def get_netmap_netinfo(
@reporter.step("Head object")
def head_object(
wallet: str,
wallet: WalletInfo,
cid: str,
oid: str,
shell: Shell,
@ -588,7 +557,6 @@ def head_object(
json_output: bool = True,
is_raw: bool = False,
is_direct: bool = False,
wallet_config: Optional[str] = None,
session: Optional[str] = None,
timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
):
@ -596,7 +564,7 @@ def head_object(
HEAD an Object.
Args:
wallet (str): wallet on whose behalf HEAD is done
wallet (WalletInfo): wallet on whose behalf HEAD is done
cid (str): ID of Container where we get the Object from
oid (str): ObjectID to HEAD
shell: executor for cli command
@ -608,7 +576,6 @@ def head_object(
turns into `--raw` key
is_direct(optional, bool): send request directly to the node or not; this flag
turns into `--ttl 1` key
wallet_config(optional, str): path to the wallet config
xhdr (optional, dict): Request X-Headers in form of Key=Value
session (optional, dict): path to a JSON-encoded container session token
timeout: Timeout for the operation.
@ -619,10 +586,9 @@ def head_object(
(str): HEAD response as a plain text
"""
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or DEFAULT_WALLET_CONFIG)
cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path)
result = cli.object.head(
rpc_endpoint=endpoint,
wallet=wallet,
cid=cid,
oid=oid,
bearer=bearer,
@ -673,7 +639,7 @@ def head_object(
@reporter.step("Run neo-go dump-keys")
def neo_go_dump_keys(shell: Shell, wallet: str) -> dict:
def neo_go_dump_keys(shell: Shell, wallet: WalletInfo) -> dict:
"""
Run neo-go dump keys command
@ -761,9 +727,7 @@ def get_object_nodes(
parsing_output = parse_cmd_table(result_object_nodes.stdout, "|")
list_object_nodes = [
node
for node in parsing_output
if node["should_contain_object"] == "true" and node["actually_contains_object"] == "true"
node for node in parsing_output if node["should_contain_object"] == "true" and node["actually_contains_object"] == "true"
]
netmap_nodes_list = parse_netmap_output(
@ -780,10 +744,7 @@ def get_object_nodes(
]
result = [
cluster_node
for netmap_node in netmap_nodes
for cluster_node in cluster.cluster_nodes
if netmap_node.node == cluster_node.host_ip
cluster_node for netmap_node in netmap_nodes for cluster_node in cluster.cluster_nodes if netmap_node.node == cluster_node.host_ip
]
return result