Move shared code to testlib

Signed-off-by: Andrey Berezin <a.berezin@yadro.com>

src/frostfs_testlib/steps/acl.py

@@ -0,0 +1,191 @@
+import base64
+import json
+import logging
+import os
+import uuid
+from time import sleep
+from typing import List, Optional, Union
+
+import base58
+
+from frostfs_testlib.cli import FrostfsCli
+from frostfs_testlib.reporter import get_reporter
+from frostfs_testlib.resources.cli import FROSTFS_CLI_EXEC
+from frostfs_testlib.resources.common import ASSETS_DIR, DEFAULT_WALLET_CONFIG
+from frostfs_testlib.shell import Shell
+from frostfs_testlib.storage.dataclasses.acl import (
+    EACL_LIFETIME,
+    FROSTFS_CONTRACT_CACHE_TIMEOUT,
+    EACLPubKey,
+    EACLRole,
+    EACLRule,
+)
+from frostfs_testlib.utils import wallet_utils
+
+reporter = get_reporter()
+logger = logging.getLogger("NeoLogger")
+
+
+@reporter.step_deco("Get extended ACL")
+def get_eacl(wallet_path: str, cid: str, shell: Shell, endpoint: str) -> Optional[str]:
+    cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, DEFAULT_WALLET_CONFIG)
+    try:
+        result = cli.container.get_eacl(wallet=wallet_path, rpc_endpoint=endpoint, cid=cid)
+    except RuntimeError as exc:
+        logger.info("Extended ACL table is not set for this container")
+        logger.info(f"Got exception while getting eacl: {exc}")
+        return None
+    if "extended ACL table is not set for this container" in result.stdout:
+        return None
+    return result.stdout
+
+
+@reporter.step_deco("Set extended ACL")
+def set_eacl(
+    wallet_path: str,
+    cid: str,
+    eacl_table_path: str,
+    shell: Shell,
+    endpoint: str,
+    session_token: Optional[str] = None,
+) -> None:
+    cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, DEFAULT_WALLET_CONFIG)
+    cli.container.set_eacl(
+        wallet=wallet_path,
+        rpc_endpoint=endpoint,
+        cid=cid,
+        table=eacl_table_path,
+        await_mode=True,
+        session=session_token,
+    )
+
+
+def _encode_cid_for_eacl(cid: str) -> str:
+    cid_base58 = base58.b58decode(cid)
+    return base64.b64encode(cid_base58).decode("utf-8")
+
+
+def create_eacl(cid: str, rules_list: List[EACLRule], shell: Shell) -> str:
+    table_file_path = os.path.join(os.getcwd(), ASSETS_DIR, f"eacl_table_{str(uuid.uuid4())}.json")
+    cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, DEFAULT_WALLET_CONFIG)
+    cli.acl.extended_create(cid=cid, out=table_file_path, rule=rules_list)
+
+    with open(table_file_path, "r") as file:
+        table_data = file.read()
+        logger.info(f"Generated eACL:\n{table_data}")
+
+    return table_file_path
+
+
+def form_bearertoken_file(
+    wif: str,
+    cid: str,
+    eacl_rule_list: List[Union[EACLRule, EACLPubKey]],
+    shell: Shell,
+    endpoint: str,
+    sign: Optional[bool] = True,
+) -> str:
+    """
+    This function fetches eACL for given <cid> on behalf of <wif>,
+    then extends it with filters taken from <eacl_rules>, signs
+    with bearer token and writes to file
+    """
+    enc_cid = _encode_cid_for_eacl(cid) if cid else None
+    file_path = os.path.join(os.getcwd(), ASSETS_DIR, str(uuid.uuid4()))
+
+    eacl = get_eacl(wif, cid, shell, endpoint)
+    json_eacl = dict()
+    if eacl:
+        eacl = eacl.replace("eACL: ", "").split("Signature")[0]
+        json_eacl = json.loads(eacl)
+    logger.info(json_eacl)
+    eacl_result = {
+        "body": {
+            "eaclTable": {"containerID": {"value": enc_cid} if cid else enc_cid, "records": []},
+            "lifetime": {"exp": EACL_LIFETIME, "nbf": "1", "iat": "0"},
+        }
+    }
+
+    assert eacl_rules, "Got empty eacl_records list"
+    for rule in eacl_rule_list:
+        op_data = {
+            "operation": rule.operation.value.upper(),
+            "action": rule.access.value.upper(),
+            "filters": rule.filters or [],
+            "targets": [],
+        }
+
+        if isinstance(rule.role, EACLRole):
+            op_data["targets"] = [{"role": rule.role.value.upper()}]
+        elif isinstance(rule.role, EACLPubKey):
+            op_data["targets"] = [{"keys": rule.role.keys}]
+
+        eacl_result["body"]["eaclTable"]["records"].append(op_data)
+
+    # Add records from current eACL
+    if "records" in json_eacl.keys():
+        for record in json_eacl["records"]:
+            eacl_result["body"]["eaclTable"]["records"].append(record)
+
+    with open(file_path, "w", encoding="utf-8") as eacl_file:
+        json.dump(eacl_result, eacl_file, ensure_ascii=False, indent=4)
+
+    logger.info(f"Got these extended ACL records: {eacl_result}")
+    if sign:
+        sign_bearer(
+            shell=shell,
+            wallet_path=wif,
+            eacl_rules_file_from=file_path,
+            eacl_rules_file_to=file_path,
+            json=True,
+        )
+    return file_path
+
+
+def eacl_rules(access: str, verbs: list, user: str) -> list[str]:
+    """
+    This function creates a list of eACL rules.
+    Args:
+        access (str): identifies if the following operation(s)
+                    is allowed or denied
+        verbs (list): a list of operations to set rules for
+        user (str): a group of users (user/others) or a wallet of
+                    a certain user for whom rules are set
+    Returns:
+        (list): a list of eACL rules
+    """
+    if user not in ("others", "user"):
+        pubkey = wallet_utils.get_wallet_public_key(user, wallet_password="")
+        user = f"pubkey:{pubkey}"
+
+    rules = []
+    for verb in verbs:
+        rule = f"{access} {verb} {user}"
+        rules.append(rule)
+    return rules
+
+
+def sign_bearer(
+    shell: Shell, wallet_path: str, eacl_rules_file_from: str, eacl_rules_file_to: str, json: bool
+) -> None:
+    frostfscli = FrostfsCli(
+        shell=shell, frostfs_cli_exec_path=FROSTFS_CLI_EXEC, config_file=DEFAULT_WALLET_CONFIG
+    )
+    frostfscli.util.sign_bearer_token(
+        wallet=wallet_path, from_file=eacl_rules_file_from, to_file=eacl_rules_file_to, json=json
+    )
+
+
+@reporter.step_deco("Wait for eACL cache expired")
+def wait_for_cache_expired():
+    sleep(FROSTFS_CONTRACT_CACHE_TIMEOUT)
+    return
+
+
+@reporter.step_deco("Return bearer token in base64 to caller")
+def bearer_token_base64_from_file(
+    bearer_path: str,
+) -> str:
+    with open(bearer_path, "rb") as file:
+        signed = file.read()
+    return base64.b64encode(signed).decode("utf-8")

src/frostfs_testlib/steps/cli/container.py

@@ -0,0 +1,359 @@
+import json
+import logging
+from dataclasses import dataclass
+from time import sleep
+from typing import Optional, Union
+
+from frostfs_testlib.cli import FrostfsCli
+from frostfs_testlib.reporter import get_reporter
+from frostfs_testlib.resources.cli import CLI_DEFAULT_TIMEOUT, FROSTFS_CLI_EXEC
+from frostfs_testlib.resources.common import DEFAULT_WALLET_CONFIG
+from frostfs_testlib.shell import Shell
+from frostfs_testlib.steps.cli.object import put_object, put_object_to_random_node
+from frostfs_testlib.storage.cluster import Cluster
+from frostfs_testlib.storage.dataclasses.storage_object_info import StorageObjectInfo
+from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
+from frostfs_testlib.utils import json_utils
+from frostfs_testlib.utils.file_utils import generate_file, get_file_hash
+
+reporter = get_reporter()
+logger = logging.getLogger("NeoLogger")
+
+
+@dataclass
+class StorageContainerInfo:
+    id: str
+    wallet_file: WalletInfo
+
+
+class StorageContainer:
+    def __init__(
+        self,
+        storage_container_info: StorageContainerInfo,
+        shell: Shell,
+        cluster: Cluster,
+    ) -> None:
+        self.shell = shell
+        self.storage_container_info = storage_container_info
+        self.cluster = cluster
+
+    def get_id(self) -> str:
+        return self.storage_container_info.id
+
+    def get_wallet_path(self) -> str:
+        return self.storage_container_info.wallet_file.path
+
+    def get_wallet_config_path(self) -> str:
+        return self.storage_container_info.wallet_file.config_path
+
+    @reporter.step_deco("Generate new object and put in container")
+    def generate_object(
+        self,
+        size: int,
+        expire_at: Optional[int] = None,
+        bearer_token: Optional[str] = None,
+        endpoint: Optional[str] = None,
+    ) -> StorageObjectInfo:
+        with reporter.step(f"Generate object with size {size}"):
+            file_path = generate_file(size)
+            file_hash = get_file_hash(file_path)
+
+        container_id = self.get_id()
+        wallet_path = self.get_wallet_path()
+        wallet_config = self.get_wallet_config_path()
+        with reporter.step(f"Put object with size {size} to container {container_id}"):
+            if endpoint:
+                object_id = put_object(
+                    wallet=wallet_path,
+                    path=file_path,
+                    cid=container_id,
+                    expire_at=expire_at,
+                    shell=self.shell,
+                    endpoint=endpoint,
+                    bearer=bearer_token,
+                    wallet_config=wallet_config,
+                )
+            else:
+                object_id = put_object_to_random_node(
+                    wallet=wallet_path,
+                    path=file_path,
+                    cid=container_id,
+                    expire_at=expire_at,
+                    shell=self.shell,
+                    cluster=self.cluster,
+                    bearer=bearer_token,
+                    wallet_config=wallet_config,
+                )
+
+            storage_object = StorageObjectInfo(
+                container_id,
+                object_id,
+                size=size,
+                wallet_file_path=wallet_path,
+                file_path=file_path,
+                file_hash=file_hash,
+            )
+
+        return storage_object
+
+
+DEFAULT_PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 4 FROM * AS X"
+SINGLE_PLACEMENT_RULE = "REP 1 IN X CBF 1 SELECT 4 FROM * AS X"
+REP_2_FOR_3_NODES_PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 3 FROM * AS X"
+
+
+@reporter.step_deco("Create Container")
+def create_container(
+    wallet: str,
+    shell: Shell,
+    endpoint: str,
+    rule: str = DEFAULT_PLACEMENT_RULE,
+    basic_acl: str = "",
+    attributes: Optional[dict] = None,
+    session_token: str = "",
+    session_wallet: str = "",
+    name: Optional[str] = None,
+    options: Optional[dict] = None,
+    await_mode: bool = True,
+    wait_for_creation: bool = True,
+    timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
+) -> str:
+    """
+    A wrapper for `frostfs-cli container create` call.
+
+    Args:
+        wallet (str): a wallet on whose behalf a container is created
+        rule (optional, str): placement rule for container
+        basic_acl (optional, str): an ACL for container, will be
+                            appended to `--basic-acl` key
+        attributes (optional, dict): container attributes , will be
+                            appended to `--attributes` key
+        session_token (optional, str): a path to session token file
+        session_wallet(optional, str): a path to the wallet which signed
+                            the session token; this parameter makes sense
+                            when paired with `session_token`
+        shell: executor for cli command
+        endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
+        options (optional, dict): any other options to pass to the call
+        name (optional, str): container name attribute
+        await_mode (bool): block execution until container is persisted
+        wait_for_creation (): Wait for container shows in container list
+        timeout: Timeout for the operation.
+
+    Returns:
+        (str): CID of the created container
+    """
+
+    cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, DEFAULT_WALLET_CONFIG)
+    result = cli.container.create(
+        rpc_endpoint=endpoint,
+        wallet=session_wallet if session_wallet else wallet,
+        policy=rule,
+        basic_acl=basic_acl,
+        attributes=attributes,
+        name=name,
+        session=session_token,
+        await_mode=await_mode,
+        timeout=timeout,
+        **options or {},
+    )
+
+    cid = _parse_cid(result.stdout)
+
+    logger.info("Container created; waiting until it is persisted in the sidechain")
+
+    if wait_for_creation:
+        wait_for_container_creation(wallet, cid, shell, endpoint)
+
+    return cid
+
+
+def wait_for_container_creation(
+    wallet: str, cid: str, shell: Shell, endpoint: str, attempts: int = 15, sleep_interval: int = 1
+):
+    for _ in range(attempts):
+        containers = list_containers(wallet, shell, endpoint)
+        if cid in containers:
+            return
+        logger.info(f"There is no {cid} in {containers} yet; sleep {sleep_interval} and continue")
+        sleep(sleep_interval)
+    raise RuntimeError(
+        f"After {attempts * sleep_interval} seconds container {cid} hasn't been persisted; exiting"
+    )
+
+
+def wait_for_container_deletion(
+    wallet: str, cid: str, shell: Shell, endpoint: str, attempts: int = 30, sleep_interval: int = 1
+):
+    for _ in range(attempts):
+        try:
+            get_container(wallet, cid, shell=shell, endpoint=endpoint)
+            sleep(sleep_interval)
+            continue
+        except Exception as err:
+            if "container not found" not in str(err):
+                raise AssertionError(f'Expected "container not found" in error, got\n{err}')
+            return
+    raise AssertionError(f"Expected container deleted during {attempts * sleep_interval} sec.")
+
+
+@reporter.step_deco("List Containers")
+def list_containers(
+    wallet: str, shell: Shell, endpoint: str, timeout: Optional[str] = CLI_DEFAULT_TIMEOUT
+) -> list[str]:
+    """
+    A wrapper for `frostfs-cli container list` call. It returns all the
+    available containers for the given wallet.
+    Args:
+        wallet (str): a wallet on whose behalf we list the containers
+        shell: executor for cli command
+        endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
+        timeout: Timeout for the operation.
+    Returns:
+        (list): list of containers
+    """
+    cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, DEFAULT_WALLET_CONFIG)
+    result = cli.container.list(rpc_endpoint=endpoint, wallet=wallet, timeout=timeout)
+    logger.info(f"Containers: \n{result}")
+    return result.stdout.split()
+
+
+@reporter.step_deco("List Objects in container")
+def list_objects(
+    wallet: str,
+    shell: Shell,
+    container_id: str,
+    endpoint: str,
+    timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
+) -> list[str]:
+    """
+    A wrapper for `frostfs-cli container list-objects` call. It returns all the
+    available objects in container.
+    Args:
+        wallet (str): a wallet on whose behalf we list the containers objects
+        shell: executor for cli command
+        container_id: cid of container
+        endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
+        timeout: Timeout for the operation.
+    Returns:
+        (list): list of containers
+    """
+    cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, DEFAULT_WALLET_CONFIG)
+    result = cli.container.list_objects(
+        rpc_endpoint=endpoint, wallet=wallet, cid=container_id, timeout=timeout
+    )
+    logger.info(f"Container objects: \n{result}")
+    return result.stdout.split()
+
+
+@reporter.step_deco("Get Container")
+def get_container(
+    wallet: str,
+    cid: str,
+    shell: Shell,
+    endpoint: str,
+    json_mode: bool = True,
+    timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
+) -> Union[dict, str]:
+    """
+    A wrapper for `frostfs-cli container get` call. It extracts container's
+    attributes and rearranges them into a more compact view.
+    Args:
+        wallet (str): path to a wallet on whose behalf we get the container
+        cid (str): ID of the container to get
+        shell: executor for cli command
+        endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
+        json_mode (bool): return container in JSON format
+        timeout: Timeout for the operation.
+    Returns:
+        (dict, str): dict of container attributes
+    """
+
+    cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, DEFAULT_WALLET_CONFIG)
+    result = cli.container.get(
+        rpc_endpoint=endpoint, wallet=wallet, cid=cid, json_mode=json_mode, timeout=timeout
+    )
+
+    if not json_mode:
+        return result.stdout
+
+    container_info = json.loads(result.stdout)
+    attributes = dict()
+    for attr in container_info["attributes"]:
+        attributes[attr["key"]] = attr["value"]
+    container_info["attributes"] = attributes
+    container_info["ownerID"] = json_utils.json_reencode(container_info["ownerID"]["value"])
+    return container_info
+
+
+@reporter.step_deco("Delete Container")
+# TODO: make the error message about a non-found container more user-friendly
+def delete_container(
+    wallet: str,
+    cid: str,
+    shell: Shell,
+    endpoint: str,
+    force: bool = False,
+    session_token: Optional[str] = None,
+    await_mode: bool = False,
+    timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
+) -> None:
+    """
+    A wrapper for `frostfs-cli container delete` call.
+    Args:
+        wallet (str): path to a wallet on whose behalf we delete the container
+        cid (str): ID of the container to delete
+        shell: executor for cli command
+        endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
+        force (bool): do not check whether container contains locks and remove immediately
+        session_token: a path to session token file
+        timeout: Timeout for the operation.
+    This function doesn't return anything.
+    """
+
+    cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, DEFAULT_WALLET_CONFIG)
+    cli.container.delete(
+        wallet=wallet,
+        cid=cid,
+        rpc_endpoint=endpoint,
+        force=force,
+        session=session_token,
+        await_mode=await_mode,
+        timeout=timeout,
+    )
+
+
+def _parse_cid(output: str) -> str:
+    """
+    Parses container ID from a given CLI output. The input string we expect:
+            container ID: 2tz86kVTDpJxWHrhw3h6PbKMwkLtBEwoqhHQCKTre1FN
+            awaiting...
+            container has been persisted on sidechain
+    We want to take 'container ID' value from the string.
+
+    Args:
+        output (str): CLI output to parse
+
+    Returns:
+        (str): extracted CID
+    """
+    try:
+        # taking first line from command's output
+        first_line = output.split("\n")[0]
+    except Exception:
+        first_line = ""
+        logger.error(f"Got empty output: {output}")
+    splitted = first_line.split(": ")
+    if len(splitted) != 2:
+        raise ValueError(f"no CID was parsed from command output: \t{first_line}")
+    return splitted[1]
+
+
+@reporter.step_deco("Search container by name")
+def search_container_by_name(wallet: str, name: str, shell: Shell, endpoint: str):
+    list_cids = list_containers(wallet, shell, endpoint)
+    for cid in list_cids:
+        cont_info = get_container(wallet, cid, shell, endpoint, True)
+        if cont_info.get("attributes", {}).get("Name", None) == name:
+            return cid
+    return None

src/frostfs_testlib/steps/cli/object.py

@@ -0,0 +1,727 @@
+import json
+import logging
+import os
+import re
+import uuid
+from typing import Any, Optional
+
+from frostfs_testlib.cli import FrostfsCli
+from frostfs_testlib.cli.neogo import NeoGo
+from frostfs_testlib.reporter import get_reporter
+from frostfs_testlib.resources.cli import CLI_DEFAULT_TIMEOUT, FROSTFS_CLI_EXEC, NEOGO_EXECUTABLE
+from frostfs_testlib.resources.common import ASSETS_DIR, DEFAULT_WALLET_CONFIG
+from frostfs_testlib.shell import Shell
+from frostfs_testlib.storage.cluster import Cluster
+from frostfs_testlib.utils import json_utils
+
+logger = logging.getLogger("NeoLogger")
+reporter = get_reporter()
+
+
+@reporter.step_deco("Get object from random node")
+def get_object_from_random_node(
+    wallet: str,
+    cid: str,
+    oid: str,
+    shell: Shell,
+    cluster: Cluster,
+    bearer: Optional[str] = None,
+    write_object: Optional[str] = None,
+    xhdr: Optional[dict] = None,
+    wallet_config: Optional[str] = None,
+    no_progress: bool = True,
+    session: Optional[str] = None,
+    timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
+) -> str:
+    """
+    GET from FrostFS random storage node
+
+    Args:
+        wallet: wallet on whose behalf GET is done
+        cid: ID of Container where we get the Object from
+        oid: Object ID
+        shell: executor for cli command
+        cluster: cluster object
+        bearer (optional, str): path to Bearer Token file, appends to `--bearer` key
+        write_object (optional, str): path to downloaded file, appends to `--file` key
+        wallet_config(optional, str): path to the wallet config
+        no_progress(optional, bool): do not show progress bar
+        xhdr (optional, dict): Request X-Headers in form of Key=Value
+        session (optional, dict): path to a JSON-encoded container session token
+        timeout: Timeout for the operation.
+    Returns:
+        (str): path to downloaded file
+    """
+    endpoint = cluster.get_random_storage_rpc_endpoint()
+    return get_object(
+        wallet,
+        cid,
+        oid,
+        shell,
+        endpoint,
+        bearer,
+        write_object,
+        xhdr,
+        wallet_config,
+        no_progress,
+        session,
+        timeout,
+    )
+
+
+@reporter.step_deco("Get object from {endpoint}")
+def get_object(
+    wallet: str,
+    cid: str,
+    oid: str,
+    shell: Shell,
+    endpoint: str,
+    bearer: Optional[str] = None,
+    write_object: Optional[str] = None,
+    xhdr: Optional[dict] = None,
+    wallet_config: Optional[str] = None,
+    no_progress: bool = True,
+    session: Optional[str] = None,
+    timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
+) -> str:
+    """
+    GET from FrostFS.
+
+    Args:
+        wallet (str): wallet on whose behalf GET is done
+        cid (str): ID of Container where we get the Object from
+        oid (str): Object ID
+        shell: executor for cli command
+        bearer: path to Bearer Token file, appends to `--bearer` key
+        write_object: path to downloaded file, appends to `--file` key
+        endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
+        wallet_config(optional, str): path to the wallet config
+        no_progress(optional, bool): do not show progress bar
+        xhdr (optional, dict): Request X-Headers in form of Key=Value
+        session (optional, dict): path to a JSON-encoded container session token
+        timeout: Timeout for the operation.
+    Returns:
+        (str): path to downloaded file
+    """
+
+    if not write_object:
+        write_object = str(uuid.uuid4())
+    file_path = os.path.join(ASSETS_DIR, write_object)
+
+    cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or DEFAULT_WALLET_CONFIG)
+    cli.object.get(
+        rpc_endpoint=endpoint,
+        wallet=wallet,
+        cid=cid,
+        oid=oid,
+        file=file_path,
+        bearer=bearer,
+        no_progress=no_progress,
+        xhdr=xhdr,
+        session=session,
+        timeout=timeout,
+    )
+
+    return file_path
+
+
+@reporter.step_deco("Get Range Hash from {endpoint}")
+def get_range_hash(
+    wallet: str,
+    cid: str,
+    oid: str,
+    range_cut: str,
+    shell: Shell,
+    endpoint: str,
+    bearer: Optional[str] = None,
+    wallet_config: Optional[str] = None,
+    xhdr: Optional[dict] = None,
+    session: Optional[str] = None,
+    timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
+):
+    """
+    GETRANGEHASH of given Object.
+
+    Args:
+        wallet: wallet on whose behalf GETRANGEHASH is done
+        cid: ID of Container where we get the Object from
+        oid: Object ID
+        shell: executor for cli command
+        bearer: path to Bearer Token file, appends to `--bearer` key
+        range_cut: Range to take hash from in the form offset1:length1,...,
+                        value to pass to the `--range` parameter
+        endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
+        wallet_config: path to the wallet config
+        xhdr: Request X-Headers in form of Key=Values
+        session: Filepath to a JSON- or binary-encoded token of the object RANGEHASH session.
+        timeout: Timeout for the operation.
+    Returns:
+        None
+    """
+    cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or DEFAULT_WALLET_CONFIG)
+    result = cli.object.hash(
+        rpc_endpoint=endpoint,
+        wallet=wallet,
+        cid=cid,
+        oid=oid,
+        range=range_cut,
+        bearer=bearer,
+        xhdr=xhdr,
+        session=session,
+        timeout=timeout,
+    )
+
+    # cutting off output about range offset and length
+    return result.stdout.split(":")[1].strip()
+
+
+@reporter.step_deco("Put object to random node")
+def put_object_to_random_node(
+    wallet: str,
+    path: str,
+    cid: str,
+    shell: Shell,
+    cluster: Cluster,
+    bearer: Optional[str] = None,
+    attributes: Optional[dict] = None,
+    xhdr: Optional[dict] = None,
+    wallet_config: Optional[str] = None,
+    expire_at: Optional[int] = None,
+    no_progress: bool = True,
+    session: Optional[str] = None,
+    timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
+):
+    """
+    PUT of given file to a random storage node.
+
+    Args:
+        wallet: wallet on whose behalf PUT is done
+        path: path to file to be PUT
+        cid: ID of Container where we get the Object from
+        shell: executor for cli command
+        cluster: cluster under test
+        bearer: path to Bearer Token file, appends to `--bearer` key
+        attributes: User attributes in form of Key1=Value1,Key2=Value2
+        cluster: cluster under test
+        wallet_config: path to the wallet config
+        no_progress: do not show progress bar
+        expire_at: Last epoch in the life of the object
+        xhdr: Request X-Headers in form of Key=Value
+        session: path to a JSON-encoded container session token
+        timeout: Timeout for the operation.
+    Returns:
+        ID of uploaded Object
+    """
+
+    endpoint = cluster.get_random_storage_rpc_endpoint()
+    return put_object(
+        wallet,
+        path,
+        cid,
+        shell,
+        endpoint,
+        bearer,
+        attributes,
+        xhdr,
+        wallet_config,
+        expire_at,
+        no_progress,
+        session,
+        timeout=timeout,
+    )
+
+
+@reporter.step_deco("Put object at {endpoint} in container {cid}")
+def put_object(
+    wallet: str,
+    path: str,
+    cid: str,
+    shell: Shell,
+    endpoint: str,
+    bearer: Optional[str] = None,
+    attributes: Optional[dict] = None,
+    xhdr: Optional[dict] = None,
+    wallet_config: Optional[str] = None,
+    expire_at: Optional[int] = None,
+    no_progress: bool = True,
+    session: Optional[str] = None,
+    timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
+):
+    """
+    PUT of given file.
+
+    Args:
+        wallet: wallet on whose behalf PUT is done
+        path: path to file to be PUT
+        cid: ID of Container where we get the Object from
+        shell: executor for cli command
+        bearer: path to Bearer Token file, appends to `--bearer` key
+        attributes: User attributes in form of Key1=Value1,Key2=Value2
+        endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
+        wallet_config: path to the wallet config
+        no_progress: do not show progress bar
+        expire_at: Last epoch in the life of the object
+        xhdr: Request X-Headers in form of Key=Value
+        session: path to a JSON-encoded container session token
+        timeout: Timeout for the operation.
+    Returns:
+        (str): ID of uploaded Object
+    """
+
+    cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or DEFAULT_WALLET_CONFIG)
+    result = cli.object.put(
+        rpc_endpoint=endpoint,
+        wallet=wallet,
+        file=path,
+        cid=cid,
+        attributes=attributes,
+        bearer=bearer,
+        expire_at=expire_at,
+        no_progress=no_progress,
+        xhdr=xhdr,
+        session=session,
+        timeout=timeout,
+    )
+
+    # Splitting CLI output to separate lines and taking the penultimate line
+    id_str = result.stdout.strip().split("\n")[-2]
+    oid = id_str.split(":")[1]
+    return oid.strip()
+
+
+@reporter.step_deco("Delete object {cid}/{oid} from {endpoint}")
+def delete_object(
+    wallet: str,
+    cid: str,
+    oid: str,
+    shell: Shell,
+    endpoint: str,
+    bearer: str = "",
+    wallet_config: Optional[str] = None,
+    xhdr: Optional[dict] = None,
+    session: Optional[str] = None,
+    timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
+):
+    """
+    DELETE an Object.
+
+    Args:
+        wallet: wallet on whose behalf DELETE is done
+        cid: ID of Container where we get the Object from
+        oid: ID of Object we are going to delete
+        shell: executor for cli command
+        bearer: path to Bearer Token file, appends to `--bearer` key
+        endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
+        wallet_config: path to the wallet config
+        xhdr: Request X-Headers in form of Key=Value
+        session: path to a JSON-encoded container session token
+        timeout: Timeout for the operation.
+    Returns:
+        (str): Tombstone ID
+    """
+
+    cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or DEFAULT_WALLET_CONFIG)
+    result = cli.object.delete(
+        rpc_endpoint=endpoint,
+        wallet=wallet,
+        cid=cid,
+        oid=oid,
+        bearer=bearer,
+        xhdr=xhdr,
+        session=session,
+        timeout=timeout,
+    )
+
+    id_str = result.stdout.split("\n")[1]
+    tombstone = id_str.split(":")[1]
+    return tombstone.strip()
+
+
+@reporter.step_deco("Get Range")
+def get_range(
+    wallet: str,
+    cid: str,
+    oid: str,
+    range_cut: str,
+    shell: Shell,
+    endpoint: str,
+    wallet_config: Optional[str] = None,
+    bearer: str = "",
+    xhdr: Optional[dict] = None,
+    session: Optional[str] = None,
+    timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
+):
+    """
+    GETRANGE an Object.
+
+    Args:
+        wallet: wallet on whose behalf GETRANGE is done
+        cid: ID of Container where we get the Object from
+        oid: ID of Object we are going to request
+        range_cut: range to take data from in the form offset:length
+        shell: executor for cli command
+        endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
+        bearer: path to Bearer Token file, appends to `--bearer` key
+        wallet_config: path to the wallet config
+        xhdr: Request X-Headers in form of Key=Value
+        session: path to a JSON-encoded container session token
+        timeout: Timeout for the operation.
+    Returns:
+        (str, bytes) - path to the file with range content and content of this file as bytes
+    """
+    range_file_path = os.path.join(ASSETS_DIR, str(uuid.uuid4()))
+
+    cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or DEFAULT_WALLET_CONFIG)
+    cli.object.range(
+        rpc_endpoint=endpoint,
+        wallet=wallet,
+        cid=cid,
+        oid=oid,
+        range=range_cut,
+        file=range_file_path,
+        bearer=bearer,
+        xhdr=xhdr,
+        session=session,
+        timeout=timeout,
+    )
+
+    with open(range_file_path, "rb") as file:
+        content = file.read()
+    return range_file_path, content
+
+
+@reporter.step_deco("Lock Object")
+def lock_object(
+    wallet: str,
+    cid: str,
+    oid: str,
+    shell: Shell,
+    endpoint: str,
+    lifetime: Optional[int] = None,
+    expire_at: Optional[int] = None,
+    address: Optional[str] = None,
+    bearer: Optional[str] = None,
+    session: Optional[str] = None,
+    wallet_config: Optional[str] = None,
+    ttl: Optional[int] = None,
+    xhdr: Optional[dict] = None,
+    timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
+) -> str:
+    """
+    Locks object in container.
+
+    Args:
+        address: Address of wallet account.
+        bearer: File with signed JSON or binary encoded bearer token.
+        cid: Container ID.
+        oid: Object ID.
+        lifetime: Lock lifetime.
+        expire_at: Lock expiration epoch.
+        shell: executor for cli command
+        endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
+        session: Path to a JSON-encoded container session token.
+        ttl: TTL value in request meta header (default 2).
+        wallet: WIF (NEP-2) string or path to the wallet or binary key.
+        xhdr: Dict with request X-Headers.
+        timeout: Timeout for the operation.
+
+    Returns:
+        Lock object ID
+    """
+
+    cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or DEFAULT_WALLET_CONFIG)
+    result = cli.object.lock(
+        rpc_endpoint=endpoint,
+        lifetime=lifetime,
+        expire_at=expire_at,
+        address=address,
+        wallet=wallet,
+        cid=cid,
+        oid=oid,
+        bearer=bearer,
+        xhdr=xhdr,
+        session=session,
+        ttl=ttl,
+        timeout=timeout,
+    )
+
+    # Splitting CLI output to separate lines and taking the penultimate line
+    id_str = result.stdout.strip().split("\n")[0]
+    oid = id_str.split(":")[1]
+    return oid.strip()
+
+
+@reporter.step_deco("Search object")
+def search_object(
+    wallet: str,
+    cid: str,
+    shell: Shell,
+    endpoint: str,
+    bearer: str = "",
+    filters: Optional[dict] = None,
+    expected_objects_list: Optional[list] = None,
+    wallet_config: Optional[str] = None,
+    xhdr: Optional[dict] = None,
+    session: Optional[str] = None,
+    phy: bool = False,
+    root: bool = False,
+    timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
+) -> list:
+    """
+    SEARCH an Object.
+
+    Args:
+        wallet: wallet on whose behalf SEARCH is done
+        cid: ID of Container where we get the Object from
+        shell: executor for cli command
+        bearer: path to Bearer Token file, appends to `--bearer` key
+        endpoint: FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
+        filters: key=value pairs to filter Objects
+        expected_objects_list: a list of ObjectIDs to compare found Objects with
+        wallet_config: path to the wallet config
+        xhdr: Request X-Headers in form of Key=Value
+        session: path to a JSON-encoded container session token
+        phy: Search physically stored objects.
+        root: Search for user objects.
+        timeout: Timeout for the operation.
+
+    Returns:
+        list of found ObjectIDs
+    """
+
+    cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or DEFAULT_WALLET_CONFIG)
+    result = cli.object.search(
+        rpc_endpoint=endpoint,
+        wallet=wallet,
+        cid=cid,
+        bearer=bearer,
+        xhdr=xhdr,
+        filters=[f"{filter_key} EQ {filter_val}" for filter_key, filter_val in filters.items()]
+        if filters
+        else None,
+        session=session,
+        phy=phy,
+        root=root,
+        timeout=timeout,
+    )
+
+    found_objects = re.findall(r"(\w{43,44})", result.stdout)
+
+    if expected_objects_list:
+        if sorted(found_objects) == sorted(expected_objects_list):
+            logger.info(
+                f"Found objects list '{found_objects}' "
+                f"is equal for expected list '{expected_objects_list}'"
+            )
+        else:
+            logger.warning(
+                f"Found object list {found_objects} "
+                f"is not equal to expected list '{expected_objects_list}'"
+            )
+
+    return found_objects
+
+
+@reporter.step_deco("Get netmap netinfo")
+def get_netmap_netinfo(
+    wallet: str,
+    shell: Shell,
+    endpoint: str,
+    wallet_config: Optional[str] = None,
+    address: Optional[str] = None,
+    ttl: Optional[int] = None,
+    xhdr: Optional[dict] = None,
+    timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
+) -> dict[str, Any]:
+    """
+    Get netmap netinfo output from node
+
+    Args:
+        wallet (str): wallet on whose behalf request is done
+        shell: executor for cli command
+        endpoint (optional, str): FrostFS endpoint to send request to, appends to `--rpc-endpoint` key
+        address: Address of wallet account
+        ttl: TTL value in request meta header (default 2)
+        wallet: Path to the wallet or binary key
+        xhdr: Request X-Headers in form of Key=Value
+        timeout: Timeout for the operation.
+
+    Returns:
+        (dict): dict of parsed command output
+    """
+
+    cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or DEFAULT_WALLET_CONFIG)
+    output = cli.netmap.netinfo(
+        wallet=wallet,
+        rpc_endpoint=endpoint,
+        address=address,
+        ttl=ttl,
+        xhdr=xhdr,
+        timeout=timeout,
+    )
+
+    settings = dict()
+
+    patterns = [
+        (re.compile("(.*): (\d+)"), int),
+        (re.compile("(.*): (false|true)"), bool),
+        (re.compile("(.*): (\d+\.\d+)"), float),
+    ]
+    for pattern, func in patterns:
+        for setting, value in re.findall(pattern, output.stdout):
+            settings[setting.lower().strip().replace(" ", "_")] = func(value)
+
+    return settings
+
+
+@reporter.step_deco("Head object")
+def head_object(
+    wallet: str,
+    cid: str,
+    oid: str,
+    shell: Shell,
+    endpoint: str,
+    bearer: str = "",
+    xhdr: Optional[dict] = None,
+    json_output: bool = True,
+    is_raw: bool = False,
+    is_direct: bool = False,
+    wallet_config: Optional[str] = None,
+    session: Optional[str] = None,
+    timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
+):
+    """
+    HEAD an Object.
+
+    Args:
+        wallet (str): wallet on whose behalf HEAD is done
+        cid (str): ID of Container where we get the Object from
+        oid (str): ObjectID to HEAD
+        shell: executor for cli command
+        bearer (optional, str): path to Bearer Token file, appends to `--bearer` key
+        endpoint(optional, str): FrostFS endpoint to send request to
+        json_output(optional, bool): return response in JSON format or not; this flag
+                                    turns into `--json` key
+        is_raw(optional, bool): send "raw" request or not; this flag
+                                    turns into `--raw` key
+        is_direct(optional, bool): send request directly to the node or not; this flag
+                                    turns into `--ttl 1` key
+        wallet_config(optional, str): path to the wallet config
+        xhdr (optional, dict): Request X-Headers in form of Key=Value
+        session (optional, dict): path to a JSON-encoded container session token
+        timeout: Timeout for the operation.
+    Returns:
+        depending on the `json_output` parameter value, the function returns
+        (dict): HEAD response in JSON format
+        or
+        (str): HEAD response as a plain text
+    """
+
+    cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet_config or DEFAULT_WALLET_CONFIG)
+    result = cli.object.head(
+        rpc_endpoint=endpoint,
+        wallet=wallet,
+        cid=cid,
+        oid=oid,
+        bearer=bearer,
+        json_mode=json_output,
+        raw=is_raw,
+        ttl=1 if is_direct else None,
+        xhdr=xhdr,
+        session=session,
+        timeout=timeout,
+    )
+
+    if not json_output:
+        return result
+
+    try:
+        decoded = json.loads(result.stdout)
+    except Exception as exc:
+        # If we failed to parse output as JSON, the cause might be
+        # the plain text string in the beginning of the output.
+        # Here we cut off first string and try to parse again.
+        logger.info(f"failed to parse output: {exc}")
+        logger.info("parsing output in another way")
+        fst_line_idx = result.stdout.find("\n")
+        decoded = json.loads(result.stdout[fst_line_idx:])
+
+    # If response is Complex Object header, it has `splitId` key
+    if "splitId" in decoded.keys():
+        logger.info("decoding split header")
+        return json_utils.decode_split_header(decoded)
+
+    # If response is Last or Linking Object header,
+    # it has `header` dictionary and non-null `split` dictionary
+    if "split" in decoded["header"].keys():
+        if decoded["header"]["split"]:
+            logger.info("decoding linking object")
+            return json_utils.decode_linking_object(decoded)
+
+    if decoded["header"]["objectType"] == "STORAGE_GROUP":
+        logger.info("decoding storage group")
+        return json_utils.decode_storage_group(decoded)
+
+    if decoded["header"]["objectType"] == "TOMBSTONE":
+        logger.info("decoding tombstone")
+        return json_utils.decode_tombstone(decoded)
+
+    logger.info("decoding simple header")
+    return json_utils.decode_simple_header(decoded)
+
+
+@reporter.step_deco("Run neo-go dump-keys")
+def neo_go_dump_keys(shell: Shell, wallet: str) -> dict:
+    """
+    Run neo-go dump keys command
+
+    Args:
+        shell: executor for cli command
+        wallet: wallet path to dump from
+    Returns:
+        dict Address:Wallet Key
+    """
+    neogo = NeoGo(shell, neo_go_exec_path=NEOGO_EXECUTABLE)
+    output = neogo.wallet.dump_keys(wallet=wallet).stdout
+    first_line = ""
+    try:
+        # taking first line from command's output contain wallet address
+        first_line = output.split("\n")[0]
+    except Exception:
+        logger.error(f"Got empty output (neo-go dump keys): {output}")
+    address_id = first_line.split()[0]
+    # taking second line from command's output contain wallet key
+    wallet_key = output.split("\n")[1]
+    return {address_id: wallet_key}
+
+
+@reporter.step_deco("Run neo-go query height")
+def neo_go_query_height(shell: Shell, endpoint: str) -> dict:
+    """
+    Run neo-go query height command
+
+    Args:
+        shell: executor for cli command
+        endpoint: endpoint to execute
+    Returns:
+        dict->
+            Latest block: {value}
+            Validated state: {value}
+
+    """
+    neogo = NeoGo(shell, neo_go_exec_path=NEOGO_EXECUTABLE)
+    output = neogo.query.height(rpc_endpoint=endpoint).stdout
+    first_line = ""
+    try:
+        # taking first line from command's output contain the latest block in blockchain
+        first_line = output.split("\n")[0]
+    except Exception:
+        logger.error(f"Got empty output (neo-go query height): {output}")
+    latest_block = first_line.split(":")
+    # taking second line from command's output contain wallet key
+    second_line = output.split("\n")[1]
+    validated_state = second_line.split(":")
+    return {
+        latest_block[0].replace(":", ""): int(latest_block[1]),
+        validated_state[0].replace(":", ""): int(validated_state[1]),
+    }

src/frostfs_testlib/steps/complex_object_actions.py

@@ -0,0 +1,210 @@
+#!/usr/bin/python3
+
+"""
+    This module contains functions which are used for Large Object assembling:
+    getting Last Object and split and getting Link Object. It is not enough to
+    simply perform a "raw" HEAD request.
+    Therefore, the reliable retrieval of the aforementioned objects must be done 
+    this way: send direct "raw" HEAD request to the every Storage Node and return 
+    the desired OID on first non-null response.
+"""
+
+import logging
+from typing import Optional, Tuple
+
+from frostfs_testlib.reporter import get_reporter
+from frostfs_testlib.resources.cli import CLI_DEFAULT_TIMEOUT
+from frostfs_testlib.resources.common import DEFAULT_WALLET_CONFIG
+from frostfs_testlib.shell import Shell
+from frostfs_testlib.steps.cli.object import head_object
+from frostfs_testlib.storage.cluster import Cluster, StorageNode
+from frostfs_testlib.storage.dataclasses.storage_object_info import StorageObjectInfo
+
+reporter = get_reporter()
+logger = logging.getLogger("NeoLogger")
+
+
+def get_storage_object_chunks(
+    storage_object: StorageObjectInfo,
+    shell: Shell,
+    cluster: Cluster,
+    timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
+) -> list[str]:
+    """
+    Get complex object split objects ids (no linker object)
+
+    Args:
+        storage_object: storage_object to get it's chunks
+        shell: client shell to do cmd requests
+        cluster: cluster object under test
+        timeout: Timeout for an operation.
+
+    Returns:
+    list of object ids of complex object chunks
+    """
+
+    with reporter.step(f"Get complex object chunks (f{storage_object.oid})"):
+        split_object_id = get_link_object(
+            storage_object.wallet_file_path,
+            storage_object.cid,
+            storage_object.oid,
+            shell,
+            cluster.services(StorageNode),
+            is_direct=False,
+            timeout=timeout,
+        )
+        head = head_object(
+            storage_object.wallet_file_path,
+            storage_object.cid,
+            split_object_id,
+            shell,
+            cluster.default_rpc_endpoint,
+            timeout=timeout,
+        )
+
+        chunks_object_ids = []
+        if "split" in head["header"] and "children" in head["header"]["split"]:
+            chunks_object_ids = head["header"]["split"]["children"]
+
+        return chunks_object_ids
+
+
+def get_complex_object_split_ranges(
+    storage_object: StorageObjectInfo,
+    shell: Shell,
+    cluster: Cluster,
+    timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
+) -> list[Tuple[int, int]]:
+
+    """
+    Get list of split ranges tuples (offset, length) of a complex object
+    For example if object size if 100 and max object size in system is 30
+    the returned list should be
+    [(0, 30), (30, 30), (60, 30), (90, 10)]
+
+    Args:
+        storage_object: storage_object to get it's chunks
+        shell: client shell to do cmd requests
+        cluster: cluster object under test
+        timeout: Timeout for an operation.
+
+    Returns:
+    list of object ids of complex object chunks
+    """
+
+    ranges: list = []
+    offset = 0
+    chunks_ids = get_storage_object_chunks(storage_object, shell, cluster)
+    for chunk_id in chunks_ids:
+        head = head_object(
+            storage_object.wallet_file_path,
+            storage_object.cid,
+            chunk_id,
+            shell,
+            cluster.default_rpc_endpoint,
+            timeout=timeout,
+        )
+
+        length = int(head["header"]["payloadLength"])
+        ranges.append((offset, length))
+
+        offset = offset + length
+
+    return ranges
+
+
+@reporter.step_deco("Get Link Object")
+def get_link_object(
+    wallet: str,
+    cid: str,
+    oid: str,
+    shell: Shell,
+    nodes: list[StorageNode],
+    bearer: str = "",
+    wallet_config: str = DEFAULT_WALLET_CONFIG,
+    is_direct: bool = True,
+    timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
+):
+    """
+    Args:
+        wallet (str): path to the wallet on whose behalf the Storage Nodes
+                        are requested
+        cid (str): Container ID which stores the Large Object
+        oid (str): Large Object ID
+        shell: executor for cli command
+        nodes: list of nodes to do search on
+        bearer (optional, str): path to Bearer token file
+        wallet_config (optional, str): path to the frostfs-cli config file
+        is_direct: send request directly to the node or not; this flag
+                   turns into `--ttl 1` key
+        timeout: Timeout for an operation.
+    Returns:
+        (str): Link Object ID
+        When no Link Object ID is found after all Storage Nodes polling,
+        the function throws an error.
+    """
+    for node in nodes:
+        endpoint = node.get_rpc_endpoint()
+        try:
+            resp = head_object(
+                wallet,
+                cid,
+                oid,
+                shell=shell,
+                endpoint=endpoint,
+                is_raw=True,
+                is_direct=is_direct,
+                bearer=bearer,
+                wallet_config=wallet_config,
+                timeout=timeout,
+            )
+            if resp["link"]:
+                return resp["link"]
+        except Exception:
+            logger.info(f"No Link Object found on {endpoint}; continue")
+    logger.error(f"No Link Object for {cid}/{oid} found among all Storage Nodes")
+    return None
+
+
+@reporter.step_deco("Get Last Object")
+def get_last_object(
+    wallet: str,
+    cid: str,
+    oid: str,
+    shell: Shell,
+    nodes: list[StorageNode],
+    timeout: Optional[str] = CLI_DEFAULT_TIMEOUT,
+) -> Optional[str]:
+    """
+    Args:
+        wallet (str): path to the wallet on whose behalf the Storage Nodes
+                        are requested
+        cid (str): Container ID which stores the Large Object
+        oid (str): Large Object ID
+        shell: executor for cli command
+        nodes: list of nodes to do search on
+        timeout: Timeout for an operation.
+    Returns:
+        (str): Last Object ID
+        When no Last Object ID is found after all Storage Nodes polling,
+        the function throws an error.
+    """
+    for node in nodes:
+        endpoint = node.get_rpc_endpoint()
+        try:
+            resp = head_object(
+                wallet,
+                cid,
+                oid,
+                shell=shell,
+                endpoint=endpoint,
+                is_raw=True,
+                is_direct=True,
+                timeout=timeout,
+            )
+            if resp["lastPart"]:
+                return resp["lastPart"]
+        except Exception:
+            logger.info(f"No Last Object found on {endpoint}; continue")
+    logger.error(f"No Last Object for {cid}/{oid} found among all Storage Nodes")
+    return None

src/frostfs_testlib/steps/epoch.py

@@ -0,0 +1,131 @@
+import logging
+from time import sleep
+from typing import Optional
+
+from frostfs_testlib.cli import FrostfsAdm, FrostfsCli, NeoGo
+from frostfs_testlib.reporter import get_reporter
+from frostfs_testlib.resources.cli import (
+    CLI_DEFAULT_TIMEOUT,
+    FROSTFS_ADM_CONFIG_PATH,
+    FROSTFS_ADM_EXEC,
+    FROSTFS_CLI_EXEC,
+    NEOGO_EXECUTABLE,
+)
+from frostfs_testlib.resources.common import MORPH_BLOCK_TIME
+from frostfs_testlib.shell import Shell
+from frostfs_testlib.steps.payment_neogo import get_contract_hash
+from frostfs_testlib.storage.cluster import Cluster, StorageNode
+from frostfs_testlib.storage.dataclasses.frostfs_services import InnerRing, MorphChain
+from frostfs_testlib.testing.test_control import wait_for_success
+from frostfs_testlib.utils import datetime_utils, wallet_utils
+
+reporter = get_reporter()
+logger = logging.getLogger("NeoLogger")
+
+
+@reporter.step_deco("Get epochs from nodes")
+def get_epochs_from_nodes(shell: Shell, cluster: Cluster) -> dict[str, int]:
+    """
+    Get current epochs on each node.
+
+    Args:
+        shell: shell to run commands on.
+        cluster: cluster under test.
+
+    Returns:
+        Dict of {node_ip: epoch}.
+    """
+    epochs_by_node = {}
+    for node in cluster.services(StorageNode):
+        epochs_by_node[node.host.config.address] = get_epoch(shell, cluster, node)
+    return epochs_by_node
+
+
+@reporter.step_deco("Ensure fresh epoch")
+def ensure_fresh_epoch(
+    shell: Shell, cluster: Cluster, alive_node: Optional[StorageNode] = None
+) -> int:
+    # ensure new fresh epoch to avoid epoch switch during test session
+    alive_node = alive_node if alive_node else cluster.services(StorageNode)[0]
+    current_epoch = get_epoch(shell, cluster, alive_node)
+    tick_epoch(shell, cluster, alive_node)
+    epoch = get_epoch(shell, cluster, alive_node)
+    assert epoch > current_epoch, "Epoch wasn't ticked"
+    return epoch
+
+
+@reporter.step_deco("Wait for epochs align in whole cluster")
+@wait_for_success(60, 5)
+def wait_for_epochs_align(shell: Shell, cluster: Cluster) -> None:
+    epochs = []
+    for node in cluster.services(StorageNode):
+        epochs.append(get_epoch(shell, cluster, node))
+    unique_epochs = list(set(epochs))
+    assert (
+        len(unique_epochs) == 1
+    ), f"unaligned epochs found,  {epochs}, count of unique epochs {len(unique_epochs)}"
+
+
+@reporter.step_deco("Get Epoch")
+def get_epoch(shell: Shell, cluster: Cluster, alive_node: Optional[StorageNode] = None):
+    alive_node = alive_node if alive_node else cluster.services(StorageNode)[0]
+    endpoint = alive_node.get_rpc_endpoint()
+    wallet_path = alive_node.get_wallet_path()
+    wallet_config = alive_node.get_wallet_config_path()
+
+    cli = FrostfsCli(shell=shell, frostfs_cli_exec_path=FROSTFS_CLI_EXEC, config_file=wallet_config)
+
+    epoch = cli.netmap.epoch(endpoint, wallet_path, timeout=CLI_DEFAULT_TIMEOUT)
+    return int(epoch.stdout)
+
+
+@reporter.step_deco("Tick Epoch")
+def tick_epoch(shell: Shell, cluster: Cluster, alive_node: Optional[StorageNode] = None):
+    """
+    Tick epoch using frostfs-adm or NeoGo if frostfs-adm is not available (DevEnv)
+    Args:
+        shell: local shell to make queries about current epoch. Remote shell will be used to tick new one
+        cluster: cluster instance under test
+        alive_node: node to send requests to (first node in cluster by default)
+    """
+
+    alive_node = alive_node if alive_node else cluster.services(StorageNode)[0]
+    remote_shell = alive_node.host.get_shell()
+
+    if FROSTFS_ADM_EXEC and FROSTFS_ADM_CONFIG_PATH:
+        # If frostfs-adm is available, then we tick epoch with it (to be consistent with UAT tests)
+        frostfs_adm = FrostfsAdm(
+            shell=remote_shell,
+            frostfs_adm_exec_path=FROSTFS_ADM_EXEC,
+            config_file=FROSTFS_ADM_CONFIG_PATH,
+        )
+        frostfs_adm.morph.force_new_epoch()
+        return
+
+    # Otherwise we tick epoch using transaction
+    cur_epoch = get_epoch(shell, cluster)
+
+    # Use first node by default
+    ir_node = cluster.services(InnerRing)[0]
+    # In case if no local_wallet_path is provided, we use wallet_path
+    ir_wallet_path = ir_node.get_wallet_path()
+    ir_wallet_pass = ir_node.get_wallet_password()
+    ir_address = wallet_utils.get_last_address_from_wallet(ir_wallet_path, ir_wallet_pass)
+
+    morph_chain = cluster.services(MorphChain)[0]
+    morph_endpoint = morph_chain.get_endpoint()
+
+    neogo = NeoGo(shell, neo_go_exec_path=NEOGO_EXECUTABLE)
+    neogo.contract.invokefunction(
+        wallet=ir_wallet_path,
+        wallet_password=ir_wallet_pass,
+        scripthash=get_contract_hash(morph_chain, "netmap.frostfs", shell=shell),
+        method="newEpoch",
+        arguments=f"int:{cur_epoch + 1}",
+        multisig_hash=f"{ir_address}:Global",
+        address=ir_address,
+        rpc_endpoint=morph_endpoint,
+        force=True,
+        gas=1,
+    )
+    sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME))

src/frostfs_testlib/steps/http/http_gate.py

@@ -0,0 +1,355 @@
+import logging
+import os
+import random
+import re
+import shutil
+import uuid
+import zipfile
+from typing import Optional
+from urllib.parse import quote_plus
+
+import requests
+
+from frostfs_testlib.reporter import get_reporter
+from frostfs_testlib.resources.common import SIMPLE_OBJECT_SIZE
+from frostfs_testlib.s3.aws_cli_client import LONG_TIMEOUT
+from frostfs_testlib.shell import Shell
+from frostfs_testlib.steps.cli.object import get_object
+from frostfs_testlib.steps.storage_policy import get_nodes_without_object
+from frostfs_testlib.storage.cluster import StorageNode
+from frostfs_testlib.utils.cli_utils import _cmd_run
+from frostfs_testlib.utils.file_utils import get_file_hash
+
+reporter = get_reporter()
+
+logger = logging.getLogger("NeoLogger")
+
+ASSETS_DIR = os.getenv("ASSETS_DIR", "TemporaryDir/")
+
+
+@reporter.step_deco("Get via HTTP Gate")
+def get_via_http_gate(cid: str, oid: str, endpoint: str, request_path: Optional[str] = None):
+    """
+    This function gets given object from HTTP gate
+    cid:          container id to get object from
+    oid:          object ID
+    endpoint:     http gate endpoint
+    request_path: (optional) http request, if ommited - use default [{endpoint}/get/{cid}/{oid}]
+    """
+
+    # if `request_path` parameter omitted, use default
+    if request_path is None:
+        request = f"{endpoint}/get/{cid}/{oid}"
+    else:
+        request = f"{endpoint}{request_path}"
+
+    resp = requests.get(request, stream=True)
+
+    if not resp.ok:
+        raise Exception(
+            f"""Failed to get object via HTTP gate:
+                request: {resp.request.path_url},
+                response: {resp.text},
+                status code: {resp.status_code} {resp.reason}"""
+        )
+
+    logger.info(f"Request: {request}")
+    _attach_allure_step(request, resp.status_code)
+
+    file_path = os.path.join(os.getcwd(), ASSETS_DIR, f"{cid}_{oid}")
+    with open(file_path, "wb") as file:
+        shutil.copyfileobj(resp.raw, file)
+    return file_path
+
+
+@reporter.step_deco("Get via Zip HTTP Gate")
+def get_via_zip_http_gate(cid: str, prefix: str, endpoint: str):
+    """
+    This function gets given object from HTTP gate
+    cid:      container id to get object from
+    prefix:   common prefix
+    endpoint: http gate endpoint
+    """
+    request = f"{endpoint}/zip/{cid}/{prefix}"
+    resp = requests.get(request, stream=True)
+
+    if not resp.ok:
+        raise Exception(
+            f"""Failed to get object via HTTP gate:
+                request: {resp.request.path_url},
+                response: {resp.text},
+                status code: {resp.status_code} {resp.reason}"""
+        )
+
+    logger.info(f"Request: {request}")
+    _attach_allure_step(request, resp.status_code)
+
+    file_path = os.path.join(os.getcwd(), ASSETS_DIR, f"{cid}_archive.zip")
+    with open(file_path, "wb") as file:
+        shutil.copyfileobj(resp.raw, file)
+
+    with zipfile.ZipFile(file_path, "r") as zip_ref:
+        zip_ref.extractall(ASSETS_DIR)
+
+    return os.path.join(os.getcwd(), ASSETS_DIR, prefix)
+
+
+@reporter.step_deco("Get via HTTP Gate by attribute")
+def get_via_http_gate_by_attribute(
+    cid: str, attribute: dict, endpoint: str, request_path: Optional[str] = None
+):
+    """
+    This function gets given object from HTTP gate
+    cid:          CID to get object from
+    attribute:    attribute {name: attribute} value pair
+    endpoint:     http gate endpoint
+    request_path: (optional) http request path, if ommited - use default [{endpoint}/get_by_attribute/{Key}/{Value}]
+    """
+    attr_name = list(attribute.keys())[0]
+    attr_value = quote_plus(str(attribute.get(attr_name)))
+    # if `request_path` parameter ommited, use default
+    if request_path is None:
+        request = f"{endpoint}/get_by_attribute/{cid}/{quote_plus(str(attr_name))}/{attr_value}"
+    else:
+        request = f"{endpoint}{request_path}"
+
+    resp = requests.get(request, stream=True)
+
+    if not resp.ok:
+        raise Exception(
+            f"""Failed to get object via HTTP gate:
+                request: {resp.request.path_url},
+                response: {resp.text},
+                status code: {resp.status_code} {resp.reason}"""
+        )
+
+    logger.info(f"Request: {request}")
+    _attach_allure_step(request, resp.status_code)
+
+    file_path = os.path.join(os.getcwd(), ASSETS_DIR, f"{cid}_{str(uuid.uuid4())}")
+    with open(file_path, "wb") as file:
+        shutil.copyfileobj(resp.raw, file)
+    return file_path
+
+
+@reporter.step_deco("Upload via HTTP Gate")
+def upload_via_http_gate(cid: str, path: str, endpoint: str, headers: Optional[dict] = None) -> str:
+    """
+    This function upload given object through HTTP gate
+    cid:      CID to get object from
+    path:     File path to upload
+    endpoint: http gate endpoint
+    headers:  Object header
+    """
+    request = f"{endpoint}/upload/{cid}"
+    files = {"upload_file": open(path, "rb")}
+    body = {"filename": path}
+    resp = requests.post(request, files=files, data=body, headers=headers)
+
+    if not resp.ok:
+        raise Exception(
+            f"""Failed to get object via HTTP gate:
+                request: {resp.request.path_url},
+                response: {resp.text},
+                status code: {resp.status_code} {resp.reason}"""
+        )
+
+    logger.info(f"Request: {request}")
+    _attach_allure_step(request, resp.json(), req_type="POST")
+
+    assert resp.json().get("object_id"), f"OID found in response {resp}"
+
+    return resp.json().get("object_id")
+
+
+@reporter.step_deco("Check is the passed object large")
+def is_object_large(filepath: str) -> bool:
+    """
+    This function check passed file size and return True if file_size > SIMPLE_OBJECT_SIZE
+    filepath: File path to check
+    """
+    file_size = os.path.getsize(filepath)
+    logger.info(f"Size= {file_size}")
+    if file_size > int(SIMPLE_OBJECT_SIZE):
+        return True
+    else:
+        return False
+
+
+@reporter.step_deco("Upload via HTTP Gate using Curl")
+def upload_via_http_gate_curl(
+    cid: str,
+    filepath: str,
+    endpoint: str,
+    headers: Optional[list] = None,
+    error_pattern: Optional[str] = None,
+) -> str:
+    """
+    This function upload given object through HTTP gate using curl utility.
+    cid: CID to get object from
+    filepath: File path to upload
+    headers: Object header
+    endpoint: http gate endpoint
+    error_pattern: [optional] expected error message from the command
+    """
+    request = f"{endpoint}/upload/{cid}"
+    attributes = ""
+    if headers:
+        # parse attributes
+        attributes = " ".join(headers)
+
+    large_object = is_object_large(filepath)
+    if large_object:
+        # pre-clean
+        _cmd_run("rm pipe -f")
+        files = f"file=@pipe;filename={os.path.basename(filepath)}"
+        cmd = f"mkfifo pipe;cat {filepath} > pipe & curl --no-buffer -F '{files}' {attributes} {request}"
+        output = _cmd_run(cmd, LONG_TIMEOUT)
+        # clean up pipe
+        _cmd_run("rm pipe")
+    else:
+        files = f"file=@{filepath};filename={os.path.basename(filepath)}"
+        cmd = f"curl -F '{files}' {attributes} {request}"
+        output = _cmd_run(cmd)
+
+    if error_pattern:
+        match = error_pattern.casefold() in str(output).casefold()
+        assert match, f"Expected {output} to match {error_pattern}"
+        return ""
+
+    oid_re = re.search(r'"object_id": "(.*)"', output)
+    if not oid_re:
+        raise AssertionError(f'Could not find "object_id" in {output}')
+    return oid_re.group(1)
+
+
+@reporter.step_deco("Get via HTTP Gate using Curl")
+def get_via_http_curl(cid: str, oid: str, endpoint: str) -> str:
+    """
+    This function gets given object from HTTP gate using curl utility.
+    cid:      CID to get object from
+    oid:      object OID
+    endpoint: http gate endpoint
+    """
+    request = f"{endpoint}/get/{cid}/{oid}"
+    file_path = os.path.join(os.getcwd(), ASSETS_DIR, f"{cid}_{oid}_{str(uuid.uuid4())}")
+
+    cmd = f"curl {request} > {file_path}"
+    _cmd_run(cmd)
+
+    return file_path
+
+
+def _attach_allure_step(request: str, status_code: int, req_type="GET"):
+    command_attachment = f"REQUEST: '{request}'\n" f"RESPONSE:\n {status_code}\n"
+    with reporter.step(f"{req_type} Request"):
+        reporter.attach(command_attachment, f"{req_type} Request")
+
+
+@reporter.step_deco("Try to get object and expect error")
+def try_to_get_object_and_expect_error(
+    cid: str, oid: str, error_pattern: str, endpoint: str
+) -> None:
+    try:
+        get_via_http_gate(cid=cid, oid=oid, endpoint=endpoint)
+        raise AssertionError(f"Expected error on getting object with cid: {cid}")
+    except Exception as err:
+        match = error_pattern.casefold() in str(err).casefold()
+        assert match, f"Expected {err} to match {error_pattern}"
+
+
+@reporter.step_deco("Verify object can be get using HTTP header attribute")
+def get_object_by_attr_and_verify_hashes(
+    oid: str, file_name: str, cid: str, attrs: dict, endpoint: str
+) -> None:
+    got_file_path_http = get_via_http_gate(cid=cid, oid=oid, endpoint=endpoint)
+    got_file_path_http_attr = get_via_http_gate_by_attribute(
+        cid=cid, attribute=attrs, endpoint=endpoint
+    )
+    assert_hashes_are_equal(file_name, got_file_path_http, got_file_path_http_attr)
+
+
+def verify_object_hash(
+    oid: str,
+    file_name: str,
+    wallet: str,
+    cid: str,
+    shell: Shell,
+    nodes: list[StorageNode],
+    endpoint: str,
+    object_getter=None,
+) -> None:
+
+    nodes_list = get_nodes_without_object(
+        wallet=wallet,
+        cid=cid,
+        oid=oid,
+        shell=shell,
+        nodes=nodes,
+    )
+    # for some reason we can face with case when nodes_list is empty due to object resides in all nodes
+    if nodes_list:
+        random_node = random.choice(nodes_list)
+    else:
+        random_node = random.choice(nodes)
+
+    object_getter = object_getter or get_via_http_gate
+
+    got_file_path = get_object(
+        wallet=wallet,
+        cid=cid,
+        oid=oid,
+        shell=shell,
+        endpoint=random_node.get_rpc_endpoint(),
+    )
+    got_file_path_http = object_getter(cid=cid, oid=oid, endpoint=endpoint)
+
+    assert_hashes_are_equal(file_name, got_file_path, got_file_path_http)
+
+
+def assert_hashes_are_equal(orig_file_name: str, got_file_1: str, got_file_2: str) -> None:
+    msg = "Expected hashes are equal for files {f1} and {f2}"
+    got_file_hash_http = get_file_hash(got_file_1)
+    assert get_file_hash(got_file_2) == got_file_hash_http, msg.format(f1=got_file_2, f2=got_file_1)
+    assert get_file_hash(orig_file_name) == got_file_hash_http, msg.format(
+        f1=orig_file_name, f2=got_file_1
+    )
+
+
+def attr_into_header(attrs: dict) -> dict:
+    return {f"X-Attribute-{_key}": _value for _key, _value in attrs.items()}
+
+
+@reporter.step_deco(
+    "Convert each attribute (Key=Value) to the following format: -H 'X-Attribute-Key: Value'"
+)
+def attr_into_str_header_curl(attrs: dict) -> list:
+    headers = []
+    for k, v in attrs.items():
+        headers.append(f"-H 'X-Attribute-{k}: {v}'")
+    logger.info(f"[List of Attrs for curl:] {headers}")
+    return headers
+
+
+@reporter.step_deco(
+    "Try to get object via http (pass http_request and optional attributes) and expect error"
+)
+def try_to_get_object_via_passed_request_and_expect_error(
+    cid: str,
+    oid: str,
+    error_pattern: str,
+    endpoint: str,
+    http_request_path: str,
+    attrs: Optional[dict] = None,
+) -> None:
+    try:
+        if attrs is None:
+            get_via_http_gate(cid=cid, oid=oid, endpoint=endpoint, request_path=http_request_path)
+        else:
+            get_via_http_gate_by_attribute(
+                cid=cid, attribute=attrs, endpoint=endpoint, request_path=http_request_path
+            )
+        raise AssertionError(f"Expected error on getting object with cid: {cid}")
+    except Exception as err:
+        match = error_pattern.casefold() in str(err).casefold()
+        assert match, f"Expected {err} to match {error_pattern}"

src/frostfs_testlib/steps/node_management.py

@@ -0,0 +1,351 @@
+import logging
+import random
+import re
+import time
+from dataclasses import dataclass
+from time import sleep
+from typing import Optional
+
+from frostfs_testlib.cli import FrostfsAdm, FrostfsCli
+from frostfs_testlib.reporter import get_reporter
+from frostfs_testlib.resources.cli import (
+    FROSTFS_ADM_CONFIG_PATH,
+    FROSTFS_ADM_EXEC,
+    FROSTFS_CLI_EXEC,
+)
+from frostfs_testlib.resources.common import MORPH_BLOCK_TIME
+from frostfs_testlib.shell import Shell
+from frostfs_testlib.steps.epoch import tick_epoch
+from frostfs_testlib.storage.cluster import Cluster, StorageNode
+from frostfs_testlib.storage.dataclasses.frostfs_services import S3Gate
+from frostfs_testlib.utils import datetime_utils
+
+reporter = get_reporter()
+logger = logging.getLogger("NeoLogger")
+
+
+@dataclass
+class HealthStatus:
+    network_status: Optional[str] = None
+    health_status: Optional[str] = None
+
+    @staticmethod
+    def from_stdout(output: str) -> "HealthStatus":
+        network, health = None, None
+        for line in output.split("\n"):
+            if "Network status" in line:
+                network = line.split(":")[-1].strip()
+            if "Health status" in line:
+                health = line.split(":")[-1].strip()
+        return HealthStatus(network, health)
+
+
+@reporter.step_deco("Stop random storage nodes")
+def stop_random_storage_nodes(number: int, nodes: list[StorageNode]) -> list[StorageNode]:
+    """
+    Shuts down the given number of randomly selected storage nodes.
+    Args:
+       number: the number of storage nodes to stop
+       nodes: the list of storage nodes to stop
+    Returns:
+        the list of nodes that were stopped
+    """
+    nodes_to_stop = random.sample(nodes, number)
+    for node in nodes_to_stop:
+        node.stop_service()
+    return nodes_to_stop
+
+
+@reporter.step_deco("Start storage node")
+def start_storage_nodes(nodes: list[StorageNode]) -> None:
+    """
+    The function starts specified storage nodes.
+    Args:
+       nodes: the list of nodes to start
+    """
+    for node in nodes:
+        node.start_service()
+
+
+@reporter.step_deco("Stop storage node")
+def stop_storage_nodes(nodes: list[StorageNode]) -> None:
+    """
+    The function starts specified storage nodes.
+    Args:
+       nodes: the list of nodes to start
+    """
+    for node in nodes:
+        node.stop_service()
+
+
+@reporter.step_deco("Get Locode from random storage node")
+def get_locode_from_random_node(cluster: Cluster) -> str:
+    node = random.choice(cluster.services(StorageNode))
+    locode = node.get_un_locode()
+    logger.info(f"Chosen '{locode}' locode from node {node}")
+    return locode
+
+
+@reporter.step_deco("Healthcheck for storage node {node}")
+def storage_node_healthcheck(node: StorageNode) -> HealthStatus:
+    """
+    The function returns storage node's health status.
+    Args:
+        node: storage node for which health status should be retrieved.
+    Returns:
+        health status as HealthStatus object.
+    """
+    command = "control healthcheck"
+    output = _run_control_command_with_retries(node, command)
+    return HealthStatus.from_stdout(output)
+
+
+@reporter.step_deco("Set status for {node}")
+def storage_node_set_status(node: StorageNode, status: str, retries: int = 0) -> None:
+    """
+    The function sets particular status for given node.
+    Args:
+        node: node for which status should be set.
+        status: online or offline.
+        retries (optional, int): number of retry attempts if it didn't work from the first time
+    """
+    command = f"control set-status --status {status}"
+    _run_control_command_with_retries(node, command, retries)
+
+
+@reporter.step_deco("Get netmap snapshot")
+def get_netmap_snapshot(node: StorageNode, shell: Shell) -> str:
+    """
+    The function returns string representation of netmap snapshot.
+    Args:
+        node: node from which netmap snapshot should be requested.
+    Returns:
+        string representation of netmap
+    """
+
+    storage_wallet_config = node.get_wallet_config_path()
+    storage_wallet_path = node.get_wallet_path()
+
+    cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, config_file=storage_wallet_config)
+    return cli.netmap.snapshot(
+        rpc_endpoint=node.get_rpc_endpoint(),
+        wallet=storage_wallet_path,
+    ).stdout
+
+
+@reporter.step_deco("Get shard list for {node}")
+def node_shard_list(node: StorageNode) -> list[str]:
+    """
+    The function returns list of shards for specified storage node.
+    Args:
+        node: node for which shards should be returned.
+    Returns:
+        list of shards.
+    """
+    command = "control shards list"
+    output = _run_control_command_with_retries(node, command)
+    return re.findall(r"Shard (.*):", output)
+
+
+@reporter.step_deco("Shard set for {node}")
+def node_shard_set_mode(node: StorageNode, shard: str, mode: str) -> str:
+    """
+    The function sets mode for specified shard.
+    Args:
+        node: node on which shard mode should be set.
+    """
+    command = f"control shards set-mode --id {shard} --mode {mode}"
+    return _run_control_command_with_retries(node, command)
+
+
+@reporter.step_deco("Drop object from {node}")
+def drop_object(node: StorageNode, cid: str, oid: str) -> str:
+    """
+    The function drops object from specified node.
+    Args:
+        node_id str: node from which object should be dropped.
+    """
+    command = f"control drop-objects  -o {cid}/{oid}"
+    return _run_control_command_with_retries(node, command)
+
+
+@reporter.step_deco("Delete data from host for node {node}")
+def delete_node_data(node: StorageNode) -> None:
+    node.stop_service()
+    node.host.delete_storage_node_data(node.name)
+    time.sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME))
+
+
+@reporter.step_deco("Exclude node {node_to_exclude} from network map")
+def exclude_node_from_network_map(
+    node_to_exclude: StorageNode,
+    alive_node: StorageNode,
+    shell: Shell,
+    cluster: Cluster,
+) -> None:
+    node_netmap_key = node_to_exclude.get_wallet_public_key()
+
+    storage_node_set_status(node_to_exclude, status="offline")
+
+    time.sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME))
+    tick_epoch(shell, cluster)
+
+    snapshot = get_netmap_snapshot(node=alive_node, shell=shell)
+    assert (
+        node_netmap_key not in snapshot
+    ), f"Expected node with key {node_netmap_key} to be absent in network map"
+
+
+@reporter.step_deco("Include node {node_to_include} into network map")
+def include_node_to_network_map(
+    node_to_include: StorageNode,
+    alive_node: StorageNode,
+    shell: Shell,
+    cluster: Cluster,
+) -> None:
+    storage_node_set_status(node_to_include, status="online")
+
+    # Per suggestion of @fyrchik we need to wait for 2 blocks after we set status and after tick epoch.
+    # First sleep can be omitted after https://github.com/TrueCloudLab/frostfs-node/issues/60 complete.
+
+    time.sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME) * 2)
+    tick_epoch(shell, cluster)
+    time.sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME) * 2)
+
+    check_node_in_map(node_to_include, shell, alive_node)
+
+
+@reporter.step_deco("Check node {node} in network map")
+def check_node_in_map(
+    node: StorageNode, shell: Shell, alive_node: Optional[StorageNode] = None
+) -> None:
+    alive_node = alive_node or node
+
+    node_netmap_key = node.get_wallet_public_key()
+    logger.info(f"Node ({node.label}) netmap key: {node_netmap_key}")
+
+    snapshot = get_netmap_snapshot(alive_node, shell)
+    assert (
+        node_netmap_key in snapshot
+    ), f"Expected node with key {node_netmap_key} to be in network map"
+
+
+@reporter.step_deco("Check node {node} NOT in network map")
+def check_node_not_in_map(
+    node: StorageNode, shell: Shell, alive_node: Optional[StorageNode] = None
+) -> None:
+    alive_node = alive_node or node
+
+    node_netmap_key = node.get_wallet_public_key()
+    logger.info(f"Node ({node.label}) netmap key: {node_netmap_key}")
+
+    snapshot = get_netmap_snapshot(alive_node, shell)
+    assert (
+        node_netmap_key not in snapshot
+    ), f"Expected node with key {node_netmap_key} to be NOT in network map"
+
+
+@reporter.step_deco("Wait for node {node} is ready")
+def wait_for_node_to_be_ready(node: StorageNode) -> None:
+    timeout, attempts = 30, 6
+    for _ in range(attempts):
+        try:
+            health_check = storage_node_healthcheck(node)
+            if health_check.health_status == "READY":
+                return
+        except Exception as err:
+            logger.warning(f"Node {node} is not ready:\n{err}")
+        sleep(timeout)
+    raise AssertionError(
+        f"Node {node} hasn't gone to the READY state after {timeout * attempts} seconds"
+    )
+
+
+@reporter.step_deco("Remove nodes from network map trough cli-adm morph command")
+def remove_nodes_from_map_morph(
+    shell: Shell,
+    cluster: Cluster,
+    remove_nodes: list[StorageNode],
+    alive_node: Optional[StorageNode] = None,
+):
+    """
+    Move node to the Offline state in the candidates list and tick an epoch to update the netmap
+    using frostfs-adm
+    Args:
+        shell: local shell to make queries about current epoch. Remote shell will be used to tick new one
+        cluster: cluster instance under test
+        alive_node: node to send requests to (first node in cluster by default)
+        remove_nodes: list of nodes which would be removed from map
+    """
+
+    alive_node = alive_node if alive_node else remove_nodes[0]
+    remote_shell = alive_node.host.get_shell()
+
+    node_netmap_keys = list(map(StorageNode.get_wallet_public_key, remove_nodes))
+    logger.info(f"Nodes netmap keys are: {' '.join(node_netmap_keys)}")
+
+    if FROSTFS_ADM_EXEC and FROSTFS_ADM_CONFIG_PATH:
+        # If frostfs-adm is available, then we tick epoch with it (to be consistent with UAT tests)
+        frostfsadm = FrostfsAdm(
+            shell=remote_shell,
+            frostfs_adm_exec_path=FROSTFS_ADM_EXEC,
+            config_file=FROSTFS_ADM_CONFIG_PATH,
+        )
+        frostfsadm.morph.remove_nodes(node_netmap_keys)
+
+
+def _run_control_command_with_retries(node: StorageNode, command: str, retries: int = 0) -> str:
+    for attempt in range(1 + retries):  # original attempt + specified retries
+        try:
+            return _run_control_command(node, command)
+        except AssertionError as err:
+            if attempt < retries:
+                logger.warning(f"Command {command} failed with error {err} and will be retried")
+                continue
+            raise AssertionError(f"Command {command} failed with error {err}") from err
+
+
+def _run_control_command(node: StorageNode, command: str) -> None:
+    host = node.host
+
+    service_config = host.get_service_config(node.name)
+    wallet_path = service_config.attributes["wallet_path"]
+    wallet_password = service_config.attributes["wallet_password"]
+    control_endpoint = service_config.attributes["control_endpoint"]
+
+    shell = host.get_shell()
+    wallet_config_path = f"/tmp/{node.name}-config.yaml"
+    wallet_config = f'password: "{wallet_password}"'
+    shell.exec(f"echo '{wallet_config}' > {wallet_config_path}")
+
+    cli_config = host.get_cli_config("frostfs-cli")
+
+    # TODO: implement cli.control
+    # cli = FrostfsCli(shell, cli_config.exec_path, wallet_config_path)
+    result = shell.exec(
+        f"{cli_config.exec_path} {command} --endpoint {control_endpoint} "
+        f"--wallet {wallet_path} --config {wallet_config_path}"
+    )
+    return result.stdout
+
+
+@reporter.step_deco("Start services s3gate ")
+def start_s3gates(cluster: Cluster) -> None:
+    """
+    The function starts specified storage nodes.
+    Args:
+       cluster: cluster instance under test
+    """
+    for gate in cluster.services(S3Gate):
+        gate.start_service()
+
+
+@reporter.step_deco("Stop services s3gate ")
+def stop_s3gates(cluster: Cluster) -> None:
+    """
+    The function starts specified storage nodes.
+    Args:
+       cluster: cluster instance under test
+    """
+    for gate in cluster.services(S3Gate):
+        gate.stop_service()

src/frostfs_testlib/steps/payment_neogo.py

@@ -0,0 +1,217 @@
+import base64
+import json
+import logging
+import re
+import time
+from typing import Optional
+
+from neo3.wallet import utils as neo3_utils
+from neo3.wallet import wallet as neo3_wallet
+
+from frostfs_testlib.cli import NeoGo
+from frostfs_testlib.reporter import get_reporter
+from frostfs_testlib.resources.cli import NEOGO_EXECUTABLE
+from frostfs_testlib.resources.common import FROSTFS_CONTRACT, GAS_HASH, MORPH_BLOCK_TIME
+from frostfs_testlib.shell import Shell
+from frostfs_testlib.storage.dataclasses.frostfs_services import MainChain, MorphChain
+from frostfs_testlib.utils import converting_utils, datetime_utils, wallet_utils
+
+reporter = get_reporter()
+logger = logging.getLogger("NeoLogger")
+
+EMPTY_PASSWORD = ""
+TX_PERSIST_TIMEOUT = 15  # seconds
+ASSET_POWER_MAINCHAIN = 10**8
+ASSET_POWER_SIDECHAIN = 10**12
+
+
+def get_nns_contract_hash(morph_chain: MorphChain) -> str:
+    return morph_chain.rpc_client.get_contract_state(1)["hash"]
+
+
+def get_contract_hash(morph_chain: MorphChain, resolve_name: str, shell: Shell) -> str:
+    nns_contract_hash = get_nns_contract_hash(morph_chain)
+    neogo = NeoGo(shell=shell, neo_go_exec_path=NEOGO_EXECUTABLE)
+    out = neogo.contract.testinvokefunction(
+        scripthash=nns_contract_hash,
+        method="resolve",
+        arguments=f"string:{resolve_name} int:16",
+        rpc_endpoint=morph_chain.get_endpoint(),
+    )
+    stack_data = json.loads(out.stdout.replace("\n", ""))["stack"][0]["value"]
+    return bytes.decode(base64.b64decode(stack_data[0]["value"]))
+
+
+@reporter.step_deco("Withdraw Mainnet Gas")
+def withdraw_mainnet_gas(shell: Shell, main_chain: MainChain, wlt: str, amount: int):
+    address = wallet_utils.get_last_address_from_wallet(wlt, EMPTY_PASSWORD)
+    scripthash = neo3_utils.address_to_script_hash(address)
+
+    neogo = NeoGo(shell=shell, neo_go_exec_path=NEOGO_EXECUTABLE)
+    out = neogo.contract.invokefunction(
+        wallet=wlt,
+        address=address,
+        rpc_endpoint=main_chain.get_endpoint(),
+        scripthash=FROSTFS_CONTRACT,
+        method="withdraw",
+        arguments=f"{scripthash} int:{amount}",
+        multisig_hash=f"{scripthash}:Global",
+        wallet_password="",
+    )
+
+    m = re.match(r"^Sent invocation transaction (\w{64})$", out.stdout)
+    if m is None:
+        raise Exception("Can not get Tx.")
+    tx = m.group(1)
+    if not transaction_accepted(main_chain, tx):
+        raise AssertionError(f"TX {tx} hasn't been processed")
+
+
+def transaction_accepted(main_chain: MainChain, tx_id: str):
+    """
+    This function returns True in case of accepted TX.
+    Args:
+        tx_id(str): transaction ID
+    Returns:
+        (bool)
+    """
+
+    try:
+        for _ in range(0, TX_PERSIST_TIMEOUT):
+            time.sleep(1)
+            resp = main_chain.rpc_client.get_transaction_height(tx_id)
+            if resp is not None:
+                logger.info(f"TX is accepted in block: {resp}")
+                return True, resp
+    except Exception as out:
+        logger.info(f"request failed with error: {out}")
+        raise out
+    return False
+
+
+@reporter.step_deco("Get FrostFS Balance")
+def get_balance(shell: Shell, morph_chain: MorphChain, wallet_path: str, wallet_password: str = ""):
+    """
+    This function returns FrostFS balance for given wallet.
+    """
+    with open(wallet_path) as wallet_file:
+        wallet = neo3_wallet.Wallet.from_json(json.load(wallet_file), password=wallet_password)
+    acc = wallet.accounts[-1]
+    payload = [{"type": "Hash160", "value": str(acc.script_hash)}]
+    try:
+        resp = morph_chain.rpc_client.invoke_function(
+            get_contract_hash(morph_chain, "balance.frostfs", shell=shell), "balanceOf", payload
+        )
+        logger.info(f"Got response \n{resp}")
+        value = int(resp["stack"][0]["value"])
+        return value / ASSET_POWER_SIDECHAIN
+    except Exception as out:
+        logger.error(f"failed to get wallet balance: {out}")
+        raise out
+
+
+@reporter.step_deco("Transfer Gas")
+def transfer_gas(
+    shell: Shell,
+    amount: int,
+    main_chain: MainChain,
+    wallet_from_path: Optional[str] = None,
+    wallet_from_password: Optional[str] = None,
+    address_from: Optional[str] = None,
+    address_to: Optional[str] = None,
+    wallet_to_path: Optional[str] = None,
+    wallet_to_password: Optional[str] = None,
+):
+    """
+    This function transfer GAS in main chain from mainnet wallet to
+    the provided wallet. If the wallet contains more than one address,
+    the assets will be transferred to the last one.
+    Args:
+        shell: Shell instance.
+        wallet_from_password: Password of the wallet; it is required to decode the wallet
+            and extract its addresses.
+        wallet_from_path: Path to chain node wallet.
+        address_from: The address of the wallet to transfer assets from.
+        wallet_to_path: The path to the wallet to transfer assets to.
+        wallet_to_password: The password to the wallet to transfer assets to.
+        address_to: The address of the wallet to transfer assets to.
+        amount: Amount of gas to transfer.
+    """
+    wallet_from_path = wallet_from_path or main_chain.get_wallet_path()
+    wallet_from_password = (
+        wallet_from_password
+        if wallet_from_password is not None
+        else main_chain.get_wallet_password()
+    )
+    address_from = address_from or wallet_utils.get_last_address_from_wallet(
+        wallet_from_path, wallet_from_password
+    )
+    address_to = address_to or wallet_utils.get_last_address_from_wallet(
+        wallet_to_path, wallet_to_password
+    )
+
+    neogo = NeoGo(shell, neo_go_exec_path=NEOGO_EXECUTABLE)
+    out = neogo.nep17.transfer(
+        rpc_endpoint=main_chain.get_endpoint(),
+        wallet=wallet_from_path,
+        wallet_password=wallet_from_password,
+        amount=amount,
+        from_address=address_from,
+        to_address=address_to,
+        token="GAS",
+        force=True,
+    )
+    txid = out.stdout.strip().split("\n")[-1]
+    if len(txid) != 64:
+        raise Exception("Got no TXID after run the command")
+    if not transaction_accepted(main_chain, txid):
+        raise AssertionError(f"TX {txid} hasn't been processed")
+    time.sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME))
+
+
+@reporter.step_deco("FrostFS Deposit")
+def deposit_gas(
+    shell: Shell,
+    main_chain: MainChain,
+    amount: int,
+    wallet_from_path: str,
+    wallet_from_password: str,
+):
+    """
+    Transferring GAS from given wallet to FrostFS contract address.
+    """
+    # get FrostFS contract address
+    deposit_addr = converting_utils.contract_hash_to_address(FROSTFS_CONTRACT)
+    logger.info(f"FrostFS contract address: {deposit_addr}")
+    address_from = wallet_utils.get_last_address_from_wallet(
+        wallet_path=wallet_from_path, wallet_password=wallet_from_password
+    )
+    transfer_gas(
+        shell=shell,
+        main_chain=main_chain,
+        amount=amount,
+        wallet_from_path=wallet_from_path,
+        wallet_from_password=wallet_from_password,
+        address_to=deposit_addr,
+        address_from=address_from,
+    )
+
+
+@reporter.step_deco("Get Mainnet Balance")
+def get_mainnet_balance(main_chain: MainChain, address: str):
+    resp = main_chain.rpc_client.get_nep17_balances(address=address)
+    logger.info(f"Got getnep17balances response: {resp}")
+    for balance in resp["balance"]:
+        if balance["assethash"] == GAS_HASH:
+            return float(balance["amount"]) / ASSET_POWER_MAINCHAIN
+    return float(0)
+
+
+@reporter.step_deco("Get Sidechain Balance")
+def get_sidechain_balance(morph_chain: MorphChain, address: str):
+    resp = morph_chain.rpc_client.get_nep17_balances(address=address)
+    logger.info(f"Got getnep17balances response: {resp}")
+    for balance in resp["balance"]:
+        if balance["assethash"] == GAS_HASH:
+            return float(balance["amount"]) / ASSET_POWER_SIDECHAIN
+    return float(0)

src/frostfs_testlib/steps/s3/s3_helper.py

@@ -0,0 +1,247 @@
+import json
+import logging
+import os
+import re
+import uuid
+from datetime import datetime, timedelta
+from typing import Optional
+
+from dateutil.parser import parse
+
+from frostfs_testlib.reporter import get_reporter
+from frostfs_testlib.resources.cli import FROSTFS_AUTHMATE_EXEC
+from frostfs_testlib.resources.common import CREDENTIALS_CREATE_TIMEOUT
+from frostfs_testlib.s3 import S3ClientWrapper, VersioningStatus
+from frostfs_testlib.storage.cluster import Cluster
+from frostfs_testlib.storage.dataclasses.frostfs_services import S3Gate
+from frostfs_testlib.utils.cli_utils import _run_with_passwd
+
+reporter = get_reporter()
+logger = logging.getLogger("NeoLogger")
+
+
+@reporter.step_deco("Expected all objects are presented in the bucket")
+def check_objects_in_bucket(
+    s3_client: S3ClientWrapper,
+    bucket: str,
+    expected_objects: list,
+    unexpected_objects: Optional[list] = None,
+) -> None:
+    unexpected_objects = unexpected_objects or []
+    bucket_objects = s3_client.list_objects(bucket)
+    assert len(bucket_objects) == len(
+        expected_objects
+    ), f"Expected {len(expected_objects)} objects in the bucket"
+    for bucket_object in expected_objects:
+        assert (
+            bucket_object in bucket_objects
+        ), f"Expected object {bucket_object} in objects list {bucket_objects}"
+
+    for bucket_object in unexpected_objects:
+        assert (
+            bucket_object not in bucket_objects
+        ), f"Expected object {bucket_object} not in objects list {bucket_objects}"
+
+
+@reporter.step_deco("Try to get object and got error")
+def try_to_get_objects_and_expect_error(
+    s3_client: S3ClientWrapper, bucket: str, object_keys: list
+) -> None:
+    for obj in object_keys:
+        try:
+            s3_client.get_object(bucket, obj)
+            raise AssertionError(f"Object {obj} found in bucket {bucket}")
+        except Exception as err:
+            assert "The specified key does not exist" in str(
+                err
+            ), f"Expected error in exception {err}"
+
+
+@reporter.step_deco("Set versioning status to '{status}' for bucket '{bucket}'")
+def set_bucket_versioning(s3_client: S3ClientWrapper, bucket: str, status: VersioningStatus):
+    s3_client.get_bucket_versioning_status(bucket)
+    s3_client.put_bucket_versioning(bucket, status=status)
+    bucket_status = s3_client.get_bucket_versioning_status(bucket)
+    assert bucket_status == status.value, f"Expected {bucket_status} status. Got {status.value}"
+
+
+def object_key_from_file_path(full_path: str) -> str:
+    return os.path.basename(full_path)
+
+
+def assert_tags(
+    actual_tags: list, expected_tags: Optional[list] = None, unexpected_tags: Optional[list] = None
+) -> None:
+    expected_tags = (
+        [{"Key": key, "Value": value} for key, value in expected_tags] if expected_tags else []
+    )
+    unexpected_tags = (
+        [{"Key": key, "Value": value} for key, value in unexpected_tags] if unexpected_tags else []
+    )
+    if expected_tags == []:
+        assert not actual_tags, f"Expected there is no tags, got {actual_tags}"
+    assert len(expected_tags) == len(actual_tags)
+    for tag in expected_tags:
+        assert tag in actual_tags, f"Tag {tag} must be in {actual_tags}"
+    for tag in unexpected_tags:
+        assert tag not in actual_tags, f"Tag {tag} should not be in {actual_tags}"
+
+
+@reporter.step_deco("Expected all tags are presented in object")
+def check_tags_by_object(
+    s3_client: S3ClientWrapper,
+    bucket: str,
+    key: str,
+    expected_tags: list,
+    unexpected_tags: Optional[list] = None,
+) -> None:
+    actual_tags = s3_client.get_object_tagging(bucket, key)
+    assert_tags(
+        expected_tags=expected_tags, unexpected_tags=unexpected_tags, actual_tags=actual_tags
+    )
+
+
+@reporter.step_deco("Expected all tags are presented in bucket")
+def check_tags_by_bucket(
+    s3_client: S3ClientWrapper,
+    bucket: str,
+    expected_tags: list,
+    unexpected_tags: Optional[list] = None,
+) -> None:
+    actual_tags = s3_client.get_bucket_tagging(bucket)
+    assert_tags(
+        expected_tags=expected_tags, unexpected_tags=unexpected_tags, actual_tags=actual_tags
+    )
+
+
+def assert_object_lock_mode(
+    s3_client: S3ClientWrapper,
+    bucket: str,
+    file_name: str,
+    object_lock_mode: str,
+    retain_until_date: datetime,
+    legal_hold_status: str = "OFF",
+    retain_period: Optional[int] = None,
+):
+    object_dict = s3_client.get_object(bucket, file_name, full_output=True)
+    assert (
+        object_dict.get("ObjectLockMode") == object_lock_mode
+    ), f"Expected Object Lock Mode is {object_lock_mode}"
+    assert (
+        object_dict.get("ObjectLockLegalHoldStatus") == legal_hold_status
+    ), f"Expected Object Lock Legal Hold Status is {legal_hold_status}"
+    object_retain_date = object_dict.get("ObjectLockRetainUntilDate")
+    retain_date = (
+        parse(object_retain_date) if isinstance(object_retain_date, str) else object_retain_date
+    )
+    if retain_until_date:
+        assert retain_date.strftime("%Y-%m-%dT%H:%M:%S") == retain_until_date.strftime(
+            "%Y-%m-%dT%H:%M:%S"
+        ), f'Expected Object Lock Retain Until Date is {str(retain_until_date.strftime("%Y-%m-%dT%H:%M:%S"))}'
+    elif retain_period:
+        last_modify_date = object_dict.get("LastModified")
+        last_modify = (
+            parse(last_modify_date) if isinstance(last_modify_date, str) else last_modify_date
+        )
+        assert (
+            retain_date - last_modify + timedelta(seconds=1)
+        ).days == retain_period, f"Expected retention period is {retain_period} days"
+
+
+def assert_s3_acl(acl_grants: list, permitted_users: str):
+    if permitted_users == "AllUsers":
+        grantees = {"AllUsers": 0, "CanonicalUser": 0}
+        for acl_grant in acl_grants:
+            if acl_grant.get("Grantee", {}).get("Type") == "Group":
+                uri = acl_grant.get("Grantee", {}).get("URI")
+                permission = acl_grant.get("Permission")
+                assert (uri, permission) == (
+                    "http://acs.amazonaws.com/groups/global/AllUsers",
+                    "FULL_CONTROL",
+                ), "All Groups should have FULL_CONTROL"
+                grantees["AllUsers"] += 1
+            if acl_grant.get("Grantee", {}).get("Type") == "CanonicalUser":
+                permission = acl_grant.get("Permission")
+                assert permission == "FULL_CONTROL", "Canonical User should have FULL_CONTROL"
+                grantees["CanonicalUser"] += 1
+        assert grantees["AllUsers"] >= 1, "All Users should have FULL_CONTROL"
+        assert grantees["CanonicalUser"] >= 1, "Canonical User should have FULL_CONTROL"
+
+    if permitted_users == "CanonicalUser":
+        for acl_grant in acl_grants:
+            if acl_grant.get("Grantee", {}).get("Type") == "CanonicalUser":
+                permission = acl_grant.get("Permission")
+                assert permission == "FULL_CONTROL", "Only CanonicalUser should have FULL_CONTROL"
+            else:
+                logger.error("FULL_CONTROL is given to All Users")
+
+
+@reporter.step_deco("Init S3 Credentials")
+def init_s3_credentials(
+    wallet_path: str,
+    cluster: Cluster,
+    s3_bearer_rules_file: str,
+    policy: Optional[dict] = None,
+):
+    bucket = str(uuid.uuid4())
+
+    s3gate_node = cluster.services(S3Gate)[0]
+    gate_public_key = s3gate_node.get_wallet_public_key()
+    cmd = (
+        f"{FROSTFS_AUTHMATE_EXEC} --debug --with-log --timeout {CREDENTIALS_CREATE_TIMEOUT} "
+        f"issue-secret --wallet {wallet_path} --gate-public-key={gate_public_key} "
+        f"--peer {cluster.default_rpc_endpoint} --container-friendly-name {bucket} "
+        f"--bearer-rules {s3_bearer_rules_file}"
+    )
+    if policy:
+        cmd += f" --container-policy {policy}'"
+    logger.info(f"Executing command: {cmd}")
+
+    try:
+        output = _run_with_passwd(cmd)
+        logger.info(f"Command completed with output: {output}")
+
+        # output contains some debug info and then several JSON structures, so we find each
+        # JSON structure by curly brackets (naive approach, but works while JSON is not nested)
+        # and then we take JSON containing secret_access_key
+        json_blocks = re.findall(r"\{.*?\}", output, re.DOTALL)
+        for json_block in json_blocks:
+            try:
+                parsed_json_block = json.loads(json_block)
+                if "secret_access_key" in parsed_json_block:
+                    return (
+                        parsed_json_block["container_id"],
+                        parsed_json_block["access_key_id"],
+                        parsed_json_block["secret_access_key"],
+                    )
+            except json.JSONDecodeError:
+                raise AssertionError(f"Could not parse info from output\n{output}")
+        raise AssertionError(f"Could not find AWS credentials in output:\n{output}")
+
+    except Exception as exc:
+        raise RuntimeError(f"Failed to init s3 credentials because of error\n{exc}") from exc
+
+
+@reporter.step_deco("Delete bucket with all objects")
+def delete_bucket_with_objects(s3_client: S3ClientWrapper, bucket: str):
+    versioning_status = s3_client.get_bucket_versioning_status(bucket)
+    if versioning_status == VersioningStatus.ENABLED.value:
+        # From versioned bucket we should delete all versions and delete markers of all objects
+        objects_versions = s3_client.list_objects_versions(bucket)
+        if objects_versions:
+            s3_client.delete_object_versions_without_dm(bucket, objects_versions)
+        objects_delete_markers = s3_client.list_delete_markers(bucket)
+        if objects_delete_markers:
+            s3_client.delete_object_versions_without_dm(bucket, objects_delete_markers)
+
+    else:
+        # From non-versioned bucket it's sufficient to delete objects by key
+        objects = s3_client.list_objects(bucket)
+        if objects:
+            s3_client.delete_objects(bucket, objects)
+        objects_delete_markers = s3_client.list_delete_markers(bucket)
+        if objects_delete_markers:
+            s3_client.delete_object_versions_without_dm(bucket, objects_delete_markers)
+
+    # Delete the bucket itself
+    s3_client.delete_bucket(bucket)

src/frostfs_testlib/steps/session_token.py

@@ -0,0 +1,287 @@
+import base64
+import json
+import logging
+import os
+import uuid
+from dataclasses import dataclass
+from enum import Enum
+from typing import Any, Optional
+
+from frostfs_testlib.cli import FrostfsCli
+from frostfs_testlib.reporter import get_reporter
+from frostfs_testlib.resources.cli import FROSTFS_CLI_EXEC
+from frostfs_testlib.resources.common import ASSETS_DIR, DEFAULT_WALLET_CONFIG
+from frostfs_testlib.shell import Shell
+from frostfs_testlib.storage.dataclasses.storage_object_info import StorageObjectInfo
+from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
+from frostfs_testlib.utils import json_utils, wallet_utils
+
+reporter = get_reporter()
+logger = logging.getLogger("NeoLogger")
+
+UNRELATED_KEY = "unrelated key in the session"
+UNRELATED_OBJECT = "unrelated object in the session"
+UNRELATED_CONTAINER = "unrelated container in the session"
+WRONG_VERB = "wrong verb of the session"
+INVALID_SIGNATURE = "invalid signature of the session data"
+
+
+class ObjectVerb(Enum):
+    PUT = "PUT"
+    DELETE = "DELETE"
+    GET = "GET"
+    RANGEHASH = "RANGEHASH"
+    RANGE = "RANGE"
+    HEAD = "HEAD"
+    SEARCH = "SEARCH"
+
+
+class ContainerVerb(Enum):
+    CREATE = "PUT"
+    DELETE = "DELETE"
+    SETEACL = "SETEACL"
+
+
+@dataclass
+class Lifetime:
+    exp: int = 100000000
+    nbf: int = 0
+    iat: int = 0
+
+
+@reporter.step_deco("Generate Session Token")
+def generate_session_token(
+    owner_wallet: WalletInfo,
+    session_wallet: WalletInfo,
+    session: dict[str, dict[str, Any]],
+    tokens_dir: str,
+    lifetime: Optional[Lifetime] = None,
+) -> str:
+    """
+    This function generates session token  and writes it to the file.
+    Args:
+        owner_wallet: wallet of container owner
+        session_wallet: wallet to which we grant the access via session token
+        session: Contains allowed operation with parameters
+        tokens_dir: Dir for token
+        lifetime: lifetime options for session
+    Returns:
+        The path to the generated session token file
+    """
+
+    file_path = os.path.join(tokens_dir, str(uuid.uuid4()))
+
+    pub_key_64 = wallet_utils.get_wallet_public_key(
+        session_wallet.path, session_wallet.password, "base64"
+    )
+
+    lifetime = lifetime or Lifetime()
+
+    session_token = {
+        "body": {
+            "id": f"{base64.b64encode(uuid.uuid4().bytes).decode('utf-8')}",
+            "ownerID": {"value": f"{json_utils.encode_for_json(owner_wallet.get_address())}"},
+            "lifetime": {
+                "exp": f"{lifetime.exp}",
+                "nbf": f"{lifetime.nbf}",
+                "iat": f"{lifetime.iat}",
+            },
+            "sessionKey": pub_key_64,
+        }
+    }
+    session_token["body"].update(session)
+
+    logger.info(f"Got this Session Token: {session_token}")
+    with open(file_path, "w", encoding="utf-8") as session_token_file:
+        json.dump(session_token, session_token_file, ensure_ascii=False, indent=4)
+
+    return file_path
+
+
+@reporter.step_deco("Generate Session Token For Container")
+def generate_container_session_token(
+    owner_wallet: WalletInfo,
+    session_wallet: WalletInfo,
+    verb: ContainerVerb,
+    tokens_dir: str,
+    lifetime: Optional[Lifetime] = None,
+    cid: Optional[str] = None,
+) -> str:
+    """
+    This function generates session token for ContainerSessionContext
+    and writes it to the file. It is able to prepare session token file
+    for a specific container (<cid>) or for every container (adds
+    "wildcard" field).
+    Args:
+        owner_wallet: wallet of container owner.
+        session_wallet: wallet to which we grant the access via session token.
+        verb: verb to grant access to.
+        lifetime: lifetime options for session.
+        cid: container ID of the container
+    Returns:
+        The path to the generated session token file
+    """
+    session = {
+        "container": {
+            "verb": verb.value,
+            "wildcard": cid is None,
+            **(
+                {"containerID": {"value": f"{json_utils.encode_for_json(cid)}"}}
+                if cid is not None
+                else {}
+            ),
+        },
+    }
+
+    return generate_session_token(
+        owner_wallet=owner_wallet,
+        session_wallet=session_wallet,
+        session=session,
+        tokens_dir=tokens_dir,
+        lifetime=lifetime,
+    )
+
+
+@reporter.step_deco("Generate Session Token For Object")
+def generate_object_session_token(
+    owner_wallet: WalletInfo,
+    session_wallet: WalletInfo,
+    oids: list[str],
+    cid: str,
+    verb: ObjectVerb,
+    tokens_dir: str,
+    lifetime: Optional[Lifetime] = None,
+) -> str:
+    """
+    This function generates session token for ObjectSessionContext
+    and writes it to the file.
+    Args:
+        owner_wallet: wallet of container owner
+        session_wallet: wallet to which we grant the access via session token
+        cid: container ID of the container
+        oids: list of objectIDs to put into session
+        verb: verb to grant access to; Valid verbs are: ObjectVerb.
+        lifetime: lifetime options for session
+    Returns:
+        The path to the generated session token file
+    """
+    session = {
+        "object": {
+            "verb": verb.value,
+            "target": {
+                "container": {"value": json_utils.encode_for_json(cid)},
+                "objects": [{"value": json_utils.encode_for_json(oid)} for oid in oids],
+            },
+        },
+    }
+
+    return generate_session_token(
+        owner_wallet=owner_wallet,
+        session_wallet=session_wallet,
+        session=session,
+        tokens_dir=tokens_dir,
+        lifetime=lifetime,
+    )
+
+
+@reporter.step_deco("Get signed token for container session")
+def get_container_signed_token(
+    owner_wallet: WalletInfo,
+    user_wallet: WalletInfo,
+    verb: ContainerVerb,
+    shell: Shell,
+    tokens_dir: str,
+    lifetime: Optional[Lifetime] = None,
+) -> str:
+    """
+    Returns signed token file path for static container session
+    """
+    session_token_file = generate_container_session_token(
+        owner_wallet=owner_wallet,
+        session_wallet=user_wallet,
+        verb=verb,
+        tokens_dir=tokens_dir,
+        lifetime=lifetime,
+    )
+    return sign_session_token(shell, session_token_file, owner_wallet)
+
+
+@reporter.step_deco("Get signed token for object session")
+def get_object_signed_token(
+    owner_wallet: WalletInfo,
+    user_wallet: WalletInfo,
+    cid: str,
+    storage_objects: list[StorageObjectInfo],
+    verb: ObjectVerb,
+    shell: Shell,
+    tokens_dir: str,
+    lifetime: Optional[Lifetime] = None,
+) -> str:
+    """
+    Returns signed token file path for static object session
+    """
+    storage_object_ids = [storage_object.oid for storage_object in storage_objects]
+    session_token_file = generate_object_session_token(
+        owner_wallet=owner_wallet,
+        session_wallet=user_wallet,
+        oids=storage_object_ids,
+        cid=cid,
+        verb=verb,
+        tokens_dir=tokens_dir,
+        lifetime=lifetime,
+    )
+    return sign_session_token(shell, session_token_file, owner_wallet)
+
+
+@reporter.step_deco("Create Session Token")
+def create_session_token(
+    shell: Shell,
+    owner: str,
+    wallet_path: str,
+    wallet_password: str,
+    rpc_endpoint: str,
+) -> str:
+    """
+    Create session token for an object.
+    Args:
+        shell: Shell instance.
+        owner: User that writes the token.
+        wallet_path: The path to wallet to which we grant the access via session token.
+        wallet_password: Wallet password.
+        rpc_endpoint: Remote node address (as 'multiaddr' or '<host>:<port>').
+    Returns:
+        The path to the generated session token file.
+    """
+    session_token = os.path.join(os.getcwd(), ASSETS_DIR, str(uuid.uuid4()))
+    frostfscli = FrostfsCli(shell=shell, frostfs_cli_exec_path=FROSTFS_CLI_EXEC)
+    frostfscli.session.create(
+        rpc_endpoint=rpc_endpoint,
+        address=owner,
+        wallet=wallet_path,
+        wallet_password=wallet_password,
+        out=session_token,
+    )
+    return session_token
+
+
+@reporter.step_deco("Sign Session Token")
+def sign_session_token(shell: Shell, session_token_file: str, wlt: WalletInfo) -> str:
+    """
+    This function signs the session token by the given wallet.
+
+    Args:
+        shell: Shell instance.
+        session_token_file: The path to the session token file.
+        wlt: The path to the signing wallet.
+
+    Returns:
+        The path to the signed token.
+    """
+    signed_token_file = os.path.join(os.getcwd(), ASSETS_DIR, str(uuid.uuid4()))
+    frostfscli = FrostfsCli(
+        shell=shell, frostfs_cli_exec_path=FROSTFS_CLI_EXEC, config_file=DEFAULT_WALLET_CONFIG
+    )
+    frostfscli.util.sign_session_token(
+        wallet=wlt.path, from_file=session_token_file, to_file=signed_token_file
+    )
+    return signed_token_file

src/frostfs_testlib/steps/storage_object.py

@@ -0,0 +1,63 @@
+import logging
+from time import sleep
+
+import pytest
+
+from frostfs_testlib.reporter import get_reporter
+from frostfs_testlib.resources.error_patterns import OBJECT_ALREADY_REMOVED
+from frostfs_testlib.shell import Shell
+from frostfs_testlib.steps.cli.object import delete_object, get_object
+from frostfs_testlib.steps.epoch import tick_epoch
+from frostfs_testlib.steps.tombstone import verify_head_tombstone
+from frostfs_testlib.storage.cluster import Cluster
+from frostfs_testlib.storage.dataclasses.storage_object_info import StorageObjectInfo
+
+reporter = get_reporter()
+logger = logging.getLogger("NeoLogger")
+
+CLEANUP_TIMEOUT = 10
+
+
+@reporter.step_deco("Delete Objects")
+def delete_objects(
+    storage_objects: list[StorageObjectInfo], shell: Shell, cluster: Cluster
+) -> None:
+    """
+    Deletes given storage objects.
+
+    Args:
+        storage_objects: list of objects to delete
+        shell: executor for cli command
+    """
+
+    with reporter.step("Delete objects"):
+        for storage_object in storage_objects:
+            storage_object.tombstone = delete_object(
+                storage_object.wallet_file_path,
+                storage_object.cid,
+                storage_object.oid,
+                shell=shell,
+                endpoint=cluster.default_rpc_endpoint,
+            )
+            verify_head_tombstone(
+                wallet_path=storage_object.wallet_file_path,
+                cid=storage_object.cid,
+                oid_ts=storage_object.tombstone,
+                oid=storage_object.oid,
+                shell=shell,
+                endpoint=cluster.default_rpc_endpoint,
+            )
+
+    tick_epoch(shell, cluster)
+    sleep(CLEANUP_TIMEOUT)
+
+    with reporter.step("Get objects and check errors"):
+        for storage_object in storage_objects:
+            with pytest.raises(Exception, match=OBJECT_ALREADY_REMOVED):
+                get_object(
+                    storage_object.wallet_file_path,
+                    storage_object.cid,
+                    storage_object.oid,
+                    shell=shell,
+                    endpoint=cluster.default_rpc_endpoint,
+                )

src/frostfs_testlib/steps/storage_policy.py

@@ -0,0 +1,173 @@
+#!/usr/bin/python3
+
+"""
+    This module contains keywords which are used for asserting
+    that storage policies are respected.
+"""
+import logging
+
+from frostfs_testlib.reporter import get_reporter
+from frostfs_testlib.resources.error_patterns import OBJECT_NOT_FOUND
+from frostfs_testlib.shell import Shell
+from frostfs_testlib.steps.cli.object import head_object
+from frostfs_testlib.steps.complex_object_actions import get_last_object
+from frostfs_testlib.storage.cluster import StorageNode
+from frostfs_testlib.utils import string_utils
+
+reporter = get_reporter()
+logger = logging.getLogger("NeoLogger")
+
+
+@reporter.step_deco("Get Object Copies")
+def get_object_copies(
+    complexity: str, wallet: str, cid: str, oid: str, shell: Shell, nodes: list[StorageNode]
+) -> int:
+    """
+    The function performs requests to all nodes of the container and
+    finds out if they store a copy of the object. The procedure is
+    different for simple and complex object, so the function requires
+    a sign of object complexity.
+    Args:
+        complexity (str): the tag of object size and complexity,
+                            [Simple|Complex]
+        wallet (str): the path to the wallet on whose behalf the
+                            copies are got
+        cid (str): ID of the container
+        oid (str): ID of the Object
+        shell: executor for cli command
+    Returns:
+        (int): the number of object copies in the container
+    """
+    return (
+        get_simple_object_copies(wallet, cid, oid, shell, nodes)
+        if complexity == "Simple"
+        else get_complex_object_copies(wallet, cid, oid, shell, nodes)
+    )
+
+
+@reporter.step_deco("Get Simple Object Copies")
+def get_simple_object_copies(
+    wallet: str, cid: str, oid: str, shell: Shell, nodes: list[StorageNode]
+) -> int:
+    """
+    To figure out the number of a simple object copies, only direct
+    HEAD requests should be made to the every node of the container.
+    We consider non-empty HEAD response as a stored object copy.
+    Args:
+        wallet (str): the path to the wallet on whose behalf the
+                            copies are got
+        cid (str): ID of the container
+        oid (str): ID of the Object
+        shell: executor for cli command
+        nodes: nodes to search on
+    Returns:
+        (int): the number of object copies in the container
+    """
+    copies = 0
+    for node in nodes:
+        try:
+            response = head_object(
+                wallet, cid, oid, shell=shell, endpoint=node.get_rpc_endpoint(), is_direct=True
+            )
+            if response:
+                logger.info(f"Found object {oid} on node {node}")
+                copies += 1
+        except Exception:
+            logger.info(f"No {oid} object copy found on {node}, continue")
+            continue
+    return copies
+
+
+@reporter.step_deco("Get Complex Object Copies")
+def get_complex_object_copies(
+    wallet: str, cid: str, oid: str, shell: Shell, nodes: list[StorageNode]
+) -> int:
+    """
+    To figure out the number of a complex object copies, we firstly
+    need to retrieve its Last object. We consider that the number of
+    complex object copies is equal to the number of its last object
+    copies. When we have the Last object ID, the task is reduced
+    to getting simple object copies.
+    Args:
+        wallet (str): the path to the wallet on whose behalf the
+                            copies are got
+        cid (str): ID of the container
+        oid (str): ID of the Object
+        shell: executor for cli command
+    Returns:
+        (int): the number of object copies in the container
+    """
+    last_oid = get_last_object(wallet, cid, oid, shell, nodes)
+    assert last_oid, f"No Last Object for {cid}/{oid} found among all Storage Nodes"
+    return get_simple_object_copies(wallet, cid, last_oid, shell, nodes)
+
+
+@reporter.step_deco("Get Nodes With Object")
+def get_nodes_with_object(
+    cid: str, oid: str, shell: Shell, nodes: list[StorageNode]
+) -> list[StorageNode]:
+    """
+    The function returns list of nodes which store
+    the given object.
+    Args:
+         cid (str): ID of the container which store the object
+         oid (str): object ID
+         shell: executor for cli command
+         nodes: nodes to find on
+    Returns:
+         (list): nodes which store the object
+    """
+
+    nodes_list = []
+    for node in nodes:
+        wallet = node.get_wallet_path()
+        wallet_config = node.get_wallet_config_path()
+        try:
+            res = head_object(
+                wallet,
+                cid,
+                oid,
+                shell=shell,
+                endpoint=node.get_rpc_endpoint(),
+                is_direct=True,
+                wallet_config=wallet_config,
+            )
+            if res is not None:
+                logger.info(f"Found object {oid} on node {node}")
+                nodes_list.append(node)
+        except Exception:
+            logger.info(f"No {oid} object copy found on {node}, continue")
+            continue
+    return nodes_list
+
+
+@reporter.step_deco("Get Nodes Without Object")
+def get_nodes_without_object(
+    wallet: str, cid: str, oid: str, shell: Shell, nodes: list[StorageNode]
+) -> list[StorageNode]:
+    """
+    The function returns list of nodes which do not store
+    the given object.
+    Args:
+         wallet (str): the path to the wallet on whose behalf
+                     we request the nodes
+         cid (str): ID of the container which store the object
+         oid (str): object ID
+         shell: executor for cli command
+    Returns:
+         (list): nodes which do not store the object
+    """
+    nodes_list = []
+    for node in nodes:
+        try:
+            res = head_object(
+                wallet, cid, oid, shell=shell, endpoint=node.get_rpc_endpoint(), is_direct=True
+            )
+            if res is None:
+                nodes_list.append(node)
+        except Exception as err:
+            if string_utils.is_str_match_pattern(err, OBJECT_NOT_FOUND):
+                nodes_list.append(node)
+            else:
+                raise Exception(f"Got error {err} on head object command") from err
+    return nodes_list

src/frostfs_testlib/steps/tombstone.py

@@ -0,0 +1,41 @@
+import json
+import logging
+
+from neo3.wallet import wallet
+
+from frostfs_testlib.reporter import get_reporter
+from frostfs_testlib.shell import Shell
+from frostfs_testlib.steps.cli.object import head_object
+
+reporter = get_reporter()
+logger = logging.getLogger("NeoLogger")
+
+
+@reporter.step_deco("Verify Head Tombstone")
+def verify_head_tombstone(
+    wallet_path: str, cid: str, oid_ts: str, oid: str, shell: Shell, endpoint: str
+):
+    header = head_object(wallet_path, cid, oid_ts, shell=shell, endpoint=endpoint)["header"]
+
+    s_oid = header["sessionToken"]["body"]["object"]["target"]["objects"]
+    logger.info(f"Header Session OIDs is {s_oid}")
+    logger.info(f"OID is {oid}")
+
+    assert header["containerID"] == cid, "Tombstone Header CID is wrong"
+
+    with open(wallet_path, "r") as file:
+        wlt_data = json.loads(file.read())
+    wlt = wallet.Wallet.from_json(wlt_data, password="")
+    addr = wlt.accounts[0].address
+
+    assert header["ownerID"] == addr, "Tombstone Owner ID is wrong"
+    assert header["objectType"] == "TOMBSTONE", "Header Type isn't Tombstone"
+    assert (
+        header["sessionToken"]["body"]["object"]["verb"] == "DELETE"
+    ), "Header Session Type isn't DELETE"
+    assert (
+        header["sessionToken"]["body"]["object"]["target"]["container"] == cid
+    ), "Header Session ID is wrong"
+    assert (
+        oid in header["sessionToken"]["body"]["object"]["target"]["objects"]
+    ), "Header Session OID is wrong"