From 13bc98eecc950018de17e33959b605beb3d1bf30 Mon Sep 17 00:00:00 2001 From: Aleksei Chetaev Date: Mon, 20 Feb 2023 00:58:07 +0100 Subject: [PATCH] Fixing imports after move utils ti frostfs-testlib Signed-off-by: Aleksei Chetaev --- pytest_tests/helpers/cluster.py | 4 +- pytest_tests/helpers/utility.py | 29 +--- pytest_tests/helpers/wallet.py | 6 +- pytest_tests/steps/session_token.py | 22 +-- pytest_tests/testsuites/acl/conftest.py | 4 +- .../acl/storage_group/test_storagegroup.py | 4 +- pytest_tests/testsuites/conftest.py | 4 +- .../network/test_node_management.py | 12 +- .../testsuites/object/test_object_lock.py | 5 +- .../services/s3_gate/test_s3_object.py | 9 +- .../test_object_session_token.py | 4 +- robot/resources/lib/python_keywords/acl.py | 6 +- .../lib/python_keywords/container.py | 4 +- .../lib/python_keywords/data_formatters.py | 50 ------- robot/resources/lib/python_keywords/epoch.py | 7 +- .../lib/python_keywords/frostfs_verbs.py | 12 +- .../lib/python_keywords/json_transformers.py | 136 ------------------ .../lib/python_keywords/node_management.py | 10 +- .../lib/python_keywords/object_access.py | 16 +-- .../lib/python_keywords/payment_neogo.py | 18 +-- .../lib/python_keywords/storage_policy.py | 4 +- 21 files changed, 78 insertions(+), 288 deletions(-) delete mode 100644 robot/resources/lib/python_keywords/data_formatters.py delete mode 100644 robot/resources/lib/python_keywords/json_transformers.py diff --git a/pytest_tests/helpers/cluster.py b/pytest_tests/helpers/cluster.py index bb52a653..20704cc9 100644 --- a/pytest_tests/helpers/cluster.py +++ b/pytest_tests/helpers/cluster.py @@ -3,11 +3,11 @@ import re from dataclasses import dataclass from typing import Any -import data_formatters import yaml from frostfs_testlib.blockchain import RPCClient from frostfs_testlib.hosting import Host, Hosting from frostfs_testlib.hosting.config import ServiceConfig +from frostfs_testlib.utils import wallet_utils @dataclass @@ -85,7 +85,7 @@ class NodeBase: def get_wallet_public_key(self): storage_wallet_path = self.get_wallet_path() storage_wallet_pass = self.get_wallet_password() - return data_formatters.get_wallet_public_key(storage_wallet_path, storage_wallet_pass) + return wallet_utils.get_wallet_public_key(storage_wallet_path, storage_wallet_pass) def _get_attribute(self, attribute_name: str, default_attribute_name: str = None) -> list[str]: config = self.host.get_service_config(self.name) diff --git a/pytest_tests/helpers/utility.py b/pytest_tests/helpers/utility.py index 3993de12..fe03d499 100644 --- a/pytest_tests/helpers/utility.py +++ b/pytest_tests/helpers/utility.py @@ -2,32 +2,7 @@ import time import allure from common import STORAGE_GC_TIME - - -def parse_time(value: str) -> int: - """Converts time interval in text form into time interval as number of seconds. - - Args: - value: time interval as text. - - Returns: - Number of seconds in the parsed time interval. - """ - value = value.lower() - - for suffix in ["s", "sec"]: - if value.endswith(suffix): - return int(value[: -len(suffix)]) - - for suffix in ["m", "min"]: - if value.endswith(suffix): - return int(value[: -len(suffix)]) * 60 - - for suffix in ["h", "hr", "hour"]: - if value.endswith(suffix): - return int(value[: -len(suffix)]) * 60 * 60 - - raise ValueError(f"Unknown units in time value '{value}'") +from frostfs_testlib.utils import datetime_utils def placement_policy_from_container(container_info: str) -> str: @@ -57,6 +32,6 @@ def placement_policy_from_container(container_info: str) -> str: def wait_for_gc_pass_on_storage_nodes() -> None: - wait_time = parse_time(STORAGE_GC_TIME) + wait_time = datetime_utils.parse_time(STORAGE_GC_TIME) with allure.step(f"Wait {wait_time}s until GC completes on storage nodes"): time.sleep(wait_time) diff --git a/pytest_tests/helpers/wallet.py b/pytest_tests/helpers/wallet.py index f1e13a2b..597490dc 100644 --- a/pytest_tests/helpers/wallet.py +++ b/pytest_tests/helpers/wallet.py @@ -5,7 +5,7 @@ from dataclasses import dataclass from cluster import Cluster, NodeBase from common import FREE_STORAGE, WALLET_CONFIG, WALLET_PASS from frostfs_testlib.shell import Shell -from frostfs_testlib.utils.wallet import get_last_address_from_wallet, init_wallet +from frostfs_testlib.utils import wallet_utils from python_keywords.payment_neogo import deposit_gas, transfer_gas @@ -28,7 +28,7 @@ class WalletFile: Returns: The address of the wallet. """ - return get_last_address_from_wallet(self.path, self.password) + return wallet_utils.get_last_address_from_wallet(self.path, self.password) class WalletFactory: @@ -47,7 +47,7 @@ class WalletFactory: WalletFile object of new wallet """ wallet_path = os.path.join(self.wallets_dir, f"{str(uuid.uuid4())}.json") - init_wallet(wallet_path, password) + wallet_utils.init_wallet(wallet_path, password) if not FREE_STORAGE: main_chain = self.cluster.main_chain_nodes[0] diff --git a/pytest_tests/steps/session_token.py b/pytest_tests/steps/session_token.py index d82e8a35..0498def3 100644 --- a/pytest_tests/steps/session_token.py +++ b/pytest_tests/steps/session_token.py @@ -8,12 +8,10 @@ from enum import Enum from typing import Any, Optional import allure -import json_transformers from common import ASSETS_DIR, FROSTFS_CLI_EXEC, WALLET_CONFIG -from data_formatters import get_wallet_public_key from frostfs_testlib.cli import FrostfsCli from frostfs_testlib.shell import Shell -from json_transformers import encode_for_json +from frostfs_testlib.utils import json_utils, wallet_utils from storage_object_info import StorageObjectInfo from wallet import WalletFile @@ -71,16 +69,16 @@ def generate_session_token( file_path = os.path.join(tokens_dir, str(uuid.uuid4())) - pub_key_64 = get_wallet_public_key(session_wallet.path, session_wallet.password, "base64") + pub_key_64 = wallet_utils.get_wallet_public_key( + session_wallet.path, session_wallet.password, "base64" + ) lifetime = lifetime or Lifetime() session_token = { "body": { "id": f"{base64.b64encode(uuid.uuid4().bytes).decode('utf-8')}", - "ownerID": { - "value": f"{json_transformers.encode_for_json(owner_wallet.get_address())}" - }, + "ownerID": {"value": f"{json_utils.encode_for_json(owner_wallet.get_address())}"}, "lifetime": { "exp": f"{lifetime.exp}", "nbf": f"{lifetime.nbf}", @@ -125,7 +123,11 @@ def generate_container_session_token( "container": { "verb": verb.value, "wildcard": cid is None, - **({"containerID": {"value": f"{encode_for_json(cid)}"}} if cid is not None else {}), + **( + {"containerID": {"value": f"{json_utils.encode_for_json(cid)}"}} + if cid is not None + else {} + ), }, } @@ -165,8 +167,8 @@ def generate_object_session_token( "object": { "verb": verb.value, "target": { - "container": {"value": encode_for_json(cid)}, - "objects": [{"value": encode_for_json(oid)} for oid in oids], + "container": {"value": json_utils.encode_for_json(cid)}, + "objects": [{"value": json_utils.encode_for_json(oid)} for oid in oids], }, }, } diff --git a/pytest_tests/testsuites/acl/conftest.py b/pytest_tests/testsuites/acl/conftest.py index 73b94383..737082d5 100644 --- a/pytest_tests/testsuites/acl/conftest.py +++ b/pytest_tests/testsuites/acl/conftest.py @@ -10,7 +10,7 @@ from common import WALLET_CONFIG, WALLET_PASS from file_helper import generate_file from frostfs_testlib.resources.common import PUBLIC_ACL from frostfs_testlib.shell import Shell -from frostfs_testlib.utils.wallet import init_wallet +from frostfs_testlib.utils import wallet_utils from python_keywords.acl import EACLRole from python_keywords.container import create_container from python_keywords.frostfs_verbs import put_object_to_random_node @@ -41,7 +41,7 @@ def wallets(default_wallet, temp_directory, cluster: Cluster) -> Wallets: os.path.join(temp_directory, f"{str(uuid.uuid4())}.json") for _ in range(2) ] for other_wallet_path in other_wallets_paths: - init_wallet(other_wallet_path, WALLET_PASS) + wallet_utils.init_wallet(other_wallet_path, WALLET_PASS) ir_node = cluster.ir_nodes[0] storage_node = cluster.storage_nodes[0] diff --git a/pytest_tests/testsuites/acl/storage_group/test_storagegroup.py b/pytest_tests/testsuites/acl/storage_group/test_storagegroup.py index 30f61fcb..44538cb8 100644 --- a/pytest_tests/testsuites/acl/storage_group/test_storagegroup.py +++ b/pytest_tests/testsuites/acl/storage_group/test_storagegroup.py @@ -9,7 +9,7 @@ from cluster_test_base import ClusterTestBase from common import ASSETS_DIR, FREE_STORAGE, WALLET_PASS from file_helper import generate_file from frostfs_testlib.resources.common import OBJECT_ACCESS_DENIED, OBJECT_NOT_FOUND -from frostfs_testlib.utils.wallet import init_wallet +from frostfs_testlib.utils import wallet_utils from python_keywords.acl import ( EACLAccess, EACLOperation, @@ -48,7 +48,7 @@ class TestStorageGroup(ClusterTestBase): def prepare_two_wallets(self, default_wallet): self.main_wallet = default_wallet self.other_wallet = os.path.join(os.getcwd(), ASSETS_DIR, f"{str(uuid.uuid4())}.json") - init_wallet(self.other_wallet, WALLET_PASS) + wallet_utils.init_wallet(self.other_wallet, WALLET_PASS) if not FREE_STORAGE: main_chain = self.cluster.main_chain_nodes[0] deposit = 30 diff --git a/pytest_tests/testsuites/conftest.py b/pytest_tests/testsuites/conftest.py index 24e95b3f..963f9acd 100644 --- a/pytest_tests/testsuites/conftest.py +++ b/pytest_tests/testsuites/conftest.py @@ -24,7 +24,7 @@ from env_properties import save_env_properties from frostfs_testlib.hosting import Hosting from frostfs_testlib.reporter import AllureHandler, get_reporter from frostfs_testlib.shell import LocalShell, Shell -from frostfs_testlib.utils.wallet import init_wallet +from frostfs_testlib.utils import wallet_utils from k6 import LoadParams from load import get_services_endpoints, prepare_k6_instances from load_params import ( @@ -242,7 +242,7 @@ def background_grpc_load(client_shell: Shell, hosting: Hosting): @allure.title("Prepare wallet and deposit") def default_wallet(client_shell: Shell, temp_directory: str, cluster: Cluster): wallet_path = os.path.join(os.getcwd(), ASSETS_DIR, f"{str(uuid.uuid4())}.json") - init_wallet(wallet_path, WALLET_PASS) + wallet_utils.init_wallet(wallet_path, WALLET_PASS) allure.attach.file(wallet_path, os.path.basename(wallet_path), allure.attachment_type.JSON) if not FREE_STORAGE: diff --git a/pytest_tests/testsuites/network/test_node_management.py b/pytest_tests/testsuites/network/test_node_management.py index 66917269..ffe7cf88 100644 --- a/pytest_tests/testsuites/network/test_node_management.py +++ b/pytest_tests/testsuites/network/test_node_management.py @@ -11,7 +11,7 @@ from common import FROSTFS_CONTRACT_CACHE_TIMEOUT, MORPH_BLOCK_TIME from epoch import tick_epoch from file_helper import generate_file from frostfs_testlib.resources.common import OBJECT_NOT_FOUND, PUBLIC_ACL -from frostfs_testlib.utils.errors import error_matches_status +from frostfs_testlib.utils import datetime_utils, string_utils from python_keywords.container import create_container, get_container from python_keywords.failover_utils import wait_object_replication from python_keywords.frostfs_verbs import ( @@ -35,7 +35,7 @@ from python_keywords.node_management import ( storage_node_set_status, ) from storage_policy import get_nodes_with_object, get_simple_object_copies -from utility import parse_time, placement_policy_from_container, wait_for_gc_pass_on_storage_nodes +from utility import placement_policy_from_container, wait_for_gc_pass_on_storage_nodes logger = logging.getLogger("NeoLogger") check_nodes: list[StorageNode] = [] @@ -109,13 +109,13 @@ class TestNodeManagement(ClusterTestBase): # We need to wait for node to establish notifications from morph-chain # Otherwise it will hang up when we will try to set status - sleep(parse_time(MORPH_BLOCK_TIME)) + sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME)) with allure.step(f"Move node {node} to online state"): storage_node_set_status(node, status="online", retries=2) check_nodes.remove(node) - sleep(parse_time(MORPH_BLOCK_TIME)) + sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME)) self.tick_epoch_with_retries(3) check_node_in_map(node, shell=self.shell, alive_node=alive_node) @@ -472,7 +472,7 @@ class TestNodeManagement(ClusterTestBase): if copies == expected_copies: break tick_epoch(self.shell, self.cluster) - sleep(parse_time(FROSTFS_CONTRACT_CACHE_TIMEOUT)) + sleep(datetime_utils.parse_time(FROSTFS_CONTRACT_CACHE_TIMEOUT)) else: raise AssertionError(f"There are no {expected_copies} copies during time") @@ -483,7 +483,7 @@ class TestNodeManagement(ClusterTestBase): checker(wallet, cid, oid, shell=self.shell, endpoint=endpoint) wait_for_gc_pass_on_storage_nodes() except Exception as err: - if error_matches_status(err, OBJECT_NOT_FOUND): + if string_utils.is_str_match_pattern(err, OBJECT_NOT_FOUND): return raise AssertionError(f'Expected "{OBJECT_NOT_FOUND}" error, got\n{err}') diff --git a/pytest_tests/testsuites/object/test_object_lock.py b/pytest_tests/testsuites/object/test_object_lock.py index 89a933a2..7794eb60 100755 --- a/pytest_tests/testsuites/object/test_object_lock.py +++ b/pytest_tests/testsuites/object/test_object_lock.py @@ -18,13 +18,14 @@ from frostfs_testlib.resources.common import ( OBJECT_NOT_FOUND, ) from frostfs_testlib.shell import Shell +from frostfs_testlib.utils import datetime_utils from node_management import drop_object from pytest import FixtureRequest from python_keywords.container import create_container from python_keywords.frostfs_verbs import delete_object, head_object, lock_object from storage_policy import get_nodes_with_object from test_control import expect_not_raises, wait_for_success -from utility import parse_time, wait_for_gc_pass_on_storage_nodes +from utility import wait_for_gc_pass_on_storage_nodes from helpers.container import StorageContainer, StorageContainerInfo from helpers.storage_object_info import LockObjectInfo, StorageObjectInfo @@ -321,7 +322,7 @@ class TestObjectLockWithGrpc(ClusterTestBase): self.cluster.default_rpc_endpoint, ) - @wait_for_success(parse_time(STORAGE_GC_TIME)) + @wait_for_success(datetime_utils.parse_time(STORAGE_GC_TIME)) def check_object_not_found(): with pytest.raises(Exception, match=OBJECT_NOT_FOUND): head_object( diff --git a/pytest_tests/testsuites/services/s3_gate/test_s3_object.py b/pytest_tests/testsuites/services/s3_gate/test_s3_object.py index 94e9e8eb..6dc89865 100644 --- a/pytest_tests/testsuites/services/s3_gate/test_s3_object.py +++ b/pytest_tests/testsuites/services/s3_gate/test_s3_object.py @@ -8,9 +8,8 @@ import allure import pytest from aws_cli_client import AwsCliClient from common import ASSETS_DIR, FREE_STORAGE, WALLET_PASS -from data_formatters import get_wallet_public_key from file_helper import concat_files, generate_file, generate_file_with_content, get_file_hash -from frostfs_testlib.utils.wallet import init_wallet +from frostfs_testlib.utils import wallet_utils from python_keywords.payment_neogo import deposit_gas, transfer_gas from s3_helper import ( assert_object_lock_mode, @@ -661,10 +660,10 @@ class TestS3GateObject(TestS3GateBase): @pytest.fixture def prepare_two_wallets(self, default_wallet, client_shell): self.main_wallet = default_wallet - self.main_public_key = get_wallet_public_key(self.main_wallet, WALLET_PASS) + self.main_public_key = wallet_utils.get_wallet_public_key(self.main_wallet, WALLET_PASS) self.other_wallet = os.path.join(os.getcwd(), ASSETS_DIR, f"{str(uuid.uuid4())}.json") - init_wallet(self.other_wallet, WALLET_PASS) - self.other_public_key = get_wallet_public_key(self.other_wallet, WALLET_PASS) + wallet_utils.init_wallet(self.other_wallet, WALLET_PASS) + self.other_public_key = wallet_utils.get_wallet_public_key(self.other_wallet, WALLET_PASS) if not FREE_STORAGE: main_chain = self.cluster.main_chain_nodes[0] diff --git a/pytest_tests/testsuites/session_token/test_object_session_token.py b/pytest_tests/testsuites/session_token/test_object_session_token.py index f181f9e2..3ef261ab 100644 --- a/pytest_tests/testsuites/session_token/test_object_session_token.py +++ b/pytest_tests/testsuites/session_token/test_object_session_token.py @@ -6,7 +6,7 @@ from cluster_test_base import ClusterTestBase from common import WALLET_PASS from file_helper import generate_file from frostfs_testlib.resources.common import SESSION_NOT_FOUND -from frostfs_testlib.utils.wallet import get_last_address_from_wallet +from frostfs_testlib.utils import wallet_utils from python_keywords.container import create_container from python_keywords.frostfs_verbs import delete_object, put_object, put_object_to_random_node @@ -38,7 +38,7 @@ class TestDynamicObjectSession(ClusterTestBase): with allure.step("Init wallet"): wallet = default_wallet - address = get_last_address_from_wallet(wallet, "") + address = wallet_utils.get_last_address_from_wallet(wallet, "") with allure.step("Nodes Settlements"): ( diff --git a/robot/resources/lib/python_keywords/acl.py b/robot/resources/lib/python_keywords/acl.py index 9b46d74b..74ae6517 100644 --- a/robot/resources/lib/python_keywords/acl.py +++ b/robot/resources/lib/python_keywords/acl.py @@ -11,9 +11,9 @@ from typing import Any, Dict, List, Optional, Union import allure import base58 from common import ASSETS_DIR, FROSTFS_CLI_EXEC, WALLET_CONFIG -from data_formatters import get_wallet_public_key from frostfs_testlib.cli import FrostfsCli from frostfs_testlib.shell import Shell +from frostfs_testlib.utils import wallet_utils logger = logging.getLogger("NeoLogger") EACL_LIFETIME = 100500 @@ -110,7 +110,7 @@ class EACLRule: role = ( self.role.value if isinstance(self.role, EACLRole) - else f'pubkey:{get_wallet_public_key(self.role, "")}' + else f'pubkey:{wallet_utils.get_wallet_public_key(self.role, "")}' ) return f'{self.access.value} {self.operation.value} {self.filters or ""} {role}' @@ -244,7 +244,7 @@ def eacl_rules(access: str, verbs: list, user: str) -> list[str]: (list): a list of eACL rules """ if user not in ("others", "user"): - pubkey = get_wallet_public_key(user, wallet_password="") + pubkey = wallet_utils.get_wallet_public_key(user, wallet_password="") user = f"pubkey:{pubkey}" rules = [] diff --git a/robot/resources/lib/python_keywords/container.py b/robot/resources/lib/python_keywords/container.py index 812b4e30..26cf2be7 100644 --- a/robot/resources/lib/python_keywords/container.py +++ b/robot/resources/lib/python_keywords/container.py @@ -10,10 +10,10 @@ from time import sleep from typing import Optional, Union import allure -import json_transformers from common import FROSTFS_CLI_EXEC, WALLET_CONFIG from frostfs_testlib.cli import FrostfsCli from frostfs_testlib.shell import Shell +from frostfs_testlib.utils import json_utils logger = logging.getLogger("NeoLogger") @@ -164,7 +164,7 @@ def get_container( for attr in container_info["attributes"]: attributes[attr["key"]] = attr["value"] container_info["attributes"] = attributes - container_info["ownerID"] = json_transformers.json_reencode(container_info["ownerID"]["value"]) + container_info["ownerID"] = json_utils.json_reencode(container_info["ownerID"]["value"]) return container_info diff --git a/robot/resources/lib/python_keywords/data_formatters.py b/robot/resources/lib/python_keywords/data_formatters.py deleted file mode 100644 index fd293fc7..00000000 --- a/robot/resources/lib/python_keywords/data_formatters.py +++ /dev/null @@ -1,50 +0,0 @@ -import base64 -import json - -import base58 -from neo3.wallet import wallet - - -def dict_to_attrs(attrs: dict) -> str: - """ - This function takes a dictionary of object's attributes and converts them - into string. The string is passed to `--attributes` key of frostfs-cli. - - Args: - attrs (dict): object attributes in {"a": "b", "c": "d"} format. - - Returns: - (str): string in "a=b,c=d" format. - """ - return ",".join(f"{key}={value}" for key, value in attrs.items()) - - -def __fix_wallet_schema(wallet: dict) -> None: - # Temporary function to fix wallets that do not conform to the schema - # TODO: get rid of it once issue is solved - if "name" not in wallet: - wallet["name"] = None - for account in wallet["accounts"]: - if "extra" not in account: - account["extra"] = None - - -def get_wallet_public_key(wallet_path: str, wallet_password: str, format: str = "hex") -> str: - # Get public key from wallet file - with open(wallet_path, "r") as file: - wallet_content = json.load(file) - __fix_wallet_schema(wallet_content) - - wallet_from_json = wallet.Wallet.from_json(wallet_content, password=wallet_password) - public_key_hex = str(wallet_from_json.accounts[0].public_key) - - # Convert public key to specified format - if format == "hex": - return public_key_hex - if format == "base58": - public_key_base58 = base58.b58encode(bytes.fromhex(public_key_hex)) - return public_key_base58.decode("utf-8") - if format == "base64": - public_key_base64 = base64.b64encode(bytes.fromhex(public_key_hex)) - return public_key_base64.decode("utf-8") - raise ValueError(f"Invalid public key format: {format}") diff --git a/robot/resources/lib/python_keywords/epoch.py b/robot/resources/lib/python_keywords/epoch.py index 889f50db..6932661c 100644 --- a/robot/resources/lib/python_keywords/epoch.py +++ b/robot/resources/lib/python_keywords/epoch.py @@ -13,10 +13,9 @@ from common import ( ) from frostfs_testlib.cli import FrostfsAdm, FrostfsCli, NeoGo from frostfs_testlib.shell import Shell -from frostfs_testlib.utils.wallet import get_last_address_from_wallet +from frostfs_testlib.utils import datetime_utils, wallet_utils from payment_neogo import get_contract_hash from test_control import wait_for_success -from utility import parse_time logger = logging.getLogger("NeoLogger") @@ -90,7 +89,7 @@ def tick_epoch(shell: Shell, cluster: Cluster, alive_node: Optional[StorageNode] # In case if no local_wallet_path is provided, we use wallet_path ir_wallet_path = ir_node.get_wallet_path() ir_wallet_pass = ir_node.get_wallet_password() - ir_address = get_last_address_from_wallet(ir_wallet_path, ir_wallet_pass) + ir_address = wallet_utils.get_last_address_from_wallet(ir_wallet_path, ir_wallet_pass) morph_chain = cluster.morph_chain_nodes[0] morph_endpoint = morph_chain.get_endpoint() @@ -108,4 +107,4 @@ def tick_epoch(shell: Shell, cluster: Cluster, alive_node: Optional[StorageNode] force=True, gas=1, ) - sleep(parse_time(MAINNET_BLOCK_TIME)) + sleep(datetime_utils.parse_time(MAINNET_BLOCK_TIME)) diff --git a/robot/resources/lib/python_keywords/frostfs_verbs.py b/robot/resources/lib/python_keywords/frostfs_verbs.py index 8855d22b..24aaee14 100644 --- a/robot/resources/lib/python_keywords/frostfs_verbs.py +++ b/robot/resources/lib/python_keywords/frostfs_verbs.py @@ -6,11 +6,11 @@ import uuid from typing import Any, Optional import allure -import json_transformers from cluster import Cluster from common import ASSETS_DIR, FROSTFS_CLI_EXEC, WALLET_CONFIG from frostfs_testlib.cli import FrostfsCli from frostfs_testlib.shell import Shell +from frostfs_testlib.utils import json_utils logger = logging.getLogger("NeoLogger") @@ -613,22 +613,22 @@ def head_object( # If response is Complex Object header, it has `splitId` key if "splitId" in decoded.keys(): logger.info("decoding split header") - return json_transformers.decode_split_header(decoded) + return json_utils.decode_split_header(decoded) # If response is Last or Linking Object header, # it has `header` dictionary and non-null `split` dictionary if "split" in decoded["header"].keys(): if decoded["header"]["split"]: logger.info("decoding linking object") - return json_transformers.decode_linking_object(decoded) + return json_utils.decode_linking_object(decoded) if decoded["header"]["objectType"] == "STORAGE_GROUP": logger.info("decoding storage group") - return json_transformers.decode_storage_group(decoded) + return json_utils.decode_storage_group(decoded) if decoded["header"]["objectType"] == "TOMBSTONE": logger.info("decoding tombstone") - return json_transformers.decode_tombstone(decoded) + return json_utils.decode_tombstone(decoded) logger.info("decoding simple header") - return json_transformers.decode_simple_header(decoded) + return json_utils.decode_simple_header(decoded) diff --git a/robot/resources/lib/python_keywords/json_transformers.py b/robot/resources/lib/python_keywords/json_transformers.py deleted file mode 100644 index ac4ba561..00000000 --- a/robot/resources/lib/python_keywords/json_transformers.py +++ /dev/null @@ -1,136 +0,0 @@ -""" - When doing requests to FrostFS, we get JSON output as an automatically decoded - structure from protobuf. Some fields are decoded with boilerplates and binary - values are Base64-encoded. - - This module contains functions which rearrange the structure and reencode binary - data from Base64 to Base58. -""" - -import base64 - -import base58 - - -def decode_simple_header(data: dict) -> dict: - """ - This function reencodes Simple Object header and its attributes. - """ - try: - data = decode_common_fields(data) - - # Normalize object attributes - data["header"]["attributes"] = { - attr["key"]: attr["value"] for attr in data["header"]["attributes"] - } - except Exception as exc: - raise ValueError(f"failed to decode JSON output: {exc}") from exc - - return data - - -def decode_split_header(data: dict) -> dict: - """ - This function rearranges Complex Object header. - The header holds SplitID, a random unique - number, which is common among all splitted objects, and IDs of the Linking - Object and the last splitted Object. - """ - try: - data["splitId"] = json_reencode(data["splitId"]) - data["lastPart"] = json_reencode(data["lastPart"]["value"]) if data["lastPart"] else None - data["link"] = json_reencode(data["link"]["value"]) if data["link"] else None - except Exception as exc: - raise ValueError(f"failed to decode JSON output: {exc}") from exc - - return data - - -def decode_linking_object(data: dict) -> dict: - """ - This function reencodes Linking Object header. - It contains IDs of child Objects and Split Chain data. - """ - try: - data = decode_simple_header(data) - split = data["header"]["split"] - split["children"] = [json_reencode(item["value"]) for item in split["children"]] - split["splitID"] = json_reencode(split["splitID"]) - split["previous"] = json_reencode(split["previous"]["value"]) if split["previous"] else None - split["parent"] = json_reencode(split["parent"]["value"]) if split["parent"] else None - except Exception as exc: - raise ValueError(f"failed to decode JSON output: {exc}") from exc - - return data - - -def decode_storage_group(data: dict) -> dict: - """ - This function reencodes Storage Group header. - """ - try: - data = decode_common_fields(data) - except Exception as exc: - raise ValueError(f"failed to decode JSON output: {exc}") from exc - - return data - - -def decode_tombstone(data: dict) -> dict: - """ - This function reencodes Tombstone header. - """ - try: - data = decode_simple_header(data) - data["header"]["sessionToken"] = decode_session_token(data["header"]["sessionToken"]) - except Exception as exc: - raise ValueError(f"failed to decode JSON output: {exc}") from exc - return data - - -def decode_session_token(data: dict) -> dict: - """ - This function reencodes a fragment of header which contains - information about session token. - """ - target = data["body"]["object"]["target"] - target["container"] = json_reencode(target["container"]["value"]) - target["objects"] = [json_reencode(obj["value"]) for obj in target["objects"]] - return data - - -def json_reencode(data: str) -> str: - """ - According to JSON protocol, binary data (Object/Container/Storage Group IDs, etc) - is converted to string via Base58 encoder. But we usually operate with Base64-encoded format. - This function reencodes given Base58 string into the Base64 one. - """ - return base58.b58encode(base64.b64decode(data)).decode("utf-8") - - -def encode_for_json(data: str) -> str: - """ - This function encodes binary data for sending them as protobuf - structures. - """ - return base64.b64encode(base58.b58decode(data)).decode("utf-8") - - -def decode_common_fields(data: dict) -> dict: - """ - Despite of type (simple/complex Object, Storage Group, etc) every Object - header contains several common fields. - This function rearranges these fields. - """ - data["objectID"] = json_reencode(data["objectID"]["value"]) - - header = data["header"] - header["containerID"] = json_reencode(header["containerID"]["value"]) - header["ownerID"] = json_reencode(header["ownerID"]["value"]) - header["payloadHash"] = json_reencode(header["payloadHash"]["sum"]) - header["version"] = f"{header['version']['major']}{header['version']['minor']}" - # Homomorphic hash is optional and its calculation might be disabled in trusted network - if header.get("homomorphicHash"): - header["homomorphicHash"] = json_reencode(header["homomorphicHash"]["sum"]) - - return data diff --git a/robot/resources/lib/python_keywords/node_management.py b/robot/resources/lib/python_keywords/node_management.py index 2a708805..efd10b7f 100644 --- a/robot/resources/lib/python_keywords/node_management.py +++ b/robot/resources/lib/python_keywords/node_management.py @@ -11,7 +11,7 @@ from common import FROSTFS_CLI_EXEC, MORPH_BLOCK_TIME from epoch import tick_epoch from frostfs_testlib.cli import FrostfsCli from frostfs_testlib.shell import Shell -from utility import parse_time +from frostfs_testlib.utils import datetime_utils logger = logging.getLogger("NeoLogger") @@ -154,7 +154,7 @@ def drop_object(node: StorageNode, cid: str, oid: str) -> str: def delete_node_data(node: StorageNode) -> None: node.stop_service() node.host.delete_storage_node_data(node.name) - time.sleep(parse_time(MORPH_BLOCK_TIME)) + time.sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME)) @allure.step("Exclude node {node_to_exclude} from network map") @@ -168,7 +168,7 @@ def exclude_node_from_network_map( storage_node_set_status(node_to_exclude, status="offline") - time.sleep(parse_time(MORPH_BLOCK_TIME)) + time.sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME)) tick_epoch(shell, cluster) snapshot = get_netmap_snapshot(node=alive_node, shell=shell) @@ -189,9 +189,9 @@ def include_node_to_network_map( # Per suggestion of @fyrchik we need to wait for 2 blocks after we set status and after tick epoch. # First sleep can be omitted after https://github.com/nspcc-dev/frostfs-node/issues/1790 complete. - time.sleep(parse_time(MORPH_BLOCK_TIME) * 2) + time.sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME) * 2) tick_epoch(shell, cluster) - time.sleep(parse_time(MORPH_BLOCK_TIME) * 2) + time.sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME) * 2) check_node_in_map(node_to_include, shell, alive_node) diff --git a/robot/resources/lib/python_keywords/object_access.py b/robot/resources/lib/python_keywords/object_access.py index dcbd5557..ba599716 100644 --- a/robot/resources/lib/python_keywords/object_access.py +++ b/robot/resources/lib/python_keywords/object_access.py @@ -5,7 +5,7 @@ from cluster import Cluster from file_helper import get_file_hash from frostfs_testlib.resources.common import OBJECT_ACCESS_DENIED from frostfs_testlib.shell import Shell -from frostfs_testlib.utils.errors import error_matches_status +from frostfs_testlib.utils import string_utils from python_keywords.frostfs_verbs import ( delete_object, get_object_from_random_node, @@ -43,7 +43,7 @@ def can_get_object( cluster=cluster, ) except OPERATION_ERROR_TYPE as err: - assert error_matches_status( + assert string_utils.is_str_match_pattern( err, OBJECT_ACCESS_DENIED ), f"Expected {err} to match {OBJECT_ACCESS_DENIED}" return False @@ -76,7 +76,7 @@ def can_put_object( cluster=cluster, ) except OPERATION_ERROR_TYPE as err: - assert error_matches_status( + assert string_utils.is_str_match_pattern( err, OBJECT_ACCESS_DENIED ), f"Expected {err} to match {OBJECT_ACCESS_DENIED}" return False @@ -106,7 +106,7 @@ def can_delete_object( endpoint=endpoint, ) except OPERATION_ERROR_TYPE as err: - assert error_matches_status( + assert string_utils.is_str_match_pattern( err, OBJECT_ACCESS_DENIED ), f"Expected {err} to match {OBJECT_ACCESS_DENIED}" return False @@ -136,7 +136,7 @@ def can_get_head_object( endpoint=endpoint, ) except OPERATION_ERROR_TYPE as err: - assert error_matches_status( + assert string_utils.is_str_match_pattern( err, OBJECT_ACCESS_DENIED ), f"Expected {err} to match {OBJECT_ACCESS_DENIED}" return False @@ -167,7 +167,7 @@ def can_get_range_of_object( endpoint=endpoint, ) except OPERATION_ERROR_TYPE as err: - assert error_matches_status( + assert string_utils.is_str_match_pattern( err, OBJECT_ACCESS_DENIED ), f"Expected {err} to match {OBJECT_ACCESS_DENIED}" return False @@ -198,7 +198,7 @@ def can_get_range_hash_of_object( endpoint=endpoint, ) except OPERATION_ERROR_TYPE as err: - assert error_matches_status( + assert string_utils.is_str_match_pattern( err, OBJECT_ACCESS_DENIED ), f"Expected {err} to match {OBJECT_ACCESS_DENIED}" return False @@ -227,7 +227,7 @@ def can_search_object( endpoint=endpoint, ) except OPERATION_ERROR_TYPE as err: - assert error_matches_status( + assert string_utils.is_str_match_pattern( err, OBJECT_ACCESS_DENIED ), f"Expected {err} to match {OBJECT_ACCESS_DENIED}" return False diff --git a/robot/resources/lib/python_keywords/payment_neogo.py b/robot/resources/lib/python_keywords/payment_neogo.py index 4efcd528..b65474a4 100644 --- a/robot/resources/lib/python_keywords/payment_neogo.py +++ b/robot/resources/lib/python_keywords/payment_neogo.py @@ -10,11 +10,9 @@ from cluster import MainChain, MorphChain from common import FROSTFS_CONTRACT, GAS_HASH, MAINNET_BLOCK_TIME, NEOGO_EXECUTABLE from frostfs_testlib.cli import NeoGo from frostfs_testlib.shell import Shell -from frostfs_testlib.utils.converters import contract_hash_to_address -from frostfs_testlib.utils.wallet import get_last_address_from_wallet +from frostfs_testlib.utils import converting_utils, datetime_utils, wallet_utils from neo3.wallet import utils as neo3_utils from neo3.wallet import wallet as neo3_wallet -from utility import parse_time logger = logging.getLogger("NeoLogger") @@ -43,7 +41,7 @@ def get_contract_hash(morph_chain: MorphChain, resolve_name: str, shell: Shell) @allure.step("Withdraw Mainnet Gas") def withdraw_mainnet_gas(shell: Shell, main_chain: MainChain, wlt: str, amount: int): - address = get_last_address_from_wallet(wlt, EMPTY_PASSWORD) + address = wallet_utils.get_last_address_from_wallet(wlt, EMPTY_PASSWORD) scripthash = neo3_utils.address_to_script_hash(address) neogo = NeoGo(shell=shell, neo_go_exec_path=NEOGO_EXECUTABLE) @@ -142,10 +140,12 @@ def transfer_gas( if wallet_from_password is not None else main_chain.get_wallet_password() ) - address_from = address_from or get_last_address_from_wallet( + address_from = address_from or wallet_utils.get_last_address_from_wallet( wallet_from_path, wallet_from_password ) - address_to = address_to or get_last_address_from_wallet(wallet_to_path, wallet_to_password) + address_to = address_to or wallet_utils.get_last_address_from_wallet( + wallet_to_path, wallet_to_password + ) neogo = NeoGo(shell, neo_go_exec_path=NEOGO_EXECUTABLE) out = neogo.nep17.transfer( @@ -163,7 +163,7 @@ def transfer_gas( raise Exception("Got no TXID after run the command") if not transaction_accepted(main_chain, txid): raise AssertionError(f"TX {txid} hasn't been processed") - time.sleep(parse_time(MAINNET_BLOCK_TIME)) + time.sleep(datetime_utils.parse_time(MAINNET_BLOCK_TIME)) @allure.step("FrostFS Deposit") @@ -178,9 +178,9 @@ def deposit_gas( Transferring GAS from given wallet to FrostFS contract address. """ # get FrostFS contract address - deposit_addr = contract_hash_to_address(FROSTFS_CONTRACT) + deposit_addr = converting_utils.contract_hash_to_address(FROSTFS_CONTRACT) logger.info(f"FrostFS contract address: {deposit_addr}") - address_from = get_last_address_from_wallet( + address_from = wallet_utils.get_last_address_from_wallet( wallet_path=wallet_from_path, wallet_password=wallet_from_password ) transfer_gas( diff --git a/robot/resources/lib/python_keywords/storage_policy.py b/robot/resources/lib/python_keywords/storage_policy.py index 267bdd80..a2d2a09d 100644 --- a/robot/resources/lib/python_keywords/storage_policy.py +++ b/robot/resources/lib/python_keywords/storage_policy.py @@ -13,7 +13,7 @@ import frostfs_verbs from cluster import StorageNode from frostfs_testlib.resources.common import OBJECT_NOT_FOUND from frostfs_testlib.shell import Shell -from frostfs_testlib.utils.errors import error_matches_status +from frostfs_testlib.utils import string_utils logger = logging.getLogger("NeoLogger") @@ -166,7 +166,7 @@ def get_nodes_without_object( if res is None: nodes_list.append(node) except Exception as err: - if error_matches_status(err, OBJECT_NOT_FOUND): + if string_utils.is_str_match_pattern(err, OBJECT_NOT_FOUND): nodes_list.append(node) else: raise Exception(f"Got error {err} on head object command") from err