TrueCloudLab/frostfs-testcases
25
0
Fork 21
Code Issues 2 Pull requests 1 Releases Activity Actions 7

Remove common erros to testlib #8

Merged
alexchetaev merged 4 commits from feature/remove_common_erros_to_testlib into master 2023-02-27 10:44:20 +00:00
Conversation 0 Commits 4 Files changed 45 +78 -288
21 changed files with 78 additions and 288 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files
Showing only changes of commit 901c390cb9 - Show all commits

4
pytest_tests/helpers/cluster.py
View file

@ -3,11 +3,11 @@ import re
from dataclasses import dataclass
from typing import Any
import data_formatters
import yaml
from frostfs_testlib.blockchain import RPCClient
from frostfs_testlib.hosting import Host, Hosting
from frostfs_testlib.hosting.config import ServiceConfig
from frostfs_testlib.utils import wallet_utils
@dataclass
@ -85,7 +85,7 @@ class NodeBase:
def get_wallet_public_key(self):
storage_wallet_path = self.get_wallet_path()
storage_wallet_pass = self.get_wallet_password()
return data_formatters.get_wallet_public_key(storage_wallet_path, storage_wallet_pass)
return wallet_utils.get_wallet_public_key(storage_wallet_path, storage_wallet_pass)
def _get_attribute(self, attribute_name: str, default_attribute_name: str = None) -> list[str]:
config = self.host.get_service_config(self.name)

29
pytest_tests/helpers/utility.py
View file

@ -2,32 +2,7 @@ import time
import allure
from common import STORAGE_GC_TIME
def parse_time(value: str) -> int:
"""Converts time interval in text form into time interval as number of seconds.
Args:
value: time interval as text.
Returns:
Number of seconds in the parsed time interval.
"""
value = value.lower()
for suffix in ["s", "sec"]:
if value.endswith(suffix):
return int(value[: -len(suffix)])
for suffix in ["m", "min"]:
if value.endswith(suffix):
return int(value[: -len(suffix)]) * 60
for suffix in ["h", "hr", "hour"]:
if value.endswith(suffix):
return int(value[: -len(suffix)]) * 60 * 60
raise ValueError(f"Unknown units in time value '{value}'")
from frostfs_testlib.utils import datetime_utils
def placement_policy_from_container(container_info: str) -> str:
@ -57,6 +32,6 @@ def placement_policy_from_container(container_info: str) -> str:
def wait_for_gc_pass_on_storage_nodes() -> None:
wait_time = parse_time(STORAGE_GC_TIME)
wait_time = datetime_utils.parse_time(STORAGE_GC_TIME)
with allure.step(f"Wait {wait_time}s until GC completes on storage nodes"):
time.sleep(wait_time)

6
pytest_tests/helpers/wallet.py
View file

@ -5,7 +5,7 @@ from dataclasses import dataclass
from cluster import Cluster, NodeBase
from common import FREE_STORAGE, WALLET_CONFIG, WALLET_PASS
from frostfs_testlib.shell import Shell
from frostfs_testlib.utils.wallet import get_last_address_from_wallet, init_wallet
from frostfs_testlib.utils import wallet_utils
from python_keywords.payment_neogo import deposit_gas, transfer_gas
@ -28,7 +28,7 @@ class WalletFile:
Returns:
The address of the wallet.
"""
return get_last_address_from_wallet(self.path, self.password)
return wallet_utils.get_last_address_from_wallet(self.path, self.password)
class WalletFactory:
@ -47,7 +47,7 @@ class WalletFactory:
WalletFile object of new wallet
"""
wallet_path = os.path.join(self.wallets_dir, f"{str(uuid.uuid4())}.json")
init_wallet(wallet_path, password)
wallet_utils.init_wallet(wallet_path, password)
if not FREE_STORAGE:
main_chain = self.cluster.main_chain_nodes[0]

22
pytest_tests/steps/session_token.py
View file

@ -8,12 +8,10 @@ from enum import Enum
from typing import Any, Optional
import allure
import json_transformers
from common import ASSETS_DIR, FROSTFS_CLI_EXEC, WALLET_CONFIG
from data_formatters import get_wallet_public_key
from frostfs_testlib.cli import FrostfsCli
from frostfs_testlib.shell import Shell
from json_transformers import encode_for_json
from frostfs_testlib.utils import json_utils, wallet_utils
from storage_object_info import StorageObjectInfo
from wallet import WalletFile
@ -71,16 +69,16 @@ def generate_session_token(
file_path = os.path.join(tokens_dir, str(uuid.uuid4()))
pub_key_64 = get_wallet_public_key(session_wallet.path, session_wallet.password, "base64")
pub_key_64 = wallet_utils.get_wallet_public_key(
session_wallet.path, session_wallet.password, "base64"
)
lifetime = lifetime or Lifetime()
session_token = {
"body": {
"id": f"{base64.b64encode(uuid.uuid4().bytes).decode('utf-8')}",
"ownerID": {
"value": f"{json_transformers.encode_for_json(owner_wallet.get_address())}"
},
"ownerID": {"value": f"{json_utils.encode_for_json(owner_wallet.get_address())}"},
"lifetime": {
"exp": f"{lifetime.exp}",
"nbf": f"{lifetime.nbf}",
@ -125,7 +123,11 @@ def generate_container_session_token(
"container": {
"verb": verb.value,
"wildcard": cid is None,
**({"containerID": {"value": f"{encode_for_json(cid)}"}} if cid is not None else {}),
**(
{"containerID": {"value": f"{json_utils.encode_for_json(cid)}"}}
if cid is not None
else {}
),
},
}
@ -165,8 +167,8 @@ def generate_object_session_token(
"object": {
"verb": verb.value,
"target": {
"container": {"value": encode_for_json(cid)},
"objects": [{"value": encode_for_json(oid)} for oid in oids],
"container": {"value": json_utils.encode_for_json(cid)},
"objects": [{"value": json_utils.encode_for_json(oid)} for oid in oids],
},
},
}

4
pytest_tests/testsuites/acl/conftest.py
View file

@ -10,7 +10,7 @@ from common import WALLET_CONFIG, WALLET_PASS
from file_helper import generate_file
from frostfs_testlib.resources.common import PUBLIC_ACL
from frostfs_testlib.shell import Shell
from frostfs_testlib.utils.wallet import init_wallet
from frostfs_testlib.utils import wallet_utils
from python_keywords.acl import EACLRole
from python_keywords.container import create_container
from python_keywords.frostfs_verbs import put_object_to_random_node
@ -41,7 +41,7 @@ def wallets(default_wallet, temp_directory, cluster: Cluster) -> Wallets:
os.path.join(temp_directory, f"{str(uuid.uuid4())}.json") for _ in range(2)
]
for other_wallet_path in other_wallets_paths:
init_wallet(other_wallet_path, WALLET_PASS)
wallet_utils.init_wallet(other_wallet_path, WALLET_PASS)
ir_node = cluster.ir_nodes[0]
storage_node = cluster.storage_nodes[0]

4
pytest_tests/testsuites/acl/storage_group/test_storagegroup.py
View file

@ -9,7 +9,7 @@ from cluster_test_base import ClusterTestBase
from common import ASSETS_DIR, FREE_STORAGE, WALLET_PASS
from file_helper import generate_file
from frostfs_testlib.resources.common import OBJECT_ACCESS_DENIED, OBJECT_NOT_FOUND
from frostfs_testlib.utils.wallet import init_wallet
from frostfs_testlib.utils import wallet_utils
from python_keywords.acl import (
EACLAccess,
EACLOperation,
@ -48,7 +48,7 @@ class TestStorageGroup(ClusterTestBase):
def prepare_two_wallets(self, default_wallet):
self.main_wallet = default_wallet
self.other_wallet = os.path.join(os.getcwd(), ASSETS_DIR, f"{str(uuid.uuid4())}.json")
init_wallet(self.other_wallet, WALLET_PASS)
wallet_utils.init_wallet(self.other_wallet, WALLET_PASS)
if not FREE_STORAGE:
main_chain = self.cluster.main_chain_nodes[0]
deposit = 30

4
pytest_tests/testsuites/conftest.py
View file

@ -24,7 +24,7 @@ from env_properties import save_env_properties
from frostfs_testlib.hosting import Hosting
from frostfs_testlib.reporter import AllureHandler, get_reporter
from frostfs_testlib.shell import LocalShell, Shell
from frostfs_testlib.utils.wallet import init_wallet
from frostfs_testlib.utils import wallet_utils
from k6 import LoadParams
from load import get_services_endpoints, prepare_k6_instances
from load_params import (
@ -242,7 +242,7 @@ def background_grpc_load(client_shell: Shell, hosting: Hosting):
@allure.title("Prepare wallet and deposit")
def default_wallet(client_shell: Shell, temp_directory: str, cluster: Cluster):
wallet_path = os.path.join(os.getcwd(), ASSETS_DIR, f"{str(uuid.uuid4())}.json")
init_wallet(wallet_path, WALLET_PASS)
wallet_utils.init_wallet(wallet_path, WALLET_PASS)
allure.attach.file(wallet_path, os.path.basename(wallet_path), allure.attachment_type.JSON)
if not FREE_STORAGE:

12
pytest_tests/testsuites/network/test_node_management.py
View file

@ -11,7 +11,7 @@ from common import FROSTFS_CONTRACT_CACHE_TIMEOUT, MORPH_BLOCK_TIME
from epoch import tick_epoch
from file_helper import generate_file
from frostfs_testlib.resources.common import OBJECT_NOT_FOUND, PUBLIC_ACL
from frostfs_testlib.utils.errors import error_matches_status
from frostfs_testlib.utils import datetime_utils, string_utils
from python_keywords.container import create_container, get_container
from python_keywords.failover_utils import wait_object_replication
from python_keywords.frostfs_verbs import (
@ -35,7 +35,7 @@ from python_keywords.node_management import (
storage_node_set_status,
)
from storage_policy import get_nodes_with_object, get_simple_object_copies
from utility import parse_time, placement_policy_from_container, wait_for_gc_pass_on_storage_nodes
from utility import placement_policy_from_container, wait_for_gc_pass_on_storage_nodes
logger = logging.getLogger("NeoLogger")
check_nodes: list[StorageNode] = []
@ -109,13 +109,13 @@ class TestNodeManagement(ClusterTestBase):
# We need to wait for node to establish notifications from morph-chain
# Otherwise it will hang up when we will try to set status
sleep(parse_time(MORPH_BLOCK_TIME))
sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME))
with allure.step(f"Move node {node} to online state"):
storage_node_set_status(node, status="online", retries=2)
check_nodes.remove(node)
sleep(parse_time(MORPH_BLOCK_TIME))
sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME))
self.tick_epoch_with_retries(3)
check_node_in_map(node, shell=self.shell, alive_node=alive_node)
@ -472,7 +472,7 @@ class TestNodeManagement(ClusterTestBase):
if copies == expected_copies:
break
tick_epoch(self.shell, self.cluster)
sleep(parse_time(FROSTFS_CONTRACT_CACHE_TIMEOUT))
sleep(datetime_utils.parse_time(FROSTFS_CONTRACT_CACHE_TIMEOUT))
else:
raise AssertionError(f"There are no {expected_copies} copies during time")
@ -483,7 +483,7 @@ class TestNodeManagement(ClusterTestBase):
checker(wallet, cid, oid, shell=self.shell, endpoint=endpoint)
wait_for_gc_pass_on_storage_nodes()
except Exception as err:
if error_matches_status(err, OBJECT_NOT_FOUND):
if string_utils.is_str_match_pattern(err, OBJECT_NOT_FOUND):
return
raise AssertionError(f'Expected "{OBJECT_NOT_FOUND}" error, got\n{err}')

5
pytest_tests/testsuites/object/test_object_lock.py
View file

@ -18,13 +18,14 @@ from frostfs_testlib.resources.common import (
OBJECT_NOT_FOUND,
)
from frostfs_testlib.shell import Shell
from frostfs_testlib.utils import datetime_utils
from node_management import drop_object
from pytest import FixtureRequest
from python_keywords.container import create_container
from python_keywords.frostfs_verbs import delete_object, head_object, lock_object
from storage_policy import get_nodes_with_object
from test_control import expect_not_raises, wait_for_success
from utility import parse_time, wait_for_gc_pass_on_storage_nodes
from utility import wait_for_gc_pass_on_storage_nodes
from helpers.container import StorageContainer, StorageContainerInfo
from helpers.storage_object_info import LockObjectInfo, StorageObjectInfo
@ -321,7 +322,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
self.cluster.default_rpc_endpoint,
)
@wait_for_success(parse_time(STORAGE_GC_TIME))
@wait_for_success(datetime_utils.parse_time(STORAGE_GC_TIME))
def check_object_not_found():
with pytest.raises(Exception, match=OBJECT_NOT_FOUND):
head_object(

9
pytest_tests/testsuites/services/s3_gate/test_s3_object.py
View file

@ -8,9 +8,8 @@ import allure
import pytest
from aws_cli_client import AwsCliClient
from common import ASSETS_DIR, FREE_STORAGE, WALLET_PASS
from data_formatters import get_wallet_public_key
from file_helper import concat_files, generate_file, generate_file_with_content, get_file_hash
from frostfs_testlib.utils.wallet import init_wallet
from frostfs_testlib.utils import wallet_utils
from python_keywords.payment_neogo import deposit_gas, transfer_gas
from s3_helper import (
assert_object_lock_mode,
@ -661,10 +660,10 @@ class TestS3GateObject(TestS3GateBase):
@pytest.fixture
def prepare_two_wallets(self, default_wallet, client_shell):
self.main_wallet = default_wallet
self.main_public_key = get_wallet_public_key(self.main_wallet, WALLET_PASS)
self.main_public_key = wallet_utils.get_wallet_public_key(self.main_wallet, WALLET_PASS)
self.other_wallet = os.path.join(os.getcwd(), ASSETS_DIR, f"{str(uuid.uuid4())}.json")
init_wallet(self.other_wallet, WALLET_PASS)
self.other_public_key = get_wallet_public_key(self.other_wallet, WALLET_PASS)
wallet_utils.init_wallet(self.other_wallet, WALLET_PASS)
self.other_public_key = wallet_utils.get_wallet_public_key(self.other_wallet, WALLET_PASS)
if not FREE_STORAGE:
main_chain = self.cluster.main_chain_nodes[0]

4
pytest_tests/testsuites/session_token/test_object_session_token.py
View file

@ -6,7 +6,7 @@ from cluster_test_base import ClusterTestBase
from common import WALLET_PASS
from file_helper import generate_file
from frostfs_testlib.resources.common import SESSION_NOT_FOUND
from frostfs_testlib.utils.wallet import get_last_address_from_wallet
from frostfs_testlib.utils import wallet_utils
from python_keywords.container import create_container
from python_keywords.frostfs_verbs import delete_object, put_object, put_object_to_random_node
@ -38,7 +38,7 @@ class TestDynamicObjectSession(ClusterTestBase):
with allure.step("Init wallet"):
wallet = default_wallet
address = get_last_address_from_wallet(wallet, "")
address = wallet_utils.get_last_address_from_wallet(wallet, "")
with allure.step("Nodes Settlements"):
(

6
robot/resources/lib/python_keywords/acl.py
View file

@ -11,9 +11,9 @@ from typing import Any, Dict, List, Optional, Union
import allure
import base58
from common import ASSETS_DIR, FROSTFS_CLI_EXEC, WALLET_CONFIG
from data_formatters import get_wallet_public_key
from frostfs_testlib.cli import FrostfsCli
from frostfs_testlib.shell import Shell
from frostfs_testlib.utils import wallet_utils
logger = logging.getLogger("NeoLogger")
EACL_LIFETIME = 100500
@ -110,7 +110,7 @@ class EACLRule:
role = (
self.role.value
if isinstance(self.role, EACLRole)
else f'pubkey:{get_wallet_public_key(self.role, "")}'
else f'pubkey:{wallet_utils.get_wallet_public_key(self.role, "")}'
)
return f'{self.access.value} {self.operation.value} {self.filters or ""} {role}'
@ -244,7 +244,7 @@ def eacl_rules(access: str, verbs: list, user: str) -> list[str]:
(list): a list of eACL rules
"""
if user not in ("others", "user"):
pubkey = get_wallet_public_key(user, wallet_password="")
pubkey = wallet_utils.get_wallet_public_key(user, wallet_password="")
user = f"pubkey:{pubkey}"
rules = []

4
robot/resources/lib/python_keywords/container.py
View file

@ -10,10 +10,10 @@ from time import sleep
from typing import Optional, Union
import allure
import json_transformers
from common import FROSTFS_CLI_EXEC, WALLET_CONFIG
from frostfs_testlib.cli import FrostfsCli
from frostfs_testlib.shell import Shell
from frostfs_testlib.utils import json_utils
logger = logging.getLogger("NeoLogger")
@ -164,7 +164,7 @@ def get_container(
for attr in container_info["attributes"]:
attributes[attr["key"]] = attr["value"]
container_info["attributes"] = attributes
container_info["ownerID"] = json_transformers.json_reencode(container_info["ownerID"]["value"])
container_info["ownerID"] = json_utils.json_reencode(container_info["ownerID"]["value"])
return container_info

50
robot/resources/lib/python_keywords/data_formatters.py
View file

@ -1,50 +0,0 @@
import base64
import json
import base58
from neo3.wallet import wallet
def dict_to_attrs(attrs: dict) -> str:
"""
This function takes a dictionary of object's attributes and converts them
into string. The string is passed to `--attributes` key of frostfs-cli.
Args:
attrs (dict): object attributes in {"a": "b", "c": "d"} format.
Returns:
(str): string in "a=b,c=d" format.
"""
return ",".join(f"{key}={value}" for key, value in attrs.items())
def __fix_wallet_schema(wallet: dict) -> None:
# Temporary function to fix wallets that do not conform to the schema
# TODO: get rid of it once issue is solved
if "name" not in wallet:
wallet["name"] = None
for account in wallet["accounts"]:
if "extra" not in account:
account["extra"] = None
def get_wallet_public_key(wallet_path: str, wallet_password: str, format: str = "hex") -> str:
# Get public key from wallet file
with open(wallet_path, "r") as file:
wallet_content = json.load(file)
__fix_wallet_schema(wallet_content)
wallet_from_json = wallet.Wallet.from_json(wallet_content, password=wallet_password)
public_key_hex = str(wallet_from_json.accounts[0].public_key)
# Convert public key to specified format
if format == "hex":
return public_key_hex
if format == "base58":
public_key_base58 = base58.b58encode(bytes.fromhex(public_key_hex))
return public_key_base58.decode("utf-8")
if format == "base64":
public_key_base64 = base64.b64encode(bytes.fromhex(public_key_hex))
return public_key_base64.decode("utf-8")
raise ValueError(f"Invalid public key format: {format}")

7
robot/resources/lib/python_keywords/epoch.py
View file

@ -13,10 +13,9 @@ from common import (
)
from frostfs_testlib.cli import FrostfsAdm, FrostfsCli, NeoGo
from frostfs_testlib.shell import Shell
from frostfs_testlib.utils.wallet import get_last_address_from_wallet
from frostfs_testlib.utils import datetime_utils, wallet_utils
from payment_neogo import get_contract_hash
from test_control import wait_for_success
from utility import parse_time
logger = logging.getLogger("NeoLogger")
@ -90,7 +89,7 @@ def tick_epoch(shell: Shell, cluster: Cluster, alive_node: Optional[StorageNode]
# In case if no local_wallet_path is provided, we use wallet_path
ir_wallet_path = ir_node.get_wallet_path()
ir_wallet_pass = ir_node.get_wallet_password()
ir_address = get_last_address_from_wallet(ir_wallet_path, ir_wallet_pass)
ir_address = wallet_utils.get_last_address_from_wallet(ir_wallet_path, ir_wallet_pass)
morph_chain = cluster.morph_chain_nodes[0]
morph_endpoint = morph_chain.get_endpoint()
@ -108,4 +107,4 @@ def tick_epoch(shell: Shell, cluster: Cluster, alive_node: Optional[StorageNode]
force=True,
gas=1,
)
sleep(parse_time(MAINNET_BLOCK_TIME))
sleep(datetime_utils.parse_time(MAINNET_BLOCK_TIME))

12
robot/resources/lib/python_keywords/frostfs_verbs.py
View file

@ -6,11 +6,11 @@ import uuid
from typing import Any, Optional
import allure
import json_transformers
from cluster import Cluster
from common import ASSETS_DIR, FROSTFS_CLI_EXEC, WALLET_CONFIG
from frostfs_testlib.cli import FrostfsCli
from frostfs_testlib.shell import Shell
from frostfs_testlib.utils import json_utils
logger = logging.getLogger("NeoLogger")
@ -613,22 +613,22 @@ def head_object(
# If response is Complex Object header, it has `splitId` key
if "splitId" in decoded.keys():
logger.info("decoding split header")
return json_transformers.decode_split_header(decoded)
return json_utils.decode_split_header(decoded)
# If response is Last or Linking Object header,
# it has `header` dictionary and non-null `split` dictionary
if "split" in decoded["header"].keys():
if decoded["header"]["split"]:
logger.info("decoding linking object")
return json_transformers.decode_linking_object(decoded)
return json_utils.decode_linking_object(decoded)
if decoded["header"]["objectType"] == "STORAGE_GROUP":
logger.info("decoding storage group")
return json_transformers.decode_storage_group(decoded)
return json_utils.decode_storage_group(decoded)
if decoded["header"]["objectType"] == "TOMBSTONE":
logger.info("decoding tombstone")
return json_transformers.decode_tombstone(decoded)
return json_utils.decode_tombstone(decoded)
logger.info("decoding simple header")
return json_transformers.decode_simple_header(decoded)
return json_utils.decode_simple_header(decoded)

136
robot/resources/lib/python_keywords/json_transformers.py
View file

@ -1,136 +0,0 @@
"""
When doing requests to FrostFS, we get JSON output as an automatically decoded
structure from protobuf. Some fields are decoded with boilerplates and binary
values are Base64-encoded.
This module contains functions which rearrange the structure and reencode binary
data from Base64 to Base58.
"""
import base64
import base58
def decode_simple_header(data: dict) -> dict:
"""
This function reencodes Simple Object header and its attributes.
"""
try:
data = decode_common_fields(data)
# Normalize object attributes
data["header"]["attributes"] = {
attr["key"]: attr["value"] for attr in data["header"]["attributes"]
}
except Exception as exc:
raise ValueError(f"failed to decode JSON output: {exc}") from exc
return data
def decode_split_header(data: dict) -> dict:
"""
This function rearranges Complex Object header.
The header holds SplitID, a random unique
number, which is common among all splitted objects, and IDs of the Linking
Object and the last splitted Object.
"""
try:
data["splitId"] = json_reencode(data["splitId"])
data["lastPart"] = json_reencode(data["lastPart"]["value"]) if data["lastPart"] else None
data["link"] = json_reencode(data["link"]["value"]) if data["link"] else None
except Exception as exc:
raise ValueError(f"failed to decode JSON output: {exc}") from exc
return data
def decode_linking_object(data: dict) -> dict:
"""
This function reencodes Linking Object header.
It contains IDs of child Objects and Split Chain data.
"""
try:
data = decode_simple_header(data)
split = data["header"]["split"]
split["children"] = [json_reencode(item["value"]) for item in split["children"]]
split["splitID"] = json_reencode(split["splitID"])
split["previous"] = json_reencode(split["previous"]["value"]) if split["previous"] else None
split["parent"] = json_reencode(split["parent"]["value"]) if split["parent"] else None
except Exception as exc:
raise ValueError(f"failed to decode JSON output: {exc}") from exc
return data
def decode_storage_group(data: dict) -> dict:
"""
This function reencodes Storage Group header.
"""
try:
data = decode_common_fields(data)
except Exception as exc:
raise ValueError(f"failed to decode JSON output: {exc}") from exc
return data
def decode_tombstone(data: dict) -> dict:
"""
This function reencodes Tombstone header.
"""
try:
data = decode_simple_header(data)
data["header"]["sessionToken"] = decode_session_token(data["header"]["sessionToken"])
except Exception as exc:
raise ValueError(f"failed to decode JSON output: {exc}") from exc
return data
def decode_session_token(data: dict) -> dict:
"""
This function reencodes a fragment of header which contains
information about session token.
"""
target = data["body"]["object"]["target"]
target["container"] = json_reencode(target["container"]["value"])
target["objects"] = [json_reencode(obj["value"]) for obj in target["objects"]]
return data
def json_reencode(data: str) -> str:
"""
According to JSON protocol, binary data (Object/Container/Storage Group IDs, etc)
is converted to string via Base58 encoder. But we usually operate with Base64-encoded format.
This function reencodes given Base58 string into the Base64 one.
"""
return base58.b58encode(base64.b64decode(data)).decode("utf-8")
def encode_for_json(data: str) -> str:
"""
This function encodes binary data for sending them as protobuf
structures.
"""
return base64.b64encode(base58.b58decode(data)).decode("utf-8")
def decode_common_fields(data: dict) -> dict:
"""
Despite of type (simple/complex Object, Storage Group, etc) every Object
header contains several common fields.
This function rearranges these fields.
"""
data["objectID"] = json_reencode(data["objectID"]["value"])
header = data["header"]
header["containerID"] = json_reencode(header["containerID"]["value"])
header["ownerID"] = json_reencode(header["ownerID"]["value"])
header["payloadHash"] = json_reencode(header["payloadHash"]["sum"])
header["version"] = f"{header['version']['major']}{header['version']['minor']}"
# Homomorphic hash is optional and its calculation might be disabled in trusted network
if header.get("homomorphicHash"):
header["homomorphicHash"] = json_reencode(header["homomorphicHash"]["sum"])
return data

10
robot/resources/lib/python_keywords/node_management.py
View file

@ -11,7 +11,7 @@ from common import FROSTFS_CLI_EXEC, MORPH_BLOCK_TIME
from epoch import tick_epoch
from frostfs_testlib.cli import FrostfsCli
from frostfs_testlib.shell import Shell
from utility import parse_time
from frostfs_testlib.utils import datetime_utils
logger = logging.getLogger("NeoLogger")
@ -154,7 +154,7 @@ def drop_object(node: StorageNode, cid: str, oid: str) -> str:
def delete_node_data(node: StorageNode) -> None:
node.stop_service()
node.host.delete_storage_node_data(node.name)
time.sleep(parse_time(MORPH_BLOCK_TIME))
time.sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME))
@allure.step("Exclude node {node_to_exclude} from network map")
@ -168,7 +168,7 @@ def exclude_node_from_network_map(
storage_node_set_status(node_to_exclude, status="offline")
time.sleep(parse_time(MORPH_BLOCK_TIME))
time.sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME))
tick_epoch(shell, cluster)
snapshot = get_netmap_snapshot(node=alive_node, shell=shell)
@ -189,9 +189,9 @@ def include_node_to_network_map(
# Per suggestion of @fyrchik we need to wait for 2 blocks after we set status and after tick epoch.
# First sleep can be omitted after https://github.com/nspcc-dev/frostfs-node/issues/1790 complete.
time.sleep(parse_time(MORPH_BLOCK_TIME) * 2)
time.sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME) * 2)
tick_epoch(shell, cluster)
time.sleep(parse_time(MORPH_BLOCK_TIME) * 2)
time.sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME) * 2)
check_node_in_map(node_to_include, shell, alive_node)

16
robot/resources/lib/python_keywords/object_access.py
View file

@ -5,7 +5,7 @@ from cluster import Cluster
from file_helper import get_file_hash
from frostfs_testlib.resources.common import OBJECT_ACCESS_DENIED
from frostfs_testlib.shell import Shell
from frostfs_testlib.utils.errors import error_matches_status
from frostfs_testlib.utils import string_utils
from python_keywords.frostfs_verbs import (
delete_object,
get_object_from_random_node,
@ -43,7 +43,7 @@ def can_get_object(
cluster=cluster,
)
except OPERATION_ERROR_TYPE as err:
assert error_matches_status(
assert string_utils.is_str_match_pattern(
err, OBJECT_ACCESS_DENIED
), f"Expected {err} to match {OBJECT_ACCESS_DENIED}"
return False
@ -76,7 +76,7 @@ def can_put_object(
cluster=cluster,
)
except OPERATION_ERROR_TYPE as err:
assert error_matches_status(
assert string_utils.is_str_match_pattern(
err, OBJECT_ACCESS_DENIED
), f"Expected {err} to match {OBJECT_ACCESS_DENIED}"
return False
@ -106,7 +106,7 @@ def can_delete_object(
endpoint=endpoint,
)
except OPERATION_ERROR_TYPE as err:
assert error_matches_status(
assert string_utils.is_str_match_pattern(
err, OBJECT_ACCESS_DENIED
), f"Expected {err} to match {OBJECT_ACCESS_DENIED}"
return False
@ -136,7 +136,7 @@ def can_get_head_object(
endpoint=endpoint,
)
except OPERATION_ERROR_TYPE as err:
assert error_matches_status(
assert string_utils.is_str_match_pattern(
err, OBJECT_ACCESS_DENIED
), f"Expected {err} to match {OBJECT_ACCESS_DENIED}"
return False
@ -167,7 +167,7 @@ def can_get_range_of_object(
endpoint=endpoint,
)
except OPERATION_ERROR_TYPE as err:
assert error_matches_status(
assert string_utils.is_str_match_pattern(
err, OBJECT_ACCESS_DENIED
), f"Expected {err} to match {OBJECT_ACCESS_DENIED}"
return False
@ -198,7 +198,7 @@ def can_get_range_hash_of_object(
endpoint=endpoint,
)
except OPERATION_ERROR_TYPE as err:
assert error_matches_status(
assert string_utils.is_str_match_pattern(
err, OBJECT_ACCESS_DENIED
), f"Expected {err} to match {OBJECT_ACCESS_DENIED}"
return False
@ -227,7 +227,7 @@ def can_search_object(
endpoint=endpoint,
)
except OPERATION_ERROR_TYPE as err:
assert error_matches_status(
assert string_utils.is_str_match_pattern(
err, OBJECT_ACCESS_DENIED
), f"Expected {err} to match {OBJECT_ACCESS_DENIED}"
return False

18
robot/resources/lib/python_keywords/payment_neogo.py
View file

@ -10,11 +10,9 @@ from cluster import MainChain, MorphChain
from common import FROSTFS_CONTRACT, GAS_HASH, MAINNET_BLOCK_TIME, NEOGO_EXECUTABLE
from frostfs_testlib.cli import NeoGo
from frostfs_testlib.shell import Shell
from frostfs_testlib.utils.converters import contract_hash_to_address
from frostfs_testlib.utils.wallet import get_last_address_from_wallet
from frostfs_testlib.utils import converting_utils, datetime_utils, wallet_utils
from neo3.wallet import utils as neo3_utils
from neo3.wallet import wallet as neo3_wallet
from utility import parse_time
logger = logging.getLogger("NeoLogger")
@ -43,7 +41,7 @@ def get_contract_hash(morph_chain: MorphChain, resolve_name: str, shell: Shell)
@allure.step("Withdraw Mainnet Gas")
def withdraw_mainnet_gas(shell: Shell, main_chain: MainChain, wlt: str, amount: int):
address = get_last_address_from_wallet(wlt, EMPTY_PASSWORD)
address = wallet_utils.get_last_address_from_wallet(wlt, EMPTY_PASSWORD)
scripthash = neo3_utils.address_to_script_hash(address)
neogo = NeoGo(shell=shell, neo_go_exec_path=NEOGO_EXECUTABLE)
@ -142,10 +140,12 @@ def transfer_gas(
if wallet_from_password is not None
else main_chain.get_wallet_password()
)
address_from = address_from or get_last_address_from_wallet(
address_from = address_from or wallet_utils.get_last_address_from_wallet(
wallet_from_path, wallet_from_password
)
address_to = address_to or get_last_address_from_wallet(wallet_to_path, wallet_to_password)
address_to = address_to or wallet_utils.get_last_address_from_wallet(
wallet_to_path, wallet_to_password
)
neogo = NeoGo(shell, neo_go_exec_path=NEOGO_EXECUTABLE)
out = neogo.nep17.transfer(
@ -163,7 +163,7 @@ def transfer_gas(
raise Exception("Got no TXID after run the command")
if not transaction_accepted(main_chain, txid):
raise AssertionError(f"TX {txid} hasn't been processed")
time.sleep(parse_time(MAINNET_BLOCK_TIME))
time.sleep(datetime_utils.parse_time(MAINNET_BLOCK_TIME))
@allure.step("FrostFS Deposit")
@ -178,9 +178,9 @@ def deposit_gas(
Transferring GAS from given wallet to FrostFS contract address.
"""
# get FrostFS contract address
deposit_addr = contract_hash_to_address(FROSTFS_CONTRACT)
deposit_addr = converting_utils.contract_hash_to_address(FROSTFS_CONTRACT)
logger.info(f"FrostFS contract address: {deposit_addr}")
address_from = get_last_address_from_wallet(
address_from = wallet_utils.get_last_address_from_wallet(
wallet_path=wallet_from_path, wallet_password=wallet_from_password
)
transfer_gas(

4
robot/resources/lib/python_keywords/storage_policy.py
View file

@ -13,7 +13,7 @@ import frostfs_verbs
from cluster import StorageNode
from frostfs_testlib.resources.common import OBJECT_NOT_FOUND
from frostfs_testlib.shell import Shell
from frostfs_testlib.utils.errors import error_matches_status
from frostfs_testlib.utils import string_utils
logger = logging.getLogger("NeoLogger")
@ -166,7 +166,7 @@ def get_nodes_without_object(
if res is None:
nodes_list.append(node)
except Exception as err:
if error_matches_status(err, OBJECT_NOT_FOUND):
if string_utils.is_str_match_pattern(err, OBJECT_NOT_FOUND):
nodes_list.append(node)
else:
raise Exception(f"Got error {err} on head object command") from err