dstepanov-yadro/frostfs-testcases
1
0
Fork 0
forked from TrueCloudLab/frostfs-testcases
Code Issues Pull requests Projects Releases Packages Wiki Activity

Fixing imports after move utils ti frostfs-testlib

Browse source 
Signed-off-by: Aleksei Chetaev <alex.chetaev@gmail.com>
This commit is contained in:
Aleksei Chetaev 2023-02-20 00:58:07 +01:00 committed by Aleksey Chetaev
parent d253e8f5fd
commit 13bc98eecc
21 changed files with 78 additions and 288 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
pytest_tests/helpers/cluster.py
View file

@ -3,11 +3,11 @@ import re
from dataclasses import dataclass
from typing import Any
import data_formatters
import yaml
from frostfs_testlib.blockchain import RPCClient
from frostfs_testlib.hosting import Host, Hosting
from frostfs_testlib.hosting.config import ServiceConfig
from frostfs_testlib.utils import wallet_utils
@dataclass
@ -85,7 +85,7 @@ class NodeBase:
def get_wallet_public_key(self):
storage_wallet_path = self.get_wallet_path()
storage_wallet_pass = self.get_wallet_password()
return data_formatters.get_wallet_public_key(storage_wallet_path, storage_wallet_pass)
return wallet_utils.get_wallet_public_key(storage_wallet_path, storage_wallet_pass)
def _get_attribute(self, attribute_name: str, default_attribute_name: str = None) -> list[str]:
config = self.host.get_service_config(self.name)

29
pytest_tests/helpers/utility.py
View file

@ -2,32 +2,7 @@ import time
import allure
from common import STORAGE_GC_TIME
def parse_time(value: str) -> int:
"""Converts time interval in text form into time interval as number of seconds.
Args:
value: time interval as text.
Returns:
Number of seconds in the parsed time interval.
"""
value = value.lower()
for suffix in ["s", "sec"]:
if value.endswith(suffix):
return int(value[: -len(suffix)])
for suffix in ["m", "min"]:
if value.endswith(suffix):
return int(value[: -len(suffix)]) * 60
for suffix in ["h", "hr", "hour"]:
if value.endswith(suffix):
return int(value[: -len(suffix)]) * 60 * 60
raise ValueError(f"Unknown units in time value '{value}'")
from frostfs_testlib.utils import datetime_utils
def placement_policy_from_container(container_info: str) -> str:
@ -57,6 +32,6 @@ def placement_policy_from_container(container_info: str) -> str:
def wait_for_gc_pass_on_storage_nodes() -> None:
wait_time = parse_time(STORAGE_GC_TIME)
wait_time = datetime_utils.parse_time(STORAGE_GC_TIME)
with allure.step(f"Wait {wait_time}s until GC completes on storage nodes"):
time.sleep(wait_time)

6
pytest_tests/helpers/wallet.py
View file

@ -5,7 +5,7 @@ from dataclasses import dataclass
from cluster import Cluster, NodeBase
from common import FREE_STORAGE, WALLET_CONFIG, WALLET_PASS
from frostfs_testlib.shell import Shell
from frostfs_testlib.utils.wallet import get_last_address_from_wallet, init_wallet
from frostfs_testlib.utils import wallet_utils
from python_keywords.payment_neogo import deposit_gas, transfer_gas
@ -28,7 +28,7 @@ class WalletFile:
Returns:
The address of the wallet.
"""
return get_last_address_from_wallet(self.path, self.password)
return wallet_utils.get_last_address_from_wallet(self.path, self.password)
class WalletFactory:
@ -47,7 +47,7 @@ class WalletFactory:
WalletFile object of new wallet
"""
wallet_path = os.path.join(self.wallets_dir, f"{str(uuid.uuid4())}.json")
init_wallet(wallet_path, password)
wallet_utils.init_wallet(wallet_path, password)
if not FREE_STORAGE:
main_chain = self.cluster.main_chain_nodes[0]

22
pytest_tests/steps/session_token.py
View file

@ -8,12 +8,10 @@ from enum import Enum
from typing import Any, Optional
import allure
import json_transformers
from common import ASSETS_DIR, FROSTFS_CLI_EXEC, WALLET_CONFIG
from data_formatters import get_wallet_public_key
from frostfs_testlib.cli import FrostfsCli
from frostfs_testlib.shell import Shell
from json_transformers import encode_for_json
from frostfs_testlib.utils import json_utils, wallet_utils
from storage_object_info import StorageObjectInfo
from wallet import WalletFile
@ -71,16 +69,16 @@ def generate_session_token(
file_path = os.path.join(tokens_dir, str(uuid.uuid4()))
pub_key_64 = get_wallet_public_key(session_wallet.path, session_wallet.password, "base64")
pub_key_64 = wallet_utils.get_wallet_public_key(
session_wallet.path, session_wallet.password, "base64"
)
lifetime = lifetime or Lifetime()
session_token = {
"body": {
"id": f"{base64.b64encode(uuid.uuid4().bytes).decode('utf-8')}",
"ownerID": {
"value": f"{json_transformers.encode_for_json(owner_wallet.get_address())}"
},
"ownerID": {"value": f"{json_utils.encode_for_json(owner_wallet.get_address())}"},
"lifetime": {
"exp": f"{lifetime.exp}",
"nbf": f"{lifetime.nbf}",
@ -125,7 +123,11 @@ def generate_container_session_token(
"container": {
"verb": verb.value,
"wildcard": cid is None,
**({"containerID": {"value": f"{encode_for_json(cid)}"}} if cid is not None else {}),
**(
{"containerID": {"value": f"{json_utils.encode_for_json(cid)}"}}
if cid is not None
else {}
),
},
}
@ -165,8 +167,8 @@ def generate_object_session_token(
"object": {
"verb": verb.value,
"target": {
"container": {"value": encode_for_json(cid)},
"objects": [{"value": encode_for_json(oid)} for oid in oids],
"container": {"value": json_utils.encode_for_json(cid)},
"objects": [{"value": json_utils.encode_for_json(oid)} for oid in oids],
},
},
}

4
pytest_tests/testsuites/acl/conftest.py
View file

@ -10,7 +10,7 @@ from common import WALLET_CONFIG, WALLET_PASS
from file_helper import generate_file
from frostfs_testlib.resources.common import PUBLIC_ACL
from frostfs_testlib.shell import Shell
from frostfs_testlib.utils.wallet import init_wallet
from frostfs_testlib.utils import wallet_utils
from python_keywords.acl import EACLRole
from python_keywords.container import create_container
from python_keywords.frostfs_verbs import put_object_to_random_node
@ -41,7 +41,7 @@ def wallets(default_wallet, temp_directory, cluster: Cluster) -> Wallets:
os.path.join(temp_directory, f"{str(uuid.uuid4())}.json") for _ in range(2)
]
for other_wallet_path in other_wallets_paths:
init_wallet(other_wallet_path, WALLET_PASS)
wallet_utils.init_wallet(other_wallet_path, WALLET_PASS)
ir_node = cluster.ir_nodes[0]
storage_node = cluster.storage_nodes[0]

4
pytest_tests/testsuites/acl/storage_group/test_storagegroup.py
View file

@ -9,7 +9,7 @@ from cluster_test_base import ClusterTestBase
from common import ASSETS_DIR, FREE_STORAGE, WALLET_PASS
from file_helper import generate_file
from frostfs_testlib.resources.common import OBJECT_ACCESS_DENIED, OBJECT_NOT_FOUND
from frostfs_testlib.utils.wallet import init_wallet
from frostfs_testlib.utils import wallet_utils
from python_keywords.acl import (
EACLAccess,
EACLOperation,
@ -48,7 +48,7 @@ class TestStorageGroup(ClusterTestBase):
def prepare_two_wallets(self, default_wallet):
self.main_wallet = default_wallet
self.other_wallet = os.path.join(os.getcwd(), ASSETS_DIR, f"{str(uuid.uuid4())}.json")
init_wallet(self.other_wallet, WALLET_PASS)
wallet_utils.init_wallet(self.other_wallet, WALLET_PASS)
if not FREE_STORAGE:
main_chain = self.cluster.main_chain_nodes[0]
deposit = 30

4
pytest_tests/testsuites/conftest.py
View file

@ -24,7 +24,7 @@ from env_properties import save_env_properties
from frostfs_testlib.hosting import Hosting
from frostfs_testlib.reporter import AllureHandler, get_reporter
from frostfs_testlib.shell import LocalShell, Shell
from frostfs_testlib.utils.wallet import init_wallet
from frostfs_testlib.utils import wallet_utils
from k6 import LoadParams
from load import get_services_endpoints, prepare_k6_instances
from load_params import (
@ -242,7 +242,7 @@ def background_grpc_load(client_shell: Shell, hosting: Hosting):
@allure.title("Prepare wallet and deposit")
def default_wallet(client_shell: Shell, temp_directory: str, cluster: Cluster):
wallet_path = os.path.join(os.getcwd(), ASSETS_DIR, f"{str(uuid.uuid4())}.json")
init_wallet(wallet_path, WALLET_PASS)
wallet_utils.init_wallet(wallet_path, WALLET_PASS)
allure.attach.file(wallet_path, os.path.basename(wallet_path), allure.attachment_type.JSON)
if not FREE_STORAGE:

12
pytest_tests/testsuites/network/test_node_management.py
View file

@ -11,7 +11,7 @@ from common import FROSTFS_CONTRACT_CACHE_TIMEOUT, MORPH_BLOCK_TIME
from epoch import tick_epoch
from file_helper import generate_file
from frostfs_testlib.resources.common import OBJECT_NOT_FOUND, PUBLIC_ACL
from frostfs_testlib.utils.errors import error_matches_status
from frostfs_testlib.utils import datetime_utils, string_utils
from python_keywords.container import create_container, get_container
from python_keywords.failover_utils import wait_object_replication
from python_keywords.frostfs_verbs import (
@ -35,7 +35,7 @@ from python_keywords.node_management import (
storage_node_set_status,
)
from storage_policy import get_nodes_with_object, get_simple_object_copies
from utility import parse_time, placement_policy_from_container, wait_for_gc_pass_on_storage_nodes
from utility import placement_policy_from_container, wait_for_gc_pass_on_storage_nodes
logger = logging.getLogger("NeoLogger")
check_nodes: list[StorageNode] = []
@ -109,13 +109,13 @@ class TestNodeManagement(ClusterTestBase):
# We need to wait for node to establish notifications from morph-chain
# Otherwise it will hang up when we will try to set status
sleep(parse_time(MORPH_BLOCK_TIME))
sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME))
with allure.step(f"Move node {node} to online state"):
storage_node_set_status(node, status="online", retries=2)
check_nodes.remove(node)
sleep(parse_time(MORPH_BLOCK_TIME))
sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME))
self.tick_epoch_with_retries(3)
check_node_in_map(node, shell=self.shell, alive_node=alive_node)
@ -472,7 +472,7 @@ class TestNodeManagement(ClusterTestBase):
if copies == expected_copies:
break
tick_epoch(self.shell, self.cluster)
sleep(parse_time(FROSTFS_CONTRACT_CACHE_TIMEOUT))
sleep(datetime_utils.parse_time(FROSTFS_CONTRACT_CACHE_TIMEOUT))
else:
raise AssertionError(f"There are no {expected_copies} copies during time")
@ -483,7 +483,7 @@ class TestNodeManagement(ClusterTestBase):
checker(wallet, cid, oid, shell=self.shell, endpoint=endpoint)
wait_for_gc_pass_on_storage_nodes()
except Exception as err:
if error_matches_status(err, OBJECT_NOT_FOUND):
if string_utils.is_str_match_pattern(err, OBJECT_NOT_FOUND):
return
raise AssertionError(f'Expected "{OBJECT_NOT_FOUND}" error, got\n{err}')

5
pytest_tests/testsuites/object/test_object_lock.py
View file

@ -18,13 +18,14 @@ from frostfs_testlib.resources.common import (
OBJECT_NOT_FOUND,
)
from frostfs_testlib.shell import Shell
from frostfs_testlib.utils import datetime_utils
from node_management import drop_object
from pytest import FixtureRequest
from python_keywords.container import create_container
from python_keywords.frostfs_verbs import delete_object, head_object, lock_object
from storage_policy import get_nodes_with_object
from test_control import expect_not_raises, wait_for_success
from utility import parse_time, wait_for_gc_pass_on_storage_nodes
from utility import wait_for_gc_pass_on_storage_nodes
from helpers.container import StorageContainer, StorageContainerInfo
from helpers.storage_object_info import LockObjectInfo, StorageObjectInfo
@ -321,7 +322,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
self.cluster.default_rpc_endpoint,
)
@wait_for_success(parse_time(STORAGE_GC_TIME))
@wait_for_success(datetime_utils.parse_time(STORAGE_GC_TIME))
def check_object_not_found():
with pytest.raises(Exception, match=OBJECT_NOT_FOUND):
head_object(

9
pytest_tests/testsuites/services/s3_gate/test_s3_object.py
View file

@ -8,9 +8,8 @@ import allure
import pytest
from aws_cli_client import AwsCliClient
from common import ASSETS_DIR, FREE_STORAGE, WALLET_PASS
from data_formatters import get_wallet_public_key
from file_helper import concat_files, generate_file, generate_file_with_content, get_file_hash
from frostfs_testlib.utils.wallet import init_wallet
from frostfs_testlib.utils import wallet_utils
from python_keywords.payment_neogo import deposit_gas, transfer_gas
from s3_helper import (
assert_object_lock_mode,
@ -661,10 +660,10 @@ class TestS3GateObject(TestS3GateBase):
@pytest.fixture
def prepare_two_wallets(self, default_wallet, client_shell):
self.main_wallet = default_wallet
self.main_public_key = get_wallet_public_key(self.main_wallet, WALLET_PASS)
self.main_public_key = wallet_utils.get_wallet_public_key(self.main_wallet, WALLET_PASS)
self.other_wallet = os.path.join(os.getcwd(), ASSETS_DIR, f"{str(uuid.uuid4())}.json")
init_wallet(self.other_wallet, WALLET_PASS)
self.other_public_key = get_wallet_public_key(self.other_wallet, WALLET_PASS)
wallet_utils.init_wallet(self.other_wallet, WALLET_PASS)
self.other_public_key = wallet_utils.get_wallet_public_key(self.other_wallet, WALLET_PASS)
if not FREE_STORAGE:
main_chain = self.cluster.main_chain_nodes[0]

4
pytest_tests/testsuites/session_token/test_object_session_token.py
View file

@ -6,7 +6,7 @@ from cluster_test_base import ClusterTestBase
from common import WALLET_PASS
from file_helper import generate_file
from frostfs_testlib.resources.common import SESSION_NOT_FOUND
from frostfs_testlib.utils.wallet import get_last_address_from_wallet
from frostfs_testlib.utils import wallet_utils
from python_keywords.container import create_container
from python_keywords.frostfs_verbs import delete_object, put_object, put_object_to_random_node
@ -38,7 +38,7 @@ class TestDynamicObjectSession(ClusterTestBase):
with allure.step("Init wallet"):
wallet = default_wallet
address = get_last_address_from_wallet(wallet, "")
address = wallet_utils.get_last_address_from_wallet(wallet, "")
with allure.step("Nodes Settlements"):
(

6
robot/resources/lib/python_keywords/acl.py
View file

@ -11,9 +11,9 @@ from typing import Any, Dict, List, Optional, Union
import allure
import base58
from common import ASSETS_DIR, FROSTFS_CLI_EXEC, WALLET_CONFIG
from data_formatters import get_wallet_public_key
from frostfs_testlib.cli import FrostfsCli
from frostfs_testlib.shell import Shell
from frostfs_testlib.utils import wallet_utils
logger = logging.getLogger("NeoLogger")
EACL_LIFETIME = 100500
@ -110,7 +110,7 @@ class EACLRule:
role = (
self.role.value
if isinstance(self.role, EACLRole)
else f'pubkey:{get_wallet_public_key(self.role, "")}'
else f'pubkey:{wallet_utils.get_wallet_public_key(self.role, "")}'
)
return f'{self.access.value} {self.operation.value} {self.filters or ""} {role}'
@ -244,7 +244,7 @@ def eacl_rules(access: str, verbs: list, user: str) -> list[str]:
(list): a list of eACL rules
"""
if user not in ("others", "user"):
pubkey = get_wallet_public_key(user, wallet_password="")
pubkey = wallet_utils.get_wallet_public_key(user, wallet_password="")
user = f"pubkey:{pubkey}"
rules = []

4
robot/resources/lib/python_keywords/container.py
View file

@ -10,10 +10,10 @@ from time import sleep
from typing import Optional, Union
import allure
import json_transformers
from common import FROSTFS_CLI_EXEC, WALLET_CONFIG
from frostfs_testlib.cli import FrostfsCli
from frostfs_testlib.shell import Shell
from frostfs_testlib.utils import json_utils
logger = logging.getLogger("NeoLogger")
@ -164,7 +164,7 @@ def get_container(
for attr in container_info["attributes"]:
attributes[attr["key"]] = attr["value"]
container_info["attributes"] = attributes
container_info["ownerID"] = json_transformers.json_reencode(container_info["ownerID"]["value"])
container_info["ownerID"] = json_utils.json_reencode(container_info["ownerID"]["value"])
return container_info

50
robot/resources/lib/python_keywords/data_formatters.py
View file

@ -1,50 +0,0 @@
import base64
import json
import base58
from neo3.wallet import wallet
def dict_to_attrs(attrs: dict) -> str:
"""
This function takes a dictionary of object's attributes and converts them
into string. The string is passed to `--attributes` key of frostfs-cli.
Args:
attrs (dict): object attributes in {"a": "b", "c": "d"} format.
Returns:
(str): string in "a=b,c=d" format.
"""
return ",".join(f"{key}={value}" for key, value in attrs.items())
def __fix_wallet_schema(wallet: dict) -> None:
# Temporary function to fix wallets that do not conform to the schema
# TODO: get rid of it once issue is solved
if "name" not in wallet:
wallet["name"] = None
for account in wallet["accounts"]:
if "extra" not in account:
account["extra"] = None
def get_wallet_public_key(wallet_path: str, wallet_password: str, format: str = "hex") -> str:
# Get public key from wallet file
with open(wallet_path, "r") as file:
wallet_content = json.load(file)
__fix_wallet_schema(wallet_content)
wallet_from_json = wallet.Wallet.from_json(wallet_content, password=wallet_password)
public_key_hex = str(wallet_from_json.accounts[0].public_key)
# Convert public key to specified format
if format == "hex":
return public_key_hex
if format == "base58":
public_key_base58 = base58.b58encode(bytes.fromhex(public_key_hex))
return public_key_base58.decode("utf-8")
if format == "base64":
public_key_base64 = base64.b64encode(bytes.fromhex(public_key_hex))
return public_key_base64.decode("utf-8")
raise ValueError(f"Invalid public key format: {format}")

7
robot/resources/lib/python_keywords/epoch.py
View file

@ -13,10 +13,9 @@ from common import (
)
from frostfs_testlib.cli import FrostfsAdm, FrostfsCli, NeoGo
from frostfs_testlib.shell import Shell
from frostfs_testlib.utils.wallet import get_last_address_from_wallet
from frostfs_testlib.utils import datetime_utils, wallet_utils
from payment_neogo import get_contract_hash
from test_control import wait_for_success
from utility import parse_time
logger = logging.getLogger("NeoLogger")
@ -90,7 +89,7 @@ def tick_epoch(shell: Shell, cluster: Cluster, alive_node: Optional[StorageNode]
# In case if no local_wallet_path is provided, we use wallet_path
ir_wallet_path = ir_node.get_wallet_path()
ir_wallet_pass = ir_node.get_wallet_password()
ir_address = get_last_address_from_wallet(ir_wallet_path, ir_wallet_pass)
ir_address = wallet_utils.get_last_address_from_wallet(ir_wallet_path, ir_wallet_pass)
morph_chain = cluster.morph_chain_nodes[0]
morph_endpoint = morph_chain.get_endpoint()
@ -108,4 +107,4 @@ def tick_epoch(shell: Shell, cluster: Cluster, alive_node: Optional[StorageNode]
force=True,
gas=1,
)
sleep(parse_time(MAINNET_BLOCK_TIME))
sleep(datetime_utils.parse_time(MAINNET_BLOCK_TIME))

12
robot/resources/lib/python_keywords/frostfs_verbs.py
View file

@ -6,11 +6,11 @@ import uuid
from typing import Any, Optional
import allure
import json_transformers
from cluster import Cluster
from common import ASSETS_DIR, FROSTFS_CLI_EXEC, WALLET_CONFIG
from frostfs_testlib.cli import FrostfsCli
from frostfs_testlib.shell import Shell
from frostfs_testlib.utils import json_utils
logger = logging.getLogger("NeoLogger")
@ -613,22 +613,22 @@ def head_object(
# If response is Complex Object header, it has `splitId` key
if "splitId" in decoded.keys():
logger.info("decoding split header")
return json_transformers.decode_split_header(decoded)
return json_utils.decode_split_header(decoded)
# If response is Last or Linking Object header,
# it has `header` dictionary and non-null `split` dictionary
if "split" in decoded["header"].keys():
if decoded["header"]["split"]:
logger.info("decoding linking object")
return json_transformers.decode_linking_object(decoded)
return json_utils.decode_linking_object(decoded)
if decoded["header"]["objectType"] == "STORAGE_GROUP":
logger.info("decoding storage group")
return json_transformers.decode_storage_group(decoded)
return json_utils.decode_storage_group(decoded)
if decoded["header"]["objectType"] == "TOMBSTONE":
logger.info("decoding tombstone")
return json_transformers.decode_tombstone(decoded)
return json_utils.decode_tombstone(decoded)
logger.info("decoding simple header")
return json_transformers.decode_simple_header(decoded)
return json_utils.decode_simple_header(decoded)

136
robot/resources/lib/python_keywords/json_transformers.py
View file

@ -1,136 +0,0 @@
"""
When doing requests to FrostFS, we get JSON output as an automatically decoded
structure from protobuf. Some fields are decoded with boilerplates and binary
values are Base64-encoded.
This module contains functions which rearrange the structure and reencode binary
data from Base64 to Base58.
"""
import base64
import base58
def decode_simple_header(data: dict) -> dict:
"""
This function reencodes Simple Object header and its attributes.
"""
try:
data = decode_common_fields(data)
# Normalize object attributes
data["header"]["attributes"] = {
attr["key"]: attr["value"] for attr in data["header"]["attributes"]
}
except Exception as exc:
raise ValueError(f"failed to decode JSON output: {exc}") from exc
return data
def decode_split_header(data: dict) -> dict:
"""
This function rearranges Complex Object header.
The header holds SplitID, a random unique
number, which is common among all splitted objects, and IDs of the Linking
Object and the last splitted Object.
"""
try:
data["splitId"] = json_reencode(data["splitId"])
data["lastPart"] = json_reencode(data["lastPart"]["value"]) if data["lastPart"] else None
data["link"] = json_reencode(data["link"]["value"]) if data["link"] else None
except Exception as exc:
raise ValueError(f"failed to decode JSON output: {exc}") from exc
return data
def decode_linking_object(data: dict) -> dict:
"""
This function reencodes Linking Object header.
It contains IDs of child Objects and Split Chain data.
"""
try:
data = decode_simple_header(data)
split = data["header"]["split"]
split["children"] = [json_reencode(item["value"]) for item in split["children"]]
split["splitID"] = json_reencode(split["splitID"])
split["previous"] = json_reencode(split["previous"]["value"]) if split["previous"] else None
split["parent"] = json_reencode(split["parent"]["value"]) if split["parent"] else None
except Exception as exc:
raise ValueError(f"failed to decode JSON output: {exc}") from exc
return data
def decode_storage_group(data: dict) -> dict:
"""
This function reencodes Storage Group header.
"""
try:
data = decode_common_fields(data)
except Exception as exc:
raise ValueError(f"failed to decode JSON output: {exc}") from exc
return data
def decode_tombstone(data: dict) -> dict:
"""
This function reencodes Tombstone header.
"""
try:
data = decode_simple_header(data)
data["header"]["sessionToken"] = decode_session_token(data["header"]["sessionToken"])
except Exception as exc:
raise ValueError(f"failed to decode JSON output: {exc}") from exc
return data
def decode_session_token(data: dict) -> dict:
"""
This function reencodes a fragment of header which contains
information about session token.
"""
target = data["body"]["object"]["target"]
target["container"] = json_reencode(target["container"]["value"])
target["objects"] = [json_reencode(obj["value"]) for obj in target["objects"]]
return data
def json_reencode(data: str) -> str:
"""
According to JSON protocol, binary data (Object/Container/Storage Group IDs, etc)
is converted to string via Base58 encoder. But we usually operate with Base64-encoded format.
This function reencodes given Base58 string into the Base64 one.
"""
return base58.b58encode(base64.b64decode(data)).decode("utf-8")
def encode_for_json(data: str) -> str:
"""
This function encodes binary data for sending them as protobuf
structures.
"""
return base64.b64encode(base58.b58decode(data)).decode("utf-8")
def decode_common_fields(data: dict) -> dict:
"""
Despite of type (simple/complex Object, Storage Group, etc) every Object
header contains several common fields.
This function rearranges these fields.
"""
data["objectID"] = json_reencode(data["objectID"]["value"])
header = data["header"]
header["containerID"] = json_reencode(header["containerID"]["value"])
header["ownerID"] = json_reencode(header["ownerID"]["value"])
header["payloadHash"] = json_reencode(header["payloadHash"]["sum"])
header["version"] = f"{header['version']['major']}{header['version']['minor']}"
# Homomorphic hash is optional and its calculation might be disabled in trusted network
if header.get("homomorphicHash"):
header["homomorphicHash"] = json_reencode(header["homomorphicHash"]["sum"])
return data

10
robot/resources/lib/python_keywords/node_management.py
View file

@ -11,7 +11,7 @@ from common import FROSTFS_CLI_EXEC, MORPH_BLOCK_TIME
from epoch import tick_epoch
from frostfs_testlib.cli import FrostfsCli
from frostfs_testlib.shell import Shell
from utility import parse_time
from frostfs_testlib.utils import datetime_utils
logger = logging.getLogger("NeoLogger")
@ -154,7 +154,7 @@ def drop_object(node: StorageNode, cid: str, oid: str) -> str:
def delete_node_data(node: StorageNode) -> None:
node.stop_service()
node.host.delete_storage_node_data(node.name)
time.sleep(parse_time(MORPH_BLOCK_TIME))
time.sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME))
@allure.step("Exclude node {node_to_exclude} from network map")
@ -168,7 +168,7 @@ def exclude_node_from_network_map(
storage_node_set_status(node_to_exclude, status="offline")
time.sleep(parse_time(MORPH_BLOCK_TIME))
time.sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME))
tick_epoch(shell, cluster)
snapshot = get_netmap_snapshot(node=alive_node, shell=shell)
@ -189,9 +189,9 @@ def include_node_to_network_map(
# Per suggestion of @fyrchik we need to wait for 2 blocks after we set status and after tick epoch.
# First sleep can be omitted after https://github.com/nspcc-dev/frostfs-node/issues/1790 complete.
time.sleep(parse_time(MORPH_BLOCK_TIME) * 2)
time.sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME) * 2)
tick_epoch(shell, cluster)
time.sleep(parse_time(MORPH_BLOCK_TIME) * 2)
time.sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME) * 2)
check_node_in_map(node_to_include, shell, alive_node)

16
robot/resources/lib/python_keywords/object_access.py
View file

@ -5,7 +5,7 @@ from cluster import Cluster
from file_helper import get_file_hash
from frostfs_testlib.resources.common import OBJECT_ACCESS_DENIED
from frostfs_testlib.shell import Shell
from frostfs_testlib.utils.errors import error_matches_status
from frostfs_testlib.utils import string_utils
from python_keywords.frostfs_verbs import (
delete_object,
get_object_from_random_node,
@ -43,7 +43,7 @@ def can_get_object(
cluster=cluster,
)
except OPERATION_ERROR_TYPE as err:
assert error_matches_status(
assert string_utils.is_str_match_pattern(
err, OBJECT_ACCESS_DENIED
), f"Expected {err} to match {OBJECT_ACCESS_DENIED}"
return False
@ -76,7 +76,7 @@ def can_put_object(
cluster=cluster,
)
except OPERATION_ERROR_TYPE as err:
assert error_matches_status(
assert string_utils.is_str_match_pattern(
err, OBJECT_ACCESS_DENIED
), f"Expected {err} to match {OBJECT_ACCESS_DENIED}"
return False
@ -106,7 +106,7 @@ def can_delete_object(
endpoint=endpoint,
)
except OPERATION_ERROR_TYPE as err:
assert error_matches_status(
assert string_utils.is_str_match_pattern(
err, OBJECT_ACCESS_DENIED
), f"Expected {err} to match {OBJECT_ACCESS_DENIED}"
return False
@ -136,7 +136,7 @@ def can_get_head_object(
endpoint=endpoint,
)
except OPERATION_ERROR_TYPE as err:
assert error_matches_status(
assert string_utils.is_str_match_pattern(
err, OBJECT_ACCESS_DENIED
), f"Expected {err} to match {OBJECT_ACCESS_DENIED}"
return False
@ -167,7 +167,7 @@ def can_get_range_of_object(
endpoint=endpoint,
)
except OPERATION_ERROR_TYPE as err:
assert error_matches_status(
assert string_utils.is_str_match_pattern(
err, OBJECT_ACCESS_DENIED
), f"Expected {err} to match {OBJECT_ACCESS_DENIED}"
return False
@ -198,7 +198,7 @@ def can_get_range_hash_of_object(
endpoint=endpoint,
)
except OPERATION_ERROR_TYPE as err:
assert error_matches_status(
assert string_utils.is_str_match_pattern(
err, OBJECT_ACCESS_DENIED
), f"Expected {err} to match {OBJECT_ACCESS_DENIED}"
return False
@ -227,7 +227,7 @@ def can_search_object(
endpoint=endpoint,
)
except OPERATION_ERROR_TYPE as err:
assert error_matches_status(
assert string_utils.is_str_match_pattern(
err, OBJECT_ACCESS_DENIED
), f"Expected {err} to match {OBJECT_ACCESS_DENIED}"
return False

18
robot/resources/lib/python_keywords/payment_neogo.py
View file

@ -10,11 +10,9 @@ from cluster import MainChain, MorphChain
from common import FROSTFS_CONTRACT, GAS_HASH, MAINNET_BLOCK_TIME, NEOGO_EXECUTABLE
from frostfs_testlib.cli import NeoGo
from frostfs_testlib.shell import Shell
from frostfs_testlib.utils.converters import contract_hash_to_address
from frostfs_testlib.utils.wallet import get_last_address_from_wallet
from frostfs_testlib.utils import converting_utils, datetime_utils, wallet_utils
from neo3.wallet import utils as neo3_utils
from neo3.wallet import wallet as neo3_wallet
from utility import parse_time
logger = logging.getLogger("NeoLogger")
@ -43,7 +41,7 @@ def get_contract_hash(morph_chain: MorphChain, resolve_name: str, shell: Shell)
@allure.step("Withdraw Mainnet Gas")
def withdraw_mainnet_gas(shell: Shell, main_chain: MainChain, wlt: str, amount: int):
address = get_last_address_from_wallet(wlt, EMPTY_PASSWORD)
address = wallet_utils.get_last_address_from_wallet(wlt, EMPTY_PASSWORD)
scripthash = neo3_utils.address_to_script_hash(address)
neogo = NeoGo(shell=shell, neo_go_exec_path=NEOGO_EXECUTABLE)
@ -142,10 +140,12 @@ def transfer_gas(
if wallet_from_password is not None
else main_chain.get_wallet_password()
)
address_from = address_from or get_last_address_from_wallet(
address_from = address_from or wallet_utils.get_last_address_from_wallet(
wallet_from_path, wallet_from_password
)
address_to = address_to or get_last_address_from_wallet(wallet_to_path, wallet_to_password)
address_to = address_to or wallet_utils.get_last_address_from_wallet(
wallet_to_path, wallet_to_password
)
neogo = NeoGo(shell, neo_go_exec_path=NEOGO_EXECUTABLE)
out = neogo.nep17.transfer(
@ -163,7 +163,7 @@ def transfer_gas(
raise Exception("Got no TXID after run the command")
if not transaction_accepted(main_chain, txid):
raise AssertionError(f"TX {txid} hasn't been processed")
time.sleep(parse_time(MAINNET_BLOCK_TIME))
time.sleep(datetime_utils.parse_time(MAINNET_BLOCK_TIME))
@allure.step("FrostFS Deposit")
@ -178,9 +178,9 @@ def deposit_gas(
Transferring GAS from given wallet to FrostFS contract address.
"""
# get FrostFS contract address
deposit_addr = contract_hash_to_address(FROSTFS_CONTRACT)
deposit_addr = converting_utils.contract_hash_to_address(FROSTFS_CONTRACT)
logger.info(f"FrostFS contract address: {deposit_addr}")
address_from = get_last_address_from_wallet(
address_from = wallet_utils.get_last_address_from_wallet(
wallet_path=wallet_from_path, wallet_password=wallet_from_password
)
transfer_gas(

4
robot/resources/lib/python_keywords/storage_policy.py
View file

@ -13,7 +13,7 @@ import frostfs_verbs
from cluster import StorageNode
from frostfs_testlib.resources.common import OBJECT_NOT_FOUND
from frostfs_testlib.shell import Shell
from frostfs_testlib.utils.errors import error_matches_status
from frostfs_testlib.utils import string_utils
logger = logging.getLogger("NeoLogger")
@ -166,7 +166,7 @@ def get_nodes_without_object(
if res is None:
nodes_list.append(node)
except Exception as err:
if error_matches_status(err, OBJECT_NOT_FOUND):
if string_utils.is_str_match_pattern(err, OBJECT_NOT_FOUND):
nodes_list.append(node)
else:
raise Exception(f"Got error {err} on head object command") from err