From 47c55f006077301be9a86e5e837b5f0fb92bc70e Mon Sep 17 00:00:00 2001 From: Vladimir Domnich Date: Mon, 1 Aug 2022 13:15:02 +0300 Subject: [PATCH] Remove redundant variables Small refactoring that includes: - Removed variables that are not used any more. - Cleanup in helper functions' names. Signed-off-by: Vladimir Domnich --- configuration/devenv.yml | 1 - .../network/test_node_management.py | 32 +++++-------- robot/resources/lib/python_keywords/acl.py | 47 +++++++++---------- .../lib/python_keywords/data_formatters.py | 38 ++++++++++++--- .../lib/python_keywords/payment_neogo.py | 8 ++-- .../lib/python_keywords/s3_gate_bucket.py | 15 +++--- robot/variables/common.py | 2 +- 7 files changed, 77 insertions(+), 66 deletions(-) diff --git a/configuration/devenv.yml b/configuration/devenv.yml index a479844b..f52ca36b 100644 --- a/configuration/devenv.yml +++ b/configuration/devenv.yml @@ -50,7 +50,6 @@ neofs_netmap: # Paths to binaries -neogo_cli_exec: 'neo-go' neogo_executable: 'neo-go' neofs_cli_exec: 'neofs-cli' diff --git a/pytest_tests/testsuites/network/test_node_management.py b/pytest_tests/testsuites/network/test_node_management.py index bebc2a07..70d4284c 100644 --- a/pytest_tests/testsuites/network/test_node_management.py +++ b/pytest_tests/testsuites/network/test_node_management.py @@ -3,11 +3,10 @@ from random import choice from time import sleep import allure -import base58 import pytest -from cli_helpers import _cmd_run +from data_formatters import get_wallet_public_key from common import (COMPLEX_OBJ_SIZE, MAINNET_BLOCK_TIME, NEOFS_CONTRACT_CACHE_TIMEOUT, - NEOFS_NETMAP_DICT, NEOGO_CLI_EXEC) + NEOFS_NETMAP_DICT, STORAGE_WALLET_PASS) from epoch import tick_epoch from python_keywords.container import create_container, get_container from python_keywords.neofs_verbs import (delete_object, get_object, @@ -82,9 +81,14 @@ def test_nodes_management(prepare_tmp_dir): random_node = choice(list(NEOFS_NETMAP_DICT)) alive_node = choice([node for node in NEOFS_NETMAP_DICT if node != random_node]) - # Calculate public key that identifies node in netmap + # Calculate public key that identifies node in netmap (we need base58-formatted key + # because keys of storage nodes are base58-encoded in netmap) random_node_wallet_path = NEOFS_NETMAP_DICT[random_node]['wallet_path'] - random_node_netmap_key = get_netmap_public_key_from_wallet(random_node_wallet_path) + random_node_netmap_key = get_wallet_public_key( + random_node_wallet_path, + STORAGE_WALLET_PASS, + format="base58" + ) with allure.step('Check node {random_node} is in netmap'): snapshot = get_netmap_snapshot(node_name=alive_node) @@ -298,7 +302,7 @@ def validate_object_copies(wallet: str, placement_rule: str, file_path: str, exp @allure.step('Wait for node {node_name} goes online') -def wait_for_node_go_online(node_name: str): +def wait_for_node_go_online(node_name: str) -> None: timeout, attempts = 5, 20 for _ in range(attempts): try: @@ -313,7 +317,8 @@ def wait_for_node_go_online(node_name: str): @allure.step('Wait for {expected_copies} object copies in the wallet') -def wait_for_expected_object_copies(wallet: str, cid: str, oid: str, expected_copies: int = 2): +def wait_for_expected_object_copies(wallet: str, cid: str, oid: str, + expected_copies: int = 2) -> None: for i in range(2): copies = get_simple_object_copies(wallet, cid, oid) if copies == expected_copies: @@ -325,7 +330,7 @@ def wait_for_expected_object_copies(wallet: str, cid: str, oid: str, expected_co @allure.step('Wait for object to be dropped') -def wait_for_obj_dropped(wallet: str, cid: str, oid: str, checker): +def wait_for_obj_dropped(wallet: str, cid: str, oid: str, checker) -> None: for _ in range(3): try: checker(wallet, cid, oid) @@ -335,14 +340,3 @@ def wait_for_obj_dropped(wallet: str, cid: str, oid: str, checker): break else: raise AssertionError(f'Object {oid} is not dropped from node') - - -def get_netmap_public_key_from_wallet(wallet_path: str) -> str: - # Get public key from wallet file (it is printed on 2nd line) - cmd = f"{NEOGO_CLI_EXEC} wallet dump-keys -w {wallet_path}" - output = _cmd_run(cmd) - public_key_hex = output.split("\n")[1].strip() - - # Encode public key in base58 (this is how it is present in netmap) - public_key_base58 = base58.b58encode(bytes.fromhex(public_key_hex)) - return public_key_base58.decode("utf-8") diff --git a/robot/resources/lib/python_keywords/acl.py b/robot/resources/lib/python_keywords/acl.py index db891a0f..6edb3f63 100644 --- a/robot/resources/lib/python_keywords/acl.py +++ b/robot/resources/lib/python_keywords/acl.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3.8 +#!/usr/bin/python3.9 import base64 import json @@ -6,11 +6,12 @@ import os import re import uuid from enum import Enum, auto +from typing import Optional import base58 from cli_helpers import _cmd_run -from common import ASSETS_DIR, NEOFS_ENDPOINT, WALLET_CONFIG -from data_formatters import pub_key_hex +from common import ASSETS_DIR, NEOFS_CLI_EXEC, NEOFS_ENDPOINT, WALLET_CONFIG +from data_formatters import get_wallet_public_key from robot.api import logger from robot.api.deco import keyword @@ -19,9 +20,6 @@ Robot Keywords and helper functions for work with NeoFS ACL. """ ROBOT_AUTO_KEYWORDS = False - -# path to neofs-cli executable -NEOFS_CLI_EXEC = os.getenv('NEOFS_CLI_EXEC', 'neofs-cli') EACL_LIFETIME = 100500 @@ -37,7 +35,7 @@ class Role(AutoName): @keyword('Get eACL') -def get_eacl(wallet_path: str, cid: str): +def get_eacl(wallet_path: str, cid: str) -> Optional[str]: cmd = ( f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet_path} ' f'container get-eacl --cid {cid} --config {WALLET_CONFIG}' @@ -54,7 +52,7 @@ def get_eacl(wallet_path: str, cid: str): @keyword('Set eACL') -def set_eacl(wallet_path: str, cid: str, eacl_table_path: str): +def set_eacl(wallet_path: str, cid: str, eacl_table_path: str) -> None: cmd = ( f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet_path} ' f'container set-eacl --cid {cid} --table {eacl_table_path} --config {WALLET_CONFIG} --await' @@ -68,23 +66,19 @@ def _encode_cid_for_eacl(cid: str) -> str: @keyword('Create eACL') -def create_eacl(cid: str, rules_list: list): - table = f"{os.getcwd()}/{ASSETS_DIR}/eacl_table_{str(uuid.uuid4())}.json" - rules = "" - for rule in rules_list: - # TODO: check if $Object: is still necessary for filtering in the newest releases - rules += f"--rule '{rule}' " - cmd = ( - f"{NEOFS_CLI_EXEC} acl extended create --cid {cid} " - f"{rules}--out {table}" - ) +def create_eacl(cid: str, rules_list: list) -> str: + table_file_path = f"{os.getcwd()}/{ASSETS_DIR}/eacl_table_{str(uuid.uuid4())}.json" + # TODO: check if $Object: is still necessary for filtering in the newest releases + rules = " ".join(f"--rule '{rule}'" for rule in rules_list) + + cmd = f"{NEOFS_CLI_EXEC} acl extended create --cid {cid} {rules} --out {table_file_path}" _cmd_run(cmd) - with open(table, 'r') as fout: - table_data = fout.read() + with open(table_file_path, 'r') as file: + table_data = file.read() logger.info(f"Generated eACL:\n{table_data}") - return table + return table_file_path @keyword('Form BearerToken File') @@ -164,8 +158,9 @@ def form_bearertoken_file(wif: str, cid: str, eacl_records: list) -> str: sign_bearer_token(wif, file_path) return file_path + @keyword('EACL Rules') -def eacl_rules(access: str, verbs: list, user: str): +def eacl_rules(access: str, verbs: list, user: str) -> list[str]: """ This function creates a list of eACL rules. Args: @@ -178,17 +173,17 @@ def eacl_rules(access: str, verbs: list, user: str): (list): a list of eACL rules """ if user not in ('others', 'user'): - pubkey = pub_key_hex(user) + pubkey = get_wallet_public_key(user, wallet_password="") user = f"pubkey:{pubkey}" rules = [] for verb in verbs: - elements = [access, verb, user] - rules.append(' '.join(elements)) + rule = f"{access} {verb} {user}" + rules.append(rule) return rules -def sign_bearer_token(wallet_path: str, eacl_rules_file: str): +def sign_bearer_token(wallet_path: str, eacl_rules_file: str) -> None: cmd = ( f'{NEOFS_CLI_EXEC} util sign bearer-token --from {eacl_rules_file} ' f'--to {eacl_rules_file} --wallet {wallet_path} --config {WALLET_CONFIG} --json' diff --git a/robot/resources/lib/python_keywords/data_formatters.py b/robot/resources/lib/python_keywords/data_formatters.py index 54fecb94..58df289c 100644 --- a/robot/resources/lib/python_keywords/data_formatters.py +++ b/robot/resources/lib/python_keywords/data_formatters.py @@ -1,4 +1,7 @@ +import base64 import json + +import base58 from neo3 import wallet @@ -16,11 +19,32 @@ def dict_to_attrs(attrs: dict) -> str: return ",".join(f"{key}={value}" for key, value in attrs.items()) -def pub_key_hex(wallet_path: str, wallet_password=""): - wallet_content = '' - with open(wallet_path) as out: - wallet_content = json.load(out) +def __fix_wallet_schema(wallet: dict) -> None: + # Temporary function to fix wallets that do not conform to the schema + # TODO: get rid of it once issue is solved + if "name" not in wallet: + wallet["name"] = None + for account in wallet["accounts"]: + if "extra" not in account: + account["extra"] = None + + +def get_wallet_public_key(wallet_path: str, wallet_password: str, format: str = "hex") -> str: + # Get public key from wallet file + with open(wallet_path, "r") as file: + wallet_content = json.load(file) + __fix_wallet_schema(wallet_content) + wallet_from_json = wallet.Wallet.from_json(wallet_content, password=wallet_password) - pub_key_64 = str(wallet_from_json.accounts[0].public_key) - - return pub_key_64 + public_key_hex = str(wallet_from_json.accounts[0].public_key) + + # Convert public key to specified format + if format == "hex": + return public_key_hex + if format == "base58": + public_key_base58 = base58.b58encode(bytes.fromhex(public_key_hex)) + return public_key_base58.decode("utf-8") + if format == "base64": + public_key_base64 = base64.b64encode(bytes.fromhex(public_key_hex)) + return public_key_base64.decode("utf-8") + raise ValueError(f"Invalid public key format: {format}") diff --git a/robot/resources/lib/python_keywords/payment_neogo.py b/robot/resources/lib/python_keywords/payment_neogo.py index d044f8dc..86eabb95 100644 --- a/robot/resources/lib/python_keywords/payment_neogo.py +++ b/robot/resources/lib/python_keywords/payment_neogo.py @@ -10,9 +10,8 @@ from robot.api.deco import keyword import contract import converters import rpc_client -from common import (GAS_HASH, MAINNET_SINGLE_ADDR, MAINNET_WALLET_PATH, - MORPH_ENDPOINT, NEO_MAINNET_ENDPOINT, NEOFS_CONTRACT, - NEOGO_CLI_EXEC) +from common import (GAS_HASH, MAINNET_SINGLE_ADDR, MAINNET_WALLET_PATH, MAINNET_WALLET_PASS, + MORPH_ENDPOINT, NEO_MAINNET_ENDPOINT, NEOFS_CONTRACT, NEOGO_EXECUTABLE) from converters import load_wallet from wallet import nep17_transfer from wrappers import run_sh_with_passwd_contract @@ -20,7 +19,6 @@ from wrappers import run_sh_with_passwd_contract ROBOT_AUTO_KEYWORDS = False EMPTY_PASSWORD = '' -MAINNET_WALLET_PASS = 'one' TX_PERSIST_TIMEOUT = 15 # seconds ASSET_POWER_MAINCHAIN = 10 ** 8 ASSET_POWER_SIDECHAIN = 10 ** 12 @@ -35,7 +33,7 @@ def withdraw_mainnet_gas(wlt: str, amount: int): scripthash = wallet.Account.address_to_script_hash(address) cmd = ( - f"{NEOGO_CLI_EXEC} contract invokefunction -w {wlt} -a {address} " + f"{NEOGO_EXECUTABLE} contract invokefunction -w {wlt} -a {address} " f"-r {NEO_MAINNET_ENDPOINT} {NEOFS_CONTRACT} withdraw {scripthash} " f"int:{amount} -- {scripthash}:Global" ) diff --git a/robot/resources/lib/python_keywords/s3_gate_bucket.py b/robot/resources/lib/python_keywords/s3_gate_bucket.py index 6fca3621..2f05ad5f 100644 --- a/robot/resources/lib/python_keywords/s3_gate_bucket.py +++ b/robot/resources/lib/python_keywords/s3_gate_bucket.py @@ -6,16 +6,17 @@ import re import uuid from enum import Enum from time import sleep +from typing import Optional import boto3 -from data_formatters import pub_key_hex -from botocore.exceptions import ClientError import urllib3 +from botocore.exceptions import ClientError from robot.api import logger from robot.api.deco import keyword from cli_helpers import _run_with_passwd, log_command_execution from common import NEOFS_ENDPOINT, S3_GATE, S3_GATE_WALLET_PATH, S3_GATE_WALLET_PASS +from data_formatters import get_wallet_public_key ########################################################## # Disabling warnings on self-signed certificate which the @@ -39,13 +40,13 @@ class VersioningStatus(Enum): @keyword('Init S3 Credentials') -def init_s3_credentials(wallet_path, s3_bearer_rules_file: str = None): +def init_s3_credentials(wallet_path, s3_bearer_rules_file: Optional[str] = None): bucket = str(uuid.uuid4()) s3_bearer_rules = s3_bearer_rules_file or 'robot/resources/files/s3_bearer_rules.json' - gate_pub_key = pub_key_hex(S3_GATE_WALLET_PATH, S3_GATE_WALLET_PASS) + gate_public_key = get_wallet_public_key(S3_GATE_WALLET_PATH, S3_GATE_WALLET_PASS) cmd = ( f'{NEOFS_EXEC} --debug --with-log --timeout {CREDENTIALS_CREATE_TIMEOUT} ' - f'issue-secret --wallet {wallet_path} --gate-public-key={gate_pub_key} ' + f'issue-secret --wallet {wallet_path} --gate-public-key={gate_public_key} ' f'--peer {NEOFS_ENDPOINT} --container-friendly-name {bucket} ' f'--bearer-rules {s3_bearer_rules}' ) @@ -158,7 +159,7 @@ def head_bucket(s3_client, bucket: str): @keyword('Set bucket versioning status') -def set_bucket_versioning(s3_client, bucket_name: str, status: VersioningStatus): +def set_bucket_versioning(s3_client, bucket_name: str, status: VersioningStatus) -> None: try: response = s3_client.put_bucket_versioning(Bucket=bucket_name, VersioningConfiguration={'Status': status.value}) log_command_execution('S3 Set bucket versioning to', response) @@ -202,7 +203,7 @@ def get_bucket_tagging(s3_client, bucket_name: str) -> list: @keyword('Delete bucket tagging') -def delete_bucket_tagging(s3_client, bucket_name: str): +def delete_bucket_tagging(s3_client, bucket_name: str) -> None: try: response = s3_client.delete_bucket_tagging(Bucket=bucket_name) log_command_execution('S3 Delete bucket tagging', response) diff --git a/robot/variables/common.py b/robot/variables/common.py index 56663578..f7cb684f 100644 --- a/robot/variables/common.py +++ b/robot/variables/common.py @@ -18,7 +18,6 @@ NEOFS_CONTRACT_CACHE_TIMEOUT = os.getenv("NEOFS_CONTRACT_CACHE_TIMEOUT", "30s") SHARD_0_GC_SLEEP = os.getenv("NEOFS_STORAGE_SHARD_0_GC_REMOVER_SLEEP_INTERVAL", "1m") NEOFS_ENDPOINT = os.getenv("NEOFS_ENDPOINT", "s01.neofs.devenv:8080") -NEOGO_CLI_EXEC = os.getenv("NEOGO_EXECUTABLE", "neo-go") NEO_MAINNET_ENDPOINT = os.getenv("NEO_MAINNET_ENDPOINT", 'http://main-chain.neofs.devenv:30333') MORPH_ENDPOINT = os.getenv("MORPH_ENDPOINT", 'http://morph-chain.neofs.devenv:30333') @@ -50,6 +49,7 @@ STORAGE_WALLET_PATH_2 = os.getenv("STORAGE_WALLET_PATH_2", f"{DEVENV_PATH}/servi STORAGE_WALLET_PATH_3 = os.getenv("STORAGE_WALLET_PATH_3", f"{DEVENV_PATH}/services/storage/wallet03.json") STORAGE_WALLET_PATH_4 = os.getenv("STORAGE_WALLET_PATH_4", f"{DEVENV_PATH}/services/storage/wallet04.json") STORAGE_WALLET_PATH = STORAGE_WALLET_PATH_1 +STORAGE_WALLET_PASS = os.getenv("STORAGE_WALLET_PASS", "") NEOFS_NETMAP_DICT = { 's01': {