Remove redundant variables

Small refactoring that includes:
 - Removed variables that are not used any more.
 - Cleanup in helper functions' names.

Signed-off-by: Vladimir Domnich <v.domnich@yadro.com>
This commit is contained in:
Vladimir Domnich 2022-08-01 13:15:02 +03:00
parent 642af0a888
commit 47c55f0060
7 changed files with 77 additions and 66 deletions

View file

@ -50,7 +50,6 @@ neofs_netmap:
# Paths to binaries # Paths to binaries
neogo_cli_exec: 'neo-go'
neogo_executable: 'neo-go' neogo_executable: 'neo-go'
neofs_cli_exec: 'neofs-cli' neofs_cli_exec: 'neofs-cli'

View file

@ -3,11 +3,10 @@ from random import choice
from time import sleep from time import sleep
import allure import allure
import base58
import pytest import pytest
from cli_helpers import _cmd_run from data_formatters import get_wallet_public_key
from common import (COMPLEX_OBJ_SIZE, MAINNET_BLOCK_TIME, NEOFS_CONTRACT_CACHE_TIMEOUT, from common import (COMPLEX_OBJ_SIZE, MAINNET_BLOCK_TIME, NEOFS_CONTRACT_CACHE_TIMEOUT,
NEOFS_NETMAP_DICT, NEOGO_CLI_EXEC) NEOFS_NETMAP_DICT, STORAGE_WALLET_PASS)
from epoch import tick_epoch from epoch import tick_epoch
from python_keywords.container import create_container, get_container from python_keywords.container import create_container, get_container
from python_keywords.neofs_verbs import (delete_object, get_object, from python_keywords.neofs_verbs import (delete_object, get_object,
@ -82,9 +81,14 @@ def test_nodes_management(prepare_tmp_dir):
random_node = choice(list(NEOFS_NETMAP_DICT)) random_node = choice(list(NEOFS_NETMAP_DICT))
alive_node = choice([node for node in NEOFS_NETMAP_DICT if node != random_node]) alive_node = choice([node for node in NEOFS_NETMAP_DICT if node != random_node])
# Calculate public key that identifies node in netmap # Calculate public key that identifies node in netmap (we need base58-formatted key
# because keys of storage nodes are base58-encoded in netmap)
random_node_wallet_path = NEOFS_NETMAP_DICT[random_node]['wallet_path'] random_node_wallet_path = NEOFS_NETMAP_DICT[random_node]['wallet_path']
random_node_netmap_key = get_netmap_public_key_from_wallet(random_node_wallet_path) random_node_netmap_key = get_wallet_public_key(
random_node_wallet_path,
STORAGE_WALLET_PASS,
format="base58"
)
with allure.step('Check node {random_node} is in netmap'): with allure.step('Check node {random_node} is in netmap'):
snapshot = get_netmap_snapshot(node_name=alive_node) snapshot = get_netmap_snapshot(node_name=alive_node)
@ -298,7 +302,7 @@ def validate_object_copies(wallet: str, placement_rule: str, file_path: str, exp
@allure.step('Wait for node {node_name} goes online') @allure.step('Wait for node {node_name} goes online')
def wait_for_node_go_online(node_name: str): def wait_for_node_go_online(node_name: str) -> None:
timeout, attempts = 5, 20 timeout, attempts = 5, 20
for _ in range(attempts): for _ in range(attempts):
try: try:
@ -313,7 +317,8 @@ def wait_for_node_go_online(node_name: str):
@allure.step('Wait for {expected_copies} object copies in the wallet') @allure.step('Wait for {expected_copies} object copies in the wallet')
def wait_for_expected_object_copies(wallet: str, cid: str, oid: str, expected_copies: int = 2): def wait_for_expected_object_copies(wallet: str, cid: str, oid: str,
expected_copies: int = 2) -> None:
for i in range(2): for i in range(2):
copies = get_simple_object_copies(wallet, cid, oid) copies = get_simple_object_copies(wallet, cid, oid)
if copies == expected_copies: if copies == expected_copies:
@ -325,7 +330,7 @@ def wait_for_expected_object_copies(wallet: str, cid: str, oid: str, expected_co
@allure.step('Wait for object to be dropped') @allure.step('Wait for object to be dropped')
def wait_for_obj_dropped(wallet: str, cid: str, oid: str, checker): def wait_for_obj_dropped(wallet: str, cid: str, oid: str, checker) -> None:
for _ in range(3): for _ in range(3):
try: try:
checker(wallet, cid, oid) checker(wallet, cid, oid)
@ -335,14 +340,3 @@ def wait_for_obj_dropped(wallet: str, cid: str, oid: str, checker):
break break
else: else:
raise AssertionError(f'Object {oid} is not dropped from node') raise AssertionError(f'Object {oid} is not dropped from node')
def get_netmap_public_key_from_wallet(wallet_path: str) -> str:
# Get public key from wallet file (it is printed on 2nd line)
cmd = f"{NEOGO_CLI_EXEC} wallet dump-keys -w {wallet_path}"
output = _cmd_run(cmd)
public_key_hex = output.split("\n")[1].strip()
# Encode public key in base58 (this is how it is present in netmap)
public_key_base58 = base58.b58encode(bytes.fromhex(public_key_hex))
return public_key_base58.decode("utf-8")

View file

@ -1,4 +1,4 @@
#!/usr/bin/python3.8 #!/usr/bin/python3.9
import base64 import base64
import json import json
@ -6,11 +6,12 @@ import os
import re import re
import uuid import uuid
from enum import Enum, auto from enum import Enum, auto
from typing import Optional
import base58 import base58
from cli_helpers import _cmd_run from cli_helpers import _cmd_run
from common import ASSETS_DIR, NEOFS_ENDPOINT, WALLET_CONFIG from common import ASSETS_DIR, NEOFS_CLI_EXEC, NEOFS_ENDPOINT, WALLET_CONFIG
from data_formatters import pub_key_hex from data_formatters import get_wallet_public_key
from robot.api import logger from robot.api import logger
from robot.api.deco import keyword from robot.api.deco import keyword
@ -19,9 +20,6 @@ Robot Keywords and helper functions for work with NeoFS ACL.
""" """
ROBOT_AUTO_KEYWORDS = False ROBOT_AUTO_KEYWORDS = False
# path to neofs-cli executable
NEOFS_CLI_EXEC = os.getenv('NEOFS_CLI_EXEC', 'neofs-cli')
EACL_LIFETIME = 100500 EACL_LIFETIME = 100500
@ -37,7 +35,7 @@ class Role(AutoName):
@keyword('Get eACL') @keyword('Get eACL')
def get_eacl(wallet_path: str, cid: str): def get_eacl(wallet_path: str, cid: str) -> Optional[str]:
cmd = ( cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet_path} ' f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet_path} '
f'container get-eacl --cid {cid} --config {WALLET_CONFIG}' f'container get-eacl --cid {cid} --config {WALLET_CONFIG}'
@ -54,7 +52,7 @@ def get_eacl(wallet_path: str, cid: str):
@keyword('Set eACL') @keyword('Set eACL')
def set_eacl(wallet_path: str, cid: str, eacl_table_path: str): def set_eacl(wallet_path: str, cid: str, eacl_table_path: str) -> None:
cmd = ( cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet_path} ' f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet_path} '
f'container set-eacl --cid {cid} --table {eacl_table_path} --config {WALLET_CONFIG} --await' f'container set-eacl --cid {cid} --table {eacl_table_path} --config {WALLET_CONFIG} --await'
@ -68,23 +66,19 @@ def _encode_cid_for_eacl(cid: str) -> str:
@keyword('Create eACL') @keyword('Create eACL')
def create_eacl(cid: str, rules_list: list): def create_eacl(cid: str, rules_list: list) -> str:
table = f"{os.getcwd()}/{ASSETS_DIR}/eacl_table_{str(uuid.uuid4())}.json" table_file_path = f"{os.getcwd()}/{ASSETS_DIR}/eacl_table_{str(uuid.uuid4())}.json"
rules = ""
for rule in rules_list:
# TODO: check if $Object: is still necessary for filtering in the newest releases # TODO: check if $Object: is still necessary for filtering in the newest releases
rules += f"--rule '{rule}' " rules = " ".join(f"--rule '{rule}'" for rule in rules_list)
cmd = (
f"{NEOFS_CLI_EXEC} acl extended create --cid {cid} " cmd = f"{NEOFS_CLI_EXEC} acl extended create --cid {cid} {rules} --out {table_file_path}"
f"{rules}--out {table}"
)
_cmd_run(cmd) _cmd_run(cmd)
with open(table, 'r') as fout: with open(table_file_path, 'r') as file:
table_data = fout.read() table_data = file.read()
logger.info(f"Generated eACL:\n{table_data}") logger.info(f"Generated eACL:\n{table_data}")
return table return table_file_path
@keyword('Form BearerToken File') @keyword('Form BearerToken File')
@ -164,8 +158,9 @@ def form_bearertoken_file(wif: str, cid: str, eacl_records: list) -> str:
sign_bearer_token(wif, file_path) sign_bearer_token(wif, file_path)
return file_path return file_path
@keyword('EACL Rules') @keyword('EACL Rules')
def eacl_rules(access: str, verbs: list, user: str): def eacl_rules(access: str, verbs: list, user: str) -> list[str]:
""" """
This function creates a list of eACL rules. This function creates a list of eACL rules.
Args: Args:
@ -178,17 +173,17 @@ def eacl_rules(access: str, verbs: list, user: str):
(list): a list of eACL rules (list): a list of eACL rules
""" """
if user not in ('others', 'user'): if user not in ('others', 'user'):
pubkey = pub_key_hex(user) pubkey = get_wallet_public_key(user, wallet_password="")
user = f"pubkey:{pubkey}" user = f"pubkey:{pubkey}"
rules = [] rules = []
for verb in verbs: for verb in verbs:
elements = [access, verb, user] rule = f"{access} {verb} {user}"
rules.append(' '.join(elements)) rules.append(rule)
return rules return rules
def sign_bearer_token(wallet_path: str, eacl_rules_file: str): def sign_bearer_token(wallet_path: str, eacl_rules_file: str) -> None:
cmd = ( cmd = (
f'{NEOFS_CLI_EXEC} util sign bearer-token --from {eacl_rules_file} ' f'{NEOFS_CLI_EXEC} util sign bearer-token --from {eacl_rules_file} '
f'--to {eacl_rules_file} --wallet {wallet_path} --config {WALLET_CONFIG} --json' f'--to {eacl_rules_file} --wallet {wallet_path} --config {WALLET_CONFIG} --json'

View file

@ -1,4 +1,7 @@
import base64
import json import json
import base58
from neo3 import wallet from neo3 import wallet
@ -16,11 +19,32 @@ def dict_to_attrs(attrs: dict) -> str:
return ",".join(f"{key}={value}" for key, value in attrs.items()) return ",".join(f"{key}={value}" for key, value in attrs.items())
def pub_key_hex(wallet_path: str, wallet_password=""): def __fix_wallet_schema(wallet: dict) -> None:
wallet_content = '' # Temporary function to fix wallets that do not conform to the schema
with open(wallet_path) as out: # TODO: get rid of it once issue is solved
wallet_content = json.load(out) if "name" not in wallet:
wallet_from_json = wallet.Wallet.from_json(wallet_content, password=wallet_password) wallet["name"] = None
pub_key_64 = str(wallet_from_json.accounts[0].public_key) for account in wallet["accounts"]:
if "extra" not in account:
account["extra"] = None
return pub_key_64
def get_wallet_public_key(wallet_path: str, wallet_password: str, format: str = "hex") -> str:
# Get public key from wallet file
with open(wallet_path, "r") as file:
wallet_content = json.load(file)
__fix_wallet_schema(wallet_content)
wallet_from_json = wallet.Wallet.from_json(wallet_content, password=wallet_password)
public_key_hex = str(wallet_from_json.accounts[0].public_key)
# Convert public key to specified format
if format == "hex":
return public_key_hex
if format == "base58":
public_key_base58 = base58.b58encode(bytes.fromhex(public_key_hex))
return public_key_base58.decode("utf-8")
if format == "base64":
public_key_base64 = base64.b64encode(bytes.fromhex(public_key_hex))
return public_key_base64.decode("utf-8")
raise ValueError(f"Invalid public key format: {format}")

View file

@ -10,9 +10,8 @@ from robot.api.deco import keyword
import contract import contract
import converters import converters
import rpc_client import rpc_client
from common import (GAS_HASH, MAINNET_SINGLE_ADDR, MAINNET_WALLET_PATH, from common import (GAS_HASH, MAINNET_SINGLE_ADDR, MAINNET_WALLET_PATH, MAINNET_WALLET_PASS,
MORPH_ENDPOINT, NEO_MAINNET_ENDPOINT, NEOFS_CONTRACT, MORPH_ENDPOINT, NEO_MAINNET_ENDPOINT, NEOFS_CONTRACT, NEOGO_EXECUTABLE)
NEOGO_CLI_EXEC)
from converters import load_wallet from converters import load_wallet
from wallet import nep17_transfer from wallet import nep17_transfer
from wrappers import run_sh_with_passwd_contract from wrappers import run_sh_with_passwd_contract
@ -20,7 +19,6 @@ from wrappers import run_sh_with_passwd_contract
ROBOT_AUTO_KEYWORDS = False ROBOT_AUTO_KEYWORDS = False
EMPTY_PASSWORD = '' EMPTY_PASSWORD = ''
MAINNET_WALLET_PASS = 'one'
TX_PERSIST_TIMEOUT = 15 # seconds TX_PERSIST_TIMEOUT = 15 # seconds
ASSET_POWER_MAINCHAIN = 10 ** 8 ASSET_POWER_MAINCHAIN = 10 ** 8
ASSET_POWER_SIDECHAIN = 10 ** 12 ASSET_POWER_SIDECHAIN = 10 ** 12
@ -35,7 +33,7 @@ def withdraw_mainnet_gas(wlt: str, amount: int):
scripthash = wallet.Account.address_to_script_hash(address) scripthash = wallet.Account.address_to_script_hash(address)
cmd = ( cmd = (
f"{NEOGO_CLI_EXEC} contract invokefunction -w {wlt} -a {address} " f"{NEOGO_EXECUTABLE} contract invokefunction -w {wlt} -a {address} "
f"-r {NEO_MAINNET_ENDPOINT} {NEOFS_CONTRACT} withdraw {scripthash} " f"-r {NEO_MAINNET_ENDPOINT} {NEOFS_CONTRACT} withdraw {scripthash} "
f"int:{amount} -- {scripthash}:Global" f"int:{amount} -- {scripthash}:Global"
) )

View file

@ -6,16 +6,17 @@ import re
import uuid import uuid
from enum import Enum from enum import Enum
from time import sleep from time import sleep
from typing import Optional
import boto3 import boto3
from data_formatters import pub_key_hex
from botocore.exceptions import ClientError
import urllib3 import urllib3
from botocore.exceptions import ClientError
from robot.api import logger from robot.api import logger
from robot.api.deco import keyword from robot.api.deco import keyword
from cli_helpers import _run_with_passwd, log_command_execution from cli_helpers import _run_with_passwd, log_command_execution
from common import NEOFS_ENDPOINT, S3_GATE, S3_GATE_WALLET_PATH, S3_GATE_WALLET_PASS from common import NEOFS_ENDPOINT, S3_GATE, S3_GATE_WALLET_PATH, S3_GATE_WALLET_PASS
from data_formatters import get_wallet_public_key
########################################################## ##########################################################
# Disabling warnings on self-signed certificate which the # Disabling warnings on self-signed certificate which the
@ -39,13 +40,13 @@ class VersioningStatus(Enum):
@keyword('Init S3 Credentials') @keyword('Init S3 Credentials')
def init_s3_credentials(wallet_path, s3_bearer_rules_file: str = None): def init_s3_credentials(wallet_path, s3_bearer_rules_file: Optional[str] = None):
bucket = str(uuid.uuid4()) bucket = str(uuid.uuid4())
s3_bearer_rules = s3_bearer_rules_file or 'robot/resources/files/s3_bearer_rules.json' s3_bearer_rules = s3_bearer_rules_file or 'robot/resources/files/s3_bearer_rules.json'
gate_pub_key = pub_key_hex(S3_GATE_WALLET_PATH, S3_GATE_WALLET_PASS) gate_public_key = get_wallet_public_key(S3_GATE_WALLET_PATH, S3_GATE_WALLET_PASS)
cmd = ( cmd = (
f'{NEOFS_EXEC} --debug --with-log --timeout {CREDENTIALS_CREATE_TIMEOUT} ' f'{NEOFS_EXEC} --debug --with-log --timeout {CREDENTIALS_CREATE_TIMEOUT} '
f'issue-secret --wallet {wallet_path} --gate-public-key={gate_pub_key} ' f'issue-secret --wallet {wallet_path} --gate-public-key={gate_public_key} '
f'--peer {NEOFS_ENDPOINT} --container-friendly-name {bucket} ' f'--peer {NEOFS_ENDPOINT} --container-friendly-name {bucket} '
f'--bearer-rules {s3_bearer_rules}' f'--bearer-rules {s3_bearer_rules}'
) )
@ -158,7 +159,7 @@ def head_bucket(s3_client, bucket: str):
@keyword('Set bucket versioning status') @keyword('Set bucket versioning status')
def set_bucket_versioning(s3_client, bucket_name: str, status: VersioningStatus): def set_bucket_versioning(s3_client, bucket_name: str, status: VersioningStatus) -> None:
try: try:
response = s3_client.put_bucket_versioning(Bucket=bucket_name, VersioningConfiguration={'Status': status.value}) response = s3_client.put_bucket_versioning(Bucket=bucket_name, VersioningConfiguration={'Status': status.value})
log_command_execution('S3 Set bucket versioning to', response) log_command_execution('S3 Set bucket versioning to', response)
@ -202,7 +203,7 @@ def get_bucket_tagging(s3_client, bucket_name: str) -> list:
@keyword('Delete bucket tagging') @keyword('Delete bucket tagging')
def delete_bucket_tagging(s3_client, bucket_name: str): def delete_bucket_tagging(s3_client, bucket_name: str) -> None:
try: try:
response = s3_client.delete_bucket_tagging(Bucket=bucket_name) response = s3_client.delete_bucket_tagging(Bucket=bucket_name)
log_command_execution('S3 Delete bucket tagging', response) log_command_execution('S3 Delete bucket tagging', response)

View file

@ -18,7 +18,6 @@ NEOFS_CONTRACT_CACHE_TIMEOUT = os.getenv("NEOFS_CONTRACT_CACHE_TIMEOUT", "30s")
SHARD_0_GC_SLEEP = os.getenv("NEOFS_STORAGE_SHARD_0_GC_REMOVER_SLEEP_INTERVAL", "1m") SHARD_0_GC_SLEEP = os.getenv("NEOFS_STORAGE_SHARD_0_GC_REMOVER_SLEEP_INTERVAL", "1m")
NEOFS_ENDPOINT = os.getenv("NEOFS_ENDPOINT", "s01.neofs.devenv:8080") NEOFS_ENDPOINT = os.getenv("NEOFS_ENDPOINT", "s01.neofs.devenv:8080")
NEOGO_CLI_EXEC = os.getenv("NEOGO_EXECUTABLE", "neo-go")
NEO_MAINNET_ENDPOINT = os.getenv("NEO_MAINNET_ENDPOINT", 'http://main-chain.neofs.devenv:30333') NEO_MAINNET_ENDPOINT = os.getenv("NEO_MAINNET_ENDPOINT", 'http://main-chain.neofs.devenv:30333')
MORPH_ENDPOINT = os.getenv("MORPH_ENDPOINT", 'http://morph-chain.neofs.devenv:30333') MORPH_ENDPOINT = os.getenv("MORPH_ENDPOINT", 'http://morph-chain.neofs.devenv:30333')
@ -50,6 +49,7 @@ STORAGE_WALLET_PATH_2 = os.getenv("STORAGE_WALLET_PATH_2", f"{DEVENV_PATH}/servi
STORAGE_WALLET_PATH_3 = os.getenv("STORAGE_WALLET_PATH_3", f"{DEVENV_PATH}/services/storage/wallet03.json") STORAGE_WALLET_PATH_3 = os.getenv("STORAGE_WALLET_PATH_3", f"{DEVENV_PATH}/services/storage/wallet03.json")
STORAGE_WALLET_PATH_4 = os.getenv("STORAGE_WALLET_PATH_4", f"{DEVENV_PATH}/services/storage/wallet04.json") STORAGE_WALLET_PATH_4 = os.getenv("STORAGE_WALLET_PATH_4", f"{DEVENV_PATH}/services/storage/wallet04.json")
STORAGE_WALLET_PATH = STORAGE_WALLET_PATH_1 STORAGE_WALLET_PATH = STORAGE_WALLET_PATH_1
STORAGE_WALLET_PASS = os.getenv("STORAGE_WALLET_PASS", "")
NEOFS_NETMAP_DICT = { NEOFS_NETMAP_DICT = {
's01': { 's01': {