anurindm/frostfs-testcases
1
0
Fork 0
forked from TrueCloudLab/frostfs-testcases
Code Pull requests Activity

Remove redundant variables

Browse source 
Small refactoring that includes:
 - Removed variables that are not used any more.
 - Cleanup in helper functions' names.

Signed-off-by: Vladimir Domnich <v.domnich@yadro.com>
This commit is contained in:
Vladimir Domnich 2022-08-01 13:15:02 +03:00
parent 642af0a888
commit 47c55f0060
7 changed files with 77 additions and 66 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
configuration/devenv.yml
View file

@ -50,7 +50,6 @@ neofs_netmap:
# Paths to binaries
neogo_cli_exec: 'neo-go'
neogo_executable: 'neo-go'
neofs_cli_exec: 'neofs-cli'

32
pytest_tests/testsuites/network/test_node_management.py
View file

@ -3,11 +3,10 @@ from random import choice
from time import sleep
import allure
import base58
import pytest
from cli_helpers import _cmd_run
from data_formatters import get_wallet_public_key
from common import (COMPLEX_OBJ_SIZE, MAINNET_BLOCK_TIME, NEOFS_CONTRACT_CACHE_TIMEOUT,
NEOFS_NETMAP_DICT, NEOGO_CLI_EXEC)
NEOFS_NETMAP_DICT, STORAGE_WALLET_PASS)
from epoch import tick_epoch
from python_keywords.container import create_container, get_container
from python_keywords.neofs_verbs import (delete_object, get_object,
@ -82,9 +81,14 @@ def test_nodes_management(prepare_tmp_dir):
random_node = choice(list(NEOFS_NETMAP_DICT))
alive_node = choice([node for node in NEOFS_NETMAP_DICT if node != random_node])
# Calculate public key that identifies node in netmap
# Calculate public key that identifies node in netmap (we need base58-formatted key
# because keys of storage nodes are base58-encoded in netmap)
random_node_wallet_path = NEOFS_NETMAP_DICT[random_node]['wallet_path']
random_node_netmap_key = get_netmap_public_key_from_wallet(random_node_wallet_path)
random_node_netmap_key = get_wallet_public_key(
random_node_wallet_path,
STORAGE_WALLET_PASS,
format="base58"
)
with allure.step('Check node {random_node} is in netmap'):
snapshot = get_netmap_snapshot(node_name=alive_node)
@ -298,7 +302,7 @@ def validate_object_copies(wallet: str, placement_rule: str, file_path: str, exp
@allure.step('Wait for node {node_name} goes online')
def wait_for_node_go_online(node_name: str):
def wait_for_node_go_online(node_name: str) -> None:
timeout, attempts = 5, 20
for _ in range(attempts):
try:
@ -313,7 +317,8 @@ def wait_for_node_go_online(node_name: str):
@allure.step('Wait for {expected_copies} object copies in the wallet')
def wait_for_expected_object_copies(wallet: str, cid: str, oid: str, expected_copies: int = 2):
def wait_for_expected_object_copies(wallet: str, cid: str, oid: str,
expected_copies: int = 2) -> None:
for i in range(2):
copies = get_simple_object_copies(wallet, cid, oid)
if copies == expected_copies:
@ -325,7 +330,7 @@ def wait_for_expected_object_copies(wallet: str, cid: str, oid: str, expected_co
@allure.step('Wait for object to be dropped')
def wait_for_obj_dropped(wallet: str, cid: str, oid: str, checker):
def wait_for_obj_dropped(wallet: str, cid: str, oid: str, checker) -> None:
for _ in range(3):
try:
checker(wallet, cid, oid)
@ -335,14 +340,3 @@ def wait_for_obj_dropped(wallet: str, cid: str, oid: str, checker):
break
else:
raise AssertionError(f'Object {oid} is not dropped from node')
def get_netmap_public_key_from_wallet(wallet_path: str) -> str:
# Get public key from wallet file (it is printed on 2nd line)
cmd = f"{NEOGO_CLI_EXEC} wallet dump-keys -w {wallet_path}"
output = _cmd_run(cmd)
public_key_hex = output.split("\n")[1].strip()
# Encode public key in base58 (this is how it is present in netmap)
public_key_base58 = base58.b58encode(bytes.fromhex(public_key_hex))
return public_key_base58.decode("utf-8")

47
robot/resources/lib/python_keywords/acl.py
View file

@ -1,4 +1,4 @@
#!/usr/bin/python3.8
#!/usr/bin/python3.9
import base64
import json
@ -6,11 +6,12 @@ import os
import re
import uuid
from enum import Enum, auto
from typing import Optional
import base58
from cli_helpers import _cmd_run
from common import ASSETS_DIR, NEOFS_ENDPOINT, WALLET_CONFIG
from data_formatters import pub_key_hex
from common import ASSETS_DIR, NEOFS_CLI_EXEC, NEOFS_ENDPOINT, WALLET_CONFIG
from data_formatters import get_wallet_public_key
from robot.api import logger
from robot.api.deco import keyword
@ -19,9 +20,6 @@ Robot Keywords and helper functions for work with NeoFS ACL.
"""
ROBOT_AUTO_KEYWORDS = False
# path to neofs-cli executable
NEOFS_CLI_EXEC = os.getenv('NEOFS_CLI_EXEC', 'neofs-cli')
EACL_LIFETIME = 100500
@ -37,7 +35,7 @@ class Role(AutoName):
@keyword('Get eACL')
def get_eacl(wallet_path: str, cid: str):
def get_eacl(wallet_path: str, cid: str) -> Optional[str]:
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet_path} '
f'container get-eacl --cid {cid} --config {WALLET_CONFIG}'
@ -54,7 +52,7 @@ def get_eacl(wallet_path: str, cid: str):
@keyword('Set eACL')
def set_eacl(wallet_path: str, cid: str, eacl_table_path: str):
def set_eacl(wallet_path: str, cid: str, eacl_table_path: str) -> None:
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet_path} '
f'container set-eacl --cid {cid} --table {eacl_table_path} --config {WALLET_CONFIG} --await'
@ -68,23 +66,19 @@ def _encode_cid_for_eacl(cid: str) -> str:
@keyword('Create eACL')
def create_eacl(cid: str, rules_list: list):
table = f"{os.getcwd()}/{ASSETS_DIR}/eacl_table_{str(uuid.uuid4())}.json"
rules = ""
for rule in rules_list:
# TODO: check if $Object: is still necessary for filtering in the newest releases
rules += f"--rule '{rule}' "
cmd = (
f"{NEOFS_CLI_EXEC} acl extended create --cid {cid} "
f"{rules}--out {table}"
)
def create_eacl(cid: str, rules_list: list) -> str:
table_file_path = f"{os.getcwd()}/{ASSETS_DIR}/eacl_table_{str(uuid.uuid4())}.json"
# TODO: check if $Object: is still necessary for filtering in the newest releases
rules = " ".join(f"--rule '{rule}'" for rule in rules_list)
cmd = f"{NEOFS_CLI_EXEC} acl extended create --cid {cid} {rules} --out {table_file_path}"
_cmd_run(cmd)
with open(table, 'r') as fout:
table_data = fout.read()
with open(table_file_path, 'r') as file:
table_data = file.read()
logger.info(f"Generated eACL:\n{table_data}")
return table
return table_file_path
@keyword('Form BearerToken File')
@ -164,8 +158,9 @@ def form_bearertoken_file(wif: str, cid: str, eacl_records: list) -> str:
sign_bearer_token(wif, file_path)
return file_path
@keyword('EACL Rules')
def eacl_rules(access: str, verbs: list, user: str):
def eacl_rules(access: str, verbs: list, user: str) -> list[str]:
"""
This function creates a list of eACL rules.
Args:
@ -178,17 +173,17 @@ def eacl_rules(access: str, verbs: list, user: str):
(list): a list of eACL rules
"""
if user not in ('others', 'user'):
pubkey = pub_key_hex(user)
pubkey = get_wallet_public_key(user, wallet_password="")
user = f"pubkey:{pubkey}"
rules = []
for verb in verbs:
elements = [access, verb, user]
rules.append(' '.join(elements))
rule = f"{access} {verb} {user}"
rules.append(rule)
return rules
def sign_bearer_token(wallet_path: str, eacl_rules_file: str):
def sign_bearer_token(wallet_path: str, eacl_rules_file: str) -> None:
cmd = (
f'{NEOFS_CLI_EXEC} util sign bearer-token --from {eacl_rules_file} '
f'--to {eacl_rules_file} --wallet {wallet_path} --config {WALLET_CONFIG} --json'

38
robot/resources/lib/python_keywords/data_formatters.py
View file

@ -1,4 +1,7 @@
import base64
import json
import base58
from neo3 import wallet
@ -16,11 +19,32 @@ def dict_to_attrs(attrs: dict) -> str:
return ",".join(f"{key}={value}" for key, value in attrs.items())
def pub_key_hex(wallet_path: str, wallet_password=""):
wallet_content = ''
with open(wallet_path) as out:
wallet_content = json.load(out)
def __fix_wallet_schema(wallet: dict) -> None:
# Temporary function to fix wallets that do not conform to the schema
# TODO: get rid of it once issue is solved
if "name" not in wallet:
wallet["name"] = None
for account in wallet["accounts"]:
if "extra" not in account:
account["extra"] = None
def get_wallet_public_key(wallet_path: str, wallet_password: str, format: str = "hex") -> str:
# Get public key from wallet file
with open(wallet_path, "r") as file:
wallet_content = json.load(file)
__fix_wallet_schema(wallet_content)
wallet_from_json = wallet.Wallet.from_json(wallet_content, password=wallet_password)
pub_key_64 = str(wallet_from_json.accounts[0].public_key)
return pub_key_64
public_key_hex = str(wallet_from_json.accounts[0].public_key)
# Convert public key to specified format
if format == "hex":
return public_key_hex
if format == "base58":
public_key_base58 = base58.b58encode(bytes.fromhex(public_key_hex))
return public_key_base58.decode("utf-8")
if format == "base64":
public_key_base64 = base64.b64encode(bytes.fromhex(public_key_hex))
return public_key_base64.decode("utf-8")
raise ValueError(f"Invalid public key format: {format}")

8
robot/resources/lib/python_keywords/payment_neogo.py
View file

@ -10,9 +10,8 @@ from robot.api.deco import keyword
import contract
import converters
import rpc_client
from common import (GAS_HASH, MAINNET_SINGLE_ADDR, MAINNET_WALLET_PATH,
MORPH_ENDPOINT, NEO_MAINNET_ENDPOINT, NEOFS_CONTRACT,
NEOGO_CLI_EXEC)
from common import (GAS_HASH, MAINNET_SINGLE_ADDR, MAINNET_WALLET_PATH, MAINNET_WALLET_PASS,
MORPH_ENDPOINT, NEO_MAINNET_ENDPOINT, NEOFS_CONTRACT, NEOGO_EXECUTABLE)
from converters import load_wallet
from wallet import nep17_transfer
from wrappers import run_sh_with_passwd_contract
@ -20,7 +19,6 @@ from wrappers import run_sh_with_passwd_contract
ROBOT_AUTO_KEYWORDS = False
EMPTY_PASSWORD = ''
MAINNET_WALLET_PASS = 'one'
TX_PERSIST_TIMEOUT = 15 # seconds
ASSET_POWER_MAINCHAIN = 10 ** 8
ASSET_POWER_SIDECHAIN = 10 ** 12
@ -35,7 +33,7 @@ def withdraw_mainnet_gas(wlt: str, amount: int):
scripthash = wallet.Account.address_to_script_hash(address)
cmd = (
f"{NEOGO_CLI_EXEC} contract invokefunction -w {wlt} -a {address} "
f"{NEOGO_EXECUTABLE} contract invokefunction -w {wlt} -a {address} "
f"-r {NEO_MAINNET_ENDPOINT} {NEOFS_CONTRACT} withdraw {scripthash} "
f"int:{amount} -- {scripthash}:Global"
)

15
robot/resources/lib/python_keywords/s3_gate_bucket.py
View file

@ -6,16 +6,17 @@ import re
import uuid
from enum import Enum
from time import sleep
from typing import Optional
import boto3
from data_formatters import pub_key_hex
from botocore.exceptions import ClientError
import urllib3
from botocore.exceptions import ClientError
from robot.api import logger
from robot.api.deco import keyword
from cli_helpers import _run_with_passwd, log_command_execution
from common import NEOFS_ENDPOINT, S3_GATE, S3_GATE_WALLET_PATH, S3_GATE_WALLET_PASS
from data_formatters import get_wallet_public_key
##########################################################
# Disabling warnings on self-signed certificate which the
@ -39,13 +40,13 @@ class VersioningStatus(Enum):
@keyword('Init S3 Credentials')
def init_s3_credentials(wallet_path, s3_bearer_rules_file: str = None):
def init_s3_credentials(wallet_path, s3_bearer_rules_file: Optional[str] = None):
bucket = str(uuid.uuid4())
s3_bearer_rules = s3_bearer_rules_file or 'robot/resources/files/s3_bearer_rules.json'
gate_pub_key = pub_key_hex(S3_GATE_WALLET_PATH, S3_GATE_WALLET_PASS)
gate_public_key = get_wallet_public_key(S3_GATE_WALLET_PATH, S3_GATE_WALLET_PASS)
cmd = (
f'{NEOFS_EXEC} --debug --with-log --timeout {CREDENTIALS_CREATE_TIMEOUT} '
f'issue-secret --wallet {wallet_path} --gate-public-key={gate_pub_key} '
f'issue-secret --wallet {wallet_path} --gate-public-key={gate_public_key} '
f'--peer {NEOFS_ENDPOINT} --container-friendly-name {bucket} '
f'--bearer-rules {s3_bearer_rules}'
)
@ -158,7 +159,7 @@ def head_bucket(s3_client, bucket: str):
@keyword('Set bucket versioning status')
def set_bucket_versioning(s3_client, bucket_name: str, status: VersioningStatus):
def set_bucket_versioning(s3_client, bucket_name: str, status: VersioningStatus) -> None:
try:
response = s3_client.put_bucket_versioning(Bucket=bucket_name, VersioningConfiguration={'Status': status.value})
log_command_execution('S3 Set bucket versioning to', response)
@ -202,7 +203,7 @@ def get_bucket_tagging(s3_client, bucket_name: str) -> list:
@keyword('Delete bucket tagging')
def delete_bucket_tagging(s3_client, bucket_name: str):
def delete_bucket_tagging(s3_client, bucket_name: str) -> None:
try:
response = s3_client.delete_bucket_tagging(Bucket=bucket_name)
log_command_execution('S3 Delete bucket tagging', response)

2
robot/variables/common.py
View file

@ -18,7 +18,6 @@ NEOFS_CONTRACT_CACHE_TIMEOUT = os.getenv("NEOFS_CONTRACT_CACHE_TIMEOUT", "30s")
SHARD_0_GC_SLEEP = os.getenv("NEOFS_STORAGE_SHARD_0_GC_REMOVER_SLEEP_INTERVAL", "1m")
NEOFS_ENDPOINT = os.getenv("NEOFS_ENDPOINT", "s01.neofs.devenv:8080")
NEOGO_CLI_EXEC = os.getenv("NEOGO_EXECUTABLE", "neo-go")
NEO_MAINNET_ENDPOINT = os.getenv("NEO_MAINNET_ENDPOINT", 'http://main-chain.neofs.devenv:30333')
MORPH_ENDPOINT = os.getenv("MORPH_ENDPOINT", 'http://morph-chain.neofs.devenv:30333')
@ -50,6 +49,7 @@ STORAGE_WALLET_PATH_2 = os.getenv("STORAGE_WALLET_PATH_2", f"{DEVENV_PATH}/servi
STORAGE_WALLET_PATH_3 = os.getenv("STORAGE_WALLET_PATH_3", f"{DEVENV_PATH}/services/storage/wallet03.json")
STORAGE_WALLET_PATH_4 = os.getenv("STORAGE_WALLET_PATH_4", f"{DEVENV_PATH}/services/storage/wallet04.json")
STORAGE_WALLET_PATH = STORAGE_WALLET_PATH_1
STORAGE_WALLET_PASS = os.getenv("STORAGE_WALLET_PASS", "")
NEOFS_NETMAP_DICT = {
's01': {