r.chernykh/frostfs-testcases
1
0
Fork 0
forked from TrueCloudLab/frostfs-testcases
Code Pull requests Activity

Some linter fixes.

Browse source 
Signed-off-by: a.y.volkov <a.y.volkov@yadro.com>
This commit is contained in:
a.y.volkov 2022-06-09 16:08:11 +03:00 committed by Anastasia Prasolova
parent e086d0d62b
commit 0e27ea02c1
19 changed files with 291 additions and 275 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
robot/resources/lib/python_keywords/acl.py
View file

@ -1,34 +1,34 @@
#!/usr/bin/python3.8
from enum import Enum, auto
import base64
import json
import os
import re
import uuid
from enum import Enum, auto
import base64
import base58
from cli_helpers import _cmd_run
from common import ASSETS_DIR, NEOFS_ENDPOINT, WALLET_PASS
from robot.api.deco import keyword
from robot.api import logger
from robot.api.deco import keyword
"""
Robot Keywords and helper functions for work with NeoFS ACL.
"""
ROBOT_AUTO_KEYWORDS = False
# path to neofs-cli executable
NEOFS_CLI_EXEC = os.getenv('NEOFS_CLI_EXEC', 'neofs-cli')
EACL_LIFETIME = 100500
class AutoName(Enum):
def _generate_next_value_(name, start, count, last_values):
return name
class Role(AutoName):
USER = auto()
SYSTEM = auto()
@ -65,6 +65,7 @@ def _encode_cid_for_eacl(cid: str) -> str:
cid_base58 = base58.b58decode(cid)
return base64.b64encode(cid_base58).decode("utf-8")
@keyword('Create eACL')
def create_eacl(cid: str, rules_list: list):
table = f"{os.getcwd()}/{ASSETS_DIR}/eacl_table_{str(uuid.uuid4())}.json"

3
robot/resources/lib/python_keywords/cli_helpers.py
View file

@ -6,8 +6,8 @@ and other CLIs.
"""
import subprocess
import pexpect
import pexpect
from robot.api import logger
ROBOT_AUTO_KEYWORDS = False
@ -35,6 +35,7 @@ def _cmd_run(cmd, timeout=30):
f"{exc.output.decode('utf-8') if type(exc.output) is bytes else exc.output}")
raise
def _run_with_passwd(cmd):
child = pexpect.spawn(cmd)
child.expect(".*")

1
robot/resources/lib/python_keywords/cli_keywords.py
View file

@ -6,6 +6,7 @@ from robot.api.deco import keyword
ROBOT_AUTO_KEYWORDS = False
@keyword('Run Process And Enter Empty Password')
def run_proccess_and_interact(cmd: str) -> str:
p = pexpect.spawn(cmd)

8
robot/resources/lib/python_keywords/complex_object_actions.py
View file

@ -10,13 +10,13 @@
first non-null response.
"""
from common import NEOFS_NETMAP
import neofs_verbs
from robot.api.deco import keyword
from robot.api import logger
from robot.api.deco import keyword
from robot.libraries.BuiltIn import BuiltIn
import neofs_verbs
from common import NEOFS_NETMAP
ROBOT_AUTO_KEYWORDS = False

11
robot/resources/lib/python_keywords/container.py
View file

@ -8,14 +8,13 @@
import json
import time
from common import NEOFS_ENDPOINT, COMMON_PLACEMENT_RULE, NEOFS_CLI_EXEC, WALLET_PASS
from cli_helpers import _cmd_run
from data_formatters import dict_to_attrs
import json_transformers
from robot.api.deco import keyword
from robot.api import logger
from robot.api.deco import keyword
import json_transformers
from cli_helpers import _cmd_run
from common import NEOFS_ENDPOINT, COMMON_PLACEMENT_RULE, NEOFS_CLI_EXEC, WALLET_PASS
from data_formatters import dict_to_attrs
ROBOT_AUTO_KEYWORDS = False

8
robot/resources/lib/python_keywords/data_formatters.py
View file

@ -7,15 +7,15 @@ from functools import reduce
def dict_to_attrs(attrs: dict):
'''
"""
This function takes dictionary of object attributes and converts them
into the string. The string is passed to `--attibutes` key of the
into the string. The string is passed to `--attributes` key of the
neofs-cli.
Args:
attrs (dict): object attirbutes in {"a": "b", "c": "d"} format.
attrs (dict): object attributes in {"a": "b", "c": "d"} format.
Returns:
(str): string in "a=b,c=d" format.
'''
"""
return reduce(lambda a, b: f"{a},{b}", map(lambda i: f"{i}={attrs[i]}", attrs))

7
robot/resources/lib/python_keywords/http_gate.py
View file

@ -3,15 +3,16 @@
import shutil
import requests
from robot.api import logger
from robot.api.deco import keyword
from robot.libraries.BuiltIn import BuiltIn
from common import HTTP_GATE
from robot.api.deco import keyword
from robot.api import logger
from robot.libraries.BuiltIn import BuiltIn
ROBOT_AUTO_KEYWORDS = False
ASSETS_DIR = BuiltIn().get_variable_value("${ASSETS_DIR}")
@keyword('Get via HTTP Gate')
def get_via_http_gate(cid: str, oid: str):
"""

47
robot/resources/lib/python_keywords/json_transformers.py
View file

@ -1,24 +1,25 @@
#!/usr/bin/python3
'''
"""
When doing requests to NeoFS, we get JSON output as an automatically decoded
structure from protobuf. Some fields are decoded with boilerplates and binary
values are Base64-encoded.
This module contains functions which rearrange the structure and reencode binary
data from Base64 to Base58.
'''
"""
import base64
import base58
ROBOT_AUTO_KEYWORDS = False
def decode_simple_header(data: dict):
'''
"""
This function reencodes Simple Object header and its attributes.
'''
"""
try:
data = decode_common_fields(data)
@ -34,12 +35,12 @@ def decode_simple_header(data: dict):
def decode_split_header(data: dict):
'''
"""
This function rearranges Complex Object header.
The header holds SplitID, a random unique
number, which is common among all splitted objects, and IDs of the Linking
Object and the last splitted Object.
'''
"""
try:
data["splitId"] = json_reencode(data["splitId"])
data["lastPart"] = (
@ -55,11 +56,12 @@ def decode_split_header(data: dict):
return data
def decode_linking_object(data: dict):
'''
"""
This function reencodes Linking Object header.
It contains IDs of child Objects and Split Chain data.
'''
"""
try:
data = decode_simple_header(data)
# reencoding Child Object IDs
@ -82,9 +84,9 @@ def decode_linking_object(data: dict):
def decode_storage_group(data: dict):
'''
"""
This function reencodes Storage Group header.
'''
"""
try:
data = decode_common_fields(data)
except Exception as exc:
@ -92,10 +94,11 @@ def decode_storage_group(data: dict):
return data
def decode_tombstone(data: dict):
'''
"""
This function reencodes Tombstone header.
'''
"""
try:
data = decode_simple_header(data)
data['header']['sessionToken'] = decode_session_token(
@ -106,39 +109,41 @@ def decode_tombstone(data: dict):
def decode_session_token(data: dict):
'''
"""
This function reencodes a fragment of header which contains
information about session token.
'''
"""
data['body']['object']['address']['containerID'] = json_reencode(
data['body']['object']['address']['containerID']['value'])
data['body']['object']['address']['objectID'] = json_reencode(
data['body']['object']['address']['objectID']['value'])
return data
def json_reencode(data: str):
'''
"""
According to JSON protocol, binary data (Object/Container/Storage Group IDs, etc)
is converted to string via Base58 encoder. But we usually operate with Base64-encoded
format.
This function reencodes given Base58 string into the Base64 one.
'''
"""
return base58.b58encode(base64.b64decode(data)).decode("utf-8")
def encode_for_json(data: str):
'''
"""
This function encodes binary data for sending them as protobuf
structures.
'''
"""
return base64.b64encode(base58.b58decode(data)).decode('utf-8')
def decode_common_fields(data: dict):
'''
"""
Despite of type (simple/complex Object, Storage Group, etc) every Object
header contains several common fields.
This function rearranges these fields.
'''
"""
# reencoding binary IDs
data["objectID"] = json_reencode(data["objectID"]["value"])
data["header"]["containerID"] = json_reencode(data["header"]["containerID"]["value"])

7
robot/resources/lib/python_keywords/neofs.py
View file

@ -5,12 +5,13 @@ import os
import random
from neo3 import wallet
from common import NEOFS_NETMAP_DICT
import neofs_verbs
from robot.api.deco import keyword
from robot.api import logger
from robot.api.deco import keyword
from robot.libraries.BuiltIn import BuiltIn
import neofs_verbs
from common import NEOFS_NETMAP_DICT
ROBOT_AUTO_KEYWORDS = False
# path to neofs-cli executable

43
robot/resources/lib/python_keywords/neofs_verbs.py
View file

@ -6,17 +6,17 @@
import json
import os
import re
import random
import re
import uuid
from common import NEOFS_ENDPOINT, ASSETS_DIR, NEOFS_NETMAP, WALLET_PASS
from cli_helpers import _cmd_run
import json_transformers
from data_formatters import dict_to_attrs
from robot.api.deco import keyword
from robot.api import logger
from robot.api.deco import keyword
import json_transformers
from cli_helpers import _cmd_run
from common import NEOFS_ENDPOINT, ASSETS_DIR, NEOFS_NETMAP, WALLET_PASS
from data_formatters import dict_to_attrs
ROBOT_AUTO_KEYWORDS = False
@ -27,7 +27,7 @@ NEOFS_CLI_EXEC = os.getenv('NEOFS_CLI_EXEC', 'neofs-cli')
@keyword('Get object')
def get_object(wallet: str, cid: str, oid: str, bearer_token: str = "",
write_object: str = "", endpoint: str = "", options: str = ""):
'''
"""
GET from NeoFS.
Args:
@ -40,7 +40,7 @@ def get_object(wallet: str, cid: str, oid: str, bearer_token: str="",
options (optional, str): any options which `neofs-cli object get` accepts
Returns:
(str): path to downloaded file
'''
"""
if not write_object:
write_object = str(uuid.uuid4())
@ -63,7 +63,7 @@ def get_object(wallet: str, cid: str, oid: str, bearer_token: str="",
@keyword('Get Range Hash')
def get_range_hash(wallet: str, cid: str, oid: str, bearer_token: str, range_cut: str,
options: str = ""):
'''
"""
GETRANGEHASH of given Object.
Args:
@ -76,7 +76,7 @@ def get_range_hash(wallet: str, cid: str, oid: str, bearer_token: str, range_cut
options (optional, str): any options which `neofs-cli object hash` accepts
Returns:
None
'''
"""
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet} '
f'object hash --cid {cid} --oid {oid} --range {range_cut} --config {WALLET_PASS} '
@ -91,7 +91,7 @@ def get_range_hash(wallet: str, cid: str, oid: str, bearer_token: str, range_cut
@keyword('Put object')
def put_object(wallet: str, path: str, cid: str, bearer: str = "", user_headers: dict = {},
endpoint: str = "", options: str = ""):
'''
"""
PUT of given file.
Args:
@ -104,7 +104,7 @@ def put_object(wallet: str, path: str, cid: str, bearer: str="", user_headers: d
options (optional, str): any options which `neofs-cli object put` accepts
Returns:
(str): ID of uploaded Object
'''
"""
if not endpoint:
endpoint = random.sample(NEOFS_NETMAP, 1)[0]
cmd = (
@ -122,7 +122,7 @@ def put_object(wallet: str, path: str, cid: str, bearer: str="", user_headers: d
@keyword('Delete object')
def delete_object(wallet: str, cid: str, oid: str, bearer: str = "", options: str = ""):
'''
"""
DELETE an Object.
Args:
@ -133,7 +133,7 @@ def delete_object(wallet: str, cid: str, oid: str, bearer: str="", options: str=
options (optional, str): any options which `neofs-cli object delete` accepts
Returns:
(str): Tombstone ID
'''
"""
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet} '
f'object delete --cid {cid} --oid {oid} {options} --config {WALLET_PASS} '
@ -150,19 +150,20 @@ def delete_object(wallet: str, cid: str, oid: str, bearer: str="", options: str=
@keyword('Get Range')
def get_range(wallet: str, cid: str, oid: str, file_path: str, bearer: str, range_cut: str,
options: str = ""):
'''
"""
GETRANGE an Object.
Args:
wallet (str): wallet on whose behalf GETRANGE is done
cid (str): ID of Container where we get the Object from
oid (str): ID of Object we are going to request
file_path (str): file path
range_cut (str): range to take data from in the form offset:length
bearer (optional, str): path to Bearer Token file, appends to `--bearer` key
options (optional, str): any options which `neofs-cli object range` accepts
Returns:
(void)
'''
"""
range_file = f"{ASSETS_DIR}/{uuid.uuid4()}"
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet} '
@ -180,7 +181,7 @@ def get_range(wallet: str, cid: str, oid: str, file_path: str, bearer: str, rang
@keyword('Search object')
def search_object(wallet: str, cid: str, keys: str = "", bearer: str = "", filters: dict = {},
expected_objects_list=[]):
'''
"""
SEARCH an Object.
Args:
@ -193,7 +194,7 @@ def search_object(wallet: str, cid: str, keys: str="", bearer: str="", filters:
expected_objects_list (optional, list): a list of ObjectIDs to compare found Objects with
Returns:
(list): list of found ObjectIDs
'''
"""
filters_result = ""
if filters:
filters_result += "--filters "
@ -224,7 +225,7 @@ def search_object(wallet: str, cid: str, keys: str="", bearer: str="", filters:
def head_object(wallet: str, cid: str, oid: str, bearer_token: str = "",
options: str = "", endpoint: str = "", json_output: bool = True,
is_raw: bool = False, is_direct: bool = False):
'''
"""
HEAD an Object.
Args:
@ -245,7 +246,7 @@ def head_object(wallet: str, cid: str, oid: str, bearer_token: str="",
(dict): HEAD response in JSON format
or
(str): HEAD response as a plain text
'''
"""
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {endpoint if endpoint else NEOFS_ENDPOINT} '
f'--wallet {wallet} --config {WALLET_PASS} '

2
robot/resources/lib/python_keywords/nodes_management.py
View file

@ -8,11 +8,11 @@
import random
import docker
from robot.api.deco import keyword
ROBOT_AUTO_KEYWORDS = False
@keyword('Stop Nodes')
def stop_nodes(number: int, nodes: list):
"""

11
robot/resources/lib/python_keywords/payment_neogo.py
View file

@ -1,16 +1,15 @@
#!/usr/bin/python3
import os
import pexpect
import re
from robot.api.deco import keyword
from robot.api import logger
import pexpect
from neo3 import wallet
from robot.api import logger
from robot.api.deco import keyword
from common import *
import rpc_client
import contract
import rpc_client
from common import *
from wrappers import run_sh_with_passwd_contract
ROBOT_AUTO_KEYWORDS = False

10
robot/resources/lib/python_keywords/s3_gate.py
View file

@ -6,12 +6,12 @@ import uuid
import boto3
import botocore
from cli_helpers import _run_with_passwd
from common import GATE_PUB_KEY, NEOFS_ENDPOINT, S3_GATE
import urllib3
from robot.api.deco import keyword
from robot.api import logger
from robot.api.deco import keyword
from cli_helpers import _run_with_passwd
from common import GATE_PUB_KEY, NEOFS_ENDPOINT, S3_GATE
##########################################################
# Disabling warnings on self-signed certificate which the
@ -24,6 +24,7 @@ CREDENTIALS_CREATE_TIMEOUT = '30s'
NEOFS_EXEC = os.getenv('NEOFS_EXEC', 'neofs-authmate')
@keyword('Init S3 Credentials')
def init_s3_credentials(wallet):
bucket = str(uuid.uuid4())
@ -179,7 +180,6 @@ def put_object_s3(s3_client, bucket, filepath):
@keyword('Head object S3')
def head_object_s3(s3_client, bucket, object_key):
try:
response = s3_client.head_object(Bucket=bucket, Key=object_key)
logger.info(f"S3 Head object result: {response}")

13
robot/resources/lib/python_keywords/session_token.py
View file

@ -10,12 +10,12 @@ import os
import uuid
from neo3 import wallet
from common import WALLET_PASS, ASSETS_DIR
from cli_helpers import _cmd_run
import json_transformers
from robot.api.deco import keyword
from robot.api import logger
from robot.api.deco import keyword
import json_transformers
from cli_helpers import _cmd_run
from common import WALLET_PASS, ASSETS_DIR
ROBOT_AUTO_KEYWORDS = False
@ -68,8 +68,7 @@ def generate_session_token(owner: str, session_wallet: str, cid: str='') -> str:
"verb": "PUT",
"wildcard": cid != '',
**({"containerID":
{"value":
f"{base64.b64encode(cid.encode('utf-8')).decode('utf-8')}"}
{"value": f"{base64.b64encode(cid.encode('utf-8')).decode('utf-8')}"}
} if cid != '' else {}
)
}

6
robot/resources/lib/python_keywords/storage_group.py
View file

@ -5,11 +5,11 @@
It contains wrappers for `neofs-cli storagegroup` verbs.
"""
from cli_helpers import _cmd_run
from common import NEOFS_CLI_EXEC, NEOFS_ENDPOINT, WALLET_PASS
from robot.api.deco import keyword
from cli_helpers import _cmd_run
from common import NEOFS_CLI_EXEC, NEOFS_ENDPOINT, WALLET_PASS
ROBOT_AUTO_KEYWORDS = False

8
robot/resources/lib/python_keywords/storage_policy.py
View file

@ -5,12 +5,12 @@
that storage policies are kept.
'''
from common import NEOFS_NETMAP
from robot.api import logger
from robot.api.deco import keyword
import complex_object_actions
import neofs_verbs
from robot.api.deco import keyword
from robot.api import logger
from common import NEOFS_NETMAP
ROBOT_AUTO_KEYWORDS = False

13
robot/resources/lib/python_keywords/utility_keywords.py
View file

@ -1,20 +1,21 @@
#!/usr/bin/python3.8
import hashlib
import os
import tarfile
import uuid
import hashlib
import docker
from common import SIMPLE_OBJ_SIZE, ASSETS_DIR
from cli_helpers import _cmd_run
from robot.api.deco import keyword
import docker
from robot.api import logger
from robot.api.deco import keyword
from robot.libraries.BuiltIn import BuiltIn
from cli_helpers import _cmd_run
from common import SIMPLE_OBJ_SIZE, ASSETS_DIR
ROBOT_AUTO_KEYWORDS = False
@keyword('Generate file')
def generate_file_and_file_hash(size: int) -> str:
"""
@ -67,6 +68,7 @@ def get_container_logs(testcase_name: str) -> None:
os.remove(file_name)
tar.close()
@keyword('Make Up')
def make_up(services: list = [], config_dict: dict = {}):
test_path = os.getcwd()
@ -87,6 +89,7 @@ def make_up(services: list=[], config_dict: dict={}):
os.chdir(test_path)
@keyword('Make Down')
def make_down(services: list = []):
test_path = os.getcwd()

15
robot/variables/common.py
View file

@ -43,22 +43,27 @@ COMMON_PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 4 FROM * AS X"
ASSETS_DIR = os.getenv("ASSETS_DIR", "TemporaryDir/")
MORPH_MAGIC = os.environ["MORPH_MAGIC"]
MORPH_MAGIC = os.getenv("MORPH_MAGIC")
GATE_PUB_KEY = '0313b1ac3a8076e155a7e797b24f0b650cccad5941ea59d7cfd51a024a8b2a06bf'
NEOFS_NETMAP_DICT = {'s01': {'rpc': 's01.neofs.devenv:8080',
STORAGE_NODE_1 = os.getenv('DATA_NODE_1', 's01.neofs.devenv:8080')
STORAGE_NODE_2 = os.getenv('DATA_NODE_2', 's02.neofs.devenv:8080')
STORAGE_NODE_3 = os.getenv('DATA_NODE_3', 's03.neofs.devenv:8080')
STORAGE_NODE_4 = os.getenv('DATA_NODE_4', 's04.neofs.devenv:8080')
NEOFS_NETMAP_DICT = {'s01': {'rpc': STORAGE_NODE_1,
'control': 's01.neofs.devenv:8081',
'wif': 'Kwk6k2eC3L3QuPvD8aiaNyoSXgQ2YL1bwS5CP1oKoA9waeAze97s',
'UN-LOCODE': 'RU MOW'},
's02': {'rpc': 's02.neofs.devenv:8080',
's02': {'rpc': STORAGE_NODE_2,
'control': 's02.neofs.devenv:8081',
'wif': 'L1NdHdnrTNGQZH1fJSrdUZJyeYFHvaQSSHZHxhK3udiGFdr5YaZ6',
'UN-LOCODE': 'RU LED'},
's03': {'rpc': 's03.neofs.devenv:8080',
's03': {'rpc': STORAGE_NODE_3,
'control': 's03.neofs.devenv:8081',
'wif': 'KzN38k39af6ACWJjK8YrnARWo86ddcc1EuBWz7xFEdcELcP3ZTym',
'UN-LOCODE': 'SE STO'},
's04': {'rpc': 's04.neofs.devenv:8080',
's04': {'rpc': STORAGE_NODE_4,
'control': 's04.neofs.devenv:8081',
'wif': 'Kzk1Z3dowAqfNyjqeYKWenZMduFV3NAKgXg9K1sA4jRKYxEc8HEW',
'UN-LOCODE': 'FI HEL'}