From a97e1ee1e98eb5c2cb87767ad629e484ccdc2ca8 Mon Sep 17 00:00:00 2001 From: anastasia prasolova Date: Thu, 26 May 2022 16:40:18 +0300 Subject: [PATCH] [#184]: keywords for nodes management out into separate library Signed-off-by: anastasia prasolova --- robot/resources/lib/python/neofs.py | 129 ------------------ .../resources/lib/python/nodes_management.py | 47 +++++++ robot/resources/lib/python/storage_policy.py | 46 +++++++ .../integration/network/netmap_simple.robot | 3 - .../integration/network/replication.robot | 10 +- .../integration/services/http_gate.robot | 1 + 6 files changed, 98 insertions(+), 138 deletions(-) create mode 100644 robot/resources/lib/python/nodes_management.py diff --git a/robot/resources/lib/python/neofs.py b/robot/resources/lib/python/neofs.py index 1c76923..829e840 100644 --- a/robot/resources/lib/python/neofs.py +++ b/robot/resources/lib/python/neofs.py @@ -1,13 +1,11 @@ #!/usr/bin/python3 import base64 -from datetime import datetime import json import os import re import random import uuid -import docker import base58 from neo3 import wallet @@ -31,64 +29,6 @@ def get_scripthash(wif: str): return str(acc.script_hash) -@keyword('Stop nodes') -def stop_nodes(down_num: int, nodes_list: list): - - # select nodes to stop from list - nodes = random.sample(nodes_list, down_num) - - for node in nodes: - m = re.search(r'(s\d+).', node) - node = m.group(1) - - client = docker.APIClient() - client.stop(node) - - return nodes - - -@keyword('Start nodes') -def start_nodes(nodes_list: list): - - for node in nodes_list: - m = re.search(r'(s\d+).', node) - node = m.group(1) - client = docker.APIClient() - client.start(node) - - -@keyword('Get nodes with object') -def get_nodes_with_object(wallet: str, cid: str, oid: str): - - nodes_list = [] - - for node in NEOFS_NETMAP: - res = _search_object(node, wallet, cid, oid) - if res: - if oid in res: - nodes_list.append(node) - - logger.info(f"Nodes with object: {nodes_list}") - return nodes_list - - -@keyword('Get nodes without object') -def get_nodes_without_object(wallet: str, cid: str, oid: str): - - nodes_list = [] - - for node in NEOFS_NETMAP: - search_res = _search_object(node, wallet, cid, oid) - if search_res: - if not re.search(fr'({oid})', search_res): - nodes_list.append(node) - else: - nodes_list.append(node) - - logger.info(f"Nodes without object: {nodes_list}") - return nodes_list - - @keyword('Verify Head Tombstone') def verify_head_tombstone(wallet: str, cid: str, oid_ts: str, oid: str, addr: str): # TODO: replace with HEAD from neofs_verbs.py @@ -165,66 +105,6 @@ def get_locode(): return locode -@keyword('Get Nodes Log Latest Timestamp') -def get_logs_latest_timestamp(): - """ - Keyword return: - nodes_logs_time -- structure (dict) of nodes container name (key) and latest logs timestamp (value) - """ - client_api = docker.APIClient() - - nodes_logs_time = dict() - - for node in NEOFS_NETMAP: - container = node.split('.')[0] - log_line = client_api.logs(container, tail=1) - - m = re.search(r'(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z)', str(log_line)) - if m is not None: - timestamp = m.group(1) - - timestamp_date = datetime.fromisoformat(timestamp[:-1]) - - nodes_logs_time[container] = timestamp_date - - logger.info(f"Latest logs timestamp list: {nodes_logs_time}") - - return nodes_logs_time - - -@keyword('Find in Nodes Log') -def find_in_nodes_log(line: str, nodes_logs_time: dict): - - client_api = docker.APIClient() - container_names = list() - - for docker_container in client_api.containers(): - container_names.append(docker_container['Names'][0][1:]) - - global_count = 0 - - for container in nodes_logs_time.keys(): - # check if container exists - if container in container_names: - # Get log since timestamp - timestamp_date = nodes_logs_time[container] - log_lines = client_api.logs(container, since=timestamp_date) - logger.info(f"Timestamp since: {timestamp_date}") - found_count = len(re.findall(line, log_lines.decode("utf-8") )) - logger.info(f"Node {container} log - found counter: {found_count}") - global_count += found_count - - else: - logger.info(f"Container {container} has not been found.") - - if global_count > 0: - logger.info(f"Expected line '{line}' has been found in the logs.") - else: - raise Exception(f"Expected line '{line}' has not been found in the logs.") - - return 1 - - @keyword('Generate Session Token') def generate_session_token(owner: str, pub_key: str, cid: str = "", wildcard: bool = False) -> str: @@ -273,12 +153,3 @@ def sign_session_token(session_token: str, wallet: str, to_file: str=''): ) logger.info(f"cmd: {cmd}") _cmd_run(cmd) - - -def _search_object(node:str, wallet: str, cid:str, oid: str): - cmd = ( - f'{NEOFS_CLI_EXEC} --rpc-endpoint {node} --wallet {wallet} --ttl 1 ' - f'object search --root --cid {cid} --oid {oid} --config {WALLET_PASS}' - ) - output = _cmd_run(cmd) - return output diff --git a/robot/resources/lib/python/nodes_management.py b/robot/resources/lib/python/nodes_management.py new file mode 100644 index 0000000..b5812d7 --- /dev/null +++ b/robot/resources/lib/python/nodes_management.py @@ -0,0 +1,47 @@ +#!/usr/bin/python3 + +""" + This module contains keywords for management test stand + nodes. It assumes that nodes are docker containers. +""" + +import random + +import docker + +from robot.api.deco import keyword + +ROBOT_AUTO_KEYWORDS = False + +@keyword('Stop Nodes') +def stop_nodes(number: int, nodes: list): + """ + The function shuts down the given number of randomly + selected nodes in docker. + Args: + number (int): the number of nodes to shut down + nodes (list): the list of nodes for possible shut down + Returns: + (list): the list of nodes which have been shut down + """ + nodes = random.sample(nodes, number) + client = docker.APIClient() + for node in nodes: + node = node.split('.')[0] + client.stop(node) + return nodes + + +@keyword('Start Nodes') +def start_nodes(nodes: list): + """ + The function raises the given nodes. + Args: + nodes (list): the list of nodes to raise + Returns: + (void) + """ + client = docker.APIClient() + for node in nodes: + node = node.split('.')[0] + client.start(node) diff --git a/robot/resources/lib/python/storage_policy.py b/robot/resources/lib/python/storage_policy.py index 41aede7..8ae7ead 100644 --- a/robot/resources/lib/python/storage_policy.py +++ b/robot/resources/lib/python/storage_policy.py @@ -77,3 +77,49 @@ def get_complex_object_copies(wallet: str, cid: str, oid: str): """ last_oid = complex_object_actions.get_last_object(wallet, cid, oid) return get_simple_object_copies(wallet, cid, last_oid) + + +@keyword('Get Nodes With Object') +def get_nodes_with_object(wallet: str, cid: str, oid: str): + """ + The function returns list of nodes which store + the given object. + Args: + wallet (str): the path to the wallet on whose behalf + we request the nodes + cid (str): ID of the container which store the object + oid (str): object ID + Returns: + (list): nodes which store the object + """ + nodes_list = [] + for node in NEOFS_NETMAP: + res = neofs_verbs.head_object(wallet, cid, oid, + endpoint=node, + is_direct=True) + if res is not None: + nodes_list.append(node) + return nodes_list + + +@keyword('Get Nodes Without Object') +def get_nodes_without_object(wallet: str, cid: str, oid: str): + """ + The function returns list of nodes which do not store + the given object. + Args: + wallet (str): the path to the wallet on whose behalf + we request the nodes + cid (str): ID of the container which store the object + oid (str): object ID + Returns: + (list): nodes which do not store the object + """ + nodes_list = [] + for node in NEOFS_NETMAP: + res = neofs_verbs.head_object(wallet, cid, oid, + endpoint=node, + is_direct=True) + if res is None: + nodes_list.append(node) + return nodes_list diff --git a/robot/testsuites/integration/network/netmap_simple.robot b/robot/testsuites/integration/network/netmap_simple.robot index 732a1d2..d66915f 100644 --- a/robot/testsuites/integration/network/netmap_simple.robot +++ b/robot/testsuites/integration/network/netmap_simple.robot @@ -1,8 +1,5 @@ *** Settings *** -Variables common.py - Library container.py -Library neofs.py Library neofs_verbs.py Library storage_policy.py Library utility_keywords.py diff --git a/robot/testsuites/integration/network/replication.robot b/robot/testsuites/integration/network/replication.robot index 0402423..e1597c6 100644 --- a/robot/testsuites/integration/network/replication.robot +++ b/robot/testsuites/integration/network/replication.robot @@ -4,8 +4,8 @@ Variables wellknown_acl.py Library container.py Library contract_keywords.py -Library neofs.py Library neofs_verbs.py +Library nodes_management.py Library storage_policy.py Library utility_keywords.py @@ -46,10 +46,9 @@ Check Replication ${COPIES} = Get Object Copies Simple ${WALLET} ${CID} ${S_OID} Should Be Equal ${EXPECTED_COPIES} ${COPIES} - @{NODES_OBJ} = Get nodes with Object ${WALLET} ${CID} ${S_OID} - ${NODES_LOG_TIME} = Get Nodes Log Latest Timestamp + @{NODES_OBJ} = Get Nodes With Object ${WALLET} ${CID} ${S_OID} - @{NODES_OBJ_STOPPED} = Stop nodes 1 ${NODES_OBJ} + @{NODES_OBJ_STOPPED} = Stop Nodes 1 ${NODES_OBJ} @{NETMAP} = Convert To List ${NEOFS_NETMAP} Remove Values From List ${NETMAP} ${NODES_OBJ_STOPPED} @@ -65,8 +64,7 @@ Check Replication Run Keyword Unless ${PASSED} Fail ... Storage policy for object ${S_OID} in container ${CID} isn't valid - Find in Nodes Log object successfully replicated ${NODES_LOG_TIME} - Start nodes ${NODES_OBJ_STOPPED} + Start Nodes ${NODES_OBJ_STOPPED} Tick Epoch # We have 2 or 3 copies. Expected behaviour: during two epochs potential 3rd copy should be removed. diff --git a/robot/testsuites/integration/services/http_gate.robot b/robot/testsuites/integration/services/http_gate.robot index 54ad29c..541d6a0 100644 --- a/robot/testsuites/integration/services/http_gate.robot +++ b/robot/testsuites/integration/services/http_gate.robot @@ -6,6 +6,7 @@ Library container.py Library neofs.py Library neofs_verbs.py Library http_gate.py +Library storage_policy.py Library utility_keywords.py Resource payment_operations.robot