[#184]: keywords for nodes management out into separate library

Signed-off-by: anastasia prasolova <anastasia@nspcc.ru>

robot/resources/lib/python/neofs.py

@@ -1,13 +1,11 @@
 #!/usr/bin/python3
 
 import base64
-from datetime import datetime
 import json
 import os
 import re
 import random
 import uuid
-import docker
 import base58
 
 from neo3 import wallet
@@ -31,64 +29,6 @@ def get_scripthash(wif: str):
     return str(acc.script_hash)
 
 
-@keyword('Stop nodes')
-def stop_nodes(down_num: int, nodes_list: list):
-
-    # select nodes to stop from list
-    nodes = random.sample(nodes_list, down_num)
-
-    for node in nodes:
-        m = re.search(r'(s\d+).', node)
-        node = m.group(1)
-
-        client = docker.APIClient()
-        client.stop(node)
-
-    return nodes
-
-
-@keyword('Start nodes')
-def start_nodes(nodes_list: list):
-
-    for node in nodes_list:
-        m = re.search(r'(s\d+).', node)
-        node = m.group(1)
-        client = docker.APIClient()
-        client.start(node)
-
-
-@keyword('Get nodes with object')
-def get_nodes_with_object(wallet: str, cid: str, oid: str):
-
-    nodes_list = []
-
-    for node in NEOFS_NETMAP:
-        res = _search_object(node, wallet, cid, oid)
-        if res:
-            if oid in res:
-                nodes_list.append(node)
-
-    logger.info(f"Nodes with object: {nodes_list}")
-    return nodes_list
-
-
-@keyword('Get nodes without object')
-def get_nodes_without_object(wallet: str, cid: str, oid: str):
-
-    nodes_list = []
-
-    for node in NEOFS_NETMAP:
-        search_res = _search_object(node, wallet, cid, oid)
-        if search_res:
-            if not re.search(fr'({oid})', search_res):
-                nodes_list.append(node)
-        else:
-            nodes_list.append(node)
-
-    logger.info(f"Nodes without object: {nodes_list}")
-    return nodes_list
-
-
 @keyword('Verify Head Tombstone')
 def verify_head_tombstone(wallet: str, cid: str, oid_ts: str, oid: str, addr: str):
     # TODO: replace with HEAD from neofs_verbs.py
@@ -165,66 +105,6 @@ def get_locode():
     return locode
 
 
-@keyword('Get Nodes Log Latest Timestamp')
-def get_logs_latest_timestamp():
-    """
-    Keyword return:
-    nodes_logs_time -- structure (dict) of nodes container name (key) and latest logs timestamp (value)
-    """
-    client_api = docker.APIClient()
-
-    nodes_logs_time = dict()
-
-    for node in NEOFS_NETMAP:
-        container = node.split('.')[0]
-        log_line = client_api.logs(container, tail=1)
-
-        m = re.search(r'(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z)', str(log_line))
-        if m is not None:
-            timestamp = m.group(1)
-
-        timestamp_date = datetime.fromisoformat(timestamp[:-1])
-
-        nodes_logs_time[container] = timestamp_date
-
-    logger.info(f"Latest logs timestamp list: {nodes_logs_time}")
-
-    return nodes_logs_time
-
-
-@keyword('Find in Nodes Log')
-def find_in_nodes_log(line: str, nodes_logs_time: dict):
-
-    client_api = docker.APIClient()
-    container_names = list()
-
-    for docker_container in client_api.containers():
-        container_names.append(docker_container['Names'][0][1:])
-
-    global_count = 0
-
-    for container in nodes_logs_time.keys():
-        # check if container exists
-        if container in container_names:
-            # Get log since timestamp
-            timestamp_date = nodes_logs_time[container]
-            log_lines = client_api.logs(container, since=timestamp_date)
-            logger.info(f"Timestamp since: {timestamp_date}")
-            found_count = len(re.findall(line, log_lines.decode("utf-8") ))
-            logger.info(f"Node {container} log - found counter: {found_count}")
-            global_count += found_count
-
-        else:
-            logger.info(f"Container {container} has not been found.")
-
-    if global_count > 0:
-        logger.info(f"Expected line '{line}' has been found in the logs.")
-    else:
-        raise Exception(f"Expected line '{line}' has not been found in the logs.")
-
-    return 1
-
-
 @keyword('Generate Session Token')
 def generate_session_token(owner: str, pub_key: str, cid: str = "", wildcard: bool = False) -> str:
 
@@ -273,12 +153,3 @@ def sign_session_token(session_token: str, wallet: str, to_file: str=''):
     )
     logger.info(f"cmd: {cmd}")
     _cmd_run(cmd)
-
-
-def _search_object(node:str, wallet: str, cid:str, oid: str):
-    cmd = (
-        f'{NEOFS_CLI_EXEC} --rpc-endpoint {node} --wallet {wallet} --ttl 1 '
-        f'object search --root --cid {cid} --oid {oid} --config {WALLET_PASS}'
-    )
-    output = _cmd_run(cmd)
-    return output

robot/resources/lib/python/nodes_management.py

@@ -0,0 +1,47 @@
+#!/usr/bin/python3
+
+"""
+    This module contains keywords for management test stand
+    nodes. It assumes that nodes are docker containers.
+"""
+
+import random
+
+import docker
+
+from robot.api.deco import keyword
+
+ROBOT_AUTO_KEYWORDS = False
+
+@keyword('Stop Nodes')
+def stop_nodes(number: int, nodes: list):
+    """
+        The function shuts down the given number of randomly
+        selected nodes in docker.
+        Args:
+           number (int): the number of nodes to shut down
+           nodes (list): the list of nodes for possible shut down
+        Returns:
+            (list): the list of nodes which have been shut down
+    """
+    nodes = random.sample(nodes, number)
+    client = docker.APIClient()
+    for node in nodes:
+        node = node.split('.')[0]
+        client.stop(node)
+    return nodes
+
+
+@keyword('Start Nodes')
+def start_nodes(nodes: list):
+    """
+        The function raises the given nodes.
+        Args:
+           nodes (list): the list of nodes to raise
+        Returns:
+            (void)
+    """
+    client = docker.APIClient()
+    for node in nodes:
+        node = node.split('.')[0]
+        client.start(node)

robot/resources/lib/python/storage_policy.py

@@ -77,3 +77,49 @@ def get_complex_object_copies(wallet: str, cid: str, oid: str):
     """
     last_oid = complex_object_actions.get_last_object(wallet, cid, oid)
     return get_simple_object_copies(wallet, cid, last_oid)
+
+
+@keyword('Get Nodes With Object')
+def get_nodes_with_object(wallet: str, cid: str, oid: str):
+    """
+       The function returns list of nodes which store
+       the given object.
+       Args:
+            wallet (str): the path to the wallet on whose behalf
+                        we request the nodes
+            cid (str): ID of the container which store the object
+            oid (str): object ID
+       Returns:
+            (list): nodes which store the object
+    """
+    nodes_list = []
+    for node in NEOFS_NETMAP:
+        res = neofs_verbs.head_object(wallet, cid, oid,
+                        endpoint=node,
+                        is_direct=True)
+        if res is not None:
+            nodes_list.append(node)
+    return nodes_list
+
+
+@keyword('Get Nodes Without Object')
+def get_nodes_without_object(wallet: str, cid: str, oid: str):
+    """
+       The function returns list of nodes which do not store
+       the given object.
+       Args:
+            wallet (str): the path to the wallet on whose behalf
+                        we request the nodes
+            cid (str): ID of the container which store the object
+            oid (str): object ID
+       Returns:
+            (list): nodes which do not store the object
+    """
+    nodes_list = []
+    for node in NEOFS_NETMAP:
+        res = neofs_verbs.head_object(wallet, cid, oid,
+                        endpoint=node,
+                        is_direct=True)
+        if res is None:
+            nodes_list.append(node)
+    return nodes_list

robot/testsuites/integration/network/netmap_simple.robot

@@ -1,8 +1,5 @@
 *** Settings ***
-Variables   common.py
-
 Library     container.py
-Library     neofs.py
 Library     neofs_verbs.py
 Library     storage_policy.py
 Library     utility_keywords.py

robot/testsuites/integration/network/replication.robot

@@ -4,8 +4,8 @@ Variables   wellknown_acl.py
 
 Library     container.py
 Library     contract_keywords.py
-Library     neofs.py
 Library     neofs_verbs.py
+Library     nodes_management.py
 Library     storage_policy.py
 Library     utility_keywords.py
 
@@ -46,10 +46,9 @@ Check Replication
     ${COPIES} =             Get Object Copies   Simple      ${WALLET}   ${CID}  ${S_OID}
                             Should Be Equal     ${EXPECTED_COPIES}  ${COPIES}
 
-    @{NODES_OBJ} =          Get nodes with Object    ${WALLET}    ${CID}    ${S_OID}
-    ${NODES_LOG_TIME} =     Get Nodes Log Latest Timestamp
+    @{NODES_OBJ} =          Get Nodes With Object    ${WALLET}    ${CID}    ${S_OID}
 
-    @{NODES_OBJ_STOPPED} =  Stop nodes          1       ${NODES_OBJ}
+    @{NODES_OBJ_STOPPED} =  Stop Nodes          1       ${NODES_OBJ}
     @{NETMAP} =             Convert To List             ${NEOFS_NETMAP}
                             Remove Values From List     ${NETMAP}   ${NODES_OBJ_STOPPED}
 
@@ -65,8 +64,7 @@ Check Replication
     Run Keyword Unless      ${PASSED}     Fail
     ...     Storage policy for object ${S_OID} in container ${CID} isn't valid
 
-    Find in Nodes Log       object successfully replicated    ${NODES_LOG_TIME}
-    Start nodes             ${NODES_OBJ_STOPPED}
+    Start Nodes             ${NODES_OBJ_STOPPED}
     Tick Epoch
 
     # We have 2 or 3 copies. Expected behaviour: during two epochs potential 3rd copy should be removed.

robot/testsuites/integration/services/http_gate.robot

@@ -6,6 +6,7 @@ Library     container.py
 Library     neofs.py
 Library     neofs_verbs.py
 Library     http_gate.py
+Library     storage_policy.py
 Library     utility_keywords.py
 
 Resource    payment_operations.robot