diff --git a/src/frostfs_testlib/hosting/interfaces.py b/src/frostfs_testlib/hosting/interfaces.py index 9dd6f3c9..daea6eb3 100644 --- a/src/frostfs_testlib/hosting/interfaces.py +++ b/src/frostfs_testlib/hosting/interfaces.py @@ -5,6 +5,7 @@ from typing import Optional from frostfs_testlib.hosting.config import CLIConfig, HostConfig, ServiceConfig from frostfs_testlib.shell.interfaces import Shell from frostfs_testlib.testing.readable import HumanReadableEnum +from frostfs_testlib.testing.test_control import retry class HostStatus(HumanReadableEnum): @@ -25,9 +26,7 @@ class Host(ABC): def __init__(self, config: HostConfig) -> None: self._config = config - self._service_config_by_name = { - service_config.name: service_config for service_config in config.services - } + self._service_config_by_name = {service_config.name: service_config for service_config in config.services} self._cli_config_by_name = {cli_config.name: cli_config for cli_config in config.clis} @property @@ -323,9 +322,7 @@ class Host(ABC): """ @abstractmethod - def wait_for_service_to_be_in_state( - self, systemd_service_name: str, expected_state: str, timeout: int - ) -> None: + def wait_for_service_to_be_in_state(self, systemd_service_name: str, expected_state: str, timeout: int) -> None: """ Waites for service to be in specified state. @@ -335,3 +332,23 @@ class Host(ABC): timeout: Seconds to wait """ + + def down_interface(self, interface: str) -> None: + shell = self.get_shell() + shell.exec(f"ip link set {interface} down") + + def up_interface(self, interface: str) -> None: + shell = self.get_shell() + shell.exec(f"ip link set {interface} up") + + def check_state(self, interface: str) -> str: + shell = self.get_shell() + return shell.exec(f"ip link show {interface} | sed -z 's/.*state \(.*\) mode .*/\\1/'").stdout.strip() + + @retry(max_attempts=5, sleep_interval=5, expected_result="UP") + def check_state_up(self, interface: str) -> str: + return self.check_state(interface=interface) + + @retry(max_attempts=5, sleep_interval=5, expected_result="DOWN") + def check_state_down(self, interface: str) -> str: + return self.check_state(interface=interface) diff --git a/src/frostfs_testlib/shell/command_inspectors.py b/src/frostfs_testlib/shell/command_inspectors.py index 8fe2f34e..00030175 100644 --- a/src/frostfs_testlib/shell/command_inspectors.py +++ b/src/frostfs_testlib/shell/command_inspectors.py @@ -9,7 +9,7 @@ class SudoInspector(CommandInspector): def inspect(self, original_command: str, command: str) -> str: if not command.startswith("sudo"): - return f"sudo {command}" + return f"sudo -i {command}" return command diff --git a/src/frostfs_testlib/steps/network.py b/src/frostfs_testlib/steps/network.py index 64e235a0..efaaf5a4 100644 --- a/src/frostfs_testlib/steps/network.py +++ b/src/frostfs_testlib/steps/network.py @@ -1,77 +1,19 @@ -from frostfs_testlib import reporter +from frostfs_testlib.shell import CommandOptions from frostfs_testlib.storage.cluster import ClusterNode -from frostfs_testlib.testing.test_control import retry -class IpTablesHelper: - @staticmethod - def drop_input_traffic_to_port(node: ClusterNode, ports: list[str]) -> None: - shell = node.host.get_shell() - for port in ports: - shell.exec(f"iptables -A INPUT -p tcp --dport {port} -j DROP") - +class IpHelper: @staticmethod def drop_input_traffic_to_node(node: ClusterNode, block_ip: list[str]) -> None: shell = node.host.get_shell() for ip in block_ip: - shell.exec(f"iptables -A INPUT -s {ip} -j DROP") - - @staticmethod - def restore_input_traffic_to_port(node: ClusterNode) -> None: - shell = node.host.get_shell() - ports = shell.exec("iptables -L --numeric | grep DROP | awk '{print $7}'").stdout.strip().split("\n") - if ports[0] == "": - return - for port in ports: - shell.exec(f"iptables -D INPUT -p tcp --dport {port.split(':')[-1]} -j DROP") + shell.exec(f"ip route add blackhole {ip}") @staticmethod def restore_input_traffic_to_node(node: ClusterNode) -> None: shell = node.host.get_shell() - unlock_ip = shell.exec("iptables -L --numeric | grep DROP | awk '{print $4}'").stdout.strip().split("\n") - if unlock_ip[0] == "": + unlock_ip = shell.exec("ip route list | grep blackhole", CommandOptions(check=False)) + if unlock_ip.return_code != 0: return - for ip in unlock_ip: - shell.exec(f"iptables -D INPUT -s {ip} -j DROP") - - -# TODO Move class to HOST -class IfUpDownHelper: - @reporter.step("Down {interface} to {node}") - def down_interface(self, node: ClusterNode, interface: str) -> None: - shell = node.host.get_shell() - shell.exec(f"ifdown {interface}") - - @reporter.step("Up {interface} to {node}") - def up_interface(self, node: ClusterNode, interface: str) -> None: - shell = node.host.get_shell() - shell.exec(f"ifup {interface}") - - @reporter.step("Up all interface to {node}") - def up_all_interface(self, node: ClusterNode) -> None: - shell = node.host.get_shell() - interfaces = list(node.host.config.interfaces.keys()) - shell.exec("ifup -av") - for name_interface in interfaces: - self.check_state_up(node, name_interface) - - @reporter.step("Down all interface to {node}") - def down_all_interface(self, node: ClusterNode) -> None: - shell = node.host.get_shell() - interfaces = list(node.host.config.interfaces.keys()) - shell.exec("ifdown -av") - for name_interface in interfaces: - self.check_state_down(node, name_interface) - - @reporter.step("Check {node} to {interface}") - def check_state(self, node: ClusterNode, interface: str) -> str: - shell = node.host.get_shell() - return shell.exec(f"ip link show {interface} | sed -z 's/.*state \(.*\) mode .*/\\1/'").stdout.strip() - - @retry(max_attempts=5, sleep_interval=5, expected_result="UP") - def check_state_up(self, node: ClusterNode, interface: str) -> str: - return self.check_state(node=node, interface=interface) - - @retry(max_attempts=5, sleep_interval=5, expected_result="DOWN") - def check_state_down(self, node: ClusterNode, interface: str) -> str: - return self.check_state(node=node, interface=interface) + for ip in unlock_ip.stdout.strip().split("\n"): + shell.exec(f"ip route del blackhole {ip.split(' ')[1]}") diff --git a/src/frostfs_testlib/storage/controllers/cluster_state_controller.py b/src/frostfs_testlib/storage/controllers/cluster_state_controller.py index 301b6364..290503ce 100644 --- a/src/frostfs_testlib/storage/controllers/cluster_state_controller.py +++ b/src/frostfs_testlib/storage/controllers/cluster_state_controller.py @@ -13,7 +13,7 @@ from frostfs_testlib.plugins import load_all from frostfs_testlib.resources.cli import FROSTFS_ADM_CONFIG_PATH, FROSTFS_ADM_EXEC, FROSTFS_CLI_EXEC from frostfs_testlib.resources.common import DEFAULT_WALLET_CONFIG, MORPH_BLOCK_TIME from frostfs_testlib.shell import CommandOptions, Shell, SshConnectionProvider -from frostfs_testlib.steps.network import IfUpDownHelper, IpTablesHelper +from frostfs_testlib.steps.network import IpHelper from frostfs_testlib.storage.cluster import Cluster, ClusterNode, S3Gate, StorageNode from frostfs_testlib.storage.controllers.disk_controller import DiskController from frostfs_testlib.storage.dataclasses.node_base import NodeBase, ServiceClass @@ -22,7 +22,6 @@ from frostfs_testlib.testing.test_control import retry, run_optionally, wait_for from frostfs_testlib.utils.datetime_utils import parse_time logger = logging.getLogger("NeoLogger") -if_up_down_helper = IfUpDownHelper() class StateManager: @@ -305,57 +304,25 @@ class ClusterStateController: [node.host.wait_success_resume_process(process_name) for node in list_nodes] self.suspended_services = {} - @reporter.step("Drop traffic to {node}, with ports - {ports}, nodes - {block_nodes}") + @reporter.step("Drop traffic to {node}, nodes - {block_nodes}") def drop_traffic( self, - mode: str, node: ClusterNode, wakeup_timeout: int, - ports: list[str] = None, + name_interface: str, block_nodes: list[ClusterNode] = None, ) -> None: - allowed_modes = ["ports", "nodes"] - assert mode in allowed_modes - - match mode: - case "ports": - IpTablesHelper.drop_input_traffic_to_port(node, ports) - case "nodes": - list_ip = self._parse_intefaces(block_nodes) - IpTablesHelper.drop_input_traffic_to_node(node, list_ip) + list_ip = self._parse_interfaces(block_nodes, name_interface) + IpHelper.drop_input_traffic_to_node(node, list_ip) time.sleep(wakeup_timeout) self.dropped_traffic.append(node) - @reporter.step("Ping traffic") - def ping_traffic( - self, - node: ClusterNode, - nodes_list: list[ClusterNode], - expect_result: int, - ) -> bool: - shell = node.host.get_shell() - options = CommandOptions(check=False) - ips = self._parse_intefaces(nodes_list) - for ip in ips: - code = shell.exec(f"ping {ip} -c 1", options).return_code - if code != expect_result: - return False - return True - @reporter.step("Start traffic to {node}") def restore_traffic( self, - mode: str, node: ClusterNode, ) -> None: - allowed_modes = ["ports", "nodes"] - assert mode in allowed_modes - - match mode: - case "ports": - IpTablesHelper.restore_input_traffic_to_port(node=node) - case "nodes": - IpTablesHelper.restore_input_traffic_to_node(node=node) + IpHelper.restore_input_traffic_to_node(node=node) @reporter.step("Restore blocked nodes") def restore_all_traffic(self): @@ -385,22 +352,25 @@ class ClusterStateController: @reporter.step("Down {interface} to {nodes}") def down_interface(self, nodes: list[ClusterNode], interface: str): for node in nodes: - if_up_down_helper.down_interface(node=node, interface=interface) - assert if_up_down_helper.check_state(node=node, interface=interface) == "DOWN" + node.host.down_interface(interface=interface) + assert node.host.check_state(interface=interface) == "DOWN" self.nodes_with_modified_interface.append(node) @reporter.step("Up {interface} to {nodes}") def up_interface(self, nodes: list[ClusterNode], interface: str): for node in nodes: - if_up_down_helper.up_interface(node=node, interface=interface) - assert if_up_down_helper.check_state(node=node, interface=interface) == "UP" + node.host.up_interface(interface=interface) + assert node.host.check_state(interface=interface) == "UP" if node in self.nodes_with_modified_interface: self.nodes_with_modified_interface.remove(node) @reporter.step("Restore interface") def restore_interfaces(self): for node in self.nodes_with_modified_interface: - if_up_down_helper.up_all_interface(node) + dict_interfaces = node.host.config.interfaces.keys() + for name_interface in dict_interfaces: + if "mgmt" not in name_interface: + node.host.up_interface(interface=name_interface) @reporter.step("Get node time") def get_node_date(self, node: ClusterNode) -> datetime: @@ -523,15 +493,14 @@ class ClusterStateController: return disk_controller def _restore_traffic_to_node(self, node): - IpTablesHelper.restore_input_traffic_to_port(node) - IpTablesHelper.restore_input_traffic_to_node(node) + IpHelper.restore_input_traffic_to_node(node) - def _parse_intefaces(self, nodes: list[ClusterNode]): + def _parse_interfaces(self, nodes: list[ClusterNode], name_interface: str): interfaces = [] for node in nodes: dict_interfaces = node.host.config.interfaces for type, ip in dict_interfaces.items(): - if "mgmt" not in type: + if name_interface in type: interfaces.append(ip) return interfaces