Keep only one ssh connection per host

Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
This commit is contained in:
Andrey Berezin 2023-10-10 17:47:46 +03:00
parent d039bcc221
commit 2c2af7f8ed
4 changed files with 261 additions and 144 deletions

View file

@ -3,7 +3,7 @@ import time
import frostfs_testlib.resources.optionals as optionals
from frostfs_testlib.reporter import get_reporter
from frostfs_testlib.shell import CommandOptions, Shell
from frostfs_testlib.shell import CommandOptions, Shell, SshConnectionProvider
from frostfs_testlib.steps.network import IfUpDownHelper, IpTablesHelper
from frostfs_testlib.storage.cluster import Cluster, ClusterNode, StorageNode
from frostfs_testlib.storage.controllers.disk_controller import DiskController
@ -37,6 +37,10 @@ class ClusterStateController:
@run_optionally(optionals.OPTIONAL_FAILOVER_ENABLED)
@reporter.step_deco("Stop host of node {node}")
def stop_node_host(self, node: ClusterNode, mode: str):
# Drop ssh connection for this node before shutdown
provider = SshConnectionProvider()
provider.drop(node.host_ip)
with reporter.step(f"Stop host {node.host.config.address}"):
node.host.stop_host(mode=mode)
wait_for_host_offline(self.shell, node.storage_node)
@ -48,6 +52,11 @@ class ClusterStateController:
nodes = (
reversed(self.cluster.cluster_nodes) if reversed_order else self.cluster.cluster_nodes
)
# Drop all ssh connections before shutdown
provider = SshConnectionProvider()
provider.drop_all()
for node in nodes:
with reporter.step(f"Stop host {node.host.config.address}"):
self.stopped_nodes.append(node)
@ -307,6 +316,10 @@ class ClusterStateController:
options = CommandOptions(close_stdin=True, timeout=1, check=False)
shell.exec('sudo sh -c "echo b > /proc/sysrq-trigger"', options)
# Drop ssh connection for this node
provider = SshConnectionProvider()
provider.drop(node.host_ip)
if wait_for_return:
# Let the things to be settled
# A little wait here to prevent ssh stuck during panic