From 6cdeb497c0ab09228f9927fce729da657a18bc04 Mon Sep 17 00:00:00 2001 From: Andrey Berezin Date: Wed, 24 May 2023 11:15:12 +0300 Subject: [PATCH] Various updates for failover and to cases Signed-off-by: Andrey Berezin --- src/frostfs_testlib/load/k6.py | 2 +- src/frostfs_testlib/load/load_verifiers.py | 41 ++++++++++++++++---- src/frostfs_testlib/resources/load_params.py | 4 +- 3 files changed, 37 insertions(+), 10 deletions(-) diff --git a/src/frostfs_testlib/load/k6.py b/src/frostfs_testlib/load/k6.py index d3939ae..24ca447 100644 --- a/src/frostfs_testlib/load/k6.py +++ b/src/frostfs_testlib/load/k6.py @@ -239,4 +239,4 @@ class K6: def __log_output(self) -> None: reporter.attach(self._k6_process.stdout(full=True), "K6 stdout") - reporter.attach(self._k6_process.stderr(full=True), "K6 stderr") + reporter.attach(f"{self._k6_process.process_dir}/stderr", "K6 stderr path") diff --git a/src/frostfs_testlib/load/load_verifiers.py b/src/frostfs_testlib/load/load_verifiers.py index becfcf7..69e9f1f 100644 --- a/src/frostfs_testlib/load/load_verifiers.py +++ b/src/frostfs_testlib/load/load_verifiers.py @@ -11,26 +11,53 @@ class LoadVerifier: self.load_params = load_params def verify_summaries(self, load_summary, verification_summary) -> None: + exceptions = [] + if not verification_summary or not load_summary: logger.info("Can't check load results due to missing summary") load_metrics = get_metrics_object(self.load_params.scenario, load_summary) - writers = self.load_params.writers or 0 + + writers = self.load_params.writers or self.load_params.preallocated_writers or 0 + readers = self.load_params.readers or self.load_params.preallocated_readers or 0 + deleters = self.load_params.deleters or self.load_params.preallocated_deleters or 0 objects_count = load_metrics.write_success_iterations fails_count = load_metrics.write_failed_iterations if writers > 0: - assert objects_count > 0, "Total put objects should be greater than 0" - assert fails_count == 0, f"There were {fails_count} failed put objects operations" + if objects_count < 1: + exceptions.append("Total put objects should be greater than 0") + if fails_count > 0: + exceptions.append(f"There were {fails_count} failed write operations") + + if readers > 0: + read_count = load_metrics.read_success_iterations + read_fails_count = load_metrics.read_failed_iterations + if read_count < 1: + exceptions.append("Total read operations should be greater than 0") + if read_fails_count > 0: + exceptions.append(f"There were {read_fails_count} failed read operations") + + if deleters > 0: + delete_count = load_metrics.delete_success_iterations + delete_fails_count = load_metrics.delete_failed_iterations + if delete_count < 1: + exceptions.append("Total delete operations should be greater than 0") + if delete_fails_count > 0: + exceptions.append(f"There were {delete_fails_count} failed delete operations") if verification_summary: verify_metrics = get_metrics_object(LoadScenario.VERIFY, verification_summary) verified_objects = verify_metrics.read_success_iterations invalid_objects = verify_metrics.read_failed_iterations - assert invalid_objects == 0, f"There were {invalid_objects} verification fails" + if invalid_objects > 0: + exceptions.append(f"There were {invalid_objects} verification fails") # Due to interruptions we may see total verified objects to be less than written on writers count - assert ( - abs(objects_count - verified_objects) <= writers - ), f"Verified objects is less than total objects. Total: {objects_count}, Verified: {verified_objects}. Writers: {writers}." + if abs(objects_count - verified_objects) > writers: + exceptions.append( + f"Verified objects is less than total objects. Total: {objects_count}, Verified: {verified_objects}. Writers: {writers}." + ) + + assert not exceptions, "\n".join(exceptions) diff --git a/src/frostfs_testlib/resources/load_params.py b/src/frostfs_testlib/resources/load_params.py index a43d20b..64f1aa4 100644 --- a/src/frostfs_testlib/resources/load_params.py +++ b/src/frostfs_testlib/resources/load_params.py @@ -7,8 +7,8 @@ LOAD_NODE_SSH_USER = os.getenv("LOAD_NODE_SSH_USER", "service") LOAD_NODE_SSH_PASSWORD = os.getenv("LOAD_NODE_SSH_PASSWORD") LOAD_NODE_SSH_PRIVATE_KEY_PATH = os.getenv("LOAD_NODE_SSH_PRIVATE_KEY_PATH") LOAD_NODE_SSH_PRIVATE_KEY_PASSPHRASE = os.getenv("LOAD_NODE_SSH_PRIVATE_KEY_PASSPHRASE") -BACKGROUND_WRITERS_COUNT = os.getenv("BACKGROUND_WRITERS_COUNT", 4) -BACKGROUND_READERS_COUNT = os.getenv("BACKGROUND_READERS_COUNT", 4) +BACKGROUND_WRITERS_COUNT = os.getenv("BACKGROUND_WRITERS_COUNT", 0) +BACKGROUND_READERS_COUNT = os.getenv("BACKGROUND_READERS_COUNT", 0) BACKGROUND_DELETERS_COUNT = os.getenv("BACKGROUND_DELETERS_COUNT", 0) BACKGROUND_LOAD_DEFAULT_TIME = os.getenv("BACKGROUND_LOAD_DEFAULT_TIME", 600) BACKGROUND_LOAD_DEFAULT_OBJECT_SIZE = os.getenv("BACKGROUND_LOAD_DEFAULT_OBJECT_SIZE", 32)