Various updates for failover and to cases

Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
This commit is contained in:
Andrey Berezin 2023-05-24 11:15:12 +03:00
parent a34c34991f
commit 6cdeb497c0
3 changed files with 37 additions and 10 deletions

View file

@ -239,4 +239,4 @@ class K6:
def __log_output(self) -> None: def __log_output(self) -> None:
reporter.attach(self._k6_process.stdout(full=True), "K6 stdout") reporter.attach(self._k6_process.stdout(full=True), "K6 stdout")
reporter.attach(self._k6_process.stderr(full=True), "K6 stderr") reporter.attach(f"{self._k6_process.process_dir}/stderr", "K6 stderr path")

View file

@ -11,26 +11,53 @@ class LoadVerifier:
self.load_params = load_params self.load_params = load_params
def verify_summaries(self, load_summary, verification_summary) -> None: def verify_summaries(self, load_summary, verification_summary) -> None:
exceptions = []
if not verification_summary or not load_summary: if not verification_summary or not load_summary:
logger.info("Can't check load results due to missing summary") logger.info("Can't check load results due to missing summary")
load_metrics = get_metrics_object(self.load_params.scenario, load_summary) load_metrics = get_metrics_object(self.load_params.scenario, load_summary)
writers = self.load_params.writers or 0
writers = self.load_params.writers or self.load_params.preallocated_writers or 0
readers = self.load_params.readers or self.load_params.preallocated_readers or 0
deleters = self.load_params.deleters or self.load_params.preallocated_deleters or 0
objects_count = load_metrics.write_success_iterations objects_count = load_metrics.write_success_iterations
fails_count = load_metrics.write_failed_iterations fails_count = load_metrics.write_failed_iterations
if writers > 0: if writers > 0:
assert objects_count > 0, "Total put objects should be greater than 0" if objects_count < 1:
assert fails_count == 0, f"There were {fails_count} failed put objects operations" exceptions.append("Total put objects should be greater than 0")
if fails_count > 0:
exceptions.append(f"There were {fails_count} failed write operations")
if readers > 0:
read_count = load_metrics.read_success_iterations
read_fails_count = load_metrics.read_failed_iterations
if read_count < 1:
exceptions.append("Total read operations should be greater than 0")
if read_fails_count > 0:
exceptions.append(f"There were {read_fails_count} failed read operations")
if deleters > 0:
delete_count = load_metrics.delete_success_iterations
delete_fails_count = load_metrics.delete_failed_iterations
if delete_count < 1:
exceptions.append("Total delete operations should be greater than 0")
if delete_fails_count > 0:
exceptions.append(f"There were {delete_fails_count} failed delete operations")
if verification_summary: if verification_summary:
verify_metrics = get_metrics_object(LoadScenario.VERIFY, verification_summary) verify_metrics = get_metrics_object(LoadScenario.VERIFY, verification_summary)
verified_objects = verify_metrics.read_success_iterations verified_objects = verify_metrics.read_success_iterations
invalid_objects = verify_metrics.read_failed_iterations invalid_objects = verify_metrics.read_failed_iterations
assert invalid_objects == 0, f"There were {invalid_objects} verification fails" if invalid_objects > 0:
exceptions.append(f"There were {invalid_objects} verification fails")
# Due to interruptions we may see total verified objects to be less than written on writers count # Due to interruptions we may see total verified objects to be less than written on writers count
assert ( if abs(objects_count - verified_objects) > writers:
abs(objects_count - verified_objects) <= writers exceptions.append(
), f"Verified objects is less than total objects. Total: {objects_count}, Verified: {verified_objects}. Writers: {writers}." f"Verified objects is less than total objects. Total: {objects_count}, Verified: {verified_objects}. Writers: {writers}."
)
assert not exceptions, "\n".join(exceptions)

View file

@ -7,8 +7,8 @@ LOAD_NODE_SSH_USER = os.getenv("LOAD_NODE_SSH_USER", "service")
LOAD_NODE_SSH_PASSWORD = os.getenv("LOAD_NODE_SSH_PASSWORD") LOAD_NODE_SSH_PASSWORD = os.getenv("LOAD_NODE_SSH_PASSWORD")
LOAD_NODE_SSH_PRIVATE_KEY_PATH = os.getenv("LOAD_NODE_SSH_PRIVATE_KEY_PATH") LOAD_NODE_SSH_PRIVATE_KEY_PATH = os.getenv("LOAD_NODE_SSH_PRIVATE_KEY_PATH")
LOAD_NODE_SSH_PRIVATE_KEY_PASSPHRASE = os.getenv("LOAD_NODE_SSH_PRIVATE_KEY_PASSPHRASE") LOAD_NODE_SSH_PRIVATE_KEY_PASSPHRASE = os.getenv("LOAD_NODE_SSH_PRIVATE_KEY_PASSPHRASE")
BACKGROUND_WRITERS_COUNT = os.getenv("BACKGROUND_WRITERS_COUNT", 4) BACKGROUND_WRITERS_COUNT = os.getenv("BACKGROUND_WRITERS_COUNT", 0)
BACKGROUND_READERS_COUNT = os.getenv("BACKGROUND_READERS_COUNT", 4) BACKGROUND_READERS_COUNT = os.getenv("BACKGROUND_READERS_COUNT", 0)
BACKGROUND_DELETERS_COUNT = os.getenv("BACKGROUND_DELETERS_COUNT", 0) BACKGROUND_DELETERS_COUNT = os.getenv("BACKGROUND_DELETERS_COUNT", 0)
BACKGROUND_LOAD_DEFAULT_TIME = os.getenv("BACKGROUND_LOAD_DEFAULT_TIME", 600) BACKGROUND_LOAD_DEFAULT_TIME = os.getenv("BACKGROUND_LOAD_DEFAULT_TIME", 600)
BACKGROUND_LOAD_DEFAULT_OBJECT_SIZE = os.getenv("BACKGROUND_LOAD_DEFAULT_OBJECT_SIZE", 32) BACKGROUND_LOAD_DEFAULT_OBJECT_SIZE = os.getenv("BACKGROUND_LOAD_DEFAULT_OBJECT_SIZE", 32)