forked from TrueCloudLab/frostfs-testlib
Various updates for failover and to cases
Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
This commit is contained in:
parent
a34c34991f
commit
6cdeb497c0
3 changed files with 37 additions and 10 deletions
|
@ -239,4 +239,4 @@ class K6:
|
||||||
|
|
||||||
def __log_output(self) -> None:
|
def __log_output(self) -> None:
|
||||||
reporter.attach(self._k6_process.stdout(full=True), "K6 stdout")
|
reporter.attach(self._k6_process.stdout(full=True), "K6 stdout")
|
||||||
reporter.attach(self._k6_process.stderr(full=True), "K6 stderr")
|
reporter.attach(f"{self._k6_process.process_dir}/stderr", "K6 stderr path")
|
||||||
|
|
|
@ -11,26 +11,53 @@ class LoadVerifier:
|
||||||
self.load_params = load_params
|
self.load_params = load_params
|
||||||
|
|
||||||
def verify_summaries(self, load_summary, verification_summary) -> None:
|
def verify_summaries(self, load_summary, verification_summary) -> None:
|
||||||
|
exceptions = []
|
||||||
|
|
||||||
if not verification_summary or not load_summary:
|
if not verification_summary or not load_summary:
|
||||||
logger.info("Can't check load results due to missing summary")
|
logger.info("Can't check load results due to missing summary")
|
||||||
|
|
||||||
load_metrics = get_metrics_object(self.load_params.scenario, load_summary)
|
load_metrics = get_metrics_object(self.load_params.scenario, load_summary)
|
||||||
writers = self.load_params.writers or 0
|
|
||||||
|
writers = self.load_params.writers or self.load_params.preallocated_writers or 0
|
||||||
|
readers = self.load_params.readers or self.load_params.preallocated_readers or 0
|
||||||
|
deleters = self.load_params.deleters or self.load_params.preallocated_deleters or 0
|
||||||
|
|
||||||
objects_count = load_metrics.write_success_iterations
|
objects_count = load_metrics.write_success_iterations
|
||||||
fails_count = load_metrics.write_failed_iterations
|
fails_count = load_metrics.write_failed_iterations
|
||||||
|
|
||||||
if writers > 0:
|
if writers > 0:
|
||||||
assert objects_count > 0, "Total put objects should be greater than 0"
|
if objects_count < 1:
|
||||||
assert fails_count == 0, f"There were {fails_count} failed put objects operations"
|
exceptions.append("Total put objects should be greater than 0")
|
||||||
|
if fails_count > 0:
|
||||||
|
exceptions.append(f"There were {fails_count} failed write operations")
|
||||||
|
|
||||||
|
if readers > 0:
|
||||||
|
read_count = load_metrics.read_success_iterations
|
||||||
|
read_fails_count = load_metrics.read_failed_iterations
|
||||||
|
if read_count < 1:
|
||||||
|
exceptions.append("Total read operations should be greater than 0")
|
||||||
|
if read_fails_count > 0:
|
||||||
|
exceptions.append(f"There were {read_fails_count} failed read operations")
|
||||||
|
|
||||||
|
if deleters > 0:
|
||||||
|
delete_count = load_metrics.delete_success_iterations
|
||||||
|
delete_fails_count = load_metrics.delete_failed_iterations
|
||||||
|
if delete_count < 1:
|
||||||
|
exceptions.append("Total delete operations should be greater than 0")
|
||||||
|
if delete_fails_count > 0:
|
||||||
|
exceptions.append(f"There were {delete_fails_count} failed delete operations")
|
||||||
|
|
||||||
if verification_summary:
|
if verification_summary:
|
||||||
verify_metrics = get_metrics_object(LoadScenario.VERIFY, verification_summary)
|
verify_metrics = get_metrics_object(LoadScenario.VERIFY, verification_summary)
|
||||||
verified_objects = verify_metrics.read_success_iterations
|
verified_objects = verify_metrics.read_success_iterations
|
||||||
invalid_objects = verify_metrics.read_failed_iterations
|
invalid_objects = verify_metrics.read_failed_iterations
|
||||||
|
|
||||||
assert invalid_objects == 0, f"There were {invalid_objects} verification fails"
|
if invalid_objects > 0:
|
||||||
|
exceptions.append(f"There were {invalid_objects} verification fails")
|
||||||
# Due to interruptions we may see total verified objects to be less than written on writers count
|
# Due to interruptions we may see total verified objects to be less than written on writers count
|
||||||
assert (
|
if abs(objects_count - verified_objects) > writers:
|
||||||
abs(objects_count - verified_objects) <= writers
|
exceptions.append(
|
||||||
), f"Verified objects is less than total objects. Total: {objects_count}, Verified: {verified_objects}. Writers: {writers}."
|
f"Verified objects is less than total objects. Total: {objects_count}, Verified: {verified_objects}. Writers: {writers}."
|
||||||
|
)
|
||||||
|
|
||||||
|
assert not exceptions, "\n".join(exceptions)
|
||||||
|
|
|
@ -7,8 +7,8 @@ LOAD_NODE_SSH_USER = os.getenv("LOAD_NODE_SSH_USER", "service")
|
||||||
LOAD_NODE_SSH_PASSWORD = os.getenv("LOAD_NODE_SSH_PASSWORD")
|
LOAD_NODE_SSH_PASSWORD = os.getenv("LOAD_NODE_SSH_PASSWORD")
|
||||||
LOAD_NODE_SSH_PRIVATE_KEY_PATH = os.getenv("LOAD_NODE_SSH_PRIVATE_KEY_PATH")
|
LOAD_NODE_SSH_PRIVATE_KEY_PATH = os.getenv("LOAD_NODE_SSH_PRIVATE_KEY_PATH")
|
||||||
LOAD_NODE_SSH_PRIVATE_KEY_PASSPHRASE = os.getenv("LOAD_NODE_SSH_PRIVATE_KEY_PASSPHRASE")
|
LOAD_NODE_SSH_PRIVATE_KEY_PASSPHRASE = os.getenv("LOAD_NODE_SSH_PRIVATE_KEY_PASSPHRASE")
|
||||||
BACKGROUND_WRITERS_COUNT = os.getenv("BACKGROUND_WRITERS_COUNT", 4)
|
BACKGROUND_WRITERS_COUNT = os.getenv("BACKGROUND_WRITERS_COUNT", 0)
|
||||||
BACKGROUND_READERS_COUNT = os.getenv("BACKGROUND_READERS_COUNT", 4)
|
BACKGROUND_READERS_COUNT = os.getenv("BACKGROUND_READERS_COUNT", 0)
|
||||||
BACKGROUND_DELETERS_COUNT = os.getenv("BACKGROUND_DELETERS_COUNT", 0)
|
BACKGROUND_DELETERS_COUNT = os.getenv("BACKGROUND_DELETERS_COUNT", 0)
|
||||||
BACKGROUND_LOAD_DEFAULT_TIME = os.getenv("BACKGROUND_LOAD_DEFAULT_TIME", 600)
|
BACKGROUND_LOAD_DEFAULT_TIME = os.getenv("BACKGROUND_LOAD_DEFAULT_TIME", 600)
|
||||||
BACKGROUND_LOAD_DEFAULT_OBJECT_SIZE = os.getenv("BACKGROUND_LOAD_DEFAULT_OBJECT_SIZE", 32)
|
BACKGROUND_LOAD_DEFAULT_OBJECT_SIZE = os.getenv("BACKGROUND_LOAD_DEFAULT_OBJECT_SIZE", 32)
|
||||||
|
|
Loading…
Reference in a new issue