From 14a5eac5b1f1f2c724678c21d0a4620dd61022a9 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 12 Jan 2024 13:16:34 +0300 Subject: [PATCH] [#117] scenarios: Refactor data generator construction 1. Make it easier to change new parameters. 2. Fix a bug where the generator was created even for read-only scenarios. Signed-off-by: Evgenii Stratonikov --- scenarios/grpc.js | 6 ++---- scenarios/grpc_car.js | 5 ++--- scenarios/http.js | 5 ++--- scenarios/libs/datagen.js | 8 ++++++++ scenarios/local.js | 5 ++--- scenarios/s3.js | 5 ++--- scenarios/s3_car.js | 5 ++--- scenarios/s3_multipart.js | 5 ++--- scenarios/s3local.js | 5 ++--- 9 files changed, 24 insertions(+), 25 deletions(-) create mode 100644 scenarios/libs/datagen.js diff --git a/scenarios/grpc.js b/scenarios/grpc.js index 874efbb..f3d8075 100644 --- a/scenarios/grpc.js +++ b/scenarios/grpc.js @@ -1,4 +1,3 @@ -import datagen from 'k6/x/frostfs/datagen'; import native from 'k6/x/frostfs/native'; import logging from 'k6/x/frostfs/logging'; import registry from 'k6/x/frostfs/registry'; @@ -7,6 +6,7 @@ import { sleep } from 'k6'; import { textSummary } from './libs/k6-summary-0.0.2.js'; import { parseEnv } from './libs/env-parser.js'; import { uuidv4 } from './libs/k6-utils-1.4.0.js'; +import { newGenerator } from './libs/datagen.js'; parseEnv(); @@ -63,13 +63,11 @@ if (registry_enabled) { ) } - -const generator = datagen.generator(1024 * parseInt(__ENV.WRITE_OBJ_SIZE), __ENV.PAYLOAD_TYPE || ""); - const scenarios = {}; const write_vu_count = parseInt(__ENV.WRITERS || '0'); const write_grpc_chunk_size = 1024 * parseInt(__ENV.GRPC_CHUNK_SIZE || '0') +const generator = newGenerator(write_vu_count > 0); if (write_vu_count > 0) { scenarios.write = { executor: 'constant-vus', diff --git a/scenarios/grpc_car.js b/scenarios/grpc_car.js index c271d6e..7f4f306 100644 --- a/scenarios/grpc_car.js +++ b/scenarios/grpc_car.js @@ -1,4 +1,3 @@ -import datagen from 'k6/x/frostfs/datagen'; import native from 'k6/x/frostfs/native'; import logging from 'k6/x/frostfs/logging'; import registry from 'k6/x/frostfs/registry'; @@ -7,6 +6,7 @@ import { sleep } from 'k6'; import { textSummary } from './libs/k6-summary-0.0.2.js'; import { parseEnv } from './libs/env-parser.js'; import { uuidv4 } from './libs/k6-utils-1.4.0.js'; +import { newGenerator } from './libs/datagen.js'; parseEnv(); @@ -63,8 +63,6 @@ if (registry_enabled) { ) } -const generator = datagen.generator(1024 * parseInt(__ENV.WRITE_OBJ_SIZE), __ENV.PAYLOAD_TYPE || ""); - const scenarios = {}; const time_unit = __ENV.TIME_UNIT || '1s'; @@ -72,6 +70,7 @@ const pre_alloc_write_vus = parseInt(__ENV.PRE_ALLOC_WRITERS || '0'); const max_write_vus = parseInt(__ENV.MAX_WRITERS || pre_alloc_write_vus); const write_rate = parseInt(__ENV.WRITE_RATE || '0'); const write_grpc_chunk_size = 1024 * parseInt(__ENV.GRPC_CHUNK_SIZE || '0') +const generator = newGenerator(write_rate > 0); if (write_rate > 0) { scenarios.write = { executor: 'constant-arrival-rate', diff --git a/scenarios/http.js b/scenarios/http.js index 110e097..d13dd5b 100644 --- a/scenarios/http.js +++ b/scenarios/http.js @@ -1,4 +1,3 @@ -import datagen from 'k6/x/frostfs/datagen'; import logging from 'k6/x/frostfs/logging'; import registry from 'k6/x/frostfs/registry'; import http from 'k6/http'; @@ -7,6 +6,7 @@ import { sleep } from 'k6'; import { textSummary } from './libs/k6-summary-0.0.2.js'; import { parseEnv } from './libs/env-parser.js'; import { uuidv4 } from './libs/k6-utils-1.4.0.js'; +import { newGenerator } from './libs/datagen.js'; parseEnv(); @@ -31,11 +31,10 @@ const obj_registry = registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : und const duration = __ENV.DURATION; -const generator = datagen.generator(1024 * parseInt(__ENV.WRITE_OBJ_SIZE), __ENV.PAYLOAD_TYPE || ""); - const scenarios = {}; const write_vu_count = parseInt(__ENV.WRITERS || '0'); +const generator = newGenerator(write_vu_count > 0); if (write_vu_count > 0) { scenarios.write = { executor: 'constant-vus', diff --git a/scenarios/libs/datagen.js b/scenarios/libs/datagen.js new file mode 100644 index 0000000..d6feba2 --- /dev/null +++ b/scenarios/libs/datagen.js @@ -0,0 +1,8 @@ +import datagen from 'k6/x/frostfs/datagen'; + +export function newGenerator(condition) { + if (condition) { + return datagen.generator(1024 * parseInt(__ENV.WRITE_OBJ_SIZE), __ENV.PAYLOAD_TYPE || ""); + } + return undefined; +} diff --git a/scenarios/local.js b/scenarios/local.js index cbcab65..56def89 100644 --- a/scenarios/local.js +++ b/scenarios/local.js @@ -1,4 +1,3 @@ -import datagen from 'k6/x/frostfs/datagen'; import local from 'k6/x/frostfs/local'; import logging from 'k6/x/frostfs/logging'; import registry from 'k6/x/frostfs/registry'; @@ -7,6 +6,7 @@ import { textSummary } from './libs/k6-summary-0.0.2.js'; import { parseEnv } from './libs/env-parser.js'; import { uuidv4 } from './libs/k6-utils-1.4.0.js'; import exec from 'k6/execution'; +import { newGenerator } from './libs/datagen.js'; parseEnv(); @@ -47,11 +47,10 @@ if (registry_enabled && delete_age) { ); } -const generator = datagen.generator(1024 * parseInt(__ENV.WRITE_OBJ_SIZE), __ENV.PAYLOAD_TYPE || ""); - const scenarios = {}; const write_vu_count = parseInt(__ENV.WRITERS || '0'); +const generator = newGenerator(write_vu_count > 0); if (write_vu_count > 0) { scenarios.write = { executor: 'constant-vus', diff --git a/scenarios/s3.js b/scenarios/s3.js index 9a2e625..deff64e 100644 --- a/scenarios/s3.js +++ b/scenarios/s3.js @@ -1,4 +1,3 @@ -import datagen from 'k6/x/frostfs/datagen'; import logging from 'k6/x/frostfs/logging'; import registry from 'k6/x/frostfs/registry'; import s3 from 'k6/x/frostfs/s3'; @@ -7,6 +6,7 @@ import { sleep } from 'k6'; import { textSummary } from './libs/k6-summary-0.0.2.js'; import { parseEnv } from './libs/env-parser.js'; import { uuidv4 } from './libs/k6-utils-1.4.0.js'; +import { newGenerator } from './libs/datagen.js'; parseEnv(); @@ -62,11 +62,10 @@ if (registry_enabled) { ) } -const generator = datagen.generator(1024 * parseInt(__ENV.WRITE_OBJ_SIZE), __ENV.PAYLOAD_TYPE || ""); - const scenarios = {}; const write_vu_count = parseInt(__ENV.WRITERS || '0'); +const generator = newGenerator(write_vu_count > 0); if (write_vu_count > 0) { scenarios.write = { executor: 'constant-vus', diff --git a/scenarios/s3_car.js b/scenarios/s3_car.js index a1f6402..1581bc9 100644 --- a/scenarios/s3_car.js +++ b/scenarios/s3_car.js @@ -1,4 +1,3 @@ -import datagen from 'k6/x/frostfs/datagen'; import logging from 'k6/x/frostfs/logging'; import registry from 'k6/x/frostfs/registry'; import s3 from 'k6/x/frostfs/s3'; @@ -7,6 +6,7 @@ import { sleep } from 'k6'; import { textSummary } from './libs/k6-summary-0.0.2.js'; import { parseEnv } from './libs/env-parser.js'; import { uuidv4 } from './libs/k6-utils-1.4.0.js'; +import { newGenerator } from './libs/datagen.js'; parseEnv(); @@ -62,14 +62,13 @@ if (registry_enabled) { ) } -const generator = datagen.generator(1024 * parseInt(__ENV.WRITE_OBJ_SIZE), __ENV.PAYLOAD_TYPE || ""); - const scenarios = {}; const time_unit = __ENV.TIME_UNIT || '1s'; const pre_alloc_write_vus = parseInt(__ENV.PRE_ALLOC_WRITERS || '0'); const max_write_vus = parseInt(__ENV.MAX_WRITERS || pre_alloc_write_vus); const write_rate = parseInt(__ENV.WRITE_RATE || '0'); +const generator = newGenerator(write_rate > 0); if (write_rate > 0) { scenarios.write = { executor: 'constant-arrival-rate', diff --git a/scenarios/s3_multipart.js b/scenarios/s3_multipart.js index ee39996..f7095b7 100644 --- a/scenarios/s3_multipart.js +++ b/scenarios/s3_multipart.js @@ -1,4 +1,3 @@ -import datagen from 'k6/x/frostfs/datagen'; import logging from 'k6/x/frostfs/logging'; import registry from 'k6/x/frostfs/registry'; import s3 from 'k6/x/frostfs/s3'; @@ -7,6 +6,7 @@ import {sleep} from 'k6'; import {textSummary} from './libs/k6-summary-0.0.2.js'; import {parseEnv} from './libs/env-parser.js'; import {uuidv4} from './libs/k6-utils-1.4.0.js'; +import { newGenerator } from './libs/datagen.js'; parseEnv(); @@ -29,8 +29,6 @@ const obj_registry = registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : und const duration = __ENV.DURATION; -const generator = datagen.generator(1024 * parseInt(__ENV.WRITE_OBJ_SIZE), __ENV.PAYLOAD_TYPE || ""); - const scenarios = {}; const write_vu_count = parseInt(__ENV.WRITERS || '0'); @@ -43,6 +41,7 @@ if (write_multipart_vu_count < 1) { throw 'number of parts (env WRITERS_MULTIPART) to upload in parallel should be greater than 0'; } +const generator = newGenerator(write_vu_count > 0 || write_multipart_vu_count > 0); if (write_vu_count > 0) { scenarios.write_multipart = { executor: 'constant-vus', diff --git a/scenarios/s3local.js b/scenarios/s3local.js index 71e89f8..6ea26d2 100644 --- a/scenarios/s3local.js +++ b/scenarios/s3local.js @@ -1,4 +1,3 @@ -import datagen from 'k6/x/frostfs/datagen'; import logging from 'k6/x/frostfs/logging'; import registry from 'k6/x/frostfs/registry'; import s3local from 'k6/x/frostfs/s3local'; @@ -7,6 +6,7 @@ import { textSummary } from './libs/k6-summary-0.0.2.js'; import { parseEnv } from './libs/env-parser.js'; import { uuidv4 } from './libs/k6-utils-1.4.0.js'; import exec from 'k6/execution'; +import { newGenerator } from './libs/datagen.js'; parseEnv(); @@ -61,11 +61,10 @@ if (registry_enabled) { const duration = __ENV.DURATION; -const generator = datagen.generator(1024 * parseInt(__ENV.WRITE_OBJ_SIZE), __ENV.PAYLOAD_TYPE || ""); - const scenarios = {}; const write_vu_count = parseInt(__ENV.WRITERS || '0'); +const generator = newGenerator(write_vu_count > 0); if (write_vu_count > 0) { scenarios.write = { executor: 'constant-vus',