From 93aaec4e0d741c12a1c91f50ac810c2008ba6e02 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 20 Mar 2024 11:04:13 +0300 Subject: [PATCH] [#131] scenarios: Format js files with clang formatter Signed-off-by: Dmitrii Stepanov --- scenarios/grpc.js | 261 ++++++++++++++++----------------- scenarios/grpc_car.js | 298 ++++++++++++++++++------------------- scenarios/http.js | 165 +++++++++++---------- scenarios/local.js | 206 +++++++++++++------------- scenarios/s3.js | 263 ++++++++++++++++----------------- scenarios/s3_car.js | 299 +++++++++++++++++++------------------- scenarios/s3_multipart.js | 117 ++++++++------- scenarios/s3local.js | 208 +++++++++++++------------- scenarios/verify.js | 215 ++++++++++++++------------- 9 files changed, 1038 insertions(+), 994 deletions(-) diff --git a/scenarios/grpc.js b/scenarios/grpc.js index 75d9622..623e17c 100644 --- a/scenarios/grpc.js +++ b/scenarios/grpc.js @@ -1,71 +1,69 @@ -import native from 'k6/x/frostfs/native'; +import {sleep} from 'k6'; +import {SharedArray} from 'k6/data'; import logging from 'k6/x/frostfs/logging'; +import native from 'k6/x/frostfs/native'; import registry from 'k6/x/frostfs/registry'; import stats from 'k6/x/frostfs/stats'; -import { SharedArray } from 'k6/data'; -import { sleep } from 'k6'; -import { textSummary } from './libs/k6-summary-0.0.2.js'; -import { parseEnv } from './libs/env-parser.js'; -import { uuidv4 } from './libs/k6-utils-1.4.0.js'; -import { newGenerator } from './libs/datagen.js'; + +import {newGenerator} from './libs/datagen.js'; +import {parseEnv} from './libs/env-parser.js'; +import {textSummary} from './libs/k6-summary-0.0.2.js'; +import {uuidv4} from './libs/k6-utils-1.4.0.js'; parseEnv(); -const obj_list = new SharedArray('obj_list', function () { - return JSON.parse(open(__ENV.PREGEN_JSON)).objects; +const obj_list = new SharedArray('obj_list', function() { + return JSON.parse(open(__ENV.PREGEN_JSON)).objects; }); -const container_list = new SharedArray('container_list', function () { - return JSON.parse(open(__ENV.PREGEN_JSON)).containers; +const container_list = new SharedArray('container_list', function() { + return JSON.parse(open(__ENV.PREGEN_JSON)).containers; }); const read_size = JSON.parse(open(__ENV.PREGEN_JSON)).obj_size; -const summary_json = __ENV.SUMMARY_JSON || "/tmp/summary.json"; +const summary_json = __ENV.SUMMARY_JSON || '/tmp/summary.json'; // Select random gRPC endpoint for current VU const grpc_endpoints = __ENV.GRPC_ENDPOINTS.split(','); -const grpc_endpoint = grpc_endpoints[Math.floor(Math.random() * grpc_endpoints.length)]; -const grpc_client = native.connect(grpc_endpoint, '', - __ENV.DIAL_TIMEOUT ? parseInt(__ENV.DIAL_TIMEOUT) : 5, +const grpc_endpoint = + grpc_endpoints[Math.floor(Math.random() * grpc_endpoints.length)]; +const grpc_client = native.connect( + grpc_endpoint, '', __ENV.DIAL_TIMEOUT ? parseInt(__ENV.DIAL_TIMEOUT) : 5, __ENV.STREAM_TIMEOUT ? parseInt(__ENV.STREAM_TIMEOUT) : 60, - __ENV.PREPARE_LOCALLY ? __ENV.PREPARE_LOCALLY.toLowerCase() === "true" : false); -const log = logging.new().withField("endpoint", grpc_endpoint); + __ENV.PREPARE_LOCALLY ? __ENV.PREPARE_LOCALLY.toLowerCase() === 'true' : + false); +const log = logging.new().withField('endpoint', grpc_endpoint); const registry_enabled = !!__ENV.REGISTRY_FILE; -const obj_registry = registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : undefined; +const obj_registry = + registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : undefined; const duration = __ENV.DURATION; if (!!__ENV.METRIC_TAGS) { - stats.setTags(__ENV.METRIC_TAGS) + stats.setTags(__ENV.METRIC_TAGS) } const delete_age = __ENV.DELETE_AGE ? parseInt(__ENV.DELETE_AGE) : undefined; let obj_to_delete_selector = undefined; if (registry_enabled && delete_age) { - obj_to_delete_selector = registry.getSelector( - __ENV.REGISTRY_FILE, - "obj_to_delete", - __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, - { - status: "created", - age: delete_age, - } - ); + obj_to_delete_selector = registry.getSelector( + __ENV.REGISTRY_FILE, 'obj_to_delete', + __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, { + status: 'created', + age: delete_age, + }); } const read_age = __ENV.READ_AGE ? parseInt(__ENV.READ_AGE) : 10; let obj_to_read_selector = undefined; if (registry_enabled) { - obj_to_read_selector = registry.getLoopedSelector( - __ENV.REGISTRY_FILE, - "obj_to_read", - __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, - { - status: "created", - age: read_age, - } - ) + obj_to_read_selector = registry.getLoopedSelector( + __ENV.REGISTRY_FILE, 'obj_to_read', + __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, { + status: 'created', + age: read_age, + }) } const scenarios = {}; @@ -74,138 +72,141 @@ const write_vu_count = parseInt(__ENV.WRITERS || '0'); const write_grpc_chunk_size = 1024 * parseInt(__ENV.GRPC_CHUNK_SIZE || '0') const generator = newGenerator(write_vu_count > 0); if (write_vu_count > 0) { - scenarios.write = { - executor: 'constant-vus', - vus: write_vu_count, - duration: `${duration}s`, - exec: 'obj_write', - gracefulStop: '5s', - }; + scenarios.write = { + executor: 'constant-vus', + vus: write_vu_count, + duration: `${duration}s`, + exec: 'obj_write', + gracefulStop: '5s', + }; } const read_vu_count = parseInt(__ENV.READERS || '0'); if (read_vu_count > 0) { - scenarios.read = { - executor: 'constant-vus', - vus: read_vu_count, - duration: `${duration}s`, - exec: 'obj_read', - gracefulStop: '5s', - }; + scenarios.read = { + executor: 'constant-vus', + vus: read_vu_count, + duration: `${duration}s`, + exec: 'obj_read', + gracefulStop: '5s', + }; } const delete_vu_count = parseInt(__ENV.DELETERS || '0'); if (delete_vu_count > 0) { - if (!obj_to_delete_selector) { - throw new Error('Positive DELETE worker number without a proper object selector'); - } + if (!obj_to_delete_selector) { + throw new Error( + 'Positive DELETE worker number without a proper object selector'); + } - scenarios.delete = { - executor: 'constant-vus', - vus: delete_vu_count, - duration: `${duration}s`, - exec: 'obj_delete', - gracefulStop: '5s', - }; + scenarios.delete = { + executor: 'constant-vus', + vus: delete_vu_count, + duration: `${duration}s`, + exec: 'obj_delete', + gracefulStop: '5s', + }; } export const options = { - scenarios, - setupTimeout: '5s', + scenarios, + setupTimeout: '5s', }; export function setup() { - const total_vu_count = write_vu_count + read_vu_count + delete_vu_count; + const total_vu_count = write_vu_count + read_vu_count + delete_vu_count; - console.log(`Pregenerated containers: ${container_list.length}`); - console.log(`Pregenerated read object size: ${read_size}`); - console.log(`Pregenerated total objects: ${obj_list.length}`); - console.log(`Reading VUs: ${read_vu_count}`); - console.log(`Writing VUs: ${write_vu_count}`); - console.log(`Deleting VUs: ${delete_vu_count}`); - console.log(`Total VUs: ${total_vu_count}`); - - const start_timestamp = Date.now() - console.log(`Load started at: ${Date(start_timestamp).toString()}`) + console.log(`Pregenerated containers: ${container_list.length}`); + console.log(`Pregenerated read object size: ${read_size}`); + console.log(`Pregenerated total objects: ${obj_list.length}`); + console.log(`Reading VUs: ${read_vu_count}`); + console.log(`Writing VUs: ${write_vu_count}`); + console.log(`Deleting VUs: ${delete_vu_count}`); + console.log(`Total VUs: ${total_vu_count}`); + + const start_timestamp = Date.now() + console.log( + `Load started at: ${Date(start_timestamp).toString()}`) } export function teardown(data) { - if (obj_registry) { - obj_registry.close(); - } - const end_timestamp = Date.now() - console.log(`Load finished at: ${Date(end_timestamp).toString()}`) + if (obj_registry) { + obj_registry.close(); + } + const end_timestamp = Date.now() + console.log( + `Load finished at: ${Date(end_timestamp).toString()}`) } export function handleSummary(data) { - return { - 'stdout': textSummary(data, { indent: ' ', enableColors: false }), - [summary_json]: JSON.stringify(data), - }; + return { + 'stdout': textSummary(data, {indent: ' ', enableColors: false}), + [summary_json]: JSON.stringify(data), + }; } export function obj_write() { - if (__ENV.SLEEP_WRITE) { - sleep(__ENV.SLEEP_WRITE); - } + if (__ENV.SLEEP_WRITE) { + sleep(__ENV.SLEEP_WRITE); + } - const headers = { - unique_header: uuidv4() - }; - const container = container_list[Math.floor(Math.random() * container_list.length)]; + const headers = {unique_header: uuidv4()}; + const container = + container_list[Math.floor(Math.random() * container_list.length)]; - const payload = generator.genPayload(); - const resp = grpc_client.put(container, headers, payload, write_grpc_chunk_size); - if (!resp.success) { - log.withField("cid", container).error(resp.error); - return; - } + const payload = generator.genPayload(); + const resp = + grpc_client.put(container, headers, payload, write_grpc_chunk_size); + if (!resp.success) { + log.withField('cid', container).error(resp.error); + return; + } - if (obj_registry) { - obj_registry.addObject(container, resp.object_id, "", "", payload.hash()); - } + if (obj_registry) { + obj_registry.addObject(container, resp.object_id, '', '', payload.hash()); + } } export function obj_read() { - if (__ENV.SLEEP_READ) { - sleep(__ENV.SLEEP_READ); - } + if (__ENV.SLEEP_READ) { + sleep(__ENV.SLEEP_READ); + } - if(obj_to_read_selector) { - const obj = obj_to_read_selector.nextObject(); - if (!obj) { - return; - } - const resp = grpc_client.get(obj.c_id, obj.o_id) - if (!resp.success) { - log.withFields({cid: obj.c_id, oid: obj.o_id}).error(resp.error); - } - return + if (obj_to_read_selector) { + const obj = obj_to_read_selector.nextObject(); + if (!obj) { + return; } - - const obj = obj_list[Math.floor(Math.random() * obj_list.length)]; - const resp = grpc_client.get(obj.container, obj.object) + const resp = grpc_client.get(obj.c_id, obj.o_id) if (!resp.success) { - log.withFields({cid: obj.container, oid: obj.object}).error(resp.error); + log.withFields({cid: obj.c_id, oid: obj.o_id}).error(resp.error); } + return + } + + const obj = obj_list[Math.floor(Math.random() * obj_list.length)]; + const resp = grpc_client.get(obj.container, obj.object) + if (!resp.success) { + log.withFields({cid: obj.container, oid: obj.object}).error(resp.error); + } } export function obj_delete() { - if (__ENV.SLEEP_DELETE) { - sleep(__ENV.SLEEP_DELETE); - } + if (__ENV.SLEEP_DELETE) { + sleep(__ENV.SLEEP_DELETE); + } - const obj = obj_to_delete_selector.nextObject(); - if (!obj) { - return; - } + const obj = obj_to_delete_selector.nextObject(); + if (!obj) { + return; + } - const resp = grpc_client.delete(obj.c_id, obj.o_id); - if (!resp.success) { - // Log errors except (2052 - object already deleted) - log.withFields({cid: obj.c_id, oid: obj.o_id}).error(resp.error); - return; - } + const resp = grpc_client.delete(obj.c_id, obj.o_id); + if (!resp.success) { + // Log errors except (2052 - object already deleted) + log.withFields({cid: obj.c_id, oid: obj.o_id}).error(resp.error); + return; + } - obj_registry.deleteObject(obj.id); + obj_registry.deleteObject(obj.id); } diff --git a/scenarios/grpc_car.js b/scenarios/grpc_car.js index 752e76a..8ee5d46 100644 --- a/scenarios/grpc_car.js +++ b/scenarios/grpc_car.js @@ -1,71 +1,69 @@ -import native from 'k6/x/frostfs/native'; +import {sleep} from 'k6'; +import {SharedArray} from 'k6/data'; import logging from 'k6/x/frostfs/logging'; +import native from 'k6/x/frostfs/native'; import registry from 'k6/x/frostfs/registry'; import stats from 'k6/x/frostfs/stats'; -import { SharedArray } from 'k6/data'; -import { sleep } from 'k6'; -import { textSummary } from './libs/k6-summary-0.0.2.js'; -import { parseEnv } from './libs/env-parser.js'; -import { uuidv4 } from './libs/k6-utils-1.4.0.js'; -import { newGenerator } from './libs/datagen.js'; + +import {newGenerator} from './libs/datagen.js'; +import {parseEnv} from './libs/env-parser.js'; +import {textSummary} from './libs/k6-summary-0.0.2.js'; +import {uuidv4} from './libs/k6-utils-1.4.0.js'; parseEnv(); -const obj_list = new SharedArray('obj_list', function () { - return JSON.parse(open(__ENV.PREGEN_JSON)).objects; +const obj_list = new SharedArray('obj_list', function() { + return JSON.parse(open(__ENV.PREGEN_JSON)).objects; }); -const container_list = new SharedArray('container_list', function () { - return JSON.parse(open(__ENV.PREGEN_JSON)).containers; +const container_list = new SharedArray('container_list', function() { + return JSON.parse(open(__ENV.PREGEN_JSON)).containers; }); const read_size = JSON.parse(open(__ENV.PREGEN_JSON)).obj_size; -const summary_json = __ENV.SUMMARY_JSON || "/tmp/summary.json"; +const summary_json = __ENV.SUMMARY_JSON || '/tmp/summary.json'; // Select random gRPC endpoint for current VU const grpc_endpoints = __ENV.GRPC_ENDPOINTS.split(','); -const grpc_endpoint = grpc_endpoints[Math.floor(Math.random() * grpc_endpoints.length)]; -const grpc_client = native.connect(grpc_endpoint, '', - __ENV.DIAL_TIMEOUT ? parseInt(__ENV.DIAL_TIMEOUT) : 5, +const grpc_endpoint = + grpc_endpoints[Math.floor(Math.random() * grpc_endpoints.length)]; +const grpc_client = native.connect( + grpc_endpoint, '', __ENV.DIAL_TIMEOUT ? parseInt(__ENV.DIAL_TIMEOUT) : 5, __ENV.STREAM_TIMEOUT ? parseInt(__ENV.STREAM_TIMEOUT) : 60, - __ENV.PREPARE_LOCALLY ? __ENV.PREPARE_LOCALLY.toLowerCase() === "true" : false); -const log = logging.new().withField("endpoint", grpc_endpoint); + __ENV.PREPARE_LOCALLY ? __ENV.PREPARE_LOCALLY.toLowerCase() === 'true' : + false); +const log = logging.new().withField('endpoint', grpc_endpoint); const registry_enabled = !!__ENV.REGISTRY_FILE; -const obj_registry = registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : undefined; +const obj_registry = + registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : undefined; const duration = __ENV.DURATION; if (!!__ENV.METRIC_TAGS) { - stats.setTags(__ENV.METRIC_TAGS) + stats.setTags(__ENV.METRIC_TAGS) } const delete_age = __ENV.DELETE_AGE ? parseInt(__ENV.DELETE_AGE) : undefined; let obj_to_delete_selector = undefined; if (registry_enabled && delete_age) { - obj_to_delete_selector = registry.getSelector( - __ENV.REGISTRY_FILE, - "obj_to_delete", - __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, - { - status: "created", - age: delete_age, - } - ); + obj_to_delete_selector = registry.getSelector( + __ENV.REGISTRY_FILE, 'obj_to_delete', + __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, { + status: 'created', + age: delete_age, + }); } const read_age = __ENV.READ_AGE ? parseInt(__ENV.READ_AGE) : 10; let obj_to_read_selector = undefined; if (registry_enabled) { - obj_to_read_selector = registry.getLoopedSelector( - __ENV.REGISTRY_FILE, - "obj_to_read", - __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, - { - status: "created", - age: read_age, - } - ) + obj_to_read_selector = registry.getLoopedSelector( + __ENV.REGISTRY_FILE, 'obj_to_read', + __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, { + status: 'created', + age: read_age, + }) } const scenarios = {}; @@ -77,160 +75,164 @@ const write_rate = parseInt(__ENV.WRITE_RATE || '0'); const write_grpc_chunk_size = 1024 * parseInt(__ENV.GRPC_CHUNK_SIZE || '0') const generator = newGenerator(write_rate > 0); if (write_rate > 0) { - scenarios.write = { - executor: 'constant-arrival-rate', - duration: `${duration}s`, - preAllocatedVUs: pre_alloc_write_vus, - maxVUs: max_write_vus, - rate: write_rate, - timeUnit: time_unit, - exec: 'obj_write', - gracefulStop: '5s', - }; + scenarios.write = { + executor: 'constant-arrival-rate', + duration: `${duration}s`, + preAllocatedVUs: pre_alloc_write_vus, + maxVUs: max_write_vus, + rate: write_rate, + timeUnit: time_unit, + exec: 'obj_write', + gracefulStop: '5s', + }; } const pre_alloc_read_vus = parseInt(__ENV.PRE_ALLOC_READERS || '0'); const max_read_vus = parseInt(__ENV.MAX_READERS || pre_alloc_read_vus); const read_rate = parseInt(__ENV.READ_RATE || '0'); if (read_rate > 0) { - scenarios.read = { - executor: 'constant-arrival-rate', - duration: `${duration}s`, - preAllocatedVUs: pre_alloc_write_vus, - maxVUs: max_read_vus, - rate: read_rate, - timeUnit: time_unit, - exec: 'obj_read', - gracefulStop: '5s', - }; + scenarios.read = { + executor: 'constant-arrival-rate', + duration: `${duration}s`, + preAllocatedVUs: pre_alloc_write_vus, + maxVUs: max_read_vus, + rate: read_rate, + timeUnit: time_unit, + exec: 'obj_read', + gracefulStop: '5s', + }; } const pre_alloc_delete_vus = parseInt(__ENV.PRE_ALLOC_DELETERS || '0'); const max_delete_vus = parseInt(__ENV.MAX_DELETERS || pre_alloc_write_vus); const delete_rate = parseInt(__ENV.DELETE_RATE || '0'); if (delete_rate > 0) { - if (!obj_to_delete_selector) { - throw new Error('Positive DELETE worker number without a proper object selector'); - } + if (!obj_to_delete_selector) { + throw new Error( + 'Positive DELETE worker number without a proper object selector'); + } - scenarios.delete = { - executor: 'constant-arrival-rate', - duration: `${duration}s`, - preAllocatedVUs: pre_alloc_delete_vus, - maxVUs: max_delete_vus, - rate: delete_rate, - timeUnit: time_unit, - exec: 'obj_delete', - gracefulStop: '5s', - }; + scenarios.delete = { + executor: 'constant-arrival-rate', + duration: `${duration}s`, + preAllocatedVUs: pre_alloc_delete_vus, + maxVUs: max_delete_vus, + rate: delete_rate, + timeUnit: time_unit, + exec: 'obj_delete', + gracefulStop: '5s', + }; } export const options = { - scenarios, - setupTimeout: '5s', + scenarios, + setupTimeout: '5s', }; export function setup() { - const total_pre_allocated_vu_count = pre_alloc_write_vus + pre_alloc_read_vus + pre_alloc_delete_vus; - const total_max_vu_count = max_read_vus + max_write_vus + max_delete_vus + const total_pre_allocated_vu_count = + pre_alloc_write_vus + pre_alloc_read_vus + pre_alloc_delete_vus; + const total_max_vu_count = max_read_vus + max_write_vus + max_delete_vus - console.log(`Pregenerated containers: ${container_list.length}`); - console.log(`Pregenerated read object size: ${read_size}`); - console.log(`Pregenerated total objects: ${obj_list.length}`); - console.log(`Pre allocated reading VUs: ${pre_alloc_read_vus}`); - console.log(`Pre allocated writing VUs: ${pre_alloc_write_vus}`); - console.log(`Pre allocated deleting VUs: ${pre_alloc_delete_vus}`); - console.log(`Total pre allocated VUs: ${total_pre_allocated_vu_count}`); - console.log(`Max reading VUs: ${max_read_vus}`); - console.log(`Max writing VUs: ${max_write_vus}`); - console.log(`Max deleting VUs: ${max_delete_vus}`); - console.log(`Total max VUs: ${total_max_vu_count}`); - console.log(`Time unit: ${time_unit}`); - console.log(`Read rate: ${read_rate}`); - console.log(`Writing rate: ${write_rate}`); - console.log(`Delete rate: ${delete_rate}`); - - const start_timestamp = Date.now() - console.log(`Load started at: ${Date(start_timestamp).toString()}`) + console.log(`Pregenerated containers: ${container_list.length}`); + console.log(`Pregenerated read object size: ${read_size}`); + console.log(`Pregenerated total objects: ${obj_list.length}`); + console.log(`Pre allocated reading VUs: ${pre_alloc_read_vus}`); + console.log(`Pre allocated writing VUs: ${pre_alloc_write_vus}`); + console.log(`Pre allocated deleting VUs: ${pre_alloc_delete_vus}`); + console.log(`Total pre allocated VUs: ${total_pre_allocated_vu_count}`); + console.log(`Max reading VUs: ${max_read_vus}`); + console.log(`Max writing VUs: ${max_write_vus}`); + console.log(`Max deleting VUs: ${max_delete_vus}`); + console.log(`Total max VUs: ${total_max_vu_count}`); + console.log(`Time unit: ${time_unit}`); + console.log(`Read rate: ${read_rate}`); + console.log(`Writing rate: ${write_rate}`); + console.log(`Delete rate: ${delete_rate}`); + + const start_timestamp = Date.now() + console.log( + `Load started at: ${Date(start_timestamp).toString()}`) } export function teardown(data) { - if (obj_registry) { - obj_registry.close(); - } - const end_timestamp = Date.now() - console.log(`Load finished at: ${Date(end_timestamp).toString()}`) + if (obj_registry) { + obj_registry.close(); + } + const end_timestamp = Date.now() + console.log( + `Load finished at: ${Date(end_timestamp).toString()}`) } export function handleSummary(data) { - return { - 'stdout': textSummary(data, { indent: ' ', enableColors: false }), - [summary_json]: JSON.stringify(data), - }; + return { + 'stdout': textSummary(data, {indent: ' ', enableColors: false}), + [summary_json]: JSON.stringify(data), + }; } export function obj_write() { - if (__ENV.SLEEP_WRITE) { - sleep(__ENV.SLEEP_WRITE); - } + if (__ENV.SLEEP_WRITE) { + sleep(__ENV.SLEEP_WRITE); + } - const headers = { - unique_header: uuidv4() - }; - const container = container_list[Math.floor(Math.random() * container_list.length)]; + const headers = {unique_header: uuidv4()}; + const container = + container_list[Math.floor(Math.random() * container_list.length)]; - const payload = generator.genPayload(); - const resp = grpc_client.put(container, headers, payload, write_grpc_chunk_size); - if (!resp.success) { - log.withField("cid", container).error(resp.error); - return; - } + const payload = generator.genPayload(); + const resp = + grpc_client.put(container, headers, payload, write_grpc_chunk_size); + if (!resp.success) { + log.withField('cid', container).error(resp.error); + return; + } - if (obj_registry) { - obj_registry.addObject(container, resp.object_id, "", "", payload.hash()); - } + if (obj_registry) { + obj_registry.addObject(container, resp.object_id, '', '', payload.hash()); + } } export function obj_read() { - if (__ENV.SLEEP_READ) { - sleep(__ENV.SLEEP_READ); - } + if (__ENV.SLEEP_READ) { + sleep(__ENV.SLEEP_READ); + } - if(obj_to_read_selector) { - const obj = obj_to_read_selector.nextObject(); - if (!obj) { - return; - } - const resp = grpc_client.get(obj.c_id, obj.o_id) - if (!resp.success) { - log.withFields({cid: obj.c_id, oid: obj.o_id}).error(resp.error); - } - return + if (obj_to_read_selector) { + const obj = obj_to_read_selector.nextObject(); + if (!obj) { + return; } - - const obj = obj_list[Math.floor(Math.random() * obj_list.length)]; - const resp = grpc_client.get(obj.container, obj.object) + const resp = grpc_client.get(obj.c_id, obj.o_id) if (!resp.success) { - log.withFields({cid: obj.container, oid: obj.object}).error(resp.error); + log.withFields({cid: obj.c_id, oid: obj.o_id}).error(resp.error); } + return + } + + const obj = obj_list[Math.floor(Math.random() * obj_list.length)]; + const resp = grpc_client.get(obj.container, obj.object) + if (!resp.success) { + log.withFields({cid: obj.container, oid: obj.object}).error(resp.error); + } } export function obj_delete() { - if (__ENV.SLEEP_DELETE) { - sleep(__ENV.SLEEP_DELETE); - } + if (__ENV.SLEEP_DELETE) { + sleep(__ENV.SLEEP_DELETE); + } - const obj = obj_to_delete_selector.nextObject(); - if (!obj) { - return; - } + const obj = obj_to_delete_selector.nextObject(); + if (!obj) { + return; + } - const resp = grpc_client.delete(obj.c_id, obj.o_id); - if (!resp.success) { - // Log errors except (2052 - object already deleted) - log.withFields({cid: obj.c_id, oid: obj.o_id}).error(resp.error); - return; - } + const resp = grpc_client.delete(obj.c_id, obj.o_id); + if (!resp.success) { + // Log errors except (2052 - object already deleted) + log.withFields({cid: obj.c_id, oid: obj.o_id}).error(resp.error); + return; + } - obj_registry.deleteObject(obj.id); + obj_registry.deleteObject(obj.id); } diff --git a/scenarios/http.js b/scenarios/http.js index f96c0af..801c31c 100644 --- a/scenarios/http.js +++ b/scenarios/http.js @@ -1,39 +1,42 @@ +import {sleep} from 'k6'; +import {SharedArray} from 'k6/data'; +import http from 'k6/http'; import logging from 'k6/x/frostfs/logging'; import registry from 'k6/x/frostfs/registry'; import stats from 'k6/x/frostfs/stats'; -import http from 'k6/http'; -import { SharedArray } from 'k6/data'; -import { sleep } from 'k6'; -import { textSummary } from './libs/k6-summary-0.0.2.js'; -import { parseEnv } from './libs/env-parser.js'; -import { uuidv4 } from './libs/k6-utils-1.4.0.js'; -import { newGenerator } from './libs/datagen.js'; + +import {newGenerator} from './libs/datagen.js'; +import {parseEnv} from './libs/env-parser.js'; +import {textSummary} from './libs/k6-summary-0.0.2.js'; +import {uuidv4} from './libs/k6-utils-1.4.0.js'; parseEnv(); -const obj_list = new SharedArray('obj_list', function () { - return JSON.parse(open(__ENV.PREGEN_JSON)).objects; +const obj_list = new SharedArray('obj_list', function() { + return JSON.parse(open(__ENV.PREGEN_JSON)).objects; }); -const container_list = new SharedArray('container_list', function () { - return JSON.parse(open(__ENV.PREGEN_JSON)).containers; +const container_list = new SharedArray('container_list', function() { + return JSON.parse(open(__ENV.PREGEN_JSON)).containers; }); const read_size = JSON.parse(open(__ENV.PREGEN_JSON)).obj_size; -const summary_json = __ENV.SUMMARY_JSON || "/tmp/summary.json"; +const summary_json = __ENV.SUMMARY_JSON || '/tmp/summary.json'; // Select random HTTP endpoint for current VU const http_endpoints = __ENV.HTTP_ENDPOINTS.split(','); -const http_endpoint = http_endpoints[Math.floor(Math.random() * http_endpoints.length)]; -const log = logging.new().withField("endpoint", http_endpoint); +const http_endpoint = + http_endpoints[Math.floor(Math.random() * http_endpoints.length)]; +const log = logging.new().withField('endpoint', http_endpoint); const registry_enabled = !!__ENV.REGISTRY_FILE; -const obj_registry = registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : undefined; +const obj_registry = + registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : undefined; const duration = __ENV.DURATION; if (!!__ENV.METRIC_TAGS) { - stats.setTags(__ENV.METRIC_TAGS) + stats.setTags(__ENV.METRIC_TAGS) } const scenarios = {}; @@ -41,94 +44,100 @@ const scenarios = {}; const write_vu_count = parseInt(__ENV.WRITERS || '0'); const generator = newGenerator(write_vu_count > 0); if (write_vu_count > 0) { - scenarios.write = { - executor: 'constant-vus', - vus: write_vu_count, - duration: `${duration}s`, - exec: 'obj_write', - gracefulStop: '5s', - } + scenarios.write = { + executor: 'constant-vus', + vus: write_vu_count, + duration: `${duration}s`, + exec: 'obj_write', + gracefulStop: '5s', + } } const read_vu_count = parseInt(__ENV.READERS || '0'); if (read_vu_count > 0) { - scenarios.read = { - executor: 'constant-vus', - vus: read_vu_count, - duration: `${duration}s`, - exec: 'obj_read', - gracefulStop: '5s', - } + scenarios.read = { + executor: 'constant-vus', + vus: read_vu_count, + duration: `${duration}s`, + exec: 'obj_read', + gracefulStop: '5s', + } } export const options = { - scenarios, - setupTimeout: '5s', + scenarios, + setupTimeout: '5s', }; export function setup() { - const total_vu_count = write_vu_count + read_vu_count; + const total_vu_count = write_vu_count + read_vu_count; - console.log(`Pregenerated containers: ${container_list.length}`); - console.log(`Pregenerated read object size: ${read_size}`); - console.log(`Pregenerated total objects: ${obj_list.length}`); - console.log(`Reading VUs: ${read_vu_count}`); - console.log(`Writing VUs: ${write_vu_count}`); - console.log(`Total VUs: ${total_vu_count}`); + console.log(`Pregenerated containers: ${container_list.length}`); + console.log(`Pregenerated read object size: ${read_size}`); + console.log(`Pregenerated total objects: ${obj_list.length}`); + console.log(`Reading VUs: ${read_vu_count}`); + console.log(`Writing VUs: ${write_vu_count}`); + console.log(`Total VUs: ${total_vu_count}`); - const start_timestamp = Date.now() - console.log(`Load started at: ${Date(start_timestamp).toString()}`) + const start_timestamp = Date.now() + console.log( + `Load started at: ${Date(start_timestamp).toString()}`) } export function teardown(data) { - if (obj_registry) { - obj_registry.close(); - } - const end_timestamp = Date.now() - console.log(`Load finished at: ${Date(end_timestamp).toString()}`) + if (obj_registry) { + obj_registry.close(); + } + const end_timestamp = Date.now() + console.log( + `Load finished at: ${Date(end_timestamp).toString()}`) } export function handleSummary(data) { - return { - 'stdout': textSummary(data, { indent: ' ', enableColors: false }), - [summary_json]: JSON.stringify(data), - }; + return { + 'stdout': textSummary(data, {indent: ' ', enableColors: false}), + [summary_json]: JSON.stringify(data), + }; } export function obj_write() { - if (__ENV.SLEEP_WRITE) { - sleep(__ENV.SLEEP_WRITE); - } + if (__ENV.SLEEP_WRITE) { + sleep(__ENV.SLEEP_WRITE); + } - const container = container_list[Math.floor(Math.random() * container_list.length)]; + const container = + container_list[Math.floor(Math.random() * container_list.length)]; - const payload = generator.genPayload(); - const data = { - field: uuidv4(), - // Because we use `file` wrapping and it is not straightforward to use streams here, - // `-e STREAMING=1` has no effect for this scenario. - file: http.file(payload.bytes(), "random.data"), - }; + const payload = generator.genPayload(); + const data = { + field: uuidv4(), + // Because we use `file` wrapping and it is not straightforward to use + // streams here, + // `-e STREAMING=1` has no effect for this scenario. + file: http.file(payload.bytes(), 'random.data'), + }; - const resp = http.post(`http://${http_endpoint}/upload/${container}`, data); - if (resp.status != 200) { - log.withFields({status: resp.status, cid: container}).error(resp.error); - return; - } - const object_id = JSON.parse(resp.body).object_id; - if (obj_registry) { - obj_registry.addObject(container, object_id, "", "", payload.hash()); - } + const resp = http.post(`http://${http_endpoint}/upload/${container}`, data); + if (resp.status != 200) { + log.withFields({status: resp.status, cid: container}).error(resp.error); + return; + } + const object_id = JSON.parse(resp.body).object_id; + if (obj_registry) { + obj_registry.addObject(container, object_id, '', '', payload.hash()); + } } export function obj_read() { - if (__ENV.SLEEP_READ) { - sleep(__ENV.SLEEP_READ); - } + if (__ENV.SLEEP_READ) { + sleep(__ENV.SLEEP_READ); + } - const obj = obj_list[Math.floor(Math.random() * obj_list.length)]; - const resp = http.get(`http://${http_endpoint}/get/${obj.container}/${obj.object}`); - if (resp.status != 200) { - log.withFields({status: resp.status, cid: obj.container, oid: obj.object}).error(resp.error); - } + const obj = obj_list[Math.floor(Math.random() * obj_list.length)]; + const resp = + http.get(`http://${http_endpoint}/get/${obj.container}/${obj.object}`); + if (resp.status != 200) { + log.withFields({status: resp.status, cid: obj.container, oid: obj.object}) + .error(resp.error); + } } diff --git a/scenarios/local.js b/scenarios/local.js index 0630304..7b3f003 100644 --- a/scenarios/local.js +++ b/scenarios/local.js @@ -1,55 +1,57 @@ +import {SharedArray} from 'k6/data'; +import exec from 'k6/execution'; import local from 'k6/x/frostfs/local'; import logging from 'k6/x/frostfs/logging'; import registry from 'k6/x/frostfs/registry'; import stats from 'k6/x/frostfs/stats'; -import { SharedArray } from 'k6/data'; -import { textSummary } from './libs/k6-summary-0.0.2.js'; -import { parseEnv } from './libs/env-parser.js'; -import { uuidv4 } from './libs/k6-utils-1.4.0.js'; -import exec from 'k6/execution'; -import { newGenerator } from './libs/datagen.js'; + +import {newGenerator} from './libs/datagen.js'; +import {parseEnv} from './libs/env-parser.js'; +import {textSummary} from './libs/k6-summary-0.0.2.js'; +import {uuidv4} from './libs/k6-utils-1.4.0.js'; parseEnv(); -const obj_list = new SharedArray('obj_list', function () { - return JSON.parse(open(__ENV.PREGEN_JSON)).objects; +const obj_list = new SharedArray('obj_list', function() { + return JSON.parse(open(__ENV.PREGEN_JSON)).objects; }); -const container_list = new SharedArray('container_list', function () { - return JSON.parse(open(__ENV.PREGEN_JSON)).containers; +const container_list = new SharedArray('container_list', function() { + return JSON.parse(open(__ENV.PREGEN_JSON)).containers; }); const read_size = JSON.parse(open(__ENV.PREGEN_JSON)).obj_size; -const summary_json = __ENV.SUMMARY_JSON || "/tmp/summary.json"; +const summary_json = __ENV.SUMMARY_JSON || '/tmp/summary.json'; const config_file = __ENV.CONFIG_FILE; const config_dir = __ENV.CONFIG_DIR; const debug_logger = (__ENV.DEBUG_LOGGER || 'false') == 'true'; -const max_total_size_gb = __ENV.MAX_TOTAL_SIZE_GB ? parseInt(__ENV.MAX_TOTAL_SIZE_GB) : 0; -const local_client = local.connect(config_file, config_dir, '', debug_logger, max_total_size_gb); -const log = logging.new().withFields({"config_file": config_file,"config_dir": config_dir}); +const max_total_size_gb = + __ENV.MAX_TOTAL_SIZE_GB ? parseInt(__ENV.MAX_TOTAL_SIZE_GB) : 0; +const local_client = + local.connect(config_file, config_dir, '', debug_logger, max_total_size_gb); +const log = logging.new().withFields( + {'config_file': config_file, 'config_dir': config_dir}); const registry_enabled = !!__ENV.REGISTRY_FILE; -const obj_registry = registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : undefined; +const obj_registry = + registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : undefined; const duration = __ENV.DURATION; if (!!__ENV.METRIC_TAGS) { - stats.setTags(__ENV.METRIC_TAGS) + stats.setTags(__ENV.METRIC_TAGS) } const delete_age = __ENV.DELETE_AGE ? parseInt(__ENV.DELETE_AGE) : undefined; let obj_to_delete_selector = undefined; if (registry_enabled && delete_age) { - obj_to_delete_selector = registry.getSelector( - __ENV.REGISTRY_FILE, - "obj_to_delete", - __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, - { - status: "created", - age: delete_age, - } - ); + obj_to_delete_selector = registry.getSelector( + __ENV.REGISTRY_FILE, 'obj_to_delete', + __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, { + status: 'created', + age: delete_age, + }); } const scenarios = {}; @@ -57,117 +59,119 @@ const scenarios = {}; const write_vu_count = parseInt(__ENV.WRITERS || '0'); const generator = newGenerator(write_vu_count > 0); if (write_vu_count > 0) { - scenarios.write = { - executor: 'constant-vus', - vus: write_vu_count, - duration: `${duration}s`, - exec: 'obj_write', - gracefulStop: '5s', - }; + scenarios.write = { + executor: 'constant-vus', + vus: write_vu_count, + duration: `${duration}s`, + exec: 'obj_write', + gracefulStop: '5s', + }; } const read_vu_count = parseInt(__ENV.READERS || '0'); if (read_vu_count > 0) { - scenarios.read = { - executor: 'constant-vus', - vus: read_vu_count, - duration: `${duration}s`, - exec: 'obj_read', - gracefulStop: '5s', - }; + scenarios.read = { + executor: 'constant-vus', + vus: read_vu_count, + duration: `${duration}s`, + exec: 'obj_read', + gracefulStop: '5s', + }; } const delete_vu_count = parseInt(__ENV.DELETERS || '0'); if (delete_vu_count > 0) { - if (!obj_to_delete_selector) { - throw new Error('Positive DELETE worker number without a proper object selector'); - } + if (!obj_to_delete_selector) { + throw new Error( + 'Positive DELETE worker number without a proper object selector'); + } - scenarios.delete = { - executor: 'constant-vus', - vus: delete_vu_count, - duration: `${duration}s`, - exec: 'obj_delete', - gracefulStop: '5s', - }; + scenarios.delete = { + executor: 'constant-vus', + vus: delete_vu_count, + duration: `${duration}s`, + exec: 'obj_delete', + gracefulStop: '5s', + }; } export const options = { - scenarios, - setupTimeout: '5s', + scenarios, + setupTimeout: '5s', }; export function setup() { - const total_vu_count = write_vu_count + read_vu_count + delete_vu_count; + const total_vu_count = write_vu_count + read_vu_count + delete_vu_count; - console.log(`Pregenerated containers: ${container_list.length}`); - console.log(`Pregenerated read object size: ${read_size}`); - console.log(`Pregenerated total objects: ${obj_list.length}`); - console.log(`Reading VUs: ${read_vu_count}`); - console.log(`Writing VUs: ${write_vu_count}`); - console.log(`Deleting VUs: ${delete_vu_count}`); - console.log(`Total VUs: ${total_vu_count}`); + console.log(`Pregenerated containers: ${container_list.length}`); + console.log(`Pregenerated read object size: ${read_size}`); + console.log(`Pregenerated total objects: ${obj_list.length}`); + console.log(`Reading VUs: ${read_vu_count}`); + console.log(`Writing VUs: ${write_vu_count}`); + console.log(`Deleting VUs: ${delete_vu_count}`); + console.log(`Total VUs: ${total_vu_count}`); - const start_timestamp = Date.now() - console.log(`Load started at: ${Date(start_timestamp).toString()}`) + const start_timestamp = Date.now() + console.log( + `Load started at: ${Date(start_timestamp).toString()}`) } export function teardown(data) { - if (obj_registry) { - obj_registry.close(); - } - const end_timestamp = Date.now() - console.log(`Load finished at: ${Date(end_timestamp).toString()}`) + if (obj_registry) { + obj_registry.close(); + } + const end_timestamp = Date.now() + console.log( + `Load finished at: ${Date(end_timestamp).toString()}`) } export function handleSummary(data) { - return { - 'stdout': textSummary(data, { indent: ' ', enableColors: false }), - [summary_json]: JSON.stringify(data), - }; + return { + 'stdout': textSummary(data, {indent: ' ', enableColors: false}), + [summary_json]: JSON.stringify(data), + }; } export function obj_write() { - const headers = { - unique_header: uuidv4() - }; - const container = container_list[Math.floor(Math.random() * container_list.length)]; + const headers = {unique_header: uuidv4()}; + const container = + container_list[Math.floor(Math.random() * container_list.length)]; - const payload = generator.genPayload(); - const resp = local_client.put(container, headers, payload); - if (!resp.success) { - if (resp.abort) { - exec.test.abort(resp.error); - } - log.withField("cid", container).error(resp.error); - return; + const payload = generator.genPayload(); + const resp = local_client.put(container, headers, payload); + if (!resp.success) { + if (resp.abort) { + exec.test.abort(resp.error); } + log.withField('cid', container).error(resp.error); + return; + } - if (obj_registry) { - obj_registry.addObject(container, resp.object_id, "", "", payload.hash()); - } + if (obj_registry) { + obj_registry.addObject(container, resp.object_id, '', '', payload.hash()); + } } export function obj_read() { - const obj = obj_list[Math.floor(Math.random() * obj_list.length)]; - const resp = local_client.get(obj.container, obj.object) - if (!resp.success) { - log.withFields({cid: obj.container, oid: obj.object}).error(resp.error); - } + const obj = obj_list[Math.floor(Math.random() * obj_list.length)]; + const resp = local_client.get(obj.container, obj.object) + if (!resp.success) { + log.withFields({cid: obj.container, oid: obj.object}).error(resp.error); + } } export function obj_delete() { - const obj = obj_to_delete_selector.nextObject(); - if (!obj) { - return; - } + const obj = obj_to_delete_selector.nextObject(); + if (!obj) { + return; + } - const resp = local_client.delete(obj.c_id, obj.o_id); - if (!resp.success) { - // Log errors except (2052 - object already deleted) - log.withFields({cid: obj.c_id, oid: obj.o_id}).error(resp.error); - return; - } + const resp = local_client.delete(obj.c_id, obj.o_id); + if (!resp.success) { + // Log errors except (2052 - object already deleted) + log.withFields({cid: obj.c_id, oid: obj.o_id}).error(resp.error); + return; + } - obj_registry.deleteObject(obj.id); + obj_registry.deleteObject(obj.id); } diff --git a/scenarios/s3.js b/scenarios/s3.js index 7da53b9..0532748 100644 --- a/scenarios/s3.js +++ b/scenarios/s3.js @@ -1,70 +1,69 @@ +import {sleep} from 'k6'; +import {SharedArray} from 'k6/data'; import logging from 'k6/x/frostfs/logging'; import registry from 'k6/x/frostfs/registry'; -import stats from 'k6/x/frostfs/stats'; import s3 from 'k6/x/frostfs/s3'; -import { SharedArray } from 'k6/data'; -import { sleep } from 'k6'; -import { textSummary } from './libs/k6-summary-0.0.2.js'; -import { parseEnv } from './libs/env-parser.js'; -import { uuidv4 } from './libs/k6-utils-1.4.0.js'; -import { newGenerator } from './libs/datagen.js'; +import stats from 'k6/x/frostfs/stats'; + +import {newGenerator} from './libs/datagen.js'; +import {parseEnv} from './libs/env-parser.js'; +import {textSummary} from './libs/k6-summary-0.0.2.js'; +import {uuidv4} from './libs/k6-utils-1.4.0.js'; parseEnv(); -const obj_list = new SharedArray('obj_list', function () { - return JSON.parse(open(__ENV.PREGEN_JSON)).objects; +const obj_list = new SharedArray('obj_list', function() { + return JSON.parse(open(__ENV.PREGEN_JSON)).objects; }); -const bucket_list = new SharedArray('bucket_list', function () { - return JSON.parse(open(__ENV.PREGEN_JSON)).buckets; +const bucket_list = new SharedArray('bucket_list', function() { + return JSON.parse(open(__ENV.PREGEN_JSON)).buckets; }); const read_size = JSON.parse(open(__ENV.PREGEN_JSON)).obj_size; -const summary_json = __ENV.SUMMARY_JSON || "/tmp/summary.json"; +const summary_json = __ENV.SUMMARY_JSON || '/tmp/summary.json'; -const no_verify_ssl = __ENV.NO_VERIFY_SSL || "true"; -const connection_args = {no_verify_ssl: no_verify_ssl} +const no_verify_ssl = __ENV.NO_VERIFY_SSL || 'true'; +const connection_args = { + no_verify_ssl: no_verify_ssl +} // Select random S3 endpoint for current VU const s3_endpoints = __ENV.S3_ENDPOINTS.split(','); -const s3_endpoint = s3_endpoints[Math.floor(Math.random() * s3_endpoints.length)]; +const s3_endpoint = + s3_endpoints[Math.floor(Math.random() * s3_endpoints.length)]; const s3_client = s3.connect(s3_endpoint, connection_args); -const log = logging.new().withField("endpoint", s3_endpoint); +const log = logging.new().withField('endpoint', s3_endpoint); const registry_enabled = !!__ENV.REGISTRY_FILE; -const obj_registry = registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : undefined; +const obj_registry = + registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : undefined; const duration = __ENV.DURATION; if (!!__ENV.METRIC_TAGS) { - stats.setTags(__ENV.METRIC_TAGS) + stats.setTags(__ENV.METRIC_TAGS) } const delete_age = __ENV.DELETE_AGE ? parseInt(__ENV.DELETE_AGE) : undefined; let obj_to_delete_selector = undefined; if (registry_enabled && delete_age) { - obj_to_delete_selector = registry.getSelector( - __ENV.REGISTRY_FILE, - "obj_to_delete", - __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, - { - status: "created", - age: delete_age, - } - ); + obj_to_delete_selector = registry.getSelector( + __ENV.REGISTRY_FILE, 'obj_to_delete', + __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, { + status: 'created', + age: delete_age, + }); } const read_age = __ENV.READ_AGE ? parseInt(__ENV.READ_AGE) : 10; let obj_to_read_selector = undefined; if (registry_enabled) { - obj_to_read_selector = registry.getLoopedSelector( - __ENV.REGISTRY_FILE, - "obj_to_read", - __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, - { - status: "created", - age: read_age, - } - ) + obj_to_read_selector = registry.getLoopedSelector( + __ENV.REGISTRY_FILE, 'obj_to_read', + __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, { + status: 'created', + age: read_age, + }) } const scenarios = {}; @@ -72,136 +71,140 @@ const scenarios = {}; const write_vu_count = parseInt(__ENV.WRITERS || '0'); const generator = newGenerator(write_vu_count > 0); if (write_vu_count > 0) { - scenarios.write = { - executor: 'constant-vus', - vus: write_vu_count, - duration: `${duration}s`, - exec: 'obj_write', - gracefulStop: '5s', - }; + scenarios.write = { + executor: 'constant-vus', + vus: write_vu_count, + duration: `${duration}s`, + exec: 'obj_write', + gracefulStop: '5s', + }; } const read_vu_count = parseInt(__ENV.READERS || '0'); if (read_vu_count > 0) { - scenarios.read = { - executor: 'constant-vus', - vus: read_vu_count, - duration: `${duration}s`, - exec: 'obj_read', - gracefulStop: '5s', - }; + scenarios.read = { + executor: 'constant-vus', + vus: read_vu_count, + duration: `${duration}s`, + exec: 'obj_read', + gracefulStop: '5s', + }; } const delete_vu_count = parseInt(__ENV.DELETERS || '0'); if (delete_vu_count > 0) { - if (!obj_to_delete_selector) { - throw 'Positive DELETE worker number without a proper object selector'; - } + if (!obj_to_delete_selector) { + throw 'Positive DELETE worker number without a proper object selector'; + } - scenarios.delete = { - executor: 'constant-vus', - vus: delete_vu_count, - duration: `${duration}s`, - exec: 'obj_delete', - gracefulStop: '5s', - }; + scenarios.delete = { + executor: 'constant-vus', + vus: delete_vu_count, + duration: `${duration}s`, + exec: 'obj_delete', + gracefulStop: '5s', + }; } export const options = { - scenarios, - setupTimeout: '5s', + scenarios, + setupTimeout: '5s', }; export function setup() { - const total_vu_count = write_vu_count + read_vu_count + delete_vu_count; + const total_vu_count = write_vu_count + read_vu_count + delete_vu_count; - console.log(`Pregenerated buckets: ${bucket_list.length}`); - console.log(`Pregenerated read object size: ${read_size}`); - console.log(`Pregenerated total objects: ${obj_list.length}`); - console.log(`Reading VUs: ${read_vu_count}`); - console.log(`Writing VUs: ${write_vu_count}`); - console.log(`Deleting VUs: ${delete_vu_count}`); - console.log(`Total VUs: ${total_vu_count}`); - - const start_timestamp = Date.now() - console.log(`Load started at: ${Date(start_timestamp).toString()}`) + console.log(`Pregenerated buckets: ${bucket_list.length}`); + console.log(`Pregenerated read object size: ${read_size}`); + console.log(`Pregenerated total objects: ${obj_list.length}`); + console.log(`Reading VUs: ${read_vu_count}`); + console.log(`Writing VUs: ${write_vu_count}`); + console.log(`Deleting VUs: ${delete_vu_count}`); + console.log(`Total VUs: ${total_vu_count}`); + + const start_timestamp = Date.now() + console.log( + `Load started at: ${Date(start_timestamp).toString()}`) } export function teardown(data) { - if (obj_registry) { - obj_registry.close(); - } - const end_timestamp = Date.now() - console.log(`Load finished at: ${Date(end_timestamp).toString()}`) + if (obj_registry) { + obj_registry.close(); + } + const end_timestamp = Date.now() + console.log( + `Load finished at: ${Date(end_timestamp).toString()}`) } export function handleSummary(data) { - return { - 'stdout': textSummary(data, { indent: ' ', enableColors: false }), - [summary_json]: JSON.stringify(data), - }; - } + return { + 'stdout': textSummary(data, {indent: ' ', enableColors: false}), + [summary_json]: JSON.stringify(data), + }; +} export function obj_write() { - if (__ENV.SLEEP_WRITE) { - sleep(__ENV.SLEEP_WRITE); - } + if (__ENV.SLEEP_WRITE) { + sleep(__ENV.SLEEP_WRITE); + } - const key = __ENV.OBJ_NAME || uuidv4(); - const bucket = bucket_list[Math.floor(Math.random() * bucket_list.length)]; + const key = __ENV.OBJ_NAME || uuidv4(); + const bucket = bucket_list[Math.floor(Math.random() * bucket_list.length)]; - const payload = generator.genPayload(); - const resp = s3_client.put(bucket, key, payload); - if (!resp.success) { - log.withFields({bucket: bucket, key: key}).error(resp.error); - return; - } + const payload = generator.genPayload(); + const resp = s3_client.put(bucket, key, payload); + if (!resp.success) { + log.withFields({bucket: bucket, key: key}).error(resp.error); + return; + } - if (obj_registry) { - obj_registry.addObject("", "", bucket, key, payload.hash()); - } + if (obj_registry) { + obj_registry.addObject('', '', bucket, key, payload.hash()); + } } export function obj_read() { - if (__ENV.SLEEP_READ) { - sleep(__ENV.SLEEP_READ); + if (__ENV.SLEEP_READ) { + sleep(__ENV.SLEEP_READ); + } + + if (obj_to_read_selector) { + const obj = obj_to_read_selector.nextObject(); + if (!obj) { + return; } - - if(obj_to_read_selector) { - const obj = obj_to_read_selector.nextObject(); - if (!obj) { - return; - } - const resp = s3_client.get(obj.s3_bucket, obj.s3_key) - if (!resp.success) { - log.withFields({bucket: obj.s3_bucket, key: obj.s3_key}).error(resp.error); - } - return - } - - const obj = obj_list[Math.floor(Math.random() * obj_list.length)]; - - const resp = s3_client.get(obj.bucket, obj.object); + const resp = s3_client.get(obj.s3_bucket, obj.s3_key) if (!resp.success) { - log.withFields({bucket: obj.bucket, key: obj.object}).error(resp.error); - } + log.withFields({bucket: obj.s3_bucket, key: obj.s3_key}) + .error(resp.error); + } + return + } + + const obj = obj_list[Math.floor(Math.random() * obj_list.length)]; + + const resp = s3_client.get(obj.bucket, obj.object); + if (!resp.success) { + log.withFields({bucket: obj.bucket, key: obj.object}).error(resp.error); + } } export function obj_delete() { - if (__ENV.SLEEP_DELETE) { - sleep(__ENV.SLEEP_DELETE); - } + if (__ENV.SLEEP_DELETE) { + sleep(__ENV.SLEEP_DELETE); + } - const obj = obj_to_delete_selector.nextObject(); - if (!obj) { - return; - } + const obj = obj_to_delete_selector.nextObject(); + if (!obj) { + return; + } - const resp = s3_client.delete(obj.s3_bucket, obj.s3_key); - if (!resp.success) { - log.withFields({bucket: obj.s3_bucket, key: obj.s3_key, op: "DELETE"}).error(resp.error); - return; - } + const resp = s3_client.delete(obj.s3_bucket, obj.s3_key); + if (!resp.success) { + log.withFields({bucket: obj.s3_bucket, key: obj.s3_key, op: 'DELETE'}) + .error(resp.error); + return; + } - obj_registry.deleteObject(obj.id); + obj_registry.deleteObject(obj.id); } diff --git a/scenarios/s3_car.js b/scenarios/s3_car.js index 6666eee..2aaa94e 100644 --- a/scenarios/s3_car.js +++ b/scenarios/s3_car.js @@ -1,70 +1,69 @@ +import {sleep} from 'k6'; +import {SharedArray} from 'k6/data'; import logging from 'k6/x/frostfs/logging'; import registry from 'k6/x/frostfs/registry'; -import stats from 'k6/x/frostfs/stats'; import s3 from 'k6/x/frostfs/s3'; -import { SharedArray } from 'k6/data'; -import { sleep } from 'k6'; -import { textSummary } from './libs/k6-summary-0.0.2.js'; -import { parseEnv } from './libs/env-parser.js'; -import { uuidv4 } from './libs/k6-utils-1.4.0.js'; -import { newGenerator } from './libs/datagen.js'; +import stats from 'k6/x/frostfs/stats'; + +import {newGenerator} from './libs/datagen.js'; +import {parseEnv} from './libs/env-parser.js'; +import {textSummary} from './libs/k6-summary-0.0.2.js'; +import {uuidv4} from './libs/k6-utils-1.4.0.js'; parseEnv(); -const obj_list = new SharedArray('obj_list', function () { - return JSON.parse(open(__ENV.PREGEN_JSON)).objects; +const obj_list = new SharedArray('obj_list', function() { + return JSON.parse(open(__ENV.PREGEN_JSON)).objects; }); -const bucket_list = new SharedArray('bucket_list', function () { - return JSON.parse(open(__ENV.PREGEN_JSON)).buckets; +const bucket_list = new SharedArray('bucket_list', function() { + return JSON.parse(open(__ENV.PREGEN_JSON)).buckets; }); const read_size = JSON.parse(open(__ENV.PREGEN_JSON)).obj_size; -const summary_json = __ENV.SUMMARY_JSON || "/tmp/summary.json"; +const summary_json = __ENV.SUMMARY_JSON || '/tmp/summary.json'; // Select random S3 endpoint for current VU const s3_endpoints = __ENV.S3_ENDPOINTS.split(','); -const s3_endpoint = s3_endpoints[Math.floor(Math.random() * s3_endpoints.length)]; -const no_verify_ssl = __ENV.NO_VERIFY_SSL || "true"; -const connection_args = {no_verify_ssl: no_verify_ssl} +const s3_endpoint = + s3_endpoints[Math.floor(Math.random() * s3_endpoints.length)]; +const no_verify_ssl = __ENV.NO_VERIFY_SSL || 'true'; +const connection_args = { + no_verify_ssl: no_verify_ssl +}; const s3_client = s3.connect(s3_endpoint, connection_args); -const log = logging.new().withField("endpoint", s3_endpoint); +const log = logging.new().withField('endpoint', s3_endpoint); const registry_enabled = !!__ENV.REGISTRY_FILE; -const obj_registry = registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : undefined; +const obj_registry = + registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : undefined; const duration = __ENV.DURATION; if (!!__ENV.METRIC_TAGS) { - stats.setTags(__ENV.METRIC_TAGS) + stats.setTags(__ENV.METRIC_TAGS) } const delete_age = __ENV.DELETE_AGE ? parseInt(__ENV.DELETE_AGE) : undefined; let obj_to_delete_selector = undefined; if (registry_enabled && delete_age) { - obj_to_delete_selector = registry.getSelector( - __ENV.REGISTRY_FILE, - "obj_to_delete", - __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, - { - status: "created", - age: delete_age, - } - ); + obj_to_delete_selector = registry.getSelector( + __ENV.REGISTRY_FILE, 'obj_to_delete', + __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, { + status: 'created', + age: delete_age, + }); } const read_age = __ENV.READ_AGE ? parseInt(__ENV.READ_AGE) : 10; let obj_to_read_selector = undefined; if (registry_enabled) { - obj_to_read_selector = registry.getLoopedSelector( - __ENV.REGISTRY_FILE, - "obj_to_read", - __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, - { - status: "created", - age: read_age, - } - ) + obj_to_read_selector = registry.getLoopedSelector( + __ENV.REGISTRY_FILE, 'obj_to_read', + __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, { + status: 'created', + age: read_age, + }) } const scenarios = {}; @@ -75,16 +74,16 @@ const max_write_vus = parseInt(__ENV.MAX_WRITERS || pre_alloc_write_vus); const write_rate = parseInt(__ENV.WRITE_RATE || '0'); const generator = newGenerator(write_rate > 0); if (write_rate > 0) { - scenarios.write = { - executor: 'constant-arrival-rate', - duration: `${duration}s`, - preAllocatedVUs: pre_alloc_write_vus, - maxVUs: max_write_vus, - rate: write_rate, - timeUnit: time_unit, - exec: 'obj_write', - gracefulStop: '5s', - }; + scenarios.write = { + executor: 'constant-arrival-rate', + duration: `${duration}s`, + preAllocatedVUs: pre_alloc_write_vus, + maxVUs: max_write_vus, + rate: write_rate, + timeUnit: time_unit, + exec: 'obj_write', + gracefulStop: '5s', + }; } @@ -92,16 +91,16 @@ const pre_alloc_read_vus = parseInt(__ENV.PRE_ALLOC_READERS || '0'); const max_read_vus = parseInt(__ENV.MAX_READERS || pre_alloc_read_vus); const read_rate = parseInt(__ENV.READ_RATE || '0'); if (read_rate > 0) { - scenarios.read = { - executor: 'constant-arrival-rate', - duration: `${duration}s`, - preAllocatedVUs: pre_alloc_write_vus, - maxVUs: max_read_vus, - rate: read_rate, - timeUnit: time_unit, - exec: 'obj_read', - gracefulStop: '5s', - }; + scenarios.read = { + executor: 'constant-arrival-rate', + duration: `${duration}s`, + preAllocatedVUs: pre_alloc_write_vus, + maxVUs: max_read_vus, + rate: read_rate, + timeUnit: time_unit, + exec: 'obj_read', + gracefulStop: '5s', + }; } @@ -109,126 +108,132 @@ const pre_alloc_delete_vus = parseInt(__ENV.PRE_ALLOC_DELETERS || '0'); const max_delete_vus = parseInt(__ENV.MAX_DELETERS || pre_alloc_write_vus); const delete_rate = parseInt(__ENV.DELETE_RATE || '0'); if (delete_rate > 0) { - if (!obj_to_delete_selector) { - throw new Error('Positive DELETE worker number without a proper object selector'); - } + if (!obj_to_delete_selector) { + throw new Error( + 'Positive DELETE worker number without a proper object selector'); + } - scenarios.delete = { - executor: 'constant-arrival-rate', - duration: `${duration}s`, - preAllocatedVUs: pre_alloc_delete_vus, - maxVUs: max_delete_vus, - rate: delete_rate, - timeUnit: time_unit, - exec: 'obj_delete', - gracefulStop: '5s', - }; + scenarios.delete = { + executor: 'constant-arrival-rate', + duration: `${duration}s`, + preAllocatedVUs: pre_alloc_delete_vus, + maxVUs: max_delete_vus, + rate: delete_rate, + timeUnit: time_unit, + exec: 'obj_delete', + gracefulStop: '5s', + }; } export const options = { - scenarios, - setupTimeout: '5s', + scenarios, + setupTimeout: '5s', }; export function setup() { - const total_pre_allocated_vu_count = pre_alloc_write_vus + pre_alloc_read_vus + pre_alloc_delete_vus; - const total_max_vu_count = max_read_vus + max_write_vus + max_delete_vus + const total_pre_allocated_vu_count = + pre_alloc_write_vus + pre_alloc_read_vus + pre_alloc_delete_vus; + const total_max_vu_count = max_read_vus + max_write_vus + max_delete_vus - console.log(`Pregenerated buckets: ${bucket_list.length}`); - console.log(`Pregenerated read object size: ${read_size}`); - console.log(`Pregenerated total objects: ${obj_list.length}`); - console.log(`Pre allocated reading VUs: ${pre_alloc_read_vus}`); - console.log(`Pre allocated writing VUs: ${pre_alloc_write_vus}`); - console.log(`Pre allocated deleting VUs: ${pre_alloc_delete_vus}`); - console.log(`Total pre allocated VUs: ${total_pre_allocated_vu_count}`); - console.log(`Max reading VUs: ${max_read_vus}`); - console.log(`Max writing VUs: ${max_write_vus}`); - console.log(`Max deleting VUs: ${max_delete_vus}`); - console.log(`Total max VUs: ${total_max_vu_count}`); - console.log(`Time unit: ${time_unit}`); - console.log(`Read rate: ${read_rate}`); - console.log(`Writing rate: ${write_rate}`); - console.log(`Delete rate: ${delete_rate}`); + console.log(`Pregenerated buckets: ${bucket_list.length}`); + console.log(`Pregenerated read object size: ${read_size}`); + console.log(`Pregenerated total objects: ${obj_list.length}`); + console.log(`Pre allocated reading VUs: ${pre_alloc_read_vus}`); + console.log(`Pre allocated writing VUs: ${pre_alloc_write_vus}`); + console.log(`Pre allocated deleting VUs: ${pre_alloc_delete_vus}`); + console.log(`Total pre allocated VUs: ${total_pre_allocated_vu_count}`); + console.log(`Max reading VUs: ${max_read_vus}`); + console.log(`Max writing VUs: ${max_write_vus}`); + console.log(`Max deleting VUs: ${max_delete_vus}`); + console.log(`Total max VUs: ${total_max_vu_count}`); + console.log(`Time unit: ${time_unit}`); + console.log(`Read rate: ${read_rate}`); + console.log(`Writing rate: ${write_rate}`); + console.log(`Delete rate: ${delete_rate}`); - const start_timestamp = Date.now() - console.log(`Load started at: ${Date(start_timestamp).toString()}`) + const start_timestamp = Date.now() + console.log( + `Load started at: ${Date(start_timestamp).toString()}`) } export function teardown(data) { - if (obj_registry) { - obj_registry.close(); - } - const end_timestamp = Date.now() - console.log(`Load finished at: ${Date(end_timestamp).toString()}`) + if (obj_registry) { + obj_registry.close(); + } + const end_timestamp = Date.now() + console.log( + `Load finished at: ${Date(end_timestamp).toString()}`) } export function handleSummary(data) { - return { - 'stdout': textSummary(data, { indent: ' ', enableColors: false }), - [summary_json]: JSON.stringify(data), - }; - } + return { + 'stdout': textSummary(data, {indent: ' ', enableColors: false}), + [summary_json]: JSON.stringify(data), + }; +} export function obj_write() { - if (__ENV.SLEEP_WRITE) { - sleep(__ENV.SLEEP_WRITE); - } + if (__ENV.SLEEP_WRITE) { + sleep(__ENV.SLEEP_WRITE); + } - const key = __ENV.OBJ_NAME || uuidv4(); - const bucket = bucket_list[Math.floor(Math.random() * bucket_list.length)]; + const key = __ENV.OBJ_NAME || uuidv4(); + const bucket = bucket_list[Math.floor(Math.random() * bucket_list.length)]; - const payload = generator.genPayload(); - const resp = s3_client.put(bucket, key, payload); - if (!resp.success) { - log.withFields({bucket: bucket, key: key}).error(resp.error); - return; - } + const payload = generator.genPayload(); + const resp = s3_client.put(bucket, key, payload); + if (!resp.success) { + log.withFields({bucket: bucket, key: key}).error(resp.error); + return; + } - if (obj_registry) { - obj_registry.addObject("", "", bucket, key, payload.hash()); - } + if (obj_registry) { + obj_registry.addObject('', '', bucket, key, payload.hash()); + } } export function obj_read() { - if (__ENV.SLEEP_READ) { - sleep(__ENV.SLEEP_READ); + if (__ENV.SLEEP_READ) { + sleep(__ENV.SLEEP_READ); + } + + if (obj_to_read_selector) { + const obj = obj_to_read_selector.nextObject(); + if (!obj) { + return; } - - if(obj_to_read_selector) { - const obj = obj_to_read_selector.nextObject(); - if (!obj) { - return; - } - const resp = s3_client.get(obj.s3_bucket, obj.s3_key) - if (!resp.success) { - log.withFields({bucket: obj.s3_bucket, key: obj.s3_key}).error(resp.error); - } - return - } - - const obj = obj_list[Math.floor(Math.random() * obj_list.length)]; - - const resp = s3_client.get(obj.bucket, obj.object); + const resp = s3_client.get(obj.s3_bucket, obj.s3_key) if (!resp.success) { - log.withFields({bucket: obj.bucket, key: obj.object}).error(resp.error); - } + log.withFields({bucket: obj.s3_bucket, key: obj.s3_key}) + .error(resp.error); + } + return + } + + const obj = obj_list[Math.floor(Math.random() * obj_list.length)]; + + const resp = s3_client.get(obj.bucket, obj.object); + if (!resp.success) { + log.withFields({bucket: obj.bucket, key: obj.object}).error(resp.error); + } } export function obj_delete() { - if (__ENV.SLEEP_DELETE) { - sleep(__ENV.SLEEP_DELETE); - } + if (__ENV.SLEEP_DELETE) { + sleep(__ENV.SLEEP_DELETE); + } - const obj = obj_to_delete_selector.nextObject(); - if (!obj) { - return; - } + const obj = obj_to_delete_selector.nextObject(); + if (!obj) { + return; + } - const resp = s3_client.delete(obj.s3_bucket, obj.s3_key); - if (!resp.success) { - log.withFields({bucket: obj.s3_bucket, key: obj.s3_key, op: "DELETE"}).error(resp.error); - return; - } + const resp = s3_client.delete(obj.s3_bucket, obj.s3_key); + if (!resp.success) { + log.withFields({bucket: obj.s3_bucket, key: obj.s3_key, op: 'DELETE'}) + .error(resp.error); + return; + } - obj_registry.deleteObject(obj.id); + obj_registry.deleteObject(obj.id); } diff --git a/scenarios/s3_multipart.js b/scenarios/s3_multipart.js index a3c1e28..2b94ddc 100644 --- a/scenarios/s3_multipart.js +++ b/scenarios/s3_multipart.js @@ -1,110 +1,119 @@ +import {sleep} from 'k6'; +import {SharedArray} from 'k6/data'; import logging from 'k6/x/frostfs/logging'; import registry from 'k6/x/frostfs/registry'; -import stats from 'k6/x/frostfs/stats'; import s3 from 'k6/x/frostfs/s3'; -import {SharedArray} from 'k6/data'; -import {sleep} from 'k6'; -import {textSummary} from './libs/k6-summary-0.0.2.js'; +import stats from 'k6/x/frostfs/stats'; + +import {newGenerator} from './libs/datagen.js'; import {parseEnv} from './libs/env-parser.js'; +import {textSummary} from './libs/k6-summary-0.0.2.js'; import {uuidv4} from './libs/k6-utils-1.4.0.js'; -import { newGenerator } from './libs/datagen.js'; parseEnv(); -const bucket_list = new SharedArray('bucket_list', function () { - return JSON.parse(open(__ENV.PREGEN_JSON)).buckets; +const bucket_list = new SharedArray('bucket_list', function() { + return JSON.parse(open(__ENV.PREGEN_JSON)).buckets; }); -const summary_json = __ENV.SUMMARY_JSON || "/tmp/summary.json"; +const summary_json = __ENV.SUMMARY_JSON || '/tmp/summary.json'; // Select random S3 endpoint for current VU const s3_endpoints = __ENV.S3_ENDPOINTS.split(','); -const s3_endpoint = s3_endpoints[Math.floor(Math.random() * s3_endpoints.length)]; -const no_verify_ssl = __ENV.NO_VERIFY_SSL || "true"; -const connection_args = {no_verify_ssl: no_verify_ssl} +const s3_endpoint = + s3_endpoints[Math.floor(Math.random() * s3_endpoints.length)]; +const no_verify_ssl = __ENV.NO_VERIFY_SSL || 'true'; +const connection_args = { + no_verify_ssl: no_verify_ssl +}; const s3_client = s3.connect(s3_endpoint, connection_args); -const log = logging.new().withField("endpoint", s3_endpoint); +const log = logging.new().withField('endpoint', s3_endpoint); const registry_enabled = !!__ENV.REGISTRY_FILE; -const obj_registry = registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : undefined; +const obj_registry = + registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : undefined; const duration = __ENV.DURATION; if (!!__ENV.METRIC_TAGS) { - stats.setTags(__ENV.METRIC_TAGS) + stats.setTags(__ENV.METRIC_TAGS) } const scenarios = {}; const write_vu_count = parseInt(__ENV.WRITERS || '0'); if (write_vu_count < 1) { - throw 'number of VUs (env WRITERS) performing write operations should be greater than 0'; + throw 'number of VUs (env WRITERS) performing write operations should be greater than 0'; } const write_multipart_vu_count = parseInt(__ENV.WRITERS_MULTIPART || '0'); if (write_multipart_vu_count < 1) { - throw 'number of parts (env WRITERS_MULTIPART) to upload in parallel should be greater than 0'; + throw 'number of parts (env WRITERS_MULTIPART) to upload in parallel should be greater than 0'; } -const generator = newGenerator(write_vu_count > 0 || write_multipart_vu_count > 0); +const generator = + newGenerator(write_vu_count > 0 || write_multipart_vu_count > 0); if (write_vu_count > 0) { - scenarios.write_multipart = { - executor: 'constant-vus', - vus: write_vu_count, - duration: `${duration}s`, - exec: 'obj_write_multipart', - gracefulStop: '5s', - }; + scenarios.write_multipart = { + executor: 'constant-vus', + vus: write_vu_count, + duration: `${duration}s`, + exec: 'obj_write_multipart', + gracefulStop: '5s', + }; } export const options = { - scenarios, - setupTimeout: '5s', + scenarios, + setupTimeout: '5s', }; export function setup() { - const total_vu_count = write_vu_count * write_multipart_vu_count; + const total_vu_count = write_vu_count * write_multipart_vu_count; - console.log(`Pregenerated buckets: ${bucket_list.length}`); - console.log(`Writing VUs: ${write_vu_count}`); - console.log(`Writing multipart VUs: ${write_multipart_vu_count}`); - console.log(`Total VUs: ${total_vu_count}`); + console.log(`Pregenerated buckets: ${bucket_list.length}`); + console.log(`Writing VUs: ${write_vu_count}`); + console.log(`Writing multipart VUs: ${write_multipart_vu_count}`); + console.log(`Total VUs: ${total_vu_count}`); } export function teardown(data) { - if (obj_registry) { - obj_registry.close(); - } + if (obj_registry) { + obj_registry.close(); + } } export function handleSummary(data) { - return { - 'stdout': textSummary(data, {indent: ' ', enableColors: false}), - [summary_json]: JSON.stringify(data), - }; + return { + 'stdout': textSummary(data, {indent: ' ', enableColors: false}), + [summary_json]: JSON.stringify(data), + }; } -const write_multipart_part_size = 1024 * parseInt(__ENV.WRITE_OBJ_PART_SIZE || '0') +const write_multipart_part_size = + 1024 * parseInt(__ENV.WRITE_OBJ_PART_SIZE || '0') if (write_multipart_part_size < 5 * 1024 * 1024) { - throw 'part size (env WRITE_OBJ_PART_SIZE * 1024) must be greater than (5 MB)'; + throw 'part size (env WRITE_OBJ_PART_SIZE * 1024) must be greater than (5 MB)'; } export function obj_write_multipart() { - if (__ENV.SLEEP_WRITE) { - sleep(__ENV.SLEEP_WRITE); - } + if (__ENV.SLEEP_WRITE) { + sleep(__ENV.SLEEP_WRITE); + } - const key = __ENV.OBJ_NAME || uuidv4(); - const bucket = bucket_list[Math.floor(Math.random() * bucket_list.length)]; + const key = __ENV.OBJ_NAME || uuidv4(); + const bucket = bucket_list[Math.floor(Math.random() * bucket_list.length)]; - const payload = generator.genPayload(); - const resp = s3_client.multipart(bucket, key, write_multipart_part_size, write_multipart_vu_count, payload); - if (!resp.success) { - log.withFields({bucket: bucket, key: key}).error(resp.error); - return; - } + const payload = generator.genPayload(); + const resp = s3_client.multipart( + bucket, key, write_multipart_part_size, write_multipart_vu_count, + payload); + if (!resp.success) { + log.withFields({bucket: bucket, key: key}).error(resp.error); + return; + } - if (obj_registry) { - obj_registry.addObject("", "", bucket, key, payload.hash()); - } + if (obj_registry) { + obj_registry.addObject('', '', bucket, key, payload.hash()); + } } diff --git a/scenarios/s3local.js b/scenarios/s3local.js index 34b7562..6eaf049 100644 --- a/scenarios/s3local.js +++ b/scenarios/s3local.js @@ -1,67 +1,70 @@ +import {SharedArray} from 'k6/data'; +import exec from 'k6/execution'; import logging from 'k6/x/frostfs/logging'; import registry from 'k6/x/frostfs/registry'; -import stats from 'k6/x/frostfs/stats'; import s3local from 'k6/x/frostfs/s3local'; -import { SharedArray } from 'k6/data'; -import { textSummary } from './libs/k6-summary-0.0.2.js'; -import { parseEnv } from './libs/env-parser.js'; -import { uuidv4 } from './libs/k6-utils-1.4.0.js'; -import exec from 'k6/execution'; -import { newGenerator } from './libs/datagen.js'; +import stats from 'k6/x/frostfs/stats'; + +import {newGenerator} from './libs/datagen.js'; +import {parseEnv} from './libs/env-parser.js'; +import {textSummary} from './libs/k6-summary-0.0.2.js'; +import {uuidv4} from './libs/k6-utils-1.4.0.js'; parseEnv(); -const obj_list = new SharedArray('obj_list', function () { - return JSON.parse(open(__ENV.PREGEN_JSON)).objects; +const obj_list = new SharedArray('obj_list', function() { + return JSON.parse(open(__ENV.PREGEN_JSON)).objects; }); -const container_list = new SharedArray('container_list', function () { - return JSON.parse(open(__ENV.PREGEN_JSON)).containers; +const container_list = new SharedArray('container_list', function() { + return JSON.parse(open(__ENV.PREGEN_JSON)).containers; }); -const bucket_list = new SharedArray('bucket_list', function () { - return JSON.parse(open(__ENV.PREGEN_JSON)).buckets; +const bucket_list = new SharedArray('bucket_list', function() { + return JSON.parse(open(__ENV.PREGEN_JSON)).buckets; }); function bucket_mapping() { - if (container_list.length != bucket_list.length) { - throw 'The number of containers and buckets in the preset file must be the same.'; - } - let mapping = {}; - for (let i = 0; i < container_list.length; ++i) { - mapping[bucket_list[i]] = container_list[i]; - } - return mapping; + if (container_list.length != bucket_list.length) { + throw 'The number of containers and buckets in the preset file must be the same.'; + } + let mapping = {}; + for (let i = 0; i < container_list.length; ++i) { + mapping[bucket_list[i]] = container_list[i]; + } + return mapping; } const read_size = JSON.parse(open(__ENV.PREGEN_JSON)).obj_size; -const summary_json = __ENV.SUMMARY_JSON || "/tmp/summary.json"; +const summary_json = __ENV.SUMMARY_JSON || '/tmp/summary.json'; const config_file = __ENV.CONFIG_FILE; const config_dir = __ENV.CONFIG_DIR; -const max_total_size_gb = __ENV.MAX_TOTAL_SIZE_GB ? parseInt(__ENV.MAX_TOTAL_SIZE_GB) : 0; -const s3_client = s3local.connect(config_file, config_dir, { - 'debug_logger': __ENV.DEBUG_LOGGER || 'false', -}, bucket_mapping(), max_total_size_gb); -const log = logging.new().withFields({"config_file": config_file,"config_dir": config_dir}); +const max_total_size_gb = + __ENV.MAX_TOTAL_SIZE_GB ? parseInt(__ENV.MAX_TOTAL_SIZE_GB) : 0; +const s3_client = s3local.connect( + config_file, config_dir, { + 'debug_logger': __ENV.DEBUG_LOGGER || 'false', + }, + bucket_mapping(), max_total_size_gb); +const log = logging.new().withFields( + {'config_file': config_file, 'config_dir': config_dir}); if (!!__ENV.METRIC_TAGS) { - stats.setTags(__ENV.METRIC_TAGS) + stats.setTags(__ENV.METRIC_TAGS) } const registry_enabled = !!__ENV.REGISTRY_FILE; -const obj_registry = registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : undefined; +const obj_registry = + registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : undefined; let obj_to_read_selector = undefined; if (registry_enabled) { - obj_to_read_selector = registry.getLoopedSelector( - __ENV.REGISTRY_FILE, - "obj_to_read", - __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, - { - status: "created", - } - ) + obj_to_read_selector = registry.getLoopedSelector( + __ENV.REGISTRY_FILE, 'obj_to_read', + __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, { + status: 'created', + }) } const duration = __ENV.DURATION; @@ -71,96 +74,99 @@ const scenarios = {}; const write_vu_count = parseInt(__ENV.WRITERS || '0'); const generator = newGenerator(write_vu_count > 0); if (write_vu_count > 0) { - scenarios.write = { - executor: 'constant-vus', - vus: write_vu_count, - duration: `${duration}s`, - exec: 'obj_write', - gracefulStop: '5s', - }; + scenarios.write = { + executor: 'constant-vus', + vus: write_vu_count, + duration: `${duration}s`, + exec: 'obj_write', + gracefulStop: '5s', + }; } const read_vu_count = parseInt(__ENV.READERS || '0'); if (read_vu_count > 0) { - scenarios.read = { - executor: 'constant-vus', - vus: read_vu_count, - duration: `${duration}s`, - exec: 'obj_read', - gracefulStop: '5s', - }; + scenarios.read = { + executor: 'constant-vus', + vus: read_vu_count, + duration: `${duration}s`, + exec: 'obj_read', + gracefulStop: '5s', + }; } export const options = { - scenarios, - setupTimeout: '5s', + scenarios, + setupTimeout: '5s', }; export function setup() { - const total_vu_count = write_vu_count + read_vu_count; + const total_vu_count = write_vu_count + read_vu_count; - console.log(`Pregenerated buckets: ${bucket_list.length}`); - console.log(`Pregenerated read object size: ${read_size}`); - console.log(`Pregenerated total objects: ${obj_list.length}`); - console.log(`Reading VUs: ${read_vu_count}`); - console.log(`Writing VUs: ${write_vu_count}`); - console.log(`Total VUs: ${total_vu_count}`); + console.log(`Pregenerated buckets: ${bucket_list.length}`); + console.log(`Pregenerated read object size: ${read_size}`); + console.log(`Pregenerated total objects: ${obj_list.length}`); + console.log(`Reading VUs: ${read_vu_count}`); + console.log(`Writing VUs: ${write_vu_count}`); + console.log(`Total VUs: ${total_vu_count}`); - const start_timestamp = Date.now() - console.log(`Load started at: ${Date(start_timestamp).toString()}`) + const start_timestamp = Date.now() + console.log( + `Load started at: ${Date(start_timestamp).toString()}`) } export function teardown(data) { - if (obj_registry) { - obj_registry.close(); - } - const end_timestamp = Date.now() - console.log(`Load finished at: ${Date(end_timestamp).toString()}`) + if (obj_registry) { + obj_registry.close(); + } + const end_timestamp = Date.now() + console.log( + `Load finished at: ${Date(end_timestamp).toString()}`) } export function handleSummary(data) { - return { - 'stdout': textSummary(data, { indent: ' ', enableColors: false }), - [summary_json]: JSON.stringify(data), - }; - } + return { + 'stdout': textSummary(data, {indent: ' ', enableColors: false}), + [summary_json]: JSON.stringify(data), + }; +} export function obj_write() { - const key = __ENV.OBJ_NAME || uuidv4(); - const bucket = bucket_list[Math.floor(Math.random() * bucket_list.length)]; + const key = __ENV.OBJ_NAME || uuidv4(); + const bucket = bucket_list[Math.floor(Math.random() * bucket_list.length)]; - const payload = generator.genPayload(); - const resp = s3_client.put(bucket, key, payload); - if (!resp.success) { - if (resp.abort) { - exec.test.abort(resp.error); - } - log.withFields({bucket: bucket, key: key}).error(resp.error); - return; + const payload = generator.genPayload(); + const resp = s3_client.put(bucket, key, payload); + if (!resp.success) { + if (resp.abort) { + exec.test.abort(resp.error); } + log.withFields({bucket: bucket, key: key}).error(resp.error); + return; + } - if (obj_registry) { - obj_registry.addObject("", "", bucket, key, payload.hash()); - } + if (obj_registry) { + obj_registry.addObject('', '', bucket, key, payload.hash()); + } } export function obj_read() { - if(obj_to_read_selector) { - const obj = obj_to_read_selector.nextObject(); - if (!obj) { - return; - } - const resp = s3_client.get(obj.s3_bucket, obj.s3_key) - if (!resp.success) { - log.withFields({bucket: obj.s3_bucket, key: obj.s3_key}).error(resp.error); - } - return + if (obj_to_read_selector) { + const obj = obj_to_read_selector.nextObject(); + if (!obj) { + return; } - - const obj = obj_list[Math.floor(Math.random() * obj_list.length)]; - - const resp = s3_client.get(obj.bucket, obj.object); + const resp = s3_client.get(obj.s3_bucket, obj.s3_key) if (!resp.success) { - log.withFields({bucket: obj.bucket, key: obj.object}).error(resp.error); - } + log.withFields({bucket: obj.s3_bucket, key: obj.s3_key}) + .error(resp.error); + } + return + } + + const obj = obj_list[Math.floor(Math.random() * obj_list.length)]; + + const resp = s3_client.get(obj.bucket, obj.object); + if (!resp.success) { + log.withFields({bucket: obj.bucket, key: obj.object}).error(resp.error); + } } diff --git a/scenarios/verify.js b/scenarios/verify.js index 44318c6..e2f0d79 100644 --- a/scenarios/verify.js +++ b/scenarios/verify.js @@ -1,20 +1,21 @@ +import {sleep} from 'k6'; +import {Counter} from 'k6/metrics'; +import logging from 'k6/x/frostfs/logging'; import native from 'k6/x/frostfs/native'; import registry from 'k6/x/frostfs/registry'; -import stats from 'k6/x/frostfs/stats'; import s3 from 'k6/x/frostfs/s3'; -import logging from 'k6/x/frostfs/logging'; -import { sleep } from 'k6'; -import { Counter } from 'k6/metrics'; -import { textSummary } from './libs/k6-summary-0.0.2.js'; -import { parseEnv } from './libs/env-parser.js'; +import stats from 'k6/x/frostfs/stats'; + +import {parseEnv} from './libs/env-parser.js'; +import {textSummary} from './libs/k6-summary-0.0.2.js'; parseEnv(); const obj_registry = registry.open(__ENV.REGISTRY_FILE); // Time limit (in seconds) for the run -const time_limit = __ENV.TIME_LIMIT || "60"; -const summary_json = __ENV.SUMMARY_JSON || "/tmp/summary.json"; +const time_limit = __ENV.TIME_LIMIT || '60'; +const summary_json = __ENV.SUMMARY_JSON || '/tmp/summary.json'; // Number of objects in each status. These counters are cumulative in a // sense that they reflect total number of objects in the registry, not just @@ -22,144 +23,148 @@ const summary_json = __ENV.SUMMARY_JSON || "/tmp/summary.json"; // This allows to run this scenario multiple times and collect overall // statistics in the final run. const obj_counters = { - verified: new Counter('verified_obj'), - skipped: new Counter('skipped_obj'), - invalid: new Counter('invalid_obj'), + verified: new Counter('verified_obj'), + skipped: new Counter('skipped_obj'), + invalid: new Counter('invalid_obj'), }; let log = logging.new(); if (!!__ENV.METRIC_TAGS) { - stats.setTags(__ENV.METRIC_TAGS) + stats.setTags(__ENV.METRIC_TAGS) } // Connect to random gRPC endpoint let grpc_client = undefined; if (__ENV.GRPC_ENDPOINTS) { - const grpcEndpoints = __ENV.GRPC_ENDPOINTS.split(','); - const grpcEndpoint = grpcEndpoints[Math.floor(Math.random() * grpcEndpoints.length)]; - log = log.withField("endpoint", grpcEndpoint); - grpc_client = native.connect(grpcEndpoint, '', - __ENV.DIAL_TIMEOUT ? parseInt(__ENV.DIAL_TIMEOUT) : 0, - __ENV.STREAM_TIMEOUT ? parseInt(__ENV.STREAM_TIMEOUT) : 0, - __ENV.PREPARE_LOCALLY ? __ENV.PREPARE_LOCALLY.toLowerCase() === "true" : false, ''); + const grpcEndpoints = __ENV.GRPC_ENDPOINTS.split(','); + const grpcEndpoint = + grpcEndpoints[Math.floor(Math.random() * grpcEndpoints.length)]; + log = log.withField('endpoint', grpcEndpoint); + grpc_client = native.connect( + grpcEndpoint, '', __ENV.DIAL_TIMEOUT ? parseInt(__ENV.DIAL_TIMEOUT) : 0, + __ENV.STREAM_TIMEOUT ? parseInt(__ENV.STREAM_TIMEOUT) : 0, + __ENV.PREPARE_LOCALLY ? __ENV.PREPARE_LOCALLY.toLowerCase() === 'true' : + false, + ''); } // Connect to random S3 endpoint let s3_client = undefined; if (__ENV.S3_ENDPOINTS) { - const no_verify_ssl = __ENV.NO_VERIFY_SSL || "true"; - const connection_args = {no_verify_ssl: no_verify_ssl} - const s3_endpoints = __ENV.S3_ENDPOINTS.split(','); - const s3_endpoint = s3_endpoints[Math.floor(Math.random() * s3_endpoints.length)]; - log = log.withField("endpoint", s3_endpoint); - s3_client = s3.connect(s3_endpoint, connection_args); + const no_verify_ssl = __ENV.NO_VERIFY_SSL || 'true'; + const connection_args = {no_verify_ssl: no_verify_ssl}; + const s3_endpoints = __ENV.S3_ENDPOINTS.split(','); + const s3_endpoint = + s3_endpoints[Math.floor(Math.random() * s3_endpoints.length)]; + log = log.withField('endpoint', s3_endpoint); + s3_client = s3.connect(s3_endpoint, connection_args); } -// We will attempt to verify every object in "created" status. The scenario will execute -// as many iterations as there are objects. Each object will have 3 retries to be verified +// We will attempt to verify every object in "created" status. The scenario will +// execute as many iterations as there are objects. Each object will have 3 +// retries to be verified const obj_to_verify_selector = registry.getSelector( - __ENV.REGISTRY_FILE, - "obj_to_verify", - __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, - { - status: "created", - } -); + __ENV.REGISTRY_FILE, 'obj_to_verify', + __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, { + status: 'created', + }); const obj_to_verify_count = obj_to_verify_selector.count(); -// Execute at least one iteration (executor shared-iterations can't run 0 iterations) +// Execute at least one iteration (executor shared-iterations can't run 0 +// iterations) const iterations = Math.max(1, obj_to_verify_count); -// Executor shared-iterations requires number of iterations to be larger than number of VUs +// Executor shared-iterations requires number of iterations to be larger than +// number of VUs const vus = Math.min(__ENV.CLIENTS, iterations); const scenarios = { - verify: { - executor: 'shared-iterations', - vus, - iterations, - maxDuration: `${time_limit}s`, - exec: 'obj_verify', - gracefulStop: '5s', - } + verify: { + executor: 'shared-iterations', + vus, + iterations, + maxDuration: `${time_limit}s`, + exec: 'obj_verify', + gracefulStop: '5s', + } }; export const options = { - scenarios, - setupTimeout: '5s', + scenarios, + setupTimeout: '5s', }; export function setup() { - // Populate counters with initial values - for (const [status, counter] of Object.entries(obj_counters)) { - const obj_selector = registry.getSelector( - __ENV.REGISTRY_FILE, - status, - __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, - { status }); - counter.add(obj_selector.count()); - } + // Populate counters with initial values + for (const [status, counter] of Object.entries(obj_counters)) { + const obj_selector = registry.getSelector( + __ENV.REGISTRY_FILE, status, + __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, {status}); + counter.add(obj_selector.count()); + } } export function handleSummary(data) { - return { - 'stdout': textSummary(data, { indent: ' ', enableColors: false }), - [summary_json]: JSON.stringify(data), - }; + return { + 'stdout': textSummary(data, {indent: ' ', enableColors: false}), + [summary_json]: JSON.stringify(data), + }; } export function obj_verify() { - if (obj_to_verify_count == 0) { - log.info("Nothing to verify"); - return; - } + if (obj_to_verify_count == 0) { + log.info('Nothing to verify'); + return; + } - if (__ENV.SLEEP) { - sleep(__ENV.SLEEP); - } + if (__ENV.SLEEP) { + sleep(__ENV.SLEEP); + } - const obj = obj_to_verify_selector.nextObject(); - if (!obj) { - log.info("All objects have been verified"); - return; - } + const obj = obj_to_verify_selector.nextObject(); + if (!obj) { + log.info('All objects have been verified'); + return; + } - const obj_status = verify_object_with_retries(obj, 3); - obj_counters[obj_status].add(1); - obj_registry.setObjectStatus(obj.id, obj.status, obj_status); + const obj_status = verify_object_with_retries(obj, 3); + obj_counters[obj_status].add(1); + obj_registry.setObjectStatus(obj.id, obj.status, obj_status); } function verify_object_with_retries(obj, attempts) { - for (let i = 0; i < attempts; i++) { - let result; - // Different name is required. - // ReferenceError: Cannot access a variable before initialization. - let lg = log; - if (obj.c_id && obj.o_id) { - lg = lg.withFields({cid: obj.c_id, oid: obj.o_id}); - result = grpc_client.verifyHash(obj.c_id, obj.o_id, obj.payload_hash); - } else if (obj.s3_bucket && obj.s3_key) { - lg = lg.withFields({bucket: obj.s3_bucket, key: obj.s3_key}); - result = s3_client.verifyHash(obj.s3_bucket, obj.s3_key, obj.payload_hash); - } else { - lg.withFields({ - cid: obj.c_id, - oid: obj.o_id, - bucket: obj.s3_bucket, - key: obj.s3_key - }).warn(`Object cannot be verified with supported protocols`); - return "skipped"; - } - - if (result.success) { - return "verified"; - } else if (result.error == "hash mismatch") { - return "invalid"; - } - - // Unless we explicitly saw that there was a hash mismatch, then we will retry after a delay - lg.error(`Verify error: ${result.error}. Object will be re-tried`); - sleep(__ENV.SLEEP); + for (let i = 0; i < attempts; i++) { + let result; + // Different name is required. + // ReferenceError: Cannot access a variable before initialization. + let lg = log; + if (obj.c_id && obj.o_id) { + lg = lg.withFields({cid: obj.c_id, oid: obj.o_id}); + result = grpc_client.verifyHash(obj.c_id, obj.o_id, obj.payload_hash); + } else if (obj.s3_bucket && obj.s3_key) { + lg = lg.withFields({bucket: obj.s3_bucket, key: obj.s3_key}); + result = + s3_client.verifyHash(obj.s3_bucket, obj.s3_key, obj.payload_hash); + } else { + lg.withFields({ + cid: obj.c_id, + oid: obj.o_id, + bucket: obj.s3_bucket, + key: obj.s3_key + }).warn(`Object cannot be verified with supported protocols`); + return 'skipped'; } - return "invalid"; + if (result.success) { + return 'verified'; + } else if (result.error == 'hash mismatch') { + return 'invalid'; + } + + // Unless we explicitly saw that there was a hash mismatch, then we will + // retry after a delay + lg.error(`Verify error: ${result.error}. Object will be re-tried`); + sleep(__ENV.SLEEP); + } + + return 'invalid'; }