From bdf4c192e17e05bf35abfc012c0691f89f476fe6 Mon Sep 17 00:00:00 2001 From: "m.malygina" Date: Thu, 6 Jun 2024 14:40:56 +0300 Subject: [PATCH] [#143] Add new s3 scenario where delete only already read objects Signed-off-by: m.malygina --- scenarios/s3_dar.js | 230 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 230 insertions(+) create mode 100644 scenarios/s3_dar.js diff --git a/scenarios/s3_dar.js b/scenarios/s3_dar.js new file mode 100644 index 0000000..34711c5 --- /dev/null +++ b/scenarios/s3_dar.js @@ -0,0 +1,230 @@ +import {sleep} from 'k6'; +import {SharedArray} from 'k6/data'; +import exec from 'k6/execution'; +import logging from 'k6/x/frostfs/logging'; +import registry from 'k6/x/frostfs/registry'; +import s3 from 'k6/x/frostfs/s3'; +import stats from 'k6/x/frostfs/stats'; + +import {newGenerator} from './libs/datagen.js'; +import {parseEnv} from './libs/env-parser.js'; +import {textSummary} from './libs/k6-summary-0.0.2.js'; +import {uuidv4} from './libs/k6-utils-1.4.0.js'; + +parseEnv(); + +const obj_list = new SharedArray( + 'obj_list', + function() { return JSON.parse(open(__ENV.PREGEN_JSON)).objects; }); + +const bucket_list = new SharedArray( + 'bucket_list', + function() { return JSON.parse(open(__ENV.PREGEN_JSON)).buckets; }); + +const read_size = JSON.parse(open(__ENV.PREGEN_JSON)).obj_size; +const summary_json = __ENV.SUMMARY_JSON || '/tmp/summary.json'; + +const no_verify_ssl = __ENV.NO_VERIFY_SSL || 'true'; + +const connection_args = { + no_verify_ssl : no_verify_ssl +} +// Select random S3 endpoint for current VU +const s3_endpoints = __ENV.S3_ENDPOINTS.split(','); +const s3_endpoint = + s3_endpoints[Math.floor(Math.random() * s3_endpoints.length)]; +const s3_client = s3.connect(s3_endpoint, connection_args); +const log = logging.new().withField('endpoint', s3_endpoint); + +const registry_enabled = !!__ENV.REGISTRY_FILE; +const obj_registry = + registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : undefined; + +const duration = __ENV.DURATION; + +if (!!__ENV.METRIC_TAGS) { + stats.setTags(__ENV.METRIC_TAGS) +} + +const read_age = __ENV.READ_AGE ? parseInt(__ENV.READ_AGE) : 10; +let obj_to_read_selector = undefined; +if (registry_enabled) { + obj_to_read_selector = registry.getSelector( + __ENV.REGISTRY_FILE, 'obj_to_read', + __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, { + status : 'created', + age : read_age, + }) +} + +const scenarios = {}; + +const write_vu_count = parseInt(__ENV.WRITERS || '0'); +const generator = newGenerator(write_vu_count > 0); +if (write_vu_count > 0) { + scenarios.write = { + executor : 'constant-vus', + vus : write_vu_count, + duration : `${duration}s`, + exec : 'obj_write', + gracefulStop : '5s', + }; +} + +const delete_age = __ENV.DELETE_AGE ? parseInt(__ENV.DELETE_AGE) : undefined; +let obj_to_delete_selector = undefined; +let obj_to_delete_exit_on_null = undefined; +if (registry_enabled && delete_age) { + obj_to_delete_exit_on_null = write_vu_count == 0; + + let constructor = obj_to_delete_exit_on_null ? registry.getOneshotSelector + : registry.getSelector; + + obj_to_delete_selector = + constructor(__ENV.REGISTRY_FILE, 'obj_to_delete', + __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, { + status : 'read', + age : delete_age, + }); +} + +const read_vu_count = parseInt(__ENV.READERS || '0'); +if (read_vu_count > 0) { + scenarios.read = { + executor : 'constant-vus', + vus : read_vu_count, + duration : `${duration}s`, + exec : 'obj_read', + gracefulStop : '5s', + }; +} + +const delete_vu_count = parseInt(__ENV.DELETERS || '0'); +if (delete_vu_count > 0) { + if (!obj_to_delete_selector) { + throw 'Positive DELETE worker number without a proper object selector'; + } + + scenarios.delete = { + executor : 'constant-vus', + vus : delete_vu_count, + duration : `${duration}s`, + exec : 'obj_delete', + gracefulStop : '5s', + }; +} + +export const options = { + scenarios, + setupTimeout : '5s', +}; + +export function setup() { + const total_vu_count = write_vu_count + read_vu_count + delete_vu_count; + + console.log(`Pregenerated buckets: ${bucket_list.length}`); + console.log(`Pregenerated read object size: ${read_size}`); + console.log(`Pregenerated total objects: ${obj_list.length}`); + console.log(`Reading VUs: ${read_vu_count}`); + console.log(`Writing VUs: ${write_vu_count}`); + console.log(`Deleting VUs: ${delete_vu_count}`); + console.log(`Total VUs: ${total_vu_count}`); + + const start_timestamp = Date.now() + console.log( + `Load started at: ${Date(start_timestamp).toString()}`) +} + +export function teardown(data) { + if (obj_registry) { + obj_registry.close(); + } + const end_timestamp = Date.now() + console.log( + `Load finished at: ${Date(end_timestamp).toString()}`) +} + +export function handleSummary(data) { + return { + 'stdout' : textSummary(data, {indent : ' ', enableColors : false}), + [summary_json] : JSON.stringify(data), + }; +} + +export function obj_write() { + if (__ENV.SLEEP_WRITE) { + sleep(__ENV.SLEEP_WRITE); + } + + const key = __ENV.OBJ_NAME || uuidv4(); + const bucket = bucket_list[Math.floor(Math.random() * bucket_list.length)]; + + const payload = generator.genPayload(); + const resp = s3_client.put(bucket, key, payload); + if (!resp.success) { + log.withFields({bucket : bucket, key : key}).error(resp.error); + return; + } + + if (obj_registry) { + obj_registry.addObject('', '', bucket, key, payload.hash()); + } +} + +export function obj_read() { + if (__ENV.SLEEP_READ) { + sleep(__ENV.SLEEP_READ); + } + + if (obj_to_read_selector) { + const obj = obj_to_read_selector.nextObject(); + if (!obj ) { + return; + } + const resp = s3_client.get(obj.s3_bucket, obj.s3_key) + if (!resp.success) { + log.withFields({bucket : obj.s3_bucket, key : obj.s3_key, status: obj.status, op: `READ`}) + .error(resp.error); + } else { + obj_registry.setObjectStatus(obj.id, obj.status, 'read'); + } + + return + } + + const obj = obj_list[Math.floor(Math.random() * obj_list.length)]; + + const resp = s3_client.get(obj.bucket, obj.object); + if (!resp.success) { + log.withFields({bucket : obj.bucket, key : obj.object}).error(resp.error); + } else { + obj_registry.setObjectStatus(obj.id, obj.status, 'read'); + } +} + +export function obj_delete() { + if (__ENV.SLEEP_DELETE) { + sleep(__ENV.SLEEP_DELETE); + } + + const obj = obj_to_delete_selector.nextObject(); + delete_object(obj) +} + +export function delete_object(obj) { + if (!obj) { + if (obj_to_delete_exit_on_null) { + exec.test.abort("No more objects to select"); + } + return; + } + + const resp = s3_client.delete(obj.s3_bucket, obj.s3_key); + if (!resp.success) { + log.withFields({bucket : obj.s3_bucket, key : obj.s3_key, op : 'DELETE'}) + .error(resp.error); + return; + } + + obj_registry.deleteObject(obj.id); +} \ No newline at end of file