forked from TrueCloudLab/xk6-frostfs
[#193] Evenly pick endpoints during VU init
Signed-off-by: a.berezin <a.berezin@yadro.com>
This commit is contained in:
parent
ebbc5bc0a7
commit
8459c46346
7 changed files with 207 additions and 70 deletions
|
@ -1,6 +1,6 @@
|
||||||
import { sleep } from 'k6';
|
import { sleep } from 'k6';
|
||||||
import { SharedArray } from 'k6/data';
|
import { SharedArray } from 'k6/data';
|
||||||
import exec from 'k6/execution';
|
import exec, {vu} from 'k6/execution';
|
||||||
import logging from 'k6/x/frostfs/logging';
|
import logging from 'k6/x/frostfs/logging';
|
||||||
import native from 'k6/x/frostfs/native';
|
import native from 'k6/x/frostfs/native';
|
||||||
import registry from 'k6/x/frostfs/registry';
|
import registry from 'k6/x/frostfs/registry';
|
||||||
|
@ -24,16 +24,9 @@ const container_list = new SharedArray(
|
||||||
const read_size = JSON.parse(open(__ENV.PREGEN_JSON)).obj_size;
|
const read_size = JSON.parse(open(__ENV.PREGEN_JSON)).obj_size;
|
||||||
const summary_json = __ENV.SUMMARY_JSON || '/tmp/summary.json';
|
const summary_json = __ENV.SUMMARY_JSON || '/tmp/summary.json';
|
||||||
|
|
||||||
// Select random gRPC endpoint for current VU
|
|
||||||
const grpc_endpoints = __ENV.GRPC_ENDPOINTS.split(',');
|
const grpc_endpoints = __ENV.GRPC_ENDPOINTS.split(',');
|
||||||
const grpc_endpoint =
|
let vu_logged = false;
|
||||||
grpc_endpoints[Math.floor(Math.random() * grpc_endpoints.length)];
|
const log = logging.new()
|
||||||
const grpc_client = native.connect(
|
|
||||||
grpc_endpoint, '', __ENV.DIAL_TIMEOUT ? parseInt(__ENV.DIAL_TIMEOUT) : 5,
|
|
||||||
__ENV.STREAM_TIMEOUT ? parseInt(__ENV.STREAM_TIMEOUT) : 60,
|
|
||||||
__ENV.PREPARE_LOCALLY ? __ENV.PREPARE_LOCALLY.toLowerCase() === 'true' : false,
|
|
||||||
1024 * parseInt(__ENV.MAX_OBJECT_SIZE || '0'));
|
|
||||||
const log = logging.new().withField('endpoint', grpc_endpoint);
|
|
||||||
|
|
||||||
const registry_enabled = !!__ENV.REGISTRY_FILE;
|
const registry_enabled = !!__ENV.REGISTRY_FILE;
|
||||||
const obj_registry =
|
const obj_registry =
|
||||||
|
@ -41,6 +34,25 @@ const obj_registry =
|
||||||
|
|
||||||
const duration = __ENV.DURATION;
|
const duration = __ENV.DURATION;
|
||||||
|
|
||||||
|
function get_endpoint() {
|
||||||
|
return grpc_endpoints[(vu.idInTest - 1) % grpc_endpoints.length];
|
||||||
|
}
|
||||||
|
|
||||||
|
function get_client(op, endpoint) {
|
||||||
|
const grpc_client = native.connect(
|
||||||
|
endpoint, '', __ENV.DIAL_TIMEOUT ? parseInt(__ENV.DIAL_TIMEOUT) : 5,
|
||||||
|
__ENV.STREAM_TIMEOUT ? parseInt(__ENV.STREAM_TIMEOUT) : 60,
|
||||||
|
__ENV.PREPARE_LOCALLY ? __ENV.PREPARE_LOCALLY.toLowerCase() === 'true' : false,
|
||||||
|
1024 * parseInt(__ENV.MAX_OBJECT_SIZE || '0'));
|
||||||
|
|
||||||
|
if (!vu_logged) {
|
||||||
|
log.withFields({op: op, vuidt: vu.idInTest, vuidi: vu.idInInstance, endpoint: endpoint}).info("VU initialized");
|
||||||
|
vu_logged = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
return grpc_client
|
||||||
|
}
|
||||||
|
|
||||||
if (!!__ENV.METRIC_TAGS) {
|
if (!!__ENV.METRIC_TAGS) {
|
||||||
stats.setTags(__ENV.METRIC_TAGS)
|
stats.setTags(__ENV.METRIC_TAGS)
|
||||||
}
|
}
|
||||||
|
@ -157,6 +169,9 @@ export function obj_write() {
|
||||||
sleep(__ENV.SLEEP_WRITE);
|
sleep(__ENV.SLEEP_WRITE);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const grpc_endpoint = get_endpoint();
|
||||||
|
const grpc_client = get_client("WRITE", grpc_endpoint)
|
||||||
|
|
||||||
const headers = { unique_header: uuidv4() };
|
const headers = { unique_header: uuidv4() };
|
||||||
const container =
|
const container =
|
||||||
container_list[Math.floor(Math.random() * container_list.length)];
|
container_list[Math.floor(Math.random() * container_list.length)];
|
||||||
|
@ -165,7 +180,7 @@ export function obj_write() {
|
||||||
const resp =
|
const resp =
|
||||||
grpc_client.put(container, headers, payload, write_grpc_chunk_size);
|
grpc_client.put(container, headers, payload, write_grpc_chunk_size);
|
||||||
if (!resp.success) {
|
if (!resp.success) {
|
||||||
log.withField('cid', container).error(resp.error);
|
log.withFields({cid: container, endpoint: grpc_endpoint}).error(resp.error);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -179,6 +194,9 @@ export function obj_read() {
|
||||||
sleep(__ENV.SLEEP_READ);
|
sleep(__ENV.SLEEP_READ);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const grpc_endpoint = get_endpoint();
|
||||||
|
const grpc_client = get_client("READ", grpc_endpoint)
|
||||||
|
|
||||||
if (obj_to_read_selector) {
|
if (obj_to_read_selector) {
|
||||||
const obj = obj_to_read_selector.nextObject();
|
const obj = obj_to_read_selector.nextObject();
|
||||||
if (!obj) {
|
if (!obj) {
|
||||||
|
@ -186,7 +204,7 @@ export function obj_read() {
|
||||||
}
|
}
|
||||||
const resp = grpc_client.get(obj.c_id, obj.o_id)
|
const resp = grpc_client.get(obj.c_id, obj.o_id)
|
||||||
if (!resp.success) {
|
if (!resp.success) {
|
||||||
log.withFields({ cid: obj.c_id, oid: obj.o_id }).error(resp.error);
|
log.withFields({cid: obj.c_id, oid: obj.o_id, endpoint: grpc_endpoint}).error(resp.error);
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
@ -194,7 +212,7 @@ export function obj_read() {
|
||||||
const obj = obj_list[Math.floor(Math.random() * obj_list.length)];
|
const obj = obj_list[Math.floor(Math.random() * obj_list.length)];
|
||||||
const resp = grpc_client.get(obj.container, obj.object)
|
const resp = grpc_client.get(obj.container, obj.object)
|
||||||
if (!resp.success) {
|
if (!resp.success) {
|
||||||
log.withFields({ cid: obj.container, oid: obj.object }).error(resp.error);
|
log.withFields({cid: obj.container, oid: obj.object, endpoint: grpc_endpoint}).error(resp.error);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -211,10 +229,13 @@ export function obj_delete() {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const grpc_endpoint = get_endpoint();
|
||||||
|
const grpc_client = get_client("DELETE", grpc_endpoint)
|
||||||
|
|
||||||
const resp = grpc_client.delete(obj.c_id, obj.o_id);
|
const resp = grpc_client.delete(obj.c_id, obj.o_id);
|
||||||
if (!resp.success) {
|
if (!resp.success) {
|
||||||
// Log errors except (2052 - object already deleted)
|
// Log errors except (2052 - object already deleted)
|
||||||
log.withFields({ cid: obj.c_id, oid: obj.o_id }).error(resp.error);
|
log.withFields({cid: obj.c_id, oid: obj.o_id, endpoint: grpc_endpoint}).error(resp.error);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1,5 +1,7 @@
|
||||||
import { sleep } from 'k6';
|
import { sleep } from 'k6';
|
||||||
import { SharedArray } from 'k6/data';
|
import { SharedArray } from 'k6/data';
|
||||||
|
import { vu } from 'k6/execution';
|
||||||
|
|
||||||
import logging from 'k6/x/frostfs/logging';
|
import logging from 'k6/x/frostfs/logging';
|
||||||
import native from 'k6/x/frostfs/native';
|
import native from 'k6/x/frostfs/native';
|
||||||
import registry from 'k6/x/frostfs/registry';
|
import registry from 'k6/x/frostfs/registry';
|
||||||
|
@ -23,16 +25,9 @@ const container_list = new SharedArray('container_list', function () {
|
||||||
const read_size = JSON.parse(open(__ENV.PREGEN_JSON)).obj_size;
|
const read_size = JSON.parse(open(__ENV.PREGEN_JSON)).obj_size;
|
||||||
const summary_json = __ENV.SUMMARY_JSON || '/tmp/summary.json';
|
const summary_json = __ENV.SUMMARY_JSON || '/tmp/summary.json';
|
||||||
|
|
||||||
// Select random gRPC endpoint for current VU
|
|
||||||
const grpc_endpoints = __ENV.GRPC_ENDPOINTS.split(',');
|
const grpc_endpoints = __ENV.GRPC_ENDPOINTS.split(',');
|
||||||
const grpc_endpoint =
|
let vu_logged = false;
|
||||||
grpc_endpoints[Math.floor(Math.random() * grpc_endpoints.length)];
|
const log = logging.new()
|
||||||
const grpc_client = native.connect(
|
|
||||||
grpc_endpoint, '', __ENV.DIAL_TIMEOUT ? parseInt(__ENV.DIAL_TIMEOUT) : 5,
|
|
||||||
__ENV.STREAM_TIMEOUT ? parseInt(__ENV.STREAM_TIMEOUT) : 60,
|
|
||||||
__ENV.PREPARE_LOCALLY ? __ENV.PREPARE_LOCALLY.toLowerCase() === 'true' : false,
|
|
||||||
1024 * parseInt(__ENV.MAX_OBJECT_SIZE || '0'));
|
|
||||||
const log = logging.new().withField('endpoint', grpc_endpoint);
|
|
||||||
|
|
||||||
const registry_enabled = !!__ENV.REGISTRY_FILE;
|
const registry_enabled = !!__ENV.REGISTRY_FILE;
|
||||||
const obj_registry =
|
const obj_registry =
|
||||||
|
@ -40,6 +35,25 @@ const obj_registry =
|
||||||
|
|
||||||
const duration = __ENV.DURATION;
|
const duration = __ENV.DURATION;
|
||||||
|
|
||||||
|
function get_endpoint() {
|
||||||
|
return grpc_endpoints[(vu.idInTest - 1) % grpc_endpoints.length];
|
||||||
|
}
|
||||||
|
|
||||||
|
function get_client(op, endpoint) {
|
||||||
|
const grpc_client = native.connect(
|
||||||
|
endpoint, '', __ENV.DIAL_TIMEOUT ? parseInt(__ENV.DIAL_TIMEOUT) : 5,
|
||||||
|
__ENV.STREAM_TIMEOUT ? parseInt(__ENV.STREAM_TIMEOUT) : 60,
|
||||||
|
__ENV.PREPARE_LOCALLY ? __ENV.PREPARE_LOCALLY.toLowerCase() === 'true' : false,
|
||||||
|
1024 * parseInt(__ENV.MAX_OBJECT_SIZE || '0'));
|
||||||
|
|
||||||
|
if (!vu_logged) {
|
||||||
|
log.withFields({op: op, vuidt: vu.idInTest, vuidi: vu.idInInstance, endpoint: endpoint}).info("VU initialized");
|
||||||
|
vu_logged = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
return grpc_client
|
||||||
|
}
|
||||||
|
|
||||||
if (!!__ENV.METRIC_TAGS) {
|
if (!!__ENV.METRIC_TAGS) {
|
||||||
stats.setTags(__ENV.METRIC_TAGS)
|
stats.setTags(__ENV.METRIC_TAGS)
|
||||||
}
|
}
|
||||||
|
@ -176,6 +190,9 @@ export function obj_write() {
|
||||||
sleep(__ENV.SLEEP_WRITE);
|
sleep(__ENV.SLEEP_WRITE);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const grpc_endpoint = get_endpoint();
|
||||||
|
const grpc_client = get_client("WRITE", grpc_endpoint)
|
||||||
|
|
||||||
const headers = { unique_header: uuidv4() };
|
const headers = { unique_header: uuidv4() };
|
||||||
const container =
|
const container =
|
||||||
container_list[Math.floor(Math.random() * container_list.length)];
|
container_list[Math.floor(Math.random() * container_list.length)];
|
||||||
|
@ -184,7 +201,7 @@ export function obj_write() {
|
||||||
const resp =
|
const resp =
|
||||||
grpc_client.put(container, headers, payload, write_grpc_chunk_size);
|
grpc_client.put(container, headers, payload, write_grpc_chunk_size);
|
||||||
if (!resp.success) {
|
if (!resp.success) {
|
||||||
log.withField('cid', container).error(resp.error);
|
log.withFields({cid: container, endpoint: grpc_endpoint}).error(resp.error);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -198,6 +215,9 @@ export function obj_read() {
|
||||||
sleep(__ENV.SLEEP_READ);
|
sleep(__ENV.SLEEP_READ);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const grpc_endpoint = get_endpoint();
|
||||||
|
const grpc_client = get_client("READ", grpc_endpoint)
|
||||||
|
|
||||||
if (obj_to_read_selector) {
|
if (obj_to_read_selector) {
|
||||||
const obj = obj_to_read_selector.nextObject();
|
const obj = obj_to_read_selector.nextObject();
|
||||||
if (!obj) {
|
if (!obj) {
|
||||||
|
@ -205,7 +225,7 @@ export function obj_read() {
|
||||||
}
|
}
|
||||||
const resp = grpc_client.get(obj.c_id, obj.o_id)
|
const resp = grpc_client.get(obj.c_id, obj.o_id)
|
||||||
if (!resp.success) {
|
if (!resp.success) {
|
||||||
log.withFields({ cid: obj.c_id, oid: obj.o_id }).error(resp.error);
|
log.withFields({cid: obj.c_id, oid: obj.o_id, endpoint: grpc_endpoint}).error(resp.error);
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
@ -213,7 +233,7 @@ export function obj_read() {
|
||||||
const obj = obj_list[Math.floor(Math.random() * obj_list.length)];
|
const obj = obj_list[Math.floor(Math.random() * obj_list.length)];
|
||||||
const resp = grpc_client.get(obj.container, obj.object)
|
const resp = grpc_client.get(obj.container, obj.object)
|
||||||
if (!resp.success) {
|
if (!resp.success) {
|
||||||
log.withFields({ cid: obj.container, oid: obj.object }).error(resp.error);
|
log.withFields({cid: obj.container, oid: obj.object, endpoint: grpc_endpoint}).error(resp.error);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -227,10 +247,13 @@ export function obj_delete() {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const grpc_endpoint = get_endpoint();
|
||||||
|
const grpc_client = get_client("DELETE", grpc_endpoint)
|
||||||
|
|
||||||
const resp = grpc_client.delete(obj.c_id, obj.o_id);
|
const resp = grpc_client.delete(obj.c_id, obj.o_id);
|
||||||
if (!resp.success) {
|
if (!resp.success) {
|
||||||
// Log errors except (2052 - object already deleted)
|
// Log errors except (2052 - object already deleted)
|
||||||
log.withFields({ cid: obj.c_id, oid: obj.o_id }).error(resp.error);
|
log.withFields({cid: obj.c_id, oid: obj.o_id, endpoint: grpc_endpoint}).error(resp.error);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1,5 +1,6 @@
|
||||||
import {sleep} from 'k6';
|
import {sleep} from 'k6';
|
||||||
import {SharedArray} from 'k6/data';
|
import {SharedArray} from 'k6/data';
|
||||||
|
import { vu } from 'k6/execution';
|
||||||
import http from 'k6/http';
|
import http from 'k6/http';
|
||||||
import logging from 'k6/x/frostfs/logging';
|
import logging from 'k6/x/frostfs/logging';
|
||||||
import registry from 'k6/x/frostfs/registry';
|
import registry from 'k6/x/frostfs/registry';
|
||||||
|
@ -23,11 +24,9 @@ const container_list = new SharedArray('container_list', function() {
|
||||||
const read_size = JSON.parse(open(__ENV.PREGEN_JSON)).obj_size;
|
const read_size = JSON.parse(open(__ENV.PREGEN_JSON)).obj_size;
|
||||||
const summary_json = __ENV.SUMMARY_JSON || '/tmp/summary.json';
|
const summary_json = __ENV.SUMMARY_JSON || '/tmp/summary.json';
|
||||||
|
|
||||||
// Select random HTTP endpoint for current VU
|
|
||||||
const http_endpoints = __ENV.HTTP_ENDPOINTS.split(',');
|
const http_endpoints = __ENV.HTTP_ENDPOINTS.split(',');
|
||||||
const http_endpoint =
|
let vu_logged = false;
|
||||||
http_endpoints[Math.floor(Math.random() * http_endpoints.length)];
|
const log = logging.new()
|
||||||
const log = logging.new().withField('endpoint', http_endpoint);
|
|
||||||
|
|
||||||
const registry_enabled = !!__ENV.REGISTRY_FILE;
|
const registry_enabled = !!__ENV.REGISTRY_FILE;
|
||||||
const obj_registry =
|
const obj_registry =
|
||||||
|
@ -35,6 +34,17 @@ const obj_registry =
|
||||||
|
|
||||||
const duration = __ENV.DURATION;
|
const duration = __ENV.DURATION;
|
||||||
|
|
||||||
|
function get_endpoint(op) {
|
||||||
|
const endpoint = http_endpoints[(vu.idInTest - 1) % http_endpoints.length];
|
||||||
|
|
||||||
|
if (!vu_logged) {
|
||||||
|
log.withFields({op: op, vuidt: vu.idInTest, vuidi: vu.idInInstance, endpoint: endpoint}).info("VU initialized");
|
||||||
|
vu_logged = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
return endpoint
|
||||||
|
}
|
||||||
|
|
||||||
if (!!__ENV.METRIC_TAGS) {
|
if (!!__ENV.METRIC_TAGS) {
|
||||||
stats.setTags(__ENV.METRIC_TAGS)
|
stats.setTags(__ENV.METRIC_TAGS)
|
||||||
}
|
}
|
||||||
|
@ -117,9 +127,11 @@ export function obj_write() {
|
||||||
file: http.file(payload.bytes(), 'random.data'),
|
file: http.file(payload.bytes(), 'random.data'),
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const http_endpoint = get_endpoint("WRITE")
|
||||||
|
|
||||||
const resp = http.post(`http://${http_endpoint}/upload/${container}`, data);
|
const resp = http.post(`http://${http_endpoint}/upload/${container}`, data);
|
||||||
if (resp.status != 200) {
|
if (resp.status != 200) {
|
||||||
log.withFields({status: resp.status, cid: container}).error(resp.error);
|
log.withFields({status: resp.status, cid: container, endpoint: http_endpoint}).error(resp.error);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const object_id = JSON.parse(resp.body).object_id;
|
const object_id = JSON.parse(resp.body).object_id;
|
||||||
|
@ -133,11 +145,13 @@ export function obj_read() {
|
||||||
sleep(__ENV.SLEEP_READ);
|
sleep(__ENV.SLEEP_READ);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const http_endpoint = get_endpoint("READ")
|
||||||
|
|
||||||
const obj = obj_list[Math.floor(Math.random() * obj_list.length)];
|
const obj = obj_list[Math.floor(Math.random() * obj_list.length)];
|
||||||
const resp =
|
const resp =
|
||||||
http.get(`http://${http_endpoint}/get/${obj.container}/${obj.object}`);
|
http.get(`http://${http_endpoint}/get/${obj.container}/${obj.object}`);
|
||||||
if (resp.status != 200) {
|
if (resp.status != 200) {
|
||||||
log.withFields({status: resp.status, cid: obj.container, oid: obj.object})
|
log.withFields({status: resp.status, cid: obj.container, oid: obj.object, endpoint: http_endpoint})
|
||||||
.error(resp.error);
|
.error(resp.error);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
import {sleep} from 'k6';
|
import {sleep} from 'k6';
|
||||||
import {SharedArray} from 'k6/data';
|
import {SharedArray} from 'k6/data';
|
||||||
import exec from 'k6/execution';
|
import exec, {vu} from 'k6/execution';
|
||||||
import logging from 'k6/x/frostfs/logging';
|
import logging from 'k6/x/frostfs/logging';
|
||||||
import registry from 'k6/x/frostfs/registry';
|
import registry from 'k6/x/frostfs/registry';
|
||||||
import s3 from 'k6/x/frostfs/s3';
|
import s3 from 'k6/x/frostfs/s3';
|
||||||
|
@ -28,19 +28,29 @@ const no_verify_ssl = __ENV.NO_VERIFY_SSL || 'true';
|
||||||
const connection_args = {
|
const connection_args = {
|
||||||
no_verify_ssl : no_verify_ssl
|
no_verify_ssl : no_verify_ssl
|
||||||
}
|
}
|
||||||
// Select random S3 endpoint for current VU
|
|
||||||
const s3_endpoints = __ENV.S3_ENDPOINTS.split(',');
|
const s3_endpoints = __ENV.S3_ENDPOINTS.split(',');
|
||||||
const s3_endpoint =
|
const log = logging.new()
|
||||||
s3_endpoints[Math.floor(Math.random() * s3_endpoints.length)];
|
|
||||||
const s3_client = s3.connect(s3_endpoint, connection_args);
|
|
||||||
const log = logging.new().withField('endpoint', s3_endpoint);
|
|
||||||
|
|
||||||
const registry_enabled = !!__ENV.REGISTRY_FILE;
|
const registry_enabled = !!__ENV.REGISTRY_FILE;
|
||||||
const obj_registry =
|
const obj_registry =
|
||||||
registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : undefined;
|
registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : undefined;
|
||||||
|
let vu_logged = false;
|
||||||
const duration = __ENV.DURATION;
|
const duration = __ENV.DURATION;
|
||||||
|
|
||||||
|
function get_endpoint() {
|
||||||
|
return s3_endpoints[(vu.idInTest - 1) % s3_endpoints.length];
|
||||||
|
}
|
||||||
|
|
||||||
|
function get_client(op, endpoint) {
|
||||||
|
const s3_client = s3.connect(endpoint, connection_args);
|
||||||
|
|
||||||
|
if (!vu_logged) {
|
||||||
|
log.withFields({op: op, vuidt: vu.idInTest, vuidi: vu.idInInstance, endpoint: endpoint}).info("VU initialized");
|
||||||
|
vu_logged = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
return s3_client;
|
||||||
|
}
|
||||||
|
|
||||||
if (!!__ENV.METRIC_TAGS) {
|
if (!!__ENV.METRIC_TAGS) {
|
||||||
stats.setTags(__ENV.METRIC_TAGS)
|
stats.setTags(__ENV.METRIC_TAGS)
|
||||||
}
|
}
|
||||||
|
@ -159,13 +169,16 @@ export function obj_write() {
|
||||||
sleep(__ENV.SLEEP_WRITE);
|
sleep(__ENV.SLEEP_WRITE);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const s3_endpoint = get_endpoint();
|
||||||
|
const s3_client = get_client("WRITE", s3_endpoint)
|
||||||
|
|
||||||
const key = generateS3Key();
|
const key = generateS3Key();
|
||||||
const bucket = bucket_list[Math.floor(Math.random() * bucket_list.length)];
|
const bucket = bucket_list[Math.floor(Math.random() * bucket_list.length)];
|
||||||
|
|
||||||
const payload = generator.genPayload();
|
const payload = generator.genPayload();
|
||||||
const resp = s3_client.put(bucket, key, payload);
|
const resp = s3_client.put(bucket, key, payload);
|
||||||
if (!resp.success) {
|
if (!resp.success) {
|
||||||
log.withFields({bucket : bucket, key : key}).error(resp.error);
|
log.withFields({bucket : bucket, key : key, endpoint: s3_endpoint}).error(resp.error);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -179,14 +192,18 @@ export function obj_read() {
|
||||||
sleep(__ENV.SLEEP_READ);
|
sleep(__ENV.SLEEP_READ);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const s3_endpoint = get_endpoint();
|
||||||
|
const s3_client = get_client("READ", s3_endpoint)
|
||||||
|
|
||||||
if (obj_to_read_selector) {
|
if (obj_to_read_selector) {
|
||||||
const obj = obj_to_read_selector.nextObject();
|
const obj = obj_to_read_selector.nextObject();
|
||||||
if (!obj) {
|
if (!obj) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const resp = s3_client.get(obj.s3_bucket, obj.s3_key)
|
const resp = s3_client.get(obj.s3_bucket, obj.s3_key)
|
||||||
if (!resp.success) {
|
if (!resp.success) {
|
||||||
log.withFields({bucket : obj.s3_bucket, key : obj.s3_key})
|
log.withFields({bucket : obj.s3_bucket, key : obj.s3_key, endpoint: s3_endpoints})
|
||||||
.error(resp.error);
|
.error(resp.error);
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
|
@ -205,6 +222,9 @@ export function obj_delete() {
|
||||||
sleep(__ENV.SLEEP_DELETE);
|
sleep(__ENV.SLEEP_DELETE);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const s3_endpoint = get_endpoint();
|
||||||
|
const s3_client = get_client("DELETE", s3_endpoint)
|
||||||
|
|
||||||
const obj = obj_to_delete_selector.nextObject();
|
const obj = obj_to_delete_selector.nextObject();
|
||||||
if (!obj) {
|
if (!obj) {
|
||||||
if (obj_to_delete_exit_on_null) {
|
if (obj_to_delete_exit_on_null) {
|
||||||
|
@ -217,7 +237,7 @@ export function obj_delete() {
|
||||||
|
|
||||||
const resp = s3_client.delete(obj.s3_bucket, obj.s3_key);
|
const resp = s3_client.delete(obj.s3_bucket, obj.s3_key);
|
||||||
if (!resp.success) {
|
if (!resp.success) {
|
||||||
log.withFields({bucket : obj.s3_bucket, key : obj.s3_key, op : 'DELETE'})
|
log.withFields({bucket : obj.s3_bucket, key : obj.s3_key, endpoint: s3_endpoint, op : 'DELETE'})
|
||||||
.error(resp.error);
|
.error(resp.error);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,5 +1,6 @@
|
||||||
import {sleep} from 'k6';
|
import {sleep} from 'k6';
|
||||||
import {SharedArray} from 'k6/data';
|
import {SharedArray} from 'k6/data';
|
||||||
|
import { vu } from 'k6/execution';
|
||||||
import logging from 'k6/x/frostfs/logging';
|
import logging from 'k6/x/frostfs/logging';
|
||||||
import registry from 'k6/x/frostfs/registry';
|
import registry from 'k6/x/frostfs/registry';
|
||||||
import s3 from 'k6/x/frostfs/s3';
|
import s3 from 'k6/x/frostfs/s3';
|
||||||
|
@ -23,16 +24,13 @@ const bucket_list = new SharedArray('bucket_list', function() {
|
||||||
const read_size = JSON.parse(open(__ENV.PREGEN_JSON)).obj_size;
|
const read_size = JSON.parse(open(__ENV.PREGEN_JSON)).obj_size;
|
||||||
const summary_json = __ENV.SUMMARY_JSON || '/tmp/summary.json';
|
const summary_json = __ENV.SUMMARY_JSON || '/tmp/summary.json';
|
||||||
|
|
||||||
// Select random S3 endpoint for current VU
|
|
||||||
const s3_endpoints = __ENV.S3_ENDPOINTS.split(',');
|
const s3_endpoints = __ENV.S3_ENDPOINTS.split(',');
|
||||||
const s3_endpoint =
|
|
||||||
s3_endpoints[Math.floor(Math.random() * s3_endpoints.length)];
|
|
||||||
const no_verify_ssl = __ENV.NO_VERIFY_SSL || 'true';
|
const no_verify_ssl = __ENV.NO_VERIFY_SSL || 'true';
|
||||||
const connection_args = {
|
const connection_args = {
|
||||||
no_verify_ssl: no_verify_ssl
|
no_verify_ssl: no_verify_ssl
|
||||||
};
|
};
|
||||||
const s3_client = s3.connect(s3_endpoint, connection_args);
|
let vu_logged = false;
|
||||||
const log = logging.new().withField('endpoint', s3_endpoint);
|
const log = logging.new()
|
||||||
|
|
||||||
const registry_enabled = !!__ENV.REGISTRY_FILE;
|
const registry_enabled = !!__ENV.REGISTRY_FILE;
|
||||||
const obj_registry =
|
const obj_registry =
|
||||||
|
@ -40,6 +38,21 @@ const obj_registry =
|
||||||
|
|
||||||
const duration = __ENV.DURATION;
|
const duration = __ENV.DURATION;
|
||||||
|
|
||||||
|
function get_endpoint() {
|
||||||
|
return s3_endpoints[(vu.idInTest - 1) % s3_endpoints.length];
|
||||||
|
}
|
||||||
|
|
||||||
|
function get_client(op, endpoint) {
|
||||||
|
const s3_client = s3.connect(endpoint, connection_args);
|
||||||
|
|
||||||
|
if (!vu_logged) {
|
||||||
|
log.withFields({op: op, vuidt: vu.idInTest, vuidi: vu.idInInstance, endpoint: endpoint}).info("VU initialized");
|
||||||
|
vu_logged = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
return s3_client;
|
||||||
|
}
|
||||||
|
|
||||||
if (!!__ENV.METRIC_TAGS) {
|
if (!!__ENV.METRIC_TAGS) {
|
||||||
stats.setTags(__ENV.METRIC_TAGS)
|
stats.setTags(__ENV.METRIC_TAGS)
|
||||||
}
|
}
|
||||||
|
@ -177,13 +190,16 @@ export function obj_write() {
|
||||||
sleep(__ENV.SLEEP_WRITE);
|
sleep(__ENV.SLEEP_WRITE);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const s3_endpoint = get_endpoint();
|
||||||
|
const s3_client = get_client("WRITE", s3_endpoint)
|
||||||
|
|
||||||
const key = generateS3Key();
|
const key = generateS3Key();
|
||||||
const bucket = bucket_list[Math.floor(Math.random() * bucket_list.length)];
|
const bucket = bucket_list[Math.floor(Math.random() * bucket_list.length)];
|
||||||
|
|
||||||
const payload = generator.genPayload();
|
const payload = generator.genPayload();
|
||||||
const resp = s3_client.put(bucket, key, payload);
|
const resp = s3_client.put(bucket, key, payload);
|
||||||
if (!resp.success) {
|
if (!resp.success) {
|
||||||
log.withFields({bucket: bucket, key: key}).error(resp.error);
|
log.withFields({bucket: bucket, key: key, endpoint: s3_endpoint}).error(resp.error);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -197,6 +213,9 @@ export function obj_read() {
|
||||||
sleep(__ENV.SLEEP_READ);
|
sleep(__ENV.SLEEP_READ);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const s3_endpoint = get_endpoint();
|
||||||
|
const s3_client = get_client("READ", s3_endpoint)
|
||||||
|
|
||||||
if (obj_to_read_selector) {
|
if (obj_to_read_selector) {
|
||||||
const obj = obj_to_read_selector.nextObject();
|
const obj = obj_to_read_selector.nextObject();
|
||||||
if (!obj) {
|
if (!obj) {
|
||||||
|
@ -214,7 +233,7 @@ export function obj_read() {
|
||||||
|
|
||||||
const resp = s3_client.get(obj.bucket, obj.object);
|
const resp = s3_client.get(obj.bucket, obj.object);
|
||||||
if (!resp.success) {
|
if (!resp.success) {
|
||||||
log.withFields({bucket: obj.bucket, key: obj.object}).error(resp.error);
|
log.withFields({bucket: obj.bucket, key: obj.object, endpoint: s3_endpoint}).error(resp.error);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -228,9 +247,12 @@ export function obj_delete() {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const s3_endpoint = get_endpoint();
|
||||||
|
const s3_client = get_client("DELETE", s3_endpoint)
|
||||||
|
|
||||||
const resp = s3_client.delete(obj.s3_bucket, obj.s3_key);
|
const resp = s3_client.delete(obj.s3_bucket, obj.s3_key);
|
||||||
if (!resp.success) {
|
if (!resp.success) {
|
||||||
log.withFields({bucket: obj.s3_bucket, key: obj.s3_key, op: 'DELETE'})
|
log.withFields({bucket: obj.s3_bucket, key: obj.s3_key, op: 'DELETE', endpoint: s3_endpoint})
|
||||||
.error(resp.error);
|
.error(resp.error);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
import {sleep} from 'k6';
|
import {sleep} from 'k6';
|
||||||
import {SharedArray} from 'k6/data';
|
import {SharedArray} from 'k6/data';
|
||||||
import exec from 'k6/execution';
|
import exec, {vu} from 'k6/execution';
|
||||||
import logging from 'k6/x/frostfs/logging';
|
import logging from 'k6/x/frostfs/logging';
|
||||||
import registry from 'k6/x/frostfs/registry';
|
import registry from 'k6/x/frostfs/registry';
|
||||||
import s3 from 'k6/x/frostfs/s3';
|
import s3 from 'k6/x/frostfs/s3';
|
||||||
|
@ -29,19 +29,31 @@ const no_verify_ssl = __ENV.NO_VERIFY_SSL || 'true';
|
||||||
const connection_args = {
|
const connection_args = {
|
||||||
no_verify_ssl : no_verify_ssl
|
no_verify_ssl : no_verify_ssl
|
||||||
}
|
}
|
||||||
// Select random S3 endpoint for current VU
|
|
||||||
const s3_endpoints = __ENV.S3_ENDPOINTS.split(',');
|
const s3_endpoints = __ENV.S3_ENDPOINTS.split(',');
|
||||||
const s3_endpoint =
|
const log = logging.new()
|
||||||
s3_endpoints[Math.floor(Math.random() * s3_endpoints.length)];
|
|
||||||
const s3_client = s3.connect(s3_endpoint, connection_args);
|
|
||||||
const log = logging.new().withField('endpoint', s3_endpoint);
|
|
||||||
|
|
||||||
const registry_enabled = !!__ENV.REGISTRY_FILE;
|
const registry_enabled = !!__ENV.REGISTRY_FILE;
|
||||||
const obj_registry =
|
const obj_registry =
|
||||||
registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : undefined;
|
registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : undefined;
|
||||||
|
let vu_logged = false;
|
||||||
const duration = __ENV.DURATION;
|
const duration = __ENV.DURATION;
|
||||||
|
|
||||||
|
function get_endpoint() {
|
||||||
|
return s3_endpoints[(vu.idInTest - 1) % s3_endpoints.length];
|
||||||
|
}
|
||||||
|
|
||||||
|
function get_client(op, endpoint) {
|
||||||
|
const s3_client = s3.connect(endpoint, connection_args);
|
||||||
|
|
||||||
|
if (!vu_logged) {
|
||||||
|
log.withFields({op: op, vuidt: vu.idInTest, vuidi: vu.idInInstance, endpoint: endpoint}).info("VU initialized");
|
||||||
|
vu_logged = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
return s3_client;
|
||||||
|
}
|
||||||
|
|
||||||
if (!!__ENV.METRIC_TAGS) {
|
if (!!__ENV.METRIC_TAGS) {
|
||||||
stats.setTags(__ENV.METRIC_TAGS)
|
stats.setTags(__ENV.METRIC_TAGS)
|
||||||
}
|
}
|
||||||
|
@ -159,13 +171,16 @@ export function obj_write() {
|
||||||
sleep(__ENV.SLEEP_WRITE);
|
sleep(__ENV.SLEEP_WRITE);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const s3_endpoint = get_endpoint();
|
||||||
|
const s3_client = get_client("WRITE", s3_endpoint)
|
||||||
|
|
||||||
const key = generateS3Key();
|
const key = generateS3Key();
|
||||||
const bucket = bucket_list[Math.floor(Math.random() * bucket_list.length)];
|
const bucket = bucket_list[Math.floor(Math.random() * bucket_list.length)];
|
||||||
|
|
||||||
const payload = generator.genPayload();
|
const payload = generator.genPayload();
|
||||||
const resp = s3_client.put(bucket, key, payload);
|
const resp = s3_client.put(bucket, key, payload);
|
||||||
if (!resp.success) {
|
if (!resp.success) {
|
||||||
log.withFields({bucket : bucket, key : key}).error(resp.error);
|
log.withFields({bucket : bucket, key : key, endpoint: s3_endpoint}).error(resp.error);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -179,6 +194,9 @@ export function obj_read() {
|
||||||
sleep(__ENV.SLEEP_READ);
|
sleep(__ENV.SLEEP_READ);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const s3_endpoint = get_endpoint();
|
||||||
|
const s3_client = get_client("READ", s3_endpoint)
|
||||||
|
|
||||||
if (obj_to_read_selector) {
|
if (obj_to_read_selector) {
|
||||||
const obj = obj_to_read_selector.nextObject();
|
const obj = obj_to_read_selector.nextObject();
|
||||||
if (!obj ) {
|
if (!obj ) {
|
||||||
|
@ -186,7 +204,7 @@ export function obj_read() {
|
||||||
}
|
}
|
||||||
const resp = s3_client.get(obj.s3_bucket, obj.s3_key)
|
const resp = s3_client.get(obj.s3_bucket, obj.s3_key)
|
||||||
if (!resp.success) {
|
if (!resp.success) {
|
||||||
log.withFields({bucket : obj.s3_bucket, key : obj.s3_key, status: obj.status, op: `READ`})
|
log.withFields({bucket : obj.s3_bucket, key : obj.s3_key, status: obj.status, op: `READ`, endpoint: s3_endpoint})
|
||||||
.error(resp.error);
|
.error(resp.error);
|
||||||
} else {
|
} else {
|
||||||
obj_registry.setObjectStatus(obj.id, obj.status, 'read');
|
obj_registry.setObjectStatus(obj.id, obj.status, 'read');
|
||||||
|
@ -199,7 +217,7 @@ export function obj_read() {
|
||||||
|
|
||||||
const resp = s3_client.get(obj.bucket, obj.object);
|
const resp = s3_client.get(obj.bucket, obj.object);
|
||||||
if (!resp.success) {
|
if (!resp.success) {
|
||||||
log.withFields({bucket : obj.bucket, key : obj.object}).error(resp.error);
|
log.withFields({bucket : obj.bucket, key : obj.object, endpoint: s3_endpoint}).error(resp.error);
|
||||||
} else {
|
} else {
|
||||||
obj_registry.setObjectStatus(obj.id, obj.status, 'read');
|
obj_registry.setObjectStatus(obj.id, obj.status, 'read');
|
||||||
}
|
}
|
||||||
|
@ -222,9 +240,12 @@ export function delete_object(obj) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const s3_endpoint = get_endpoint();
|
||||||
|
const s3_client = get_client("DELETE", s3_endpoint)
|
||||||
|
|
||||||
const resp = s3_client.delete(obj.s3_bucket, obj.s3_key);
|
const resp = s3_client.delete(obj.s3_bucket, obj.s3_key);
|
||||||
if (!resp.success) {
|
if (!resp.success) {
|
||||||
log.withFields({bucket : obj.s3_bucket, key : obj.s3_key, op : 'DELETE'})
|
log.withFields({bucket : obj.s3_bucket, key : obj.s3_key, op : 'DELETE', endpoint: s3_endpoint})
|
||||||
.error(resp.error);
|
.error(resp.error);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,5 +1,6 @@
|
||||||
import {sleep} from 'k6';
|
import {sleep} from 'k6';
|
||||||
import {SharedArray} from 'k6/data';
|
import {SharedArray} from 'k6/data';
|
||||||
|
import { vu } from 'k6/execution';
|
||||||
import logging from 'k6/x/frostfs/logging';
|
import logging from 'k6/x/frostfs/logging';
|
||||||
import registry from 'k6/x/frostfs/registry';
|
import registry from 'k6/x/frostfs/registry';
|
||||||
import s3 from 'k6/x/frostfs/s3';
|
import s3 from 'k6/x/frostfs/s3';
|
||||||
|
@ -18,16 +19,13 @@ const bucket_list = new SharedArray('bucket_list', function() {
|
||||||
|
|
||||||
const summary_json = __ENV.SUMMARY_JSON || '/tmp/summary.json';
|
const summary_json = __ENV.SUMMARY_JSON || '/tmp/summary.json';
|
||||||
|
|
||||||
// Select random S3 endpoint for current VU
|
|
||||||
const s3_endpoints = __ENV.S3_ENDPOINTS.split(',');
|
const s3_endpoints = __ENV.S3_ENDPOINTS.split(',');
|
||||||
const s3_endpoint =
|
|
||||||
s3_endpoints[Math.floor(Math.random() * s3_endpoints.length)];
|
|
||||||
const no_verify_ssl = __ENV.NO_VERIFY_SSL || 'true';
|
const no_verify_ssl = __ENV.NO_VERIFY_SSL || 'true';
|
||||||
const connection_args = {
|
const connection_args = {
|
||||||
no_verify_ssl: no_verify_ssl
|
no_verify_ssl: no_verify_ssl
|
||||||
};
|
};
|
||||||
const s3_client = s3.connect(s3_endpoint, connection_args);
|
let vu_logged = false;
|
||||||
const log = logging.new().withField('endpoint', s3_endpoint);
|
const log = logging.new();
|
||||||
|
|
||||||
const registry_enabled = !!__ENV.REGISTRY_FILE;
|
const registry_enabled = !!__ENV.REGISTRY_FILE;
|
||||||
const obj_registry =
|
const obj_registry =
|
||||||
|
@ -35,6 +33,21 @@ const obj_registry =
|
||||||
|
|
||||||
const duration = __ENV.DURATION;
|
const duration = __ENV.DURATION;
|
||||||
|
|
||||||
|
function get_endpoint() {
|
||||||
|
return s3_endpoints[(vu.idInTest - 1) % s3_endpoints.length];
|
||||||
|
}
|
||||||
|
|
||||||
|
function get_client(op, endpoint) {
|
||||||
|
const s3_client = s3.connect(endpoint, connection_args);
|
||||||
|
|
||||||
|
if (!vu_logged) {
|
||||||
|
log.withFields({op: op, vuidt: vu.idInTest, vuidi: vu.idInInstance, endpoint: endpoint}).info("VU initialized");
|
||||||
|
vu_logged = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
return s3_client;
|
||||||
|
}
|
||||||
|
|
||||||
if (!!__ENV.METRIC_TAGS) {
|
if (!!__ENV.METRIC_TAGS) {
|
||||||
stats.setTags(__ENV.METRIC_TAGS)
|
stats.setTags(__ENV.METRIC_TAGS)
|
||||||
}
|
}
|
||||||
|
@ -101,6 +114,9 @@ export function obj_write_multipart() {
|
||||||
sleep(__ENV.SLEEP_WRITE);
|
sleep(__ENV.SLEEP_WRITE);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const s3_endpoint = get_endpoint();
|
||||||
|
const s3_client = get_client("WRITE", s3_endpoint)
|
||||||
|
|
||||||
const key = generateS3Key();
|
const key = generateS3Key();
|
||||||
const bucket = bucket_list[Math.floor(Math.random() * bucket_list.length)];
|
const bucket = bucket_list[Math.floor(Math.random() * bucket_list.length)];
|
||||||
|
|
||||||
|
@ -109,7 +125,7 @@ export function obj_write_multipart() {
|
||||||
bucket, key, write_multipart_part_size, write_multipart_vu_count,
|
bucket, key, write_multipart_part_size, write_multipart_vu_count,
|
||||||
payload);
|
payload);
|
||||||
if (!resp.success) {
|
if (!resp.success) {
|
||||||
log.withFields({bucket: bucket, key: key}).error(resp.error);
|
log.withFields({bucket: bucket, key: key, endpoint: s3_endpoint}).error(resp.error);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Add table
Reference in a new issue