2022-12-29 14:52:55 +00:00
|
|
|
import native from 'k6/x/frostfs/native';
|
|
|
|
import registry from 'k6/x/frostfs/registry';
|
|
|
|
import s3 from 'k6/x/frostfs/s3';
|
2023-05-23 08:35:30 +00:00
|
|
|
import logging from 'k6/x/frostfs/logging';
|
2022-09-02 19:23:33 +00:00
|
|
|
import { sleep } from 'k6';
|
2022-09-21 13:22:33 +00:00
|
|
|
import { Counter } from 'k6/metrics';
|
2023-03-01 08:32:13 +00:00
|
|
|
import { textSummary } from './libs/k6-summary-0.0.2.js';
|
2023-03-14 15:04:37 +00:00
|
|
|
import { parseEnv } from './libs/env-parser.js';
|
|
|
|
|
|
|
|
parseEnv();
|
2022-09-02 19:23:33 +00:00
|
|
|
|
2022-09-21 05:45:23 +00:00
|
|
|
const obj_registry = registry.open(__ENV.REGISTRY_FILE);
|
|
|
|
|
2022-09-02 19:23:33 +00:00
|
|
|
// Time limit (in seconds) for the run
|
|
|
|
const time_limit = __ENV.TIME_LIMIT || "60";
|
2023-03-01 08:32:13 +00:00
|
|
|
const summary_json = __ENV.SUMMARY_JSON || "/tmp/summary.json";
|
2022-09-02 19:23:33 +00:00
|
|
|
|
2022-09-27 16:01:43 +00:00
|
|
|
// Number of objects in each status. These counters are cumulative in a
|
|
|
|
// sense that they reflect total number of objects in the registry, not just
|
|
|
|
// number of objects that were processed by specific run of this scenario.
|
|
|
|
// This allows to run this scenario multiple times and collect overall
|
|
|
|
// statistics in the final run.
|
2022-09-21 13:22:33 +00:00
|
|
|
const obj_counters = {
|
|
|
|
verified: new Counter('verified_obj'),
|
|
|
|
skipped: new Counter('skipped_obj'),
|
|
|
|
invalid: new Counter('invalid_obj'),
|
|
|
|
};
|
|
|
|
|
2023-05-23 08:35:30 +00:00
|
|
|
let log = logging.new();
|
|
|
|
|
2022-09-02 19:23:33 +00:00
|
|
|
// Connect to random gRPC endpoint
|
|
|
|
let grpc_client = undefined;
|
|
|
|
if (__ENV.GRPC_ENDPOINTS) {
|
|
|
|
const grpcEndpoints = __ENV.GRPC_ENDPOINTS.split(',');
|
|
|
|
const grpcEndpoint = grpcEndpoints[Math.floor(Math.random() * grpcEndpoints.length)];
|
2023-05-23 08:35:30 +00:00
|
|
|
log = log.withField("endpoint", grpcEndpoint);
|
2023-07-07 10:16:54 +00:00
|
|
|
grpc_client = native.connect(grpcEndpoint, '',
|
|
|
|
__ENV.DIAL_TIMEOUT ? parseInt(__ENV.DIAL_TIMEOUT) : 0,
|
|
|
|
__ENV.STREAM_TIMEOUT ? parseInt(__ENV.STREAM_TIMEOUT) : 0,
|
|
|
|
__ENV.PREPARE_LOCALLY ? __ENV.PREPARE_LOCALLY.toLowerCase() === "true" : false, '');
|
2022-09-02 19:23:33 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Connect to random S3 endpoint
|
|
|
|
let s3_client = undefined;
|
|
|
|
if (__ENV.S3_ENDPOINTS) {
|
2023-07-13 12:50:03 +00:00
|
|
|
const no_verify_ssl = __ENV.NO_VERIFY_SSL || "true";
|
|
|
|
const connection_args = {no_verify_ssl: no_verify_ssl}
|
2022-09-02 19:23:33 +00:00
|
|
|
const s3_endpoints = __ENV.S3_ENDPOINTS.split(',');
|
|
|
|
const s3_endpoint = s3_endpoints[Math.floor(Math.random() * s3_endpoints.length)];
|
2023-05-23 08:35:30 +00:00
|
|
|
log = log.withField("endpoint", s3_endpoint);
|
2023-07-13 12:50:03 +00:00
|
|
|
s3_client = s3.connect(s3_endpoint, connection_args);
|
2022-09-02 19:23:33 +00:00
|
|
|
}
|
|
|
|
|
2022-09-21 13:22:33 +00:00
|
|
|
// We will attempt to verify every object in "created" status. The scenario will execute
|
2022-09-26 18:05:28 +00:00
|
|
|
// as many iterations as there are objects. Each object will have 3 retries to be verified
|
|
|
|
const obj_to_verify_selector = registry.getSelector(
|
|
|
|
__ENV.REGISTRY_FILE,
|
|
|
|
"obj_to_verify",
|
2022-11-10 06:23:33 +00:00
|
|
|
__ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0,
|
2022-09-26 18:05:28 +00:00
|
|
|
{
|
|
|
|
status: "created",
|
|
|
|
}
|
|
|
|
);
|
|
|
|
const obj_to_verify_count = obj_to_verify_selector.count();
|
|
|
|
// Execute at least one iteration (executor shared-iterations can't run 0 iterations)
|
|
|
|
const iterations = Math.max(1, obj_to_verify_count);
|
2022-09-22 16:57:21 +00:00
|
|
|
// Executor shared-iterations requires number of iterations to be larger than number of VUs
|
|
|
|
const vus = Math.min(__ENV.CLIENTS, iterations);
|
|
|
|
|
2022-09-02 19:23:33 +00:00
|
|
|
const scenarios = {
|
|
|
|
verify: {
|
2022-09-21 13:22:33 +00:00
|
|
|
executor: 'shared-iterations',
|
2022-09-22 16:57:21 +00:00
|
|
|
vus,
|
|
|
|
iterations,
|
2022-09-21 13:22:33 +00:00
|
|
|
maxDuration: `${time_limit}s`,
|
2022-09-02 19:23:33 +00:00
|
|
|
exec: 'obj_verify',
|
|
|
|
gracefulStop: '5s',
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
export const options = {
|
2022-09-22 16:57:21 +00:00
|
|
|
scenarios,
|
2022-09-02 19:23:33 +00:00
|
|
|
setupTimeout: '5s',
|
|
|
|
};
|
|
|
|
|
2022-09-21 13:22:33 +00:00
|
|
|
export function setup() {
|
|
|
|
// Populate counters with initial values
|
|
|
|
for (const [status, counter] of Object.entries(obj_counters)) {
|
2022-11-10 06:23:33 +00:00
|
|
|
const obj_selector = registry.getSelector(
|
|
|
|
__ENV.REGISTRY_FILE,
|
|
|
|
status,
|
|
|
|
__ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0,
|
|
|
|
{ status });
|
2022-09-26 18:05:28 +00:00
|
|
|
counter.add(obj_selector.count());
|
2022-09-21 13:22:33 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-03-01 08:32:13 +00:00
|
|
|
export function handleSummary(data) {
|
|
|
|
return {
|
|
|
|
'stdout': textSummary(data, { indent: ' ', enableColors: false }),
|
|
|
|
[summary_json]: JSON.stringify(data),
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
2022-09-02 19:23:33 +00:00
|
|
|
export function obj_verify() {
|
2023-05-23 12:57:43 +00:00
|
|
|
if (obj_to_verify_count == 0) {
|
|
|
|
log.info("Nothing to verify");
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2022-09-02 19:23:33 +00:00
|
|
|
if (__ENV.SLEEP) {
|
|
|
|
sleep(__ENV.SLEEP);
|
|
|
|
}
|
|
|
|
|
2023-07-19 14:23:38 +00:00
|
|
|
const obj = obj_to_verify_selector.nextObject(5);
|
2022-09-02 19:23:33 +00:00
|
|
|
if (!obj) {
|
2023-05-23 08:35:30 +00:00
|
|
|
log.info("All objects have been verified");
|
2022-09-21 13:22:33 +00:00
|
|
|
return;
|
2022-09-02 19:23:33 +00:00
|
|
|
}
|
|
|
|
|
2022-09-21 13:22:33 +00:00
|
|
|
const obj_status = verify_object_with_retries(obj, 3);
|
|
|
|
obj_counters[obj_status].add(1);
|
2023-05-18 06:42:35 +00:00
|
|
|
obj_registry.setObjectStatus(obj.id, obj.status, obj_status);
|
2022-09-21 13:22:33 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
function verify_object_with_retries(obj, attempts) {
|
|
|
|
for (let i = 0; i < attempts; i++) {
|
|
|
|
let result;
|
2023-05-23 08:35:30 +00:00
|
|
|
// Different name is required.
|
|
|
|
// ReferenceError: Cannot access a variable before initialization.
|
|
|
|
let lg = log;
|
2022-09-21 13:22:33 +00:00
|
|
|
if (obj.c_id && obj.o_id) {
|
2023-05-23 08:35:30 +00:00
|
|
|
lg = lg.withFields({cid: obj.c_id, oid: obj.o_id});
|
2022-09-21 13:22:33 +00:00
|
|
|
result = grpc_client.verifyHash(obj.c_id, obj.o_id, obj.payload_hash);
|
|
|
|
} else if (obj.s3_bucket && obj.s3_key) {
|
2023-05-23 08:35:30 +00:00
|
|
|
lg = lg.withFields({bucket: obj.s3_bucket, key: obj.s3_key});
|
2022-09-21 13:22:33 +00:00
|
|
|
result = s3_client.verifyHash(obj.s3_bucket, obj.s3_key, obj.payload_hash);
|
|
|
|
} else {
|
2023-05-23 08:35:30 +00:00
|
|
|
lg.withFields({
|
|
|
|
cid: obj.c_id,
|
|
|
|
oid: obj.o_id,
|
|
|
|
bucket: obj.s3_bucket,
|
|
|
|
key: obj.s3_key
|
|
|
|
}).warn(`Object cannot be verified with supported protocols`);
|
2022-09-21 13:22:33 +00:00
|
|
|
return "skipped";
|
|
|
|
}
|
2022-09-02 19:23:33 +00:00
|
|
|
|
2022-09-21 13:22:33 +00:00
|
|
|
if (result.success) {
|
|
|
|
return "verified";
|
|
|
|
} else if (result.error == "hash mismatch") {
|
|
|
|
return "invalid";
|
|
|
|
}
|
|
|
|
|
|
|
|
// Unless we explicitly saw that there was a hash mismatch, then we will retry after a delay
|
2023-05-23 08:35:30 +00:00
|
|
|
lg.error(`Verify error: ${result.error}. Object will be re-tried`);
|
2022-09-21 13:22:33 +00:00
|
|
|
sleep(__ENV.SLEEP);
|
2022-09-02 19:23:33 +00:00
|
|
|
}
|
2022-09-21 13:22:33 +00:00
|
|
|
|
|
|
|
return "invalid";
|
2022-09-02 19:23:33 +00:00
|
|
|
}
|