TrueCloudLab/xk6-frostfs
42
1
Fork 20
Code Issues 34 Pull requests 3 Releases Activity Actions

Allow to specify max_obj_size #145

Merged
fyrchik merged 2 commits from dstepanov-yadro/xk6-frostfs:feat/max_obj_size into master 2024-09-04 19:51:18 +00:00
Conversation 0 Commits 2 Files changed 8 +148 -127
8 changed files with 148 additions and 127 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
README.md
View file

@ -48,10 +48,11 @@ Create native client with `connect` method. Arguments:
- dial timeout in seconds (0 for the default value) - dial timeout in seconds (0 for the default value)
- stream timeout in seconds (0 for the default value) - stream timeout in seconds (0 for the default value)
- generate object header on the client side (for big object - split locally too) - generate object header on the client side (for big object - split locally too)
- max size for generated object header on the client side (for big object - the size that the object is splitted into)
```js ```js
import native from 'k6/x/frostfs/native'; import native from 'k6/x/frostfs/native';
const frostfs_cli = native.connect("s01.frostfs.devenv:8080", "", 0, 0, false) const frostfs_cli = native.connect("s01.frostfs.devenv:8080", "", 0, 0, false, 0)
``` ```
### Methods ### Methods

6
examples/native.js
View file

@ -3,11 +3,11 @@ import { fail } from "k6";
import { uuidv4 } from '../scenarios/libs/k6-utils-1.4.0.js'; import { uuidv4 } from '../scenarios/libs/k6-utils-1.4.0.js';
const payload = open('../go.sum', 'b'); const payload = open('../go.sum', 'b');
const frostfs_cli = native.connect("s01.frostfs.devenv:8080", "1dd37fba80fec4e6a6f13fd708d8dcb3b29def768017052f6c930fa1c5d90bbb", 0, 0, false) const frostfs_cli = native.connect("s01.frostfs.devenv:8080", "1dd37fba80fec4e6a6f13fd708d8dcb3b29def768017052f6c930fa1c5d90bbb", 0, 0, false, 0)
export const options = { export const options = {
stages: [ stages: [
{duration: '30s', target: 10}, { duration: '30s', target: 10 },
], ],
}; };
@ -24,7 +24,7 @@ export function setup() {
fail(res.error) fail(res.error)
} }
console.info("created container", res.container_id) console.info("created container", res.container_id)
return {container_id: res.container_id} return { container_id: res.container_id }
} }
export default function (data) { export default function (data) {

6
examples/native_onsite.js
View file

@ -3,7 +3,7 @@ import { uuidv4 } from '../scenarios/libs/k6-utils-1.4.0.js';
const payload = open('../go.sum', 'b'); const payload = open('../go.sum', 'b');
const container = "AjSxSNNXbJUDPqqKYm1VbFVDGCakbpUNH8aGjPmGAH3B" const container = "AjSxSNNXbJUDPqqKYm1VbFVDGCakbpUNH8aGjPmGAH3B"
const frostfs_cli = native.connect("s01.frostfs.devenv:8080", "", 0, 0, false) const frostfs_cli = native.connect("s01.frostfs.devenv:8080", "", 0, 0, false, 0)
const frostfs_obj = frostfs_cli.onsite(container, payload) const frostfs_obj = frostfs_cli.onsite(container, payload)
export const options = { export const options = {
@ -14,11 +14,11 @@ export const options = {
export default function () { export default function () {
let headers = { let headers = {
'unique_header': uuidv4() 'unique_header': uuidv4()
} }
let resp = frostfs_obj.put(headers) let resp = frostfs_obj.put(headers)
if (resp.success) { if (resp.success) {
frostfs_cli.get(container, resp.object_id) frostfs_cli.get(container, resp.object_id)
} else { } else {
console.log(resp.error) console.log(resp.error)
} }

12
internal/native/client.go
View file

@ -35,6 +35,7 @@ type (
tok session.Object tok session.Object
cli *client.Client cli *client.Client
prepareLocally bool prepareLocally bool
maxObjSize uint64
} }
PutResponse struct { PutResponse struct {
@ -71,6 +72,7 @@ type (
hdr object.Object hdr object.Object
payload []byte payload []byte
prepareLocally bool prepareLocally bool
maxObjSize uint64
} }
) )
@ -103,7 +105,7 @@ func (c *Client) Put(containerID string, headers map[string]string, payload data
o.SetOwnerID(owner) o.SetOwnerID(owner)
o.SetAttributes(attrs...) o.SetAttributes(attrs...)
resp, err := put(c.vu, c.cli, c.prepareLocally, &tok, &o, payload, chunkSize) resp, err := put(c.vu, c.cli, c.prepareLocally, &tok, &o, payload, chunkSize, c.maxObjSize)
if err != nil { if err != nil {
return PutResponse{Success: false, Error: err.Error()} return PutResponse{Success: false, Error: err.Error()}
} }
@ -373,6 +375,7 @@ func (c *Client) Onsite(containerID string, payload datagen.Payload) PreparedObj
hdr: *obj, hdr: *obj,
payload: data, payload: data,
prepareLocally: c.prepareLocally, prepareLocally: c.prepareLocally,
maxObjSize: c.maxObjSize,
} }
} }
@ -398,7 +401,7 @@ func (p PreparedObject) Put(headers map[string]string) PutResponse {
return PutResponse{Success: false, Error: err.Error()} return PutResponse{Success: false, Error: err.Error()}
} }
_, err = put(p.vu, p.cli, p.prepareLocally, nil, &obj, datagen.NewFixedPayload(p.payload), 0) _, err = put(p.vu, p.cli, p.prepareLocally, nil, &obj, datagen.NewFixedPayload(p.payload), 0, p.maxObjSize)
if err != nil { if err != nil {
return PutResponse{Success: false, Error: err.Error()} return PutResponse{Success: false, Error: err.Error()}
} }
@ -413,7 +416,7 @@ func (s epochSource) CurrentEpoch() uint64 {
} }
func put(vu modules.VU, cli *client.Client, prepareLocally bool, tok *session.Object, func put(vu modules.VU, cli *client.Client, prepareLocally bool, tok *session.Object,
hdr *object.Object, payload datagen.Payload, chunkSize int, hdr *object.Object, payload datagen.Payload, chunkSize int, maxObjSize uint64,
) (*client.ResObjectPut, error) { ) (*client.ResObjectPut, error) {
bufSize := defaultBufferSize bufSize := defaultBufferSize
if chunkSize > 0 { if chunkSize > 0 {
@ -441,6 +444,9 @@ func put(vu modules.VU, cli *client.Client, prepareLocally bool, tok *session.Ob
prm.MaxSize = res.Info().MaxObjectSize() prm.MaxSize = res.Info().MaxObjectSize()
prm.EpochSource = epochSource(res.Info().CurrentEpoch()) prm.EpochSource = epochSource(res.Info().CurrentEpoch())
prm.WithoutHomomorphHash = true prm.WithoutHomomorphHash = true
if maxObjSize > 0 {
prm.MaxSize = maxObjSize
}
} }
objectWriter, err := cli.ObjectPutInit(vu.Context(), prm) objectWriter, err := cli.ObjectPutInit(vu.Context(), prm)

17
internal/native/native.go
View file

@ -52,13 +52,17 @@ func (n *Native) Exports() modules.Exports {
return modules.Exports{Default: n} return modules.Exports{Default: n}
} }
func (n *Native) Connect(endpoint, hexPrivateKey string, dialTimeout, streamTimeout int, prepareLocally bool) (*Client, error) { func (n *Native) Connect(endpoint, hexPrivateKey string, dialTimeout, streamTimeout int, prepareLocally bool, maxObjSize int) (*Client, error) {
var ( var (
cli client.Client cli client.Client
pk *keys.PrivateKey pk *keys.PrivateKey
err error err error
) )
if maxObjSize < 0 {
return nil, fmt.Errorf("max object size value must be positive")
}
pk, err = keys.NewPrivateKey() pk, err = keys.NewPrivateKey()
if len(hexPrivateKey) != 0 { if len(hexPrivateKey) != 0 {
pk, err = keys.NewPrivateKeyFromHex(hexPrivateKey) pk, err = keys.NewPrivateKeyFromHex(hexPrivateKey)
@ -114,6 +118,16 @@ func (n *Native) Connect(endpoint, hexPrivateKey string, dialTimeout, streamTime
tok.SetAuthKey(&key) tok.SetAuthKey(&key)
tok.SetExp(exp) tok.SetExp(exp)
if prepareLocally && maxObjSize > 0 {
res, err := cli.NetworkInfo(n.vu.Context(), client.PrmNetworkInfo{})
if err != nil {
return nil, err
}
if uint64(maxObjSize) > res.Info().MaxObjectSize() {
fyrchik marked this conversation as resolved
fyrchik commented 2024-06-14 08:16:14 +00:00
Review
Copy link

We fetch network info in put(), why not validate there?
We can use the same client and multiple sizes in the same scenario, so maxObjSize is unrelated to Connect

We fetch network info in `put()`, why not validate there? We can use the same client and multiple sizes in the same scenario, so `maxObjSize` is unrelated to `Connect`
return nil, fmt.Errorf("max object size must be not greater than %d bytes", res.Info().MaxObjectSize())
}
}
// register metrics // register metrics
objPutSuccess, _ = stats.Registry.NewMetric("frostfs_obj_put_success", metrics.Counter) objPutSuccess, _ = stats.Registry.NewMetric("frostfs_obj_put_success", metrics.Counter)
@ -140,5 +154,6 @@ func (n *Native) Connect(endpoint, hexPrivateKey string, dialTimeout, streamTime
tok: tok, tok: tok,
cli: &cli, cli: &cli,
prepareLocally: prepareLocally, prepareLocally: prepareLocally,
maxObjSize: uint64(maxObjSize),
}, nil }, nil
} }

108
scenarios/grpc.js
View file

@ -1,25 +1,25 @@
import {sleep} from 'k6'; import { sleep } from 'k6';
import {SharedArray} from 'k6/data'; import { SharedArray } from 'k6/data';
import exec from 'k6/execution'; import exec from 'k6/execution';
import logging from 'k6/x/frostfs/logging'; import logging from 'k6/x/frostfs/logging';
import native from 'k6/x/frostfs/native'; import native from 'k6/x/frostfs/native';
import registry from 'k6/x/frostfs/registry'; import registry from 'k6/x/frostfs/registry';
import stats from 'k6/x/frostfs/stats'; import stats from 'k6/x/frostfs/stats';
import {newGenerator} from './libs/datagen.js'; import { newGenerator } from './libs/datagen.js';
import {parseEnv} from './libs/env-parser.js'; import { parseEnv } from './libs/env-parser.js';
import {textSummary} from './libs/k6-summary-0.0.2.js'; import { textSummary } from './libs/k6-summary-0.0.2.js';
import {uuidv4} from './libs/k6-utils-1.4.0.js'; import { uuidv4 } from './libs/k6-utils-1.4.0.js';
parseEnv(); parseEnv();
const obj_list = new SharedArray( const obj_list = new SharedArray(
'obj_list', 'obj_list',
function() { return JSON.parse(open(__ENV.PREGEN_JSON)).objects; }); function () { return JSON.parse(open(__ENV.PREGEN_JSON)).objects; });
const container_list = new SharedArray( const container_list = new SharedArray(
'container_list', 'container_list',
function() { return JSON.parse(open(__ENV.PREGEN_JSON)).containers; }); function () { return JSON.parse(open(__ENV.PREGEN_JSON)).containers; });
const read_size = JSON.parse(open(__ENV.PREGEN_JSON)).obj_size; const read_size = JSON.parse(open(__ENV.PREGEN_JSON)).obj_size;
const summary_json = __ENV.SUMMARY_JSON || '/tmp/summary.json'; const summary_json = __ENV.SUMMARY_JSON || '/tmp/summary.json';
@ -27,17 +27,17 @@ const summary_json = __ENV.SUMMARY_JSON || '/tmp/summary.json';
// Select random gRPC endpoint for current VU // Select random gRPC endpoint for current VU
const grpc_endpoints = __ENV.GRPC_ENDPOINTS.split(','); const grpc_endpoints = __ENV.GRPC_ENDPOINTS.split(',');
const grpc_endpoint = const grpc_endpoint =
grpc_endpoints[Math.floor(Math.random() * grpc_endpoints.length)]; grpc_endpoints[Math.floor(Math.random() * grpc_endpoints.length)];
const grpc_client = native.connect( const grpc_client = native.connect(
grpc_endpoint, '', __ENV.DIAL_TIMEOUT ? parseInt(__ENV.DIAL_TIMEOUT) : 5, grpc_endpoint, '', __ENV.DIAL_TIMEOUT ? parseInt(__ENV.DIAL_TIMEOUT) : 5,
__ENV.STREAM_TIMEOUT ? parseInt(__ENV.STREAM_TIMEOUT) : 60, __ENV.STREAM_TIMEOUT ? parseInt(__ENV.STREAM_TIMEOUT) : 60,
__ENV.PREPARE_LOCALLY ? __ENV.PREPARE_LOCALLY.toLowerCase() === 'true' __ENV.PREPARE_LOCALLY ? __ENV.PREPARE_LOCALLY.toLowerCase() === 'true' : false,
: false); 1024 * parseInt(__ENV.MAX_OBJECT_SIZE || '0'));
const log = logging.new().withField('endpoint', grpc_endpoint); const log = logging.new().withField('endpoint', grpc_endpoint);
const registry_enabled = !!__ENV.REGISTRY_FILE; const registry_enabled = !!__ENV.REGISTRY_FILE;
const obj_registry = const obj_registry =
registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : undefined; registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : undefined;
const duration = __ENV.DURATION; const duration = __ENV.DURATION;
@ -49,11 +49,11 @@ const read_age = __ENV.READ_AGE ? parseInt(__ENV.READ_AGE) : 10;
let obj_to_read_selector = undefined; let obj_to_read_selector = undefined;
if (registry_enabled) { if (registry_enabled) {
obj_to_read_selector = registry.getLoopedSelector( obj_to_read_selector = registry.getLoopedSelector(
__ENV.REGISTRY_FILE, 'obj_to_read', __ENV.REGISTRY_FILE, 'obj_to_read',
__ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, { __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, {
status : 'created', status: 'created',
age : read_age, age: read_age,
}) })
} }
const scenarios = {}; const scenarios = {};
@ -63,11 +63,11 @@ const write_grpc_chunk_size = 1024 * parseInt(__ENV.GRPC_CHUNK_SIZE || '0')
const generator = newGenerator(write_vu_count > 0); const generator = newGenerator(write_vu_count > 0);
if (write_vu_count > 0) { if (write_vu_count > 0) {
scenarios.write = { scenarios.write = {
executor : 'constant-vus', executor: 'constant-vus',
vus : write_vu_count, vus: write_vu_count,
duration : `${duration}s`, duration: `${duration}s`,
exec : 'obj_write', exec: 'obj_write',
gracefulStop : '5s', gracefulStop: '5s',
}; };
} }
@ -78,24 +78,24 @@ if (registry_enabled && delete_age) {
obj_to_delete_exit_on_null = write_vu_count == 0; obj_to_delete_exit_on_null = write_vu_count == 0;
let constructor = obj_to_delete_exit_on_null ? registry.getOneshotSelector let constructor = obj_to_delete_exit_on_null ? registry.getOneshotSelector
: registry.getSelector; : registry.getSelector;
obj_to_delete_selector = obj_to_delete_selector =
constructor(__ENV.REGISTRY_FILE, 'obj_to_delete', constructor(__ENV.REGISTRY_FILE, 'obj_to_delete',
__ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, { __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, {
status : 'created', status: 'created',
age : delete_age, age: delete_age,
}); });
} }
const read_vu_count = parseInt(__ENV.READERS || '0'); const read_vu_count = parseInt(__ENV.READERS || '0');
if (read_vu_count > 0) { if (read_vu_count > 0) {
scenarios.read = { scenarios.read = {
executor : 'constant-vus', executor: 'constant-vus',
vus : read_vu_count, vus: read_vu_count,
duration : `${duration}s`, duration: `${duration}s`,
exec : 'obj_read', exec: 'obj_read',
gracefulStop : '5s', gracefulStop: '5s',
}; };
} }
@ -103,21 +103,21 @@ const delete_vu_count = parseInt(__ENV.DELETERS || '0');
if (delete_vu_count > 0) { if (delete_vu_count > 0) {
if (!obj_to_delete_selector) { if (!obj_to_delete_selector) {
throw new Error( throw new Error(
'Positive DELETE worker number without a proper object selector'); 'Positive DELETE worker number without a proper object selector');
} }
scenarios.delete = { scenarios.delete = {
executor : 'constant-vus', executor: 'constant-vus',
vus : delete_vu_count, vus: delete_vu_count,
duration : `${duration}s`, duration: `${duration}s`,
exec : 'obj_delete', exec: 'obj_delete',
gracefulStop : '5s', gracefulStop: '5s',
}; };
} }
export const options = { export const options = {
scenarios, scenarios,
setupTimeout : '5s', setupTimeout: '5s',
}; };
export function setup() { export function setup() {
@ -133,7 +133,7 @@ export function setup() {
const start_timestamp = Date.now() const start_timestamp = Date.now()
console.log( console.log(
`Load started at: ${Date(start_timestamp).toString()}`) `Load started at: ${Date(start_timestamp).toString()}`)
} }
export function teardown(data) { export function teardown(data) {
@ -142,13 +142,13 @@ export function teardown(data) {
} }
const end_timestamp = Date.now() const end_timestamp = Date.now()
console.log( console.log(
`Load finished at: ${Date(end_timestamp).toString()}`) `Load finished at: ${Date(end_timestamp).toString()}`)
} }
export function handleSummary(data) { export function handleSummary(data) {
return { return {
'stdout' : textSummary(data, {indent : ' ', enableColors : false}), 'stdout': textSummary(data, { indent: ' ', enableColors: false }),
[summary_json] : JSON.stringify(data), [summary_json]: JSON.stringify(data),
}; };
} }
@ -157,13 +157,13 @@ export function obj_write() {
sleep(__ENV.SLEEP_WRITE); sleep(__ENV.SLEEP_WRITE);
} }
const headers = {unique_header : uuidv4()}; const headers = { unique_header: uuidv4() };
const container = const container =
container_list[Math.floor(Math.random() * container_list.length)]; container_list[Math.floor(Math.random() * container_list.length)];
const payload = generator.genPayload(); const payload = generator.genPayload();
const resp = const resp =
grpc_client.put(container, headers, payload, write_grpc_chunk_size); grpc_client.put(container, headers, payload, write_grpc_chunk_size);
if (!resp.success) { if (!resp.success) {
log.withField('cid', container).error(resp.error); log.withField('cid', container).error(resp.error);
return; return;
@ -186,7 +186,7 @@ export function obj_read() {
} }
const resp = grpc_client.get(obj.c_id, obj.o_id) const resp = grpc_client.get(obj.c_id, obj.o_id)
if (!resp.success) { if (!resp.success) {
log.withFields({cid : obj.c_id, oid : obj.o_id}).error(resp.error); log.withFields({ cid: obj.c_id, oid: obj.o_id }).error(resp.error);
} }
return return
} }
@ -194,7 +194,7 @@ export function obj_read() {
const obj = obj_list[Math.floor(Math.random() * obj_list.length)]; const obj = obj_list[Math.floor(Math.random() * obj_list.length)];
const resp = grpc_client.get(obj.container, obj.object) const resp = grpc_client.get(obj.container, obj.object)
if (!resp.success) { if (!resp.success) {
log.withFields({cid : obj.container, oid : obj.object}).error(resp.error); log.withFields({ cid: obj.container, oid: obj.object }).error(resp.error);
} }
} }
@ -214,7 +214,7 @@ export function obj_delete() {
const resp = grpc_client.delete(obj.c_id, obj.o_id); const resp = grpc_client.delete(obj.c_id, obj.o_id);
if (!resp.success) { if (!resp.success) {
// Log errors except (2052 - object already deleted) // Log errors except (2052 - object already deleted)
log.withFields({cid : obj.c_id, oid : obj.o_id}).error(resp.error); log.withFields({ cid: obj.c_id, oid: obj.o_id }).error(resp.error);
return; return;
} }

70
scenarios/grpc_car.js
View file

@ -1,22 +1,22 @@
import {sleep} from 'k6'; import { sleep } from 'k6';
import {SharedArray} from 'k6/data'; import { SharedArray } from 'k6/data';
import logging from 'k6/x/frostfs/logging'; import logging from 'k6/x/frostfs/logging';
import native from 'k6/x/frostfs/native'; import native from 'k6/x/frostfs/native';
import registry from 'k6/x/frostfs/registry'; import registry from 'k6/x/frostfs/registry';
import stats from 'k6/x/frostfs/stats'; import stats from 'k6/x/frostfs/stats';
import {newGenerator} from './libs/datagen.js'; import { newGenerator } from './libs/datagen.js';
import {parseEnv} from './libs/env-parser.js'; import { parseEnv } from './libs/env-parser.js';
import {textSummary} from './libs/k6-summary-0.0.2.js'; import { textSummary } from './libs/k6-summary-0.0.2.js';
import {uuidv4} from './libs/k6-utils-1.4.0.js'; import { uuidv4 } from './libs/k6-utils-1.4.0.js';
parseEnv(); parseEnv();
const obj_list = new SharedArray('obj_list', function() { const obj_list = new SharedArray('obj_list', function () {
return JSON.parse(open(__ENV.PREGEN_JSON)).objects; return JSON.parse(open(__ENV.PREGEN_JSON)).objects;
}); });
const container_list = new SharedArray('container_list', function() { const container_list = new SharedArray('container_list', function () {
return JSON.parse(open(__ENV.PREGEN_JSON)).containers; return JSON.parse(open(__ENV.PREGEN_JSON)).containers;
}); });
@ -26,17 +26,17 @@ const summary_json = __ENV.SUMMARY_JSON || '/tmp/summary.json';
// Select random gRPC endpoint for current VU // Select random gRPC endpoint for current VU
const grpc_endpoints = __ENV.GRPC_ENDPOINTS.split(','); const grpc_endpoints = __ENV.GRPC_ENDPOINTS.split(',');
const grpc_endpoint = const grpc_endpoint =
grpc_endpoints[Math.floor(Math.random() * grpc_endpoints.length)]; grpc_endpoints[Math.floor(Math.random() * grpc_endpoints.length)];
const grpc_client = native.connect( const grpc_client = native.connect(
grpc_endpoint, '', __ENV.DIAL_TIMEOUT ? parseInt(__ENV.DIAL_TIMEOUT) : 5, grpc_endpoint, '', __ENV.DIAL_TIMEOUT ? parseInt(__ENV.DIAL_TIMEOUT) : 5,
__ENV.STREAM_TIMEOUT ? parseInt(__ENV.STREAM_TIMEOUT) : 60, __ENV.STREAM_TIMEOUT ? parseInt(__ENV.STREAM_TIMEOUT) : 60,
__ENV.PREPARE_LOCALLY ? __ENV.PREPARE_LOCALLY.toLowerCase() === 'true' : __ENV.PREPARE_LOCALLY ? __ENV.PREPARE_LOCALLY.toLowerCase() === 'true' : false,
false); 1024 * parseInt(__ENV.MAX_OBJECT_SIZE || '0'));
const log = logging.new().withField('endpoint', grpc_endpoint); const log = logging.new().withField('endpoint', grpc_endpoint);
const registry_enabled = !!__ENV.REGISTRY_FILE; const registry_enabled = !!__ENV.REGISTRY_FILE;
const obj_registry = const obj_registry =
registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : undefined; registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : undefined;
const duration = __ENV.DURATION; const duration = __ENV.DURATION;
@ -48,22 +48,22 @@ const delete_age = __ENV.DELETE_AGE ? parseInt(__ENV.DELETE_AGE) : undefined;
let obj_to_delete_selector = undefined; let obj_to_delete_selector = undefined;
if (registry_enabled && delete_age) { if (registry_enabled && delete_age) {
obj_to_delete_selector = registry.getSelector( obj_to_delete_selector = registry.getSelector(
__ENV.REGISTRY_FILE, 'obj_to_delete', __ENV.REGISTRY_FILE, 'obj_to_delete',
__ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, { __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, {
status: 'created', status: 'created',
age: delete_age, age: delete_age,
}); });
} }
const read_age = __ENV.READ_AGE ? parseInt(__ENV.READ_AGE) : 10; const read_age = __ENV.READ_AGE ? parseInt(__ENV.READ_AGE) : 10;
let obj_to_read_selector = undefined; let obj_to_read_selector = undefined;
if (registry_enabled) { if (registry_enabled) {
obj_to_read_selector = registry.getLoopedSelector( obj_to_read_selector = registry.getLoopedSelector(
__ENV.REGISTRY_FILE, 'obj_to_read', __ENV.REGISTRY_FILE, 'obj_to_read',
__ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, { __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, {
status: 'created', status: 'created',
age: read_age, age: read_age,
}) })
} }
const scenarios = {}; const scenarios = {};
@ -109,7 +109,7 @@ const delete_rate = parseInt(__ENV.DELETE_RATE || '0');
if (delete_rate > 0) { if (delete_rate > 0) {
if (!obj_to_delete_selector) { if (!obj_to_delete_selector) {
throw new Error( throw new Error(
'Positive DELETE worker number without a proper object selector'); 'Positive DELETE worker number without a proper object selector');
} }
scenarios.delete = { scenarios.delete = {
@ -131,7 +131,7 @@ export const options = {
export function setup() { export function setup() {
const total_pre_allocated_vu_count = const total_pre_allocated_vu_count =
pre_alloc_write_vus + pre_alloc_read_vus + pre_alloc_delete_vus; pre_alloc_write_vus + pre_alloc_read_vus + pre_alloc_delete_vus;
const total_max_vu_count = max_read_vus + max_write_vus + max_delete_vus const total_max_vu_count = max_read_vus + max_write_vus + max_delete_vus
console.log(`Pregenerated containers: ${container_list.length}`); console.log(`Pregenerated containers: ${container_list.length}`);
@ -152,7 +152,7 @@ export function setup() {
const start_timestamp = Date.now() const start_timestamp = Date.now()
console.log( console.log(
`Load started at: ${Date(start_timestamp).toString()}`) `Load started at: ${Date(start_timestamp).toString()}`)
} }
export function teardown(data) { export function teardown(data) {
@ -161,12 +161,12 @@ export function teardown(data) {
} }
const end_timestamp = Date.now() const end_timestamp = Date.now()
console.log( console.log(
`Load finished at: ${Date(end_timestamp).toString()}`) `Load finished at: ${Date(end_timestamp).toString()}`)
} }
export function handleSummary(data) { export function handleSummary(data) {
return { return {
'stdout': textSummary(data, {indent: ' ', enableColors: false}), 'stdout': textSummary(data, { indent: ' ', enableColors: false }),
[summary_json]: JSON.stringify(data), [summary_json]: JSON.stringify(data),
}; };
} }
@ -176,13 +176,13 @@ export function obj_write() {
sleep(__ENV.SLEEP_WRITE); sleep(__ENV.SLEEP_WRITE);
} }
const headers = {unique_header: uuidv4()}; const headers = { unique_header: uuidv4() };
const container = const container =
container_list[Math.floor(Math.random() * container_list.length)]; container_list[Math.floor(Math.random() * container_list.length)];
const payload = generator.genPayload(); const payload = generator.genPayload();
const resp = const resp =
grpc_client.put(container, headers, payload, write_grpc_chunk_size); grpc_client.put(container, headers, payload, write_grpc_chunk_size);
if (!resp.success) { if (!resp.success) {
log.withField('cid', container).error(resp.error); log.withField('cid', container).error(resp.error);
return; return;
@ -205,7 +205,7 @@ export function obj_read() {
} }
const resp = grpc_client.get(obj.c_id, obj.o_id) const resp = grpc_client.get(obj.c_id, obj.o_id)
if (!resp.success) { if (!resp.success) {
log.withFields({cid: obj.c_id, oid: obj.o_id}).error(resp.error); log.withFields({ cid: obj.c_id, oid: obj.o_id }).error(resp.error);
} }
return return
} }
@ -213,7 +213,7 @@ export function obj_read() {
const obj = obj_list[Math.floor(Math.random() * obj_list.length)]; const obj = obj_list[Math.floor(Math.random() * obj_list.length)];
const resp = grpc_client.get(obj.container, obj.object) const resp = grpc_client.get(obj.container, obj.object)
if (!resp.success) { if (!resp.success) {
log.withFields({cid: obj.container, oid: obj.object}).error(resp.error); log.withFields({ cid: obj.container, oid: obj.object }).error(resp.error);
} }
} }
@ -230,7 +230,7 @@ export function obj_delete() {
const resp = grpc_client.delete(obj.c_id, obj.o_id); const resp = grpc_client.delete(obj.c_id, obj.o_id);
if (!resp.success) { if (!resp.success) {
// Log errors except (2052 - object already deleted) // Log errors except (2052 - object already deleted)
log.withFields({cid: obj.c_id, oid: obj.o_id}).error(resp.error); log.withFields({ cid: obj.c_id, oid: obj.o_id }).error(resp.error);
return; return;
} }

53
scenarios/verify.js
View file

@ -1,13 +1,13 @@
import {sleep} from 'k6'; import { sleep } from 'k6';
import {Counter} from 'k6/metrics'; import { Counter } from 'k6/metrics';
import logging from 'k6/x/frostfs/logging'; import logging from 'k6/x/frostfs/logging';
import native from 'k6/x/frostfs/native'; import native from 'k6/x/frostfs/native';
import registry from 'k6/x/frostfs/registry'; import registry from 'k6/x/frostfs/registry';
import s3 from 'k6/x/frostfs/s3'; import s3 from 'k6/x/frostfs/s3';
import stats from 'k6/x/frostfs/stats'; import stats from 'k6/x/frostfs/stats';
import {parseEnv} from './libs/env-parser.js'; import { parseEnv } from './libs/env-parser.js';
import {textSummary} from './libs/k6-summary-0.0.2.js'; import { textSummary } from './libs/k6-summary-0.0.2.js';
parseEnv(); parseEnv();
@ -39,24 +39,23 @@ let grpc_client = undefined;
if (__ENV.GRPC_ENDPOINTS) { if (__ENV.GRPC_ENDPOINTS) {
const grpcEndpoints = __ENV.GRPC_ENDPOINTS.split(','); const grpcEndpoints = __ENV.GRPC_ENDPOINTS.split(',');
const grpcEndpoint = const grpcEndpoint =
grpcEndpoints[Math.floor(Math.random() * grpcEndpoints.length)]; grpcEndpoints[Math.floor(Math.random() * grpcEndpoints.length)];
log = log.withField('endpoint', grpcEndpoint); log = log.withField('endpoint', grpcEndpoint);
grpc_client = native.connect( grpc_client = native.connect(
grpcEndpoint, '', __ENV.DIAL_TIMEOUT ? parseInt(__ENV.DIAL_TIMEOUT) : 0, grpcEndpoint, '', __ENV.DIAL_TIMEOUT ? parseInt(__ENV.DIAL_TIMEOUT) : 0,
__ENV.STREAM_TIMEOUT ? parseInt(__ENV.STREAM_TIMEOUT) : 0, __ENV.STREAM_TIMEOUT ? parseInt(__ENV.STREAM_TIMEOUT) : 0,
__ENV.PREPARE_LOCALLY ? __ENV.PREPARE_LOCALLY.toLowerCase() === 'true' : __ENV.PREPARE_LOCALLY ? __ENV.PREPARE_LOCALLY.toLowerCase() === 'true' : false,
false, 1024 * parseInt(__ENV.MAX_OBJECT_SIZE || '0'));
'');
} }
// Connect to random S3 endpoint // Connect to random S3 endpoint
let s3_client = undefined; let s3_client = undefined;
if (__ENV.S3_ENDPOINTS) { if (__ENV.S3_ENDPOINTS) {
const no_verify_ssl = __ENV.NO_VERIFY_SSL || 'true'; const no_verify_ssl = __ENV.NO_VERIFY_SSL || 'true';
const connection_args = {no_verify_ssl: no_verify_ssl}; const connection_args = { no_verify_ssl: no_verify_ssl };
const s3_endpoints = __ENV.S3_ENDPOINTS.split(','); const s3_endpoints = __ENV.S3_ENDPOINTS.split(',');
const s3_endpoint = const s3_endpoint =
s3_endpoints[Math.floor(Math.random() * s3_endpoints.length)]; s3_endpoints[Math.floor(Math.random() * s3_endpoints.length)];
log = log.withField('endpoint', s3_endpoint); log = log.withField('endpoint', s3_endpoint);
s3_client = s3.connect(s3_endpoint, connection_args); s3_client = s3.connect(s3_endpoint, connection_args);
} }
@ -65,10 +64,10 @@ if (__ENV.S3_ENDPOINTS) {
// execute as many iterations as there are objects. Each object will have 3 // execute as many iterations as there are objects. Each object will have 3
// retries to be verified // retries to be verified
const obj_to_verify_selector = registry.getSelector( const obj_to_verify_selector = registry.getSelector(
__ENV.REGISTRY_FILE, 'obj_to_verify', __ENV.REGISTRY_FILE, 'obj_to_verify',
__ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, { __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, {
status: 'created', status: 'created',
}); });
const obj_to_verify_count = obj_to_verify_selector.count(); const obj_to_verify_count = obj_to_verify_selector.count();
// Execute at least one iteration (executor shared-iterations can't run 0 // Execute at least one iteration (executor shared-iterations can't run 0
// iterations) // iterations)
@ -97,15 +96,15 @@ export function setup() {
// Populate counters with initial values // Populate counters with initial values
for (const [status, counter] of Object.entries(obj_counters)) { for (const [status, counter] of Object.entries(obj_counters)) {
const obj_selector = registry.getSelector( const obj_selector = registry.getSelector(
__ENV.REGISTRY_FILE, status, __ENV.REGISTRY_FILE, status,
__ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, {status}); __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, { status });
counter.add(obj_selector.count()); counter.add(obj_selector.count());
} }
} }
export function handleSummary(data) { export function handleSummary(data) {
return { return {
'stdout': textSummary(data, {indent: ' ', enableColors: false}), 'stdout': textSummary(data, { indent: ' ', enableColors: false }),
[summary_json]: JSON.stringify(data), [summary_json]: JSON.stringify(data),
}; };
} }
@ -138,19 +137,19 @@ function verify_object_with_retries(obj, attempts) {
// ReferenceError: Cannot access a variable before initialization. // ReferenceError: Cannot access a variable before initialization.
let lg = log; let lg = log;
if (obj.c_id && obj.o_id) { if (obj.c_id && obj.o_id) {
lg = lg.withFields({cid: obj.c_id, oid: obj.o_id}); lg = lg.withFields({ cid: obj.c_id, oid: obj.o_id });
result = grpc_client.verifyHash(obj.c_id, obj.o_id, obj.payload_hash); result = grpc_client.verifyHash(obj.c_id, obj.o_id, obj.payload_hash);
} else if (obj.s3_bucket && obj.s3_key) { } else if (obj.s3_bucket && obj.s3_key) {
lg = lg.withFields({bucket: obj.s3_bucket, key: obj.s3_key}); lg = lg.withFields({ bucket: obj.s3_bucket, key: obj.s3_key });
result = result =
s3_client.verifyHash(obj.s3_bucket, obj.s3_key, obj.payload_hash); s3_client.verifyHash(obj.s3_bucket, obj.s3_key, obj.payload_hash);
} else { } else {
lg.withFields({ lg.withFields({
cid: obj.c_id, cid: obj.c_id,
oid: obj.o_id, oid: obj.o_id,
bucket: obj.s3_bucket, bucket: obj.s3_bucket,
key: obj.s3_key key: obj.s3_key
}).warn(`Object cannot be verified with supported protocols`); }).warn(`Object cannot be verified with supported protocols`);
return 'skipped'; return 'skipped';
} }