Use separate sleep intervals for VUs

To fine-tune read/write/delete load we need to have separate sleep
intervales for readers/writers/deleters.

The changes were originally authored by anatoly@nspcc.ru <anatoly@nspcc.ru>

Signed-off-by: Vladimir Domnich <v.domnich@yadro.com>
This commit is contained in:
anatoly@nspcc.ru 2022-08-11 16:56:39 +03:00 committed by Anatoly Bogatyrev
parent 7875aa9c02
commit ea604ee8ee
5 changed files with 34 additions and 30 deletions

View file

@ -99,8 +99,8 @@ export function teardown(data) {
}
export function obj_write() {
if (__ENV.SLEEP) {
sleep(__ENV.SLEEP);
if (__ENV.SLEEP_WRITE) {
sleep(__ENV.SLEEP_WRITE);
}
const headers = {
@ -121,8 +121,8 @@ export function obj_write() {
}
export function obj_read() {
if (__ENV.SLEEP) {
sleep(__ENV.SLEEP);
if (__ENV.SLEEP_READ) {
sleep(__ENV.SLEEP_READ);
}
const obj = obj_list[Math.floor(Math.random() * obj_list.length)];
@ -133,8 +133,8 @@ export function obj_read() {
}
export function obj_delete() {
if (__ENV.SLEEP) {
sleep(__ENV.SLEEP);
if (__ENV.SLEEP_DELETE) {
sleep(__ENV.SLEEP_DELETE);
}
const obj = obj_to_delete_selector.nextObject();

View file

@ -72,8 +72,8 @@ export function teardown(data) {
}
export function obj_write() {
if (__ENV.SLEEP) {
sleep(__ENV.SLEEP);
if (__ENV.SLEEP_WRITE) {
sleep(__ENV.SLEEP_WRITE);
}
const container = container_list[Math.floor(Math.random() * container_list.length)];
@ -96,8 +96,8 @@ export function obj_write() {
}
export function obj_read() {
if (__ENV.SLEEP) {
sleep(__ENV.SLEEP);
if (__ENV.SLEEP_READ) {
sleep(__ENV.SLEEP_READ);
}
const obj = obj_list[Math.floor(Math.random() * obj_list.length)];

View file

@ -1,25 +1,26 @@
#!/usr/bin/python3
from multiprocessing import Process
import uuid
import shlex
from subprocess import check_output, CalledProcessError, STDOUT
import argparse
import json
import os
import argparse, sys
import os
import shlex
from concurrent.futures import ProcessPoolExecutor
from subprocess import check_output, CalledProcessError, STDOUT
parser=argparse.ArgumentParser()
parser = argparse.ArgumentParser()
parser.add_argument('--size', help='Upload objects size in kb')
parser.add_argument('--containers', help='Number of containers to create')
parser.add_argument('--out', help='JSON file with output')
parser.add_argument('--preload_obj', help='Number of pre-loaded objects')
parser.add_argument(
"--policy",
help="Container placement policy",
default="REP 1 IN X CBF 1 SELECT 1 FROM * AS X"
)
parser.add_argument('--endpoint', help='Node address')
parser.add_argument('--update', help='Save existed containers')
args=parser.parse_args()
args = parser.parse_args()
print(args)
@ -33,7 +34,6 @@ def main():
with open(args.out) as f:
data_json = json.load(f)
container_list = data_json['containers']
# Get CID list
else:
print(f"Create containers: {args.containers}")
with ProcessPoolExecutor(max_workers=10) as executor:
@ -78,6 +78,7 @@ def random_payload(payload_filepath):
with open('%s'%payload_filepath, 'wb') as fout:
fout.write(os.urandom(1024*int(args.size)))
def execute_cmd(cmd_line):
args = shlex.split(cmd_line)
output = ""
@ -93,11 +94,11 @@ def execute_cmd(cmd_line):
def create_container():
cmd_line = f"neofs-cli --rpc-endpoint {args.endpoint} container create -g --policy 'REP 1 IN X CBF 1 SELECT 1 FROM * AS X' --basic-acl public-read-write --await"
cmd_line = f"neofs-cli --rpc-endpoint {args.endpoint} container create -g --policy {args.policy} --basic-acl public-read-write --await"
output, success = execute_cmd(cmd_line)
if not success:
print(f" > Container has not been created.")
print(f" > Container has not been created.")
else:
try:
fst_str = output.split('\n')[0]

View file

@ -11,7 +11,8 @@ Scenarios `grpc.js`, `http.js` and `s3.js` support the following options:
* `REGISTRY_FILE` - if set, all produced objects will be stored in database for subsequent verification. Database file name will be set to the value of `REGISTRY_FILE`.
* `WRITE_OBJ_SIZE` - object size in kb for write(PUT) operations.
* `PREGEN_JSON` - path to json file with pre-generated containers and objects (in case of http scenario we use json pre-generated for grpc scenario).
* `SLEEP` - time interval (in seconds) between VU iterations.
* `SLEEP_WRITE` - time interval (in seconds) between writing VU iterations.
* `SLEEP_READ` - time interval (in seconds) between reading VU iterations.
Examples of how to use these options are provided below for each scenario.
@ -35,6 +36,7 @@ Options (in addition to the common options):
* `GRPC_ENDPOINTS` - GRPC endpoints of neoFS in format `host:port`. To specify multiple endpoints separate them by comma.
* `DELETERS` - number of VUs performing delete operations (using deleters requires that options `DELETE_AGE` and `REGISTRY_FILE` are specified as well).
* `DELETE_AGE` - age of object in seconds before which it can not be deleted. This parameter can be used to control how many objects we have in the system under load.
* `SLEEP_DELETE` - time interval (in seconds) between deleting VU iterations.
## HTTP
@ -90,6 +92,7 @@ Options (in addition to the common options):
* `S3_ENDPOINTS` - endpoints of S3 gateways in format `host:port`. To specify multiple endpoints separate them by comma.
* `DELETERS` - number of VUs performing delete operations (using deleters requires that options `DELETE_AGE` and `REGISTRY_FILE` are specified as well).
* `DELETE_AGE` - age of object in seconds before which it can not be deleted. This parameter can be used to control how many objects we have in the system under load.
* `SLEEP_DELETE` - time interval (in seconds) between deleting VU iterations.
* `OBJ_NAME` - if specified, this name will be used for all write operations instead of random generation.
## Verify

View file

@ -98,8 +98,8 @@ export function teardown(data) {
}
export function obj_write() {
if (__ENV.SLEEP) {
sleep(__ENV.SLEEP);
if (__ENV.SLEEP_WRITE) {
sleep(__ENV.SLEEP_WRITE);
}
const key = __ENV.OBJ_NAME || uuidv4();
@ -118,8 +118,8 @@ export function obj_write() {
}
export function obj_read() {
if (__ENV.SLEEP) {
sleep(__ENV.SLEEP);
if (__ENV.SLEEP_READ) {
sleep(__ENV.SLEEP_READ);
}
const obj = obj_list[Math.floor(Math.random() * obj_list.length)];
@ -131,8 +131,8 @@ export function obj_read() {
}
export function obj_delete() {
if (__ENV.SLEEP) {
sleep(__ENV.SLEEP);
if (__ENV.SLEEP_DELETE) {
sleep(__ENV.SLEEP_DELETE);
}
const obj = obj_to_delete_selector.nextObject();