Use separate sleep intervals for VUs

To fine-tune read/write/delete load we need to have separate sleep
intervales for readers/writers/deleters.

The changes were originally authored by anatoly@nspcc.ru <anatoly@nspcc.ru>

Signed-off-by: Vladimir Domnich <v.domnich@yadro.com>
This commit is contained in:
anatoly@nspcc.ru 2022-08-11 16:56:39 +03:00 committed by Anatoly Bogatyrev
parent 7875aa9c02
commit ea604ee8ee
5 changed files with 34 additions and 30 deletions

View file

@ -99,8 +99,8 @@ export function teardown(data) {
} }
export function obj_write() { export function obj_write() {
if (__ENV.SLEEP) { if (__ENV.SLEEP_WRITE) {
sleep(__ENV.SLEEP); sleep(__ENV.SLEEP_WRITE);
} }
const headers = { const headers = {
@ -121,8 +121,8 @@ export function obj_write() {
} }
export function obj_read() { export function obj_read() {
if (__ENV.SLEEP) { if (__ENV.SLEEP_READ) {
sleep(__ENV.SLEEP); sleep(__ENV.SLEEP_READ);
} }
const obj = obj_list[Math.floor(Math.random() * obj_list.length)]; const obj = obj_list[Math.floor(Math.random() * obj_list.length)];
@ -133,8 +133,8 @@ export function obj_read() {
} }
export function obj_delete() { export function obj_delete() {
if (__ENV.SLEEP) { if (__ENV.SLEEP_DELETE) {
sleep(__ENV.SLEEP); sleep(__ENV.SLEEP_DELETE);
} }
const obj = obj_to_delete_selector.nextObject(); const obj = obj_to_delete_selector.nextObject();

View file

@ -72,8 +72,8 @@ export function teardown(data) {
} }
export function obj_write() { export function obj_write() {
if (__ENV.SLEEP) { if (__ENV.SLEEP_WRITE) {
sleep(__ENV.SLEEP); sleep(__ENV.SLEEP_WRITE);
} }
const container = container_list[Math.floor(Math.random() * container_list.length)]; const container = container_list[Math.floor(Math.random() * container_list.length)];
@ -96,8 +96,8 @@ export function obj_write() {
} }
export function obj_read() { export function obj_read() {
if (__ENV.SLEEP) { if (__ENV.SLEEP_READ) {
sleep(__ENV.SLEEP); sleep(__ENV.SLEEP_READ);
} }
const obj = obj_list[Math.floor(Math.random() * obj_list.length)]; const obj = obj_list[Math.floor(Math.random() * obj_list.length)];

View file

@ -1,25 +1,26 @@
#!/usr/bin/python3 #!/usr/bin/python3
from multiprocessing import Process import argparse
import uuid
import shlex
from subprocess import check_output, CalledProcessError, STDOUT
import json import json
import os import os
import argparse, sys import shlex
from concurrent.futures import ProcessPoolExecutor from concurrent.futures import ProcessPoolExecutor
from subprocess import check_output, CalledProcessError, STDOUT
parser=argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument('--size', help='Upload objects size in kb') parser.add_argument('--size', help='Upload objects size in kb')
parser.add_argument('--containers', help='Number of containers to create') parser.add_argument('--containers', help='Number of containers to create')
parser.add_argument('--out', help='JSON file with output') parser.add_argument('--out', help='JSON file with output')
parser.add_argument('--preload_obj', help='Number of pre-loaded objects') parser.add_argument('--preload_obj', help='Number of pre-loaded objects')
parser.add_argument(
"--policy",
help="Container placement policy",
default="REP 1 IN X CBF 1 SELECT 1 FROM * AS X"
)
parser.add_argument('--endpoint', help='Node address') parser.add_argument('--endpoint', help='Node address')
parser.add_argument('--update', help='Save existed containers') parser.add_argument('--update', help='Save existed containers')
args=parser.parse_args() args = parser.parse_args()
print(args) print(args)
@ -33,7 +34,6 @@ def main():
with open(args.out) as f: with open(args.out) as f:
data_json = json.load(f) data_json = json.load(f)
container_list = data_json['containers'] container_list = data_json['containers']
# Get CID list
else: else:
print(f"Create containers: {args.containers}") print(f"Create containers: {args.containers}")
with ProcessPoolExecutor(max_workers=10) as executor: with ProcessPoolExecutor(max_workers=10) as executor:
@ -78,6 +78,7 @@ def random_payload(payload_filepath):
with open('%s'%payload_filepath, 'wb') as fout: with open('%s'%payload_filepath, 'wb') as fout:
fout.write(os.urandom(1024*int(args.size))) fout.write(os.urandom(1024*int(args.size)))
def execute_cmd(cmd_line): def execute_cmd(cmd_line):
args = shlex.split(cmd_line) args = shlex.split(cmd_line)
output = "" output = ""
@ -93,11 +94,11 @@ def execute_cmd(cmd_line):
def create_container(): def create_container():
cmd_line = f"neofs-cli --rpc-endpoint {args.endpoint} container create -g --policy 'REP 1 IN X CBF 1 SELECT 1 FROM * AS X' --basic-acl public-read-write --await" cmd_line = f"neofs-cli --rpc-endpoint {args.endpoint} container create -g --policy {args.policy} --basic-acl public-read-write --await"
output, success = execute_cmd(cmd_line) output, success = execute_cmd(cmd_line)
if not success: if not success:
print(f" > Container has not been created.") print(f" > Container has not been created.")
else: else:
try: try:
fst_str = output.split('\n')[0] fst_str = output.split('\n')[0]

View file

@ -11,7 +11,8 @@ Scenarios `grpc.js`, `http.js` and `s3.js` support the following options:
* `REGISTRY_FILE` - if set, all produced objects will be stored in database for subsequent verification. Database file name will be set to the value of `REGISTRY_FILE`. * `REGISTRY_FILE` - if set, all produced objects will be stored in database for subsequent verification. Database file name will be set to the value of `REGISTRY_FILE`.
* `WRITE_OBJ_SIZE` - object size in kb for write(PUT) operations. * `WRITE_OBJ_SIZE` - object size in kb for write(PUT) operations.
* `PREGEN_JSON` - path to json file with pre-generated containers and objects (in case of http scenario we use json pre-generated for grpc scenario). * `PREGEN_JSON` - path to json file with pre-generated containers and objects (in case of http scenario we use json pre-generated for grpc scenario).
* `SLEEP` - time interval (in seconds) between VU iterations. * `SLEEP_WRITE` - time interval (in seconds) between writing VU iterations.
* `SLEEP_READ` - time interval (in seconds) between reading VU iterations.
Examples of how to use these options are provided below for each scenario. Examples of how to use these options are provided below for each scenario.
@ -35,6 +36,7 @@ Options (in addition to the common options):
* `GRPC_ENDPOINTS` - GRPC endpoints of neoFS in format `host:port`. To specify multiple endpoints separate them by comma. * `GRPC_ENDPOINTS` - GRPC endpoints of neoFS in format `host:port`. To specify multiple endpoints separate them by comma.
* `DELETERS` - number of VUs performing delete operations (using deleters requires that options `DELETE_AGE` and `REGISTRY_FILE` are specified as well). * `DELETERS` - number of VUs performing delete operations (using deleters requires that options `DELETE_AGE` and `REGISTRY_FILE` are specified as well).
* `DELETE_AGE` - age of object in seconds before which it can not be deleted. This parameter can be used to control how many objects we have in the system under load. * `DELETE_AGE` - age of object in seconds before which it can not be deleted. This parameter can be used to control how many objects we have in the system under load.
* `SLEEP_DELETE` - time interval (in seconds) between deleting VU iterations.
## HTTP ## HTTP
@ -90,6 +92,7 @@ Options (in addition to the common options):
* `S3_ENDPOINTS` - endpoints of S3 gateways in format `host:port`. To specify multiple endpoints separate them by comma. * `S3_ENDPOINTS` - endpoints of S3 gateways in format `host:port`. To specify multiple endpoints separate them by comma.
* `DELETERS` - number of VUs performing delete operations (using deleters requires that options `DELETE_AGE` and `REGISTRY_FILE` are specified as well). * `DELETERS` - number of VUs performing delete operations (using deleters requires that options `DELETE_AGE` and `REGISTRY_FILE` are specified as well).
* `DELETE_AGE` - age of object in seconds before which it can not be deleted. This parameter can be used to control how many objects we have in the system under load. * `DELETE_AGE` - age of object in seconds before which it can not be deleted. This parameter can be used to control how many objects we have in the system under load.
* `SLEEP_DELETE` - time interval (in seconds) between deleting VU iterations.
* `OBJ_NAME` - if specified, this name will be used for all write operations instead of random generation. * `OBJ_NAME` - if specified, this name will be used for all write operations instead of random generation.
## Verify ## Verify

View file

@ -98,8 +98,8 @@ export function teardown(data) {
} }
export function obj_write() { export function obj_write() {
if (__ENV.SLEEP) { if (__ENV.SLEEP_WRITE) {
sleep(__ENV.SLEEP); sleep(__ENV.SLEEP_WRITE);
} }
const key = __ENV.OBJ_NAME || uuidv4(); const key = __ENV.OBJ_NAME || uuidv4();
@ -118,8 +118,8 @@ export function obj_write() {
} }
export function obj_read() { export function obj_read() {
if (__ENV.SLEEP) { if (__ENV.SLEEP_READ) {
sleep(__ENV.SLEEP); sleep(__ENV.SLEEP_READ);
} }
const obj = obj_list[Math.floor(Math.random() * obj_list.length)]; const obj = obj_list[Math.floor(Math.random() * obj_list.length)];
@ -131,8 +131,8 @@ export function obj_read() {
} }
export function obj_delete() { export function obj_delete() {
if (__ENV.SLEEP) { if (__ENV.SLEEP_DELETE) {
sleep(__ENV.SLEEP); sleep(__ENV.SLEEP_DELETE);
} }
const obj = obj_to_delete_selector.nextObject(); const obj = obj_to_delete_selector.nextObject();