diff --git a/.forgejo/workflows/tests.yml b/.forgejo/workflows/tests.yml index 42fe353..fdf5301 100644 --- a/.forgejo/workflows/tests.yml +++ b/.forgejo/workflows/tests.yml @@ -20,6 +20,27 @@ jobs: - name: Run linters run: make lint + check-format: + name: Check format + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Install deps + run: | + apt update + apt install -y clang-format + + - name: Run clang-format + run: clang-format -i **/*.js + + - name: Print diff + run: git diff HEAD + + - name: Check that nothing has changed + run: git diff-index --exit-code HEAD + tests: name: Tests runs-on: ubuntu-latest diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..2deaadc --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,55 @@ +ci: + autofix_prs: false + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: check-added-large-files + - id: check-case-conflict + - id: check-executables-have-shebangs + - id: check-shebang-scripts-are-executable + - id: check-merge-conflict + - id: check-json + - id: check-xml + - id: check-yaml + - id: trailing-whitespace + args: [--markdown-linebreak-ext=md] + - id: end-of-file-fixer + exclude: "(.key|.svg)$" + + - repo: https://github.com/shellcheck-py/shellcheck-py + rev: v0.9.0.6 + hooks: + - id: shellcheck + + - repo: local + hooks: + - id: make-lint + name: Run Make Lint + entry: make lint + language: system + pass_filenames: false + + - repo: local + hooks: + - id: make-format + name: Run Make format + entry: make format + language: system + pass_filenames: false + + - repo: local + hooks: + - id: go-unit-tests + name: go unit tests + entry: make test GOFLAGS='' + pass_filenames: false + types: [go] + language: system + + - repo: https://github.com/TekWizely/pre-commit-golang + rev: v1.0.0-rc.1 + hooks: + - id: go-staticcheck-repo-mod + - id: go-mod-tidy diff --git a/Makefile b/Makefile index b834155..baea79a 100644 --- a/Makefile +++ b/Makefile @@ -62,10 +62,19 @@ cover: @go test -v -race ./... -coverprofile=coverage.txt -covermode=atomic @go tool cover -html=coverage.txt -o coverage.html +# Install gofumpt +fumpt-install: + @echo "⇒ Processing fumpt install" + @rm -rf $(GOFUMPT_DIR) + @mkdir $(GOFUMPT_DIR) + @GOBIN=$(GOFUMPT_VERSION_DIR) go install mvdan.cc/gofumpt@$(GOFUMPT_VERSION) + # Reformat code format: @echo "⇒ Processing gofmt check" @gofmt -s -w ./ + @echo "⇒ Processing clang format" + @clang-format -i **/*.js # Run linters lint: diff --git a/examples/local.js b/examples/local.js index 79161b9..7d01d11 100644 --- a/examples/local.js +++ b/examples/local.js @@ -1,24 +1,28 @@ import local from 'k6/x/frostfs/local'; -import { uuidv4 } from '../scenarios/libs/k6-utils-1.4.0.js'; + +import {uuidv4} from '../scenarios/libs/k6-utils-1.4.0.js'; const payload = open('../go.sum', 'b'); -const local_cli = local.connect("/path/to/config.yaml", "/path/to/config/dir", "", false) +const local_cli = + local.connect("/path/to/config.yaml", "/path/to/config/dir", "", false) export const options = { - stages: [ - {duration: '30s', target: 10}, - ], + stages : [ + {duration : '30s', target : 10}, + ], }; -export default function () { - let headers = { - 'unique_header': uuidv4() - } - const container_id = '6BVPPXQewRJ6J5EYmAPLczXxNocS7ikyF7amS2esWQnb'; - let resp = local_cli.put(container_id, headers, payload) - if (resp.success) { - local_cli.get(container_id, resp.object_id) - } else { - console.log(resp.error) - } +export default function() { + let headers = { + 'unique_header' : uuidv4() + } + + const container_id = '6BVPPXQewRJ6J5EYmAPLczXxNocS7ikyF7amS2esWQnb'; + let resp = local_cli.put(container_id, headers, payload) + if (resp.success) { + local_cli.get(container_id, resp.object_id) + } + else { + console.log(resp.error) + } } diff --git a/examples/native.js b/examples/native.js index d971909..c4f644d 100644 --- a/examples/native.js +++ b/examples/native.js @@ -1,40 +1,43 @@ +import {fail} from "k6"; import native from 'k6/x/frostfs/native'; -import { fail } from "k6"; -import { uuidv4 } from '../scenarios/libs/k6-utils-1.4.0.js'; + +import {uuidv4} from '../scenarios/libs/k6-utils-1.4.0.js'; const payload = open('../go.sum', 'b'); -const frostfs_cli = native.connect("s01.frostfs.devenv:8080", "1dd37fba80fec4e6a6f13fd708d8dcb3b29def768017052f6c930fa1c5d90bbb", 0, 0, false, 0) +const frostfs_cli = native.connect( + "s01.frostfs.devenv:8080", + "1dd37fba80fec4e6a6f13fd708d8dcb3b29def768017052f6c930fa1c5d90bbb", 0, 0, + false, 0) export const options = { - stages: [ - { duration: '30s', target: 10 }, - ], + stages : [ + {duration : '30s', target : 10}, + ], }; export function setup() { - const params = { - acl: 'public-read-write', - placement_policy: 'REP 3', - name: 'container-name', - name_global_scope: 'false' - } + const params = { + acl : 'public-read-write', + placement_policy : 'REP 3', + name : 'container-name', + name_global_scope : 'false' + } - const res = frostfs_cli.putContainer(params) - if (!res.success) { - fail(res.error) - } - console.info("created container", res.container_id) - return { container_id: res.container_id } + const res = frostfs_cli.putContainer(params) + if (!res.success) { + fail(res.error) + } + console.info("created container", res.container_id) + return { container_id: res.container_id } } -export default function (data) { - let headers = { - 'unique_header': uuidv4() - } - let resp = frostfs_cli.put(data.container_id, headers, payload) - if (resp.success) { - frostfs_cli.get(data.container_id, resp.object_id) - } else { - console.log(resp.error) - } +export default function(data) { + let headers = {'unique_header' : uuidv4()} let resp = + frostfs_cli.put(data.container_id, headers, payload) + if (resp.success) { + frostfs_cli.get(data.container_id, resp.object_id) + } + else { + console.log(resp.error) + } } diff --git a/examples/native_onsite.js b/examples/native_onsite.js index ba71460..ee8f1f6 100644 --- a/examples/native_onsite.js +++ b/examples/native_onsite.js @@ -1,25 +1,27 @@ import native from 'k6/x/frostfs/native'; -import { uuidv4 } from '../scenarios/libs/k6-utils-1.4.0.js'; + +import {uuidv4} from '../scenarios/libs/k6-utils-1.4.0.js'; const payload = open('../go.sum', 'b'); const container = "AjSxSNNXbJUDPqqKYm1VbFVDGCakbpUNH8aGjPmGAH3B" -const frostfs_cli = native.connect("s01.frostfs.devenv:8080", "", 0, 0, false, 0) +const frostfs_cli = + native.connect("s01.frostfs.devenv:8080", "", 0, 0, false, 0) const frostfs_obj = frostfs_cli.onsite(container, payload) export const options = { - stages: [ - { duration: '30s', target: 10 }, - ], + stages : [ + {duration : '30s', target : 10}, + ], }; -export default function () { - let headers = { - 'unique_header': uuidv4() - } - let resp = frostfs_obj.put(headers) - if (resp.success) { - frostfs_cli.get(container, resp.object_id) - } else { - console.log(resp.error) - } +export default function() { + let headers = {'unique_header' : uuidv4()} + + let resp = frostfs_obj.put(headers) + if (resp.success) { + frostfs_cli.get(container, resp.object_id) + } + else { + console.log(resp.error) + } } diff --git a/examples/s3.js b/examples/s3.js index 6e73ec6..85e5973 100644 --- a/examples/s3.js +++ b/examples/s3.js @@ -1,33 +1,35 @@ +import {fail} from 'k6' import s3 from 'k6/x/frostfs/s3'; -import { fail } from 'k6' -import { uuidv4 } from '../scenarios/libs/k6-utils-1.4.0.js'; + +import {uuidv4} from '../scenarios/libs/k6-utils-1.4.0.js'; const payload = open('../go.sum', 'b'); const bucket = "cats" -const s3_cli = s3.connect("https://s3.frostfs.devenv:8080", {'no_verify_ssl': 'true'}) +const s3_cli = + s3.connect("https://s3.frostfs.devenv:8080", {'no_verify_ssl' : 'true'}) export const options = { - stages: [ - {duration: '30s', target: 10}, - ], + stages : [ + {duration : '30s', target : 10}, + ], }; export function setup() { - const params = { - acl: 'private', - lock_enabled: 'true', - location_constraint: 'ru' - } + const params = { + acl : 'private', + lock_enabled : 'true', + location_constraint : 'ru' + } - const res = s3_cli.createBucket(bucket, params) - if (!res.success) { - fail(res.error) - } + const res = s3_cli.createBucket(bucket, params) + if (!res.success) { + fail(res.error) + } } -export default function () { - const key = uuidv4(); - if (s3_cli.put(bucket, key, payload).success) { - s3_cli.get(bucket, key) - } +export default function() { + const key = uuidv4(); + if (s3_cli.put(bucket, key, payload).success) { + s3_cli.get(bucket, key) + } } diff --git a/examples/s3local.js b/examples/s3local.js index a88cb55..0e0f337 100644 --- a/examples/s3local.js +++ b/examples/s3local.js @@ -1,15 +1,17 @@ import s3local from 'k6/x/frostfs/s3local'; -import { uuidv4 } from '../scenarios/libs/k6-utils-1.4.0.js'; + +import {uuidv4} from '../scenarios/libs/k6-utils-1.4.0.js'; const bucket = "testbucket" const payload = open('../go.sum', 'b'); -const s3local_cli = s3local.connect("path/to/storage/config.yml", "path/to/storage/config/dir", {}, { - 'testbucket': 'GBQDDUM1hdodXmiRHV57EUkFWJzuntsG8BG15wFSwam6', -}); +const s3local_cli = s3local.connect( + "path/to/storage/config.yml", "path/to/storage/config/dir", {}, { + 'testbucket' : 'GBQDDUM1hdodXmiRHV57EUkFWJzuntsG8BG15wFSwam6', + }); -export default function () { - const key = uuidv4(); - if (s3local_cli.put(bucket, key, payload).success) { - s3local_cli.get(bucket, key) - } +export default function() { + const key = uuidv4(); + if (s3local_cli.put(bucket, key, payload).success) { + s3local_cli.get(bucket, key) + } } diff --git a/scenarios/grpc.js b/scenarios/grpc.js index 973cf25..32d5ea2 100644 --- a/scenarios/grpc.js +++ b/scenarios/grpc.js @@ -1,25 +1,25 @@ -import { sleep } from 'k6'; -import { SharedArray } from 'k6/data'; +import {sleep} from 'k6'; +import {SharedArray} from 'k6/data'; import exec from 'k6/execution'; import logging from 'k6/x/frostfs/logging'; import native from 'k6/x/frostfs/native'; import registry from 'k6/x/frostfs/registry'; import stats from 'k6/x/frostfs/stats'; -import { newGenerator } from './libs/datagen.js'; -import { parseEnv } from './libs/env-parser.js'; -import { textSummary } from './libs/k6-summary-0.0.2.js'; -import { uuidv4 } from './libs/k6-utils-1.4.0.js'; +import {newGenerator} from './libs/datagen.js'; +import {parseEnv} from './libs/env-parser.js'; +import {textSummary} from './libs/k6-summary-0.0.2.js'; +import {uuidv4} from './libs/k6-utils-1.4.0.js'; parseEnv(); const obj_list = new SharedArray( - 'obj_list', - function () { return JSON.parse(open(__ENV.PREGEN_JSON)).objects; }); + 'obj_list', + function() { return JSON.parse(open(__ENV.PREGEN_JSON)).objects; }); const container_list = new SharedArray( - 'container_list', - function () { return JSON.parse(open(__ENV.PREGEN_JSON)).containers; }); + 'container_list', + function() { return JSON.parse(open(__ENV.PREGEN_JSON)).containers; }); const read_size = JSON.parse(open(__ENV.PREGEN_JSON)).obj_size; const summary_json = __ENV.SUMMARY_JSON || '/tmp/summary.json'; @@ -27,17 +27,18 @@ const summary_json = __ENV.SUMMARY_JSON || '/tmp/summary.json'; // Select random gRPC endpoint for current VU const grpc_endpoints = __ENV.GRPC_ENDPOINTS.split(','); const grpc_endpoint = - grpc_endpoints[Math.floor(Math.random() * grpc_endpoints.length)]; + grpc_endpoints[Math.floor(Math.random() * grpc_endpoints.length)]; const grpc_client = native.connect( - grpc_endpoint, '', __ENV.DIAL_TIMEOUT ? parseInt(__ENV.DIAL_TIMEOUT) : 5, - __ENV.STREAM_TIMEOUT ? parseInt(__ENV.STREAM_TIMEOUT) : 60, - __ENV.PREPARE_LOCALLY ? __ENV.PREPARE_LOCALLY.toLowerCase() === 'true' : false, - 1024 * parseInt(__ENV.MAX_OBJECT_SIZE || '0')); + grpc_endpoint, '', __ENV.DIAL_TIMEOUT ? parseInt(__ENV.DIAL_TIMEOUT) : 5, + __ENV.STREAM_TIMEOUT ? parseInt(__ENV.STREAM_TIMEOUT) : 60, + __ENV.PREPARE_LOCALLY ? __ENV.PREPARE_LOCALLY.toLowerCase() === 'true' + : false, + 1024 * parseInt(__ENV.MAX_OBJECT_SIZE || '0')); const log = logging.new().withField('endpoint', grpc_endpoint); const registry_enabled = !!__ENV.REGISTRY_FILE; const obj_registry = - registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : undefined; + registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : undefined; const duration = __ENV.DURATION; @@ -49,11 +50,11 @@ const read_age = __ENV.READ_AGE ? parseInt(__ENV.READ_AGE) : 10; let obj_to_read_selector = undefined; if (registry_enabled) { obj_to_read_selector = registry.getLoopedSelector( - __ENV.REGISTRY_FILE, 'obj_to_read', - __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, { - status: 'created', - age: read_age, - }) + __ENV.REGISTRY_FILE, 'obj_to_read', + __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, { + status : 'created', + age : read_age, + }) } const scenarios = {}; @@ -63,11 +64,11 @@ const write_grpc_chunk_size = 1024 * parseInt(__ENV.GRPC_CHUNK_SIZE || '0') const generator = newGenerator(write_vu_count > 0); if (write_vu_count > 0) { scenarios.write = { - executor: 'constant-vus', - vus: write_vu_count, - duration: `${duration}s`, - exec: 'obj_write', - gracefulStop: '5s', + executor : 'constant-vus', + vus : write_vu_count, + duration : `${duration}s`, + exec : 'obj_write', + gracefulStop : '5s', }; } @@ -78,24 +79,24 @@ if (registry_enabled && delete_age) { obj_to_delete_exit_on_null = write_vu_count == 0; let constructor = obj_to_delete_exit_on_null ? registry.getOneshotSelector - : registry.getSelector; + : registry.getSelector; obj_to_delete_selector = - constructor(__ENV.REGISTRY_FILE, 'obj_to_delete', - __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, { - status: 'created', - age: delete_age, - }); + constructor(__ENV.REGISTRY_FILE, 'obj_to_delete', + __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, { + status : 'created', + age : delete_age, + }); } const read_vu_count = parseInt(__ENV.READERS || '0'); if (read_vu_count > 0) { scenarios.read = { - executor: 'constant-vus', - vus: read_vu_count, - duration: `${duration}s`, - exec: 'obj_read', - gracefulStop: '5s', + executor : 'constant-vus', + vus : read_vu_count, + duration : `${duration}s`, + exec : 'obj_read', + gracefulStop : '5s', }; } @@ -103,21 +104,21 @@ const delete_vu_count = parseInt(__ENV.DELETERS || '0'); if (delete_vu_count > 0) { if (!obj_to_delete_selector) { throw new Error( - 'Positive DELETE worker number without a proper object selector'); + 'Positive DELETE worker number without a proper object selector'); } scenarios.delete = { - executor: 'constant-vus', - vus: delete_vu_count, - duration: `${duration}s`, - exec: 'obj_delete', - gracefulStop: '5s', + executor : 'constant-vus', + vus : delete_vu_count, + duration : `${duration}s`, + exec : 'obj_delete', + gracefulStop : '5s', }; } export const options = { scenarios, - setupTimeout: '5s', + setupTimeout : '5s', }; export function setup() { @@ -133,7 +134,7 @@ export function setup() { const start_timestamp = Date.now() console.log( - `Load started at: ${Date(start_timestamp).toString()}`) + `Load started at: ${Date(start_timestamp).toString()}`) } export function teardown(data) { @@ -142,13 +143,13 @@ export function teardown(data) { } const end_timestamp = Date.now() console.log( - `Load finished at: ${Date(end_timestamp).toString()}`) + `Load finished at: ${Date(end_timestamp).toString()}`) } export function handleSummary(data) { return { - 'stdout': textSummary(data, { indent: ' ', enableColors: false }), - [summary_json]: JSON.stringify(data), + 'stdout' : textSummary(data, {indent : ' ', enableColors : false}), + [summary_json] : JSON.stringify(data), }; } @@ -157,13 +158,13 @@ export function obj_write() { sleep(__ENV.SLEEP_WRITE); } - const headers = { unique_header: uuidv4() }; + const headers = {unique_header : uuidv4()}; const container = - container_list[Math.floor(Math.random() * container_list.length)]; + container_list[Math.floor(Math.random() * container_list.length)]; const payload = generator.genPayload(); const resp = - grpc_client.put(container, headers, payload, write_grpc_chunk_size); + grpc_client.put(container, headers, payload, write_grpc_chunk_size); if (!resp.success) { log.withField('cid', container).error(resp.error); return; @@ -186,7 +187,7 @@ export function obj_read() { } const resp = grpc_client.get(obj.c_id, obj.o_id) if (!resp.success) { - log.withFields({ cid: obj.c_id, oid: obj.o_id }).error(resp.error); + log.withFields({cid : obj.c_id, oid : obj.o_id}).error(resp.error); } return } @@ -194,7 +195,7 @@ export function obj_read() { const obj = obj_list[Math.floor(Math.random() * obj_list.length)]; const resp = grpc_client.get(obj.container, obj.object) if (!resp.success) { - log.withFields({ cid: obj.container, oid: obj.object }).error(resp.error); + log.withFields({cid : obj.container, oid : obj.object}).error(resp.error); } } @@ -214,7 +215,7 @@ export function obj_delete() { const resp = grpc_client.delete(obj.c_id, obj.o_id); if (!resp.success) { // Log errors except (2052 - object already deleted) - log.withFields({ cid: obj.c_id, oid: obj.o_id }).error(resp.error); + log.withFields({cid : obj.c_id, oid : obj.o_id}).error(resp.error); return; } diff --git a/scenarios/grpc_car.js b/scenarios/grpc_car.js index d866610..13b0bf8 100644 --- a/scenarios/grpc_car.js +++ b/scenarios/grpc_car.js @@ -1,24 +1,24 @@ -import { sleep } from 'k6'; -import { SharedArray } from 'k6/data'; +import {sleep} from 'k6'; +import {SharedArray} from 'k6/data'; import logging from 'k6/x/frostfs/logging'; import native from 'k6/x/frostfs/native'; import registry from 'k6/x/frostfs/registry'; import stats from 'k6/x/frostfs/stats'; -import { newGenerator } from './libs/datagen.js'; -import { parseEnv } from './libs/env-parser.js'; -import { textSummary } from './libs/k6-summary-0.0.2.js'; -import { uuidv4 } from './libs/k6-utils-1.4.0.js'; +import {newGenerator} from './libs/datagen.js'; +import {parseEnv} from './libs/env-parser.js'; +import {textSummary} from './libs/k6-summary-0.0.2.js'; +import {uuidv4} from './libs/k6-utils-1.4.0.js'; parseEnv(); -const obj_list = new SharedArray('obj_list', function () { - return JSON.parse(open(__ENV.PREGEN_JSON)).objects; -}); +const obj_list = new SharedArray( + 'obj_list', + function() { return JSON.parse(open(__ENV.PREGEN_JSON)).objects; }); -const container_list = new SharedArray('container_list', function () { - return JSON.parse(open(__ENV.PREGEN_JSON)).containers; -}); +const container_list = new SharedArray( + 'container_list', + function() { return JSON.parse(open(__ENV.PREGEN_JSON)).containers; }); const read_size = JSON.parse(open(__ENV.PREGEN_JSON)).obj_size; const summary_json = __ENV.SUMMARY_JSON || '/tmp/summary.json'; @@ -26,17 +26,18 @@ const summary_json = __ENV.SUMMARY_JSON || '/tmp/summary.json'; // Select random gRPC endpoint for current VU const grpc_endpoints = __ENV.GRPC_ENDPOINTS.split(','); const grpc_endpoint = - grpc_endpoints[Math.floor(Math.random() * grpc_endpoints.length)]; + grpc_endpoints[Math.floor(Math.random() * grpc_endpoints.length)]; const grpc_client = native.connect( - grpc_endpoint, '', __ENV.DIAL_TIMEOUT ? parseInt(__ENV.DIAL_TIMEOUT) : 5, - __ENV.STREAM_TIMEOUT ? parseInt(__ENV.STREAM_TIMEOUT) : 60, - __ENV.PREPARE_LOCALLY ? __ENV.PREPARE_LOCALLY.toLowerCase() === 'true' : false, - 1024 * parseInt(__ENV.MAX_OBJECT_SIZE || '0')); + grpc_endpoint, '', __ENV.DIAL_TIMEOUT ? parseInt(__ENV.DIAL_TIMEOUT) : 5, + __ENV.STREAM_TIMEOUT ? parseInt(__ENV.STREAM_TIMEOUT) : 60, + __ENV.PREPARE_LOCALLY ? __ENV.PREPARE_LOCALLY.toLowerCase() === 'true' + : false, + 1024 * parseInt(__ENV.MAX_OBJECT_SIZE || '0')); const log = logging.new().withField('endpoint', grpc_endpoint); const registry_enabled = !!__ENV.REGISTRY_FILE; const obj_registry = - registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : undefined; + registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : undefined; const duration = __ENV.DURATION; @@ -48,22 +49,22 @@ const delete_age = __ENV.DELETE_AGE ? parseInt(__ENV.DELETE_AGE) : undefined; let obj_to_delete_selector = undefined; if (registry_enabled && delete_age) { obj_to_delete_selector = registry.getSelector( - __ENV.REGISTRY_FILE, 'obj_to_delete', - __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, { - status: 'created', - age: delete_age, - }); + __ENV.REGISTRY_FILE, 'obj_to_delete', + __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, { + status : 'created', + age : delete_age, + }); } const read_age = __ENV.READ_AGE ? parseInt(__ENV.READ_AGE) : 10; let obj_to_read_selector = undefined; if (registry_enabled) { obj_to_read_selector = registry.getLoopedSelector( - __ENV.REGISTRY_FILE, 'obj_to_read', - __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, { - status: 'created', - age: read_age, - }) + __ENV.REGISTRY_FILE, 'obj_to_read', + __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, { + status : 'created', + age : read_age, + }) } const scenarios = {}; @@ -76,14 +77,14 @@ const write_grpc_chunk_size = 1024 * parseInt(__ENV.GRPC_CHUNK_SIZE || '0') const generator = newGenerator(write_rate > 0); if (write_rate > 0) { scenarios.write = { - executor: 'constant-arrival-rate', - duration: `${duration}s`, - preAllocatedVUs: pre_alloc_write_vus, - maxVUs: max_write_vus, - rate: write_rate, - timeUnit: time_unit, - exec: 'obj_write', - gracefulStop: '5s', + executor : 'constant-arrival-rate', + duration : `${duration}s`, + preAllocatedVUs : pre_alloc_write_vus, + maxVUs : max_write_vus, + rate : write_rate, + timeUnit : time_unit, + exec : 'obj_write', + gracefulStop : '5s', }; } @@ -92,14 +93,14 @@ const max_read_vus = parseInt(__ENV.MAX_READERS || pre_alloc_read_vus); const read_rate = parseInt(__ENV.READ_RATE || '0'); if (read_rate > 0) { scenarios.read = { - executor: 'constant-arrival-rate', - duration: `${duration}s`, - preAllocatedVUs: pre_alloc_write_vus, - maxVUs: max_read_vus, - rate: read_rate, - timeUnit: time_unit, - exec: 'obj_read', - gracefulStop: '5s', + executor : 'constant-arrival-rate', + duration : `${duration}s`, + preAllocatedVUs : pre_alloc_write_vus, + maxVUs : max_read_vus, + rate : read_rate, + timeUnit : time_unit, + exec : 'obj_read', + gracefulStop : '5s', }; } @@ -109,29 +110,29 @@ const delete_rate = parseInt(__ENV.DELETE_RATE || '0'); if (delete_rate > 0) { if (!obj_to_delete_selector) { throw new Error( - 'Positive DELETE worker number without a proper object selector'); + 'Positive DELETE worker number without a proper object selector'); } scenarios.delete = { - executor: 'constant-arrival-rate', - duration: `${duration}s`, - preAllocatedVUs: pre_alloc_delete_vus, - maxVUs: max_delete_vus, - rate: delete_rate, - timeUnit: time_unit, - exec: 'obj_delete', - gracefulStop: '5s', + executor : 'constant-arrival-rate', + duration : `${duration}s`, + preAllocatedVUs : pre_alloc_delete_vus, + maxVUs : max_delete_vus, + rate : delete_rate, + timeUnit : time_unit, + exec : 'obj_delete', + gracefulStop : '5s', }; } export const options = { scenarios, - setupTimeout: '5s', + setupTimeout : '5s', }; export function setup() { const total_pre_allocated_vu_count = - pre_alloc_write_vus + pre_alloc_read_vus + pre_alloc_delete_vus; + pre_alloc_write_vus + pre_alloc_read_vus + pre_alloc_delete_vus; const total_max_vu_count = max_read_vus + max_write_vus + max_delete_vus console.log(`Pregenerated containers: ${container_list.length}`); @@ -152,7 +153,7 @@ export function setup() { const start_timestamp = Date.now() console.log( - `Load started at: ${Date(start_timestamp).toString()}`) + `Load started at: ${Date(start_timestamp).toString()}`) } export function teardown(data) { @@ -161,13 +162,13 @@ export function teardown(data) { } const end_timestamp = Date.now() console.log( - `Load finished at: ${Date(end_timestamp).toString()}`) + `Load finished at: ${Date(end_timestamp).toString()}`) } export function handleSummary(data) { return { - 'stdout': textSummary(data, { indent: ' ', enableColors: false }), - [summary_json]: JSON.stringify(data), + 'stdout' : textSummary(data, {indent : ' ', enableColors : false}), + [summary_json] : JSON.stringify(data), }; } @@ -176,13 +177,13 @@ export function obj_write() { sleep(__ENV.SLEEP_WRITE); } - const headers = { unique_header: uuidv4() }; + const headers = {unique_header : uuidv4()}; const container = - container_list[Math.floor(Math.random() * container_list.length)]; + container_list[Math.floor(Math.random() * container_list.length)]; const payload = generator.genPayload(); const resp = - grpc_client.put(container, headers, payload, write_grpc_chunk_size); + grpc_client.put(container, headers, payload, write_grpc_chunk_size); if (!resp.success) { log.withField('cid', container).error(resp.error); return; @@ -205,7 +206,7 @@ export function obj_read() { } const resp = grpc_client.get(obj.c_id, obj.o_id) if (!resp.success) { - log.withFields({ cid: obj.c_id, oid: obj.o_id }).error(resp.error); + log.withFields({cid : obj.c_id, oid : obj.o_id}).error(resp.error); } return } @@ -213,7 +214,7 @@ export function obj_read() { const obj = obj_list[Math.floor(Math.random() * obj_list.length)]; const resp = grpc_client.get(obj.container, obj.object) if (!resp.success) { - log.withFields({ cid: obj.container, oid: obj.object }).error(resp.error); + log.withFields({cid : obj.container, oid : obj.object}).error(resp.error); } } @@ -230,7 +231,7 @@ export function obj_delete() { const resp = grpc_client.delete(obj.c_id, obj.o_id); if (!resp.success) { // Log errors except (2052 - object already deleted) - log.withFields({ cid: obj.c_id, oid: obj.o_id }).error(resp.error); + log.withFields({cid : obj.c_id, oid : obj.o_id}).error(resp.error); return; } diff --git a/scenarios/http.js b/scenarios/http.js index 801c31c..fa54318 100644 --- a/scenarios/http.js +++ b/scenarios/http.js @@ -12,13 +12,13 @@ import {uuidv4} from './libs/k6-utils-1.4.0.js'; parseEnv(); -const obj_list = new SharedArray('obj_list', function() { - return JSON.parse(open(__ENV.PREGEN_JSON)).objects; -}); +const obj_list = new SharedArray( + 'obj_list', + function() { return JSON.parse(open(__ENV.PREGEN_JSON)).objects; }); -const container_list = new SharedArray('container_list', function() { - return JSON.parse(open(__ENV.PREGEN_JSON)).containers; -}); +const container_list = new SharedArray( + 'container_list', + function() { return JSON.parse(open(__ENV.PREGEN_JSON)).containers; }); const read_size = JSON.parse(open(__ENV.PREGEN_JSON)).obj_size; const summary_json = __ENV.SUMMARY_JSON || '/tmp/summary.json'; @@ -45,28 +45,28 @@ const write_vu_count = parseInt(__ENV.WRITERS || '0'); const generator = newGenerator(write_vu_count > 0); if (write_vu_count > 0) { scenarios.write = { - executor: 'constant-vus', - vus: write_vu_count, - duration: `${duration}s`, - exec: 'obj_write', - gracefulStop: '5s', + executor : 'constant-vus', + vus : write_vu_count, + duration : `${duration}s`, + exec : 'obj_write', + gracefulStop : '5s', } } const read_vu_count = parseInt(__ENV.READERS || '0'); if (read_vu_count > 0) { scenarios.read = { - executor: 'constant-vus', - vus: read_vu_count, - duration: `${duration}s`, - exec: 'obj_read', - gracefulStop: '5s', + executor : 'constant-vus', + vus : read_vu_count, + duration : `${duration}s`, + exec : 'obj_read', + gracefulStop : '5s', } } export const options = { scenarios, - setupTimeout: '5s', + setupTimeout : '5s', }; export function setup() { @@ -95,8 +95,8 @@ export function teardown(data) { export function handleSummary(data) { return { - 'stdout': textSummary(data, {indent: ' ', enableColors: false}), - [summary_json]: JSON.stringify(data), + 'stdout' : textSummary(data, {indent : ' ', enableColors : false}), + [summary_json] : JSON.stringify(data), }; } @@ -110,16 +110,16 @@ export function obj_write() { const payload = generator.genPayload(); const data = { - field: uuidv4(), + field : uuidv4(), // Because we use `file` wrapping and it is not straightforward to use // streams here, // `-e STREAMING=1` has no effect for this scenario. - file: http.file(payload.bytes(), 'random.data'), + file : http.file(payload.bytes(), 'random.data'), }; const resp = http.post(`http://${http_endpoint}/upload/${container}`, data); if (resp.status != 200) { - log.withFields({status: resp.status, cid: container}).error(resp.error); + log.withFields({status : resp.status, cid : container}).error(resp.error); return; } const object_id = JSON.parse(resp.body).object_id; @@ -137,7 +137,8 @@ export function obj_read() { const resp = http.get(`http://${http_endpoint}/get/${obj.container}/${obj.object}`); if (resp.status != 200) { - log.withFields({status: resp.status, cid: obj.container, oid: obj.object}) + log.withFields( + {status : resp.status, cid : obj.container, oid : obj.object}) .error(resp.error); } } diff --git a/scenarios/local.js b/scenarios/local.js index 7b3f003..d20ab51 100644 --- a/scenarios/local.js +++ b/scenarios/local.js @@ -12,13 +12,13 @@ import {uuidv4} from './libs/k6-utils-1.4.0.js'; parseEnv(); -const obj_list = new SharedArray('obj_list', function() { - return JSON.parse(open(__ENV.PREGEN_JSON)).objects; -}); +const obj_list = new SharedArray( + 'obj_list', + function() { return JSON.parse(open(__ENV.PREGEN_JSON)).objects; }); -const container_list = new SharedArray('container_list', function() { - return JSON.parse(open(__ENV.PREGEN_JSON)).containers; -}); +const container_list = new SharedArray( + 'container_list', + function() { return JSON.parse(open(__ENV.PREGEN_JSON)).containers; }); const read_size = JSON.parse(open(__ENV.PREGEN_JSON)).obj_size; const summary_json = __ENV.SUMMARY_JSON || '/tmp/summary.json'; @@ -31,7 +31,7 @@ const max_total_size_gb = const local_client = local.connect(config_file, config_dir, '', debug_logger, max_total_size_gb); const log = logging.new().withFields( - {'config_file': config_file, 'config_dir': config_dir}); + {'config_file' : config_file, 'config_dir' : config_dir}); const registry_enabled = !!__ENV.REGISTRY_FILE; const obj_registry = @@ -49,8 +49,8 @@ if (registry_enabled && delete_age) { obj_to_delete_selector = registry.getSelector( __ENV.REGISTRY_FILE, 'obj_to_delete', __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, { - status: 'created', - age: delete_age, + status : 'created', + age : delete_age, }); } @@ -60,22 +60,22 @@ const write_vu_count = parseInt(__ENV.WRITERS || '0'); const generator = newGenerator(write_vu_count > 0); if (write_vu_count > 0) { scenarios.write = { - executor: 'constant-vus', - vus: write_vu_count, - duration: `${duration}s`, - exec: 'obj_write', - gracefulStop: '5s', + executor : 'constant-vus', + vus : write_vu_count, + duration : `${duration}s`, + exec : 'obj_write', + gracefulStop : '5s', }; } const read_vu_count = parseInt(__ENV.READERS || '0'); if (read_vu_count > 0) { scenarios.read = { - executor: 'constant-vus', - vus: read_vu_count, - duration: `${duration}s`, - exec: 'obj_read', - gracefulStop: '5s', + executor : 'constant-vus', + vus : read_vu_count, + duration : `${duration}s`, + exec : 'obj_read', + gracefulStop : '5s', }; } @@ -87,17 +87,17 @@ if (delete_vu_count > 0) { } scenarios.delete = { - executor: 'constant-vus', - vus: delete_vu_count, - duration: `${duration}s`, - exec: 'obj_delete', - gracefulStop: '5s', + executor : 'constant-vus', + vus : delete_vu_count, + duration : `${duration}s`, + exec : 'obj_delete', + gracefulStop : '5s', }; } export const options = { scenarios, - setupTimeout: '5s', + setupTimeout : '5s', }; export function setup() { @@ -127,13 +127,13 @@ export function teardown(data) { export function handleSummary(data) { return { - 'stdout': textSummary(data, {indent: ' ', enableColors: false}), - [summary_json]: JSON.stringify(data), + 'stdout' : textSummary(data, {indent : ' ', enableColors : false}), + [summary_json] : JSON.stringify(data), }; } export function obj_write() { - const headers = {unique_header: uuidv4()}; + const headers = {unique_header : uuidv4()}; const container = container_list[Math.floor(Math.random() * container_list.length)]; @@ -156,7 +156,7 @@ export function obj_read() { const obj = obj_list[Math.floor(Math.random() * obj_list.length)]; const resp = local_client.get(obj.container, obj.object) if (!resp.success) { - log.withFields({cid: obj.container, oid: obj.object}).error(resp.error); + log.withFields({cid : obj.container, oid : obj.object}).error(resp.error); } } @@ -169,7 +169,7 @@ export function obj_delete() { const resp = local_client.delete(obj.c_id, obj.o_id); if (!resp.success) { // Log errors except (2052 - object already deleted) - log.withFields({cid: obj.c_id, oid: obj.o_id}).error(resp.error); + log.withFields({cid : obj.c_id, oid : obj.o_id}).error(resp.error); return; } diff --git a/scenarios/s3.js b/scenarios/s3.js index 2445b1a..28d6cd3 100644 --- a/scenarios/s3.js +++ b/scenarios/s3.js @@ -6,10 +6,10 @@ import registry from 'k6/x/frostfs/registry'; import s3 from 'k6/x/frostfs/s3'; import stats from 'k6/x/frostfs/stats'; -import {generateS3Key} from './libs/keygen.js'; +import {newGenerator} from './libs/datagen.js'; import {parseEnv} from './libs/env-parser.js'; import {textSummary} from './libs/k6-summary-0.0.2.js'; -import {newGenerator} from './libs/datagen.js'; +import {generateS3Key} from './libs/keygen.js'; parseEnv(); @@ -133,7 +133,7 @@ export function setup() { console.log( `Load started at: ${Date(start_timestamp).toString()}`) - if (delete_vu_count > 0){ + if (delete_vu_count > 0) { obj_to_delete_selector.sync.add(delete_vu_count) } } diff --git a/scenarios/s3_car.js b/scenarios/s3_car.js index af89e09..ac00f4f 100644 --- a/scenarios/s3_car.js +++ b/scenarios/s3_car.js @@ -5,20 +5,20 @@ import registry from 'k6/x/frostfs/registry'; import s3 from 'k6/x/frostfs/s3'; import stats from 'k6/x/frostfs/stats'; -import {generateS3Key} from './libs/keygen.js'; import {newGenerator} from './libs/datagen.js'; import {parseEnv} from './libs/env-parser.js'; import {textSummary} from './libs/k6-summary-0.0.2.js'; +import {generateS3Key} from './libs/keygen.js'; parseEnv(); -const obj_list = new SharedArray('obj_list', function() { - return JSON.parse(open(__ENV.PREGEN_JSON)).objects; -}); +const obj_list = new SharedArray( + 'obj_list', + function() { return JSON.parse(open(__ENV.PREGEN_JSON)).objects; }); -const bucket_list = new SharedArray('bucket_list', function() { - return JSON.parse(open(__ENV.PREGEN_JSON)).buckets; -}); +const bucket_list = new SharedArray( + 'bucket_list', + function() { return JSON.parse(open(__ENV.PREGEN_JSON)).buckets; }); const read_size = JSON.parse(open(__ENV.PREGEN_JSON)).obj_size; const summary_json = __ENV.SUMMARY_JSON || '/tmp/summary.json'; @@ -29,7 +29,7 @@ const s3_endpoint = s3_endpoints[Math.floor(Math.random() * s3_endpoints.length)]; const no_verify_ssl = __ENV.NO_VERIFY_SSL || 'true'; const connection_args = { - no_verify_ssl: no_verify_ssl + no_verify_ssl : no_verify_ssl }; const s3_client = s3.connect(s3_endpoint, connection_args); const log = logging.new().withField('endpoint', s3_endpoint); @@ -50,8 +50,8 @@ if (registry_enabled && delete_age) { obj_to_delete_selector = registry.getSelector( __ENV.REGISTRY_FILE, 'obj_to_delete', __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, { - status: 'created', - age: delete_age, + status : 'created', + age : delete_age, }); } @@ -61,8 +61,8 @@ if (registry_enabled) { obj_to_read_selector = registry.getLoopedSelector( __ENV.REGISTRY_FILE, 'obj_to_read', __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, { - status: 'created', - age: read_age, + status : 'created', + age : read_age, }) } @@ -75,35 +75,33 @@ const write_rate = parseInt(__ENV.WRITE_RATE || '0'); const generator = newGenerator(write_rate > 0); if (write_rate > 0) { scenarios.write = { - executor: 'constant-arrival-rate', - duration: `${duration}s`, - preAllocatedVUs: pre_alloc_write_vus, - maxVUs: max_write_vus, - rate: write_rate, - timeUnit: time_unit, - exec: 'obj_write', - gracefulStop: '5s', + executor : 'constant-arrival-rate', + duration : `${duration}s`, + preAllocatedVUs : pre_alloc_write_vus, + maxVUs : max_write_vus, + rate : write_rate, + timeUnit : time_unit, + exec : 'obj_write', + gracefulStop : '5s', }; } - const pre_alloc_read_vus = parseInt(__ENV.PRE_ALLOC_READERS || '0'); const max_read_vus = parseInt(__ENV.MAX_READERS || pre_alloc_read_vus); const read_rate = parseInt(__ENV.READ_RATE || '0'); if (read_rate > 0) { scenarios.read = { - executor: 'constant-arrival-rate', - duration: `${duration}s`, - preAllocatedVUs: pre_alloc_write_vus, - maxVUs: max_read_vus, - rate: read_rate, - timeUnit: time_unit, - exec: 'obj_read', - gracefulStop: '5s', + executor : 'constant-arrival-rate', + duration : `${duration}s`, + preAllocatedVUs : pre_alloc_write_vus, + maxVUs : max_read_vus, + rate : read_rate, + timeUnit : time_unit, + exec : 'obj_read', + gracefulStop : '5s', }; } - const pre_alloc_delete_vus = parseInt(__ENV.PRE_ALLOC_DELETERS || '0'); const max_delete_vus = parseInt(__ENV.MAX_DELETERS || pre_alloc_write_vus); const delete_rate = parseInt(__ENV.DELETE_RATE || '0'); @@ -114,20 +112,20 @@ if (delete_rate > 0) { } scenarios.delete = { - executor: 'constant-arrival-rate', - duration: `${duration}s`, - preAllocatedVUs: pre_alloc_delete_vus, - maxVUs: max_delete_vus, - rate: delete_rate, - timeUnit: time_unit, - exec: 'obj_delete', - gracefulStop: '5s', + executor : 'constant-arrival-rate', + duration : `${duration}s`, + preAllocatedVUs : pre_alloc_delete_vus, + maxVUs : max_delete_vus, + rate : delete_rate, + timeUnit : time_unit, + exec : 'obj_delete', + gracefulStop : '5s', }; } export const options = { scenarios, - setupTimeout: '5s', + setupTimeout : '5s', }; export function setup() { @@ -167,8 +165,8 @@ export function teardown(data) { export function handleSummary(data) { return { - 'stdout': textSummary(data, {indent: ' ', enableColors: false}), - [summary_json]: JSON.stringify(data), + 'stdout' : textSummary(data, {indent : ' ', enableColors : false}), + [summary_json] : JSON.stringify(data), }; } @@ -183,7 +181,7 @@ export function obj_write() { const payload = generator.genPayload(); const resp = s3_client.put(bucket, key, payload); if (!resp.success) { - log.withFields({bucket: bucket, key: key}).error(resp.error); + log.withFields({bucket : bucket, key : key}).error(resp.error); return; } @@ -204,7 +202,7 @@ export function obj_read() { } const resp = s3_client.get(obj.s3_bucket, obj.s3_key) if (!resp.success) { - log.withFields({bucket: obj.s3_bucket, key: obj.s3_key}) + log.withFields({bucket : obj.s3_bucket, key : obj.s3_key}) .error(resp.error); } return @@ -214,7 +212,7 @@ export function obj_read() { const resp = s3_client.get(obj.bucket, obj.object); if (!resp.success) { - log.withFields({bucket: obj.bucket, key: obj.object}).error(resp.error); + log.withFields({bucket : obj.bucket, key : obj.object}).error(resp.error); } } @@ -230,7 +228,7 @@ export function obj_delete() { const resp = s3_client.delete(obj.s3_bucket, obj.s3_key); if (!resp.success) { - log.withFields({bucket: obj.s3_bucket, key: obj.s3_key, op: 'DELETE'}) + log.withFields({bucket : obj.s3_bucket, key : obj.s3_key, op : 'DELETE'}) .error(resp.error); return; } diff --git a/scenarios/s3_dar.js b/scenarios/s3_dar.js index d9a71da..1733f31 100644 --- a/scenarios/s3_dar.js +++ b/scenarios/s3_dar.js @@ -6,10 +6,10 @@ import registry from 'k6/x/frostfs/registry'; import s3 from 'k6/x/frostfs/s3'; import stats from 'k6/x/frostfs/stats'; -import {generateS3Key} from './libs/keygen.js'; import {newGenerator} from './libs/datagen.js'; import {parseEnv} from './libs/env-parser.js'; import {textSummary} from './libs/k6-summary-0.0.2.js'; +import {generateS3Key} from './libs/keygen.js'; parseEnv(); @@ -27,7 +27,7 @@ const summary_json = __ENV.SUMMARY_JSON || '/tmp/summary.json'; const no_verify_ssl = __ENV.NO_VERIFY_SSL || 'true'; const connection_args = { - no_verify_ssl : no_verify_ssl + no_verify_ssl : no_verify_ssl } // Select random S3 endpoint for current VU const s3_endpoints = __ENV.S3_ENDPOINTS.split(','); @@ -43,18 +43,18 @@ const obj_registry = const duration = __ENV.DURATION; if (!!__ENV.METRIC_TAGS) { - stats.setTags(__ENV.METRIC_TAGS) + stats.setTags(__ENV.METRIC_TAGS) } const read_age = __ENV.READ_AGE ? parseInt(__ENV.READ_AGE) : 10; let obj_to_read_selector = undefined; if (registry_enabled) { - obj_to_read_selector = registry.getSelector( - __ENV.REGISTRY_FILE, 'obj_to_read', - __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, { - status : 'created', - age : read_age, - }) + obj_to_read_selector = registry.getSelector( + __ENV.REGISTRY_FILE, 'obj_to_read', + __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, { + status : 'created', + age : read_age, + }) } const scenarios = {}; @@ -62,172 +62,176 @@ const scenarios = {}; const write_vu_count = parseInt(__ENV.WRITERS || '0'); const generator = newGenerator(write_vu_count > 0); if (write_vu_count > 0) { - scenarios.write = { - executor : 'constant-vus', - vus : write_vu_count, - duration : `${duration}s`, - exec : 'obj_write', - gracefulStop : '5s', - }; + scenarios.write = { + executor : 'constant-vus', + vus : write_vu_count, + duration : `${duration}s`, + exec : 'obj_write', + gracefulStop : '5s', + }; } const read_vu_count = parseInt(__ENV.READERS || '0'); if (read_vu_count > 0) { - scenarios.read = { - executor : 'constant-vus', - vus : read_vu_count, - duration : `${duration}s`, - exec : 'obj_read', - gracefulStop : '5s', - }; + scenarios.read = { + executor : 'constant-vus', + vus : read_vu_count, + duration : `${duration}s`, + exec : 'obj_read', + gracefulStop : '5s', + }; } const delete_age = __ENV.DELETE_AGE ? parseInt(__ENV.DELETE_AGE) : undefined; let obj_to_delete_selector = undefined; let obj_to_delete_exit_on_null = undefined; -if (registry_enabled ) { - obj_to_delete_exit_on_null = (write_vu_count == 0) && (read_vu_count == 0) +if (registry_enabled) { + obj_to_delete_exit_on_null = (write_vu_count == 0) && (read_vu_count == 0) - let constructor = obj_to_delete_exit_on_null ? registry.getOneshotSelector - : registry.getSelector; + let constructor = obj_to_delete_exit_on_null ? registry.getOneshotSelector + : registry.getSelector; - obj_to_delete_selector = - constructor(__ENV.REGISTRY_FILE, 'obj_to_delete', - __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, { - status : 'read', - age : delete_age, - }); + obj_to_delete_selector = + constructor(__ENV.REGISTRY_FILE, 'obj_to_delete', + __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, { + status : 'read', + age : delete_age, + }); } - - const delete_vu_count = parseInt(__ENV.DELETERS || '0'); if (delete_vu_count > 0) { - if (!obj_to_delete_selector) { - throw 'Positive DELETE worker number without a proper object selector'; - } + if (!obj_to_delete_selector) { + throw 'Positive DELETE worker number without a proper object selector'; + } - scenarios.delete = { - executor : 'constant-vus', - vus : delete_vu_count, - duration : `${duration}s`, - exec : 'obj_delete', - gracefulStop : '5s', - }; + scenarios.delete = { + executor : 'constant-vus', + vus : delete_vu_count, + duration : `${duration}s`, + exec : 'obj_delete', + gracefulStop : '5s', + }; } export const options = { - scenarios, - setupTimeout : '5s', + scenarios, + setupTimeout : '5s', }; export function setup() { - const total_vu_count = write_vu_count + read_vu_count + delete_vu_count; + const total_vu_count = write_vu_count + read_vu_count + delete_vu_count; - console.log(`Pregenerated buckets: ${bucket_list.length}`); - console.log(`Pregenerated read object size: ${read_size}`); - console.log(`Pregenerated total objects: ${obj_list.length}`); - console.log(`Reading VUs: ${read_vu_count}`); - console.log(`Writing VUs: ${write_vu_count}`); - console.log(`Deleting VUs: ${delete_vu_count}`); - console.log(`Total VUs: ${total_vu_count}`); + console.log(`Pregenerated buckets: ${bucket_list.length}`); + console.log(`Pregenerated read object size: ${read_size}`); + console.log(`Pregenerated total objects: ${obj_list.length}`); + console.log(`Reading VUs: ${read_vu_count}`); + console.log(`Writing VUs: ${write_vu_count}`); + console.log(`Deleting VUs: ${delete_vu_count}`); + console.log(`Total VUs: ${total_vu_count}`); - const start_timestamp = Date.now() - console.log( - `Load started at: ${Date(start_timestamp).toString()}`) + const start_timestamp = Date.now() + console.log( + `Load started at: ${Date(start_timestamp).toString()}`) } export function teardown(data) { - if (obj_registry) { - obj_registry.close(); - } - const end_timestamp = Date.now() - console.log( - `Load finished at: ${Date(end_timestamp).toString()}`) + if (obj_registry) { + obj_registry.close(); + } + const end_timestamp = Date.now() + console.log( + `Load finished at: ${Date(end_timestamp).toString()}`) } export function handleSummary(data) { - return { - 'stdout' : textSummary(data, {indent : ' ', enableColors : false}), - [summary_json] : JSON.stringify(data), - }; + return { + 'stdout' : textSummary(data, {indent : ' ', enableColors : false}), + [summary_json] : JSON.stringify(data), + }; } export function obj_write() { - if (__ENV.SLEEP_WRITE) { - sleep(__ENV.SLEEP_WRITE); - } + if (__ENV.SLEEP_WRITE) { + sleep(__ENV.SLEEP_WRITE); + } - const key = generateS3Key(); - const bucket = bucket_list[Math.floor(Math.random() * bucket_list.length)]; + const key = generateS3Key(); + const bucket = bucket_list[Math.floor(Math.random() * bucket_list.length)]; - const payload = generator.genPayload(); - const resp = s3_client.put(bucket, key, payload); - if (!resp.success) { - log.withFields({bucket : bucket, key : key}).error(resp.error); - return; - } + const payload = generator.genPayload(); + const resp = s3_client.put(bucket, key, payload); + if (!resp.success) { + log.withFields({bucket : bucket, key : key}).error(resp.error); + return; + } - if (obj_registry) { - obj_registry.addObject('', '', bucket, key, payload.hash()); - } + if (obj_registry) { + obj_registry.addObject('', '', bucket, key, payload.hash()); + } } export function obj_read() { - if (__ENV.SLEEP_READ) { - sleep(__ENV.SLEEP_READ); + if (__ENV.SLEEP_READ) { + sleep(__ENV.SLEEP_READ); + } + + if (obj_to_read_selector) { + const obj = obj_to_read_selector.nextObject(); + if (!obj) { + return; } - - if (obj_to_read_selector) { - const obj = obj_to_read_selector.nextObject(); - if (!obj ) { - return; - } - const resp = s3_client.get(obj.s3_bucket, obj.s3_key) - if (!resp.success) { - log.withFields({bucket : obj.s3_bucket, key : obj.s3_key, status: obj.status, op: `READ`}) - .error(resp.error); - } else { - obj_registry.setObjectStatus(obj.id, obj.status, 'read'); - } - - return - } - - const obj = obj_list[Math.floor(Math.random() * obj_list.length)]; - - const resp = s3_client.get(obj.bucket, obj.object); + const resp = s3_client.get(obj.s3_bucket, obj.s3_key) if (!resp.success) { - log.withFields({bucket : obj.bucket, key : obj.object}).error(resp.error); - } else { - obj_registry.setObjectStatus(obj.id, obj.status, 'read'); + log.withFields({ + bucket : obj.s3_bucket, + key : obj.s3_key, + status : obj.status, + op : `READ` + }) + .error(resp.error); } + else { + obj_registry.setObjectStatus(obj.id, obj.status, 'read'); + } + + return + } + + const obj = obj_list[Math.floor(Math.random() * obj_list.length)]; + + const resp = s3_client.get(obj.bucket, obj.object); + if (!resp.success) { + log.withFields({bucket : obj.bucket, key : obj.object}).error(resp.error); + } else { + obj_registry.setObjectStatus(obj.id, obj.status, 'read'); + } } export function obj_delete() { - if (__ENV.SLEEP_DELETE) { - sleep(__ENV.SLEEP_DELETE); - } + if (__ENV.SLEEP_DELETE) { + sleep(__ENV.SLEEP_DELETE); + } - const obj = obj_to_delete_selector.nextObject(); - delete_object(obj) + const obj = obj_to_delete_selector.nextObject(); + delete_object(obj) } export function delete_object(obj) { - if (!obj) { - if (obj_to_delete_exit_on_null) { - exec.test.abort("No more objects to select"); - } - return; + if (!obj) { + if (obj_to_delete_exit_on_null) { + exec.test.abort("No more objects to select"); } + return; + } - const resp = s3_client.delete(obj.s3_bucket, obj.s3_key); - if (!resp.success) { - log.withFields({bucket : obj.s3_bucket, key : obj.s3_key, op : 'DELETE'}) - .error(resp.error); - return; - } + const resp = s3_client.delete(obj.s3_bucket, obj.s3_key); + if (!resp.success) { + log.withFields({bucket : obj.s3_bucket, key : obj.s3_key, op : 'DELETE'}) + .error(resp.error); + return; + } - obj_registry.deleteObject(obj.id); + obj_registry.deleteObject(obj.id); } \ No newline at end of file diff --git a/scenarios/s3_multipart.js b/scenarios/s3_multipart.js index 10a6d62..dc6dd01 100644 --- a/scenarios/s3_multipart.js +++ b/scenarios/s3_multipart.js @@ -5,16 +5,16 @@ import registry from 'k6/x/frostfs/registry'; import s3 from 'k6/x/frostfs/s3'; import stats from 'k6/x/frostfs/stats'; -import {generateS3Key} from './libs/keygen.js'; import {newGenerator} from './libs/datagen.js'; import {parseEnv} from './libs/env-parser.js'; import {textSummary} from './libs/k6-summary-0.0.2.js'; +import {generateS3Key} from './libs/keygen.js'; parseEnv(); -const bucket_list = new SharedArray('bucket_list', function() { - return JSON.parse(open(__ENV.PREGEN_JSON)).buckets; -}); +const bucket_list = new SharedArray( + 'bucket_list', + function() { return JSON.parse(open(__ENV.PREGEN_JSON)).buckets; }); const summary_json = __ENV.SUMMARY_JSON || '/tmp/summary.json'; @@ -24,7 +24,7 @@ const s3_endpoint = s3_endpoints[Math.floor(Math.random() * s3_endpoints.length)]; const no_verify_ssl = __ENV.NO_VERIFY_SSL || 'true'; const connection_args = { - no_verify_ssl: no_verify_ssl + no_verify_ssl : no_verify_ssl }; const s3_client = s3.connect(s3_endpoint, connection_args); const log = logging.new().withField('endpoint', s3_endpoint); @@ -55,17 +55,17 @@ const generator = newGenerator(write_vu_count > 0 || write_multipart_vu_count > 0); if (write_vu_count > 0) { scenarios.write_multipart = { - executor: 'constant-vus', - vus: write_vu_count, - duration: `${duration}s`, - exec: 'obj_write_multipart', - gracefulStop: '5s', + executor : 'constant-vus', + vus : write_vu_count, + duration : `${duration}s`, + exec : 'obj_write_multipart', + gracefulStop : '5s', }; } export const options = { scenarios, - setupTimeout: '5s', + setupTimeout : '5s', }; export function setup() { @@ -85,8 +85,8 @@ export function teardown(data) { export function handleSummary(data) { return { - 'stdout': textSummary(data, {indent: ' ', enableColors: false}), - [summary_json]: JSON.stringify(data), + 'stdout' : textSummary(data, {indent : ' ', enableColors : false}), + [summary_json] : JSON.stringify(data), }; } @@ -105,11 +105,10 @@ export function obj_write_multipart() { const bucket = bucket_list[Math.floor(Math.random() * bucket_list.length)]; const payload = generator.genPayload(); - const resp = s3_client.multipart( - bucket, key, write_multipart_part_size, write_multipart_vu_count, - payload); + const resp = s3_client.multipart(bucket, key, write_multipart_part_size, + write_multipart_vu_count, payload); if (!resp.success) { - log.withFields({bucket: bucket, key: key}).error(resp.error); + log.withFields({bucket : bucket, key : key}).error(resp.error); return; } diff --git a/scenarios/s3local.js b/scenarios/s3local.js index f17a556..b0fa174 100644 --- a/scenarios/s3local.js +++ b/scenarios/s3local.js @@ -5,25 +5,25 @@ import registry from 'k6/x/frostfs/registry'; import s3local from 'k6/x/frostfs/s3local'; import stats from 'k6/x/frostfs/stats'; -import {generateS3Key} from './libs/keygen.js'; import {newGenerator} from './libs/datagen.js'; import {parseEnv} from './libs/env-parser.js'; import {textSummary} from './libs/k6-summary-0.0.2.js'; import {uuidv4} from './libs/k6-utils-1.4.0.js'; +import {generateS3Key} from './libs/keygen.js'; parseEnv(); -const obj_list = new SharedArray('obj_list', function() { - return JSON.parse(open(__ENV.PREGEN_JSON)).objects; -}); +const obj_list = new SharedArray( + 'obj_list', + function() { return JSON.parse(open(__ENV.PREGEN_JSON)).objects; }); -const container_list = new SharedArray('container_list', function() { - return JSON.parse(open(__ENV.PREGEN_JSON)).containers; -}); +const container_list = new SharedArray( + 'container_list', + function() { return JSON.parse(open(__ENV.PREGEN_JSON)).containers; }); -const bucket_list = new SharedArray('bucket_list', function() { - return JSON.parse(open(__ENV.PREGEN_JSON)).buckets; -}); +const bucket_list = new SharedArray( + 'bucket_list', + function() { return JSON.parse(open(__ENV.PREGEN_JSON)).buckets; }); function bucket_mapping() { if (container_list.length != bucket_list.length) { @@ -43,13 +43,12 @@ const config_file = __ENV.CONFIG_FILE; const config_dir = __ENV.CONFIG_DIR; const max_total_size_gb = __ENV.MAX_TOTAL_SIZE_GB ? parseInt(__ENV.MAX_TOTAL_SIZE_GB) : 0; -const s3_client = s3local.connect( - config_file, config_dir, { - 'debug_logger': __ENV.DEBUG_LOGGER || 'false', - }, - bucket_mapping(), max_total_size_gb); +const s3_client = s3local.connect(config_file, config_dir, { + 'debug_logger' : __ENV.DEBUG_LOGGER || 'false', +}, + bucket_mapping(), max_total_size_gb); const log = logging.new().withFields( - {'config_file': config_file, 'config_dir': config_dir}); + {'config_file' : config_file, 'config_dir' : config_dir}); if (!!__ENV.METRIC_TAGS) { stats.setTags(__ENV.METRIC_TAGS) @@ -64,7 +63,7 @@ if (registry_enabled) { obj_to_read_selector = registry.getLoopedSelector( __ENV.REGISTRY_FILE, 'obj_to_read', __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, { - status: 'created', + status : 'created', }) } @@ -76,28 +75,28 @@ const write_vu_count = parseInt(__ENV.WRITERS || '0'); const generator = newGenerator(write_vu_count > 0); if (write_vu_count > 0) { scenarios.write = { - executor: 'constant-vus', - vus: write_vu_count, - duration: `${duration}s`, - exec: 'obj_write', - gracefulStop: '5s', + executor : 'constant-vus', + vus : write_vu_count, + duration : `${duration}s`, + exec : 'obj_write', + gracefulStop : '5s', }; } const read_vu_count = parseInt(__ENV.READERS || '0'); if (read_vu_count > 0) { scenarios.read = { - executor: 'constant-vus', - vus: read_vu_count, - duration: `${duration}s`, - exec: 'obj_read', - gracefulStop: '5s', + executor : 'constant-vus', + vus : read_vu_count, + duration : `${duration}s`, + exec : 'obj_read', + gracefulStop : '5s', }; } export const options = { scenarios, - setupTimeout: '5s', + setupTimeout : '5s', }; export function setup() { @@ -126,8 +125,8 @@ export function teardown(data) { export function handleSummary(data) { return { - 'stdout': textSummary(data, {indent: ' ', enableColors: false}), - [summary_json]: JSON.stringify(data), + 'stdout' : textSummary(data, {indent : ' ', enableColors : false}), + [summary_json] : JSON.stringify(data), }; } @@ -141,7 +140,7 @@ export function obj_write() { if (resp.abort) { exec.test.abort(resp.error); } - log.withFields({bucket: bucket, key: key}).error(resp.error); + log.withFields({bucket : bucket, key : key}).error(resp.error); return; } @@ -158,7 +157,7 @@ export function obj_read() { } const resp = s3_client.get(obj.s3_bucket, obj.s3_key) if (!resp.success) { - log.withFields({bucket: obj.s3_bucket, key: obj.s3_key}) + log.withFields({bucket : obj.s3_bucket, key : obj.s3_key}) .error(resp.error); } return @@ -168,6 +167,6 @@ export function obj_read() { const resp = s3_client.get(obj.bucket, obj.object); if (!resp.success) { - log.withFields({bucket: obj.bucket, key: obj.object}).error(resp.error); + log.withFields({bucket : obj.bucket, key : obj.object}).error(resp.error); } } diff --git a/scenarios/verify.js b/scenarios/verify.js index 15a0055..1acb854 100644 --- a/scenarios/verify.js +++ b/scenarios/verify.js @@ -1,13 +1,13 @@ -import { sleep } from 'k6'; -import { Counter } from 'k6/metrics'; +import {sleep} from 'k6'; +import {Counter} from 'k6/metrics'; import logging from 'k6/x/frostfs/logging'; import native from 'k6/x/frostfs/native'; import registry from 'k6/x/frostfs/registry'; import s3 from 'k6/x/frostfs/s3'; import stats from 'k6/x/frostfs/stats'; -import { parseEnv } from './libs/env-parser.js'; -import { textSummary } from './libs/k6-summary-0.0.2.js'; +import {parseEnv} from './libs/env-parser.js'; +import {textSummary} from './libs/k6-summary-0.0.2.js'; parseEnv(); @@ -23,9 +23,9 @@ const summary_json = __ENV.SUMMARY_JSON || '/tmp/summary.json'; // This allows to run this scenario multiple times and collect overall // statistics in the final run. const obj_counters = { - verified: new Counter('verified_obj'), - skipped: new Counter('skipped_obj'), - invalid: new Counter('invalid_obj'), + verified : new Counter('verified_obj'), + skipped : new Counter('skipped_obj'), + invalid : new Counter('invalid_obj'), }; let log = logging.new(); @@ -39,23 +39,24 @@ let grpc_client = undefined; if (__ENV.GRPC_ENDPOINTS) { const grpcEndpoints = __ENV.GRPC_ENDPOINTS.split(','); const grpcEndpoint = - grpcEndpoints[Math.floor(Math.random() * grpcEndpoints.length)]; + grpcEndpoints[Math.floor(Math.random() * grpcEndpoints.length)]; log = log.withField('endpoint', grpcEndpoint); grpc_client = native.connect( - grpcEndpoint, '', __ENV.DIAL_TIMEOUT ? parseInt(__ENV.DIAL_TIMEOUT) : 0, - __ENV.STREAM_TIMEOUT ? parseInt(__ENV.STREAM_TIMEOUT) : 0, - __ENV.PREPARE_LOCALLY ? __ENV.PREPARE_LOCALLY.toLowerCase() === 'true' : false, - 1024 * parseInt(__ENV.MAX_OBJECT_SIZE || '0')); + grpcEndpoint, '', __ENV.DIAL_TIMEOUT ? parseInt(__ENV.DIAL_TIMEOUT) : 0, + __ENV.STREAM_TIMEOUT ? parseInt(__ENV.STREAM_TIMEOUT) : 0, + __ENV.PREPARE_LOCALLY ? __ENV.PREPARE_LOCALLY.toLowerCase() === 'true' + : false, + 1024 * parseInt(__ENV.MAX_OBJECT_SIZE || '0')); } // Connect to random S3 endpoint let s3_client = undefined; if (__ENV.S3_ENDPOINTS) { const no_verify_ssl = __ENV.NO_VERIFY_SSL || 'true'; - const connection_args = { no_verify_ssl: no_verify_ssl }; + const connection_args = {no_verify_ssl : no_verify_ssl}; const s3_endpoints = __ENV.S3_ENDPOINTS.split(','); const s3_endpoint = - s3_endpoints[Math.floor(Math.random() * s3_endpoints.length)]; + s3_endpoints[Math.floor(Math.random() * s3_endpoints.length)]; log = log.withField('endpoint', s3_endpoint); s3_client = s3.connect(s3_endpoint, connection_args); } @@ -64,10 +65,10 @@ if (__ENV.S3_ENDPOINTS) { // execute as many iterations as there are objects. Each object will have 3 // retries to be verified const obj_to_verify_selector = registry.getSelector( - __ENV.REGISTRY_FILE, 'obj_to_verify', - __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, { - status: 'created', -}); + __ENV.REGISTRY_FILE, 'obj_to_verify', + __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, { + status : 'created', + }); const obj_to_verify_count = obj_to_verify_selector.count(); // Execute at least one iteration (executor shared-iterations can't run 0 // iterations) @@ -77,35 +78,35 @@ const iterations = Math.max(1, obj_to_verify_count); const vus = Math.min(__ENV.CLIENTS, iterations); const scenarios = { - verify: { - executor: 'shared-iterations', + verify : { + executor : 'shared-iterations', vus, iterations, - maxDuration: `${time_limit}s`, - exec: 'obj_verify', - gracefulStop: '5s', + maxDuration : `${time_limit}s`, + exec : 'obj_verify', + gracefulStop : '5s', } }; export const options = { scenarios, - setupTimeout: '5s', + setupTimeout : '5s', }; export function setup() { // Populate counters with initial values for (const [status, counter] of Object.entries(obj_counters)) { const obj_selector = registry.getSelector( - __ENV.REGISTRY_FILE, status, - __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, { status }); + __ENV.REGISTRY_FILE, status, + __ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0, {status}); counter.add(obj_selector.count()); } } export function handleSummary(data) { return { - 'stdout': textSummary(data, { indent: ' ', enableColors: false }), - [summary_json]: JSON.stringify(data), + 'stdout' : textSummary(data, {indent : ' ', enableColors : false}), + [summary_json] : JSON.stringify(data), }; } @@ -137,19 +138,19 @@ function verify_object_with_retries(obj, attempts) { // ReferenceError: Cannot access a variable before initialization. let lg = log; if (obj.c_id && obj.o_id) { - lg = lg.withFields({ cid: obj.c_id, oid: obj.o_id }); + lg = lg.withFields({cid : obj.c_id, oid : obj.o_id}); result = grpc_client.verifyHash(obj.c_id, obj.o_id, obj.payload_hash); } else if (obj.s3_bucket && obj.s3_key) { - lg = lg.withFields({ bucket: obj.s3_bucket, key: obj.s3_key }); + lg = lg.withFields({bucket : obj.s3_bucket, key : obj.s3_key}); result = - s3_client.verifyHash(obj.s3_bucket, obj.s3_key, obj.payload_hash); + s3_client.verifyHash(obj.s3_bucket, obj.s3_key, obj.payload_hash); } else { lg.withFields({ - cid: obj.c_id, - oid: obj.o_id, - bucket: obj.s3_bucket, - key: obj.s3_key - }).warn(`Object cannot be verified with supported protocols`); + cid : obj.c_id, + oid : obj.o_id, + bucket : obj.s3_bucket, + key : obj.s3_key + }).warn(`Object cannot be verified with supported protocols`); return 'skipped'; }