[#19] Add json output for k6 scenarios

This is required for autotests to be able to parse summaries and do checks

Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
This commit is contained in:
Andrey Berezin 2023-03-01 11:32:13 +03:00 committed by fyrchik
parent bed807b09f
commit 614bee3581
5 changed files with 513 additions and 0 deletions

View file

@ -4,6 +4,7 @@ import logging from 'k6/x/frostfs/logging';
import registry from 'k6/x/frostfs/registry'; import registry from 'k6/x/frostfs/registry';
import { SharedArray } from 'k6/data'; import { SharedArray } from 'k6/data';
import { sleep } from 'k6'; import { sleep } from 'k6';
import { textSummary } from './libs/k6-summary-0.0.2.js';
const obj_list = new SharedArray('obj_list', function () { const obj_list = new SharedArray('obj_list', function () {
return JSON.parse(open(__ENV.PREGEN_JSON)).objects; return JSON.parse(open(__ENV.PREGEN_JSON)).objects;
@ -14,6 +15,7 @@ const container_list = new SharedArray('container_list', function () {
}); });
const read_size = JSON.parse(open(__ENV.PREGEN_JSON)).obj_size; const read_size = JSON.parse(open(__ENV.PREGEN_JSON)).obj_size;
const summary_json = __ENV.SUMMARY_JSON || "/tmp/summary.json";
// Select random gRPC endpoint for current VU // Select random gRPC endpoint for current VU
const grpc_endpoints = __ENV.GRPC_ENDPOINTS.split(','); const grpc_endpoints = __ENV.GRPC_ENDPOINTS.split(',');
@ -105,6 +107,13 @@ export function teardown(data) {
} }
} }
export function handleSummary(data) {
return {
'stdout': textSummary(data, { indent: ' ', enableColors: false }),
[summary_json]: JSON.stringify(data),
};
}
export function obj_write() { export function obj_write() {
if (__ENV.SLEEP_WRITE) { if (__ENV.SLEEP_WRITE) {
sleep(__ENV.SLEEP_WRITE); sleep(__ENV.SLEEP_WRITE);

View file

@ -3,6 +3,7 @@ import registry from 'k6/x/frostfs/registry';
import http from 'k6/http'; import http from 'k6/http';
import { SharedArray } from 'k6/data'; import { SharedArray } from 'k6/data';
import { sleep } from 'k6'; import { sleep } from 'k6';
import { textSummary } from './libs/k6-summary-0.0.2.js';
const obj_list = new SharedArray('obj_list', function () { const obj_list = new SharedArray('obj_list', function () {
return JSON.parse(open(__ENV.PREGEN_JSON)).objects; return JSON.parse(open(__ENV.PREGEN_JSON)).objects;
@ -13,6 +14,7 @@ const container_list = new SharedArray('container_list', function () {
}); });
const read_size = JSON.parse(open(__ENV.PREGEN_JSON)).obj_size; const read_size = JSON.parse(open(__ENV.PREGEN_JSON)).obj_size;
const summary_json = __ENV.SUMMARY_JSON || "/tmp/summary.json";
// Select random HTTP endpoint for current VU // Select random HTTP endpoint for current VU
const http_endpoints = __ENV.HTTP_ENDPOINTS.split(','); const http_endpoints = __ENV.HTTP_ENDPOINTS.split(',');
@ -72,6 +74,13 @@ export function teardown(data) {
} }
} }
export function handleSummary(data) {
return {
'stdout': textSummary(data, { indent: ' ', enableColors: false }),
[summary_json]: JSON.stringify(data),
};
}
export function obj_write() { export function obj_write() {
if (__ENV.SLEEP_WRITE) { if (__ENV.SLEEP_WRITE) {
sleep(__ENV.SLEEP_WRITE); sleep(__ENV.SLEEP_WRITE);

View file

@ -0,0 +1,477 @@
var forEach = function (obj, callback) {
for (var key in obj) {
if (obj.hasOwnProperty(key)) {
if (callback(key, obj[key])) {
break
}
}
}
}
var palette = {
bold: 1,
faint: 2,
red: 31,
green: 32,
cyan: 36,
//TODO: add others?
}
var groupPrefix = '█'
var detailsPrefix = '↳'
var succMark = '✓'
var failMark = '✗'
var defaultOptions = {
indent: ' ',
enableColors: true,
summaryTimeUnit: null,
summaryTrendStats: null,
}
// strWidth tries to return the actual width the string will take up on the
// screen, without any terminal formatting, unicode ligatures, etc.
function strWidth(s) {
// TODO: determine if NFC or NFKD are not more appropriate? or just give up? https://hsivonen.fi/string-length/
var data = s.normalize('NFKC') // This used to be NFKD in Go, but this should be better
var inEscSeq = false
var inLongEscSeq = false
var width = 0
for (var char of data) {
if (char.done) {
break
}
// Skip over ANSI escape codes.
if (char == '\x1b') {
inEscSeq = true
continue
}
if (inEscSeq && char == '[') {
inLongEscSeq = true
continue
}
if (inEscSeq && inLongEscSeq && char.charCodeAt(0) >= 0x40 && char.charCodeAt(0) <= 0x7e) {
inEscSeq = false
inLongEscSeq = false
continue
}
if (inEscSeq && !inLongEscSeq && char.charCodeAt(0) >= 0x40 && char.charCodeAt(0) <= 0x5f) {
inEscSeq = false
continue
}
if (!inEscSeq && !inLongEscSeq) {
width++
}
}
return width
}
function summarizeCheck(indent, check, decorate) {
if (check.fails == 0) {
return decorate(indent + succMark + ' ' + check.name, palette.green)
}
var succPercent = Math.floor((100 * check.passes) / (check.passes + check.fails))
return decorate(
indent +
failMark +
' ' +
check.name +
'\n' +
indent +
' ' +
detailsPrefix +
' ' +
succPercent +
'% — ' +
succMark +
' ' +
check.passes +
' / ' +
failMark +
' ' +
check.fails,
palette.red
)
}
function summarizeGroup(indent, group, decorate) {
var result = []
if (group.name != '') {
result.push(indent + groupPrefix + ' ' + group.name + '\n')
indent = indent + ' '
}
for (var i = 0; i < group.checks.length; i++) {
result.push(summarizeCheck(indent, group.checks[i], decorate))
}
if (group.checks.length > 0) {
result.push('')
}
for (var i = 0; i < group.groups.length; i++) {
Array.prototype.push.apply(result, summarizeGroup(indent, group.groups[i], decorate))
}
return result
}
function displayNameForMetric(name) {
var subMetricPos = name.indexOf('{')
if (subMetricPos >= 0) {
return '{ ' + name.substring(subMetricPos + 1, name.length - 1) + ' }'
}
return name
}
function indentForMetric(name) {
if (name.indexOf('{') >= 0) {
return ' '
}
return ''
}
function humanizeBytes(bytes) {
var units = ['B', 'kB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB']
var base = 1000
if (bytes < 10) {
return bytes + ' B'
}
var e = Math.floor(Math.log(bytes) / Math.log(base))
var suffix = units[e | 0]
var val = Math.floor((bytes / Math.pow(base, e)) * 10 + 0.5) / 10
return val.toFixed(val < 10 ? 1 : 0) + ' ' + suffix
}
var unitMap = {
s: { unit: 's', coef: 0.001 },
ms: { unit: 'ms', coef: 1 },
us: { unit: 'µs', coef: 1000 },
}
function toFixedNoTrailingZeros(val, prec) {
// TODO: figure out something better?
return parseFloat(val.toFixed(prec)).toString()
}
function toFixedNoTrailingZerosTrunc(val, prec) {
var mult = Math.pow(10, prec)
return toFixedNoTrailingZeros(Math.trunc(mult * val) / mult, prec)
}
function humanizeGenericDuration(dur) {
if (dur === 0) {
return '0s'
}
if (dur < 0.001) {
// smaller than a microsecond, print nanoseconds
return Math.trunc(dur * 1000000) + 'ns'
}
if (dur < 1) {
// smaller than a millisecond, print microseconds
return toFixedNoTrailingZerosTrunc(dur * 1000, 2) + 'µs'
}
if (dur < 1000) {
// duration is smaller than a second
return toFixedNoTrailingZerosTrunc(dur, 2) + 'ms'
}
var result = toFixedNoTrailingZerosTrunc((dur % 60000) / 1000, dur > 60000 ? 0 : 2) + 's'
var rem = Math.trunc(dur / 60000)
if (rem < 1) {
// less than a minute
return result
}
result = (rem % 60) + 'm' + result
rem = Math.trunc(rem / 60)
if (rem < 1) {
// less than an hour
return result
}
return rem + 'h' + result
}
function humanizeDuration(dur, timeUnit) {
if (timeUnit !== '' && unitMap.hasOwnProperty(timeUnit)) {
return (dur * unitMap[timeUnit].coef).toFixed(2) + unitMap[timeUnit].unit
}
return humanizeGenericDuration(dur)
}
function humanizeValue(val, metric, timeUnit) {
if (metric.type == 'rate') {
// Truncate instead of round when decreasing precision to 2 decimal places
return (Math.trunc(val * 100 * 100) / 100).toFixed(2) + '%'
}
switch (metric.contains) {
case 'data':
return humanizeBytes(val)
case 'time':
return humanizeDuration(val, timeUnit)
default:
return toFixedNoTrailingZeros(val, 6)
}
}
function nonTrendMetricValueForSum(metric, timeUnit) {
switch (metric.type) {
case 'counter':
return [
humanizeValue(metric.values.count, metric, timeUnit),
humanizeValue(metric.values.rate, metric, timeUnit) + '/s',
]
case 'gauge':
return [
humanizeValue(metric.values.value, metric, timeUnit),
'min=' + humanizeValue(metric.values.min, metric, timeUnit),
'max=' + humanizeValue(metric.values.max, metric, timeUnit),
]
case 'rate':
return [
humanizeValue(metric.values.rate, metric, timeUnit),
succMark + ' ' + metric.values.passes,
failMark + ' ' + metric.values.fails,
]
default:
return ['[no data]']
}
}
function summarizeMetrics(options, data, decorate) {
var indent = options.indent + ' '
var result = []
var names = []
var nameLenMax = 0
var nonTrendValues = {}
var nonTrendValueMaxLen = 0
var nonTrendExtras = {}
var nonTrendExtraMaxLens = [0, 0]
var trendCols = {}
var numTrendColumns = options.summaryTrendStats.length
var trendColMaxLens = new Array(numTrendColumns).fill(0)
forEach(data.metrics, function (name, metric) {
names.push(name)
// When calculating widths for metrics, account for the indentation on submetrics.
var displayName = indentForMetric(name) + displayNameForMetric(name)
var displayNameWidth = strWidth(displayName)
if (displayNameWidth > nameLenMax) {
nameLenMax = displayNameWidth
}
if (metric.type == 'trend') {
var cols = []
for (var i = 0; i < numTrendColumns; i++) {
var tc = options.summaryTrendStats[i]
var value = metric.values[tc]
if (tc === 'count') {
value = value.toString()
} else {
value = humanizeValue(value, metric, options.summaryTimeUnit)
}
var valLen = strWidth(value)
if (valLen > trendColMaxLens[i]) {
trendColMaxLens[i] = valLen
}
cols[i] = value
}
trendCols[name] = cols
return
}
var values = nonTrendMetricValueForSum(metric, options.summaryTimeUnit)
nonTrendValues[name] = values[0]
var valueLen = strWidth(values[0])
if (valueLen > nonTrendValueMaxLen) {
nonTrendValueMaxLen = valueLen
}
nonTrendExtras[name] = values.slice(1)
for (var i = 1; i < values.length; i++) {
var extraLen = strWidth(values[i])
if (extraLen > nonTrendExtraMaxLens[i - 1]) {
nonTrendExtraMaxLens[i - 1] = extraLen
}
}
})
// sort all metrics but keep sub metrics grouped with their parent metrics
names.sort(function (metric1, metric2) {
var parent1 = metric1.split('{', 1)[0]
var parent2 = metric2.split('{', 1)[0]
var result = parent1.localeCompare(parent2)
if (result !== 0) {
return result
}
var sub1 = metric1.substring(parent1.length)
var sub2 = metric2.substring(parent2.length)
return sub1.localeCompare(sub2)
})
var getData = function (name) {
if (trendCols.hasOwnProperty(name)) {
var cols = trendCols[name]
var tmpCols = new Array(numTrendColumns)
for (var i = 0; i < cols.length; i++) {
tmpCols[i] =
options.summaryTrendStats[i] +
'=' +
decorate(cols[i], palette.cyan) +
' '.repeat(trendColMaxLens[i] - strWidth(cols[i]))
}
return tmpCols.join(' ')
}
var value = nonTrendValues[name]
var fmtData = decorate(value, palette.cyan) + ' '.repeat(nonTrendValueMaxLen - strWidth(value))
var extras = nonTrendExtras[name]
if (extras.length == 1) {
fmtData = fmtData + ' ' + decorate(extras[0], palette.cyan, palette.faint)
} else if (extras.length > 1) {
var parts = new Array(extras.length)
for (var i = 0; i < extras.length; i++) {
parts[i] =
decorate(extras[i], palette.cyan, palette.faint) +
' '.repeat(nonTrendExtraMaxLens[i] - strWidth(extras[i]))
}
fmtData = fmtData + ' ' + parts.join(' ')
}
return fmtData
}
for (var name of names) {
var metric = data.metrics[name]
var mark = ' '
var markColor = function (text) {
return text
} // noop
if (metric.thresholds) {
mark = succMark
markColor = function (text) {
return decorate(text, palette.green)
}
forEach(metric.thresholds, function (name, threshold) {
if (!threshold.ok) {
mark = failMark
markColor = function (text) {
return decorate(text, palette.red)
}
return true // break
}
})
}
var fmtIndent = indentForMetric(name)
var fmtName = displayNameForMetric(name)
fmtName =
fmtName +
decorate(
'.'.repeat(nameLenMax - strWidth(fmtName) - strWidth(fmtIndent) + 3) + ':',
palette.faint
)
result.push(indent + fmtIndent + markColor(mark) + ' ' + fmtName + ' ' + getData(name))
}
return result
}
function generateTextSummary(data, options) {
var mergedOpts = Object.assign({}, defaultOptions, data.options, options)
var lines = []
// TODO: move all of these functions into an object with methods?
var decorate = function (text) {
return text
}
if (mergedOpts.enableColors) {
decorate = function (text, color /*, ...rest*/) {
var result = '\x1b[' + color
for (var i = 2; i < arguments.length; i++) {
result += ';' + arguments[i]
}
return result + 'm' + text + '\x1b[0m'
}
}
Array.prototype.push.apply(
lines,
summarizeGroup(mergedOpts.indent + ' ', data.root_group, decorate)
)
Array.prototype.push.apply(lines, summarizeMetrics(mergedOpts, data, decorate))
return lines.join('\n')
}
exports.humanizeValue = humanizeValue
exports.textSummary = generateTextSummary
var replacements = {
'&': '&amp;',
'<': '&lt;',
'>': '&gt;',
"'": '&#39;',
'"': '&quot;',
}
function escapeHTML(str) {
// TODO: something more robust?
return str.replace(/[&<>'"]/g, function (char) {
return replacements[char]
})
}
function generateJUnitXML(data, options) {
var failures = 0
var cases = []
forEach(data.metrics, function (metricName, metric) {
if (!metric.thresholds) {
return
}
forEach(metric.thresholds, function (thresholdName, threshold) {
if (threshold.ok) {
cases.push(
'<testcase name="' + escapeHTML(metricName) + ' - ' + escapeHTML(thresholdName) + '" />'
)
} else {
failures++
cases.push(
'<testcase name="' +
escapeHTML(metricName) +
' - ' +
escapeHTML(thresholdName) +
'"><failure message="failed" /></testcase>'
)
}
})
})
var name = options && options.name ? escapeHTML(options.name) : 'k6 thresholds'
return (
'<?xml version="1.0"?>\n<testsuites tests="' +
cases.length +
'" failures="' +
failures +
'">\n' +
'<testsuite name="' +
name +
'" tests="' +
cases.length +
'" failures="' +
failures +
'">' +
cases.join('\n') +
'\n</testsuite >\n</testsuites >'
)
}
exports.jUnit = generateJUnitXML

View file

@ -3,6 +3,7 @@ import registry from 'k6/x/frostfs/registry';
import s3 from 'k6/x/frostfs/s3'; import s3 from 'k6/x/frostfs/s3';
import { SharedArray } from 'k6/data'; import { SharedArray } from 'k6/data';
import { sleep } from 'k6'; import { sleep } from 'k6';
import { textSummary } from './libs/k6-summary-0.0.2.js';
const obj_list = new SharedArray('obj_list', function () { const obj_list = new SharedArray('obj_list', function () {
return JSON.parse(open(__ENV.PREGEN_JSON)).objects; return JSON.parse(open(__ENV.PREGEN_JSON)).objects;
@ -13,6 +14,7 @@ const bucket_list = new SharedArray('bucket_list', function () {
}); });
const read_size = JSON.parse(open(__ENV.PREGEN_JSON)).obj_size; const read_size = JSON.parse(open(__ENV.PREGEN_JSON)).obj_size;
const summary_json = __ENV.SUMMARY_JSON || "/tmp/summary.json";
// Select random S3 endpoint for current VU // Select random S3 endpoint for current VU
const s3_endpoints = __ENV.S3_ENDPOINTS.split(','); const s3_endpoints = __ENV.S3_ENDPOINTS.split(',');
@ -103,6 +105,13 @@ export function teardown(data) {
} }
} }
export function handleSummary(data) {
return {
'stdout': textSummary(data, { indent: ' ', enableColors: false }),
[summary_json]: JSON.stringify(data),
};
}
export function obj_write() { export function obj_write() {
if (__ENV.SLEEP_WRITE) { if (__ENV.SLEEP_WRITE) {
sleep(__ENV.SLEEP_WRITE); sleep(__ENV.SLEEP_WRITE);

View file

@ -3,11 +3,13 @@ import registry from 'k6/x/frostfs/registry';
import s3 from 'k6/x/frostfs/s3'; import s3 from 'k6/x/frostfs/s3';
import { sleep } from 'k6'; import { sleep } from 'k6';
import { Counter } from 'k6/metrics'; import { Counter } from 'k6/metrics';
import { textSummary } from './libs/k6-summary-0.0.2.js';
const obj_registry = registry.open(__ENV.REGISTRY_FILE); const obj_registry = registry.open(__ENV.REGISTRY_FILE);
// Time limit (in seconds) for the run // Time limit (in seconds) for the run
const time_limit = __ENV.TIME_LIMIT || "60"; const time_limit = __ENV.TIME_LIMIT || "60";
const summary_json = __ENV.SUMMARY_JSON || "/tmp/summary.json";
// Number of objects in each status. These counters are cumulative in a // Number of objects in each status. These counters are cumulative in a
// sense that they reflect total number of objects in the registry, not just // sense that they reflect total number of objects in the registry, not just
@ -80,6 +82,13 @@ export function setup() {
} }
} }
export function handleSummary(data) {
return {
'stdout': textSummary(data, { indent: ' ', enableColors: false }),
[summary_json]: JSON.stringify(data),
};
}
export function obj_verify() { export function obj_verify() {
if (__ENV.SLEEP) { if (__ENV.SLEEP) {
sleep(__ENV.SLEEP); sleep(__ENV.SLEEP);