TrueCloudLab/xk6-frostfs
42
1
Fork 20
Code Issues 33 Pull requests 1 Releases Activity Actions

Prepare k6 to export metrics #120

Merged
fyrchik merged 3 commits from dstepanov-yadro/xk6-frostfs:feat/metrics into master 2024-09-04 19:51:18 +00:00
Conversation 1 Commits 3 Files changed 18 +205 -83
18 changed files with 205 additions and 83 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
internal/local/local.go
View file

@ -60,9 +60,9 @@ var (
_ modules.Module = &RootModule{}
_ modules.Instance = &Local{}
objPutTotal, objPutFails, objPutDuration *metrics.Metric
objGetTotal, objGetFails, objGetDuration *metrics.Metric
objDeleteTotal, objDeleteFails, objDeleteDuration *metrics.Metric
objPutSuccess, objPutFails, objPutDuration, objPutData *metrics.Metric
objGetSuccess, objGetFails, objGetDuration, objGetData *metrics.Metric
objDeleteSuccess, objDeleteFails, objDeleteDuration *metrics.Metric
)
func init() {
@ -173,18 +173,19 @@ func (s *Local) Connect(configFile, configDir, hexKey string, debug bool, maxSiz
}
// Register metrics.
registry := metrics.NewRegistry()
objPutTotal, _ = registry.NewMetric("local_obj_put_total", metrics.Counter)
objPutFails, _ = registry.NewMetric("local_obj_put_fails", metrics.Counter)
objPutDuration, _ = registry.NewMetric("local_obj_put_duration", metrics.Trend, metrics.Time)
objPutSuccess, _ = stats.Registry.NewMetric("local_obj_put_success", metrics.Counter)
objPutFails, _ = stats.Registry.NewMetric("local_obj_put_fails", metrics.Counter)
objPutDuration, _ = stats.Registry.NewMetric("local_obj_put_duration", metrics.Trend, metrics.Time)
objPutData, _ = stats.Registry.NewMetric("local_obj_put_bytes", metrics.Counter, metrics.Data)
objGetTotal, _ = registry.NewMetric("local_obj_get_total", metrics.Counter)
objGetFails, _ = registry.NewMetric("local_obj_get_fails", metrics.Counter)
objGetDuration, _ = registry.NewMetric("local_obj_get_duration", metrics.Trend, metrics.Time)
objGetSuccess, _ = stats.Registry.NewMetric("local_obj_get_success", metrics.Counter)
objGetFails, _ = stats.Registry.NewMetric("local_obj_get_fails", metrics.Counter)
objGetDuration, _ = stats.Registry.NewMetric("local_obj_get_duration", metrics.Trend, metrics.Time)
objGetData, _ = stats.Registry.NewMetric("local_obj_get_bytes", metrics.Counter, metrics.Data)
objDeleteTotal, _ = registry.NewMetric("local_obj_delete_total", metrics.Counter)
objDeleteFails, _ = registry.NewMetric("local_obj_delete_fails", metrics.Counter)
objDeleteDuration, _ = registry.NewMetric("local_obj_delete_duration", metrics.Trend, metrics.Time)
objDeleteSuccess, _ = stats.Registry.NewMetric("local_obj_delete_success", metrics.Counter)
objDeleteFails, _ = stats.Registry.NewMetric("local_obj_delete_fails", metrics.Counter)
objDeleteDuration, _ = stats.Registry.NewMetric("local_obj_delete_duration", metrics.Trend, metrics.Time)
// Create raw client backed by local storage engine.
rc := rawclient.New(ng,
@ -193,25 +194,27 @@ func (s *Local) Connect(configFile, configDir, hexKey string, debug bool, maxSiz
if err != nil {
stats.Report(s.vu, objPutFails, 1)
} else {
stats.Report(s.vu, objPutTotal, 1)
stats.Report(s.vu, objPutSuccess, 1)
stats.ReportDataSent(s.vu, float64(sz))
stats.Report(s.vu, objPutDuration, metrics.D(dt))
stats.Report(s.vu, objPutData, float64(sz))
}
}),
rawclient.WithGetHandler(func(sz uint64, err error, dt time.Duration) {
if err != nil {
stats.Report(s.vu, objGetFails, 1)
} else {
stats.Report(s.vu, objGetTotal, 1)
stats.Report(s.vu, objGetSuccess, 1)
stats.Report(s.vu, objGetDuration, metrics.D(dt))
stats.ReportDataReceived(s.vu, float64(sz))
stats.Report(s.vu, objGetData, float64(sz))
}
}),
rawclient.WithDeleteHandler(func(err error, dt time.Duration) {
if err != nil {
stats.Report(s.vu, objDeleteFails, 1)
} else {
stats.Report(s.vu, objDeleteTotal, 1)
stats.Report(s.vu, objDeleteSuccess, 1)
stats.Report(s.vu, objDeleteDuration, metrics.D(dt))
}
}),

8
internal/native/client.go
View file

@ -137,7 +137,7 @@ func (c *Client) Delete(containerID string, objectID string) DeleteResponse {
return DeleteResponse{Success: false, Error: err.Error()}
}
stats.Report(c.vu, objDeleteTotal, 1)
stats.Report(c.vu, objDeleteSuccess, 1)
stats.Report(c.vu, objDeleteDuration, metrics.D(time.Since(start)))
return DeleteResponse{Success: true}
}
@ -171,9 +171,10 @@ func (c *Client) Get(containerID, objectID string) GetResponse {
return GetResponse{Success: false, Error: err.Error()}
}
stats.Report(c.vu, objGetTotal, 1)
stats.Report(c.vu, objGetSuccess, 1)
stats.Report(c.vu, objGetDuration, metrics.D(time.Since(start)))
stats.ReportDataReceived(c.vu, float64(objSize))
stats.Report(c.vu, objGetData, float64(objSize))
return GetResponse{Success: true}
}
@ -468,9 +469,10 @@ func put(vu modules.VU, cli *client.Client, prepareLocally bool, tok *session.Ob
return nil, err
}
stats.Report(vu, objPutTotal, 1)
stats.Report(vu, objPutSuccess, 1)
stats.ReportDataSent(vu, float64(sz))
stats.Report(vu, objPutDuration, metrics.D(time.Since(start)))
stats.Report(vu, objPutData, float64(sz))
return resp, nil
}

37
internal/native/native.go
View file

@ -8,6 +8,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
"git.frostfs.info/TrueCloudLab/xk6-frostfs/internal/stats"
"github.com/google/uuid"
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
"go.k6.io/k6/js/modules"
@ -28,10 +29,10 @@ var (
_ modules.Instance = &Native{}
_ modules.Module = &RootModule{}
objPutTotal, objPutFails, objPutDuration *metrics.Metric
objGetTotal, objGetFails, objGetDuration *metrics.Metric
objDeleteTotal, objDeleteFails, objDeleteDuration *metrics.Metric
cnrPutTotal, cnrPutFails, cnrPutDuration *metrics.Metric
objPutSuccess, objPutFails, objPutDuration, objPutData *metrics.Metric
objGetSuccess, objGetFails, objGetDuration, objGetData *metrics.Metric
objDeleteSuccess, objDeleteFails, objDeleteDuration *metrics.Metric
cnrPutTotal, cnrPutFails, cnrPutDuration *metrics.Metric
)
func init() {
@ -115,22 +116,24 @@ func (n *Native) Connect(endpoint, hexPrivateKey string, dialTimeout, streamTime
tok.SetExp(exp)
// register metrics
registry := metrics.NewRegistry()
objPutTotal, _ = registry.NewMetric("frostfs_obj_put_total", metrics.Counter)
objPutFails, _ = registry.NewMetric("frostfs_obj_put_fails", metrics.Counter)
objPutDuration, _ = registry.NewMetric("frostfs_obj_put_duration", metrics.Trend, metrics.Time)
objGetTotal, _ = registry.NewMetric("frostfs_obj_get_total", metrics.Counter)
objGetFails, _ = registry.NewMetric("frostfs_obj_get_fails", metrics.Counter)
objGetDuration, _ = registry.NewMetric("frostfs_obj_get_duration", metrics.Trend, metrics.Time)
objPutSuccess, _ = stats.Registry.NewMetric("frostfs_obj_put_success", metrics.Counter)
objPutFails, _ = stats.Registry.NewMetric("frostfs_obj_put_fails", metrics.Counter)
objPutDuration, _ = stats.Registry.NewMetric("frostfs_obj_put_duration", metrics.Trend, metrics.Time)
objPutData, _ = stats.Registry.NewMetric("frostfs_obj_put_bytes", metrics.Counter, metrics.Data)
objDeleteTotal, _ = registry.NewMetric("frostfs_obj_delete_total", metrics.Counter)
objDeleteFails, _ = registry.NewMetric("frostfs_obj_delete_fails", metrics.Counter)
objDeleteDuration, _ = registry.NewMetric("frostfs_obj_delete_duration", metrics.Trend, metrics.Time)
objGetSuccess, _ = stats.Registry.NewMetric("frostfs_obj_get_success", metrics.Counter)
objGetFails, _ = stats.Registry.NewMetric("frostfs_obj_get_fails", metrics.Counter)
objGetDuration, _ = stats.Registry.NewMetric("frostfs_obj_get_duration", metrics.Trend, metrics.Time)
objGetData, _ = stats.Registry.NewMetric("frostfs_obj_get_bytes", metrics.Counter, metrics.Data)
cnrPutTotal, _ = registry.NewMetric("frostfs_cnr_put_total", metrics.Counter)
cnrPutFails, _ = registry.NewMetric("frostfs_cnr_put_fails", metrics.Counter)
cnrPutDuration, _ = registry.NewMetric("frostfs_cnr_put_duration", metrics.Trend, metrics.Time)
objDeleteSuccess, _ = stats.Registry.NewMetric("frostfs_obj_delete_success", metrics.Counter)
objDeleteFails, _ = stats.Registry.NewMetric("frostfs_obj_delete_fails", metrics.Counter)
objDeleteDuration, _ = stats.Registry.NewMetric("frostfs_obj_delete_duration", metrics.Trend, metrics.Time)
cnrPutTotal, _ = stats.Registry.NewMetric("frostfs_cnr_put_total", metrics.Counter)
cnrPutFails, _ = stats.Registry.NewMetric("frostfs_cnr_put_fails", metrics.Counter)
cnrPutDuration, _ = stats.Registry.NewMetric("frostfs_cnr_put_duration", metrics.Trend, metrics.Time)
return &Client{
vu: n.vu,

15
internal/s3/client.go
View file

@ -65,9 +65,10 @@ func (c *Client) Put(bucket, key string, payload datagen.Payload) PutResponse {
return PutResponse{Success: false, Error: err.Error()}
}
stats.Report(c.vu, objPutTotal, 1)
stats.Report(c.vu, objPutSuccess, 1)
stats.ReportDataSent(c.vu, float64(sz))
stats.Report(c.vu, objPutDuration, metrics.D(time.Since(start)))
stats.Report(c.vu, objPutData, float64(sz))
return PutResponse{Success: true}
}
@ -98,9 +99,10 @@ func (c *Client) Multipart(bucket, key string, objPartSize, concurrency int, pay
return PutResponse{Success: false, Error: err.Error()}
}
stats.Report(c.vu, objPutTotal, 1)
stats.Report(c.vu, objPutSuccess, 1)
stats.ReportDataSent(c.vu, float64(sz))
stats.Report(c.vu, objPutDuration, metrics.D(time.Since(start)))
stats.Report(c.vu, objPutData, float64(sz))
return PutResponse{Success: true}
}
@ -116,7 +118,7 @@ func (c *Client) Delete(bucket, key string) DeleteResponse {
return DeleteResponse{Success: false, Error: err.Error()}
}
stats.Report(c.vu, objDeleteTotal, 1)
stats.Report(c.vu, objDeleteSuccess, 1)
stats.Report(c.vu, objDeleteDuration, metrics.D(time.Since(start)))
return DeleteResponse{Success: true}
}
@ -124,7 +126,7 @@ func (c *Client) Delete(bucket, key string) DeleteResponse {
func (c *Client) Get(bucket, key string) GetResponse {
start := time.Now()
var objSize = 0
objSize := 0
err := get(c.cli, bucket, key, func(chunk []byte) {
objSize += len(chunk)
})
@ -133,9 +135,10 @@ func (c *Client) Get(bucket, key string) GetResponse {
return GetResponse{Success: false, Error: err.Error()}
}
stats.Report(c.vu, objGetTotal, 1)
stats.Report(c.vu, objGetSuccess, 1)
stats.Report(c.vu, objGetDuration, metrics.D(time.Since(start)))
stats.ReportDataReceived(c.vu, float64(objSize))
stats.Report(c.vu, objGetData, float64(objSize))
return GetResponse{Success: true}
}
@ -212,7 +215,7 @@ func (c *Client) CreateBucket(bucket string, params map[string]string) CreateBuc
return CreateBucketResponse{Success: false, Error: err.Error()}
}
stats.Report(c.vu, createBucketTotal, 1)
stats.Report(c.vu, createBucketSuccess, 1)
stats.Report(c.vu, createBucketDuration, metrics.D(time.Since(start)))
return CreateBucketResponse{Success: true}
}

36
internal/s3/s3.go
View file

@ -7,6 +7,7 @@ import (
"strconv"
"time"
"git.frostfs.info/TrueCloudLab/xk6-frostfs/internal/stats"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/service/s3"
@ -28,10 +29,10 @@ var (
_ modules.Instance = &S3{}
_ modules.Module = &RootModule{}
objPutTotal, objPutFails, objPutDuration *metrics.Metric
objGetTotal, objGetFails, objGetDuration *metrics.Metric
objDeleteTotal, objDeleteFails, objDeleteDuration *metrics.Metric
createBucketTotal, createBucketFails, createBucketDuration *metrics.Metric
objPutSuccess, objPutFails, objPutDuration, objPutData *metrics.Metric
objGetSuccess, objGetFails, objGetDuration, objGetData *metrics.Metric
objDeleteSuccess, objDeleteFails, objDeleteDuration *metrics.Metric
createBucketSuccess, createBucketFails, createBucketDuration *metrics.Metric
)
func init() {
@ -94,22 +95,23 @@ func (s *S3) Connect(endpoint string, params map[string]string) (*Client, error)
})
// register metrics
registry := metrics.NewRegistry()
objPutTotal, _ = registry.NewMetric("aws_obj_put_total", metrics.Counter)
objPutFails, _ = registry.NewMetric("aws_obj_put_fails", metrics.Counter)
objPutDuration, _ = registry.NewMetric("aws_obj_put_duration", metrics.Trend, metrics.Time)
objPutSuccess, _ = stats.Registry.NewMetric("aws_obj_put_success", metrics.Counter)
objPutFails, _ = stats.Registry.NewMetric("aws_obj_put_fails", metrics.Counter)
objPutDuration, _ = stats.Registry.NewMetric("aws_obj_put_duration", metrics.Trend, metrics.Time)
objPutData, _ = stats.Registry.NewMetric("aws_obj_put_bytes", metrics.Counter, metrics.Data)
objGetTotal, _ = registry.NewMetric("aws_obj_get_total", metrics.Counter)
objGetFails, _ = registry.NewMetric("aws_obj_get_fails", metrics.Counter)
objGetDuration, _ = registry.NewMetric("aws_obj_get_duration", metrics.Trend, metrics.Time)
objGetSuccess, _ = stats.Registry.NewMetric("aws_obj_get_success", metrics.Counter)
objGetFails, _ = stats.Registry.NewMetric("aws_obj_get_fails", metrics.Counter)
objGetDuration, _ = stats.Registry.NewMetric("aws_obj_get_duration", metrics.Trend, metrics.Time)
objGetData, _ = stats.Registry.NewMetric("aws_obj_get_bytes", metrics.Counter, metrics.Data)
objDeleteTotal, _ = registry.NewMetric("aws_obj_delete_total", metrics.Counter)
objDeleteFails, _ = registry.NewMetric("aws_obj_delete_fails", metrics.Counter)
objDeleteDuration, _ = registry.NewMetric("aws_obj_delete_duration", metrics.Trend, metrics.Time)
objDeleteSuccess, _ = stats.Registry.NewMetric("aws_obj_delete_success", metrics.Counter)
objDeleteFails, _ = stats.Registry.NewMetric("aws_obj_delete_fails", metrics.Counter)
objDeleteDuration, _ = stats.Registry.NewMetric("aws_obj_delete_duration", metrics.Trend, metrics.Time)
createBucketTotal, _ = registry.NewMetric("aws_create_bucket_total", metrics.Counter)
createBucketFails, _ = registry.NewMetric("aws_create_bucket_fails", metrics.Counter)
createBucketDuration, _ = registry.NewMetric("aws_create_bucket_duration", metrics.Trend, metrics.Time)
createBucketSuccess, _ = stats.Registry.NewMetric("aws_create_bucket_success", metrics.Counter)
createBucketFails, _ = stats.Registry.NewMetric("aws_create_bucket_fails", metrics.Counter)
createBucketDuration, _ = stats.Registry.NewMetric("aws_create_bucket_duration", metrics.Trend, metrics.Time)
return &Client{
vu: s.vu,

6
internal/s3local/client.go
View file

@ -68,8 +68,9 @@ func (c *Client) Put(bucket, key string, payload datagen.Payload) PutResponse {
}
stats.Report(c.vu, objPutDuration, metrics.D(time.Since(start)))
stats.Report(c.vu, objPutTotal, 1)
stats.Report(c.vu, objPutSuccess, 1)
stats.ReportDataSent(c.vu, float64(prm.Size))
stats.Report(c.vu, objPutData, float64(prm.Size))
return PutResponse{Success: true}
}
@ -115,8 +116,9 @@ func (c *Client) Get(bucket, key string) GetResponse {
}
stats.Report(c.vu, objGetDuration, metrics.D(time.Since(start)))
stats.Report(c.vu, objGetTotal, 1)
stats.Report(c.vu, objGetSuccess, 1)
stats.ReportDataReceived(c.vu, wr.total)
stats.Report(c.vu, objGetData, wr.total)
return GetResponse{Success: true}
}

44
internal/s3local/local.go
View file

@ -32,10 +32,10 @@ var (
_ modules.Module = &RootModule{}
_ modules.Instance = &Local{}
internalObjPutTotal, internalObjPutFails, internalObjPutDuration *metrics.Metric
internalObjGetTotal, internalObjGetFails, internalObjGetDuration *metrics.Metric
objPutTotal, objPutFails, objPutDuration *metrics.Metric
objGetTotal, objGetFails, objGetDuration *metrics.Metric
internalObjPutSuccess, internalObjPutFails, internalObjPutDuration, internalObjPutData *metrics.Metric
internalObjGetSuccess, internalObjGetFails, internalObjGetDuration, internalObjGetData *metrics.Metric
objPutSuccess, objPutFails, objPutDuration, objPutData *metrics.Metric
objGetSuccess, objGetFails, objGetDuration, objGetData *metrics.Metric
)
func init() {
@ -88,23 +88,25 @@ func (s *Local) Connect(configFile string, configDir string, params map[string]s
}
// Register metrics.
registry := metrics.NewRegistry()
internalObjPutSuccess, _ = stats.Registry.NewMetric("s3local_internal_obj_put_success", metrics.Counter)
internalObjPutFails, _ = stats.Registry.NewMetric("s3local_internal_obj_put_fails", metrics.Counter)
internalObjPutDuration, _ = stats.Registry.NewMetric("s3local_internal_obj_put_duration", metrics.Trend, metrics.Time)
internalObjPutData, _ = stats.Registry.NewMetric("s3local_internal_obj_put_bytes", metrics.Counter, metrics.Data)
internalObjPutTotal, _ = registry.NewMetric("s3local_internal_obj_put_total", metrics.Counter)
internalObjPutFails, _ = registry.NewMetric("s3local_internal_obj_put_fails", metrics.Counter)
internalObjPutDuration, _ = registry.NewMetric("s3local_internal_obj_put_duration", metrics.Trend, metrics.Time)
internalObjGetSuccess, _ = stats.Registry.NewMetric("s3local_internal_obj_get_success", metrics.Counter)
internalObjGetFails, _ = stats.Registry.NewMetric("s3local_internal_obj_get_fails", metrics.Counter)
internalObjGetDuration, _ = stats.Registry.NewMetric("s3local_internal_obj_get_duration", metrics.Trend, metrics.Time)
internalObjGetData, _ = stats.Registry.NewMetric("s3local_internal_obj_get_bytes", metrics.Counter, metrics.Data)
internalObjGetTotal, _ = registry.NewMetric("s3local_internal_obj_get_total", metrics.Counter)
internalObjGetFails, _ = registry.NewMetric("s3local_internal_obj_get_fails", metrics.Counter)
internalObjGetDuration, _ = registry.NewMetric("s3local_internal_obj_get_duration", metrics.Trend, metrics.Time)
objPutSuccess, _ = stats.Registry.NewMetric("s3local_obj_put_success", metrics.Counter)
objPutFails, _ = stats.Registry.NewMetric("s3local_obj_put_fails", metrics.Counter)
objPutDuration, _ = stats.Registry.NewMetric("s3local_obj_put_duration", metrics.Trend, metrics.Time)
objPutData, _ = stats.Registry.NewMetric("s3local_obj_put_bytes", metrics.Counter, metrics.Data)
objPutTotal, _ = registry.NewMetric("s3local_obj_put_total", metrics.Counter)
objPutFails, _ = registry.NewMetric("s3local_obj_put_fails", metrics.Counter)
objPutDuration, _ = registry.NewMetric("s3local_obj_put_duration", metrics.Trend, metrics.Time)
objGetTotal, _ = registry.NewMetric("s3local_obj_get_total", metrics.Counter)
objGetFails, _ = registry.NewMetric("s3local_obj_get_fails", metrics.Counter)
objGetDuration, _ = registry.NewMetric("s3local_obj_get_duration", metrics.Trend, metrics.Time)
objGetSuccess, _ = stats.Registry.NewMetric("s3local_obj_get_success", metrics.Counter)
objGetFails, _ = stats.Registry.NewMetric("s3local_obj_get_fails", metrics.Counter)
objGetDuration, _ = stats.Registry.NewMetric("s3local_obj_get_duration", metrics.Trend, metrics.Time)
objGetData, _ = stats.Registry.NewMetric("s3local_obj_get_bytes", metrics.Counter, metrics.Data)
// Create S3 layer backed by local storage engine and tree service.
ng, limiter, err := s.l.ResolveEngine(s.l.VU().Context(), configFile, configDir, *debugLogger, maxSizeGB)
@ -124,16 +126,18 @@ func (s *Local) Connect(configFile string, configDir string, params map[string]s
if err != nil {
stats.Report(s.l.VU(), internalObjPutFails, 1)
} else {
stats.Report(s.l.VU(), internalObjPutTotal, 1)
stats.Report(s.l.VU(), internalObjPutSuccess, 1)
stats.Report(s.l.VU(), internalObjPutDuration, metrics.D(dt))
stats.Report(s.l.VU(), internalObjPutData, float64(sz))
}
}),
rawclient.WithGetHandler(func(sz uint64, err error, dt time.Duration) {
if err != nil {
stats.Report(s.l.VU(), internalObjGetFails, 1)
} else {
stats.Report(s.l.VU(), internalObjGetTotal, 1)
stats.Report(s.l.VU(), internalObjGetSuccess, 1)
stats.Report(s.l.VU(), internalObjGetDuration, metrics.D(dt))
stats.Report(s.l.VU(), internalObjGetData, float64(sz))
}
}),
)

46
internal/stats/stats.go
View file

@ -1,16 +1,54 @@
package stats
import (
"strings"
"time"
"go.k6.io/k6/js/modules"
"go.k6.io/k6/metrics"
)
// RootModule is the global module object type. It is instantiated once per test
// run and will be used to create k6/x/frostfs/stats module instances for each VU.
type RootModule struct {
Instance string
}
var (
tagSet *metrics.TagSet
Registry *metrics.Registry
)
func init() {
Registry = metrics.NewRegistry()
tagSet = Registry.RootTagSet()
modules.Register("k6/x/frostfs/stats", &RootModule{})
}
// SetTags sets additional tags to custom metrics.
// Format: "key1:value1;key2:value2".
// Panics if input has invalid format.
func (m *RootModule) SetTags(labels string) {
kv := make(map[string]string)
pairs := strings.Split(labels, ";")
for _, pair := range pairs {
items := strings.Split(pair, ":")
if len(items) != 2 {
panic("invalid labels format")
}
kv[strings.TrimSpace(items[0])] = strings.TrimSpace(items[1])
}
for k, v := range kv {
tagSet = tagSet.With(k, v)
}
}
func Report(vu modules.VU, metric *metrics.Metric, value float64) {
metrics.PushIfNotDone(vu.Context(), vu.State().Samples, metrics.Sample{
TimeSeries: metrics.TimeSeries{
Metric: metric,
Tags: tagSet,
},
Time: time.Now(),
Value: value,
@ -22,9 +60,11 @@ func ReportDataReceived(vu modules.VU, value float64) {
metrics.Sample{
TimeSeries: metrics.TimeSeries{
Metric: &metrics.Metric{},
Tags: tagSet,
},
Value: value,
Time: time.Now()},
Time: time.Now(),
},
)
}
@ -34,8 +74,10 @@ func ReportDataSent(vu modules.VU, value float64) {
metrics.Sample{
TimeSeries: metrics.TimeSeries{
Metric: &metrics.Metric{},
Tags: tagSet,
},
Value: value,
Time: time.Now()},
Time: time.Now(),
},
)
}

5
scenarios/grpc.js
View file

@ -1,6 +1,7 @@
import native from 'k6/x/frostfs/native';
import logging from 'k6/x/frostfs/logging';
import registry from 'k6/x/frostfs/registry';
import stats from 'k6/x/frostfs/stats';
import { SharedArray } from 'k6/data';
import { sleep } from 'k6';
import { textSummary } from './libs/k6-summary-0.0.2.js';
@ -35,6 +36,10 @@ const obj_registry = registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : und
const duration = __ENV.DURATION;
if (!!__ENV.METRIC_TAGS) {
stats.setTags(__ENV.METRIC_TAGS)
}
const delete_age = __ENV.DELETE_AGE ? parseInt(__ENV.DELETE_AGE) : undefined;
let obj_to_delete_selector = undefined;
if (registry_enabled && delete_age) {

5
scenarios/grpc_car.js
View file

@ -1,6 +1,7 @@
import native from 'k6/x/frostfs/native';
import logging from 'k6/x/frostfs/logging';
import registry from 'k6/x/frostfs/registry';
import stats from 'k6/x/frostfs/stats';
import { SharedArray } from 'k6/data';
import { sleep } from 'k6';
import { textSummary } from './libs/k6-summary-0.0.2.js';
@ -35,6 +36,10 @@ const obj_registry = registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : und
const duration = __ENV.DURATION;
if (!!__ENV.METRIC_TAGS) {
stats.setTags(__ENV.METRIC_TAGS)
}
const delete_age = __ENV.DELETE_AGE ? parseInt(__ENV.DELETE_AGE) : undefined;
let obj_to_delete_selector = undefined;
if (registry_enabled && delete_age) {

5
scenarios/http.js
View file

@ -1,5 +1,6 @@
import logging from 'k6/x/frostfs/logging';
import registry from 'k6/x/frostfs/registry';
import stats from 'k6/x/frostfs/stats';
import http from 'k6/http';
import { SharedArray } from 'k6/data';
import { sleep } from 'k6';
@ -31,6 +32,10 @@ const obj_registry = registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : und
const duration = __ENV.DURATION;
if (!!__ENV.METRIC_TAGS) {
stats.setTags(__ENV.METRIC_TAGS)
}
const scenarios = {};
const write_vu_count = parseInt(__ENV.WRITERS || '0');

5
scenarios/local.js
View file

@ -1,6 +1,7 @@
import local from 'k6/x/frostfs/local';
import logging from 'k6/x/frostfs/logging';
import registry from 'k6/x/frostfs/registry';
import stats from 'k6/x/frostfs/stats';
import { SharedArray } from 'k6/data';
import { textSummary } from './libs/k6-summary-0.0.2.js';
import { parseEnv } from './libs/env-parser.js';
@ -33,6 +34,10 @@ const obj_registry = registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : und
const duration = __ENV.DURATION;
if (!!__ENV.METRIC_TAGS) {
stats.setTags(__ENV.METRIC_TAGS)
}
const delete_age = __ENV.DELETE_AGE ? parseInt(__ENV.DELETE_AGE) : undefined;
let obj_to_delete_selector = undefined;
if (registry_enabled && delete_age) {

16
scenarios/run_scenarios.md
View file

@ -20,6 +20,7 @@ Scenarios `grpc.js`, `local.js`, `http.js` and `s3.js` support the following opt
* `SELECTION_SIZE` - size of batch to select for deletion (default: 1000).
* `PAYLOAD_TYPE` - type of an object payload ("random" or "text", default: "random").
* `STREAMING` - if set, the payload is generated on the fly and is not read into memory fully.
* `METRIC_TAGS` - `instance` tag value.
Additionally, the profiling extension can be enabled to generate CPU and memory profiles which can be inspected with `go tool pprof file.prof`:
```shell
@ -186,6 +187,21 @@ Options (in addition to the common options):
* `OBJ_NAME` - if specified, this name will be used for all write operations instead of random generation.
* `MAX_TOTAL_SIZE_GB` - if specified, max payload size in GB of the storage engine. If the storage engine is already full, no new objects will be saved.
## Export metrics
To export metrics to Prometheus (also Grafana and Victoria Metrics support Prometheus format), you need to run `k6` with an option `-o experimental-prometheus-rw` and
an environment variable `K6_PROMETHEUS_RW_SERVER_URL` whose value corresponds to the URL for the remote write endpoint.
To specify percentiles for trend metrics, use an environment variable `K6_PROMETHEUS_RW_TREND_STATS`.
See [k6 docs](https://k6.io/docs/results-output/real-time/prometheus-remote-write/) for a list of all possible options.
To distinct metrics from different loaders, use an option `METRIC_TAGS`. These tags does not apply to builtin `k6` metrics.
Example:
```bash
K6_PROMETHEUS_RW_SERVER_URL=http://host:8428/api/v1/write \
K6_PROMETHEUS_RW_TREND_STATS="p(95),p(99),min,max" \
./k6 run ... -o experimental-prometheus-rw -e METRIC_TAGS="instance:server1;run:run1" scenario.js
```
## Verify
This scenario allows to verify that objects created by a previous run are really stored in the system and their data is not corrupted. Running this scenario assumes that you've already run gRPC or HTTP or S3 scenario with option `REGISTRY_FILE`.

5
scenarios/s3.js
View file

@ -1,5 +1,6 @@
import logging from 'k6/x/frostfs/logging';
import registry from 'k6/x/frostfs/registry';
import stats from 'k6/x/frostfs/stats';
import s3 from 'k6/x/frostfs/s3';
import { SharedArray } from 'k6/data';
import { sleep } from 'k6';
@ -34,6 +35,10 @@ const obj_registry = registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : und
const duration = __ENV.DURATION;
if (!!__ENV.METRIC_TAGS) {
stats.setTags(__ENV.METRIC_TAGS)
}
const delete_age = __ENV.DELETE_AGE ? parseInt(__ENV.DELETE_AGE) : undefined;
let obj_to_delete_selector = undefined;
if (registry_enabled && delete_age) {

5
scenarios/s3_car.js
View file

@ -1,5 +1,6 @@
import logging from 'k6/x/frostfs/logging';
import registry from 'k6/x/frostfs/registry';
import stats from 'k6/x/frostfs/stats';
import s3 from 'k6/x/frostfs/s3';
import { SharedArray } from 'k6/data';
import { sleep } from 'k6';
@ -34,6 +35,10 @@ const obj_registry = registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : und
const duration = __ENV.DURATION;
if (!!__ENV.METRIC_TAGS) {
stats.setTags(__ENV.METRIC_TAGS)
}
const delete_age = __ENV.DELETE_AGE ? parseInt(__ENV.DELETE_AGE) : undefined;
let obj_to_delete_selector = undefined;
if (registry_enabled && delete_age) {

5
scenarios/s3_multipart.js
View file

@ -1,5 +1,6 @@
import logging from 'k6/x/frostfs/logging';
import registry from 'k6/x/frostfs/registry';
import stats from 'k6/x/frostfs/stats';
import s3 from 'k6/x/frostfs/s3';
import {SharedArray} from 'k6/data';
import {sleep} from 'k6';
@ -29,6 +30,10 @@ const obj_registry = registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : und
const duration = __ENV.DURATION;
if (!!__ENV.METRIC_TAGS) {
stats.setTags(__ENV.METRIC_TAGS)
}
const scenarios = {};
const write_vu_count = parseInt(__ENV.WRITERS || '0');

5
scenarios/s3local.js
View file

@ -1,5 +1,6 @@
import logging from 'k6/x/frostfs/logging';
import registry from 'k6/x/frostfs/registry';
import stats from 'k6/x/frostfs/stats';
import s3local from 'k6/x/frostfs/s3local';
import { SharedArray } from 'k6/data';
import { textSummary } from './libs/k6-summary-0.0.2.js';
@ -44,6 +45,10 @@ const s3_client = s3local.connect(config_file, config_dir, {
}, bucket_mapping(), max_total_size_gb);
const log = logging.new().withFields({"config_file": config_file,"config_dir": config_dir});
if (!!__ENV.METRIC_TAGS) {
stats.setTags(__ENV.METRIC_TAGS)
}
const registry_enabled = !!__ENV.REGISTRY_FILE;
const obj_registry = registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : undefined;

5
scenarios/verify.js
View file

@ -1,5 +1,6 @@
import native from 'k6/x/frostfs/native';
import registry from 'k6/x/frostfs/registry';
import stats from 'k6/x/frostfs/stats';
import s3 from 'k6/x/frostfs/s3';
import logging from 'k6/x/frostfs/logging';
import { sleep } from 'k6';
@ -28,6 +29,10 @@ const obj_counters = {
let log = logging.new();
if (!!__ENV.METRIC_TAGS) {
stats.setTags(__ENV.METRIC_TAGS)
}
// Connect to random gRPC endpoint
let grpc_client = undefined;
if (__ENV.GRPC_ENDPOINTS) {